@es3d/es3d 1.0.1 → 1.1.0

package/docs/SPEC.md

@@ -403,11 +403,20 @@ The title names the phase (`Parse`, `Reference`, `Geometry`, `Render`, `File`) a
 
 Unknown names get a suggestion when a known name is within a small edit distance or shares a prefix (`raduis` → `radius`, `spotlight` → `spot`, `sphear` → `sphere`).
 
+The reference reader's human-readable report (`formatDiagnostics`, and the text from `validate`) shows a repeated warning once. Warnings group when they share a code, the element name written in the message (its first `<name>`), and the same message. The report keeps the first location and then says how many times that warning occurred:
+
+```
+Reported 906 times. Showing the first.
+```
+
+Every warning remains its own entry in the diagnostic list. `validate --json` prints that list. Errors are listed one by one.
+
 ## 16. Extensibility and versioning
 
 - Elements whose names start with `x-` (and their subtrees) are vendor extensions: readers that do not understand them skip them with `W_EXTENSION`. Unknown elements without the prefix are errors, so typos are never mistaken for extensions.
 - Unknown attributes are always errors. (A future version may reserve an `x-` attribute prefix as well.)
 - `version="1.x"` documents must remain readable by version-1 readers; incompatible changes require version 2, which version-1 readers reject with `E_VERSION`.
+- A processor that rewrites a document keeps `x-` elements, and keeps any element whose id is named by a token in an `x-` attribute. The reference optimizer treats a whitespace-separated token (an optional leading `#` is ignored) as a reference when that id exists, and updates the token when it merges a duplicate material.
 
 ## 17. File names and media types
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@es3d/es3d",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "ES3D (Extensible Scalable 3D): a declarative, SVG-inspired 3D scene format with a WebGPU/WebGL2 reference renderer, editor and converter.",
   "type": "module",
   "license": "Apache-2.0",
@@ -28,7 +28,8 @@
   ],
   "main": "src/index.js",
   "exports": {
-    ".": "./src/index.js"
+    ".": "./src/index.js",
+    "./stage": "./browser/es3d-stage.min.js"
   },
   "bin": {
     "es3d": "scripts/es3d-cli.mjs"
@@ -40,6 +41,7 @@
     "!src/bench/",
     "!src/main.js",
     "scripts/es3d-cli.mjs",
+    "browser/es3d-stage.min.js",
     "docs/SPEC.md",
     "docs/REFERENCE.md"
   ],

package/scripts/es3d-cli.mjs

@@ -4,7 +4,7 @@
 //   npm run es3d -- validate scene.es3d
 //   npm run es3d -- fmt scene.es3d -o pretty.es3d
 //   npm run es3d -- stats scene.es3d
-//   npm run es3d -- glb scene.es3d --instancing -o scene.glb
+//   npm run es3d -- glb scene.es3d -o scene.glb
 //   npm run es3d -- import drawing.svg -o drawing.es3d
 //
 // Run without arguments for the full command list. Input "-" reads stdin;
@@ -13,21 +13,21 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import zlib from 'node:zlib';
-import { parse, detectSyntax } from '../src/format/parse.js';
+import { parse, detectSyntax, formatDiagnostics } from '../src/format/parse.js';
 import { serialize } from '../src/format/serializer.js';
 import { compile } from '../src/pipeline.js';
 import { exportOBJ } from '../src/io/obj.js';
-import { exportGLB } from '../src/io/gltf.js';
 import { exportSTL } from '../src/io/stl.js';
 import { exportSTEP } from '../src/io/step.js';
-import { detectKind, importFiles } from '../src/io/convert.js';
+import { detectKind, importFiles, exportFormat } from '../src/io/convert.js';
 import { exportableInstances } from '../src/io/external-scene.js';
 import { optimize, expandProcedural } from '../src/optimize/optimizer.js';
 
 const HELP = `es3d <command> [options] <input>
 
 Commands
-  validate   parse, validate and build the scene; print every diagnostic (exit 1 on errors)
+  validate   parse, validate and build the scene; print diagnostics (exit 1 on errors).
+             Repeated warnings are grouped; --json lists every one
   fmt        pretty-print, keeping the syntax (markup or compact)
   minify     minified markup (short attribute aliases, no comments)
   compact    convert to the compact syntax           (--min for one line)
@@ -36,21 +36,27 @@ Commands
   optimize   run the optimizer and print its report to stderr
   expand     replace <repeat>/<scatter> with explicit elements
   obj        export Wavefront OBJ (+ .mtl next to -o) (--lod <0-3>)
-  glb        export glTF binary                      (--lod <0-3>, --instancing, --quantize 16)
+  glb        packed glTF binary                     (instances, LODs, lights, cameras, animation)
   stl        export STL for 3D printing, Z-up        (--lod <0-3>, --ascii)
   step       export tessellated STEP (AP242), Z-up  (--lod <0-3>)
-  import     convert SVG, OBJ (+MTL), STL, STEP or glTF (.gltf + .bin, .glb) to ES3D:
+  x3d        export X3D Classic (.x3dv)            (--lod <0-3>, --flatten)
+  usd        export USD ASCII (.usda)              (--lod <0-3>, --flatten)
+  scad       export an OpenSCAD program            (--lod <0-3>, --flatten)
+  js         export a three.js module              (--lod <0-3>, --flatten)
+  import     convert SVG, OBJ (+MTL), STL, STEP, glTF, X3D, USDA or OpenSCAD to ES3D:
              SVG shapes are extruded (--size, --depth, --bevel, --flat, --no-strokes, --plate),
              STEP solids become ES3D shapes (boxes, extrusions, revolves, paths; free-form faces a mesh),
-             meshes become <mesh> elements (--precision, --smooth, --up y for Y-up STL or STEP)
+             meshes become <mesh> elements (--precision, --smooth, --up y for Y-up STL or STEP),
+             OpenSCAD programs become shapes (difference and intersection are kept and noted, not evaluated)
   stats      sizes, element / instance / triangle counts and pipeline timings
 
 Options
   -o, --out <file>   output file (default: stdout for text)
   --precision <n>    round numbers to n decimals when writing text
-  --lod <n>          level of detail for mesh exports (default 0 = finest)
-  --instancing       GLB: use EXT_mesh_gpu_instancing
-  --quantize         GLB: KHR_mesh_quantization (int16 positions, int8 normals)
+  --lod <n>          level of detail for mesh export (default 0 = finest)
+  --flatten          x3d, usd, scad, js: write triangles instead of primitives
+  --instancing       glb always writes EXT_mesh_gpu_instancing
+  --quantize         glb always writes KHR_mesh_quantization
   --min / --pretty   force minified / pretty output
   --json             validate/stats: machine readable output
   --ascii            STL: write ASCII instead of binary
@@ -65,7 +71,7 @@ Options
   -h, --help         this text`;
 
 function parseArgs(argv) {
-  const o = { cmd: null, input: null, out: null, precision: null, lod: 0, instancing: false, quantize: 0, min: false, pretty: false, json: false, ascii: false, svg: {}, mesh: {}, stl: {} };
+  const o = { cmd: null, input: null, out: null, precision: null, lod: 0, instancing: false, quantize: 0, min: false, pretty: false, json: false, ascii: false, flatten: false, svg: {}, mesh: {}, stl: {} };
   for (let i = 0; i < argv.length; i++) {
     const a = argv[i];
     const val = () => {
@@ -82,6 +88,7 @@ function parseArgs(argv) {
     else if (a === '--pretty') o.pretty = true;
     else if (a === '--json') o.json = true;
     else if (a === '--ascii') o.ascii = true;
+    else if (a === '--flatten') o.flatten = true;
     else if (a === '--size') o.svg.size = Number(val());
     else if (a === '--depth') o.svg.depth = Number(val()) / 100;
     else if (a === '--bevel') o.svg.bevel = Number(val()) / 100;
@@ -132,9 +139,7 @@ function write(opts, data) {
 }
 
 function report(doc) {
-  const out = [];
-  for (const d of doc.diagnostics) out.push(d.format(doc.lineIndex));
-  return out.join('\n\n');
+  return formatDiagnostics(doc);
 }
 
 /** Parse and validate, or exit with the diagnostics. */
@@ -144,7 +149,8 @@ function parsed(input) {
     process.stderr.write(report(doc) + '\n');
     process.exit(1);
   }
-  for (const w of doc.warnings || []) process.stderr.write(w.format(doc.lineIndex) + '\n\n');
+  const warnings = formatDiagnostics(doc.warnings || [], doc.lineIndex);
+  if (warnings) process.stderr.write(warnings + '\n\n');
   return doc;
 }
 
@@ -152,7 +158,7 @@ function compiled(input) {
   const r = compile(input);
   const errors = r.diagnostics.filter((d) => d.severity === 'error');
   if (errors.length) {
-    process.stderr.write(r.diagnostics.map((d) => d.format(r.doc.lineIndex)).join('\n\n') + '\n');
+    process.stderr.write(formatDiagnostics(r) + '\n');
     process.exit(1);
   }
   return r;
@@ -264,7 +270,8 @@ async function main() {
       const warnings = r.diagnostics.length - errors;
       if (opts.json) console.log(JSON.stringify({ ok: errors === 0, diagnostics: r.diagnostics.map((d) => d.toJSON()) }, null, 2));
       else {
-        if (r.diagnostics.length) process.stderr.write(r.diagnostics.map((d) => d.format(r.doc.lineIndex)).join('\n\n') + '\n\n');
+        const text = formatDiagnostics(r);
+        if (text) process.stderr.write(text + '\n\n');
         console.log(`${opts.input}: ${errors} error${errors === 1 ? '' : 's'}, ${warnings} warning${warnings === 1 ? '' : 's'}${errors ? '' : `, ${r.scene.count.toLocaleString('en-US')} instances`}`);
       }
       process.exit(errors ? 1 : 0);
@@ -312,8 +319,21 @@ async function main() {
       break;
     }
     case 'glb': {
-      const r = compiled(es3dInput(bytes));
-      write(opts, exportGLB(r.scene, (e, l) => r.cache.mesh(e, l), { lod: opts.lod, instancing: opts.instancing, quantize: opts.quantize > 0 }));
+      let exportPackedGLB;
+      try {
+        ({ exportPackedGLB } = await import('../src/io/gltf-pack.js'));
+      } catch (e) {
+        fail(`glb could not load the packer (${e.message})`, 1);
+      }
+      const baseDir = opts.input === '-' ? '' : path.dirname(path.resolve(opts.input));
+      const sourceName = opts.input === '-' ? 'stdin' : `${path.basename(path.dirname(opts.input))}/${path.basename(opts.input)}`;
+      let file;
+      try {
+        file = await exportPackedGLB(es3dInput(bytes), { baseDir, sourceName });
+      } catch (e) {
+        fail(e.message, 1);
+      }
+      write(opts, file);
       break;
     }
     case 'stl': {
@@ -343,6 +363,22 @@ async function main() {
       process.stderr.write(`${out.triangles.toLocaleString('en-US')} triangles\n`);
       break;
     }
+    case 'x3d':
+    case 'usd':
+    case 'scad':
+    case 'js': {
+      const format = { x3d: 'x3dv', usd: 'usda', scad: 'scad', js: 'js' }[opts.cmd];
+      const base = opts.out ? path.basename(opts.out).replace(/\.[^.]+$/, '') : 'scene';
+      let out;
+      try {
+        out = exportFormat(es3dInput(bytes), format, { baseName: base, lod: opts.lod, structure: opts.flatten ? 'flatten' : 'keep' });
+      } catch (e) {
+        fail(e.message, 1);
+      }
+      write(opts, out.files[0].data);
+      for (const note of out.notes || []) process.stderr.write(`${note}\n`);
+      break;
+    }
     case 'import': {
       const name = opts.input === '-' ? 'stdin' : path.basename(opts.input);
       const files = [{ name, bytes }];

package/src/format/parse.js

@@ -86,7 +86,47 @@ export function parseStrict(input, opts = {}) {
   return doc;
 }
 
-/** Human readable report of all diagnostics. */
-export function formatDiagnostics(doc) {
-  return doc.diagnostics.map((d) => d.format(doc.lineIndex)).join('\n\n');
+/** Element name written in a message, if it has one (`<x-anchors>`). */
+function mentionedElement(message) {
+  const m = /<([^<>\s]+)>/.exec(message || '');
+  return m ? m[1] : '';
+}
+
+/**
+ * Human-readable report of diagnostics.
+ * Repeated warnings that share a code, the element name in the message, and
+ * the same message are shown once, at the first location, with a count.
+ * Errors are listed one by one. The diagnostic objects are not changed.
+ * @param {object|Array} source a document, a compile result, or a diagnostic list
+ * @param {import('./errors.js').LineIndex} [lineIndex]
+ */
+export function formatDiagnostics(source, lineIndex) {
+  if (!source) return '';
+  const list = Array.isArray(source) ? source : source.diagnostics || [];
+  const index = lineIndex || source.lineIndex || (source.doc && source.doc.lineIndex) || null;
+  const seen = new Map();
+  const items = [];
+  for (const d of list) {
+    if (!d || d.severity !== 'warning') {
+      items.push(d);
+      continue;
+    }
+    const key = `${d.code}\0${mentionedElement(d.message)}\0${d.message}`;
+    const prev = seen.get(key);
+    if (prev) {
+      prev.count++;
+      continue;
+    }
+    const slot = { d, count: 1 };
+    seen.set(key, slot);
+    items.push(slot);
+  }
+  return items
+    .map((item) => {
+      if (!item || !item.d) return item.format(index);
+      const text = item.d.format(index);
+      if (item.count < 2) return text;
+      return `${text}\n\nReported ${item.count} times. Showing the first.`;
+    })
+    .join('\n\n');
 }

package/src/index.js

@@ -1,16 +1,16 @@
 // Public API of the ES3D reference implementation.
 //
-//   import { compile, createRenderer, Camera } from '@es3d/es3d';
+//   import { compile, createRenderer, Camera, formatDiagnostics } from '@es3d/es3d';
 //
 //   const result = compile(text);              // parse + validate + build + flatten
-//   if (!result.ok) console.error(result.diagnostics.map((d) => d.format()).join('\n\n'));
+//   if (!result.ok) console.error(formatDiagnostics(result));
 //   const renderer = await createRenderer(canvas);   // WebGPU, else WebGL2
 //   renderer.setScene(result.scene, (entry, lod) => result.cache.mesh(entry, lod));
 //   const camera = new Camera().frame(result.scene.bounds);
 //   renderer.render(camera, { width: canvas.clientWidth, height: canvas.clientHeight });
 //
-//   const svg = await importFiles([{ name: 'logo.svg', bytes }]);   // SVG, OBJ, STL, STEP, glTF → ES3D
-//   const { files } = exportFormat(svg.text, 'glb');                 // ES3D → glTF, OBJ, STL, STEP
+//   const svg = await importFiles([{ name: 'logo.svg', bytes }]);   // SVG, OBJ, STL, STEP, glTF, X3D, USDA, OpenSCAD → ES3D
+//   const { files } = exportFormat(svg.text, 'glb');                 // ES3D → glTF, OBJ, STL, STEP, X3D, USD, OpenSCAD, three.js
 //
 // The format, scene, geometry and I/O modules run in Node as well; only the
 // renderer and controls need a browser.
@@ -25,7 +25,9 @@ export { ELEMENTS, elementSpec, attrSpec } from './format/schema.js';
 // scene pipeline
 export { compile, generateGeometry, GeometryCache } from './pipeline.js';
 export { openGeometryStore, indexedDBStore, memoryStore, restoreGeometry, saveGeometry, persistGeometry, geometryKey, GEOMETRY_REVISION } from './cache/geometry-store.js';
-export { RenderScene, updateDynamic, FLAG_MIRRORED, FLAG_NO_COLLIDE } from './scene/flatten.js';
+export { RenderScene, updateDynamic, applyLocalMatrix, applyLocalMatrices, FLAG_MIRRORED, FLAG_NO_COLLIDE } from './scene/flatten.js';
+export { mulberry32 } from './math/random.js';
+export { es3dSin, es3dCos } from './math/trig.js';
 export { pick, raycast } from './scene/picking.js';
 export { applyLimits, sweepCamera, constrainCamera } from './scene/camera-constraints.js';
 
@@ -43,5 +45,10 @@ export { exportOBJ, parseOBJ, objToScene } from './io/obj.js';
 export { exportGLB, parseGLB, glbToScene, isGLB, parseGLTF, gltfToScene } from './io/gltf.js';
 export { exportSTL, parseSTL, stlToScene } from './io/stl.js';
 export { exportSTEP, parseSTEP, stepToScene, isSTEP } from './io/step.js';
+export { exportX3D, importX3D } from './io/x3d.js';
+export { exportUSD, importUSD } from './io/usd.js';
+export { exportOpenSCAD } from './io/openscad.js';
+export { importOpenSCAD } from './io/openscad-import.js';
+export { exportThree } from './io/threejs.js';
 export { sceneToMeshDocument } from './io/mesh-import.js';
 export { optimize, expandProcedural } from './optimize/optimizer.js';

package/src/io/convert.js

@@ -16,6 +16,12 @@ import { svgToES3D } from './svg-import.js';
 import { sceneToMeshDocument } from './mesh-import.js';
 import { compile } from '../pipeline.js';
 import { exportableInstances } from './external-scene.js';
+import { buildExportModel } from './export-model.js';
+import { exportX3D, importX3D } from './x3d.js';
+import { exportUSD, importUSD } from './usd.js';
+import { exportOpenSCAD } from './openscad.js';
+import { importOpenSCAD } from './openscad-import.js';
+import { exportThree } from './threejs.js';
 
 /** OBJ and STL repeat every copy's triangles; beyond this the file gets too large for a browser tab. */
 const OBJ_TRIANGLE_LIMIT = 4e6;
@@ -28,6 +34,12 @@ export const INPUT_KINDS = {
   gltf: 'glTF (JSON)',
   glb: 'glTF binary (GLB)',
   step: 'STEP',
+  x3dv: 'X3D Classic',
+  x3d: 'X3D XML',
+  usda: 'USD ASCII',
+  usdc: 'USD crate',
+  usdz: 'USDZ',
+  scad: 'OpenSCAD',
 };
 
 export const OUTPUT_FORMATS = {
@@ -38,6 +50,10 @@ export const OUTPUT_FORMATS = {
   obj: { label: 'Wavefront OBJ + MTL (.obj)', ext: '.obj', mime: 'model/obj', text: true },
   stl: { label: 'STL for 3D printing (.stl)', ext: '.stl', mime: 'model/stl' },
   step: { label: 'STEP, tessellated (.step)', ext: '.step', mime: 'model/step', text: true },
+  x3dv: { label: 'X3D Classic (.x3dv)', ext: '.x3dv', mime: 'model/x3d+vrml', text: true },
+  usda: { label: 'USD ASCII (.usda)', ext: '.usda', mime: 'model/vnd.usda', text: true },
+  scad: { label: 'OpenSCAD (.scad)', ext: '.scad', mime: 'text/x-openscad', text: true },
+  js: { label: 'three.js module (.js)', ext: '.js', mime: 'text/javascript', text: true },
 };
 
 /** Files that only accompany a main file (OBJ materials, glTF buffers and textures). */
@@ -61,6 +77,11 @@ export function detectKind(name, bytes) {
   if (ext === 'stl' || isAsciiSTL(bytes)) return 'stl';
   if (ext === 'gltf' || /^\s*\{[\s\S]*"asset"\s*:/.test(head)) return 'gltf';
   if (ext === 'obj' || /^\s*(#[^\n]*\n\s*)*(mtllib|o|g|v|vt|vn|f|usemtl|s)\s/.test(head)) return 'obj';
+  if (ext === 'x3dv' || /^#X3D\b/.test(head)) return 'x3dv';
+  if (ext === 'x3d' || /<X3D[\s>]/i.test(head)) return 'x3d';
+  if (ext === 'usda' || /^#usda\b/.test(head)) return 'usda';
+  if (ext === 'usdc' || ext === 'usdz') return ext;
+  if (ext === 'scad') return 'scad';
   if (ext === 'step' || ext === 'stp' || ext === 'p21') return 'step';
   return null;
 }
@@ -93,7 +114,7 @@ export async function importFiles(files, opts = {}) {
   const main = list.find((f) => !COMPANION.test(f.name)) || list[0];
   const companions = list.filter((f) => f !== main);
   const kind = detectKind(main.name, main.bytes);
-  if (!kind) throw new Error(`${main.name}: unknown file type. The converter reads SVG, OBJ, STL, STEP (.step, .stp) and glTF (.gltf, .glb) and ES3D (.es3d) files.`);
+  if (!kind) throw new Error(`${main.name}: unknown file type. The converter reads SVG, OBJ, STL, STEP (.step, .stp), glTF (.gltf, .glb), X3D (.x3d, .x3dv), USDA, OpenSCAD (.scad) and ES3D (.es3d) files.`);
   const title = main.name.replace(/\.[^.]+$/, '') || 'untitled';
   const svgOpts = { ...IMPORT_DEFAULTS.svg, ...(opts.svg || {}) };
   const meshOpts = { ...IMPORT_DEFAULTS.mesh, ...(opts.mesh || {}) };
@@ -165,6 +186,23 @@ export async function importFiles(files, opts = {}) {
       text = meshDocument(s, hasNormals);
       break;
     }
+    case 'x3dv':
+    case 'x3d':
+    case 'usda': {
+      const imported = kind === 'usda' ? importUSD(decode(main.bytes)) : importX3D(decode(main.bytes));
+      text = imported.text;
+      notes.push(...imported.notes);
+      break;
+    }
+    case 'usdc':
+    case 'usdz':
+      throw new Error('USDC and USDZ are not read here. This tool reads and writes USDA text; convert a crate with usdcat if you have the USD tools.');
+    case 'scad': {
+      const imported = importOpenSCAD(decode(main.bytes), { files: companions });
+      text = imported.text;
+      notes.push(...imported.notes);
+      break;
+    }
   }
   return { kind, name: main.name, title, text, warnings, notes };
 }
@@ -186,11 +224,13 @@ function describeErrors(diagnostics, lineIndex) {
  * Write an ES3D document in another format.
  * @param {string|Uint8Array} source ES3D text
  * @param {keyof OUTPUT_FORMATS} format
- * @param {{baseName?: string, lod?: number, compiled?: object, instancing?: boolean}} [opts]
- *        compiled: an existing compile() result for the same source (saves compiling twice)
- * @returns {{files: {name: string, data: Uint8Array|string, mime: string}[], triangles?: number}}
+ * @param {{baseName?: string, lod?: number, compiled?: object, instancing?: boolean, structure?: 'keep'|'flatten'}} [opts]
+ *        compiled: an existing compile() result for the same source (saves compiling twice).
+ *        structure applies to X3D, USD, OpenSCAD and three.js: keep (the default) writes
+ *        primitives and the target's own reuse; flatten writes the same triangles as glTF and OBJ.
+ * @returns {{files: {name: string, data: Uint8Array|string, mime: string}[], triangles?: number, notes?: string[]}}
  */
-export function exportFormat(source, format, { baseName = 'scene', lod = 0, compiled = null, instancing = true } = {}) {
+export function exportFormat(source, format, { baseName = 'scene', lod = 0, compiled = null, instancing = true, structure = 'keep' } = {}) {
   const spec = OUTPUT_FORMATS[format];
   if (!spec) throw new Error(`Unknown output format ${format}.`);
   const name = baseName + spec.ext;
@@ -206,6 +246,14 @@ export function exportFormat(source, format, { baseName = 'scene', lod = 0, comp
       const r = compiled || compile(source);
       if (!r.ok) throw new Error(describeErrors(r.diagnostics, r.doc && r.doc.lineIndex));
       const meshOf = (e, l) => r.cache.mesh(e, Math.min(l, e.lods - 1));
+      if (format === 'x3dv' || format === 'usda' || format === 'scad' || format === 'js') {
+        const model = buildExportModel(r, meshOf, { lod, structure: structure === 'flatten' ? 'flatten' : 'keep' });
+        if (model.meshTriangles > OBJ_TRIANGLE_LIMIT) {
+          throw new Error(`At this level of detail the scene has ${model.meshTriangles.toLocaleString('en-US')} triangles, too many for this file (limit ${OBJ_TRIANGLE_LIMIT.toLocaleString('en-US')}). Choose a coarser detail.`);
+        }
+        const written = format === 'x3dv' ? exportX3D(model) : format === 'usda' ? exportUSD(model) : format === 'scad' ? exportOpenSCAD(model) : exportThree(model);
+        return { ...one(written.text), notes: written.notes };
+      }
       if (format === 'glb') return one(exportGLB(r.scene, meshOf, { lod, instancing }));
       if (format === 'obj') {
         let triangles = 0;