ducjs 2.4.0 → 3.0.3

package/dist/lazy-files.js

@@ -1,207 +1,135 @@
+import { wasmGetExternalFile, wasmListExternalFiles } from "./wasm";
 /**
- * LazyExternalFileStore — Zero-copy, on-demand access to external file data from a FlatBuffer.
+ * Provides lazy access to external files embedded inside a `.duc` buffer.
  *
- * Instead of eagerly parsing and copying every external file's binary data into JS memory,
- * this store keeps a reference to the original FlatBuffer Uint8Array and reads file bytes
- * only when explicitly requested. FlatBuffer `dataArray()` returns a zero-copy view
- * (a Uint8Array pointing into the original buffer), so no allocation occurs until the
- * consumer actually needs the data.
- *
- * Memory lifecycle:
- *   1. On parse: only metadata (~200 bytes per file) enters JS heap.
- *   2. On demand: `getFileData(fileId)` reads the zero-copy slice from the buffer.
- *   3. The caller (renderer/worker) uses the data, then lets it GC naturally.
- *   4. If the store is released, the buffer reference is dropped, freeing everything.
- *
- * This is the key to supporting 1000s of external files without RAM bloat.
+ * Instead of loading all file blobs into memory at parse time, this store
+ * keeps a reference to the raw `.duc` buffer and fetches individual files
+ * on demand via WASM calls.
  */
-var __rest = (this && this.__rest) || function (s, e) {
-    var t = {};
-    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
-        t[p] = s[p];
-    if (s != null && typeof Object.getOwnPropertySymbols === "function")
-        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
-            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
-                t[p[i]] = s[p[i]];
-        }
-    return t;
-};
-import * as flatbuffers from "flatbuffers";
-import { ExportedDataState as ExportedDataStateFb } from "./flatbuffers/duc";
 export class LazyExternalFileStore {
     constructor(buffer) {
-        /** Map from file id → lazy entry */
-        this._entries = new Map();
-        /** Map from element key → file id (the external_files vector uses element id as key) */
-        this._keyToFileId = new Map();
-        /**
-         * Files that were added at runtime (e.g. user uploading a new image).
-         * These aren't in the original FlatBuffer so we hold their data directly.
-         */
-        this._runtimeFiles = new Map();
-        this._buffer = buffer;
-        this._byteBuffer = new flatbuffers.ByteBuffer(buffer);
-        this._dataState = ExportedDataStateFb.getRootAsExportedDataState(this._byteBuffer);
-        this._indexMetadata();
-        console.info(`[LazyExternalFileStore] indexed ${this._entries.size} files from ${buffer.byteLength} byte buffer, ids: [${[...this._entries.keys()].map(k => k.slice(0, 12)).join(', ')}]`);
+        this.metadataCache = null;
+        this.runtimeFiles = new Map();
+        this.buffer = buffer;
     }
-    _indexMetadata() {
-        if (!this._dataState)
-            return;
-        const count = this._dataState.externalFilesLength();
-        for (let i = 0; i < count; i++) {
-            const entry = this._dataState.externalFiles(i);
-            if (!entry)
-                continue;
-            const key = entry.key();
-            const fileData = entry.value();
-            if (!key || !fileData)
-                continue;
-            const id = fileData.id();
-            if (!id)
-                continue;
-            const metadata = {
-                id,
-                mimeType: fileData.mimeType() || "application/octet-stream",
-                created: Number(fileData.created()),
-                lastRetrieved: Number(fileData.lastRetrieved()) || undefined,
-            };
-            const lazyEntry = { metadata, vectorIndex: i };
-            this._entries.set(id, lazyEntry);
-            this._keyToFileId.set(key, id);
-        }
+    get isReleased() {
+        return this.buffer === null;
     }
-    /** Total number of external files */
     get size() {
-        return this._entries.size + this._runtimeFiles.size;
+        return this.getMetadataMap().size + this.runtimeFiles.size;
     }
-    /** Whether a file with the given id exists */
+    /** Check if a file with this ID exists. */
     has(fileId) {
-        return this._entries.has(fileId) || this._runtimeFiles.has(fileId);
+        return this.runtimeFiles.has(fileId) || this.getMetadataMap().has(fileId);
     }
-    /** Get metadata only (no binary data copied) — ~200 bytes per file */
+    /** Get metadata (without data blob) for a specific file. */
     getMetadata(fileId) {
-        var _a, _b;
-        const runtime = this._runtimeFiles.get(fileId);
-        if (runtime) {
-            const { data: _ } = runtime, meta = __rest(runtime, ["data"]);
-            return meta;
+        const rt = this.runtimeFiles.get(fileId);
+        if (rt) {
+            return {
+                id: rt.id,
+                mimeType: rt.mimeType,
+                created: rt.created,
+                lastRetrieved: rt.lastRetrieved,
+                version: rt.version,
+            };
         }
-        return (_b = (_a = this._entries.get(fileId)) === null || _a === void 0 ? void 0 : _a.metadata) !== null && _b !== void 0 ? _b : null;
+        return this.getMetadataMap().get(fileId);
     }
-    /** Get all metadata entries (for UI listing, etc.) */
+    /** Get metadata for all files. */
     getAllMetadata() {
-        const result = {};
-        for (const [id, entry] of this._entries) {
-            result[id] = entry.metadata;
+        const result = [];
+        for (const meta of this.getMetadataMap().values()) {
+            result.push(meta);
         }
-        for (const [id, file] of this._runtimeFiles) {
-            const { data: _ } = file, meta = __rest(file, ["data"]);
-            result[id] = meta;
+        for (const [id, data] of this.runtimeFiles) {
+            if (!this.getMetadataMap().has(id)) {
+                result.push({
+                    id: data.id,
+                    mimeType: data.mimeType,
+                    created: data.created,
+                    lastRetrieved: data.lastRetrieved,
+                    version: data.version,
+                });
+            }
         }
         return result;
     }
-    /**
-     * Get full file data (metadata + binary bytes) ON DEMAND.
-     *
-     * For files from the original FlatBuffer, this returns a zero-copy Uint8Array
-     * view into the original buffer — no allocation for the file bytes themselves.
-     * The view is valid as long as this store hasn't been released.
-     *
-     * For runtime-added files, returns the data directly.
-     */
+    /** Fetch the full file data (including blob) for a specific file. */
     getFileData(fileId) {
-        const runtime = this._runtimeFiles.get(fileId);
-        if (runtime)
-            return runtime;
-        const entry = this._entries.get(fileId);
-        if (!entry || !this._dataState)
-            return null;
-        const fbEntry = this._dataState.externalFiles(entry.vectorIndex);
-        if (!fbEntry)
+        const rt = this.runtimeFiles.get(fileId);
+        if (rt)
+            return rt;
+        if (!this.buffer)
             return null;
-        const fileData = fbEntry.value();
-        if (!fileData)
+        const result = wasmGetExternalFile(this.buffer, fileId);
+        if (!result)
             return null;
-        const data = fileData.dataArray();
-        if (!data)
-            return null;
-        return Object.assign(Object.assign({}, entry.metadata), { data });
+        // WASM returns DucExternalFileEntry { key, value: DucExternalFileData }.
+        // Unwrap to get the actual file data.
+        const entry = result;
+        if (entry.value && typeof entry.value === "object") {
+            return entry.value;
+        }
+        return entry;
     }
-    /**
-     * Get a detached copy of the file data (allocates new ArrayBuffer).
-     * Use this when you need to transfer data to a worker or keep it beyond store lifetime.
-     */
+    /** Fetch file data and return a copy of the data buffer (safe for transfer). */
     getFileDataCopy(fileId) {
-        const fileDataRef = this.getFileData(fileId);
-        if (!fileDataRef)
+        const data = this.getFileData(fileId);
+        if (!data)
             return null;
-        return Object.assign(Object.assign({}, fileDataRef), { data: new Uint8Array(fileDataRef.data) });
+        return Object.assign(Object.assign({}, data), { data: new Uint8Array(data.data) });
     }
-    /**
-     * Add a file at runtime (user upload, paste, etc.).
-     * These files are held in memory since they aren't in the FlatBuffer.
-     */
-    addRuntimeFile(fileData) {
-        this._runtimeFiles.set(fileData.id, fileData);
+    /** Add a file at runtime (not persisted in .duc until next serialize). */
+    addRuntimeFile(fileId, data) {
+        this.runtimeFiles.set(fileId, data);
     }
-    /** Remove a runtime-added file */
+    /** Remove a runtime file. */
     removeRuntimeFile(fileId) {
-        this._runtimeFiles.delete(fileId);
+        return this.runtimeFiles.delete(fileId);
     }
-    /**
-     * Export all files as a standard DucExternalFiles record.
-     * This COPIES all file data eagerly — use only for serialization.
-     */
+    /** Export all files eagerly as a DucExternalFiles record. */
     toExternalFiles() {
         const result = {};
-        for (const [key, fileId] of this._keyToFileId) {
-            const fileData = this.getFileData(fileId);
-            if (fileData) {
-                result[key] = fileData;
+        if (this.buffer) {
+            const metas = this.getMetadataMap();
+            for (const [id] of metas) {
+                const data = this.getFileData(id);
+                if (data) {
+                    result[id] = data;
+                }
             }
         }
-        for (const [id, file] of this._runtimeFiles) {
-            result[id] = file;
+        for (const [id, data] of this.runtimeFiles) {
+            result[id] = data;
         }
         return result;
     }
-    /**
-     * Merge runtime files from the given DucExternalFiles map.
-     * Only adds files not already present in the store.
-     */
+    /** Merge files from another source (only adds missing). */
     mergeFiles(files) {
-        for (const [_key, fileData] of Object.entries(files)) {
-            if (!this.has(fileData.id)) {
-                this.addRuntimeFile(fileData);
+        for (const [id, data] of Object.entries(files)) {
+            if (!this.has(id)) {
+                this.runtimeFiles.set(id, data);
             }
         }
     }
-    /** Estimated RAM usage for metadata only (not counting the backing buffer) */
-    get estimatedMetadataBytes() {
-        var _a, _b;
-        let bytes = 0;
-        for (const [, entry] of this._entries) {
-            bytes += 200 + entry.metadata.id.length * 2 + entry.metadata.mimeType.length * 2;
-        }
-        for (const [, file] of this._runtimeFiles) {
-            bytes += 200 + ((_b = (_a = file.data) === null || _a === void 0 ? void 0 : _a.byteLength) !== null && _b !== void 0 ? _b : 0);
-        }
-        return bytes;
-    }
-    /**
-     * Release the FlatBuffer reference. After this, only runtime-added files remain accessible.
-     * Call this when switching documents or when the store is no longer needed.
-     */
+    /** Release the underlying buffer to free memory. */
     release() {
-        this._buffer = null;
-        this._byteBuffer = null;
-        this._dataState = null;
-        this._entries.clear();
-        this._keyToFileId.clear();
+        this.buffer = null;
+        this.metadataCache = null;
     }
-    /** Whether the store has been released */
-    get isReleased() {
-        return this._buffer === null;
+    getMetadataMap() {
+        if (!this.metadataCache) {
+            this.metadataCache = new Map();
+            if (this.buffer) {
+                const metas = wasmListExternalFiles(this.buffer);
+                if (metas) {
+                    for (const meta of metas) {
+                        this.metadataCache.set(meta.id, meta);
+                    }
+                }
+            }
+        }
+        return this.metadataCache;
     }
 }

package/dist/parse.d.ts

@@ -1,87 +1,21 @@
-import { FileSystemHandle } from 'browser-fs-access';
-import { CustomHatchPattern as CustomHatchPatternFb, DimensionToleranceStyle as DimensionToleranceStyleFb, DocumentGridConfig as DocumentGridConfigFb, DucBlock as DucBlockFb, DucDimensionStyle as DucDimensionStyleFb, DucDocStyle as DucDocStyleFb, DucExternalFileEntry, DucFeatureControlFrameStyle as DucFeatureControlFrameStyleFb, DucGlobalState as DucGlobalStateFb, DucGroup as DucGroupFb, DucHatchStyle as DucHatchStyleFb, DucHead as DucHeadFb, DucImageFilter as DucImageFilterFb, DucLayer as DucLayerFb, DucLeaderStyle as DucLeaderStyleFb, DucLine as DucLineFb, DucLineReference as DucLineReferenceFb, DucLocalState as DucLocalStateFb, DucPath as DucPathFb, DucPlotStyle as DucPlotStyleFb, DucPointBinding as DucPointBindingFb, DucPoint as DucPointFb, DucRegion as DucRegionFb, DucStackLikeStyles as DucStackLikeStylesFb, DucTableCellStyle as DucTableCellStyleFb, DucTableStyle as DucTableStyleFb, DucTextStyle as DucTextStyleFb, DucViewportStyle as DucViewportStyleFb, DucXRayStyle as DucXRayStyleFb, ElementBackground as ElementBackgroundFb, ElementContentBase as ElementContentBaseFb, ElementStroke as ElementStrokeFb, ElementWrapper, ExportedDataState as ExportedDataStateFb, GeometricPoint as GeometricPointFb, HatchPatternLine as HatchPatternLineFb, Margins as MarginsFb, PrimaryUnits as PrimaryUnitsFb, Standard as StandardFb, StrokeSides as StrokeSidesFb, StrokeStyle as StrokeStyleFb, TilingProperties as TilingPropertiesFb, VersionGraph as VersionGraphFb, _DucElementBase as _DucElementBaseFb, _DucElementStylesBase as _DucElementStylesBaseFb, _DucLinearElementBase as _DucLinearElementBaseFb, _DucStackBase as _DucStackBaseFb, _DucStackElementBase as _DucStackElementBaseFb } from "./flatbuffers/duc";
-import { RestoreConfig, RestoredDataState } from "./restore";
-import { Standard, StandardUnits } from "./technical";
-import { CustomHatchPattern, Dictionary, DocumentGridConfig, DucBlock, DucDimensionStyle, DucDocStyle, DucElement, DucExternalFileMetadata, DucExternalFiles, DucFeatureControlFrameStyle, DucGlobalState, DucGroup, DucHatchStyle, DucHead, DucImageFilter, DucLayer, DucLeaderStyle, DucLine, DucLineReference, DucLocalState, DucPath, DucPlotStyle, DucPoint, DucPointBinding, DucRegion, DucStackLikeStyles, DucTableCellStyle, DucTableStyle, DucTextStyle, DucViewportStyle, DucXRayStyle, ElementBackground, ElementContentBase, ElementStroke, GeometricPoint, HatchPatternLine, PlotLayout, StrokeSides, StrokeStyle, TilingProperties, VersionGraph, _DucElementBase, _DucElementStylesBase, _DucLinearElementBase, _DucStackBase, _DucStackElementBase } from "./types";
-export declare function parseGeometricPoint(point: GeometricPointFb): GeometricPoint;
-export declare function parsePoint(point: DucPointFb): DucPoint;
-export declare function parseMargins(margins: MarginsFb): PlotLayout["margins"];
-export declare function parseDocumentGridConfig(gridConfig: DocumentGridConfigFb): DocumentGridConfig;
-export declare function parseHead(head: DucHeadFb): DucHead;
-export declare function parseHatchPatternLine(line: HatchPatternLineFb): HatchPatternLine;
-export declare function parseCustomHatchPattern(pattern: CustomHatchPatternFb): CustomHatchPattern;
-export declare function parseHatchStyle(hatch: DucHatchStyleFb): DucHatchStyle;
-export declare function parseTilingProperties(tiling: TilingPropertiesFb): TilingProperties;
-export declare function parseImageFilter(filter: DucImageFilterFb): DucImageFilter;
-export declare function parseElementContentBase(content: ElementContentBaseFb): ElementContentBase;
-export declare function parseStrokeStyle(style: StrokeStyleFb): StrokeStyle;
-export declare function parseStrokeSides(sides: StrokeSidesFb): StrokeSides;
-export declare function parseElementStroke(stroke: ElementStrokeFb): ElementStroke;
-export declare function parseElementBackground(background: ElementBackgroundFb): ElementBackground;
-export declare function parseElementStylesBase(styles: _DucElementStylesBaseFb): _DucElementStylesBase;
-export declare function parseElementBase(base: _DucElementBaseFb): _DucElementBase;
-export declare function parseLineReference(ref: DucLineReferenceFb): DucLineReference;
-export declare function parseLine(line: DucLineFb): DucLine;
-export declare function parsePath(path: DucPathFb): DucPath;
-export declare function parsePointBinding(binding: DucPointBindingFb): DucPointBinding;
-export declare function parseLinearElementBase(base: _DucLinearElementBaseFb): _DucLinearElementBase;
-export declare function parseStackLikeStyles(styles: DucStackLikeStylesFb): DucStackLikeStyles;
-export declare function parseStackBase(base: _DucStackBaseFb): _DucStackBase;
-export declare function parseStackElementBase(base: _DucStackElementBaseFb): _DucStackElementBase;
-export declare function parseTextStyle(style: DucTextStyleFb): DucTextStyle;
-export declare function parseTableCellStyle(style: DucTableCellStyleFb): DucTableCellStyle;
-export declare function parseTableStyle(style: DucTableStyleFb): DucTableStyle;
-export declare function parsePlotStyle(style: DucPlotStyleFb): DucPlotStyle;
-export declare function parseViewportStyle(style: DucViewportStyleFb): DucViewportStyle;
-export declare function parseXRayStyle(style: DucXRayStyleFb): DucXRayStyle;
-export declare function parseLeaderStyle(style: DucLeaderStyleFb): DucLeaderStyle;
-export declare function parseDimensionToleranceStyle(style: DimensionToleranceStyleFb): DucDimensionStyle["tolerance"];
-export declare function parseDimensionStyle(style: DucDimensionStyleFb): DucDimensionStyle;
-export declare function parseFeatureControlFrameStyle(style: DucFeatureControlFrameStyleFb): DucFeatureControlFrameStyle;
-export declare function parseDocStyle(style: DucDocStyleFb): DucDocStyle;
-export declare function parseElementFromBinary(wrapper: ElementWrapper): DucElement | null;
-export declare function parseBlockFromBinary(block: DucBlockFb): DucBlock;
-export declare function parseDictionaryFromBinary(data: ExportedDataStateFb): Dictionary;
-export declare function parseExternalFilesFromBinary(entry: DucExternalFileEntry): DucExternalFiles;
-/**
- * Parse only metadata (no binary data) from an external file entry.
- * Used by the lazy file store to avoid copying file bytes into JS memory.
- */
-export declare function parseExternalFileMetadataFromBinary(entry: DucExternalFileEntry): {
-    key: string;
-    metadata: DucExternalFileMetadata;
-} | null;
-export declare function parseGlobalStateFromBinary(state: DucGlobalStateFb): DucGlobalState;
-export declare function parseGroupFromBinary(group: DucGroupFb): DucGroup;
-export declare function parseLayerFromBinary(layer: DucLayerFb): DucLayer;
-export declare function parseLocalStateFromBinary(state: DucLocalStateFb): DucLocalState;
-export declare function parseRegionFromBinary(region: DucRegionFb): DucRegion;
-export declare function parsePrimaryUnits(units: PrimaryUnitsFb): StandardUnits["primaryUnits"];
-export declare function parseStandardFromBinary(standard: StandardFb): Standard;
-export declare function parseThumbnailFromBinary(data: ExportedDataStateFb): Uint8Array | undefined;
-export declare function parseVersionGraphFromBinary(graph: VersionGraphFb | null): VersionGraph | null;
-export declare const parseDuc: (blob: Blob | File, fileHandle?: FileSystemHandle | null, restoreConfig?: RestoreConfig) => Promise<RestoredDataState>;
-import { LazyExternalFileStore } from "./lazy-files";
-export type LazyRestoredDataState = Omit<RestoredDataState, 'files'> & {
-    /** Lazy file store: only metadata is in memory, file bytes are read on-demand from the buffer */
+import { type ElementsConfig, type RestoreConfig, type RestoredDataState } from "./restore";
+export type { RestoredDataState };
+export type LazyRestoredDataState = RestoredDataState & {
     lazyFileStore: LazyExternalFileStore;
-    /** Legacy `files` field — always empty. Use lazyFileStore instead. */
-    files: DucExternalFiles;
 };
+import { LazyExternalFileStore } from "./lazy-files";
+export { LazyExternalFileStore };
 /**
- * Parse a .duc binary with lazy external file loading.
+ * Parse a `.duc` file (Blob/File) into a RestoredDataState.
  *
- * This is identical to `parseDuc` except:
- *  - External file BYTES are NOT copied into JS memory.
- *  - Only file metadata (~200 bytes per file) is parsed.
- *  - A `LazyExternalFileStore` is returned for on-demand data access.
- *  - The store holds a reference to the original Uint8Array buffer.
- *
- * Memory comparison for 500 PDFs averaging 5MB each:
- *  - parseDuc: files field holds 2.5GB of ArrayBuffer data in JS heap
- *  - parseDucLazy: ~100KB of metadata + the original buffer (referenced, not copied)
- *
- * @param buffer - The raw .duc file bytes (from storage/IndexedDB/filesystem)
- * @param restoreConfig - Optional restore configuration
+ * 1. Read file bytes
+ * 2. WASM parse (SQLite → Rust → JS via serde-wasm-bindgen)
+ * 3. Element fixups (stack element flattening)
+ * 4. restore() for defaults & migrations
+ */
+export declare function parseDuc(blob: Blob | File, _fileHandle?: FileSystemFileHandle | null, elementsConfig?: ElementsConfig, restoreConfig?: RestoreConfig): Promise<RestoredDataState>;
+/**
+ * Parse a `.duc` file lazily — returns everything EXCEPT external file data blobs.
+ * Use `LazyExternalFileStore` for on-demand file access.
  */
-export declare const parseDucLazy: (buffer: Uint8Array, restoreConfig?: RestoreConfig) => Promise<LazyRestoredDataState>;
+export declare function parseDucLazy(buffer: Uint8Array, elementsConfig?: ElementsConfig, restoreConfig?: RestoreConfig): Promise<LazyRestoredDataState>;