@bldrs-ai/conway 1.389.1221 → 1.394.1504

package/compiled/src/compat/web-ifc/store_preview_channel.js

@@ -0,0 +1,524 @@
+import { CanonicalMeshType } from "../../index.js";
+import IfcStepModel from "../../ifc/ifc_step_model.js";
+import { IfcGeometryExtraction } from "../../ifc/ifc_geometry_extraction.js";
+import { IfcBuildingStorey, IfcProduct, IfcProject, IfcRelAggregates, IfcUnitAssignment, } from "../../ifc/ifc4_gen/index.js";
+import { cursorIterator } from "../../indexing/cursor_utilities.js";
+import { WindowedStepBufferProvider } from "../../step/step_buffer_provider.js";
+import * as glmatrix from "gl-matrix";
+import { composeTransformF64, deriveCoordinationF64, IDENTITY_MAT4, NORMALIZE_MAT_F64, } from "./coordination_f64.js";
+import { emitSpatialStructureImposters } from "./spatial_imposter.js";
+import { releaseModelGeometry, } from "./streamed_preview_channel.js";
+/* eslint-disable no-magic-numbers */
+const TICK_INTERVAL_MS = 150;
+const TICK_INTERVAL_MAX_MS = 600;
+const TICK_INTERVAL_GROWTH = 1.1;
+/**
+ * Extraction + capture budget per tick, mirroring
+ * {@link StreamedPreviewChannel}'s. The store path used to extract
+ * exactly ONE product per tick on the interval below, i.e. 2-7
+ * products/second — which is why PSB's preview arrived girder by girder
+ * where the resident path delivered batches (conway#518).
+ */
+const TICK_BUDGET_MS = 20;
+/**
+ * Products a single tick may attempt, whatever the clock says.
+ *
+ * `deferDanglingPlacements` (Share#1744) defers any product whose
+ * placement records the prefix does not hold yet, and Revit writes
+ * per-product placements toward the file tail — so early ticks meet long
+ * runs of products that cannot extract. Each such attempt still pages
+ * source through the windowed provider, so a run of them is not free;
+ * this caps the run rather than letting the deadline be the only bound.
+ */
+const TICK_MAX_ATTEMPTS = 32;
+const FIRST_GENERATION_MIN_RECORDS = 1024;
+const GENERATION_GROWTH_FACTOR = 2;
+const DEFAULT_MAX_PREVIEW_UNITS = 512;
+const DEFAULT_MAX_PREVIEW_BYTES = 48 * 1024 * 1024;
+const FLOATS_PER_VERTEX = 6;
+const BYTES_PER_FLOAT = 4;
+const DEFAULT_COLOR = [0.8, 0.8, 0.8, 1];
+const FLUSH_BUDGET_MS = 100;
+/**
+ * Parse-time placed-mesh preview for a windowed (store-backed) open.
+ *
+ * Twin of {@link StreamedPreviewChannel}: same capture math and
+ * COORDINATE_TO_ORIGIN frame, same time-budgeted tick, but the prefix
+ * model pages product closures from `store` instead of a resident
+ * buffer. After each product the pins drop, so peak source residency
+ * stays the windowed LRU — not the file.
+ *
+ * It also runs the spatial-structure walk off its own prefix
+ * generations, so the building's skeleton goes up in the first seconds
+ * of a long parse rather than after it — see
+ * {@link maybeEmitEarlySpatialPlates_}.
+ */
+export class StorePreviewChannel {
+    /**
+     * @param store_ External source (OPFS / file).
+     * @param sink_ Live columnar sink the parse is filling.
+     * @param conwaywasm_ Shared wasm wrapper.
+     * @param coordinateToOrigin_ Open COORDINATE_TO_ORIGIN.
+     * @param onMesh_ Preview consumer.
+     * @param maxUnits Cap on products preview-extracted.
+     * @param maxBytes Cap on copied payload bytes.
+     * @param firstGenerationMinRecords Records before the first snapshot.
+     */
+    constructor(store_, sink_, conwaywasm_, coordinateToOrigin_, onMesh_, maxUnits = DEFAULT_MAX_PREVIEW_UNITS, maxBytes = DEFAULT_MAX_PREVIEW_BYTES, firstGenerationMinRecords = FIRST_GENERATION_MIN_RECORDS) {
+        this.store_ = store_;
+        this.sink_ = sink_;
+        this.conwaywasm_ = conwaywasm_;
+        this.coordinateToOrigin_ = coordinateToOrigin_;
+        this.onMesh_ = onMesh_;
+        this.maxUnits = maxUnits;
+        this.maxBytes = maxBytes;
+        this.firstGenerationMinRecords = firstGenerationMinRecords;
+        this.stopped_ = false;
+        this.emittedUnits_ = 0;
+        this.emittedBytes_ = 0;
+        this.unitOrdinal_ = 0;
+        this.lastInlineTick_ = 0;
+        this.tickIntervalMs_ = TICK_INTERVAL_MS;
+        this.lastSnapshotRecords_ = 0;
+        this.lastFailedSnapshotRecords_ = 0;
+        // Fields rather than the constants directly, so a test can lift the
+        // wall-clock bounds the way ifc_api_preview_channel.test.ts already
+        // lifts the streamed channel's: asserting "this tick ran N products"
+        // against a real 20ms budget is a coin flip on a loaded runner.
+        this.tickBudgetMs_ = TICK_BUDGET_MS;
+        this.tickMaxAttempts_ = TICK_MAX_ATTEMPTS;
+        /** Set once a prefix-generation spatial walk has emitted a plate. */
+        this.earlyPlatesEmitted_ = 0;
+        /**
+         * Whether those plates were composed under the walk's own fallback
+         * frame because no preview instance had latched one yet. They get one
+         * re-emission on a later generation once a real frame exists — see
+         * {@link maybeEmitEarlySpatialPlates_}.
+         */
+        this.earlyPlatesUsedFallbackFrame_ = false;
+        this.emittedGeometry_ = new Set();
+    }
+    get capped() {
+        return this.emittedUnits_ >= this.maxUnits ||
+            this.emittedBytes_ >= this.maxBytes;
+    }
+    get emittedUnits() {
+        return this.emittedUnits_;
+    }
+    /** Products the current generation would run (test seam). */
+    get productCount() {
+        return this.generation_?.products.length ?? 0;
+    }
+    /**
+     * Plates the prefix-generation spatial walk emitted, 0 while it has
+     * not succeeded (see {@link maybeEmitEarlySpatialPlates_}).
+     */
+    get earlyPlateCount() {
+        return this.earlyPlatesEmitted_;
+    }
+    /**
+     * One cadence-gated, time-budgeted tick. No-op until the interval
+     * elapses or the index can support a generation. Safe to call from
+     * parse progress.
+     *
+     * **Cadence and budget interact once, not twice.** The interval is
+     * measured tick-start to tick-start (`lastInlineTick_` is stamped
+     * before the work, as on the streamed channel), so a tick that spends
+     * its whole budget shortens the gap to the next one by that budget
+     * rather than adding to it. And the decay is charged only to ticks
+     * that actually attempted a product: a tick that found no generation —
+     * which is every tick through the opening seconds of a big parse, when
+     * the index has not yet reached FIRST_GENERATION_MIN_RECORDS — used to
+     * decay the interval anyway, so by the time there was anything to
+     * extract the cadence had already cooled toward
+     * TICK_INTERVAL_MAX_MS. Deliberately divergent from
+     * {@link StreamedPreviewChannel}, which decays unconditionally; the
+     * cost is that a channel which never finds a generation keeps probing
+     * at TICK_INTERVAL_MS, and a probe that finds nothing is a record
+     * count compared against two thresholds.
+     *
+     * @return {Promise<void>} Settles when this tick's products (if any)
+     * are captured.
+     */
+    async maybeTickAsync() {
+        if (this.stopped_ || this.capped) {
+            return;
+        }
+        const now = Date.now();
+        if (now - this.lastInlineTick_ < this.tickIntervalMs_) {
+            return;
+        }
+        this.lastInlineTick_ = now;
+        try {
+            if ((await this.tickBudgeted_()) > 0) {
+                this.tickIntervalMs_ = Math.min(this.tickIntervalMs_ * TICK_INTERVAL_GROWTH, TICK_INTERVAL_MAX_MS);
+            }
+        }
+        catch {
+            this.stopped_ = true;
+        }
+    }
+    /**
+     * Attempt products until the time budget or the attempt cap runs out,
+     * whichever comes first.
+     *
+     * @return {Promise<number>} Products attempted this tick.
+     */
+    async tickBudgeted_() {
+        const deadline = Date.now() + this.tickBudgetMs_;
+        let attempts = 0;
+        while (!this.capped &&
+            attempts < this.tickMaxAttempts_ &&
+            Date.now() < deadline) {
+            if (!(await this.tickOnce_())) {
+                break;
+            }
+            ++attempts;
+        }
+        return attempts;
+    }
+    /**
+     * After parse: extract a short burst so small files (and prefixes
+     * that never crossed a progress tick) still emit placed meshes
+     * before open returns.
+     *
+     * @return {Promise<void>} Settles when the burst ends.
+     */
+    async flushAsync() {
+        if (this.stopped_ || this.capped) {
+            return;
+        }
+        const deadline = Date.now() + FLUSH_BUDGET_MS;
+        try {
+            while (!this.capped && Date.now() < deadline) {
+                const did = await this.tickOnce_();
+                if (!did) {
+                    break;
+                }
+            }
+        }
+        catch {
+            this.stopped_ = true;
+        }
+    }
+    /** Drop the throwaway prefix generation. Idempotent. */
+    stop() {
+        this.stopped_ = true;
+        this.disposeGeneration_();
+    }
+    /**
+     * Test seam: drain until the current sink is exhausted or a cap hits.
+     *
+     * @return {Promise<void>} Settles when idle.
+     */
+    async drainForTest() {
+        this.lastInlineTick_ = 0;
+        this.tickIntervalMs_ = 0;
+        for (;;) {
+            if (this.capped) {
+                return;
+            }
+            const did = await this.tickOnce_();
+            if (!did) {
+                return;
+            }
+        }
+    }
+    /**
+     * @return {Promise<boolean>} True when a product was attempted.
+     */
+    async tickOnce_() {
+        if (!(await this.ensureGeneration_())) {
+            return false;
+        }
+        const active = this.generation_;
+        if (this.unitOrdinal_ >= active.products.length) {
+            return false;
+        }
+        const localID = active.products[this.unitOrdinal_++];
+        const model = active.model;
+        const pins = new Set();
+        const leafSpans = [];
+        try {
+            await active.extraction.ensureResidentForProductExtract(localID, pins, leafSpans);
+            if (active.extraction.extractProductGeometryByLocalID(localID)) {
+                this.captureNewInstances_();
+                ++this.emittedUnits_;
+            }
+        }
+        catch {
+            // Forward-ref / unplaced — durable pump extracts later.
+        }
+        finally {
+            model.releaseSourceViews(pins);
+            model.unpinLocalIDs(pins);
+            for (const span of leafSpans) {
+                model.unpinAddressRange(span.address, span.length);
+            }
+        }
+        return true;
+    }
+    /**
+     * @return {Promise<boolean>} True when a generation with pending units exists.
+     */
+    async ensureGeneration_() {
+        const active = this.generation_;
+        if (active !== void 0 && this.unitOrdinal_ < active.products.length) {
+            return true;
+        }
+        const records = this.sink_.topLevelCount;
+        if (records < this.firstGenerationMinRecords) {
+            return false;
+        }
+        if (active !== void 0 &&
+            records < this.lastSnapshotRecords_ * GENERATION_GROWTH_FACTOR) {
+            return false;
+        }
+        if (records < this.lastFailedSnapshotRecords_ * GENERATION_GROWTH_FACTOR) {
+            return false;
+        }
+        try {
+            const columns = this.sink_.snapshot();
+            const provider = new WindowedStepBufferProvider(this.store_);
+            const model = new IfcStepModel(void 0, columns, provider);
+            const products = [];
+            for (const localID of cursorIterator(model.typeIndex.cursor(...IfcProduct.query))) {
+                products.push(localID);
+            }
+            if (products.length === 0) {
+                this.lastFailReason = `no products in ${records} records`;
+                return false;
+            }
+            if (products.length <= this.unitOrdinal_) {
+                this.lastFailReason = `ordinal ${this.unitOrdinal_} >= ${products.length}`;
+                return false;
+            }
+            const extraction = new IfcGeometryExtraction(this.conwaywasm_, model);
+            extraction.quietRecoverableLogging = true;
+            extraction.deferDanglingPlacements = true;
+            const prepPins = await extraction.ensureResidentForDemandPrep();
+            try {
+                extraction.prepareDemandExtraction(true);
+            }
+            finally {
+                model.releaseSourceViews(prepPins);
+                model.unpinLocalIDs(prepPins);
+            }
+            this.disposeGeneration_();
+            this.generation_ = {
+                model,
+                extraction,
+                products,
+                capturedCounts: new Map(),
+            };
+            this.lastSnapshotRecords_ = records;
+            this.lastFailedSnapshotRecords_ = 0;
+            this.lastFailReason = void 0;
+            await this.maybeEmitEarlySpatialPlates_(model, extraction);
+            return true;
+        }
+        catch (error) {
+            this.lastFailedSnapshotRecords_ = records;
+            this.lastFailReason = error instanceof Error ? error.message : String(error);
+            return false;
+        }
+    }
+    /**
+     * Put the spatial skeleton up from a PREFIX generation, seconds into
+     * the parse instead of after it (conway#518).
+     *
+     * The spatial records sit at the file head on Revit exports — on PSB
+     * `IFCSITE` is record #211 — and a prefix generation is already
+     * everything the walk needs: a columns snapshot over a windowed
+     * provider, with `prepareDemandExtraction(true)` done, which is what
+     * makes `getLinearScalingFactor()` report real units rather than 1.
+     * So the plates can lead the first extracted product rather than
+     * trailing the whole parse, which is what the store path's three
+     * stacked latency gates (one product per tick, early products
+     * deferring on late placements, the walk waiting for the full index)
+     * added up to: a blank screen through the first ~60% of a 16.7 s
+     * parse.
+     *
+     * **Runs at most once successfully, on the parse's cooperative path.**
+     * The walk is reused as-is, with no internal budget: it awaits a
+     * residency ensure per spatial node and per sampled product placement
+     * (PRODUCT_SAMPLE per node), so its cost tracks the prefix's spatial
+     * tree, not the file. On the first qualifying generation that tree is
+     * small by construction — the storeys exist but few
+     * `IfcRelContainedInSpatialStructure` do — which is the same property
+     * that makes these early plates coarse. Retries are bounded without a
+     * counter: generations only rebuild on a GENERATION_GROWTH_FACTOR
+     * doubling of the index, so there are O(log records) of them.
+     *
+     * **The end-of-parse walk in the proxy is the refresh, not a
+     * duplicate.** It re-emits the same expressIDs with contained-product
+     * samples the prefix did not hold and under the frame the channel
+     * latched, and the consumer contract is that an `aabb` payload
+     * REPLACES the prior plate for its expressID. Double emission is the
+     * design; see PreviewMeshPayload.
+     *
+     * **Known gap in that contract, and it is a gap, not a subtlety.**
+     * Replacement only reaches nodes the final walk still emits, and the
+     * emit set is not stable between the two: `aabbMostlyEqual` collapses
+     * a single-child wrapper (Project over Site over Building) once its
+     * box matches its child's, which a prefix generation's degenerate
+     * boxes may not yet do. A wrapper emitted here and collapsed there is
+     * never replaced and never removed — the payload contract has no
+     * delete — so its early box lingers as one small plate until Share
+     * tears the preview scenery down at load end. Bounded (the wrapper
+     * chain is a handful of nodes, and the collapse needs exactly one
+     * child), but real. Closing it properly needs either a delete/
+     * generation channel in the payload contract or an emit set the two
+     * walks agree on, and both are consumer-side changes.
+     *
+     * @param model The prefix model.
+     * @param extraction Its prepared extraction.
+     * @return {Promise<void>} Settles when the walk has run or been skipped.
+     */
+    async maybeEmitEarlySpatialPlates_(model, extraction) {
+        // Same choice the proxy's imposterCoordination() makes: the latched
+        // preview frame when the open asked to coordinate, else identity.
+        const coordination = this.coordinateToOrigin_ ?
+            this.coordinationMatrix : IDENTITY_MAT4;
+        // This runs as a generation is BUILT, before that tick captures its
+        // first product — and capture is the only thing that latches a
+        // frame. So the first walk on a coordinating open always finds
+        // `undefined` here and takes the walk's fallback derivation, which
+        // its own docs describe as capable of a full site-offset on a file
+        // that bakes absolute coordinates into vertices behind identity
+        // placements. Waiting for a latched frame instead would put the
+        // skeleton back behind the deferred-placement gate this whole change
+        // exists to get in front of, so: emit now, and re-emit ONCE on a
+        // later generation as soon as a real frame exists. That second
+        // emission is exactly what the replace-by-expressID contract is for.
+        const usingFallbackFrame = coordination === void 0;
+        const canImproveFrame = this.earlyPlatesUsedFallbackFrame_ && !usingFallbackFrame;
+        if (this.earlyPlatesEmitted_ > 0 && !canImproveFrame) {
+            return;
+        }
+        try {
+            // Cheap prefix-sum reads, so this is affordable per generation.
+            // No storeys or no aggregate chain means the walk would emit
+            // nothing but still pay for the type scans. IfcProject and
+            // IfcUnitAssignment are in the guard for a different reason:
+            // `extractLinearScalingFactor` returns with the factor still at 1
+            // when it cannot resolve the project's UnitsInContext (it only
+            // logs), and a silent 1 on a millimetre model scales the whole
+            // skeleton 1000x — the conway#515 symptom. The project alone is
+            // not enough: a valid file may forward-reference its
+            // IfcUnitAssignment, so a prefix can hold the project while the
+            // units record is still ahead of the parse cursor, and because a
+            // successful walk latches `earlyPlatesEmitted_`, mis-scaled early
+            // plates would stand until the frame-improvement re-emit or the
+            // post-parse refresh (Codex review on #519). The guard therefore
+            // defers plates until the units record is indexed; a file with
+            // genuinely no unit assignment never emits EARLY plates and is
+            // covered by the end-of-parse walk instead.
+            if (model.typeCount(IfcProject) < 1 ||
+                model.typeCount(IfcUnitAssignment) < 1 ||
+                model.typeCount(IfcBuildingStorey) < 1 ||
+                model.typeCount(IfcRelAggregates) < 1) {
+                return;
+            }
+            const emitted = await emitSpatialStructureImposters(model, this.onMesh_, coordination, extraction.getLinearScalingFactor());
+            if (emitted > 0) {
+                this.earlyPlatesEmitted_ = emitted;
+                this.earlyPlatesUsedFallbackFrame_ = usingFallbackFrame;
+            }
+        }
+        catch {
+            // A failed early walk retries on the next generation; the
+            // end-of-parse walk covers it regardless.
+        }
+    }
+    disposeGeneration_() {
+        const active = this.generation_;
+        if (active === void 0) {
+            return;
+        }
+        try {
+            releaseModelGeometry(active.model.geometry);
+        }
+        catch {
+            // Never let a free break the open.
+        }
+        this.generation_ = void 0;
+    }
+    /**
+     * Copy newly extracted instances out of wasm — same placement math
+     * as {@link StreamedPreviewChannel}.
+     */
+    captureNewInstances_() {
+        const active = this.generation_;
+        if (active === void 0) {
+            return;
+        }
+        const { extraction, capturedCounts } = active;
+        const scene = extraction.scene;
+        const seenThisPass = new Map();
+        for (const walked of scene.walk()) {
+            const [, nativeTransform, geometry, material, entity] = walked;
+            if (entity?.localID === void 0 || entity.expressID === void 0) {
+                continue;
+            }
+            const walkIndex = seenThisPass.get(entity.localID) ?? 0;
+            seenThisPass.set(entity.localID, walkIndex + 1);
+            if (walkIndex < (capturedCounts.get(entity.localID) ?? 0)) {
+                continue;
+            }
+            capturedCounts.set(entity.localID, walkIndex + 1);
+            if (geometry.type !== CanonicalMeshType.BUFFER_GEOMETRY ||
+                geometry.temporary) {
+                continue;
+            }
+            const material_ = material ?? {
+                name: "",
+                baseColor: DEFAULT_COLOR,
+                legacyColor: DEFAULT_COLOR,
+                doubleSided: true,
+                blend: 0,
+            };
+            let nativePt;
+            if (this.coordinationMatrix === void 0 && this.coordinateToOrigin_) {
+                nativePt = geometry.geometry.getPoint(0);
+            }
+            const center = geometry.geometry.normalize();
+            const geometryExpressID = active.model.getElementByLocalID(geometry.localID)?.expressID ??
+                geometry.localID;
+            const geometryTransform = nativeTransform?.getValues();
+            if (this.coordinationMatrix === void 0 && this.coordinateToOrigin_) {
+                this.coordinationMatrix = deriveCoordinationF64(geometryTransform, nativePt, NORMALIZE_MAT_F64, extraction.getLinearScalingFactor());
+            }
+            const coordination = this.coordinationMatrix ?? glmatrix.mat4.create();
+            const newTransform = composeTransformF64(coordination, geometryTransform, center);
+            const payload = {
+                expressID: entity.expressID,
+                geometryExpressID,
+                color: {
+                    x: material_.legacyColor[0],
+                    y: material_.legacyColor[1],
+                    z: material_.legacyColor[2],
+                    w: material_.legacyColor[3],
+                },
+                flatTransformation: Array.from(newTransform),
+            };
+            if (!this.emittedGeometry_.has(geometryExpressID)) {
+                const nativeGeometry = geometry.geometry;
+                const vertexData = this.conwaywasm_.floatHeapSlice(nativeGeometry.GetVertexData(), nativeGeometry.GetVertexDataSize()).slice();
+                const indexData = this.conwaywasm_.uint32HeapSlice(nativeGeometry.GetIndexData(), nativeGeometry.GetIndexDataSize()).slice();
+                if (vertexData.length < FLOATS_PER_VERTEX || indexData.length === 0) {
+                    continue;
+                }
+                payload.vertexData = vertexData;
+                payload.indexData = indexData;
+                this.emittedGeometry_.add(geometryExpressID);
+                this.emittedBytes_ +=
+                    (vertexData.length + indexData.length) * BYTES_PER_FLOAT;
+            }
+            this.onMesh_(payload);
+            if (this.emittedBytes_ >= this.maxBytes) {
+                return;
+            }
+        }
+    }
+}

package/compiled/src/compat/web-ifc/store_preview_channel.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=store_preview_channel.test.d.ts.map

package/compiled/src/compat/web-ifc/store_preview_channel.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store_preview_channel.test.d.ts","sourceRoot":"","sources":["../../../../src/compat/web-ifc/store_preview_channel.test.ts"],"names":[],"mappings":""}