@sixb/core 0.1.15 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/index.js +25 -25
- package/dist/actions/types/context.d.ts +2 -2
- package/dist/actions/types/context.d.ts.map +1 -1
- package/dist/actions/worker.js +1 -1
- package/dist/agents/index.js +41 -40
- package/dist/agents/index.js.map +1 -1
- package/dist/auth/index.js +5 -5
- package/dist/authorization/index.js +2 -2
- package/dist/blob-storage/download-urls.d.ts +41 -0
- package/dist/blob-storage/download-urls.d.ts.map +1 -0
- package/dist/blob-storage/execution.d.ts +9 -1
- package/dist/blob-storage/execution.d.ts.map +1 -1
- package/dist/blob-storage/index.js +5 -3
- package/dist/blob-storage/index.js.map +1 -1
- package/dist/blob-storage/internal.d.ts +2 -0
- package/dist/blob-storage/internal.d.ts.map +1 -0
- package/dist/blob-storage/internal.js +11 -0
- package/dist/blob-storage/internal.js.map +9 -0
- package/dist/bootstrap/index.js +45 -44
- package/dist/bootstrap/index.js.map +1 -1
- package/dist/{chunk-wem5vqjw.js → chunk-10styjgk.js} +18 -13
- package/dist/{chunk-wem5vqjw.js.map → chunk-10styjgk.js.map} +3 -3
- package/dist/{chunk-hgqpcmyr.js → chunk-1hqwggsn.js} +47 -19
- package/dist/chunk-1hqwggsn.js.map +12 -0
- package/dist/{chunk-q3grgzp1.js → chunk-23xs4qp9.js} +2 -2
- package/dist/{chunk-vef6qa14.js → chunk-2am3db31.js} +2 -2
- package/dist/{chunk-fj1dvp37.js → chunk-2vhg6ysa.js} +1 -1
- package/dist/{chunk-p0nbkys4.js → chunk-33saed7g.js} +1 -1
- package/dist/{chunk-mdnw7nj8.js → chunk-3rrezgyc.js} +33 -8
- package/dist/chunk-3rrezgyc.js.map +15 -0
- package/dist/{chunk-jgwb25gc.js → chunk-4as3wt1c.js} +13 -5
- package/dist/chunk-4as3wt1c.js.map +12 -0
- package/dist/chunk-4ra628t7.js +88 -0
- package/dist/chunk-4ra628t7.js.map +10 -0
- package/dist/{chunk-mmac1mp6.js → chunk-52c5zb0z.js} +2 -2
- package/dist/{chunk-8fjmqjyx.js → chunk-5be2ath4.js} +1 -1
- package/dist/{chunk-2fv8sajb.js → chunk-5thzcb8g.js} +2 -2
- package/dist/{chunk-fjw417jf.js → chunk-5yd8cchh.js} +1143 -784
- package/dist/chunk-5yd8cchh.js.map +66 -0
- package/dist/{chunk-2tz5vnvz.js → chunk-6wbm6ymz.js} +235 -41
- package/dist/{chunk-2tz5vnvz.js.map → chunk-6wbm6ymz.js.map} +8 -6
- package/dist/{chunk-py2dw8cw.js → chunk-7ecp4v7a.js} +26 -19
- package/dist/chunk-7ecp4v7a.js.map +17 -0
- package/dist/{chunk-f85hpqnm.js → chunk-99nd4rc1.js} +3 -3
- package/dist/{chunk-f85hpqnm.js.map → chunk-99nd4rc1.js.map} +3 -3
- package/dist/{chunk-1jvq82rp.js → chunk-9g6jz9kk.js} +5 -5
- package/dist/{chunk-39tqv9hz.js → chunk-9se093qz.js} +5 -6
- package/dist/{chunk-39tqv9hz.js.map → chunk-9se093qz.js.map} +3 -4
- package/dist/{chunk-8w27xf9e.js → chunk-a7er86jd.js} +1 -1
- package/dist/{chunk-zdkcdshk.js → chunk-aq39q04r.js} +41 -3
- package/dist/{chunk-zdkcdshk.js.map → chunk-aq39q04r.js.map} +4 -3
- package/dist/chunk-b9affek3.js +261 -0
- package/dist/chunk-b9affek3.js.map +12 -0
- package/dist/{chunk-bxx3ms5t.js → chunk-c2rga5cx.js} +2 -2
- package/dist/{chunk-dsfw1bdr.js → chunk-cnb5kv05.js} +2 -2
- package/dist/chunk-d62kjtbg.js.map +2 -2
- package/dist/{chunk-dqkd3kdj.js → chunk-dam456ve.js} +234 -35
- package/dist/{chunk-dqkd3kdj.js.map → chunk-dam456ve.js.map} +7 -5
- package/dist/{chunk-n5t4mq6z.js → chunk-e67ynxcr.js} +3 -3
- package/dist/{chunk-xnx96gfy.js → chunk-ere85b8y.js} +17 -8
- package/dist/chunk-ere85b8y.js.map +15 -0
- package/dist/{chunk-k4kkp58d.js → chunk-ewqa58h9.js} +12 -2
- package/dist/chunk-ewqa58h9.js.map +11 -0
- package/dist/{chunk-6q7c68yx.js → chunk-f1xh7a32.js} +3 -3
- package/dist/{chunk-z708gemf.js → chunk-fggq3d9k.js} +450 -799
- package/dist/chunk-fggq3d9k.js.map +49 -0
- package/dist/{chunk-7jc21943.js → chunk-gf7yht60.js} +4 -4
- package/dist/chunk-h8teetr0.js +9 -0
- package/dist/chunk-h8teetr0.js.map +10 -0
- package/dist/chunk-hn2r7z41.js.map +1 -1
- package/dist/{chunk-ffwvdgk1.js → chunk-jjrn5v0p.js} +1 -1
- package/dist/{chunk-r5sc1d2b.js → chunk-jnfxmr3f.js} +1 -1
- package/dist/{chunk-y0zmae99.js → chunk-k4g51s47.js} +3 -3
- package/dist/{chunk-a0073121.js → chunk-kpx7jrmn.js} +3 -1
- package/dist/chunk-kpx7jrmn.js.map +12 -0
- package/dist/{chunk-p1683c0v.js → chunk-kyjcwqz6.js} +5 -2
- package/dist/{chunk-p1683c0v.js.map → chunk-kyjcwqz6.js.map} +3 -3
- package/dist/{chunk-3c90jvfj.js → chunk-pja9jfz6.js} +742 -583
- package/dist/chunk-pja9jfz6.js.map +19 -0
- package/dist/chunk-pqajwytd.js.map +1 -1
- package/dist/{chunk-mr60an99.js → chunk-qczk96d9.js} +22 -2
- package/dist/{chunk-mr60an99.js.map → chunk-qczk96d9.js.map} +3 -3
- package/dist/{chunk-s3csryxe.js → chunk-rj69qcrd.js} +8 -2
- package/dist/chunk-rj69qcrd.js.map +11 -0
- package/dist/{chunk-d8yes9yx.js → chunk-rpxz486p.js} +32 -5
- package/dist/{chunk-d8yes9yx.js.map → chunk-rpxz486p.js.map} +4 -4
- package/dist/{chunk-drz59ve0.js → chunk-rx7h5fdx.js} +1 -1
- package/dist/{chunk-ky7zpbtt.js → chunk-s7ftxrdr.js} +106 -8
- package/dist/chunk-s7ftxrdr.js.map +22 -0
- package/dist/{chunk-yfppyaty.js → chunk-sk0hcjsz.js} +1 -1
- package/dist/{chunk-qne45xqq.js → chunk-svabwqdq.js} +2 -2
- package/dist/{chunk-xr7y35yj.js → chunk-t9shqx3w.js} +2 -2
- package/dist/{chunk-4jph1t3v.js → chunk-tjjgy3z4.js} +27 -24
- package/dist/{chunk-4jph1t3v.js.map → chunk-tjjgy3z4.js.map} +4 -4
- package/dist/{chunk-ex8s8wsk.js → chunk-vp3z1ckh.js} +42 -10
- package/dist/{chunk-ex8s8wsk.js.map → chunk-vp3z1ckh.js.map} +7 -6
- package/dist/{chunk-adbve9zt.js → chunk-vre348sb.js} +2 -2
- package/dist/{chunk-a4g5yjm0.js → chunk-w9yycd3t.js} +1 -1
- package/dist/{chunk-g4rbm4a6.js → chunk-wt4szfec.js} +379 -331
- package/dist/chunk-wt4szfec.js.map +14 -0
- package/dist/{chunk-wd34wnsy.js → chunk-xcy5ddha.js} +6 -6
- package/dist/{chunk-3nx8vj5a.js → chunk-xyvy73tj.js} +5 -3
- package/dist/chunk-xyvy73tj.js.map +12 -0
- package/dist/datasets/write.js +1 -1
- package/dist/execution/agent.js +38 -37
- package/dist/execution/agent.js.map +1 -1
- package/dist/execution/authorized-object-reader.d.ts +3 -1
- package/dist/execution/authorized-object-reader.d.ts.map +1 -1
- package/dist/execution/primitive.js +36 -35
- package/dist/execution/primitive.js.map +2 -2
- package/dist/execution/request.js +37 -36
- package/dist/execution/request.js.map +2 -2
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -59
- package/dist/index.js.map +1 -1
- package/dist/lake-storage/in-memory.d.ts +5 -1
- package/dist/lake-storage/in-memory.d.ts.map +1 -1
- package/dist/lake-storage/index.d.ts +1 -1
- package/dist/lake-storage/index.d.ts.map +1 -1
- package/dist/lake-storage/index.js +2 -2
- package/dist/lake-storage/types.d.ts +26 -0
- package/dist/lake-storage/types.d.ts.map +1 -1
- package/dist/maintenance/ontology-maintenance.d.ts +5 -1
- package/dist/maintenance/ontology-maintenance.d.ts.map +1 -1
- package/dist/maintenance/types.d.ts +1 -4
- package/dist/maintenance/types.d.ts.map +1 -1
- package/dist/materialization/event-envelopes.d.ts +20 -0
- package/dist/materialization/event-envelopes.d.ts.map +1 -0
- package/dist/materialization/events.d.ts +29 -3
- package/dist/materialization/events.d.ts.map +1 -1
- package/dist/materialization/index.d.ts +4 -1
- package/dist/materialization/index.d.ts.map +1 -1
- package/dist/materialization/index.js +9 -3
- package/dist/materialization/index.js.map +1 -1
- package/dist/materializer/edits/plan.d.ts +2 -10
- package/dist/materializer/edits/plan.d.ts.map +1 -1
- package/dist/materializer/effective/build-events.d.ts +5 -21
- package/dist/materializer/effective/build-events.d.ts.map +1 -1
- package/dist/materializer/effective/vector-indexing.d.ts +2 -3
- package/dist/materializer/effective/vector-indexing.d.ts.map +1 -1
- package/dist/materializer/effective/vectors.d.ts +2 -3
- package/dist/materializer/effective/vectors.d.ts.map +1 -1
- package/dist/materializer/execution/scope.d.ts +1 -11
- package/dist/materializer/execution/scope.d.ts.map +1 -1
- package/dist/materializer/execution/work-executor.d.ts +5 -7
- package/dist/materializer/execution/work-executor.d.ts.map +1 -1
- package/dist/materializer/execution/work-records.d.ts +1 -3
- package/dist/materializer/execution/work-records.d.ts.map +1 -1
- package/dist/materializer/index.js +31 -32
- package/dist/materializer/index.js.map +1 -1
- package/dist/materializer/projections/replace.d.ts.map +1 -1
- package/dist/materializer/projections/replacement-plan.d.ts +8 -11
- package/dist/materializer/projections/replacement-plan.d.ts.map +1 -1
- package/dist/materializer/shared/batching.d.ts +5 -0
- package/dist/materializer/shared/batching.d.ts.map +1 -1
- package/dist/materializer/telemetry/plan.d.ts +2 -8
- package/dist/materializer/telemetry/plan.d.ts.map +1 -1
- package/dist/models/audio/catalog.d.ts +20 -0
- package/dist/models/audio/catalog.d.ts.map +1 -0
- package/dist/models/audio/errors.d.ts +8 -0
- package/dist/models/audio/errors.d.ts.map +1 -0
- package/dist/models/audio/file.d.ts +9 -0
- package/dist/models/audio/file.d.ts.map +1 -0
- package/dist/models/audio/runtime.d.ts +8 -0
- package/dist/models/audio/runtime.d.ts.map +1 -0
- package/dist/models/audio/types.d.ts +60 -0
- package/dist/models/audio/types.d.ts.map +1 -0
- package/dist/models/catalog.d.ts +23 -1
- package/dist/models/catalog.d.ts.map +1 -1
- package/dist/models/definitions.d.ts +1 -1
- package/dist/models/definitions.d.ts.map +1 -1
- package/dist/models/embedding-model.d.ts +4 -0
- package/dist/models/embedding-model.d.ts.map +1 -1
- package/dist/models/events.d.ts +3 -0
- package/dist/models/events.d.ts.map +1 -1
- package/dist/models/execution/index.js +22 -22
- package/dist/models/execution/model-call-admission.d.ts +6 -1
- package/dist/models/execution/model-call-admission.d.ts.map +1 -1
- package/dist/models/execution/model-call-limits.d.ts.map +1 -1
- package/dist/models/execution/model-call-recorder.d.ts.map +1 -1
- package/dist/models/execution/reranking.d.ts +4 -0
- package/dist/models/execution/reranking.d.ts.map +1 -0
- package/dist/models/execution/usage.d.ts.map +1 -1
- package/dist/models/execution.d.ts +2 -1
- package/dist/models/execution.d.ts.map +1 -1
- package/dist/models/generation-types.d.ts +2 -0
- package/dist/models/generation-types.d.ts.map +1 -1
- package/dist/models/index.d.ts +7 -1
- package/dist/models/index.d.ts.map +1 -1
- package/dist/models/index.js +7 -3
- package/dist/models/index.js.map +1 -1
- package/dist/models/pricing.d.ts +2 -2
- package/dist/models/pricing.d.ts.map +1 -1
- package/dist/models/reranking-model.d.ts +49 -0
- package/dist/models/reranking-model.d.ts.map +1 -0
- package/dist/objects/index.js +25 -25
- package/dist/objects/query/executor.d.ts +2 -1
- package/dist/objects/query/executor.d.ts.map +1 -1
- package/dist/objects/query/index.d.ts +1 -1
- package/dist/objects/query/index.d.ts.map +1 -1
- package/dist/objects/query/index.js +24 -24
- package/dist/objects/query/index.js.map +1 -1
- package/dist/objects/query/ir.d.ts +8 -1
- package/dist/objects/query/ir.d.ts.map +1 -1
- package/dist/objects/query/normalize.d.ts.map +1 -1
- package/dist/objects/query/planner.d.ts +1 -1
- package/dist/objects/query/planner.d.ts.map +1 -1
- package/dist/objects/query/primary-id.d.ts +7 -0
- package/dist/objects/query/primary-id.d.ts.map +1 -0
- package/dist/objects/query/reranking-executor.d.ts +13 -0
- package/dist/objects/query/reranking-executor.d.ts.map +1 -0
- package/dist/objects/query/reranking-limits.d.ts +3 -0
- package/dist/objects/query/reranking-limits.d.ts.map +1 -0
- package/dist/objects/query/reranking-query.d.ts +4 -0
- package/dist/objects/query/reranking-query.d.ts.map +1 -0
- package/dist/objects/query/validate.d.ts.map +1 -1
- package/dist/objects/sdk/query-builder.d.ts +8 -0
- package/dist/objects/sdk/query-builder.d.ts.map +1 -1
- package/dist/objects/sdk/query-executor.d.ts +2 -2
- package/dist/objects/sdk/query-executor.d.ts.map +1 -1
- package/dist/objects/sdk/runtime-query-executor.d.ts.map +1 -1
- package/dist/objects/vectors/indexing-dispatch.d.ts +21 -0
- package/dist/objects/vectors/indexing-dispatch.d.ts.map +1 -0
- package/dist/objects/vectors/indexing-generate.d.ts.map +1 -1
- package/dist/objects/vectors/profile.d.ts.map +1 -1
- package/dist/objects/vectors/query.d.ts +1 -1
- package/dist/objects/vectors/query.d.ts.map +1 -1
- package/dist/objects/vectors/resolve-query.d.ts.map +1 -1
- package/dist/objects/vectors/types.d.ts +1 -1
- package/dist/objects/vectors/types.d.ts.map +1 -1
- package/dist/ontology/builders.d.ts +13 -4
- package/dist/ontology/builders.d.ts.map +1 -1
- package/dist/ontology/index.d.ts +1 -1
- package/dist/ontology/index.d.ts.map +1 -1
- package/dist/ontology/index.js +3 -3
- package/dist/ontology/internal.js +2 -2
- package/dist/ontology/query-capabilities.d.ts +1 -0
- package/dist/ontology/query-capabilities.d.ts.map +1 -1
- package/dist/ontology/types.d.ts +20 -0
- package/dist/ontology/types.d.ts.map +1 -1
- package/dist/ontology/validation/query.d.ts.map +1 -1
- package/dist/ontology/vector-profiles.d.ts.map +1 -1
- package/dist/pipelines/index.js +23 -23
- package/dist/projections/internal.js +22 -22
- package/dist/query.d.ts +4 -2
- package/dist/query.d.ts.map +1 -1
- package/dist/query.js +2 -2
- package/dist/queues/in-memory.d.ts +3 -0
- package/dist/queues/in-memory.d.ts.map +1 -1
- package/dist/queues/index.js +1 -1
- package/dist/queues/types.d.ts +4 -0
- package/dist/queues/types.d.ts.map +1 -1
- package/dist/runtime/host.d.ts +1 -0
- package/dist/runtime/host.d.ts.map +1 -1
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.d.ts.map +1 -1
- package/dist/runtime/internal.d.ts +1 -0
- package/dist/runtime/internal.d.ts.map +1 -1
- package/dist/runtime/internal.js +28 -26
- package/dist/runtime/internal.js.map +1 -1
- package/dist/runtime/public-origin.d.ts +10 -0
- package/dist/runtime/public-origin.d.ts.map +1 -0
- package/dist/runtime/sixb.d.ts +1 -0
- package/dist/runtime/sixb.d.ts.map +1 -1
- package/dist/runtime/types.d.ts +12 -2
- package/dist/runtime/types.d.ts.map +1 -1
- package/dist/sandboxes/index.js +5 -5
- package/dist/shares/internal.js +19 -19
- package/dist/storage/agents/provider.js +2 -2
- package/dist/storage/ai-cost/analytics.d.ts +1 -0
- package/dist/storage/ai-cost/analytics.d.ts.map +1 -1
- package/dist/storage/ai-cost/provider.js +3 -3
- package/dist/storage/ai-cost/types.d.ts +1 -1
- package/dist/storage/ai-cost/types.d.ts.map +1 -1
- package/dist/storage/ai-cost/validation.d.ts.map +1 -1
- package/dist/storage/ai-limits/enforcement.js +2 -2
- package/dist/storage/ai-limits/in-memory.d.ts.map +1 -1
- package/dist/storage/ai-limits/provider.d.ts +7 -2
- package/dist/storage/ai-limits/provider.d.ts.map +1 -1
- package/dist/storage/ai-limits/provider.js +4 -2
- package/dist/storage/ai-limits/provider.js.map +1 -1
- package/dist/storage/ai-limits/types.d.ts +3 -0
- package/dist/storage/ai-limits/types.d.ts.map +1 -1
- package/dist/storage/ai-usage/record.d.ts.map +1 -1
- package/dist/storage/ai-usage/types.d.ts +8 -4
- package/dist/storage/ai-usage/types.d.ts.map +1 -1
- package/dist/storage/ai-usage/usage.d.ts.map +1 -1
- package/dist/storage/file-download-grants/in-memory.d.ts +11 -0
- package/dist/storage/file-download-grants/in-memory.d.ts.map +1 -0
- package/dist/storage/file-download-grants/index.d.ts +3 -0
- package/dist/storage/file-download-grants/index.d.ts.map +1 -0
- package/dist/storage/file-download-grants/record.d.ts +11 -0
- package/dist/storage/file-download-grants/record.d.ts.map +1 -0
- package/dist/storage/file-download-grants/types.d.ts +44 -0
- package/dist/storage/file-download-grants/types.d.ts.map +1 -0
- package/dist/storage/file-upload-sessions/provider.js +6 -6
- package/dist/storage/in-memory/index.d.ts +4 -0
- package/dist/storage/in-memory/index.d.ts.map +1 -1
- package/dist/storage/index.d.ts +5 -3
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +30 -30
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/internal.d.ts +1 -0
- package/dist/storage/internal.d.ts.map +1 -1
- package/dist/storage/internal.js +8 -4
- package/dist/storage/internal.js.map +1 -1
- package/dist/storage/migrations/index.d.ts +1 -1
- package/dist/storage/migrations/index.d.ts.map +1 -1
- package/dist/storage/migrations/migrate-storage.d.ts +3 -3
- package/dist/storage/migrations/migrate-storage.d.ts.map +1 -1
- package/dist/storage/migrations/types.d.ts +7 -0
- package/dist/storage/migrations/types.d.ts.map +1 -1
- package/dist/storage/objects/in-memory/query.d.ts.map +1 -1
- package/dist/storage/objects/in-memory/selected-reader.d.ts.map +1 -1
- package/dist/storage/objects/in-memory/storage.d.ts.map +1 -1
- package/dist/storage/objects/query-preparation.d.ts +39 -0
- package/dist/storage/objects/query-preparation.d.ts.map +1 -0
- package/dist/storage/objects/types.d.ts +2 -0
- package/dist/storage/objects/types.d.ts.map +1 -1
- package/dist/storage/ontology/commits.d.ts +1 -6
- package/dist/storage/ontology/commits.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/index.d.ts +4 -2
- package/dist/storage/ontology/in-memory/index.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/materializations-state.d.ts +0 -2
- package/dist/storage/ontology/in-memory/materializations-state.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/materializations-work.d.ts +1 -3
- package/dist/storage/ontology/in-memory/materializations-work.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/materializations.d.ts +23 -57
- package/dist/storage/ontology/in-memory/materializations.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/outbox.d.ts +3 -2
- package/dist/storage/ontology/in-memory/outbox.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/replacement-plans.d.ts +104 -0
- package/dist/storage/ontology/in-memory/replacement-plans.d.ts.map +1 -0
- package/dist/storage/ontology/in-memory/shared-state.d.ts +17 -2
- package/dist/storage/ontology/in-memory/shared-state.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/source-roots.d.ts +3 -1
- package/dist/storage/ontology/in-memory/source-roots.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/sources.d.ts +1 -2
- package/dist/storage/ontology/in-memory/sources.d.ts.map +1 -1
- package/dist/storage/ontology/in-memory/state-reader.d.ts +25 -0
- package/dist/storage/ontology/in-memory/state-reader.d.ts.map +1 -0
- package/dist/storage/ontology/index.d.ts +6 -3
- package/dist/storage/ontology/index.d.ts.map +1 -1
- package/dist/storage/ontology/materializations.d.ts +40 -59
- package/dist/storage/ontology/materializations.d.ts.map +1 -1
- package/dist/storage/ontology/outbox.d.ts +4 -6
- package/dist/storage/ontology/outbox.d.ts.map +1 -1
- package/dist/storage/ontology/provider-work.d.ts +114 -3
- package/dist/storage/ontology/provider-work.d.ts.map +1 -1
- package/dist/storage/ontology/provider.d.ts +14 -39
- package/dist/storage/ontology/provider.d.ts.map +1 -1
- package/dist/storage/ontology/provider.js +26 -12
- package/dist/storage/ontology/provider.js.map +1 -1
- package/dist/storage/ontology/replacement-plans.d.ts +103 -0
- package/dist/storage/ontology/replacement-plans.d.ts.map +1 -0
- package/dist/storage/ontology/sources.d.ts +0 -8
- package/dist/storage/ontology/sources.d.ts.map +1 -1
- package/dist/storage/operation-scope.d.ts.map +1 -1
- package/dist/storage/operation-scope.js +1 -1
- package/dist/storage/projection-runs/in-memory.d.ts.map +1 -1
- package/dist/storage/projection-runs/types.d.ts +2 -0
- package/dist/storage/projection-runs/types.d.ts.map +1 -1
- package/dist/storage/types.d.ts +2 -0
- package/dist/storage/types.d.ts.map +1 -1
- package/dist/storage/workflow-runs/provider.js +6 -6
- package/dist/syncs/index.js +24 -24
- package/dist/testing/ai-cost-storage-contract.d.ts.map +1 -1
- package/dist/testing/ai-limit-storage-contract.d.ts.map +1 -1
- package/dist/testing/ai-usage-storage-contract.d.ts.map +1 -1
- package/dist/testing/file-download-grant-storage-contract.d.ts +9 -0
- package/dist/testing/file-download-grant-storage-contract.d.ts.map +1 -0
- package/dist/testing/index.d.ts +1 -0
- package/dist/testing/index.d.ts.map +1 -1
- package/dist/testing/index.js +2548 -1489
- package/dist/testing/index.js.map +14 -13
- package/dist/testing/lake-storage-contract.d.ts.map +1 -1
- package/dist/testing/materializer-storage-contract.d.ts +14 -0
- package/dist/testing/materializer-storage-contract.d.ts.map +1 -1
- package/dist/testing/object-read-scope-contract.d.ts.map +1 -1
- package/dist/testing/ontology-contract-fixture.d.ts +5 -1
- package/dist/testing/ontology-contract-fixture.d.ts.map +1 -1
- package/dist/testing/ontology-storage-contract.d.ts.map +1 -1
- package/dist/testing/projection-run-storage-contract.d.ts.map +1 -1
- package/dist/testing/queues-contract.d.ts.map +1 -1
- package/dist/webhooks/internal.js +20 -20
- package/dist/workers/index.js +15 -17
- package/dist/workers/index.js.map +3 -3
- package/dist/workers/queue-worker.d.ts +3 -2
- package/dist/workers/queue-worker.d.ts.map +1 -1
- package/dist/workflows/index.js +42 -41
- package/dist/workflows/index.js.map +1 -1
- package/package.json +7 -1
- package/src/actions/types/context.ts +5 -2
- package/src/blob-storage/download-urls.ts +138 -0
- package/src/blob-storage/execution.ts +32 -1
- package/src/blob-storage/internal.ts +1 -0
- package/src/execution/authorized-object-reader.ts +8 -1
- package/src/index.ts +25 -0
- package/src/lake-storage/in-memory.ts +38 -9
- package/src/lake-storage/index.ts +2 -0
- package/src/lake-storage/types.ts +30 -0
- package/src/maintenance/ontology-maintenance.ts +50 -23
- package/src/maintenance/types.ts +1 -4
- package/src/materialization/event-envelopes.ts +168 -0
- package/src/materialization/events.ts +40 -4
- package/src/materialization/index.ts +9 -1
- package/src/materializer/README.md +55 -10
- package/src/materializer/edits/commit.ts +3 -19
- package/src/materializer/edits/plan.ts +20 -71
- package/src/materializer/effective/build-events.ts +42 -136
- package/src/materializer/effective/vector-indexing.ts +2 -3
- package/src/materializer/effective/vectors.ts +2 -3
- package/src/materializer/execution/scope.ts +1 -38
- package/src/materializer/execution/work-executor.ts +21 -115
- package/src/materializer/execution/work-records.ts +0 -15
- package/src/materializer/projections/replace.ts +106 -63
- package/src/materializer/projections/replacement-plan.ts +73 -218
- package/src/materializer/shared/batching.ts +7 -0
- package/src/materializer/telemetry/append.ts +2 -14
- package/src/materializer/telemetry/plan.ts +6 -43
- package/src/models/audio/catalog.ts +90 -0
- package/src/models/audio/errors.ts +19 -0
- package/src/models/audio/file.ts +78 -0
- package/src/models/audio/runtime.ts +182 -0
- package/src/models/audio/types.ts +69 -0
- package/src/models/catalog.ts +52 -5
- package/src/models/definitions.ts +8 -1
- package/src/models/embedding-model.ts +12 -0
- package/src/models/events.ts +3 -0
- package/src/models/execution/README.md +20 -1
- package/src/models/execution/embedding.ts +5 -1
- package/src/models/execution/model-call-admission.ts +6 -1
- package/src/models/execution/model-call-limits.ts +21 -11
- package/src/models/execution/model-call-recorder.ts +1 -0
- package/src/models/execution/model-call-recovery.ts +1 -0
- package/src/models/execution/reranking.ts +137 -0
- package/src/models/execution/usage.ts +1 -0
- package/src/models/execution.ts +5 -1
- package/src/models/generation-types.ts +2 -0
- package/src/models/index.ts +32 -0
- package/src/models/pricing.ts +2 -1
- package/src/models/reranking-model.ts +114 -0
- package/src/objects/query/executor.ts +42 -2
- package/src/objects/query/explain.ts +8 -0
- package/src/objects/query/index.ts +1 -0
- package/src/objects/query/ir.ts +10 -0
- package/src/objects/query/normalize.ts +2 -0
- package/src/objects/query/planner.ts +12 -1
- package/src/objects/query/primary-id.ts +56 -0
- package/src/objects/query/reranking-executor.ts +91 -0
- package/src/objects/query/reranking-limits.ts +3 -0
- package/src/objects/query/reranking-query.ts +8 -0
- package/src/objects/query/validate.ts +33 -10
- package/src/objects/sdk/query-builder.ts +15 -2
- package/src/objects/sdk/query-executor.ts +2 -2
- package/src/objects/sdk/runtime-query-executor.ts +7 -4
- package/src/objects/vectors/README.md +14 -2
- package/src/objects/vectors/handle.ts +1 -1
- package/src/objects/vectors/indexing-dispatch.ts +83 -0
- package/src/objects/vectors/indexing-generate.ts +5 -1
- package/src/objects/vectors/profile.ts +2 -0
- package/src/objects/vectors/query.ts +21 -4
- package/src/objects/vectors/resolve-query.ts +9 -5
- package/src/objects/vectors/types.ts +3 -1
- package/src/ontology/builders.ts +22 -7
- package/src/ontology/index.ts +2 -0
- package/src/ontology/query-capabilities.ts +1 -1
- package/src/ontology/types.ts +21 -0
- package/src/ontology/validation/query.ts +57 -0
- package/src/ontology/vector-profiles.ts +1 -0
- package/src/query.ts +4 -0
- package/src/queues/in-memory.ts +54 -19
- package/src/queues/types.ts +4 -0
- package/src/runtime/host.ts +28 -1
- package/src/runtime/index.ts +1 -0
- package/src/runtime/internal.ts +1 -0
- package/src/runtime/public-origin.ts +33 -0
- package/src/runtime/sixb.ts +13 -2
- package/src/runtime/types.ts +12 -2
- package/src/storage/ai-cost/analytics.ts +2 -0
- package/src/storage/ai-cost/codec.ts +2 -1
- package/src/storage/ai-cost/types.ts +1 -0
- package/src/storage/ai-cost/validation.ts +21 -8
- package/src/storage/ai-limits/in-memory.ts +10 -2
- package/src/storage/ai-limits/provider.ts +29 -2
- package/src/storage/ai-limits/types.ts +3 -0
- package/src/storage/ai-usage/record.ts +9 -0
- package/src/storage/ai-usage/types.ts +8 -4
- package/src/storage/ai-usage/usage.ts +2 -0
- package/src/storage/file-download-grants/in-memory.ts +73 -0
- package/src/storage/file-download-grants/index.ts +8 -0
- package/src/storage/file-download-grants/record.ts +47 -0
- package/src/storage/file-download-grants/types.ts +48 -0
- package/src/storage/in-memory/index.ts +13 -2
- package/src/storage/index.ts +31 -13
- package/src/storage/internal.ts +1 -0
- package/src/storage/migrations/index.ts +1 -0
- package/src/storage/migrations/migrate-storage.ts +36 -12
- package/src/storage/migrations/types.ts +8 -0
- package/src/storage/objects/in-memory/query.ts +2 -0
- package/src/storage/objects/in-memory/selected-reader.ts +1 -0
- package/src/storage/objects/in-memory/storage.ts +1 -0
- package/src/storage/objects/query-preparation.ts +77 -0
- package/src/storage/objects/types.ts +2 -0
- package/src/storage/ontology/commits.ts +1 -6
- package/src/storage/ontology/in-memory/index.ts +28 -9
- package/src/storage/ontology/in-memory/materializations-finalization.ts +35 -68
- package/src/storage/ontology/in-memory/materializations-state.ts +0 -22
- package/src/storage/ontology/in-memory/materializations-work.ts +0 -45
- package/src/storage/ontology/in-memory/materializations.ts +172 -653
- package/src/storage/ontology/in-memory/outbox.ts +37 -29
- package/src/storage/ontology/in-memory/replacement-plans.ts +697 -0
- package/src/storage/ontology/in-memory/shared-state.ts +24 -2
- package/src/storage/ontology/in-memory/source-roots.ts +7 -2
- package/src/storage/ontology/in-memory/sources.ts +7 -33
- package/src/storage/ontology/in-memory/state-reader.ts +245 -0
- package/src/storage/ontology/index.ts +19 -13
- package/src/storage/ontology/materializations.ts +45 -74
- package/src/storage/ontology/outbox.ts +4 -7
- package/src/storage/ontology/provider-header-validation.ts +0 -57
- package/src/storage/ontology/provider-work.ts +381 -90
- package/src/storage/ontology/provider.ts +74 -171
- package/src/storage/ontology/replacement-plans.ts +117 -0
- package/src/storage/ontology/sources.ts +0 -12
- package/src/storage/operation-scope.ts +33 -1
- package/src/storage/projection-runs/in-memory.ts +7 -1
- package/src/storage/projection-runs/types.ts +2 -0
- package/src/storage/types.ts +2 -0
- package/src/testing/ai-cost-storage-contract.ts +1 -0
- package/src/testing/ai-limit-storage-contract.ts +62 -0
- package/src/testing/ai-usage-storage-contract.ts +31 -0
- package/src/testing/file-download-grant-storage-contract.ts +139 -0
- package/src/testing/index.ts +4 -0
- package/src/testing/lake-storage-contract.ts +87 -1
- package/src/testing/materializer-storage-contract.ts +1008 -1
- package/src/testing/object-read-scope-contract.ts +1 -0
- package/src/testing/ontology-contract-fixture.ts +11 -87
- package/src/testing/ontology-storage-contract.ts +70 -39
- package/src/testing/projection-run-storage-contract.ts +14 -0
- package/src/testing/queues-contract.ts +59 -0
- package/src/workers/queue-worker.ts +19 -16
- package/dist/chunk-3c90jvfj.js.map +0 -15
- package/dist/chunk-3nx8vj5a.js.map +0 -12
- package/dist/chunk-7ytf06d3.js +0 -113
- package/dist/chunk-7ytf06d3.js.map +0 -10
- package/dist/chunk-a0073121.js.map +0 -12
- package/dist/chunk-fjw417jf.js.map +0 -62
- package/dist/chunk-g4rbm4a6.js.map +0 -15
- package/dist/chunk-hgqpcmyr.js.map +0 -12
- package/dist/chunk-jgwb25gc.js.map +0 -12
- package/dist/chunk-k4kkp58d.js.map +0 -11
- package/dist/chunk-ky7zpbtt.js.map +0 -19
- package/dist/chunk-mdnw7nj8.js.map +0 -15
- package/dist/chunk-py2dw8cw.js.map +0 -17
- package/dist/chunk-s3csryxe.js.map +0 -11
- package/dist/chunk-xnx96gfy.js.map +0 -15
- package/dist/chunk-z708gemf.js.map +0 -50
- package/dist/materializer/execution/plan-stream.d.ts +0 -8
- package/dist/materializer/execution/plan-stream.d.ts.map +0 -1
- package/src/materializer/execution/plan-stream.ts +0 -121
- /package/dist/{chunk-q3grgzp1.js.map → chunk-23xs4qp9.js.map} +0 -0
- /package/dist/{chunk-vef6qa14.js.map → chunk-2am3db31.js.map} +0 -0
- /package/dist/{chunk-fj1dvp37.js.map → chunk-2vhg6ysa.js.map} +0 -0
- /package/dist/{chunk-p0nbkys4.js.map → chunk-33saed7g.js.map} +0 -0
- /package/dist/{chunk-mmac1mp6.js.map → chunk-52c5zb0z.js.map} +0 -0
- /package/dist/{chunk-8fjmqjyx.js.map → chunk-5be2ath4.js.map} +0 -0
- /package/dist/{chunk-2fv8sajb.js.map → chunk-5thzcb8g.js.map} +0 -0
- /package/dist/{chunk-1jvq82rp.js.map → chunk-9g6jz9kk.js.map} +0 -0
- /package/dist/{chunk-8w27xf9e.js.map → chunk-a7er86jd.js.map} +0 -0
- /package/dist/{chunk-bxx3ms5t.js.map → chunk-c2rga5cx.js.map} +0 -0
- /package/dist/{chunk-dsfw1bdr.js.map → chunk-cnb5kv05.js.map} +0 -0
- /package/dist/{chunk-n5t4mq6z.js.map → chunk-e67ynxcr.js.map} +0 -0
- /package/dist/{chunk-6q7c68yx.js.map → chunk-f1xh7a32.js.map} +0 -0
- /package/dist/{chunk-7jc21943.js.map → chunk-gf7yht60.js.map} +0 -0
- /package/dist/{chunk-ffwvdgk1.js.map → chunk-jjrn5v0p.js.map} +0 -0
- /package/dist/{chunk-r5sc1d2b.js.map → chunk-jnfxmr3f.js.map} +0 -0
- /package/dist/{chunk-y0zmae99.js.map → chunk-k4g51s47.js.map} +0 -0
- /package/dist/{chunk-drz59ve0.js.map → chunk-rx7h5fdx.js.map} +0 -0
- /package/dist/{chunk-yfppyaty.js.map → chunk-sk0hcjsz.js.map} +0 -0
- /package/dist/{chunk-qne45xqq.js.map → chunk-svabwqdq.js.map} +0 -0
- /package/dist/{chunk-xr7y35yj.js.map → chunk-t9shqx3w.js.map} +0 -0
- /package/dist/{chunk-adbve9zt.js.map → chunk-vre348sb.js.map} +0 -0
- /package/dist/{chunk-a4g5yjm0.js.map → chunk-w9yycd3t.js.map} +0 -0
- /package/dist/{chunk-wd34wnsy.js.map → chunk-xcy5ddha.js.map} +0 -0
|
@@ -15,16 +15,21 @@ import {
|
|
|
15
15
|
paginate,
|
|
16
16
|
InMemoryAuthStorage2,
|
|
17
17
|
principalsEqual2
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-svabwqdq.js";
|
|
19
19
|
import {
|
|
20
20
|
ObjectReadLimitExceededError2,
|
|
21
21
|
snapshotObjectReadExecutionLimits2,
|
|
22
22
|
assertObjectReadFacetCount2,
|
|
23
23
|
assertObjectReadOutputWithinLimit2
|
|
24
|
-
} from "./chunk-
|
|
24
|
+
} from "./chunk-5be2ath4.js";
|
|
25
25
|
import {
|
|
26
26
|
assertObjectReaderProject2
|
|
27
27
|
} from "./chunk-v4jbb3rs.js";
|
|
28
|
+
import {
|
|
29
|
+
queryScalarKindForSchema,
|
|
30
|
+
resolvePropertyQueryCapabilities2,
|
|
31
|
+
resolveQuerySchema
|
|
32
|
+
} from "./chunk-rpxz486p.js";
|
|
28
33
|
import {
|
|
29
34
|
AgentStorageError2,
|
|
30
35
|
transitionAgentThreadSandbox2,
|
|
@@ -48,10 +53,11 @@ import {
|
|
|
48
53
|
buildAiModelCallGroups,
|
|
49
54
|
normalizeAiModelCallCostRecord2,
|
|
50
55
|
aiModelCallCostMatchesUsage2
|
|
51
|
-
} from "./chunk-
|
|
56
|
+
} from "./chunk-ere85b8y.js";
|
|
52
57
|
import {
|
|
53
58
|
AiLimitStorageError2,
|
|
54
59
|
aiLimitCalendarMonth2,
|
|
60
|
+
aiLimitMeterApplies2,
|
|
55
61
|
AiLimitOperationLock2,
|
|
56
62
|
aiLimitSubjectKey2,
|
|
57
63
|
normalizeAiLimitAmount2,
|
|
@@ -73,42 +79,63 @@ import {
|
|
|
73
79
|
cloneAiLimitPolicy2,
|
|
74
80
|
assertNonBlank2,
|
|
75
81
|
cloneValidDate2
|
|
76
|
-
} from "./chunk-
|
|
82
|
+
} from "./chunk-4as3wt1c.js";
|
|
77
83
|
import {
|
|
78
84
|
isAbandonedFileUploadSession,
|
|
79
85
|
shouldDeleteFileUploadSession
|
|
80
86
|
} from "./chunk-h89wz928.js";
|
|
81
87
|
import {
|
|
82
88
|
DurableFileUploadSessions2
|
|
83
|
-
} from "./chunk-
|
|
89
|
+
} from "./chunk-33saed7g.js";
|
|
84
90
|
import {
|
|
85
91
|
ExecutionStorageError2,
|
|
86
92
|
prepareExecutionRecord2,
|
|
87
93
|
cloneExecutionRecord
|
|
88
94
|
} from "./chunk-tw03bsmb.js";
|
|
89
95
|
import {
|
|
96
|
+
objectRefKey2,
|
|
97
|
+
linkRefKey2,
|
|
98
|
+
projectionEntityKey2,
|
|
99
|
+
telemetryPointKey2,
|
|
100
|
+
objectRefSortKey2,
|
|
101
|
+
linkRefSortKey2,
|
|
102
|
+
linkScopeKey,
|
|
103
|
+
linkScopeSortKey2,
|
|
104
|
+
telemetryPointSortKey2,
|
|
90
105
|
sha256Canonical2,
|
|
91
|
-
createEventId2
|
|
106
|
+
createEventId2
|
|
107
|
+
} from "./chunk-b9affek3.js";
|
|
108
|
+
import {
|
|
109
|
+
assertNonblank2,
|
|
110
|
+
assertPositiveInteger2,
|
|
92
111
|
assertTimestamp2,
|
|
93
112
|
invalidCorrelation2,
|
|
113
|
+
assertMaterializationHeader2,
|
|
94
114
|
assertPageRows2,
|
|
95
115
|
materializationChunkRows2,
|
|
96
|
-
|
|
97
|
-
|
|
116
|
+
materializationPlanChunk2,
|
|
117
|
+
materializationOutboxWrite2,
|
|
118
|
+
claimedOutboxRows2,
|
|
119
|
+
assertReplacementPlanCommit2,
|
|
120
|
+
prepareReplacementIdentities2,
|
|
121
|
+
projectionChangeCounts,
|
|
122
|
+
createCardinalityValidator2,
|
|
98
123
|
workUniquenessKey2,
|
|
99
124
|
comparePlanWork2,
|
|
100
125
|
compareCardinalityWork2,
|
|
101
126
|
compareEventWork2,
|
|
102
|
-
assertPlanChunkCorrelations2,
|
|
103
|
-
assertMaterializationHeader2,
|
|
104
127
|
startScopeAccumulator2,
|
|
105
128
|
appendScopeSnapshot2,
|
|
106
129
|
finishScopeAccumulator2,
|
|
130
|
+
beginMaterializationVectorChanges2,
|
|
131
|
+
beginMaterializationApply2,
|
|
107
132
|
ProviderMaterializationTransactionLifecycle2,
|
|
133
|
+
prepareMaterializationWork2,
|
|
108
134
|
uniqueSorted2,
|
|
109
135
|
assertExpectedObjectRevision2,
|
|
110
136
|
assertExpectedLinkRevision2,
|
|
111
137
|
assertExpectedLinkScopeRevision2,
|
|
138
|
+
duplicateMaterializationWork2,
|
|
112
139
|
sourceStageRoots2,
|
|
113
140
|
sourceMaterializationIdentity2,
|
|
114
141
|
assertSourceBeginInput2,
|
|
@@ -119,18 +146,7 @@ import {
|
|
|
119
146
|
assertSourceStagedRow2,
|
|
120
147
|
isExactStagingManifest2,
|
|
121
148
|
sourceConflict2
|
|
122
|
-
} from "./chunk-
|
|
123
|
-
import {
|
|
124
|
-
objectRefKey2,
|
|
125
|
-
linkRefKey2,
|
|
126
|
-
projectionEntityKey2,
|
|
127
|
-
telemetryPointKey2,
|
|
128
|
-
objectRefSortKey2,
|
|
129
|
-
linkRefSortKey2,
|
|
130
|
-
linkScopeKey,
|
|
131
|
-
linkScopeSortKey2,
|
|
132
|
-
telemetryPointSortKey2
|
|
133
|
-
} from "./chunk-7ytf06d3.js";
|
|
149
|
+
} from "./chunk-wt4szfec.js";
|
|
134
150
|
import {
|
|
135
151
|
createStorageOperationScope2,
|
|
136
152
|
createOperationScopedFacade2,
|
|
@@ -139,7 +155,7 @@ import {
|
|
|
139
155
|
createOntologyOperationScope2,
|
|
140
156
|
createObjectOperationScope2,
|
|
141
157
|
createWorkflowRunOperationScope2
|
|
142
|
-
} from "./chunk-
|
|
158
|
+
} from "./chunk-qczk96d9.js";
|
|
143
159
|
import {
|
|
144
160
|
PipelineRunError2,
|
|
145
161
|
assertPipelineRunExecution2
|
|
@@ -187,7 +203,7 @@ import {
|
|
|
187
203
|
WorkflowRunError2,
|
|
188
204
|
assertWorkflowRunExecution2,
|
|
189
205
|
assertWorkflowAgentNodeRunExecution2
|
|
190
|
-
} from "./chunk-
|
|
206
|
+
} from "./chunk-t9shqx3w.js";
|
|
191
207
|
import {
|
|
192
208
|
ShareGrantStorageError2,
|
|
193
209
|
normalizeShareGrantCreate2,
|
|
@@ -206,21 +222,22 @@ import {
|
|
|
206
222
|
normalizeRevokeShareSessionInput2,
|
|
207
223
|
cloneShareSessionRecord2
|
|
208
224
|
} from "./chunk-z8fx5cgb.js";
|
|
225
|
+
import {
|
|
226
|
+
ConnectorConnectionStorageError2,
|
|
227
|
+
CONNECTOR_CONNECTION_RUN_FAILURE_CODES2
|
|
228
|
+
} from "./chunk-2c4j899j.js";
|
|
209
229
|
import {
|
|
210
230
|
actionSubjectsEqual2,
|
|
211
231
|
actionRunPhaseRecordsEqual2,
|
|
212
232
|
canRequeueActionRunAfterEnqueueFailure2,
|
|
213
233
|
isTerminalActionRun2,
|
|
214
234
|
StorageTransactionError2,
|
|
235
|
+
cloneFileDownloadGrantRecord,
|
|
215
236
|
canRequeuePipelineRunAfterEnqueueFailure2,
|
|
216
237
|
canRequeueSyncRunAfterEnqueueFailure2,
|
|
217
238
|
createTransactionStorageProxy2,
|
|
218
239
|
throwNestedStorageTransaction2
|
|
219
|
-
} from "./chunk-
|
|
220
|
-
import {
|
|
221
|
-
ConnectorConnectionStorageError2,
|
|
222
|
-
CONNECTOR_CONNECTION_RUN_FAILURE_CODES2
|
|
223
|
-
} from "./chunk-2c4j899j.js";
|
|
240
|
+
} from "./chunk-aq39q04r.js";
|
|
224
241
|
import {
|
|
225
242
|
assertPinnedDatasetWatermark2
|
|
226
243
|
} from "./chunk-6aawea28.js";
|
|
@@ -1264,8 +1281,9 @@ class InMemoryAiLimitStorage2 {
|
|
|
1264
1281
|
const unavailablePolicies = [];
|
|
1265
1282
|
const unavailableReasons = new Set;
|
|
1266
1283
|
for (const policy of enabledPolicies) {
|
|
1267
|
-
if (!subjectKeys.has(aiLimitSubjectKey2(policy.subject)))
|
|
1284
|
+
if (!subjectKeys.has(aiLimitSubjectKey2(policy.subject)) || !aiLimitMeterApplies2(policy.limit.meter, request.modelKind)) {
|
|
1268
1285
|
continue;
|
|
1286
|
+
}
|
|
1269
1287
|
const limit = normalizeAiLimitAmount2(policy.limit);
|
|
1270
1288
|
const estimate = estimates.get(aiLimitAmountKey2(limit));
|
|
1271
1289
|
const status = await this.policyStatus(policy, request.period);
|
|
@@ -1301,7 +1319,7 @@ class InMemoryAiLimitStorage2 {
|
|
|
1301
1319
|
resetAt: new Date(Math.min(...exhaustedPolicies.map((status) => status.period.resetAt.getTime())))
|
|
1302
1320
|
};
|
|
1303
1321
|
}
|
|
1304
|
-
const buckets = aiLimitReservationBuckets2(enabledPolicies, request.subjects, request.estimates);
|
|
1322
|
+
const buckets = aiLimitReservationBuckets2(enabledPolicies, request.subjects, request.estimates, request.modelKind);
|
|
1305
1323
|
if (buckets.length === 0)
|
|
1306
1324
|
return { status: "notRequired" };
|
|
1307
1325
|
for (const bucket of buckets) {
|
|
@@ -1541,6 +1559,7 @@ class InMemoryAiLimitStorage2 {
|
|
|
1541
1559
|
attempt: record.attempt,
|
|
1542
1560
|
callId: record.callId,
|
|
1543
1561
|
occurredAt: new Date(record.occurredAt),
|
|
1562
|
+
...record.modelKind === undefined ? {} : { modelKind: record.modelKind },
|
|
1544
1563
|
...record.usage.totalTokens === undefined ? {} : { totalTokens: record.usage.totalTokens },
|
|
1545
1564
|
requesterGroupIds: [...record.requesterGroupIds],
|
|
1546
1565
|
...requester?.type === "user" || requester?.type === "serviceAccount" ? { requester: { type: requester.type, id: requester.id } } : {},
|
|
@@ -2561,6 +2580,50 @@ class InMemoryExecutionStorage {
|
|
|
2561
2580
|
}
|
|
2562
2581
|
}
|
|
2563
2582
|
|
|
2583
|
+
// src/storage/file-download-grants/in-memory.ts
|
|
2584
|
+
class InMemoryFileDownloadGrantStorage {
|
|
2585
|
+
rows = new Map;
|
|
2586
|
+
async create(record) {
|
|
2587
|
+
const duplicate = [...this.rows.values()].some((current) => current.projectId === record.projectId && (current.id === record.id || current.tokenHash === record.tokenHash));
|
|
2588
|
+
if (duplicate) {
|
|
2589
|
+
throw new Error(`[Sixb] File download grant '${record.id}' conflicts with an existing one.`);
|
|
2590
|
+
}
|
|
2591
|
+
this.rows.set(rowKey(record.projectId, record.id), cloneFileDownloadGrantRecord(record));
|
|
2592
|
+
}
|
|
2593
|
+
async findByTokenHash(input) {
|
|
2594
|
+
for (const record of this.rows.values()) {
|
|
2595
|
+
if (record.projectId === input.projectId && record.tokenHash === input.tokenHash) {
|
|
2596
|
+
return cloneFileDownloadGrantRecord(record);
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
return null;
|
|
2600
|
+
}
|
|
2601
|
+
async revoke(input) {
|
|
2602
|
+
const key = rowKey(input.projectId, input.id);
|
|
2603
|
+
const current = this.rows.get(key);
|
|
2604
|
+
if (!current)
|
|
2605
|
+
return null;
|
|
2606
|
+
if (!current.revokedAt) {
|
|
2607
|
+
this.rows.set(key, { ...current, revokedAt: new Date(input.revokedAt) });
|
|
2608
|
+
}
|
|
2609
|
+
return cloneFileDownloadGrantRecord(this.rows.get(key));
|
|
2610
|
+
}
|
|
2611
|
+
async deleteExpired(input) {
|
|
2612
|
+
const expired = [...this.rows.entries()].filter(([, record]) => record.projectId === input.projectId && record.expiresAt.getTime() < input.expiredBefore.getTime()).slice(0, input.limit);
|
|
2613
|
+
for (const [key] of expired)
|
|
2614
|
+
this.rows.delete(key);
|
|
2615
|
+
return expired.length;
|
|
2616
|
+
}
|
|
2617
|
+
snapshot() {
|
|
2618
|
+
return new Map([...this.rows].map(([key, record]) => [key, cloneFileDownloadGrantRecord(record)]));
|
|
2619
|
+
}
|
|
2620
|
+
restore(snapshot) {
|
|
2621
|
+
this.rows = new Map([...snapshot].map(([key, record]) => [key, cloneFileDownloadGrantRecord(record)]));
|
|
2622
|
+
}
|
|
2623
|
+
}
|
|
2624
|
+
function rowKey(projectId, id) {
|
|
2625
|
+
return `${projectId}\x00${id}`;
|
|
2626
|
+
}
|
|
2564
2627
|
// src/objects/vectors/query.ts
|
|
2565
2628
|
function hasVectorProfile2(query) {
|
|
2566
2629
|
if (query.kind === "vector")
|
|
@@ -2572,12 +2635,27 @@ function hasVectorProfile2(query) {
|
|
|
2572
2635
|
function isVectorProfileQuery2(query) {
|
|
2573
2636
|
if (query.kind === "limit" || query.kind === "project")
|
|
2574
2637
|
return isVectorProfileQuery2(query.input);
|
|
2638
|
+
if (query.kind === "rerank") {
|
|
2639
|
+
return query.input.kind === "vector" && isVectorProfileQuery2(query.input);
|
|
2640
|
+
}
|
|
2575
2641
|
if (query.kind !== "vector" || !query.profile)
|
|
2576
2642
|
return false;
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2643
|
+
return isVectorCandidateQuery(query.input);
|
|
2644
|
+
}
|
|
2645
|
+
function isVectorCandidateQuery(query) {
|
|
2646
|
+
switch (query.kind) {
|
|
2647
|
+
case "start":
|
|
2648
|
+
return !query.includeSubtypes;
|
|
2649
|
+
case "refs":
|
|
2650
|
+
return true;
|
|
2651
|
+
case "filter":
|
|
2652
|
+
case "traverse":
|
|
2653
|
+
return isVectorCandidateQuery(query.input);
|
|
2654
|
+
case "set":
|
|
2655
|
+
return query.inputs.every(isVectorCandidateQuery);
|
|
2656
|
+
default:
|
|
2657
|
+
return false;
|
|
2658
|
+
}
|
|
2581
2659
|
}
|
|
2582
2660
|
|
|
2583
2661
|
// src/storage/objects/keys.ts
|
|
@@ -2743,6 +2821,7 @@ function vectorConfiguration(profile) {
|
|
|
2743
2821
|
model: profile.model.modelId,
|
|
2744
2822
|
representation: profile.model.definition.representation ?? { name: profile.model.modelId },
|
|
2745
2823
|
dimensions: profile.model.definition.dimensions,
|
|
2824
|
+
...profile.model.definition.inputMode === "asymmetric" ? { inputMode: "asymmetric" } : {},
|
|
2746
2825
|
metric: "cosine",
|
|
2747
2826
|
precision: "float32",
|
|
2748
2827
|
normalization: "l2-v1"
|
|
@@ -2850,6 +2929,8 @@ function assertLinkQueryLimit(limit) {
|
|
|
2850
2929
|
}
|
|
2851
2930
|
function evaluateObjectQuery(query, source) {
|
|
2852
2931
|
switch (query.kind) {
|
|
2932
|
+
case "rerank":
|
|
2933
|
+
throw new Error("[Sixb] Reranking must execute in the model runtime, not storage.");
|
|
2853
2934
|
case "start":
|
|
2854
2935
|
return evaluateStart(query.objectTypeId, source);
|
|
2855
2936
|
case "refs":
|
|
@@ -3521,6 +3602,7 @@ function createInMemorySelectedReader(source, params, queryCapabilities) {
|
|
|
3521
3602
|
const universe = resolveSelectedReadUniverse(source, plan, limits);
|
|
3522
3603
|
const result = evaluateObjectQuery(stripOuterRowShape(input.query), universe);
|
|
3523
3604
|
return visible(structuredClone({
|
|
3605
|
+
total: result.entries.length,
|
|
3524
3606
|
facets: buildFacetResults(result.entries.map((entry) => entry.row), input.facets)
|
|
3525
3607
|
}));
|
|
3526
3608
|
},
|
|
@@ -3759,6 +3841,7 @@ class InMemoryObjectStorage {
|
|
|
3759
3841
|
async facetObjects(params) {
|
|
3760
3842
|
const result = evaluateObjectQuery(stripOuterRowShape(params.query), this.readSource(params.projectId));
|
|
3761
3843
|
return {
|
|
3844
|
+
total: result.entries.length,
|
|
3762
3845
|
facets: buildFacetResults(result.entries.map((entry) => entry.row), params.facets)
|
|
3763
3846
|
};
|
|
3764
3847
|
}
|
|
@@ -3945,7 +4028,9 @@ function createInMemoryOntologyState() {
|
|
|
3945
4028
|
objectOverrides: new Map,
|
|
3946
4029
|
linkOverrides: new Map,
|
|
3947
4030
|
linkSlotOverrides: new Map,
|
|
3948
|
-
outbox: new Map
|
|
4031
|
+
outbox: new Map,
|
|
4032
|
+
replacementPlans: new Map,
|
|
4033
|
+
commitTouches: new Map
|
|
3949
4034
|
};
|
|
3950
4035
|
}
|
|
3951
4036
|
function projectEntityKey(projectId, entityKey) {
|
|
@@ -4333,55 +4418,8 @@ function storedSourceLink(sourceId, materializationId, row) {
|
|
|
4333
4418
|
return null;
|
|
4334
4419
|
return storedSource(sourceId, materializationId, row);
|
|
4335
4420
|
}
|
|
4336
|
-
function addReplacementLink(replacement, ref, diffRequired) {
|
|
4337
|
-
const key = linkRefKey2(ref);
|
|
4338
|
-
const existing = replacement.links.get(key);
|
|
4339
|
-
if (existing) {
|
|
4340
|
-
existing.diffRequired ||= diffRequired;
|
|
4341
|
-
if (diffRequired)
|
|
4342
|
-
replacement.affectedScopes.add(linkScopeSortKey2(ref.source, ref.linkId));
|
|
4343
|
-
return;
|
|
4344
|
-
}
|
|
4345
|
-
replacement.links.set(key, {
|
|
4346
|
-
ref: structuredClone(ref),
|
|
4347
|
-
sortKey: linkRefSortKey2(ref),
|
|
4348
|
-
diffRequired
|
|
4349
|
-
});
|
|
4350
|
-
if (diffRequired)
|
|
4351
|
-
replacement.affectedScopes.add(linkScopeSortKey2(ref.source, ref.linkId));
|
|
4352
|
-
}
|
|
4353
4421
|
|
|
4354
4422
|
// src/storage/ontology/in-memory/materializations-work.ts
|
|
4355
|
-
function assertChunkSequence(session, chunk) {
|
|
4356
|
-
const planItems = materializationPlanItems2(chunk);
|
|
4357
|
-
const applyStream = session.workStreams.apply;
|
|
4358
|
-
const appliedStart = session.appliedPlanItems.length;
|
|
4359
|
-
if (planItems.length > 0 && (!applyStream.started || appliedStart + planItems.length > applyStream.emittedCount)) {
|
|
4360
|
-
invalidCorrelation2("Materialization plan items cannot be applied before they are streamed.");
|
|
4361
|
-
}
|
|
4362
|
-
for (let offset = 0;offset < planItems.length; offset += 1) {
|
|
4363
|
-
const expected = session.applyWork[appliedStart + offset]?.item;
|
|
4364
|
-
if (!expected || stableJsonStringify2(planItems[offset]) !== stableJsonStringify2(expected)) {
|
|
4365
|
-
invalidCorrelation2("Materialization plan items must be applied in exact streamed order.");
|
|
4366
|
-
}
|
|
4367
|
-
}
|
|
4368
|
-
const eventStream = session.workStreams.event;
|
|
4369
|
-
const appliedOutboxCount = session.outboxEnvelopes.size;
|
|
4370
|
-
if (chunk.outbox.length > 0 && (!eventStream.started || appliedOutboxCount + chunk.outbox.length > eventStream.emittedCount)) {
|
|
4371
|
-
invalidCorrelation2("Materialization events cannot be applied before they are streamed.");
|
|
4372
|
-
}
|
|
4373
|
-
for (let offset = 0;offset < chunk.outbox.length; offset += 1) {
|
|
4374
|
-
const expected = session.eventWork[appliedOutboxCount + offset];
|
|
4375
|
-
const actual = chunk.outbox[offset]?.envelope;
|
|
4376
|
-
if (!expected || !actual) {
|
|
4377
|
-
invalidCorrelation2("Materialization outbox events must follow exact streamed order.");
|
|
4378
|
-
}
|
|
4379
|
-
const { id: _id, commitOrdinal, ...actualDraft } = actual;
|
|
4380
|
-
if (commitOrdinal !== appliedOutboxCount + offset || stableJsonStringify2(actualDraft) !== stableJsonStringify2(expected.draft)) {
|
|
4381
|
-
invalidCorrelation2("Materialization outbox events must follow exact streamed order.");
|
|
4382
|
-
}
|
|
4383
|
-
}
|
|
4384
|
-
}
|
|
4385
4423
|
function assertLastCommit(value, expected, label) {
|
|
4386
4424
|
if ((value?.lastCommitId ?? null) !== expected) {
|
|
4387
4425
|
throw new MaterializationConflictError2("effective-state", `Expected ${label} changed.`);
|
|
@@ -4392,11 +4430,11 @@ function assertLastCommit(value, expected, label) {
|
|
|
4392
4430
|
function assertFinalizationCorrelations(session, finalization, state, objects) {
|
|
4393
4431
|
const { commit } = session.header;
|
|
4394
4432
|
const { result, sourceActivations } = finalization;
|
|
4395
|
-
if (result.commitId !== commit.id || result.kind !== commit.intent.kind || result.committedAt !== commit.committedAt || result.created !== true || !Number.isSafeInteger(result.eventCount) || result.eventCount < 0 || result.eventCount !== session.
|
|
4433
|
+
if (result.commitId !== commit.id || result.kind !== commit.intent.kind || result.committedAt !== commit.committedAt || result.created !== true || !Number.isSafeInteger(result.eventCount) || result.eventCount < 0 || result.eventCount !== session.outboxWrites.size) {
|
|
4396
4434
|
invalidCorrelation2("Materialization result does not correlate with its commit intent.");
|
|
4397
4435
|
}
|
|
4398
4436
|
for (let ordinal = 0;ordinal < result.eventCount; ordinal += 1) {
|
|
4399
|
-
if (!session.
|
|
4437
|
+
if (!session.outboxWrites.has(ordinal)) {
|
|
4400
4438
|
invalidCorrelation2("Outbox event ordinals must be contiguous from zero.");
|
|
4401
4439
|
}
|
|
4402
4440
|
}
|
|
@@ -4418,7 +4456,7 @@ function assertFinalizationCorrelations(session, finalization, state, objects) {
|
|
|
4418
4456
|
assertSourceActivationCorrelation(activation, session, state);
|
|
4419
4457
|
}
|
|
4420
4458
|
if (sourceActivations[0]) {
|
|
4421
|
-
|
|
4459
|
+
assertPlanCoverage(session, sourceActivations[0]);
|
|
4422
4460
|
}
|
|
4423
4461
|
assertFinalizedWork(session, state, objects);
|
|
4424
4462
|
if (commit.intent.kind === "projection") {
|
|
@@ -4558,16 +4596,8 @@ function assertSourceDatasetWatermark(active, next) {
|
|
|
4558
4596
|
assertPinnedDatasetWatermark2(active, next, "Source activation");
|
|
4559
4597
|
}
|
|
4560
4598
|
function assertFinalizedWork(session, state, objects) {
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
if (expectedPlanItems.length > 0 && !session.workStreams.apply.completed) {
|
|
4564
|
-
invalidCorrelation2("Materialization plan work was not fully streamed.");
|
|
4565
|
-
}
|
|
4566
|
-
if (stableJsonStringify2(appliedPlanItems) !== stableJsonStringify2(expectedPlanItems)) {
|
|
4567
|
-
invalidCorrelation2("Materialization plan work was not applied exactly once.");
|
|
4568
|
-
}
|
|
4569
|
-
if (session.cardinalityWork.length > 0 && !session.workStreams.cardinality.completed) {
|
|
4570
|
-
invalidCorrelation2("Materialization cardinality work was not fully validated.");
|
|
4599
|
+
if (!session.applied) {
|
|
4600
|
+
invalidCorrelation2("A materialization plan must apply before it finalizes.");
|
|
4571
4601
|
}
|
|
4572
4602
|
assertFinalCardinality(session.cardinalityWork, session.header.commit.projectId, objects);
|
|
4573
4603
|
if (session.header.commit.intent.kind === "telemetry") {
|
|
@@ -4577,24 +4607,20 @@ function assertFinalizedWork(session, state, objects) {
|
|
|
4577
4607
|
}
|
|
4578
4608
|
}
|
|
4579
4609
|
const expectedEvents = [...session.eventWork].sort(compareEventWork2);
|
|
4580
|
-
if (
|
|
4581
|
-
invalidCorrelation2("Materialization event work was not fully drained.");
|
|
4582
|
-
}
|
|
4583
|
-
if (session.outboxEnvelopes.size !== expectedEvents.length) {
|
|
4610
|
+
if (session.outboxWrites.size !== expectedEvents.length) {
|
|
4584
4611
|
invalidCorrelation2("Materialization event work was not fully written to the outbox.");
|
|
4585
4612
|
}
|
|
4586
4613
|
for (let ordinal = 0;ordinal < expectedEvents.length; ordinal += 1) {
|
|
4587
4614
|
const expected = expectedEvents[ordinal];
|
|
4588
|
-
const actual = session.
|
|
4615
|
+
const actual = session.outboxWrites.get(ordinal);
|
|
4589
4616
|
if (!expected || !actual) {
|
|
4590
4617
|
invalidCorrelation2("Materialization event work was not fully written to the outbox.");
|
|
4591
4618
|
}
|
|
4592
|
-
|
|
4593
|
-
if (commitOrdinal !== ordinal || id !== createEventId2(session.header.commit.projectId, session.header.commit.id, ordinal) || stableJsonStringify2(actualDraft) !== stableJsonStringify2(expected.draft)) {
|
|
4619
|
+
if (actual.commitOrdinal !== ordinal || actual.id !== createEventId2(session.header.commit.projectId, session.header.commit.id, ordinal) || stableJsonStringify2(actual.event) !== stableJsonStringify2(expected.draft)) {
|
|
4594
4620
|
invalidCorrelation2("Materialization outbox event does not match its staged event work.");
|
|
4595
4621
|
}
|
|
4596
4622
|
const persisted = state.outbox.get(outboxKey(session.header.commit.projectId, actual.id));
|
|
4597
|
-
if (!persisted || stableJsonStringify2(persisted.
|
|
4623
|
+
if (!persisted || stableJsonStringify2(persisted.event) !== stableJsonStringify2(actual.event)) {
|
|
4598
4624
|
invalidCorrelation2("Materialization outbox event was not persisted.");
|
|
4599
4625
|
}
|
|
4600
4626
|
}
|
|
@@ -4602,26 +4628,15 @@ function assertFinalizedWork(session, state, objects) {
|
|
|
4602
4628
|
function assertFinalCardinality(records, projectId, objects) {
|
|
4603
4629
|
const effectiveOccupants = records.filter((record) => record.view === "effective");
|
|
4604
4630
|
const scopes = new Map;
|
|
4605
|
-
let currentScope = null;
|
|
4606
|
-
let occupiedCount = 0;
|
|
4607
4631
|
for (const record of effectiveOccupants) {
|
|
4608
|
-
if (record.scopeSortKey !== currentScope) {
|
|
4609
|
-
currentScope = record.scopeSortKey;
|
|
4610
|
-
occupiedCount = 0;
|
|
4611
|
-
}
|
|
4612
4632
|
const scope = scopes.get(record.scopeSortKey) ?? {
|
|
4613
4633
|
source: structuredClone(record.ref.source),
|
|
4614
4634
|
linkId: record.ref.linkId,
|
|
4615
4635
|
occupiedLinkKeys: new Set
|
|
4616
4636
|
};
|
|
4617
4637
|
scopes.set(record.scopeSortKey, scope);
|
|
4618
|
-
if (record.occupied)
|
|
4619
|
-
occupiedCount += 1;
|
|
4638
|
+
if (record.occupied)
|
|
4620
4639
|
scope.occupiedLinkKeys.add(record.linkSortKey);
|
|
4621
|
-
if (occupiedCount > 1) {
|
|
4622
|
-
invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
|
|
4623
|
-
}
|
|
4624
|
-
}
|
|
4625
4640
|
}
|
|
4626
4641
|
const adapter = getInMemoryObjectMaterializerAdapter(objects);
|
|
4627
4642
|
for (const scope of scopes.values()) {
|
|
@@ -4634,21 +4649,14 @@ function assertFinalCardinality(records, projectId, objects) {
|
|
|
4634
4649
|
}
|
|
4635
4650
|
}
|
|
4636
4651
|
}
|
|
4637
|
-
function
|
|
4638
|
-
const
|
|
4639
|
-
if (!
|
|
4640
|
-
invalidCorrelation2("Source activation does not match the replacement
|
|
4641
|
-
}
|
|
4642
|
-
if (activation.projectionKind === "object" && (!replacement.objectStreamCompleted || !replacement.linkStreamCompleted)) {
|
|
4643
|
-
invalidCorrelation2("Object projection replacement state was not fully streamed.");
|
|
4644
|
-
}
|
|
4645
|
-
if (activation.projectionKind === "link" && !replacement.linkStreamCompleted) {
|
|
4646
|
-
invalidCorrelation2("Link projection replacement state was not fully streamed.");
|
|
4652
|
+
function assertPlanCoverage(session, activation) {
|
|
4653
|
+
const plan = session.plan;
|
|
4654
|
+
if (!plan || plan.sourceId !== activation.source.projectionId || plan.materializationId !== activation.materializationId || plan.projectionKind !== activation.projectionKind) {
|
|
4655
|
+
invalidCorrelation2("Source activation does not match the replacement plan the session applies.");
|
|
4647
4656
|
}
|
|
4648
|
-
const
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
invalidCorrelation2("Projection replacement classification coverage does not match its streamed state.");
|
|
4657
|
+
const expected = (kind) => [...plan.identities.values()].filter((identity) => identity.entity.kind === kind && identity.diffRequired).map((identity) => identity.entity.kind === "object" ? objectRefKey2(identity.entity.ref) : linkRefKey2(identity.entity.ref)).sort();
|
|
4658
|
+
if (stableJsonStringify2(classificationKeys(session, "object")) !== stableJsonStringify2(expected("object")) || stableJsonStringify2(classificationKeys(session, "link")) !== stableJsonStringify2(expected("link")) || classificationKeys(session, "point").length > 0) {
|
|
4659
|
+
invalidCorrelation2("Projection replacement classification coverage does not match its plan.");
|
|
4652
4660
|
}
|
|
4653
4661
|
}
|
|
4654
4662
|
function classificationKeys(session, entityKind) {
|
|
@@ -4675,7 +4683,7 @@ function* previousSourceRows(state, candidate) {
|
|
|
4675
4683
|
}
|
|
4676
4684
|
}
|
|
4677
4685
|
}
|
|
4678
|
-
function activateSourceRoots(state, candidate, activation) {
|
|
4686
|
+
function activateSourceRoots(state, candidate, activation, touches) {
|
|
4679
4687
|
if (candidate.base && (candidate.base.materializationId !== activation.expected.activeMaterializationId || candidate.base.lastCommitId !== activation.expected.lastCommitId)) {
|
|
4680
4688
|
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
4681
4689
|
}
|
|
@@ -4698,6 +4706,7 @@ function activateSourceRoots(state, candidate, activation) {
|
|
|
4698
4706
|
if (sources.size === 0)
|
|
4699
4707
|
state.activeSourceRows.delete(indexKey);
|
|
4700
4708
|
if (row.assertion.kind === "link") {
|
|
4709
|
+
touches.links.add(linkRefKey2(row.assertion.ref));
|
|
4701
4710
|
const scopeKey = projectEntityKey(candidate.projectId, linkScopeSortKey2(row.assertion.ref.source, row.assertion.ref.linkId));
|
|
4702
4711
|
const scope = state.activeSourceLinkScopes.get(scopeKey);
|
|
4703
4712
|
scope.delete(JSON.stringify([candidate.source.projectionId, entityKey]));
|
|
@@ -4723,6 +4732,7 @@ function activateSourceRoots(state, candidate, activation) {
|
|
|
4723
4732
|
sources.set(candidate.source.projectionId, row);
|
|
4724
4733
|
state.activeSourceRows.set(indexKey, sources);
|
|
4725
4734
|
if (row.assertion.kind === "link") {
|
|
4735
|
+
touches.links.add(linkRefKey2(row.assertion.ref));
|
|
4726
4736
|
const scopeKey = projectEntityKey(candidate.projectId, linkScopeSortKey2(row.assertion.ref.source, row.assertion.ref.linkId));
|
|
4727
4737
|
const scope = state.activeSourceLinkScopes.get(scopeKey) ?? new Map;
|
|
4728
4738
|
scope.set(JSON.stringify([candidate.source.projectionId, entityKey]), row);
|
|
@@ -4734,325 +4744,749 @@ function activateSourceRoots(state, candidate, activation) {
|
|
|
4734
4744
|
state.activeSourceHeads.set(sourceKey, candidate.materializationId);
|
|
4735
4745
|
}
|
|
4736
4746
|
|
|
4737
|
-
// src/storage/ontology/in-memory/
|
|
4738
|
-
|
|
4747
|
+
// src/storage/ontology/in-memory/replacement-plans.ts
|
|
4748
|
+
function emptyCommitTouches(projectId) {
|
|
4749
|
+
return { projectId, objects: new Set, links: new Set, scopes: new Set };
|
|
4750
|
+
}
|
|
4751
|
+
function touchPlanItem(touches, item) {
|
|
4752
|
+
switch (item.kind) {
|
|
4753
|
+
case "object-override-upsert":
|
|
4754
|
+
case "object-override-delete":
|
|
4755
|
+
case "object-delete":
|
|
4756
|
+
touches.objects.add(objectRefKey2(item.value.ref));
|
|
4757
|
+
return;
|
|
4758
|
+
case "object-upsert":
|
|
4759
|
+
touches.objects.add(objectRefKey2(item.value.row.ref));
|
|
4760
|
+
return;
|
|
4761
|
+
case "point-upsert":
|
|
4762
|
+
touches.objects.add(objectRefKey2(item.value.point.series.object));
|
|
4763
|
+
return;
|
|
4764
|
+
case "link-override-upsert":
|
|
4765
|
+
case "link-override-delete":
|
|
4766
|
+
case "link-delete":
|
|
4767
|
+
touches.links.add(linkRefKey2(item.value.ref));
|
|
4768
|
+
return;
|
|
4769
|
+
case "link-upsert":
|
|
4770
|
+
touches.links.add(linkRefKey2(item.value.row.ref));
|
|
4771
|
+
return;
|
|
4772
|
+
case "link-slot-override-upsert":
|
|
4773
|
+
case "link-slot-override-delete":
|
|
4774
|
+
touches.scopes.add(linkScopeKey(item.value.ref.source, item.value.ref.linkId));
|
|
4775
|
+
return;
|
|
4776
|
+
}
|
|
4777
|
+
}
|
|
4778
|
+
function replacementPlanKey(input) {
|
|
4779
|
+
return sourceMaterializationKey(input.projectId, input.source.projectionId, input.materializationId);
|
|
4780
|
+
}
|
|
4781
|
+
|
|
4782
|
+
class InMemoryOntologyReplacementPlanStorage {
|
|
4739
4783
|
state;
|
|
4740
4784
|
objects;
|
|
4741
|
-
|
|
4742
|
-
|
|
4743
|
-
|
|
4744
|
-
executionExists;
|
|
4785
|
+
reader;
|
|
4786
|
+
runRootOperation;
|
|
4787
|
+
assertExecution;
|
|
4745
4788
|
hooks;
|
|
4746
|
-
|
|
4747
|
-
liveSessions = new Set;
|
|
4748
|
-
constructor(state, objects, timeseries, getTransactionToken, getMaterializationLifecycle, executionExists, hooks = {}) {
|
|
4789
|
+
constructor(state, objects, reader, runRootOperation, assertExecution, hooks = {}) {
|
|
4749
4790
|
this.state = state;
|
|
4750
4791
|
this.objects = objects;
|
|
4751
|
-
this.
|
|
4752
|
-
this.
|
|
4753
|
-
this.
|
|
4754
|
-
this.executionExists = executionExists;
|
|
4792
|
+
this.reader = reader;
|
|
4793
|
+
this.runRootOperation = runRootOperation;
|
|
4794
|
+
this.assertExecution = assertExecution;
|
|
4755
4795
|
this.hooks = hooks;
|
|
4756
4796
|
}
|
|
4757
|
-
async
|
|
4758
|
-
|
|
4759
|
-
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
|
|
4763
|
-
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
for (const expected of input.expected.sources)
|
|
4769
|
-
this.assertSource(expected, input.commit.projectId);
|
|
4770
|
-
for (const expected of input.expected.objects)
|
|
4771
|
-
await this.assertObject(expected, input.commit.projectId);
|
|
4772
|
-
for (const expected of input.expected.links)
|
|
4773
|
-
await this.assertLink(expected, input.commit.projectId);
|
|
4774
|
-
const expectedScopeRevisions = new Map;
|
|
4775
|
-
for (const expected of input.expected.linkScopes) {
|
|
4776
|
-
const key = linkScopeSortKey2(expected.source, expected.linkId);
|
|
4777
|
-
const current = expectedScopeRevisions.get(key) ?? this.computeEffectiveLinkScope(input.commit.projectId, expected.source, expected.linkId);
|
|
4778
|
-
expectedScopeRevisions.set(key, current);
|
|
4779
|
-
assertExpectedLinkScopeRevision2(current.fingerprint, expected);
|
|
4780
|
-
}
|
|
4781
|
-
for (const expected of input.expected.points)
|
|
4782
|
-
this.assertPoint(expected, input.commit.projectId);
|
|
4783
|
-
const providerToken = {};
|
|
4784
|
-
const session = {
|
|
4785
|
-
providerToken,
|
|
4786
|
-
header: structuredClone(input),
|
|
4787
|
-
transactionToken,
|
|
4788
|
-
lifecycle,
|
|
4789
|
-
active: true,
|
|
4790
|
-
writeOrdinal: 0,
|
|
4791
|
-
work: new Map,
|
|
4792
|
-
workUniqueKeys: new Set,
|
|
4793
|
-
applyWork: [],
|
|
4794
|
-
cardinalityWork: [],
|
|
4795
|
-
eventWork: [],
|
|
4796
|
-
appliedPlanItems: [],
|
|
4797
|
-
outboxEnvelopes: new Map,
|
|
4798
|
-
workStreams: {
|
|
4799
|
-
apply: { started: false, completed: false, emittedCount: 0 },
|
|
4800
|
-
cardinality: { started: false, completed: false, emittedCount: 0 },
|
|
4801
|
-
event: { started: false, completed: false, emittedCount: 0 }
|
|
4802
|
-
},
|
|
4803
|
-
workSealed: false,
|
|
4804
|
-
incidentLinksByObject: null,
|
|
4805
|
-
linkSlotStates: new Map,
|
|
4806
|
-
objectExistence: new Map,
|
|
4807
|
-
replacement: null
|
|
4808
|
-
};
|
|
4809
|
-
this.sessions.set(providerToken, session);
|
|
4810
|
-
this.liveSessions.add(session);
|
|
4811
|
-
lifecycle.register(providerToken);
|
|
4812
|
-
return { providerToken };
|
|
4813
|
-
}
|
|
4814
|
-
deactivateTransaction(transactionToken) {
|
|
4815
|
-
for (const session of this.liveSessions) {
|
|
4816
|
-
if (session.transactionToken !== transactionToken)
|
|
4817
|
-
continue;
|
|
4818
|
-
this.releaseSession(session);
|
|
4819
|
-
}
|
|
4820
|
-
}
|
|
4821
|
-
releaseSession(session) {
|
|
4822
|
-
session.active = false;
|
|
4823
|
-
session.work.clear();
|
|
4824
|
-
session.workUniqueKeys.clear();
|
|
4825
|
-
session.applyWork.length = 0;
|
|
4826
|
-
session.cardinalityWork.length = 0;
|
|
4827
|
-
session.eventWork.length = 0;
|
|
4828
|
-
session.appliedPlanItems.length = 0;
|
|
4829
|
-
session.outboxEnvelopes.clear();
|
|
4830
|
-
session.incidentLinksByObject = null;
|
|
4831
|
-
session.linkSlotStates = null;
|
|
4832
|
-
session.objectExistence.clear();
|
|
4833
|
-
session.replacement = null;
|
|
4834
|
-
this.liveSessions.delete(session);
|
|
4835
|
-
session.lifecycle.complete(session.providerToken);
|
|
4836
|
-
}
|
|
4837
|
-
async* streamState(input) {
|
|
4838
|
-
const session = this.requireSession(input.session);
|
|
4839
|
-
assertPageRows2(input.pageRows);
|
|
4840
|
-
for await (const request of input.requests) {
|
|
4841
|
-
this.requireSession(input.session);
|
|
4842
|
-
this.hooks.beforeRead?.("state.read");
|
|
4843
|
-
const objectRefs = uniqueSorted2(request.objects, objectRefKey2, objectRefSortKey2);
|
|
4844
|
-
for (let offset = 0;offset < objectRefs.length; offset += input.pageRows) {
|
|
4845
|
-
this.requireSession(input.session);
|
|
4846
|
-
const objects = [];
|
|
4847
|
-
for (const ref of objectRefs.slice(offset, offset + input.pageRows)) {
|
|
4848
|
-
objects.push(await this.objectState(session, ref));
|
|
4849
|
-
}
|
|
4850
|
-
this.requireSession(input.session);
|
|
4851
|
-
this.hooks.observeBuffer?.("state.object.page", objects.length);
|
|
4852
|
-
yield { objects, links: [], linkScopes: [], points: [] };
|
|
4853
|
-
}
|
|
4854
|
-
const linkRefs = uniqueSorted2(request.links, linkRefKey2, linkRefSortKey2);
|
|
4855
|
-
for (let offset = 0;offset < linkRefs.length; offset += input.pageRows) {
|
|
4856
|
-
this.requireSession(input.session);
|
|
4857
|
-
const links = [];
|
|
4858
|
-
for (const ref of linkRefs.slice(offset, offset + input.pageRows)) {
|
|
4859
|
-
links.push(await this.linkState(session, ref));
|
|
4860
|
-
}
|
|
4861
|
-
this.requireSession(input.session);
|
|
4862
|
-
this.hooks.observeBuffer?.("state.link.page", links.length);
|
|
4863
|
-
yield { objects: [], links, linkScopes: [], points: [] };
|
|
4864
|
-
}
|
|
4865
|
-
const incidentRefs = this.incidentLinkRefs(session, request.incidentObjects);
|
|
4866
|
-
for (let offset = 0;offset < incidentRefs.length; offset += input.pageRows) {
|
|
4867
|
-
this.requireSession(input.session);
|
|
4868
|
-
const refs = incidentRefs.slice(offset, offset + input.pageRows);
|
|
4869
|
-
const links = [];
|
|
4870
|
-
for (const ref of refs)
|
|
4871
|
-
links.push(await this.linkState(session, ref));
|
|
4872
|
-
this.requireSession(input.session);
|
|
4873
|
-
this.hooks.observeBuffer?.("state.incident-link.page", links.length);
|
|
4874
|
-
yield { objects: [], links, linkScopes: [], points: [] };
|
|
4875
|
-
}
|
|
4876
|
-
for (const scope of uniqueBy(request.linkScopes, (value) => linkScopeSortKey2(value.source, value.linkId))) {
|
|
4877
|
-
this.requireSession(input.session);
|
|
4878
|
-
const value = this.linkSlotState(session, scope.source, scope.linkId);
|
|
4879
|
-
yield { objects: [], links: [], linkScopes: [value], points: [] };
|
|
4797
|
+
async open(input) {
|
|
4798
|
+
return this.runRootOperation(async () => {
|
|
4799
|
+
assertReplacementPlanCommit2(input.commit);
|
|
4800
|
+
const candidate = await this.requireCandidate(input);
|
|
4801
|
+
const key = replacementPlanKey(input);
|
|
4802
|
+
const active = findActiveSourceMaterialization(this.state, input.projectId, input.source.projectionId);
|
|
4803
|
+
const replacedMaterializationId = active?.materializationId ?? null;
|
|
4804
|
+
const replacedLastCommitId = active?.lastCommitId ?? null;
|
|
4805
|
+
const existing = this.state.replacementPlans.get(key);
|
|
4806
|
+
if (existing?.replacedMaterializationId === replacedMaterializationId && existing.replacedLastCommitId === replacedLastCommitId) {
|
|
4807
|
+
return { committedAt: existing.committedAt };
|
|
4880
4808
|
}
|
|
4881
|
-
const
|
|
4882
|
-
|
|
4883
|
-
|
|
4884
|
-
|
|
4885
|
-
|
|
4886
|
-
|
|
4887
|
-
|
|
4888
|
-
|
|
4809
|
+
const plan = {
|
|
4810
|
+
projectId: input.projectId,
|
|
4811
|
+
sourceId: input.source.projectionId,
|
|
4812
|
+
materializationId: input.materializationId,
|
|
4813
|
+
projectionKind: candidate.projectionKind,
|
|
4814
|
+
commitId: input.commit.id,
|
|
4815
|
+
committedAt: input.commit.committedAt,
|
|
4816
|
+
replacedMaterializationId,
|
|
4817
|
+
replacedLastCommitId,
|
|
4818
|
+
watermark: this.state.commitsById.size,
|
|
4819
|
+
owned: new Set,
|
|
4820
|
+
identities: new Map,
|
|
4821
|
+
workKeys: new Set
|
|
4822
|
+
};
|
|
4823
|
+
for (const rows of [
|
|
4824
|
+
previousSourceRows(this.state, candidate),
|
|
4825
|
+
candidate.rowsByEntity.values()
|
|
4826
|
+
]) {
|
|
4827
|
+
for (const row of rows) {
|
|
4828
|
+
plan.owned.add(projectionEntityKey2(row.assertion));
|
|
4829
|
+
addIdentity(plan, row.assertion, true);
|
|
4889
4830
|
}
|
|
4890
|
-
this.requireSession(input.session);
|
|
4891
|
-
this.hooks.observeBuffer?.("state.point.page", points.length);
|
|
4892
|
-
if (points.length > 0)
|
|
4893
|
-
yield { objects: [], links: [], linkScopes: [], points };
|
|
4894
4831
|
}
|
|
4895
|
-
|
|
4832
|
+
this.state.replacementPlans.set(key, plan);
|
|
4833
|
+
return { committedAt: plan.committedAt };
|
|
4834
|
+
});
|
|
4896
4835
|
}
|
|
4897
|
-
async*
|
|
4898
|
-
const session = this.requireSession(input.session);
|
|
4836
|
+
async* streamState(input) {
|
|
4899
4837
|
assertPageRows2(input.pageRows);
|
|
4900
|
-
|
|
4901
|
-
|
|
4902
|
-
|
|
4838
|
+
this.hooks.beforeRead?.(`replacement-plan.${input.entityKind}`);
|
|
4839
|
+
const pending = await this.runRootOperation(async () => {
|
|
4840
|
+
const { plan } = await this.requirePlan(input);
|
|
4841
|
+
if (input.entityKind === "object" && plan.projectionKind !== "object") {
|
|
4903
4842
|
throw new MaterializationConflictError2("source-materialization", "Link projection replacement cannot stream object state.");
|
|
4904
4843
|
}
|
|
4905
|
-
if (
|
|
4906
|
-
|
|
4907
|
-
|
|
4908
|
-
this.hooks.beforeRead?.("source-replacement.object");
|
|
4909
|
-
for (let offset = 0;offset < replacement.orderedObjects.length; offset += input.pageRows) {
|
|
4910
|
-
this.requireSession(input.session);
|
|
4911
|
-
const selected = replacement.orderedObjects.slice(offset, offset + input.pageRows);
|
|
4912
|
-
const objects = [];
|
|
4913
|
-
for (const entry of selected) {
|
|
4914
|
-
const key = projectionEntityKey2({ kind: "object", ref: entry.ref });
|
|
4915
|
-
const base = await this.objectState(session, entry.ref);
|
|
4916
|
-
objects.push({
|
|
4917
|
-
ref: base.ref,
|
|
4918
|
-
candidateSource: storedSourceObject(input.source.projectionId, replacement.candidate.materializationId, replacement.candidate.rowsByEntity.get(key)),
|
|
4919
|
-
override: base.override,
|
|
4920
|
-
effective: base.effective,
|
|
4921
|
-
latestTelemetry: base.latestTelemetry
|
|
4922
|
-
});
|
|
4844
|
+
if (input.entityKind === "link") {
|
|
4845
|
+
if (unplanned(plan, "object").length > 0) {
|
|
4846
|
+
throw new MaterializationConflictError2("effective-state", "Object projection replacement must plan every object before its links.");
|
|
4923
4847
|
}
|
|
4924
|
-
this.
|
|
4925
|
-
this.hooks.observeBuffer?.("replacement.object.page", objects.length);
|
|
4926
|
-
yield { objects, links: [] };
|
|
4848
|
+
this.expandLinks(plan);
|
|
4927
4849
|
}
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
replacement.linkStreamStarted = true;
|
|
4938
|
-
this.hooks.beforeRead?.("source-replacement.link");
|
|
4939
|
-
this.expandReplacementLinks(session, replacement);
|
|
4940
|
-
replacement.orderedLinks ??= [...replacement.links.values()].sort((left, right) => left.sortKey.localeCompare(right.sortKey));
|
|
4941
|
-
for (let offset = 0;offset < replacement.orderedLinks.length; offset += input.pageRows) {
|
|
4942
|
-
this.requireSession(input.session);
|
|
4943
|
-
const selected = replacement.orderedLinks.slice(offset, offset + input.pageRows);
|
|
4944
|
-
const links = [];
|
|
4945
|
-
for (const entry of selected) {
|
|
4946
|
-
const key = projectionEntityKey2({ kind: "link", ref: entry.ref });
|
|
4947
|
-
const base = await this.linkState(session, entry.ref);
|
|
4948
|
-
const ownedByReplacement = replacement.owned.has(key);
|
|
4949
|
-
links.push({
|
|
4950
|
-
ref: base.ref,
|
|
4951
|
-
candidateSource: ownedByReplacement ? storedSourceLink(input.source.projectionId, replacement.candidate.materializationId, replacement.candidate.rowsByEntity.get(key)) : base.source,
|
|
4952
|
-
override: base.override,
|
|
4953
|
-
slotOverride: base.slotOverride,
|
|
4954
|
-
effective: base.effective,
|
|
4955
|
-
diffRequired: entry.diffRequired
|
|
4850
|
+
return unplanned(plan, input.entityKind).map((identity) => projectionEntityKey2(identity.entity));
|
|
4851
|
+
});
|
|
4852
|
+
for (let offset = 0;offset < pending.length; offset += input.pageRows) {
|
|
4853
|
+
const keys = pending.slice(offset, offset + input.pageRows);
|
|
4854
|
+
const page = await this.runRootOperation(async () => {
|
|
4855
|
+
const { plan, candidate } = await this.requirePlan(input);
|
|
4856
|
+
const selected = keys.flatMap((key) => {
|
|
4857
|
+
const identity = plan.identities.get(key);
|
|
4858
|
+
return identity?.plannedRevision === null ? [identity] : [];
|
|
4956
4859
|
});
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4860
|
+
if (selected.length === 0)
|
|
4861
|
+
return null;
|
|
4862
|
+
return input.entityKind === "object" ? this.objectPage(plan, candidate, selected) : this.linkPage(plan, candidate, selected);
|
|
4863
|
+
});
|
|
4864
|
+
if (!page)
|
|
4865
|
+
continue;
|
|
4866
|
+
this.hooks.observeBuffer?.(`replacement.${input.entityKind}.page`, page.objects.length + page.links.length);
|
|
4867
|
+
yield page;
|
|
4961
4868
|
}
|
|
4962
|
-
this.requireSession(input.session);
|
|
4963
|
-
replacement.linkStreamCompleted = true;
|
|
4964
4869
|
}
|
|
4965
|
-
async
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
4972
|
-
|
|
4973
|
-
|
|
4974
|
-
|
|
4975
|
-
|
|
4976
|
-
|
|
4977
|
-
|
|
4978
|
-
|
|
4979
|
-
|
|
4870
|
+
async stage(input) {
|
|
4871
|
+
await this.runRootOperation(async () => {
|
|
4872
|
+
const { plan } = await this.requirePlan(input);
|
|
4873
|
+
const updates = prepareReplacementIdentities2(input, plan).map((prepared) => {
|
|
4874
|
+
const current = plan.identities.get(prepared.entityKey);
|
|
4875
|
+
if (!current || current.readRevision === null || current.plannedRevision !== null) {
|
|
4876
|
+
throw new MaterializationValidationError2(`Replacement identity ${prepared.entityKey} is not streamed and still to plan.`);
|
|
4877
|
+
}
|
|
4878
|
+
return { current, records: structuredClone(prepared.records) };
|
|
4879
|
+
});
|
|
4880
|
+
const claimed = new Set;
|
|
4881
|
+
for (const { records } of updates) {
|
|
4882
|
+
for (const record of records) {
|
|
4883
|
+
for (const key of workKeys(record)) {
|
|
4884
|
+
if (claimed.has(key) || plan.workKeys.has(key)) {
|
|
4885
|
+
throw duplicateMaterializationWork2(record.recordKey);
|
|
4886
|
+
}
|
|
4887
|
+
claimed.add(key);
|
|
4888
|
+
}
|
|
4889
|
+
}
|
|
4980
4890
|
}
|
|
4981
|
-
|
|
4982
|
-
|
|
4983
|
-
|
|
4984
|
-
|
|
4985
|
-
|
|
4986
|
-
for (const record of cloned) {
|
|
4987
|
-
session.work.set(record.recordKey, record);
|
|
4988
|
-
session.workUniqueKeys.add(workUniquenessKey2(record));
|
|
4989
|
-
if (record.kind === "plan")
|
|
4990
|
-
session.applyWork.push(record);
|
|
4991
|
-
if (record.kind === "cardinality")
|
|
4992
|
-
session.cardinalityWork.push(record);
|
|
4993
|
-
if (record.kind === "event")
|
|
4994
|
-
session.eventWork.push(record);
|
|
4995
|
-
if (record.kind === "object-existence") {
|
|
4996
|
-
session.objectExistence.set(objectRefKey2(record.ref), record);
|
|
4891
|
+
const staged = updates.flatMap(({ records }) => records);
|
|
4892
|
+
this.hooks.observeWork?.(staged);
|
|
4893
|
+
for (const { current, records } of updates) {
|
|
4894
|
+
current.records = records;
|
|
4895
|
+
current.plannedRevision = current.readRevision;
|
|
4997
4896
|
}
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
|
|
5003
|
-
|
|
4897
|
+
for (const key of claimed)
|
|
4898
|
+
plan.workKeys.add(key);
|
|
4899
|
+
this.hooks.observeBuffer?.("plan.stage", updates.length);
|
|
4900
|
+
});
|
|
4901
|
+
}
|
|
4902
|
+
async refresh(input) {
|
|
4903
|
+
const { plan } = await this.requirePlan(input);
|
|
4904
|
+
const pending = [...plan.identities.values()].filter((identity) => identity.plannedRevision === null);
|
|
4905
|
+
if (pending.length > 0)
|
|
4906
|
+
return { fresh: false, unplanned: pending.length };
|
|
4907
|
+
let unplannedCount = 0;
|
|
4908
|
+
if (this.state.commitsById.size !== plan.watermark) {
|
|
4909
|
+
const touched = emptyCommitTouches(plan.projectId);
|
|
4910
|
+
for (let ordinal = plan.watermark + 1;ordinal <= this.state.commitsById.size; ordinal += 1) {
|
|
4911
|
+
const touches = this.state.commitTouches.get(ordinal);
|
|
4912
|
+
if (touches?.projectId !== plan.projectId)
|
|
4913
|
+
continue;
|
|
4914
|
+
for (const kind of ["objects", "links", "scopes"]) {
|
|
4915
|
+
for (const key of touches[kind])
|
|
4916
|
+
touched[kind].add(key);
|
|
5004
4917
|
}
|
|
5005
4918
|
}
|
|
4919
|
+
for (const identity of plan.identities.values()) {
|
|
4920
|
+
if (!touchesIdentity(touched, identity.entity))
|
|
4921
|
+
continue;
|
|
4922
|
+
if (this.revision(plan, identity) === identity.plannedRevision)
|
|
4923
|
+
continue;
|
|
4924
|
+
unplan(plan, identity);
|
|
4925
|
+
unplannedCount += 1;
|
|
4926
|
+
}
|
|
4927
|
+
unplannedCount += this.expandTouched(plan, touched);
|
|
4928
|
+
plan.watermark = this.state.commitsById.size;
|
|
5006
4929
|
}
|
|
5007
|
-
|
|
4930
|
+
if (unplannedCount > 0)
|
|
4931
|
+
return { fresh: false, unplanned: unplannedCount };
|
|
4932
|
+
return {
|
|
4933
|
+
fresh: true,
|
|
4934
|
+
counts: projectionChangeCounts([...plan.identities.values()].flatMap((identity) => identity.records))
|
|
4935
|
+
};
|
|
4936
|
+
}
|
|
4937
|
+
async purge(input) {
|
|
4938
|
+
assertNonblank2(input.projectId, "Replacement plan purge project id");
|
|
4939
|
+
assertPositiveInteger2(input.limit, "Replacement plan purge limit");
|
|
4940
|
+
return this.runRootOperation(() => {
|
|
4941
|
+
let horizon = this.state.commitsById.size;
|
|
4942
|
+
for (const plan of this.state.replacementPlans.values()) {
|
|
4943
|
+
if (plan.projectId === input.projectId)
|
|
4944
|
+
horizon = Math.min(horizon, plan.watermark);
|
|
4945
|
+
}
|
|
4946
|
+
let deleted = 0;
|
|
4947
|
+
for (const [ordinal, touches] of this.state.commitTouches) {
|
|
4948
|
+
if (deleted === input.limit)
|
|
4949
|
+
break;
|
|
4950
|
+
if (ordinal > horizon || touches.projectId !== input.projectId)
|
|
4951
|
+
continue;
|
|
4952
|
+
this.state.commitTouches.delete(ordinal);
|
|
4953
|
+
deleted += 1;
|
|
4954
|
+
}
|
|
4955
|
+
return deleted;
|
|
4956
|
+
});
|
|
4957
|
+
}
|
|
4958
|
+
async objectPage(plan, candidate, selected) {
|
|
4959
|
+
const objects = [];
|
|
4960
|
+
for (const identity of selected) {
|
|
4961
|
+
const ref = identity.entity.ref;
|
|
4962
|
+
const base = await this.reader.objectState(plan.projectId, ref);
|
|
4963
|
+
objects.push({
|
|
4964
|
+
ref: base.ref,
|
|
4965
|
+
candidateSource: storedSourceObject(plan.sourceId, plan.materializationId, candidate.rowsByEntity.get(projectionEntityKey2(identity.entity))),
|
|
4966
|
+
override: base.override,
|
|
4967
|
+
effective: base.effective,
|
|
4968
|
+
latestTelemetry: base.latestTelemetry
|
|
4969
|
+
});
|
|
4970
|
+
identity.readRevision = this.revision(plan, identity);
|
|
4971
|
+
}
|
|
4972
|
+
return { objects, links: [], endpoints: [] };
|
|
4973
|
+
}
|
|
4974
|
+
async linkPage(plan, candidate, selected) {
|
|
4975
|
+
const links = [];
|
|
4976
|
+
const endpoints = new Map;
|
|
4977
|
+
for (const identity of selected) {
|
|
4978
|
+
const ref = identity.entity.ref;
|
|
4979
|
+
const key = projectionEntityKey2(identity.entity);
|
|
4980
|
+
const base = await this.reader.linkState(plan.projectId, ref);
|
|
4981
|
+
links.push({
|
|
4982
|
+
ref: base.ref,
|
|
4983
|
+
candidateSource: plan.owned.has(key) ? storedSourceLink(plan.sourceId, plan.materializationId, candidate.rowsByEntity.get(key)) : base.source,
|
|
4984
|
+
override: base.override,
|
|
4985
|
+
slotOverride: base.slotOverride,
|
|
4986
|
+
effective: base.effective,
|
|
4987
|
+
diffRequired: identity.diffRequired
|
|
4988
|
+
});
|
|
4989
|
+
for (const endpoint of [ref.source, ref.target]) {
|
|
4990
|
+
endpoints.set(objectRefKey2(endpoint), {
|
|
4991
|
+
ref: structuredClone(endpoint),
|
|
4992
|
+
exists: this.endpointExists(plan, endpoint)
|
|
4993
|
+
});
|
|
4994
|
+
}
|
|
4995
|
+
identity.readRevision = this.revision(plan, identity);
|
|
4996
|
+
}
|
|
4997
|
+
return { objects: [], links, endpoints: [...endpoints.values()] };
|
|
4998
|
+
}
|
|
4999
|
+
endpointExists(plan, ref) {
|
|
5000
|
+
const planned = plan.identities.get(projectionEntityKey2({ kind: "object", ref }));
|
|
5001
|
+
const existence = planned?.records.find((record) => record.kind === "object-existence");
|
|
5002
|
+
if (existence?.kind === "object-existence")
|
|
5003
|
+
return existence.exists;
|
|
5004
|
+
return getInMemoryObjectMaterializerAdapter(this.objects).getExactObjectRow(plan.projectId, ref.objectTypeId, ref.primaryId) !== null;
|
|
5005
|
+
}
|
|
5006
|
+
revision(plan, identity) {
|
|
5007
|
+
if (identity.entity.kind === "object") {
|
|
5008
|
+
return stableJsonStringify2(this.objectRevision(plan.projectId, identity.entity.ref));
|
|
5009
|
+
}
|
|
5010
|
+
const ref = identity.entity.ref;
|
|
5011
|
+
const projectId = plan.projectId;
|
|
5012
|
+
const row = getInMemoryObjectMaterializerAdapter(this.objects).getExactLinkRow(projectId, {
|
|
5013
|
+
sourceTypeId: ref.source.objectTypeId,
|
|
5014
|
+
sourceId: ref.source.primaryId,
|
|
5015
|
+
linkId: ref.linkId,
|
|
5016
|
+
targetTypeId: ref.target.objectTypeId,
|
|
5017
|
+
targetId: ref.target.primaryId
|
|
5018
|
+
});
|
|
5019
|
+
const live = this.reader.findActiveLinkSource(projectId, ref);
|
|
5020
|
+
return stableJsonStringify2([
|
|
5021
|
+
row?.lastCommitId ?? null,
|
|
5022
|
+
this.state.linkOverrides.get(projectEntityKey(projectId, linkRefKey2(ref)))?.lastCommitId ?? null,
|
|
5023
|
+
this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(ref.source, ref.linkId)))?.lastCommitId ?? null,
|
|
5024
|
+
live ? [live.source.projectionId, live.materializationId, live.stagingOrdinal] : null,
|
|
5025
|
+
this.endpointRevision(plan, ref.source),
|
|
5026
|
+
this.endpointRevision(plan, ref.target)
|
|
5027
|
+
]);
|
|
5028
|
+
}
|
|
5029
|
+
endpointRevision(plan, ref) {
|
|
5030
|
+
if (plan.identities.has(projectionEntityKey2({ kind: "object", ref }))) {
|
|
5031
|
+
return this.objectRevision(plan.projectId, ref);
|
|
5032
|
+
}
|
|
5033
|
+
return getInMemoryObjectMaterializerAdapter(this.objects).getExactObjectRow(plan.projectId, ref.objectTypeId, ref.primaryId) !== null;
|
|
5034
|
+
}
|
|
5035
|
+
objectRevision(projectId, ref) {
|
|
5036
|
+
const row = getInMemoryObjectMaterializerAdapter(this.objects).getExactObjectRow(projectId, ref.objectTypeId, ref.primaryId);
|
|
5037
|
+
return [
|
|
5038
|
+
row ? [row.version, row.lastCommitId] : null,
|
|
5039
|
+
this.state.objectOverrides.get(projectEntityKey(projectId, objectRefKey2(ref)))?.lastCommitId ?? null,
|
|
5040
|
+
this.reader.latestTelemetry(projectId, ref).map((point) => [point.series.propertyId, point.at, point.lastCommitId])
|
|
5041
|
+
];
|
|
5042
|
+
}
|
|
5043
|
+
expandLinks(plan) {
|
|
5044
|
+
const projectId = plan.projectId;
|
|
5045
|
+
const incident = new Set;
|
|
5046
|
+
for (const identity of plan.identities.values()) {
|
|
5047
|
+
if (identity.entity.kind !== "object" || identity.expanded)
|
|
5048
|
+
continue;
|
|
5049
|
+
if (identity.plannedRevision === null)
|
|
5050
|
+
continue;
|
|
5051
|
+
identity.expanded = true;
|
|
5052
|
+
for (const record of identity.records) {
|
|
5053
|
+
if (record.kind === "incident-object")
|
|
5054
|
+
incident.add(objectRefKey2(record.ref));
|
|
5055
|
+
}
|
|
5056
|
+
}
|
|
5057
|
+
let added = 0;
|
|
5058
|
+
const consider = (ref, diffRequired) => {
|
|
5059
|
+
if (addIdentity(plan, { kind: "link", ref }, diffRequired))
|
|
5060
|
+
added += 1;
|
|
5061
|
+
};
|
|
5062
|
+
if (incident.size > 0) {
|
|
5063
|
+
const index = this.reader.incidentLinkIndex(projectId);
|
|
5064
|
+
for (const key of incident) {
|
|
5065
|
+
for (const ref of index.get(key) ?? [])
|
|
5066
|
+
consider(ref, true);
|
|
5067
|
+
}
|
|
5068
|
+
}
|
|
5069
|
+
const adapter = getInMemoryObjectMaterializerAdapter(this.objects);
|
|
5070
|
+
const scopes = new Map;
|
|
5071
|
+
for (const identity of plan.identities.values()) {
|
|
5072
|
+
if (identity.entity.kind !== "link" || !identity.diffRequired || identity.expanded)
|
|
5073
|
+
continue;
|
|
5074
|
+
identity.expanded = true;
|
|
5075
|
+
const ref = identity.entity.ref;
|
|
5076
|
+
scopes.set(linkScopeSortKey2(ref.source, ref.linkId), ref);
|
|
5077
|
+
}
|
|
5078
|
+
for (const scope of scopes.values()) {
|
|
5079
|
+
adapter.visitExactScopeLinks(projectId, scope.source.objectTypeId, scope.source.primaryId, scope.linkId, (row) => consider(linkRef(row), false));
|
|
5080
|
+
const override = this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(scope.source, scope.linkId)));
|
|
5081
|
+
if (override)
|
|
5082
|
+
consider({ ...scope, target: override.value.target }, false);
|
|
5083
|
+
}
|
|
5084
|
+
return added;
|
|
5085
|
+
}
|
|
5086
|
+
expandTouched(plan, touched) {
|
|
5087
|
+
const projectId = plan.projectId;
|
|
5088
|
+
const incident = new Set;
|
|
5089
|
+
const scopes = new Set;
|
|
5090
|
+
for (const identity of plan.identities.values()) {
|
|
5091
|
+
for (const record of identity.records) {
|
|
5092
|
+
if (record.kind === "incident-object")
|
|
5093
|
+
incident.add(objectRefKey2(record.ref));
|
|
5094
|
+
}
|
|
5095
|
+
if (identity.entity.kind === "link" && identity.diffRequired) {
|
|
5096
|
+
scopes.add(linkScopeKey(identity.entity.ref.source, identity.entity.ref.linkId));
|
|
5097
|
+
}
|
|
5098
|
+
}
|
|
5099
|
+
let added = 0;
|
|
5100
|
+
const consider = (ref, member) => {
|
|
5101
|
+
if (incident.has(objectRefKey2(ref.source)) || incident.has(objectRefKey2(ref.target))) {
|
|
5102
|
+
if (addIdentity(plan, { kind: "link", ref }, true))
|
|
5103
|
+
added += 1;
|
|
5104
|
+
} else if (member && scopes.has(linkScopeKey(ref.source, ref.linkId))) {
|
|
5105
|
+
if (addIdentity(plan, { kind: "link", ref }, false))
|
|
5106
|
+
added += 1;
|
|
5107
|
+
}
|
|
5108
|
+
};
|
|
5109
|
+
const adapter = getInMemoryObjectMaterializerAdapter(this.objects);
|
|
5110
|
+
for (const key of touched.links) {
|
|
5111
|
+
const ref = parseLinkKey(key);
|
|
5112
|
+
const effective = adapter.getExactLinkRow(projectId, {
|
|
5113
|
+
sourceTypeId: ref.source.objectTypeId,
|
|
5114
|
+
sourceId: ref.source.primaryId,
|
|
5115
|
+
linkId: ref.linkId,
|
|
5116
|
+
targetTypeId: ref.target.objectTypeId,
|
|
5117
|
+
targetId: ref.target.primaryId
|
|
5118
|
+
}) !== null;
|
|
5119
|
+
if (effective || this.state.linkOverrides.has(projectEntityKey(projectId, key)) || this.state.activeSourceRows.has(projectEntityKey(projectId, projectionEntityKey2({ kind: "link", ref })))) {
|
|
5120
|
+
consider(ref, effective);
|
|
5121
|
+
}
|
|
5122
|
+
}
|
|
5123
|
+
for (const key of touched.scopes) {
|
|
5124
|
+
const override = this.state.linkSlotOverrides.get(projectEntityKey(projectId, key));
|
|
5125
|
+
if (override)
|
|
5126
|
+
consider({ ...override.ref, target: override.value.target }, true);
|
|
5127
|
+
}
|
|
5128
|
+
return added;
|
|
5129
|
+
}
|
|
5130
|
+
async requireCandidate(input) {
|
|
5131
|
+
const candidate = this.state.sourceMaterializations.get(replacementPlanKey(input));
|
|
5132
|
+
if (!candidate || candidate.status !== "ready" || candidate.projectionRunId !== input.execution.projectionRunId || candidate.executionToken !== input.execution.executionToken) {
|
|
5133
|
+
throw new MaterializationConflictError2("source-materialization", `Candidate source materialization '${input.materializationId}' is missing, not ready, or owned by another execution.`);
|
|
5134
|
+
}
|
|
5135
|
+
await this.assertExecution({
|
|
5136
|
+
projectId: input.projectId,
|
|
5137
|
+
source: input.source,
|
|
5138
|
+
execution: input.execution
|
|
5139
|
+
});
|
|
5140
|
+
return candidate;
|
|
5141
|
+
}
|
|
5142
|
+
async requirePlan(input) {
|
|
5143
|
+
const candidate = await this.requireCandidate(input);
|
|
5144
|
+
const plan = this.state.replacementPlans.get(replacementPlanKey(input));
|
|
5145
|
+
if (!plan) {
|
|
5146
|
+
throw new MaterializationConflictError2("source-materialization", `Candidate source materialization '${input.materializationId}' has no open plan.`);
|
|
5147
|
+
}
|
|
5148
|
+
return { plan, candidate };
|
|
5149
|
+
}
|
|
5150
|
+
}
|
|
5151
|
+
function addIdentity(plan, entity, diffRequired) {
|
|
5152
|
+
const key = projectionEntityKey2(entity);
|
|
5153
|
+
const existing = plan.identities.get(key);
|
|
5154
|
+
if (existing) {
|
|
5155
|
+
if (!diffRequired || existing.diffRequired)
|
|
5156
|
+
return false;
|
|
5157
|
+
existing.diffRequired = true;
|
|
5158
|
+
unplan(plan, existing);
|
|
5159
|
+
return true;
|
|
5160
|
+
}
|
|
5161
|
+
plan.identities.set(key, {
|
|
5162
|
+
entity: structuredClone({ kind: entity.kind, ref: entity.ref }),
|
|
5163
|
+
sortKey: entity.kind === "object" ? objectRefSortKey2(entity.ref) : linkRefSortKey2(entity.ref),
|
|
5164
|
+
diffRequired,
|
|
5165
|
+
expanded: false,
|
|
5166
|
+
readRevision: null,
|
|
5167
|
+
plannedRevision: null,
|
|
5168
|
+
records: []
|
|
5169
|
+
});
|
|
5170
|
+
return true;
|
|
5171
|
+
}
|
|
5172
|
+
function unplan(plan, identity) {
|
|
5173
|
+
for (const record of identity.records) {
|
|
5174
|
+
for (const key of workKeys(record))
|
|
5175
|
+
plan.workKeys.delete(key);
|
|
5176
|
+
}
|
|
5177
|
+
identity.readRevision = null;
|
|
5178
|
+
identity.plannedRevision = null;
|
|
5179
|
+
identity.expanded = false;
|
|
5180
|
+
identity.records = [];
|
|
5181
|
+
}
|
|
5182
|
+
function touchesIdentity(touched, entity) {
|
|
5183
|
+
if (entity.kind === "object")
|
|
5184
|
+
return touched.objects.has(objectRefKey2(entity.ref));
|
|
5185
|
+
const { ref } = entity;
|
|
5186
|
+
return touched.links.has(linkRefKey2(ref)) || touched.scopes.has(linkScopeKey(ref.source, ref.linkId)) || touched.objects.has(objectRefKey2(ref.source)) || touched.objects.has(objectRefKey2(ref.target));
|
|
5187
|
+
}
|
|
5188
|
+
function parseLinkKey(key) {
|
|
5189
|
+
const [sourceTypeId, sourceId, linkId, targetTypeId, targetId] = JSON.parse(key);
|
|
5190
|
+
return {
|
|
5191
|
+
source: { objectTypeId: sourceTypeId, primaryId: sourceId },
|
|
5192
|
+
linkId,
|
|
5193
|
+
target: { objectTypeId: targetTypeId, primaryId: targetId }
|
|
5194
|
+
};
|
|
5195
|
+
}
|
|
5196
|
+
function workKeys(record) {
|
|
5197
|
+
return [`record:${record.recordKey}`, `unique:${workUniquenessKey2(record)}`];
|
|
5198
|
+
}
|
|
5199
|
+
function unplanned(plan, kind) {
|
|
5200
|
+
return [...plan.identities.values()].filter((identity) => identity.entity.kind === kind && identity.plannedRevision === null).sort((left, right) => left.sortKey.localeCompare(right.sortKey));
|
|
5201
|
+
}
|
|
5202
|
+
|
|
5203
|
+
// src/storage/ontology/in-memory/vectors.ts
|
|
5204
|
+
function vectorObjectKey(projectId, ref) {
|
|
5205
|
+
return JSON.stringify([projectId, ref.objectTypeId, ref.primaryId]);
|
|
5206
|
+
}
|
|
5207
|
+
|
|
5208
|
+
class InMemoryOntologyVectorStorage {
|
|
5209
|
+
state;
|
|
5210
|
+
assertSession;
|
|
5211
|
+
runRootOperation;
|
|
5212
|
+
constructor(state, assertSession, runRootOperation) {
|
|
5213
|
+
this.state = state;
|
|
5214
|
+
this.assertSession = assertSession;
|
|
5215
|
+
this.runRootOperation = runRootOperation;
|
|
5216
|
+
}
|
|
5217
|
+
async list(input) {
|
|
5218
|
+
return this.listBatch({ projectId: input.projectId, refs: [input.ref] });
|
|
5219
|
+
}
|
|
5220
|
+
async listBatch(input) {
|
|
5221
|
+
return this.runRootOperation(() => [...new Set(input.refs.map((ref) => vectorObjectKey(input.projectId, ref)))].flatMap((key) => [...this.state.get(key)?.values() ?? []].map(({ values: _values, ...metadata }) => structuredClone(metadata))));
|
|
5222
|
+
}
|
|
5223
|
+
async removeBatch(input) {
|
|
5224
|
+
this.assertSession(input.session, input.projectId);
|
|
5225
|
+
for (const entry of input.entries) {
|
|
5226
|
+
await this.remove({ session: input.session, projectId: input.projectId, ...entry });
|
|
5227
|
+
}
|
|
5228
|
+
}
|
|
5229
|
+
async write(input) {
|
|
5230
|
+
this.assertSession(input.session, input.projectId, input.value.lastCommitId);
|
|
5231
|
+
const key = vectorObjectKey(input.projectId, input.value.ref);
|
|
5232
|
+
const profiles = this.state.get(key) ?? new Map;
|
|
5233
|
+
this.assertRevision(profiles.get(input.value.profile), input.expectedCommitId);
|
|
5234
|
+
profiles.set(input.value.profile, structuredClone(input.value));
|
|
5235
|
+
this.state.set(key, profiles);
|
|
5236
|
+
}
|
|
5237
|
+
async writeBatch(input) {
|
|
5238
|
+
for (const entry of input.entries) {
|
|
5239
|
+
await this.write({ session: input.session, projectId: input.projectId, ...entry });
|
|
5240
|
+
}
|
|
5241
|
+
}
|
|
5242
|
+
async remove(input) {
|
|
5243
|
+
this.assertSession(input.session, input.projectId);
|
|
5244
|
+
const key = vectorObjectKey(input.projectId, input.ref);
|
|
5245
|
+
const profiles = this.state.get(key);
|
|
5246
|
+
this.assertRevision(profiles?.get(input.profile), input.expectedCommitId);
|
|
5247
|
+
profiles?.delete(input.profile);
|
|
5248
|
+
if (profiles?.size === 0)
|
|
5249
|
+
this.state.delete(key);
|
|
5250
|
+
}
|
|
5251
|
+
assertRevision(current, expected) {
|
|
5252
|
+
if ((current?.lastCommitId ?? null) !== expected) {
|
|
5253
|
+
throw new MaterializationConflictError2("effective-state", "Vector changed since it was prepared.");
|
|
5254
|
+
}
|
|
5255
|
+
}
|
|
5256
|
+
}
|
|
5257
|
+
|
|
5258
|
+
// src/storage/ontology/in-memory/materializations.ts
|
|
5259
|
+
class InMemoryOntologyMaterializationStorage {
|
|
5260
|
+
state;
|
|
5261
|
+
objects;
|
|
5262
|
+
timeseries;
|
|
5263
|
+
reader;
|
|
5264
|
+
getTransactionToken;
|
|
5265
|
+
getMaterializationLifecycle;
|
|
5266
|
+
readExecution;
|
|
5267
|
+
hooks;
|
|
5268
|
+
sessions = new WeakMap;
|
|
5269
|
+
liveSessions = new Set;
|
|
5270
|
+
constructor(state, objects, timeseries, reader, getTransactionToken, getMaterializationLifecycle, readExecution, hooks = {}) {
|
|
5271
|
+
this.state = state;
|
|
5272
|
+
this.objects = objects;
|
|
5273
|
+
this.timeseries = timeseries;
|
|
5274
|
+
this.reader = reader;
|
|
5275
|
+
this.getTransactionToken = getTransactionToken;
|
|
5276
|
+
this.getMaterializationLifecycle = getMaterializationLifecycle;
|
|
5277
|
+
this.readExecution = readExecution;
|
|
5278
|
+
this.hooks = hooks;
|
|
5008
5279
|
}
|
|
5009
|
-
async
|
|
5280
|
+
async begin(input) {
|
|
5281
|
+
const transactionToken = this.getTransactionToken();
|
|
5282
|
+
const lifecycle = this.getMaterializationLifecycle();
|
|
5283
|
+
if (!transactionToken || !lifecycle) {
|
|
5284
|
+
throw new MaterializationValidationError2("Materialization sessions require an active storage transaction.");
|
|
5285
|
+
}
|
|
5286
|
+
assertMaterializationHeader2(input);
|
|
5287
|
+
if (!await this.readExecution(input.commit.projectId, input.commit.executionId)) {
|
|
5288
|
+
throw new MaterializationValidationError2(`Ontology commit execution '${input.commit.executionId}' does not exist in project '${input.commit.projectId}'.`);
|
|
5289
|
+
}
|
|
5290
|
+
this.assertCommitAbsent(input);
|
|
5291
|
+
for (const expected of input.expected.sources)
|
|
5292
|
+
this.assertSource(expected, input.commit.projectId);
|
|
5293
|
+
for (const expected of input.expected.objects)
|
|
5294
|
+
await this.assertObject(expected, input.commit.projectId);
|
|
5295
|
+
for (const expected of input.expected.links)
|
|
5296
|
+
await this.assertLink(expected, input.commit.projectId);
|
|
5297
|
+
const expectedScopeRevisions = new Map;
|
|
5298
|
+
for (const expected of input.expected.linkScopes) {
|
|
5299
|
+
const key = linkScopeSortKey2(expected.source, expected.linkId);
|
|
5300
|
+
const current = expectedScopeRevisions.get(key) ?? this.reader.effectiveLinkScope(input.commit.projectId, expected.source, expected.linkId);
|
|
5301
|
+
expectedScopeRevisions.set(key, current);
|
|
5302
|
+
assertExpectedLinkScopeRevision2(current.fingerprint, expected);
|
|
5303
|
+
}
|
|
5304
|
+
for (const expected of input.expected.points)
|
|
5305
|
+
this.assertPoint(expected, input.commit.projectId);
|
|
5306
|
+
const plan = input.plan ? this.requirePlan(input) : null;
|
|
5307
|
+
const providerToken = {};
|
|
5308
|
+
const session = {
|
|
5309
|
+
providerToken,
|
|
5310
|
+
header: structuredClone(input),
|
|
5311
|
+
transactionToken,
|
|
5312
|
+
lifecycle,
|
|
5313
|
+
active: true,
|
|
5314
|
+
writeOrdinal: 0,
|
|
5315
|
+
work: new Map,
|
|
5316
|
+
workUniqueKeys: new Set,
|
|
5317
|
+
applyWork: [],
|
|
5318
|
+
cardinalityWork: [],
|
|
5319
|
+
eventWork: [],
|
|
5320
|
+
outboxWrites: new Map,
|
|
5321
|
+
workSealed: false,
|
|
5322
|
+
vectorChangesStreamed: false,
|
|
5323
|
+
applied: false,
|
|
5324
|
+
appliedEventCount: 0,
|
|
5325
|
+
incidentLinksByObject: null,
|
|
5326
|
+
linkSlotStates: new Map,
|
|
5327
|
+
plan
|
|
5328
|
+
};
|
|
5329
|
+
if (plan) {
|
|
5330
|
+
for (const identity of plan.identities.values()) {
|
|
5331
|
+
for (const record of structuredClone(identity.records))
|
|
5332
|
+
this.addWork(session, record);
|
|
5333
|
+
}
|
|
5334
|
+
}
|
|
5335
|
+
this.sessions.set(providerToken, session);
|
|
5336
|
+
this.liveSessions.add(session);
|
|
5337
|
+
lifecycle.register(providerToken);
|
|
5338
|
+
return { providerToken };
|
|
5339
|
+
}
|
|
5340
|
+
deactivateTransaction(transactionToken) {
|
|
5341
|
+
for (const session of this.liveSessions) {
|
|
5342
|
+
if (session.transactionToken !== transactionToken)
|
|
5343
|
+
continue;
|
|
5344
|
+
this.releaseSession(session);
|
|
5345
|
+
}
|
|
5346
|
+
}
|
|
5347
|
+
releaseSession(session) {
|
|
5348
|
+
session.active = false;
|
|
5349
|
+
session.work.clear();
|
|
5350
|
+
session.workUniqueKeys.clear();
|
|
5351
|
+
session.applyWork.length = 0;
|
|
5352
|
+
session.cardinalityWork.length = 0;
|
|
5353
|
+
session.eventWork.length = 0;
|
|
5354
|
+
session.outboxWrites.clear();
|
|
5355
|
+
session.incidentLinksByObject = null;
|
|
5356
|
+
session.linkSlotStates = null;
|
|
5357
|
+
this.liveSessions.delete(session);
|
|
5358
|
+
session.lifecycle.complete(session.providerToken);
|
|
5359
|
+
}
|
|
5360
|
+
async* streamState(input) {
|
|
5010
5361
|
const session = this.requireSession(input.session);
|
|
5011
5362
|
assertPageRows2(input.pageRows);
|
|
5012
|
-
const
|
|
5013
|
-
if (stream.started) {
|
|
5014
|
-
throw new MaterializationConflictError2("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
|
|
5015
|
-
}
|
|
5016
|
-
stream.started = true;
|
|
5017
|
-
if (!session.workSealed) {
|
|
5018
|
-
session.applyWork.sort(comparePlanWork2);
|
|
5019
|
-
session.cardinalityWork.sort(compareCardinalityWork2);
|
|
5020
|
-
session.eventWork.sort(compareEventWork2);
|
|
5021
|
-
session.workSealed = true;
|
|
5022
|
-
}
|
|
5023
|
-
const records = input.order === "apply" ? session.applyWork : input.order === "cardinality" ? session.cardinalityWork : session.eventWork;
|
|
5024
|
-
for (let offset = 0;offset < records.length; offset += input.pageRows) {
|
|
5363
|
+
for await (const request of input.requests) {
|
|
5025
5364
|
this.requireSession(input.session);
|
|
5026
|
-
|
|
5027
|
-
|
|
5028
|
-
|
|
5029
|
-
|
|
5365
|
+
this.hooks.beforeRead?.("state.read");
|
|
5366
|
+
const objectRefs = uniqueSorted2(request.objects, objectRefKey2, objectRefSortKey2);
|
|
5367
|
+
for (let offset = 0;offset < objectRefs.length; offset += input.pageRows) {
|
|
5368
|
+
this.requireSession(input.session);
|
|
5369
|
+
const objects = [];
|
|
5370
|
+
for (const ref of objectRefs.slice(offset, offset + input.pageRows)) {
|
|
5371
|
+
objects.push(await this.reader.objectState(session.header.commit.projectId, ref));
|
|
5372
|
+
}
|
|
5373
|
+
this.requireSession(input.session);
|
|
5374
|
+
this.hooks.observeBuffer?.("state.object.page", objects.length);
|
|
5375
|
+
yield { objects, links: [], linkScopes: [], points: [] };
|
|
5376
|
+
}
|
|
5377
|
+
const linkRefs = uniqueSorted2(request.links, linkRefKey2, linkRefSortKey2);
|
|
5378
|
+
for (let offset = 0;offset < linkRefs.length; offset += input.pageRows) {
|
|
5379
|
+
this.requireSession(input.session);
|
|
5380
|
+
const links = [];
|
|
5381
|
+
for (const ref of linkRefs.slice(offset, offset + input.pageRows)) {
|
|
5382
|
+
links.push(await this.reader.linkState(session.header.commit.projectId, ref));
|
|
5383
|
+
}
|
|
5384
|
+
this.requireSession(input.session);
|
|
5385
|
+
this.hooks.observeBuffer?.("state.link.page", links.length);
|
|
5386
|
+
yield { objects: [], links, linkScopes: [], points: [] };
|
|
5387
|
+
}
|
|
5388
|
+
const incidentRefs = this.incidentLinkRefs(session, request.incidentObjects);
|
|
5389
|
+
for (let offset = 0;offset < incidentRefs.length; offset += input.pageRows) {
|
|
5390
|
+
this.requireSession(input.session);
|
|
5391
|
+
const refs = incidentRefs.slice(offset, offset + input.pageRows);
|
|
5392
|
+
const links = [];
|
|
5393
|
+
for (const ref of refs)
|
|
5394
|
+
links.push(await this.reader.linkState(session.header.commit.projectId, ref));
|
|
5395
|
+
this.requireSession(input.session);
|
|
5396
|
+
this.hooks.observeBuffer?.("state.incident-link.page", links.length);
|
|
5397
|
+
yield { objects: [], links, linkScopes: [], points: [] };
|
|
5398
|
+
}
|
|
5399
|
+
for (const scope of uniqueBy(request.linkScopes, (value) => linkScopeSortKey2(value.source, value.linkId))) {
|
|
5400
|
+
this.requireSession(input.session);
|
|
5401
|
+
const value = this.linkSlotState(session, scope.source, scope.linkId);
|
|
5402
|
+
yield { objects: [], links: [], linkScopes: [value], points: [] };
|
|
5403
|
+
}
|
|
5404
|
+
const requestedPoints = uniqueSorted2(request.points, (point) => telemetryPointKey2(point.series, point.at), (point) => telemetryPointSortKey2(point.series, point.at));
|
|
5405
|
+
for (let offset = 0;offset < requestedPoints.length; offset += input.pageRows) {
|
|
5406
|
+
this.requireSession(input.session);
|
|
5407
|
+
const points = [];
|
|
5408
|
+
for (const point of requestedPoints.slice(offset, offset + input.pageRows)) {
|
|
5409
|
+
const stored = getInMemoryTimeseriesMaterializerAdapter(this.timeseries).getExactPoint(session.header.commit.projectId, point.series, point.at);
|
|
5410
|
+
if (stored)
|
|
5411
|
+
points.push(storedPoint(stored));
|
|
5412
|
+
}
|
|
5413
|
+
this.requireSession(input.session);
|
|
5414
|
+
this.hooks.observeBuffer?.("state.point.page", points.length);
|
|
5415
|
+
if (points.length > 0)
|
|
5416
|
+
yield { objects: [], links: [], linkScopes: [], points };
|
|
5417
|
+
}
|
|
5030
5418
|
}
|
|
5031
|
-
this.requireSession(input.session);
|
|
5032
|
-
stream.completed = true;
|
|
5033
5419
|
}
|
|
5034
|
-
async
|
|
5420
|
+
async stageWork(input) {
|
|
5035
5421
|
const session = this.requireSession(input.session);
|
|
5036
|
-
const
|
|
5037
|
-
|
|
5038
|
-
|
|
5039
|
-
if (record) {
|
|
5040
|
-
result.push({ ref: structuredClone(record.ref), exists: record.exists });
|
|
5422
|
+
for (const { record, uniqueKey } of prepareMaterializationWork2(session, input)) {
|
|
5423
|
+
if (session.work.has(record.recordKey) || session.workUniqueKeys.has(uniqueKey)) {
|
|
5424
|
+
throw duplicateMaterializationWork2(record.recordKey);
|
|
5041
5425
|
}
|
|
5042
5426
|
}
|
|
5043
|
-
|
|
5427
|
+
const cloned = input.records.map((record) => structuredClone(record));
|
|
5428
|
+
this.hooks.observeWork?.(cloned);
|
|
5429
|
+
for (const record of cloned)
|
|
5430
|
+
this.addWork(session, record);
|
|
5431
|
+
this.hooks.observeBuffer?.("work.stage", cloned.length);
|
|
5044
5432
|
}
|
|
5045
|
-
|
|
5433
|
+
addWork(session, record) {
|
|
5434
|
+
session.work.set(record.recordKey, record);
|
|
5435
|
+
session.workUniqueKeys.add(workUniquenessKey2(record));
|
|
5436
|
+
if (record.kind === "plan")
|
|
5437
|
+
session.applyWork.push(record);
|
|
5438
|
+
if (record.kind === "cardinality")
|
|
5439
|
+
session.cardinalityWork.push(record);
|
|
5440
|
+
if (record.kind === "event")
|
|
5441
|
+
session.eventWork.push(record);
|
|
5442
|
+
}
|
|
5443
|
+
async* streamVectorChanges(input) {
|
|
5046
5444
|
const session = this.requireSession(input.session);
|
|
5445
|
+
assertPageRows2(input.pageRows);
|
|
5446
|
+
this.sealWork(session, beginMaterializationVectorChanges2);
|
|
5047
5447
|
const projectId = session.header.commit.projectId;
|
|
5048
|
-
|
|
5049
|
-
|
|
5050
|
-
|
|
5448
|
+
const profiled = new Set(input.objectTypeIds);
|
|
5449
|
+
const changes = session.applyWork.flatMap((record) => {
|
|
5450
|
+
const item = record.item;
|
|
5451
|
+
if (item.kind !== "object-upsert" && item.kind !== "object-delete")
|
|
5452
|
+
return [];
|
|
5453
|
+
const ref = item.kind === "object-upsert" ? item.value.row.ref : item.value.ref;
|
|
5454
|
+
return profiled.has(ref.objectTypeId) || (this.state.vectors.get(vectorObjectKey(projectId, ref))?.size ?? 0) > 0 ? [item] : [];
|
|
5455
|
+
});
|
|
5456
|
+
for (let offset = 0;offset < changes.length; offset += input.pageRows) {
|
|
5457
|
+
this.requireSession(input.session);
|
|
5458
|
+
yield { items: structuredClone(changes.slice(offset, offset + input.pageRows)) };
|
|
5459
|
+
}
|
|
5460
|
+
this.requireSession(input.session);
|
|
5461
|
+
}
|
|
5462
|
+
async apply(input) {
|
|
5463
|
+
const session = this.requireSession(input.session);
|
|
5464
|
+
this.sealWork(session, beginMaterializationApply2);
|
|
5465
|
+
const cardinality = createCardinalityValidator2();
|
|
5466
|
+
for (const record of session.cardinalityWork)
|
|
5467
|
+
cardinality.accept(record);
|
|
5468
|
+
for (let start = 0;start < session.applyWork.length; ) {
|
|
5469
|
+
const phase = session.applyWork[start].applyPhase;
|
|
5470
|
+
let end = start;
|
|
5471
|
+
while (session.applyWork[end]?.applyPhase === phase)
|
|
5472
|
+
end += 1;
|
|
5473
|
+
const items = session.applyWork.slice(start, end).map((record) => record.item);
|
|
5474
|
+
this.writeChunk(session, materializationPlanChunk2(items));
|
|
5475
|
+
start = end;
|
|
5476
|
+
}
|
|
5477
|
+
const outbox = session.eventWork.map((record, ordinal) => materializationOutboxWrite2(session.header.commit, record.draft, ordinal));
|
|
5478
|
+
this.writeChunk(session, materializationPlanChunk2([], outbox));
|
|
5479
|
+
session.appliedEventCount = outbox.length;
|
|
5480
|
+
return { eventCount: outbox.length };
|
|
5481
|
+
}
|
|
5482
|
+
writeChunk(session, chunk) {
|
|
5483
|
+
const projectId = session.header.commit.projectId;
|
|
5484
|
+
this.hooks.observeBuffer?.("apply.chunk", materializationChunkRows2(chunk));
|
|
5051
5485
|
const write = (boundary, apply) => {
|
|
5052
5486
|
this.hooks.beforeWrite?.(boundary, session.writeOrdinal++);
|
|
5053
5487
|
apply();
|
|
5054
5488
|
};
|
|
5055
|
-
for (const item of
|
|
5489
|
+
for (const item of chunk.overrides.objects.upserts)
|
|
5056
5490
|
write("override.object.upsert", () => {
|
|
5057
5491
|
const key = projectEntityKey(projectId, objectRefKey2(item.ref));
|
|
5058
5492
|
assertLastCommit(this.state.objectOverrides.get(key), item.expectedLastCommitId, "object override");
|
|
@@ -5065,13 +5499,13 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5065
5499
|
updatedAt: item.updatedAt
|
|
5066
5500
|
}));
|
|
5067
5501
|
});
|
|
5068
|
-
for (const item of
|
|
5502
|
+
for (const item of chunk.overrides.objects.deletes)
|
|
5069
5503
|
write("override.object.delete", () => {
|
|
5070
5504
|
const key = projectEntityKey(projectId, objectRefKey2(item.ref));
|
|
5071
5505
|
assertLastCommit(this.state.objectOverrides.get(key), item.expectedLastCommitId, "object override");
|
|
5072
5506
|
this.state.objectOverrides.delete(key);
|
|
5073
5507
|
});
|
|
5074
|
-
for (const item of
|
|
5508
|
+
for (const item of chunk.overrides.links.edges.upserts)
|
|
5075
5509
|
write("override.link.upsert", () => {
|
|
5076
5510
|
const key = projectEntityKey(projectId, linkRefKey2(item.ref));
|
|
5077
5511
|
assertLastCommit(this.state.linkOverrides.get(key), item.expectedLastCommitId, "link edge override");
|
|
@@ -5083,13 +5517,13 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5083
5517
|
updatedAt: item.updatedAt
|
|
5084
5518
|
}));
|
|
5085
5519
|
});
|
|
5086
|
-
for (const item of
|
|
5520
|
+
for (const item of chunk.overrides.links.edges.deletes)
|
|
5087
5521
|
write("override.link.delete", () => {
|
|
5088
5522
|
const key = projectEntityKey(projectId, linkRefKey2(item.ref));
|
|
5089
5523
|
assertLastCommit(this.state.linkOverrides.get(key), item.expectedLastCommitId, "link edge override");
|
|
5090
5524
|
this.state.linkOverrides.delete(key);
|
|
5091
5525
|
});
|
|
5092
|
-
for (const item of
|
|
5526
|
+
for (const item of chunk.overrides.links.slots.upserts)
|
|
5093
5527
|
write("override.link-slot.upsert", () => {
|
|
5094
5528
|
const key = projectEntityKey(projectId, linkScopeKey(item.ref.source, item.ref.linkId));
|
|
5095
5529
|
assertLastCommit(this.state.linkSlotOverrides.get(key), item.expectedLastCommitId, "link slot override");
|
|
@@ -5101,13 +5535,13 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5101
5535
|
updatedAt: item.updatedAt
|
|
5102
5536
|
}));
|
|
5103
5537
|
});
|
|
5104
|
-
for (const item of
|
|
5538
|
+
for (const item of chunk.overrides.links.slots.deletes)
|
|
5105
5539
|
write("override.link-slot.delete", () => {
|
|
5106
5540
|
const key = projectEntityKey(projectId, linkScopeKey(item.ref.source, item.ref.linkId));
|
|
5107
5541
|
assertLastCommit(this.state.linkSlotOverrides.get(key), item.expectedLastCommitId, "link slot override");
|
|
5108
5542
|
this.state.linkSlotOverrides.delete(key);
|
|
5109
5543
|
});
|
|
5110
|
-
for (const item of
|
|
5544
|
+
for (const item of chunk.effective.linkDeletes)
|
|
5111
5545
|
write("effective.link.delete", () => {
|
|
5112
5546
|
this.assertLinkSync(item.expected, projectId);
|
|
5113
5547
|
getInMemoryObjectMaterializerAdapter(this.objects).deleteExactLink({
|
|
@@ -5119,44 +5553,42 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5119
5553
|
targetId: item.ref.target.primaryId
|
|
5120
5554
|
});
|
|
5121
5555
|
});
|
|
5122
|
-
for (const item of
|
|
5556
|
+
for (const item of chunk.effective.objectDeletes)
|
|
5123
5557
|
write("effective.object.delete", () => {
|
|
5124
5558
|
this.assertObjectSync(item.expected, projectId);
|
|
5125
5559
|
getInMemoryObjectMaterializerAdapter(this.objects).deleteExactObject(projectId, item.ref.objectTypeId, item.ref.primaryId);
|
|
5126
5560
|
});
|
|
5127
|
-
for (const item of
|
|
5561
|
+
for (const item of chunk.effective.objectUpserts)
|
|
5128
5562
|
write("effective.object.upsert", () => {
|
|
5129
5563
|
this.assertObjectSync(item.expected, projectId);
|
|
5130
5564
|
getInMemoryObjectMaterializerAdapter(this.objects).applyExactObject(item.row, projectId);
|
|
5131
5565
|
});
|
|
5132
|
-
for (const item of
|
|
5566
|
+
for (const item of chunk.effective.linkUpserts)
|
|
5133
5567
|
write("effective.link.upsert", () => {
|
|
5134
5568
|
this.assertLinkSync(item.expected, projectId);
|
|
5135
5569
|
getInMemoryObjectMaterializerAdapter(this.objects).applyExactLink(item.row, projectId);
|
|
5136
5570
|
});
|
|
5137
|
-
for (const item of
|
|
5571
|
+
for (const item of chunk.timeseries.pointUpserts)
|
|
5138
5572
|
write("timeseries.point.upsert", () => {
|
|
5139
5573
|
this.assertPoint(item.expected, projectId);
|
|
5140
5574
|
getInMemoryTimeseriesMaterializerAdapter(this.timeseries).applyExactPoint(projectId, item.point);
|
|
5141
5575
|
});
|
|
5142
|
-
for (const item of
|
|
5576
|
+
for (const item of chunk.outbox)
|
|
5143
5577
|
write("outbox.insert", () => {
|
|
5144
|
-
const envelope = item.envelope;
|
|
5145
|
-
if (envelope.projectId !== projectId || envelope.commitId !== session.header.commit.id) {
|
|
5146
|
-
throw new MaterializationValidationError2("Outbox event does not correlate with its materialization commit.");
|
|
5147
|
-
}
|
|
5148
|
-
assertTimestamp(item.availableAt, "Outbox availableAt");
|
|
5149
5578
|
assertTimestamp(item.createdAt, "Outbox createdAt");
|
|
5150
|
-
|
|
5151
|
-
const key = outboxKey(projectId, envelope.id);
|
|
5579
|
+
const key = outboxKey(projectId, item.id);
|
|
5152
5580
|
if (this.state.outbox.has(key)) {
|
|
5153
|
-
throw new MaterializationConflictError2("effective-state", `Duplicate outbox event '${
|
|
5581
|
+
throw new MaterializationConflictError2("effective-state", `Duplicate outbox event '${item.id}'.`);
|
|
5154
5582
|
}
|
|
5155
|
-
if (session.
|
|
5583
|
+
if (session.outboxWrites.has(item.commitOrdinal))
|
|
5156
5584
|
throw new MaterializationConflictError2("effective-state", "Duplicate outbox commit ordinal.");
|
|
5157
5585
|
this.state.outbox.set(key, {
|
|
5158
|
-
|
|
5159
|
-
|
|
5586
|
+
projectId,
|
|
5587
|
+
id: item.id,
|
|
5588
|
+
commitId: session.header.commit.id,
|
|
5589
|
+
commitOrdinal: item.commitOrdinal,
|
|
5590
|
+
event: structuredClone(item.event),
|
|
5591
|
+
availableAt: item.createdAt,
|
|
5160
5592
|
attempts: 0,
|
|
5161
5593
|
leaseId: null,
|
|
5162
5594
|
leaseExpiresAt: null,
|
|
@@ -5164,11 +5596,8 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5164
5596
|
lastFailure: null,
|
|
5165
5597
|
createdAt: item.createdAt
|
|
5166
5598
|
});
|
|
5167
|
-
session.
|
|
5599
|
+
session.outboxWrites.set(item.commitOrdinal, structuredClone(item));
|
|
5168
5600
|
});
|
|
5169
|
-
for (const item of materializationPlanItems2(input.chunk)) {
|
|
5170
|
-
session.appliedPlanItems.push(stableJsonStringify2(item));
|
|
5171
|
-
}
|
|
5172
5601
|
}
|
|
5173
5602
|
async finalize(input) {
|
|
5174
5603
|
const session = this.requireSession(input.session);
|
|
@@ -5176,6 +5605,9 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5176
5605
|
this.assertCommitAbsent(session.header);
|
|
5177
5606
|
assertFinalizationCorrelations(session, input.finalization, this.state, this.objects);
|
|
5178
5607
|
this.hooks.beforeWrite?.("finalize", session.writeOrdinal++);
|
|
5608
|
+
const touches = emptyCommitTouches(commit.projectId);
|
|
5609
|
+
for (const record of session.applyWork)
|
|
5610
|
+
touchPlanItem(touches, record.item);
|
|
5179
5611
|
for (const activation of input.finalization.sourceActivations) {
|
|
5180
5612
|
this.hooks.beforeWrite?.("source.activate", session.writeOrdinal++);
|
|
5181
5613
|
assertTimestamp(activation.datasetVersion.createdAt, "Source dataset version createdAt");
|
|
@@ -5195,7 +5627,7 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5195
5627
|
updatedAt: activation.updatedAt
|
|
5196
5628
|
});
|
|
5197
5629
|
}
|
|
5198
|
-
activateSourceRoots(this.state, candidate, activation);
|
|
5630
|
+
activateSourceRoots(this.state, candidate, activation, touches);
|
|
5199
5631
|
this.state.sourceMaterializations.set(candidateKey, {
|
|
5200
5632
|
...candidate,
|
|
5201
5633
|
status: "active",
|
|
@@ -5205,11 +5637,19 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5205
5637
|
updatedAt: activation.updatedAt
|
|
5206
5638
|
});
|
|
5207
5639
|
}
|
|
5640
|
+
if (session.plan) {
|
|
5641
|
+
this.state.replacementPlans.delete(replacementPlanKey({
|
|
5642
|
+
projectId: commit.projectId,
|
|
5643
|
+
source: { projectionId: session.plan.sourceId },
|
|
5644
|
+
materializationId: session.plan.materializationId
|
|
5645
|
+
}));
|
|
5646
|
+
}
|
|
5208
5647
|
const record = {
|
|
5209
5648
|
...structuredClone(commit),
|
|
5210
5649
|
result: structuredClone(input.finalization.result)
|
|
5211
5650
|
};
|
|
5212
5651
|
this.state.commitsById.set(commitKey(commit.projectId, commit.id), record);
|
|
5652
|
+
this.state.commitTouches.set(this.state.commitsById.size, touches);
|
|
5213
5653
|
this.state.commitIdByIdempotency.set(idempotencyKey(commit.projectId, commit.idempotencyKey), commit.id);
|
|
5214
5654
|
const origin = ontologyCommitOriginSelector(commit.origin);
|
|
5215
5655
|
if (origin) {
|
|
@@ -5224,6 +5664,29 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5224
5664
|
throw new MaterializationValidationError2("Vector mutation does not belong to this materialization session.");
|
|
5225
5665
|
}
|
|
5226
5666
|
}
|
|
5667
|
+
sealWork(session, begin) {
|
|
5668
|
+
const sealed = session.workSealed;
|
|
5669
|
+
begin(session);
|
|
5670
|
+
if (sealed)
|
|
5671
|
+
return;
|
|
5672
|
+
session.applyWork.sort(comparePlanWork2);
|
|
5673
|
+
session.cardinalityWork.sort(compareCardinalityWork2);
|
|
5674
|
+
session.eventWork.sort(compareEventWork2);
|
|
5675
|
+
}
|
|
5676
|
+
requirePlan(header) {
|
|
5677
|
+
const { commit, plan: ref } = header;
|
|
5678
|
+
const plan = ref ? this.state.replacementPlans.get(replacementPlanKey({ projectId: commit.projectId, ...ref })) : undefined;
|
|
5679
|
+
if (!plan || commit.intent.kind !== "projection") {
|
|
5680
|
+
throw new MaterializationConflictError2("source-materialization", "A plan-bound session needs the open plan of a projection candidate.");
|
|
5681
|
+
}
|
|
5682
|
+
if (plan.commitId !== commit.id || plan.committedAt !== commit.committedAt) {
|
|
5683
|
+
invalidCorrelation2("A plan-bound session must begin with the commit its plan carries.");
|
|
5684
|
+
}
|
|
5685
|
+
if (plan.watermark !== this.state.commitsById.size || [...plan.identities.values()].some((identity) => identity.plannedRevision === null)) {
|
|
5686
|
+
invalidCorrelation2("A replacement plan applies only fully planned and just refreshed.");
|
|
5687
|
+
}
|
|
5688
|
+
return plan;
|
|
5689
|
+
}
|
|
5227
5690
|
requireSession(session) {
|
|
5228
5691
|
const value = this.sessions.get(session.providerToken);
|
|
5229
5692
|
if (!value || !value.active || !this.getTransactionToken() || this.getTransactionToken() !== value.transactionToken) {
|
|
@@ -5231,115 +5694,6 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5231
5694
|
}
|
|
5232
5695
|
return value;
|
|
5233
5696
|
}
|
|
5234
|
-
requireReplacement(session, input) {
|
|
5235
|
-
if (session.replacement) {
|
|
5236
|
-
if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
|
|
5237
|
-
throw new MaterializationConflictError2("source-materialization", "Materialization session already owns another replacement union.");
|
|
5238
|
-
}
|
|
5239
|
-
return session.replacement;
|
|
5240
|
-
}
|
|
5241
|
-
const projectId = session.header.commit.projectId;
|
|
5242
|
-
const sourceId = input.source.projectionId;
|
|
5243
|
-
const candidate = this.state.sourceMaterializations.get(sourceMaterializationKey(projectId, sourceId, input.candidateMaterializationId));
|
|
5244
|
-
if (!candidate || candidate.status !== "ready") {
|
|
5245
|
-
throw new MaterializationConflictError2("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
|
|
5246
|
-
}
|
|
5247
|
-
const previous = findActiveSourceMaterialization(this.state, projectId, sourceId);
|
|
5248
|
-
if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projectionKind !== candidate.projectionKind)) {
|
|
5249
|
-
throw new MaterializationConflictError2("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
|
|
5250
|
-
}
|
|
5251
|
-
const replacement = {
|
|
5252
|
-
sourceId,
|
|
5253
|
-
candidateMaterializationId: input.candidateMaterializationId,
|
|
5254
|
-
owned: new Set,
|
|
5255
|
-
candidate,
|
|
5256
|
-
objects: new Map,
|
|
5257
|
-
links: new Map,
|
|
5258
|
-
orderedObjects: [],
|
|
5259
|
-
orderedLinks: null,
|
|
5260
|
-
affectedScopes: new Set,
|
|
5261
|
-
incidentObjects: new Map,
|
|
5262
|
-
objectStreamStarted: false,
|
|
5263
|
-
objectStreamCompleted: false,
|
|
5264
|
-
linkStreamStarted: false,
|
|
5265
|
-
linkStreamCompleted: false,
|
|
5266
|
-
linksExpanded: false
|
|
5267
|
-
};
|
|
5268
|
-
for (const rows of [
|
|
5269
|
-
previousSourceRows(this.state, candidate),
|
|
5270
|
-
candidate.rowsByEntity.values()
|
|
5271
|
-
]) {
|
|
5272
|
-
for (const row of rows) {
|
|
5273
|
-
replacement.owned.add(projectionEntityKey2(row.assertion));
|
|
5274
|
-
if (row.assertion.kind === "object") {
|
|
5275
|
-
const ref = row.assertion.ref;
|
|
5276
|
-
replacement.objects.set(objectRefKey2(ref), { ref, sortKey: objectRefSortKey2(ref) });
|
|
5277
|
-
} else {
|
|
5278
|
-
addReplacementLink(replacement, row.assertion.ref, true);
|
|
5279
|
-
}
|
|
5280
|
-
}
|
|
5281
|
-
}
|
|
5282
|
-
replacement.orderedObjects = [...replacement.objects.values()].sort((left, right) => left.sortKey.localeCompare(right.sortKey));
|
|
5283
|
-
session.replacement = replacement;
|
|
5284
|
-
return replacement;
|
|
5285
|
-
}
|
|
5286
|
-
expandReplacementLinks(session, replacement) {
|
|
5287
|
-
if (replacement.linksExpanded)
|
|
5288
|
-
return;
|
|
5289
|
-
const projectId = session.header.commit.projectId;
|
|
5290
|
-
const incident = replacement.incidentObjects;
|
|
5291
|
-
const touchesIncident = (ref) => incident.has(objectRefKey2(ref.source)) || incident.has(objectRefKey2(ref.target));
|
|
5292
|
-
const adapter = getInMemoryObjectMaterializerAdapter(this.objects);
|
|
5293
|
-
if (incident.size > 0) {
|
|
5294
|
-
adapter.visitExactLinks(projectId, (row) => {
|
|
5295
|
-
const ref = linkRef(row);
|
|
5296
|
-
if (touchesIncident(ref))
|
|
5297
|
-
addReplacementLink(replacement, ref, true);
|
|
5298
|
-
});
|
|
5299
|
-
for (const override of this.state.linkOverrides.values()) {
|
|
5300
|
-
if (override.projectId !== projectId)
|
|
5301
|
-
continue;
|
|
5302
|
-
if (touchesIncident(override.ref))
|
|
5303
|
-
addReplacementLink(replacement, override.ref, true);
|
|
5304
|
-
}
|
|
5305
|
-
for (const override of this.state.linkSlotOverrides.values()) {
|
|
5306
|
-
if (override.projectId !== projectId)
|
|
5307
|
-
continue;
|
|
5308
|
-
const ref = {
|
|
5309
|
-
source: override.ref.source,
|
|
5310
|
-
linkId: override.ref.linkId,
|
|
5311
|
-
target: override.value.target
|
|
5312
|
-
};
|
|
5313
|
-
if (touchesIncident(ref))
|
|
5314
|
-
addReplacementLink(replacement, ref, true);
|
|
5315
|
-
}
|
|
5316
|
-
for (const [scopeKey, rows] of this.state.activeSourceLinkScopes) {
|
|
5317
|
-
if (JSON.parse(scopeKey)[0] !== projectId)
|
|
5318
|
-
continue;
|
|
5319
|
-
for (const row of rows.values()) {
|
|
5320
|
-
if (row.assertion.kind === "link" && touchesIncident(row.assertion.ref)) {
|
|
5321
|
-
addReplacementLink(replacement, row.assertion.ref, true);
|
|
5322
|
-
}
|
|
5323
|
-
}
|
|
5324
|
-
}
|
|
5325
|
-
}
|
|
5326
|
-
const scopes = new Map;
|
|
5327
|
-
for (const { ref } of replacement.links.values()) {
|
|
5328
|
-
const key = linkScopeSortKey2(ref.source, ref.linkId);
|
|
5329
|
-
if (replacement.affectedScopes.has(key))
|
|
5330
|
-
scopes.set(key, ref);
|
|
5331
|
-
}
|
|
5332
|
-
for (const scope of scopes.values()) {
|
|
5333
|
-
adapter.visitExactScopeLinks(projectId, scope.source.objectTypeId, scope.source.primaryId, scope.linkId, (row) => {
|
|
5334
|
-
addReplacementLink(replacement, linkRef(row), false);
|
|
5335
|
-
});
|
|
5336
|
-
const override = this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(scope.source, scope.linkId)));
|
|
5337
|
-
if (override)
|
|
5338
|
-
addReplacementLink(replacement, { ...scope, target: override.value.target }, false);
|
|
5339
|
-
}
|
|
5340
|
-
replacement.linksExpanded = true;
|
|
5341
|
-
replacement.orderedLinks = null;
|
|
5342
|
-
}
|
|
5343
5697
|
assertCommitAbsent(header) {
|
|
5344
5698
|
if (this.state.commitsById.has(commitKey(header.commit.projectId, header.commit.id))) {
|
|
5345
5699
|
throw new MaterializationConflictError2("idempotency", `Ontology commit '${header.commit.id}' already exists.`);
|
|
@@ -5384,98 +5738,20 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5384
5738
|
throw new MaterializationConflictError2("timeseries-point", `Telemetry point ${telemetryPointKey2(expected.series, expected.at)} changed.`);
|
|
5385
5739
|
}
|
|
5386
5740
|
}
|
|
5387
|
-
async objectState(session, ref) {
|
|
5388
|
-
const projectId = session.header.commit.projectId;
|
|
5389
|
-
const row = await this.objects.getByPrimaryId({
|
|
5390
|
-
projectId,
|
|
5391
|
-
objectTypeId: ref.objectTypeId,
|
|
5392
|
-
primaryId: ref.primaryId
|
|
5393
|
-
});
|
|
5394
|
-
return {
|
|
5395
|
-
ref: structuredClone(ref),
|
|
5396
|
-
source: this.findActiveObjectSource(projectId, ref),
|
|
5397
|
-
override: structuredClone(publicObjectOverride(this.state.objectOverrides.get(projectEntityKey(projectId, objectRefKey2(ref))))),
|
|
5398
|
-
effective: row ? objectSnapshot(row) : null,
|
|
5399
|
-
latestTelemetry: getInMemoryTimeseriesMaterializerAdapter(this.timeseries).listLatestForObject(projectId, ref.objectTypeId, ref.primaryId).map(storedPoint)
|
|
5400
|
-
};
|
|
5401
|
-
}
|
|
5402
|
-
async linkState(session, ref) {
|
|
5403
|
-
const projectId = session.header.commit.projectId;
|
|
5404
|
-
const row = getInMemoryObjectMaterializerAdapter(this.objects).getExactLinkRow(projectId, {
|
|
5405
|
-
sourceTypeId: ref.source.objectTypeId,
|
|
5406
|
-
sourceId: ref.source.primaryId,
|
|
5407
|
-
linkId: ref.linkId,
|
|
5408
|
-
targetTypeId: ref.target.objectTypeId,
|
|
5409
|
-
targetId: ref.target.primaryId
|
|
5410
|
-
});
|
|
5411
|
-
return {
|
|
5412
|
-
ref: structuredClone(ref),
|
|
5413
|
-
source: this.findActiveLinkSource(projectId, ref),
|
|
5414
|
-
override: structuredClone(publicLinkOverride(this.state.linkOverrides.get(projectEntityKey(projectId, linkRefKey2(ref))))),
|
|
5415
|
-
slotOverride: structuredClone(publicLinkSlotOverride(this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(ref.source, ref.linkId))))),
|
|
5416
|
-
effective: row ? linkSnapshot(row) : null
|
|
5417
|
-
};
|
|
5418
|
-
}
|
|
5419
5741
|
linkSlotState(session, source, linkId) {
|
|
5420
5742
|
session.linkSlotStates ??= new Map;
|
|
5421
5743
|
const key = linkScopeSortKey2(source, linkId);
|
|
5422
5744
|
const existing = session.linkSlotStates.get(key);
|
|
5423
5745
|
if (existing)
|
|
5424
5746
|
return structuredClone(existing);
|
|
5425
|
-
const computed = this.
|
|
5426
|
-
session.linkSlotStates.set(key, computed);
|
|
5427
|
-
return structuredClone(computed);
|
|
5428
|
-
}
|
|
5429
|
-
computeLinkSlotState(projectId, source, linkId) {
|
|
5430
|
-
const value = this.computeEffectiveLinkScope(projectId, source, linkId);
|
|
5431
|
-
return {
|
|
5432
|
-
...value,
|
|
5433
|
-
sourceAssertion: this.findActiveLinkScopeSource(projectId, source, linkId),
|
|
5434
|
-
override: structuredClone(publicLinkSlotOverride(this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(source, linkId)))))
|
|
5435
|
-
};
|
|
5436
|
-
}
|
|
5437
|
-
computeEffectiveLinkScope(projectId, source, linkId) {
|
|
5438
|
-
const rows = [];
|
|
5439
|
-
getInMemoryObjectMaterializerAdapter(this.objects).visitExactScopeLinks(projectId, source.objectTypeId, source.primaryId, linkId, (row) => rows.push(row));
|
|
5440
|
-
rows.sort((left, right) => linkRefSortKey2(linkRef(left)).localeCompare(linkRefSortKey2(linkRef(right))));
|
|
5441
|
-
const accumulator = startScopeAccumulator2(source, linkId);
|
|
5442
|
-
for (let offset = 0;offset < rows.length; offset += 1000) {
|
|
5443
|
-
const page = rows.slice(offset, offset + 1000);
|
|
5444
|
-
this.hooks.observeBuffer?.("state.link-scope.page", page.length);
|
|
5445
|
-
for (const row of page) {
|
|
5446
|
-
appendScopeSnapshot2(accumulator, linkSnapshot(row));
|
|
5447
|
-
}
|
|
5448
|
-
}
|
|
5449
|
-
const effective = rows.length === 1 ? linkSnapshot(rows[0]) : null;
|
|
5450
|
-
return {
|
|
5451
|
-
...finishScopeAccumulator2(accumulator),
|
|
5452
|
-
effective
|
|
5453
|
-
};
|
|
5454
|
-
}
|
|
5455
|
-
findActiveLinkScopeSource(projectId, source, linkId) {
|
|
5456
|
-
const rows = this.state.activeSourceLinkScopes.get(projectEntityKey(projectId, linkScopeSortKey2(source, linkId)));
|
|
5457
|
-
if (rows && rows.size > 1)
|
|
5458
|
-
throw new MaterializationConflictError2("source-materialization", `Multiple active source links assert cardinality-one scope '${source.objectTypeId}.${linkId}'.`);
|
|
5459
|
-
return structuredClone(rows?.values().next().value ?? null);
|
|
5460
|
-
}
|
|
5461
|
-
findActiveObjectSource(projectId, ref) {
|
|
5462
|
-
const found = this.findActiveSource(projectId, projectionEntityKey2({ kind: "object", ref }));
|
|
5463
|
-
return found?.assertion.kind === "object" ? found : null;
|
|
5464
|
-
}
|
|
5465
|
-
findActiveLinkSource(projectId, ref) {
|
|
5466
|
-
const found = this.findActiveSource(projectId, projectionEntityKey2({ kind: "link", ref }));
|
|
5467
|
-
return found?.assertion.kind === "link" ? found : null;
|
|
5468
|
-
}
|
|
5469
|
-
findActiveSource(projectId, entityKey) {
|
|
5470
|
-
const rows = this.state.activeSourceRows.get(projectEntityKey(projectId, entityKey));
|
|
5471
|
-
if (rows && rows.size > 1)
|
|
5472
|
-
throw new MaterializationConflictError2("source-materialization", `Multiple active sources assert ${entityKey}.`);
|
|
5473
|
-
return structuredClone(rows?.values().next().value ?? null);
|
|
5747
|
+
const computed = this.reader.linkSlotState(session.header.commit.projectId, source, linkId);
|
|
5748
|
+
session.linkSlotStates.set(key, computed);
|
|
5749
|
+
return structuredClone(computed);
|
|
5474
5750
|
}
|
|
5475
5751
|
incidentLinkRefs(session, objects) {
|
|
5476
5752
|
if (objects.length === 0)
|
|
5477
5753
|
return [];
|
|
5478
|
-
session.incidentLinksByObject ??= this.
|
|
5754
|
+
session.incidentLinksByObject ??= this.reader.incidentLinkIndex(session.header.commit.projectId);
|
|
5479
5755
|
const selected = new Map;
|
|
5480
5756
|
for (const object of objects) {
|
|
5481
5757
|
for (const ref of session.incidentLinksByObject.get(objectRefKey2(object)) ?? []) {
|
|
@@ -5484,59 +5760,22 @@ class InMemoryOntologyMaterializationStorage {
|
|
|
5484
5760
|
}
|
|
5485
5761
|
return [...selected.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([, ref]) => ref);
|
|
5486
5762
|
}
|
|
5487
|
-
buildIncidentLinkIndex(projectId) {
|
|
5488
|
-
const mutable = new Map;
|
|
5489
|
-
const consider = (ref) => {
|
|
5490
|
-
const sortKey = linkRefSortKey2(ref);
|
|
5491
|
-
for (const endpoint of [ref.source, ref.target]) {
|
|
5492
|
-
const key = objectRefKey2(endpoint);
|
|
5493
|
-
const links = mutable.get(key) ?? new Map;
|
|
5494
|
-
links.set(sortKey, structuredClone(ref));
|
|
5495
|
-
mutable.set(key, links);
|
|
5496
|
-
}
|
|
5497
|
-
};
|
|
5498
|
-
getInMemoryObjectMaterializerAdapter(this.objects).visitExactLinks(projectId, (row) => consider(linkRef(row)));
|
|
5499
|
-
for (const override of this.state.linkOverrides.values()) {
|
|
5500
|
-
if (override.projectId === projectId)
|
|
5501
|
-
consider(override.ref);
|
|
5502
|
-
}
|
|
5503
|
-
for (const override of this.state.linkSlotOverrides.values()) {
|
|
5504
|
-
if (override.projectId !== projectId)
|
|
5505
|
-
continue;
|
|
5506
|
-
consider({
|
|
5507
|
-
source: override.ref.source,
|
|
5508
|
-
linkId: override.ref.linkId,
|
|
5509
|
-
target: override.value.target
|
|
5510
|
-
});
|
|
5511
|
-
}
|
|
5512
|
-
for (const [scopeKey, rows] of this.state.activeSourceLinkScopes) {
|
|
5513
|
-
if (JSON.parse(scopeKey)[0] !== projectId)
|
|
5514
|
-
continue;
|
|
5515
|
-
for (const row of rows.values()) {
|
|
5516
|
-
if (row.assertion.kind === "link")
|
|
5517
|
-
consider(row.assertion.ref);
|
|
5518
|
-
}
|
|
5519
|
-
}
|
|
5520
|
-
const index = new Map;
|
|
5521
|
-
for (const [objectKey, refs] of mutable) {
|
|
5522
|
-
index.set(objectKey, [...refs.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([, ref]) => ref));
|
|
5523
|
-
}
|
|
5524
|
-
return index;
|
|
5525
|
-
}
|
|
5526
5763
|
}
|
|
5527
5764
|
|
|
5528
5765
|
// src/storage/ontology/in-memory/outbox.ts
|
|
5529
5766
|
class InMemoryOntologyOutboxStorage {
|
|
5530
5767
|
state;
|
|
5531
5768
|
runRootOperation;
|
|
5532
|
-
|
|
5769
|
+
readExecution;
|
|
5770
|
+
constructor(state, runRootOperation, readExecution) {
|
|
5533
5771
|
this.state = state;
|
|
5534
5772
|
this.runRootOperation = runRootOperation;
|
|
5773
|
+
this.readExecution = readExecution;
|
|
5535
5774
|
}
|
|
5536
5775
|
async claim(input) {
|
|
5537
5776
|
return this.runRootOperation(() => this.claimUnlocked(input));
|
|
5538
5777
|
}
|
|
5539
|
-
claimUnlocked(input) {
|
|
5778
|
+
async claimUnlocked(input) {
|
|
5540
5779
|
assertNonblank(input.projectId, "Ontology outbox project id");
|
|
5541
5780
|
assertPositiveLimit(input.limit);
|
|
5542
5781
|
const now = assertTimestamp(input.now, "Ontology outbox claim time");
|
|
@@ -5547,7 +5786,7 @@ class InMemoryOntologyOutboxStorage {
|
|
|
5547
5786
|
}
|
|
5548
5787
|
const rows = [];
|
|
5549
5788
|
for (const row of this.state.outbox.values()) {
|
|
5550
|
-
if (row.
|
|
5789
|
+
if (row.projectId !== input.projectId)
|
|
5551
5790
|
continue;
|
|
5552
5791
|
assertPairedLease(row);
|
|
5553
5792
|
if (row.publishedAt !== null || Date.parse(row.availableAt) > now)
|
|
@@ -5556,16 +5795,23 @@ class InMemoryOntologyOutboxStorage {
|
|
|
5556
5795
|
continue;
|
|
5557
5796
|
insertBounded(rows, row, input.limit, compareClaimRows);
|
|
5558
5797
|
}
|
|
5559
|
-
|
|
5560
|
-
|
|
5561
|
-
|
|
5562
|
-
|
|
5563
|
-
|
|
5564
|
-
|
|
5565
|
-
|
|
5566
|
-
|
|
5567
|
-
|
|
5568
|
-
|
|
5798
|
+
const claimed = rows.map((row) => ({
|
|
5799
|
+
...row,
|
|
5800
|
+
attempts: row.attempts + 1,
|
|
5801
|
+
leaseId: input.leaseId,
|
|
5802
|
+
leaseExpiresAt: input.leaseExpiresAt
|
|
5803
|
+
}));
|
|
5804
|
+
const contexts = new Map;
|
|
5805
|
+
for (const commitId of new Set(claimed.map((row) => row.commitId))) {
|
|
5806
|
+
const commit = this.state.commitsById.get(commitKey(input.projectId, commitId));
|
|
5807
|
+
const execution = commit && await this.readExecution(input.projectId, commit.executionId);
|
|
5808
|
+
if (commit && execution)
|
|
5809
|
+
contexts.set(commitId, { commit, execution });
|
|
5810
|
+
}
|
|
5811
|
+
const records = claimedOutboxRows2(structuredClone(claimed), (commitId) => contexts.get(commitId));
|
|
5812
|
+
for (const row of claimed)
|
|
5813
|
+
this.state.outbox.set(outboxKey(input.projectId, row.id), row);
|
|
5814
|
+
return records;
|
|
5569
5815
|
}
|
|
5570
5816
|
async markPublished(input) {
|
|
5571
5817
|
await this.runRootOperation(() => this.markPublishedUnlocked(input));
|
|
@@ -5613,7 +5859,7 @@ class InMemoryOntologyOutboxStorage {
|
|
|
5613
5859
|
let retryingCount = 0;
|
|
5614
5860
|
let maxAttempts = 0;
|
|
5615
5861
|
for (const row of this.state.outbox.values()) {
|
|
5616
|
-
if (row.
|
|
5862
|
+
if (row.projectId !== input.projectId || row.publishedAt !== null)
|
|
5617
5863
|
continue;
|
|
5618
5864
|
pendingCount += 1;
|
|
5619
5865
|
if (row.attempts > 0)
|
|
@@ -5633,7 +5879,7 @@ class InMemoryOntologyOutboxStorage {
|
|
|
5633
5879
|
const rows = [];
|
|
5634
5880
|
for (const entry of this.state.outbox.entries()) {
|
|
5635
5881
|
const [, row] = entry;
|
|
5636
|
-
if (row.
|
|
5882
|
+
if (row.projectId !== input.projectId || row.publishedAt === null || Date.parse(row.publishedAt) >= cutoff) {
|
|
5637
5883
|
continue;
|
|
5638
5884
|
}
|
|
5639
5885
|
insertBounded(rows, entry, input.limit, ([, left], [, right]) => comparePublishedRows(left, right));
|
|
@@ -5663,10 +5909,10 @@ class InMemoryOntologyOutboxStorage {
|
|
|
5663
5909
|
}
|
|
5664
5910
|
}
|
|
5665
5911
|
function compareClaimRows(left, right) {
|
|
5666
|
-
return left.createdAt.localeCompare(right.createdAt) || left.
|
|
5912
|
+
return left.createdAt.localeCompare(right.createdAt) || left.commitId.localeCompare(right.commitId) || left.commitOrdinal - right.commitOrdinal;
|
|
5667
5913
|
}
|
|
5668
5914
|
function comparePublishedRows(left, right) {
|
|
5669
|
-
return (left.publishedAt ?? "").localeCompare(right.publishedAt ?? "") || left.
|
|
5915
|
+
return (left.publishedAt ?? "").localeCompare(right.publishedAt ?? "") || left.id.localeCompare(right.id);
|
|
5670
5916
|
}
|
|
5671
5917
|
function assertPositiveLimit(limit) {
|
|
5672
5918
|
if (!Number.isSafeInteger(limit) || limit <= 0) {
|
|
@@ -5887,23 +6133,6 @@ class InMemoryOntologySourceStorage {
|
|
|
5887
6133
|
}
|
|
5888
6134
|
return { rowsDeleted, materializationsDeleted };
|
|
5889
6135
|
}
|
|
5890
|
-
async summarizeTerminal(input) {
|
|
5891
|
-
return this.runRootOperation(() => {
|
|
5892
|
-
assertNonblank(input.projectId, "Terminal source summary project id");
|
|
5893
|
-
let count = 0;
|
|
5894
|
-
let oldestTerminalAt = null;
|
|
5895
|
-
for (const materialization of this.state.sourceMaterializations.values()) {
|
|
5896
|
-
if (materialization.projectId !== input.projectId || materialization.status !== "superseded" && materialization.status !== "abandoned" || materialization.terminalAt === null) {
|
|
5897
|
-
continue;
|
|
5898
|
-
}
|
|
5899
|
-
count += 1;
|
|
5900
|
-
if (oldestTerminalAt === null || materialization.terminalAt < oldestTerminalAt) {
|
|
5901
|
-
oldestTerminalAt = materialization.terminalAt;
|
|
5902
|
-
}
|
|
5903
|
-
}
|
|
5904
|
-
return { count, oldestTerminalAt };
|
|
5905
|
-
});
|
|
5906
|
-
}
|
|
5907
6136
|
async assertCurrentExecution(input) {
|
|
5908
6137
|
await this.assertExecution({
|
|
5909
6138
|
projectId: input.projectId,
|
|
@@ -5954,7 +6183,9 @@ class InMemoryOntologySourceStorage {
|
|
|
5954
6183
|
terminalAt: abandonedAt,
|
|
5955
6184
|
updatedAt: abandonedAt
|
|
5956
6185
|
};
|
|
5957
|
-
|
|
6186
|
+
const key = sourceMaterializationKey(materialization.projectId, materialization.source.projectionId, materialization.materializationId);
|
|
6187
|
+
this.state.sourceMaterializations.set(key, abandoned);
|
|
6188
|
+
this.state.replacementPlans.delete(key);
|
|
5958
6189
|
return sourceMaterializationRecord(abandoned);
|
|
5959
6190
|
}
|
|
5960
6191
|
findRunCandidates(projectId, sourceId, projectionRunId) {
|
|
@@ -6101,6 +6332,138 @@ function compareTerminalMaterializations([, left], [, right]) {
|
|
|
6101
6332
|
return (left.terminalAt ?? "").localeCompare(right.terminalAt ?? "") || left.source.projectionId.localeCompare(right.source.projectionId) || left.materializationId.localeCompare(right.materializationId);
|
|
6102
6333
|
}
|
|
6103
6334
|
|
|
6335
|
+
// src/storage/ontology/in-memory/state-reader.ts
|
|
6336
|
+
class InMemoryMaterializationStateReader {
|
|
6337
|
+
state;
|
|
6338
|
+
objects;
|
|
6339
|
+
timeseries;
|
|
6340
|
+
hooks;
|
|
6341
|
+
constructor(state, objects, timeseries, hooks = {}) {
|
|
6342
|
+
this.state = state;
|
|
6343
|
+
this.objects = objects;
|
|
6344
|
+
this.timeseries = timeseries;
|
|
6345
|
+
this.hooks = hooks;
|
|
6346
|
+
}
|
|
6347
|
+
async objectState(projectId, ref) {
|
|
6348
|
+
const row = await this.objects.getByPrimaryId({
|
|
6349
|
+
projectId,
|
|
6350
|
+
objectTypeId: ref.objectTypeId,
|
|
6351
|
+
primaryId: ref.primaryId
|
|
6352
|
+
});
|
|
6353
|
+
return {
|
|
6354
|
+
ref: structuredClone(ref),
|
|
6355
|
+
source: this.findActiveObjectSource(projectId, ref),
|
|
6356
|
+
override: structuredClone(publicObjectOverride(this.state.objectOverrides.get(projectEntityKey(projectId, objectRefKey2(ref))))),
|
|
6357
|
+
effective: row ? objectSnapshot(row) : null,
|
|
6358
|
+
latestTelemetry: getInMemoryTimeseriesMaterializerAdapter(this.timeseries).listLatestForObject(projectId, ref.objectTypeId, ref.primaryId).map(storedPoint)
|
|
6359
|
+
};
|
|
6360
|
+
}
|
|
6361
|
+
latestTelemetry(projectId, ref) {
|
|
6362
|
+
return getInMemoryTimeseriesMaterializerAdapter(this.timeseries).listLatestForObject(projectId, ref.objectTypeId, ref.primaryId).map(storedPoint);
|
|
6363
|
+
}
|
|
6364
|
+
async linkState(projectId, ref) {
|
|
6365
|
+
const row = getInMemoryObjectMaterializerAdapter(this.objects).getExactLinkRow(projectId, {
|
|
6366
|
+
sourceTypeId: ref.source.objectTypeId,
|
|
6367
|
+
sourceId: ref.source.primaryId,
|
|
6368
|
+
linkId: ref.linkId,
|
|
6369
|
+
targetTypeId: ref.target.objectTypeId,
|
|
6370
|
+
targetId: ref.target.primaryId
|
|
6371
|
+
});
|
|
6372
|
+
return {
|
|
6373
|
+
ref: structuredClone(ref),
|
|
6374
|
+
source: this.findActiveLinkSource(projectId, ref),
|
|
6375
|
+
override: structuredClone(publicLinkOverride(this.state.linkOverrides.get(projectEntityKey(projectId, linkRefKey2(ref))))),
|
|
6376
|
+
slotOverride: structuredClone(publicLinkSlotOverride(this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(ref.source, ref.linkId))))),
|
|
6377
|
+
effective: row ? linkSnapshot(row) : null
|
|
6378
|
+
};
|
|
6379
|
+
}
|
|
6380
|
+
linkSlotState(projectId, source, linkId) {
|
|
6381
|
+
const value = this.effectiveLinkScope(projectId, source, linkId);
|
|
6382
|
+
return {
|
|
6383
|
+
...value,
|
|
6384
|
+
sourceAssertion: this.findActiveLinkScopeSource(projectId, source, linkId),
|
|
6385
|
+
override: structuredClone(publicLinkSlotOverride(this.state.linkSlotOverrides.get(projectEntityKey(projectId, linkScopeKey(source, linkId)))))
|
|
6386
|
+
};
|
|
6387
|
+
}
|
|
6388
|
+
effectiveLinkScope(projectId, source, linkId) {
|
|
6389
|
+
const rows = [];
|
|
6390
|
+
getInMemoryObjectMaterializerAdapter(this.objects).visitExactScopeLinks(projectId, source.objectTypeId, source.primaryId, linkId, (row) => rows.push(row));
|
|
6391
|
+
rows.sort((left, right) => linkRefSortKey2(linkRef(left)).localeCompare(linkRefSortKey2(linkRef(right))));
|
|
6392
|
+
const accumulator = startScopeAccumulator2(source, linkId);
|
|
6393
|
+
for (let offset = 0;offset < rows.length; offset += 1000) {
|
|
6394
|
+
const page = rows.slice(offset, offset + 1000);
|
|
6395
|
+
this.hooks.observeBuffer?.("state.link-scope.page", page.length);
|
|
6396
|
+
for (const row of page) {
|
|
6397
|
+
appendScopeSnapshot2(accumulator, linkSnapshot(row));
|
|
6398
|
+
}
|
|
6399
|
+
}
|
|
6400
|
+
const effective = rows.length === 1 ? linkSnapshot(rows[0]) : null;
|
|
6401
|
+
return {
|
|
6402
|
+
...finishScopeAccumulator2(accumulator),
|
|
6403
|
+
effective
|
|
6404
|
+
};
|
|
6405
|
+
}
|
|
6406
|
+
findActiveLinkScopeSource(projectId, source, linkId) {
|
|
6407
|
+
const rows = this.state.activeSourceLinkScopes.get(projectEntityKey(projectId, linkScopeSortKey2(source, linkId)));
|
|
6408
|
+
if (rows && rows.size > 1)
|
|
6409
|
+
throw new MaterializationConflictError2("source-materialization", `Multiple active source links assert cardinality-one scope '${source.objectTypeId}.${linkId}'.`);
|
|
6410
|
+
return structuredClone(rows?.values().next().value ?? null);
|
|
6411
|
+
}
|
|
6412
|
+
findActiveObjectSource(projectId, ref) {
|
|
6413
|
+
const found = this.findActiveSource(projectId, projectionEntityKey2({ kind: "object", ref }));
|
|
6414
|
+
return found?.assertion.kind === "object" ? found : null;
|
|
6415
|
+
}
|
|
6416
|
+
findActiveLinkSource(projectId, ref) {
|
|
6417
|
+
const found = this.findActiveSource(projectId, projectionEntityKey2({ kind: "link", ref }));
|
|
6418
|
+
return found?.assertion.kind === "link" ? found : null;
|
|
6419
|
+
}
|
|
6420
|
+
findActiveSource(projectId, entityKey) {
|
|
6421
|
+
const rows = this.state.activeSourceRows.get(projectEntityKey(projectId, entityKey));
|
|
6422
|
+
if (rows && rows.size > 1)
|
|
6423
|
+
throw new MaterializationConflictError2("source-materialization", `Multiple active sources assert ${entityKey}.`);
|
|
6424
|
+
return structuredClone(rows?.values().next().value ?? null);
|
|
6425
|
+
}
|
|
6426
|
+
incidentLinkIndex(projectId) {
|
|
6427
|
+
const mutable = new Map;
|
|
6428
|
+
const consider = (ref) => {
|
|
6429
|
+
const sortKey = linkRefSortKey2(ref);
|
|
6430
|
+
for (const endpoint of [ref.source, ref.target]) {
|
|
6431
|
+
const key = objectRefKey2(endpoint);
|
|
6432
|
+
const links = mutable.get(key) ?? new Map;
|
|
6433
|
+
links.set(sortKey, structuredClone(ref));
|
|
6434
|
+
mutable.set(key, links);
|
|
6435
|
+
}
|
|
6436
|
+
};
|
|
6437
|
+
getInMemoryObjectMaterializerAdapter(this.objects).visitExactLinks(projectId, (row) => consider(linkRef(row)));
|
|
6438
|
+
for (const override of this.state.linkOverrides.values()) {
|
|
6439
|
+
if (override.projectId === projectId)
|
|
6440
|
+
consider(override.ref);
|
|
6441
|
+
}
|
|
6442
|
+
for (const override of this.state.linkSlotOverrides.values()) {
|
|
6443
|
+
if (override.projectId !== projectId)
|
|
6444
|
+
continue;
|
|
6445
|
+
consider({
|
|
6446
|
+
source: override.ref.source,
|
|
6447
|
+
linkId: override.ref.linkId,
|
|
6448
|
+
target: override.value.target
|
|
6449
|
+
});
|
|
6450
|
+
}
|
|
6451
|
+
for (const [scopeKey, rows] of this.state.activeSourceLinkScopes) {
|
|
6452
|
+
if (JSON.parse(scopeKey)[0] !== projectId)
|
|
6453
|
+
continue;
|
|
6454
|
+
for (const row of rows.values()) {
|
|
6455
|
+
if (row.assertion.kind === "link")
|
|
6456
|
+
consider(row.assertion.ref);
|
|
6457
|
+
}
|
|
6458
|
+
}
|
|
6459
|
+
const index = new Map;
|
|
6460
|
+
for (const [objectKey, refs] of mutable) {
|
|
6461
|
+
index.set(objectKey, [...refs.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([, ref]) => ref));
|
|
6462
|
+
}
|
|
6463
|
+
return index;
|
|
6464
|
+
}
|
|
6465
|
+
}
|
|
6466
|
+
|
|
6104
6467
|
// src/storage/ontology/in-memory/testing.ts
|
|
6105
6468
|
var testingAdapters = new WeakMap;
|
|
6106
6469
|
function registerInMemoryOntologyStorageTestingAdapter(storage, adapter) {
|
|
@@ -6229,61 +6592,6 @@ class InMemoryVectorIndexingStorage {
|
|
|
6229
6592
|
}
|
|
6230
6593
|
}
|
|
6231
6594
|
|
|
6232
|
-
// src/storage/ontology/in-memory/vectors.ts
|
|
6233
|
-
function vectorObjectKey(projectId, ref) {
|
|
6234
|
-
return JSON.stringify([projectId, ref.objectTypeId, ref.primaryId]);
|
|
6235
|
-
}
|
|
6236
|
-
|
|
6237
|
-
class InMemoryOntologyVectorStorage {
|
|
6238
|
-
state;
|
|
6239
|
-
assertSession;
|
|
6240
|
-
runRootOperation;
|
|
6241
|
-
constructor(state, assertSession, runRootOperation) {
|
|
6242
|
-
this.state = state;
|
|
6243
|
-
this.assertSession = assertSession;
|
|
6244
|
-
this.runRootOperation = runRootOperation;
|
|
6245
|
-
}
|
|
6246
|
-
async list(input) {
|
|
6247
|
-
return this.listBatch({ projectId: input.projectId, refs: [input.ref] });
|
|
6248
|
-
}
|
|
6249
|
-
async listBatch(input) {
|
|
6250
|
-
return this.runRootOperation(() => [...new Set(input.refs.map((ref) => vectorObjectKey(input.projectId, ref)))].flatMap((key) => [...this.state.get(key)?.values() ?? []].map(({ values: _values, ...metadata }) => structuredClone(metadata))));
|
|
6251
|
-
}
|
|
6252
|
-
async removeBatch(input) {
|
|
6253
|
-
this.assertSession(input.session, input.projectId);
|
|
6254
|
-
for (const entry of input.entries) {
|
|
6255
|
-
await this.remove({ session: input.session, projectId: input.projectId, ...entry });
|
|
6256
|
-
}
|
|
6257
|
-
}
|
|
6258
|
-
async write(input) {
|
|
6259
|
-
this.assertSession(input.session, input.projectId, input.value.lastCommitId);
|
|
6260
|
-
const key = vectorObjectKey(input.projectId, input.value.ref);
|
|
6261
|
-
const profiles = this.state.get(key) ?? new Map;
|
|
6262
|
-
this.assertRevision(profiles.get(input.value.profile), input.expectedCommitId);
|
|
6263
|
-
profiles.set(input.value.profile, structuredClone(input.value));
|
|
6264
|
-
this.state.set(key, profiles);
|
|
6265
|
-
}
|
|
6266
|
-
async writeBatch(input) {
|
|
6267
|
-
for (const entry of input.entries) {
|
|
6268
|
-
await this.write({ session: input.session, projectId: input.projectId, ...entry });
|
|
6269
|
-
}
|
|
6270
|
-
}
|
|
6271
|
-
async remove(input) {
|
|
6272
|
-
this.assertSession(input.session, input.projectId);
|
|
6273
|
-
const key = vectorObjectKey(input.projectId, input.ref);
|
|
6274
|
-
const profiles = this.state.get(key);
|
|
6275
|
-
this.assertRevision(profiles?.get(input.profile), input.expectedCommitId);
|
|
6276
|
-
profiles?.delete(input.profile);
|
|
6277
|
-
if (profiles?.size === 0)
|
|
6278
|
-
this.state.delete(key);
|
|
6279
|
-
}
|
|
6280
|
-
assertRevision(current, expected) {
|
|
6281
|
-
if ((current?.lastCommitId ?? null) !== expected) {
|
|
6282
|
-
throw new MaterializationConflictError2("effective-state", "Vector changed since it was prepared.");
|
|
6283
|
-
}
|
|
6284
|
-
}
|
|
6285
|
-
}
|
|
6286
|
-
|
|
6287
6595
|
// src/storage/ontology/in-memory/index.ts
|
|
6288
6596
|
class InMemoryOntologyStorage {
|
|
6289
6597
|
state = createInMemoryOntologyState();
|
|
@@ -6292,6 +6600,7 @@ class InMemoryOntologyStorage {
|
|
|
6292
6600
|
commits;
|
|
6293
6601
|
sources;
|
|
6294
6602
|
materializations;
|
|
6603
|
+
replacementPlans;
|
|
6295
6604
|
outbox;
|
|
6296
6605
|
testHooks = {};
|
|
6297
6606
|
testingAdapter = {
|
|
@@ -6304,13 +6613,16 @@ class InMemoryOntologyStorage {
|
|
|
6304
6613
|
getInMemoryObjectMaterializerAdapter(objects).setVectorReader((projectId, ref, profile) => this.state.vectors.get(vectorObjectKey(projectId, ref))?.get(profile));
|
|
6305
6614
|
this.commits = new InMemoryOntologyCommitStorage(this.state, options.runRootOperation);
|
|
6306
6615
|
this.sources = new InMemoryOntologySourceStorage(this.state, options.runRootOperation, options.assertSourceMaterializationExecution);
|
|
6307
|
-
this.outbox = new InMemoryOntologyOutboxStorage(this.state, options.runRootOperation);
|
|
6308
|
-
|
|
6616
|
+
this.outbox = new InMemoryOntologyOutboxStorage(this.state, options.runRootOperation, options.readExecution);
|
|
6617
|
+
const hooks = {
|
|
6309
6618
|
beforeRead: (boundary) => this.testHooks.beforeRead?.(boundary),
|
|
6310
6619
|
beforeWrite: (boundary, ordinal) => this.testHooks.beforeWrite?.(boundary, ordinal),
|
|
6311
6620
|
observeBuffer: (boundary, rows) => this.testHooks.observeBuffer?.(boundary, rows),
|
|
6312
6621
|
observeWork: (records) => this.testHooks.observeWork?.(records)
|
|
6313
|
-
}
|
|
6622
|
+
};
|
|
6623
|
+
const reader = new InMemoryMaterializationStateReader(this.state, objects, timeseries, hooks);
|
|
6624
|
+
this.materializations = new InMemoryOntologyMaterializationStorage(this.state, objects, timeseries, reader, options.getTransactionToken, options.getMaterializationLifecycle, options.readExecution, hooks);
|
|
6625
|
+
this.replacementPlans = new InMemoryOntologyReplacementPlanStorage(this.state, objects, reader, options.runRootOperation, options.assertSourceMaterializationExecution, hooks);
|
|
6314
6626
|
registerInMemoryOntologyStorageTestingAdapter(this, this.testingAdapter);
|
|
6315
6627
|
}
|
|
6316
6628
|
setTestHooks(hooks) {
|
|
@@ -6827,14 +7139,16 @@ class InMemoryProjectionRunStorage2 {
|
|
|
6827
7139
|
assertProjectionRunNonEmpty2(input.projectId, "projectId");
|
|
6828
7140
|
assertProjectionRunListWindow2(input.limit, "limit");
|
|
6829
7141
|
assertProjectionRunListWindow2(input.offset, "offset");
|
|
6830
|
-
if (input.statuses?.length === 0)
|
|
7142
|
+
if (input.statuses?.length === 0 || input.errorCodes?.length === 0) {
|
|
6831
7143
|
return { runs: [], hasMore: false, total: 0 };
|
|
7144
|
+
}
|
|
6832
7145
|
const order = input.order ?? "desc";
|
|
6833
7146
|
const offset = input.offset ?? 0;
|
|
6834
7147
|
const limit = input.limit ?? this.rows.size;
|
|
6835
7148
|
const statuses = input.statuses ? new Set(input.statuses) : null;
|
|
6836
7149
|
const objectTypeIds = input.objectTypeIds ? new Set(input.objectTypeIds) : null;
|
|
6837
|
-
const
|
|
7150
|
+
const errorCodes = input.errorCodes ? new Set(input.errorCodes) : null;
|
|
7151
|
+
const filtered = [...this.rows.values()].filter((record) => record.projectId === input.projectId).filter((record) => input.projectionId ? record.identity.projectionId === input.projectionId : true).filter((record) => input.projectionKind ? record.identity.projectionKind === input.projectionKind : true).filter((record) => input.datasetId ? record.identity.datasetVersion.datasetId === input.datasetId : true).filter((record) => input.datasetVersionId ? record.identity.datasetVersion.versionId === input.datasetVersionId : true).filter((record) => objectTypeIds ? projectionRunObjectTypesVisible2(record, (id) => objectTypeIds.has(id)) : true).filter((record) => statuses ? statuses.has(record.status) : true).filter((record) => errorCodes ? record.error !== undefined && errorCodes.has(record.error.code) : true).filter((record) => matchesRunListDateFilters(record, {
|
|
6838
7152
|
statuses,
|
|
6839
7153
|
startedAfter: input.startedAfter,
|
|
6840
7154
|
startedBefore: input.startedBefore
|
|
@@ -7018,7 +7332,7 @@ class InMemoryShareGrantStorage2 {
|
|
|
7018
7332
|
rows = new Map;
|
|
7019
7333
|
async create(input) {
|
|
7020
7334
|
const record = normalizeShareGrantCreate2(input);
|
|
7021
|
-
const key =
|
|
7335
|
+
const key = rowKey2(record.projectId, record.id);
|
|
7022
7336
|
const duplicateTokenHash = [...this.rows.values()].some((current) => current.projectId === record.projectId && current.tokenHash === record.tokenHash);
|
|
7023
7337
|
if (this.rows.has(key) || duplicateTokenHash) {
|
|
7024
7338
|
throw new ShareGrantStorageError2("duplicate", `[Sixb] Share grant '${record.id}' conflicts with an existing record.`);
|
|
@@ -7028,7 +7342,7 @@ class InMemoryShareGrantStorage2 {
|
|
|
7028
7342
|
}
|
|
7029
7343
|
async getById(input) {
|
|
7030
7344
|
const normalized = normalizeGetShareGrantByIdInput2(input);
|
|
7031
|
-
const record = this.rows.get(
|
|
7345
|
+
const record = this.rows.get(rowKey2(normalized.projectId, normalized.id));
|
|
7032
7346
|
return record ? cloneShareGrantRecord2(record) : null;
|
|
7033
7347
|
}
|
|
7034
7348
|
async list(input) {
|
|
@@ -7044,7 +7358,7 @@ class InMemoryShareGrantStorage2 {
|
|
|
7044
7358
|
}
|
|
7045
7359
|
async revoke(input) {
|
|
7046
7360
|
const normalized = normalizeRevokeShareGrantInput2(input);
|
|
7047
|
-
const key =
|
|
7361
|
+
const key = rowKey2(normalized.projectId, normalized.id);
|
|
7048
7362
|
const current = this.rows.get(key);
|
|
7049
7363
|
if (!current)
|
|
7050
7364
|
return null;
|
|
@@ -7072,7 +7386,7 @@ function compareShareGrants(left, right) {
|
|
|
7072
7386
|
return created;
|
|
7073
7387
|
return left.id === right.id ? 0 : left.id < right.id ? 1 : -1;
|
|
7074
7388
|
}
|
|
7075
|
-
function
|
|
7389
|
+
function rowKey2(projectId, id) {
|
|
7076
7390
|
return `${projectId}\x00${id}`;
|
|
7077
7391
|
}
|
|
7078
7392
|
// src/storage/share-sessions/in-memory.ts
|
|
@@ -7088,7 +7402,7 @@ class InMemoryShareSessionStorage2 {
|
|
|
7088
7402
|
if (!grant) {
|
|
7089
7403
|
throw new ShareSessionStorageError2("invalid_input", `[Sixb] Share grant '${record.grantId}' does not exist in project '${record.projectId}'.`);
|
|
7090
7404
|
}
|
|
7091
|
-
const key =
|
|
7405
|
+
const key = rowKey3(record.projectId, record.id);
|
|
7092
7406
|
const duplicateTokenHash = [...this.rows.values()].some((current) => current.projectId === record.projectId && current.tokenHash === record.tokenHash);
|
|
7093
7407
|
if (this.rows.has(key) || duplicateTokenHash) {
|
|
7094
7408
|
throw new ShareSessionStorageError2("duplicate", `[Sixb] Share session '${record.id}' conflicts with an existing record.`);
|
|
@@ -7098,12 +7412,12 @@ class InMemoryShareSessionStorage2 {
|
|
|
7098
7412
|
}
|
|
7099
7413
|
async getById(input) {
|
|
7100
7414
|
const normalized = normalizeGetShareSessionByIdInput2(input);
|
|
7101
|
-
const record = this.rows.get(
|
|
7415
|
+
const record = this.rows.get(rowKey3(normalized.projectId, normalized.id));
|
|
7102
7416
|
return record ? cloneShareSessionRecord2(record) : null;
|
|
7103
7417
|
}
|
|
7104
7418
|
async renewIfValid(input) {
|
|
7105
7419
|
const normalized = normalizeRenewShareSessionIfValidInput2(input);
|
|
7106
|
-
const key =
|
|
7420
|
+
const key = rowKey3(normalized.projectId, normalized.id);
|
|
7107
7421
|
const current = this.rows.get(key);
|
|
7108
7422
|
if (!current || current.grantId !== normalized.grantId || current.tokenHash !== normalized.tokenHash || current.revokedAt !== undefined || current.createdAt.getTime() > normalized.now.getTime() || current.expiresAt.getTime() <= normalized.now.getTime() || current.absoluteExpiresAt.getTime() <= normalized.now.getTime()) {
|
|
7109
7423
|
return null;
|
|
@@ -7115,7 +7429,7 @@ class InMemoryShareSessionStorage2 {
|
|
|
7115
7429
|
}
|
|
7116
7430
|
async revoke(input) {
|
|
7117
7431
|
const normalized = normalizeRevokeShareSessionInput2(input);
|
|
7118
|
-
const key =
|
|
7432
|
+
const key = rowKey3(normalized.projectId, normalized.id);
|
|
7119
7433
|
const current = this.rows.get(key);
|
|
7120
7434
|
if (!current)
|
|
7121
7435
|
return null;
|
|
@@ -7133,7 +7447,7 @@ class InMemoryShareSessionStorage2 {
|
|
|
7133
7447
|
this.rows = new Map([...snapshot].map(([key, record]) => [key, parseShareSessionRecord2(record)]));
|
|
7134
7448
|
}
|
|
7135
7449
|
}
|
|
7136
|
-
function
|
|
7450
|
+
function rowKey3(projectId, id) {
|
|
7137
7451
|
return `${projectId}\x00${id}`;
|
|
7138
7452
|
}
|
|
7139
7453
|
// src/storage/sync-runs/types.ts
|
|
@@ -8160,6 +8474,7 @@ class InMemoryStorage2 {
|
|
|
8160
8474
|
webhookRunStorage = new InMemoryWebhookRunStorage2(this.executionStorage);
|
|
8161
8475
|
rulesStorage = new InMemoryRulesStorage2;
|
|
8162
8476
|
fileUploadSessionStorage = new InMemoryFileUploadSessions2;
|
|
8477
|
+
fileDownloadGrantStorage = new InMemoryFileDownloadGrantStorage;
|
|
8163
8478
|
connectorConnectionStorage;
|
|
8164
8479
|
auth;
|
|
8165
8480
|
executions;
|
|
@@ -8178,6 +8493,7 @@ class InMemoryStorage2 {
|
|
|
8178
8493
|
shareGrants;
|
|
8179
8494
|
shareSessions;
|
|
8180
8495
|
fileUploadSessions;
|
|
8496
|
+
fileDownloadGrants;
|
|
8181
8497
|
connectorConnections;
|
|
8182
8498
|
constructor(options = {}) {
|
|
8183
8499
|
this.connectorConnectionStorage = new InMemoryConnectorConnectionStorage2(options.connectorConnections);
|
|
@@ -8201,13 +8517,14 @@ class InMemoryStorage2 {
|
|
|
8201
8517
|
this.shareGrants = createOperationScopedFacade2(this.shareGrantStorage, scope);
|
|
8202
8518
|
this.shareSessions = createOperationScopedFacade2(this.shareSessionStorage, scope);
|
|
8203
8519
|
this.fileUploadSessions = createOperationScopedFacade2(this.fileUploadSessionStorage, scope);
|
|
8520
|
+
this.fileDownloadGrants = createOperationScopedFacade2(this.fileDownloadGrantStorage, scope);
|
|
8204
8521
|
this.connectorConnections = createOperationScopedFacade2(this.connectorConnectionStorage, scope);
|
|
8205
8522
|
this.ontologyStorage = new InMemoryOntologyStorage(this.objectStorage, this.timeseriesStorage, {
|
|
8206
8523
|
runRootOperation: async (run) => run(),
|
|
8207
8524
|
getTransactionToken: () => this.getActiveTransactionToken(),
|
|
8208
8525
|
getMaterializationLifecycle: () => this.getActiveMaterializationLifecycle(),
|
|
8209
8526
|
assertSourceMaterializationExecution: (input) => this.projectionRunStorage.assertSourceMaterializationExecutionUnlocked(input),
|
|
8210
|
-
|
|
8527
|
+
readExecution: (projectId, executionId) => this.executionStorage.getById({ projectId, id: executionId })
|
|
8211
8528
|
});
|
|
8212
8529
|
this.ontology = createOntologyOperationScope2(this.ontologyStorage, scope);
|
|
8213
8530
|
this.ontologyStorage.registerTestingAlias(this.ontology);
|
|
@@ -8361,6 +8678,7 @@ class InMemoryStorage2 {
|
|
|
8361
8678
|
shareGrants: this.shareGrantStorage,
|
|
8362
8679
|
shareSessions: this.shareSessionStorage,
|
|
8363
8680
|
fileUploadSessions: this.fileUploadSessionStorage,
|
|
8681
|
+
fileDownloadGrants: this.fileDownloadGrantStorage,
|
|
8364
8682
|
connectorConnections: this.connectorConnectionStorage,
|
|
8365
8683
|
ping: async () => {
|
|
8366
8684
|
return;
|
|
@@ -8392,6 +8710,7 @@ class InMemoryStorage2 {
|
|
|
8392
8710
|
shareGrants: this.shareGrantStorage.snapshot(),
|
|
8393
8711
|
shareSessions: this.shareSessionStorage.snapshot(),
|
|
8394
8712
|
fileUploadSessions: this.fileUploadSessionStorage.snapshot(),
|
|
8713
|
+
fileDownloadGrants: this.fileDownloadGrantStorage.snapshot(),
|
|
8395
8714
|
connectorConnections: this.connectorConnectionStorage.snapshot()
|
|
8396
8715
|
};
|
|
8397
8716
|
}
|
|
@@ -8416,25 +8735,65 @@ class InMemoryStorage2 {
|
|
|
8416
8735
|
this.shareGrantStorage.restore(snapshot.shareGrants);
|
|
8417
8736
|
this.shareSessionStorage.restore(snapshot.shareSessions);
|
|
8418
8737
|
this.fileUploadSessionStorage.restore(snapshot.fileUploadSessions);
|
|
8738
|
+
this.fileDownloadGrantStorage.restore(snapshot.fileDownloadGrants);
|
|
8419
8739
|
this.connectorConnectionStorage.restore(snapshot.connectorConnections);
|
|
8420
8740
|
}
|
|
8421
8741
|
}
|
|
8742
|
+
// src/storage/objects/query-preparation.ts
|
|
8743
|
+
function isQueryPreparationCapableStorage(storage) {
|
|
8744
|
+
return "prepareObjectQueries" in storage && typeof storage.prepareObjectQueries === "function";
|
|
8745
|
+
}
|
|
8746
|
+
async function prepareObjectQueries2(input) {
|
|
8747
|
+
if (!isQueryPreparationCapableStorage(input.storage)) {
|
|
8748
|
+
throw new Error("[Sixb] This storage provider does not support object query preparation");
|
|
8749
|
+
}
|
|
8750
|
+
const values = input.ontology.getValueTypesById();
|
|
8751
|
+
const objectTypes = input.ontology.listObjectTypes().map((type) => ({
|
|
8752
|
+
objectTypeId: type.id,
|
|
8753
|
+
properties: type.properties.filter((property) => property.mode !== "telemetry" && !property.primary).map((property) => {
|
|
8754
|
+
const capabilities = resolvePropertyQueryCapabilities2(property, values);
|
|
8755
|
+
const schema = resolveQuerySchema(property.schema, values);
|
|
8756
|
+
return {
|
|
8757
|
+
propertyId: property.id,
|
|
8758
|
+
scalarKind: schema ? queryScalarKindForSchema(schema) : undefined,
|
|
8759
|
+
filterable: capabilities.operators.includes("eq"),
|
|
8760
|
+
sortable: capabilities.sortable,
|
|
8761
|
+
text: capabilities.text
|
|
8762
|
+
};
|
|
8763
|
+
}).filter((property) => property.filterable || property.sortable || property.text),
|
|
8764
|
+
indexes: type.query?.indexes ?? []
|
|
8765
|
+
}));
|
|
8766
|
+
return input.storage.prepareObjectQueries({ projectId: input.projectId, objectTypes });
|
|
8767
|
+
}
|
|
8768
|
+
|
|
8422
8769
|
// src/storage/migrations/migrate-storage.ts
|
|
8423
8770
|
function isMigrationCapableStorage2(storage) {
|
|
8424
8771
|
return Array.isArray(storage.migrators);
|
|
8425
8772
|
}
|
|
8426
|
-
async function migrateStorage2(storage) {
|
|
8427
|
-
if (!isMigrationCapableStorage2(storage) || storage.migrators.length === 0) {
|
|
8428
|
-
return { status: "skipped", reports: [] };
|
|
8429
|
-
}
|
|
8773
|
+
async function migrateStorage2(storage, options) {
|
|
8430
8774
|
const reports = [];
|
|
8431
|
-
|
|
8432
|
-
|
|
8775
|
+
if (isMigrationCapableStorage2(storage)) {
|
|
8776
|
+
for (const migrator of storage.migrators) {
|
|
8777
|
+
reports.push(await migrator.migrate());
|
|
8778
|
+
}
|
|
8433
8779
|
}
|
|
8434
|
-
|
|
8435
|
-
|
|
8436
|
-
|
|
8437
|
-
|
|
8780
|
+
let queries;
|
|
8781
|
+
if (options && isQueryPreparationCapableStorage(storage)) {
|
|
8782
|
+
queries = await prepareObjectQueries2({ ...options, storage });
|
|
8783
|
+
for (const warning of queries.warnings) {
|
|
8784
|
+
console.warn(`[Sixb] ${warning}`);
|
|
8785
|
+
}
|
|
8786
|
+
}
|
|
8787
|
+
let status = "skipped";
|
|
8788
|
+
if (reports.length > 0 || queries) {
|
|
8789
|
+
status = "current";
|
|
8790
|
+
}
|
|
8791
|
+
if (reports.some((report) => report.status === "migrated") || queries?.status === "prepared") {
|
|
8792
|
+
status = "migrated";
|
|
8793
|
+
}
|
|
8794
|
+
if (queries)
|
|
8795
|
+
return { status, reports, queries };
|
|
8796
|
+
return { status, reports };
|
|
8438
8797
|
}
|
|
8439
8798
|
|
|
8440
8799
|
// src/storage/migrations/check-schema.ts
|
|
@@ -8635,6 +8994,6 @@ var VECTOR_INDEXING_FAILURE_CODES2 = [
|
|
|
8635
8994
|
"vector.outcome_unknown",
|
|
8636
8995
|
"internal.unexpected"
|
|
8637
8996
|
];
|
|
8638
|
-
export { InMemoryActionRunStorage2, AGENT_MESSAGE_CONTENT_VERSION2, InMemoryAgentStorage2, AiCostStorageError2, InMemoryAiCostStorage2, InMemoryAiLimitStorage2, InMemoryConnectorConnectionStorage2, InMemoryFileUploadSessions2, hasVectorProfile2, isVectorProfileQuery2, objectBatchKey2, linkBatchKey2, objectLinkCursor2, compareQueryScalarValues, queryScalarValuesEqual, MAX_VECTOR_K, vectorConfiguration, vectorSources, normalizeVector2, vectorSearchCandidateLimit2, PIPELINE_RUN_FAILURE_CODES2, InMemoryPipelineRunStorage2, InMemoryProjectionRunStorage2, InMemoryRulesStorage2, InMemoryShareGrantStorage2, InMemoryShareSessionStorage2, SYNC_RUN_FAILURE_CODES2, InMemorySyncRunStorage2, WEBHOOK_RUN_FAILURE_CODES2, canRetryWebhookRun2, InMemoryWebhookRunStorage2, WorkflowInterventionError2, InMemoryWorkflowInterventionStorage2, WORKFLOW_RUN_FAILURE_CODES2, InMemoryWorkflowRunStorage2, InMemoryWorkflowNodeRunStorage2, InMemoryWorkflowAgentNodeRunStorage2, InMemoryStorage2, isMigrationCapableStorage2, migrateStorage2, checkStorageSchema2, step2, defineMigrations2, runMigrationSet2, describeMigrationHistory2, VECTOR_INDEXING_FAILURE_CODES2 };
|
|
8997
|
+
export { InMemoryActionRunStorage2, AGENT_MESSAGE_CONTENT_VERSION2, InMemoryAgentStorage2, AiCostStorageError2, InMemoryAiCostStorage2, InMemoryAiLimitStorage2, InMemoryConnectorConnectionStorage2, InMemoryFileUploadSessions2, hasVectorProfile2, isVectorProfileQuery2, objectBatchKey2, linkBatchKey2, objectLinkCursor2, compareQueryScalarValues, queryScalarValuesEqual, MAX_VECTOR_K, vectorConfiguration, vectorSources, normalizeVector2, vectorSearchCandidateLimit2, PIPELINE_RUN_FAILURE_CODES2, InMemoryPipelineRunStorage2, InMemoryProjectionRunStorage2, InMemoryRulesStorage2, InMemoryShareGrantStorage2, InMemoryShareSessionStorage2, SYNC_RUN_FAILURE_CODES2, InMemorySyncRunStorage2, WEBHOOK_RUN_FAILURE_CODES2, canRetryWebhookRun2, InMemoryWebhookRunStorage2, WorkflowInterventionError2, InMemoryWorkflowInterventionStorage2, WORKFLOW_RUN_FAILURE_CODES2, InMemoryWorkflowRunStorage2, InMemoryWorkflowNodeRunStorage2, InMemoryWorkflowAgentNodeRunStorage2, InMemoryStorage2, prepareObjectQueries2, isMigrationCapableStorage2, migrateStorage2, checkStorageSchema2, step2, defineMigrations2, runMigrationSet2, describeMigrationHistory2, VECTOR_INDEXING_FAILURE_CODES2 };
|
|
8639
8998
|
|
|
8640
|
-
//# debugId=
|
|
8999
|
+
//# debugId=06195CF0CDEA109D64756E2164756E21
|