@sixb/sqlite 0.1.16 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/ai-limit-storage.d.ts.map +1 -1
  2. package/dist/ai-usage-storage.d.ts.map +1 -1
  3. package/dist/execution-storage.d.ts +28 -0
  4. package/dist/execution-storage.d.ts.map +1 -1
  5. package/dist/file-download-grant-storage.d.ts +11 -0
  6. package/dist/file-download-grant-storage.d.ts.map +1 -0
  7. package/dist/index.d.ts +2 -0
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +1362 -579
  10. package/dist/index.js.map +27 -25
  11. package/dist/migrations.d.ts.map +1 -1
  12. package/dist/objects/query-compiler.d.ts.map +1 -1
  13. package/dist/objects/reader.d.ts.map +1 -1
  14. package/dist/objects/storage.d.ts.map +1 -1
  15. package/dist/ontology-storage/index.d.ts +2 -0
  16. package/dist/ontology-storage/index.d.ts.map +1 -1
  17. package/dist/ontology-storage/materialization-session.d.ts +17 -37
  18. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  19. package/dist/ontology-storage/materialization-state.d.ts +3 -26
  20. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  21. package/dist/ontology-storage/materialization-writer.d.ts +1 -1
  22. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  23. package/dist/ontology-storage/materializations.d.ts +3 -6
  24. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  25. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  26. package/dist/ontology-storage/replacement-plans.d.ts +88 -0
  27. package/dist/ontology-storage/replacement-plans.d.ts.map +1 -0
  28. package/dist/ontology-storage/shared.d.ts +8 -5
  29. package/dist/ontology-storage/shared.d.ts.map +1 -1
  30. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
  31. package/dist/ontology-storage/source-roots.d.ts +15 -2
  32. package/dist/ontology-storage/source-roots.d.ts.map +1 -1
  33. package/dist/ontology-storage/sources.d.ts +1 -2
  34. package/dist/ontology-storage/sources.d.ts.map +1 -1
  35. package/dist/projection-run-storage.d.ts.map +1 -1
  36. package/dist/rules-storage.d.ts.map +1 -1
  37. package/dist/testing.d.ts +4 -1
  38. package/dist/testing.d.ts.map +1 -1
  39. package/package.json +3 -3
  40. package/src/ai-cost-storage.ts +10 -0
  41. package/src/ai-limit-storage.ts +13 -2
  42. package/src/ai-usage-storage.ts +9 -1
  43. package/src/execution-storage.ts +7 -2
  44. package/src/file-download-grant-storage.ts +76 -0
  45. package/src/index.ts +10 -3
  46. package/src/migrations/053-replacement-plans.sql +73 -0
  47. package/src/migrations/054-slim-ontology-outbox.sql +21 -0
  48. package/src/migrations/055-commit-touches.sql +31 -0
  49. package/src/migrations/056-audio-transcription.sql +10 -0
  50. package/src/migrations/057-reranking-model-kind.sql +16 -0
  51. package/src/migrations/058-file-download-grants.sql +21 -0
  52. package/src/migrations/059-drained-source-versions.sql +11 -0
  53. package/src/migrations.ts +115 -0
  54. package/src/objects/query-compiler.ts +5 -0
  55. package/src/objects/reader.ts +1 -0
  56. package/src/objects/storage.ts +5 -11
  57. package/src/ontology-storage/index.ts +6 -1
  58. package/src/ontology-storage/materialization-session.ts +120 -211
  59. package/src/ontology-storage/materialization-state.ts +5 -261
  60. package/src/ontology-storage/materialization-writer.ts +11 -13
  61. package/src/ontology-storage/materializations.ts +133 -210
  62. package/src/ontology-storage/outbox.ts +50 -16
  63. package/src/ontology-storage/replacement-plans.ts +1064 -0
  64. package/src/ontology-storage/shared.ts +17 -12
  65. package/src/ontology-storage/source-cleanup.ts +24 -9
  66. package/src/ontology-storage/source-roots.ts +38 -14
  67. package/src/ontology-storage/sources.ts +0 -26
  68. package/src/projection-run-storage.ts +7 -1
  69. package/src/rules-storage.ts +9 -14
  70. package/src/testing.ts +6 -4
package/dist/index.js CHANGED
@@ -4670,6 +4670,172 @@ DROP TABLE legacy_ontology_sources;
4670
4670
  -- describe empty tables forever and steer the planner to partial indexes over primary keys.
4671
4671
  `;
4672
4672
 
4673
+ // src/migrations/053-replacement-plans.sql
4674
+ var _053_replacement_plans_default = `-- The plan of a ready replacement candidate, built outside its commit transaction. Its rows are a
4675
+ -- cache the commit verifies again, keyed by the candidate's source version, which outlives it:
4676
+ -- maintenance deletes a plan once its candidate is no longer ready, and only then the version.
4677
+ CREATE TABLE ontology_replacement_plans (
4678
+ version_id INTEGER PRIMARY KEY REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
4679
+ project_id TEXT NOT NULL,
4680
+ -- The commit the plan's work carries, at the time fixed when the plan opened.
4681
+ commit_id TEXT NOT NULL,
4682
+ committed_at TEXT NOT NULL,
4683
+ -- The active source the plan replaces, NULL when none was: a plan resumed once that moved
4684
+ -- starts over.
4685
+ replaced_materialization_id TEXT,
4686
+ replaced_last_commit_id TEXT,
4687
+ -- The last commit row every planned identity is known fresh against.
4688
+ watermark INTEGER NOT NULL
4689
+ );
4690
+
4691
+ -- One row per entity the plan decides. \`read_revision\` is the revision of the inputs its state was
4692
+ -- last read at; \`planned_revision\` the one its work was planned from, NULL while still to plan.
4693
+ -- \`classified\` and \`change\` summarize that work for the commit's change counts.
4694
+ CREATE TABLE ontology_replacement_plan_identities (
4695
+ version_id INTEGER NOT NULL,
4696
+ entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
4697
+ identity_key TEXT NOT NULL,
4698
+ sort_key TEXT NOT NULL,
4699
+ diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
4700
+ read_revision TEXT,
4701
+ planned_revision TEXT,
4702
+ classified INTEGER NOT NULL DEFAULT 0 CHECK (classified IN (0, 1)),
4703
+ change TEXT CHECK (change IN ('created', 'updated', 'deleted')),
4704
+ PRIMARY KEY (version_id, entity_kind, identity_key)
4705
+ );
4706
+ -- The identities still to plan, in planning order: a round that plans a few again finds them
4707
+ -- without walking the rest.
4708
+ CREATE INDEX idx_ontology_replacement_plan_identities_unplanned
4709
+ ON ontology_replacement_plan_identities (version_id, entity_kind, sort_key)
4710
+ WHERE planned_revision IS NULL;
4711
+
4712
+ -- Planned work, with the same columns as a session's work table plus the identity it belongs to.
4713
+ CREATE TABLE ontology_replacement_plan_work (
4714
+ work_id INTEGER NOT NULL,
4715
+ entity_kind TEXT NOT NULL,
4716
+ identity_key TEXT NOT NULL,
4717
+ record_key TEXT NOT NULL,
4718
+ unique_key TEXT NOT NULL,
4719
+ kind TEXT NOT NULL,
4720
+ lane TEXT NOT NULL,
4721
+ major_order INTEGER NOT NULL,
4722
+ minor_order INTEGER NOT NULL,
4723
+ sort_one TEXT NOT NULL,
4724
+ sort_two TEXT NOT NULL,
4725
+ classification_entity_kind TEXT,
4726
+ classification_identity_key TEXT,
4727
+ cardinality_view TEXT,
4728
+ cardinality_occupied INTEGER,
4729
+ cardinality_source_type_id TEXT,
4730
+ cardinality_source_primary_id TEXT,
4731
+ cardinality_link_id TEXT,
4732
+ cardinality_target_type_id TEXT,
4733
+ cardinality_target_primary_id TEXT,
4734
+ payload TEXT NOT NULL CHECK (json_valid(payload)),
4735
+ PRIMARY KEY (work_id, record_key),
4736
+ UNIQUE (work_id, unique_key)
4737
+ );
4738
+ CREATE INDEX idx_ontology_replacement_plan_work_identity
4739
+ ON ontology_replacement_plan_work (work_id, entity_kind, identity_key);
4740
+ -- The few objects whose existence the plan flips, which link expansion starts from.
4741
+ CREATE INDEX idx_ontology_replacement_plan_work_incident
4742
+ ON ontology_replacement_plan_work (work_id) WHERE kind = 'incident-object';
4743
+ CREATE INDEX idx_ontology_replacement_plan_work_lane
4744
+ ON ontology_replacement_plan_work (
4745
+ work_id, lane, major_order, minor_order, sort_one, sort_two, record_key
4746
+ );
4747
+ `;
4748
+
4749
+ // src/migrations/054-slim-ontology-outbox.sql
4750
+ var _054_slim_ontology_outbox_default = `-- An outbox event stores what changed and nothing it can be rebuilt from: delivery rebuilds the
4751
+ -- event from its commit and the execution that commit ran under. The commit also stops copying
4752
+ -- \`requested_by\` and \`executor\` from that execution (047): the execution is canonical, immutable,
4753
+ -- and already referenced by the commit. The migration step checks beforehand that every stored
4754
+ -- event rebuilds exactly as it was stored.
4755
+ --
4756
+ -- SQLite cannot add a NOT NULL column without a default; every insert supplies the event, and the
4757
+ -- UPDATE below replaces the placeholder on every row.
4758
+ ALTER TABLE ontology_outbox ADD COLUMN event TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(event));
4759
+ UPDATE ontology_outbox SET event = json_object(
4760
+ 'payload', CASE
4761
+ WHEN envelope ->> '$.type' IN ('object.created', 'link.created')
4762
+ THEN json_remove(envelope -> '$.payload', '$.propertyChanges')
4763
+ ELSE envelope -> '$.payload'
4764
+ END,
4765
+ 'type', envelope ->> '$.type'
4766
+ );
4767
+ ALTER TABLE ontology_outbox DROP COLUMN envelope;
4768
+
4769
+ ALTER TABLE ontology_commits DROP COLUMN requested_by;
4770
+ ALTER TABLE ontology_commits DROP COLUMN executor;
4771
+ `;
4772
+
4773
+ // src/migrations/055-commit-touches.sql
4774
+ var _055_commit_touches_default = `-- Every commit records the entities whose plan inputs it changed, so a replacement plan's refresh
4775
+ -- rechecks only what the commits since it was last fresh touched, instead of the whole plan.
4776
+ CREATE TABLE ontology_commit_touches (
4777
+ project_id TEXT NOT NULL,
4778
+ commit_id TEXT NOT NULL,
4779
+ entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link', 'scope')),
4780
+ -- \`objectRefKey\`, \`linkRefKey\` or \`linkScopeKey\`.
4781
+ identity_key TEXT NOT NULL
4782
+ );
4783
+ CREATE INDEX idx_ontology_commit_touches_commit ON ontology_commit_touches (project_id, commit_id);
4784
+
4785
+ -- Existing plans predate the touches of the commits since their watermark: they plan again.
4786
+ DELETE FROM ontology_replacement_plan_work;
4787
+ DELETE FROM ontology_replacement_plan_identities;
4788
+ DELETE FROM ontology_replacement_plans;
4789
+
4790
+ -- Link expansion starts only from what changed since it last ran: \`expansions\` counts the runs,
4791
+ -- and an identity's \`expand_at\` is the run that has yet to start from it (an object once planned,
4792
+ -- a link once it needs a diff). \`source_key\` and \`target_key\` find a link from either endpoint.
4793
+ ALTER TABLE ontology_replacement_plans ADD COLUMN expansions INTEGER NOT NULL DEFAULT 0;
4794
+ ALTER TABLE ontology_replacement_plan_identities ADD COLUMN expand_at INTEGER;
4795
+ ALTER TABLE ontology_replacement_plan_identities ADD COLUMN source_key TEXT;
4796
+ ALTER TABLE ontology_replacement_plan_identities ADD COLUMN target_key TEXT;
4797
+ CREATE INDEX idx_ontology_replacement_plan_identities_expand
4798
+ ON ontology_replacement_plan_identities (version_id, entity_kind, expand_at);
4799
+ CREATE INDEX idx_ontology_replacement_plan_identities_source
4800
+ ON ontology_replacement_plan_identities (version_id, source_key)
4801
+ WHERE source_key IS NOT NULL;
4802
+ CREATE INDEX idx_ontology_replacement_plan_identities_target
4803
+ ON ontology_replacement_plan_identities (version_id, target_key)
4804
+ WHERE target_key IS NOT NULL;
4805
+ `;
4806
+
4807
+ // src/migrations/056-audio-transcription.sql
4808
+ var _056_audio_transcription_default = `ALTER TABLE ai_model_call_usage
4809
+ ADD COLUMN audio_duration_ms INTEGER
4810
+ CHECK (audio_duration_ms IS NULL OR audio_duration_ms >= 0);
4811
+
4812
+ ALTER TABLE ai_model_call_usage
4813
+ ADD COLUMN model_kind TEXT
4814
+ CHECK (
4815
+ model_kind IS NULL
4816
+ OR model_kind IN ('language', 'image', 'video', 'embedding', 'decision', 'transcription')
4817
+ );
4818
+ `;
4819
+
4820
+ // src/migrations/057-reranking-model-kind.sql
4821
+ var _057_reranking_model_kind_default = `-- SQLite cannot alter a CHECK constraint. Swapping the column avoids rebuilding a table that
4822
+ -- usage groups, costs and limit reservations reference.
4823
+ ALTER TABLE ai_model_call_usage
4824
+ ADD COLUMN model_kind_next TEXT
4825
+ CHECK (
4826
+ model_kind_next IS NULL
4827
+ OR model_kind_next IN (
4828
+ 'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'reranking'
4829
+ )
4830
+ );
4831
+
4832
+ UPDATE ai_model_call_usage SET model_kind_next = model_kind;
4833
+
4834
+ ALTER TABLE ai_model_call_usage DROP COLUMN model_kind;
4835
+
4836
+ ALTER TABLE ai_model_call_usage RENAME COLUMN model_kind_next TO model_kind;
4837
+ `;
4838
+
4673
4839
  // src/migrations/037-execution-requester-groups.sql
4674
4840
  var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids TEXT NOT NULL DEFAULT '[]';
4675
4841
 
@@ -4790,6 +4956,44 @@ CREATE INDEX idx_file_upload_sessions_abandoned
4790
4956
  WHERE status = 'pending' AND provider_upload IS NOT NULL;
4791
4957
  `;
4792
4958
 
4959
+ // src/migrations/058-file-download-grants.sql
4960
+ var _058_file_download_grants_default = `-- URLs issued by blobs.createDownloadUrl(). Only the SHA-256 of each URL's token is stored.
4961
+ -- Timestamps are ISO-8601 UTC text and compare lexicographically.
4962
+ CREATE TABLE file_download_grants (
4963
+ project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
4964
+ id TEXT NOT NULL CHECK (length(trim(id)) > 0),
4965
+ token_hash TEXT NOT NULL CHECK (
4966
+ length(token_hash) = 64
4967
+ AND token_hash = lower(token_hash)
4968
+ AND token_hash NOT GLOB '*[^0-9a-f]*'
4969
+ ),
4970
+ file TEXT NOT NULL CHECK (json_type(file) = 'object'),
4971
+ execution_id TEXT NOT NULL CHECK (length(trim(execution_id)) > 0),
4972
+ created_at TEXT NOT NULL,
4973
+ expires_at TEXT NOT NULL,
4974
+ revoked_at TEXT,
4975
+ PRIMARY KEY (project_id, id),
4976
+ UNIQUE (project_id, token_hash),
4977
+ CHECK (expires_at > created_at)
4978
+ );
4979
+
4980
+ CREATE INDEX idx_file_download_grants_expiry ON file_download_grants (project_id, expires_at);
4981
+ `;
4982
+
4983
+ // src/migrations/059-drained-source-versions.sql
4984
+ var _059_drained_source_versions_default = `-- A superseded version's \`root_count\` counts the roots it still holds: cleanup lowers it as it
4985
+ -- deletes them. A version that reaches zero is found through this index, instead of every
4986
+ -- superseded version being walked and probed for roots on every maintenance pass.
4987
+ UPDATE ontology_sources
4988
+ SET root_count = (SELECT count(*) FROM ontology_source_roots AS roots
4989
+ WHERE roots.version_id = ontology_sources.version_id)
4990
+ WHERE status = 'superseded' AND root_count <> (SELECT count(*) FROM ontology_source_roots AS roots
4991
+ WHERE roots.version_id = ontology_sources.version_id);
4992
+ CREATE INDEX idx_ontology_sources_drained
4993
+ ON ontology_sources (project_id, terminal_at, source_id, materialization_id)
4994
+ WHERE status = 'superseded' AND root_count = 0;
4995
+ `;
4996
+
4793
4997
  // src/migrations.ts
4794
4998
  var MIGRATIONS_TABLE_SQL = `
4795
4999
  CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -4883,7 +5087,20 @@ var sqliteStorageMigrations = defineMigrations({
4883
5087
  if (after.versions !== before.versions || after.roots !== before.roots || after.rows !== before.rows) {
4884
5088
  throw new Error("[SixbSqliteStorage] Source versions, roots or rows were lost while compacting source storage.");
4885
5089
  }
4886
- }, { checksum: checksum(_052_compact_source_storage_default) })
5090
+ }, { checksum: checksum(_052_compact_source_storage_default) }),
5091
+ sqliteSql("053-replacement-plans", _053_replacement_plans_default),
5092
+ sqliteStep("054-slim-ontology-outbox", (db) => {
5093
+ const mismatched = unrebuildableOutboxEvents(db);
5094
+ if (mismatched > 0) {
5095
+ throw new Error(`[SixbSqliteStorage] ${mismatched} outbox events would not rebuild from their commit.`);
5096
+ }
5097
+ db.run(_054_slim_ontology_outbox_default);
5098
+ }, { checksum: checksum(_054_slim_ontology_outbox_default) }),
5099
+ sqliteSql("055-commit-touches", _055_commit_touches_default),
5100
+ sqliteSql("056-audio-transcription", _056_audio_transcription_default),
5101
+ sqliteSql("057-reranking-model-kind", _057_reranking_model_kind_default),
5102
+ sqliteSql("058-file-download-grants", _058_file_download_grants_default),
5103
+ sqliteSql("059-drained-source-versions", _059_drained_source_versions_default)
4887
5104
  ]
4888
5105
  });
4889
5106
  function sqliteStoragePath(basePath) {
@@ -5003,6 +5220,82 @@ function sourceStorageCounts(db) {
5003
5220
  (SELECT count(*) FROM ontology_source_roots) AS roots,
5004
5221
  (SELECT count(*) FROM ontology_source_rows) AS rows`).get();
5005
5222
  }
5223
+ function unrebuildableOutboxEvents(db) {
5224
+ const row = db.query(`SELECT COUNT(*) AS mismatched
5225
+ FROM ontology_outbox AS outbox
5226
+ LEFT JOIN ontology_commits AS commits
5227
+ ON commits.project_id = outbox.project_id AND commits.id = outbox.commit_id
5228
+ LEFT JOIN executions
5229
+ ON executions.project_id = commits.project_id AND executions.id = commits.execution_id
5230
+ WHERE executions.id IS NULL
5231
+ OR outbox.envelope ->> '$.id' IS NOT outbox.id
5232
+ OR outbox.envelope ->> '$.commitOrdinal' IS NOT outbox.commit_ordinal
5233
+ OR outbox.envelope ->> '$.commitId' IS NOT commits.id
5234
+ OR outbox.envelope ->> '$.projectId' IS NOT commits.project_id
5235
+ OR outbox.envelope ->> '$.schemaVersion' IS NOT 1
5236
+ OR outbox.envelope ->> '$.occurredAt' IS NOT commits.committed_at
5237
+ OR outbox.envelope -> '$.origin' IS NOT json(commits.origin)
5238
+ OR outbox.envelope ->> '$.correlationId' IS NOT executions.correlation_id
5239
+ OR outbox.envelope -> '$.requestedBy' IS NOT CASE
5240
+ WHEN executions.requested_by_user_id IS NOT NULL
5241
+ THEN json_object('id', executions.requested_by_user_id, 'type', 'user')
5242
+ WHEN executions.requested_by_service_account_id IS NOT NULL
5243
+ THEN json_object('id', executions.requested_by_service_account_id, 'type', 'serviceAccount')
5244
+ END
5245
+ OR outbox.envelope -> '$.executor' IS NOT CASE executions.executor_kind
5246
+ WHEN 'request' THEN json_object('requestId', executions.executor_id, 'type', 'request')
5247
+ WHEN 'agent' THEN json_object('runId', executions.executor_id, 'type', 'agent')
5248
+ WHEN 'kernel' THEN json_object(
5249
+ 'operation',
5250
+ CASE executions.authority_kernel_operation
5251
+ WHEN 'ontology.recover'
5252
+ THEN json_object('recoveryId', executions.executor_id, 'type', 'ontology.recover')
5253
+ ELSE json_object('indexingId', executions.executor_id, 'type', 'ontology.indexVectors')
5254
+ END,
5255
+ 'type', 'kernel'
5256
+ )
5257
+ ELSE json_object(
5258
+ 'id', executions.authority_primitive_id,
5259
+ 'kind', executions.executor_kind,
5260
+ 'runId', executions.executor_id,
5261
+ 'type', 'primitive'
5262
+ )
5263
+ END
5264
+ OR outbox.envelope ->> '$.topic' IS NOT CASE substr(outbox.envelope ->> '$.type', 1, 4)
5265
+ WHEN 'obje' THEN 'objects'
5266
+ WHEN 'link' THEN 'links'
5267
+ WHEN 'tele' THEN 'telemetry'
5268
+ END
5269
+ OR outbox.envelope ->> '$.partitionKey' IS NOT CASE outbox.envelope ->> '$.topic'
5270
+ WHEN 'objects' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
5271
+ (outbox.envelope ->> '$.payload.primaryId')
5272
+ WHEN 'links' THEN (outbox.envelope ->> '$.payload.sourceTypeId') || ':' ||
5273
+ (outbox.envelope ->> '$.payload.sourceId') || ':' ||
5274
+ (outbox.envelope ->> '$.payload.linkId')
5275
+ WHEN 'telemetry' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
5276
+ (outbox.envelope ->> '$.payload.objectId') || ':' ||
5277
+ (outbox.envelope ->> '$.payload.propertyId')
5278
+ END
5279
+ OR (
5280
+ outbox.envelope ->> '$.type' IN ('object.created', 'link.created')
5281
+ AND (
5282
+ (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.propertyChanges'))
5283
+ IS NOT (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.properties'))
5284
+ OR EXISTS (
5285
+ SELECT 1
5286
+ FROM json_each(outbox.envelope, '$.payload.propertyChanges') AS change
5287
+ LEFT JOIN json_each(outbox.envelope, '$.payload.properties') AS property
5288
+ ON property.key = change.key
5289
+ WHERE property.key IS NULL
5290
+ OR change.value ->> '$.operation' IS NOT 'created'
5291
+ OR json_type(change.value, '$.after') IS NOT property.type
5292
+ OR change.value ->> '$.after' IS NOT property.value
5293
+ OR (SELECT COUNT(*) FROM json_each(change.value)) IS NOT 2
5294
+ )
5295
+ )
5296
+ )`).get();
5297
+ return row.mismatched;
5298
+ }
5006
5299
  function checksum(value) {
5007
5300
  return createHash("sha256").update(value).digest("hex");
5008
5301
  }
@@ -6592,6 +6885,7 @@ class SqliteAiUsageStorage {
6592
6885
  provider_id,
6593
6886
  provider_ids,
6594
6887
  requested_model_id,
6888
+ model_kind,
6595
6889
  requested_reasoning,
6596
6890
  response_model_id,
6597
6891
  response_id,
@@ -6603,14 +6897,15 @@ class SqliteAiUsageStorage {
6603
6897
  cache_write_input_tokens,
6604
6898
  text_output_tokens,
6605
6899
  reasoning_output_tokens,
6900
+ audio_duration_ms,
6606
6901
  reporting_status,
6607
6902
  raw_usage,
6608
6903
  occurred_at,
6609
6904
  recorded_at
6610
6905
  ) VALUES (
6611
- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
6906
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
6612
6907
  )
6613
- `).run(record.projectId, record.id, record.executionId, record.attempt, record.callId, record.providerId, record.providerIds === undefined ? null : JSON.stringify(record.providerIds), record.requestedModelId, record.requestedReasoning === undefined ? null : JSON.stringify(record.requestedReasoning), record.responseModelId ?? null, record.responseId, record.usage.inputTokens ?? null, record.usage.outputTokens ?? null, record.usage.totalTokens ?? null, record.usage.uncachedInputTokens ?? null, record.usage.cacheReadInputTokens ?? null, record.usage.cacheWriteInputTokens ?? null, record.usage.textOutputTokens ?? null, record.usage.reasoningOutputTokens ?? null, record.usage.reportingStatus, record.rawUsage === undefined ? null : JSON.stringify(record.rawUsage), record.occurredAt.toISOString(), record.recordedAt.toISOString());
6908
+ `).run(record.projectId, record.id, record.executionId, record.attempt, record.callId, record.providerId, record.providerIds === undefined ? null : JSON.stringify(record.providerIds), record.requestedModelId, record.modelKind ?? null, record.requestedReasoning === undefined ? null : JSON.stringify(record.requestedReasoning), record.responseModelId ?? null, record.responseId, record.usage.inputTokens ?? null, record.usage.outputTokens ?? null, record.usage.totalTokens ?? null, record.usage.uncachedInputTokens ?? null, record.usage.cacheReadInputTokens ?? null, record.usage.cacheWriteInputTokens ?? null, record.usage.textOutputTokens ?? null, record.usage.reasoningOutputTokens ?? null, record.usage.audioDurationMs ?? null, record.usage.reportingStatus, record.rawUsage === undefined ? null : JSON.stringify(record.rawUsage), record.occurredAt.toISOString(), record.recordedAt.toISOString());
6614
6909
  }
6615
6910
  findByIdempotencyKey(record) {
6616
6911
  const row = this.connection.db.query(`
@@ -6645,6 +6940,7 @@ class SqliteAiUsageStorage {
6645
6940
  providerId: row.provider_id,
6646
6941
  ...row.provider_ids === null ? {} : { providerIds: normalizeModelProviderIds(JSON.parse(row.provider_ids)) },
6647
6942
  requestedModelId: row.requested_model_id,
6943
+ ...row.model_kind === null ? {} : { modelKind: row.model_kind },
6648
6944
  ...row.requested_reasoning === null ? {} : { requestedReasoning: requestedReasoningFromRow(row.requested_reasoning) },
6649
6945
  ...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
6650
6946
  responseId: row.response_id,
@@ -6701,7 +6997,8 @@ function usageFromRow(row) {
6701
6997
  ...row.cache_read_input_tokens === null ? {} : { cacheReadInputTokens: row.cache_read_input_tokens },
6702
6998
  ...row.cache_write_input_tokens === null ? {} : { cacheWriteInputTokens: row.cache_write_input_tokens },
6703
6999
  ...row.text_output_tokens === null ? {} : { textOutputTokens: row.text_output_tokens },
6704
- ...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens }
7000
+ ...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens },
7001
+ ...row.audio_duration_ms === null ? {} : { audioDurationMs: row.audio_duration_ms }
6705
7002
  };
6706
7003
  }
6707
7004
  function requestedReasoningFromRow(value) {
@@ -6882,6 +7179,7 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
6882
7179
  usage.cache_write_input_tokens,
6883
7180
  usage.text_output_tokens,
6884
7181
  usage.reasoning_output_tokens,
7182
+ usage.audio_duration_ms,
6885
7183
  usage.reporting_status,
6886
7184
  CASE
6887
7185
  WHEN direct_agent.kind = 'conversation' THEN 'agent'
@@ -6966,6 +7264,8 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
6966
7264
  CAST(SUM(text_output_tokens) AS TEXT) AS text_output_tokens_sum,
6967
7265
  CAST(COUNT(reasoning_output_tokens) AS TEXT) AS reasoning_output_tokens_count,
6968
7266
  CAST(SUM(reasoning_output_tokens) AS TEXT) AS reasoning_output_tokens_sum,
7267
+ CAST(COUNT(audio_duration_ms) AS TEXT) AS audio_duration_ms_count,
7268
+ CAST(SUM(audio_duration_ms) AS TEXT) AS audio_duration_ms_sum,
6969
7269
  CAST(SUM(CASE WHEN valuation_status = 'rated' THEN 1 ELSE 0 END) AS TEXT)
6970
7270
  AS rated_call_count,
6971
7271
  CAST(SUM(CASE WHEN valuation_status = 'unpriceable' THEN 1 ELSE 0 END) AS TEXT)
@@ -7011,7 +7311,8 @@ function aggregateFragmentFromRow(row) {
7011
7311
  cacheReadInputTokens: meterFragment(row.cache_read_input_tokens_count, row.cache_read_input_tokens_sum),
7012
7312
  cacheWriteInputTokens: meterFragment(row.cache_write_input_tokens_count, row.cache_write_input_tokens_sum),
7013
7313
  textOutputTokens: meterFragment(row.text_output_tokens_count, row.text_output_tokens_sum),
7014
- reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum)
7314
+ reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum),
7315
+ audioDurationMs: meterFragment(row.audio_duration_ms_count, row.audio_duration_ms_sum)
7015
7316
  },
7016
7317
  costs: {
7017
7318
  ...row.amount_currency === null || row.amount_high === null || row.amount_middle === null || row.amount_low === null ? {} : {
@@ -7067,6 +7368,7 @@ function usageFromRow2(row, requesterGroupIds) {
7067
7368
  providerId: row.provider_id,
7068
7369
  ...row.provider_ids === null ? {} : { providerIds: normalizeModelProviderIds2(JSON.parse(row.provider_ids)) },
7069
7370
  requestedModelId: row.requested_model_id,
7371
+ ...row.model_kind === null ? {} : { modelKind: row.model_kind },
7070
7372
  ...row.requested_reasoning === null ? {} : { requestedReasoning: requestedReasoningFromRow(row.requested_reasoning) },
7071
7373
  ...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
7072
7374
  responseId: row.response_id,
@@ -7079,6 +7381,7 @@ function usageFromRow2(row, requesterGroupIds) {
7079
7381
  ...row.cache_write_input_tokens === null ? {} : { cacheWriteInputTokens: row.cache_write_input_tokens },
7080
7382
  ...row.text_output_tokens === null ? {} : { textOutputTokens: row.text_output_tokens },
7081
7383
  ...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens },
7384
+ ...row.audio_duration_ms === null ? {} : { audioDurationMs: row.audio_duration_ms },
7082
7385
  reportingStatus: row.reporting_status
7083
7386
  },
7084
7387
  ...row.raw_usage === null ? {} : { rawUsage: JSON.parse(row.raw_usage) },
@@ -7219,6 +7522,7 @@ function required(value) {
7219
7522
  // src/ai-limit-storage.ts
7220
7523
  import {
7221
7524
  aiLimitAmountKey,
7525
+ aiLimitMeterApplies,
7222
7526
  aiLimitQuantityFromAmount,
7223
7527
  aiLimitReservationBuckets,
7224
7528
  aiLimitReservationRequestKey,
@@ -7327,8 +7631,9 @@ class SqliteAiLimitStorage {
7327
7631
  const unavailablePolicies = [];
7328
7632
  const unavailableReasons = new Set;
7329
7633
  for (const policy of enabledPolicies) {
7330
- if (!subjectKeys.has(aiLimitSubjectKey(policy.subject)))
7634
+ if (!subjectKeys.has(aiLimitSubjectKey(policy.subject)) || !aiLimitMeterApplies(policy.limit.meter, request.modelKind)) {
7331
7635
  continue;
7636
+ }
7332
7637
  const limit = normalizeAiLimitAmount(policy.limit);
7333
7638
  const estimate = estimates.get(aiLimitAmountKey(limit));
7334
7639
  const status = this.statusForPolicy(policy, request.period);
@@ -7362,7 +7667,7 @@ class SqliteAiLimitStorage {
7362
7667
  resetAt: new Date(Math.min(...exhaustedPolicies.map((status) => status.period.resetAt.getTime())))
7363
7668
  };
7364
7669
  }
7365
- const buckets = aiLimitReservationBuckets(enabledPolicies, request.subjects, request.estimates);
7670
+ const buckets = aiLimitReservationBuckets(enabledPolicies, request.subjects, request.estimates, request.modelKind);
7366
7671
  if (buckets.length === 0)
7367
7672
  return { status: "notRequired" };
7368
7673
  for (const bucket of buckets) {
@@ -7576,6 +7881,7 @@ class SqliteAiLimitStorage {
7576
7881
  SELECT usage.project_id, usage.id AS usage_record_id, usage.execution_id,
7577
7882
  usage.attempt, usage.call_id, usage.occurred_at,
7578
7883
  CAST(usage.total_tokens AS TEXT) AS total_tokens,
7884
+ usage.model_kind,
7579
7885
  executions.requested_by_user_id,
7580
7886
  executions.requested_by_service_account_id,
7581
7887
  valuations.status AS valuation_status,
@@ -7614,6 +7920,7 @@ class SqliteAiLimitStorage {
7614
7920
  SELECT usage.project_id, usage.id AS usage_record_id, usage.execution_id,
7615
7921
  usage.attempt, usage.call_id, usage.occurred_at,
7616
7922
  CAST(usage.total_tokens AS TEXT) AS total_tokens,
7923
+ usage.model_kind,
7617
7924
  executions.requested_by_user_id,
7618
7925
  executions.requested_by_service_account_id,
7619
7926
  valuations.status AS valuation_status,
@@ -7691,6 +7998,7 @@ function accountingEntryFromRow(row, subject, requesterGroupIds = subject?.type
7691
7998
  attempt: row.attempt,
7692
7999
  callId: row.call_id,
7693
8000
  occurredAt: new Date(row.occurred_at),
8001
+ ...row.model_kind === null ? {} : { modelKind: row.model_kind },
7694
8002
  ...totalTokens !== undefined && Number.isSafeInteger(totalTokens) ? { totalTokens } : {},
7695
8003
  requesterGroupIds,
7696
8004
  ...requester ? { requester } : {},
@@ -10410,9 +10718,12 @@ class SqliteExecutionStorage {
10410
10718
  FROM executions
10411
10719
  WHERE project_id = ? AND id = ?
10412
10720
  `).get(params.projectId, params.id);
10413
- return row ? executionRecordFromStorageRow(toStorageRow(row)) : null;
10721
+ return row ? sqliteExecutionRecord(row) : null;
10414
10722
  }
10415
10723
  }
10724
+ function sqliteExecutionRecord(row) {
10725
+ return executionRecordFromStorageRow(toStorageRow(row));
10726
+ }
10416
10727
  function toStorageRow(row) {
10417
10728
  return {
10418
10729
  projectId: row.project_id,
@@ -10444,6 +10755,43 @@ function isDuplicateExecutionError(error) {
10444
10755
  return error instanceof Error && error.message.includes("UNIQUE constraint failed: executions.project_id, executions.id");
10445
10756
  }
10446
10757
 
10758
+ // src/file-download-grant-storage.ts
10759
+ import { parseFileDownloadGrantRow } from "@sixb/core/internal/storage";
10760
+ var GRANT_COLUMNS = `
10761
+ project_id AS projectId, id, token_hash AS tokenHash, file, execution_id AS executionId,
10762
+ created_at AS createdAt, expires_at AS expiresAt, revoked_at AS revokedAt
10763
+ `;
10764
+
10765
+ class SqliteFileDownloadGrantStorage {
10766
+ db;
10767
+ constructor(connection) {
10768
+ this.db = connection.db;
10769
+ }
10770
+ async create(record) {
10771
+ this.db.query(`INSERT INTO file_download_grants (
10772
+ project_id, id, token_hash, file, execution_id, created_at, expires_at
10773
+ ) VALUES (?, ?, ?, ?, ?, ?, ?)`).run(record.projectId, record.id, record.tokenHash, JSON.stringify(record.file), record.executionId, record.createdAt.toISOString(), record.expiresAt.toISOString());
10774
+ }
10775
+ async findByTokenHash(input) {
10776
+ const row = this.db.query(`SELECT ${GRANT_COLUMNS} FROM file_download_grants WHERE project_id = ? AND token_hash = ?`).get(input.projectId, input.tokenHash);
10777
+ return row ? parseFileDownloadGrantRow(row) : null;
10778
+ }
10779
+ async revoke(input) {
10780
+ const row = this.db.query(`UPDATE file_download_grants SET revoked_at = COALESCE(revoked_at, ?)
10781
+ WHERE project_id = ? AND id = ?
10782
+ RETURNING ${GRANT_COLUMNS}`).get(input.revokedAt.toISOString(), input.projectId, input.id);
10783
+ return row ? parseFileDownloadGrantRow(row) : null;
10784
+ }
10785
+ async deleteExpired(input) {
10786
+ return this.db.query(`DELETE FROM file_download_grants WHERE rowid IN (
10787
+ SELECT rowid FROM file_download_grants
10788
+ WHERE project_id = ? AND expires_at < ?
10789
+ ORDER BY expires_at
10790
+ LIMIT ?
10791
+ )`).run(input.projectId, input.expiredBefore.toISOString(), input.limit).changes;
10792
+ }
10793
+ }
10794
+
10447
10795
  // src/file-upload-session-storage.ts
10448
10796
  import {
10449
10797
  DurableFileUploadSessions,
@@ -10584,6 +10932,8 @@ function exactContext(ctx) {
10584
10932
  }
10585
10933
  function compileObjectQueryInternal(projectId, query, ctx) {
10586
10934
  switch (query.kind) {
10935
+ case "rerank":
10936
+ throw new ObjectQueryExecutionError("reranking_requires_runtime", "Reranking must execute in the model runtime, not storage.");
10587
10937
  case "start":
10588
10938
  return compileStart(projectId, query, ctx);
10589
10939
  case "refs":
@@ -11716,6 +12066,7 @@ class SqliteObjectReader {
11716
12066
  source
11717
12067
  });
11718
12068
  return {
12069
+ total: readTotal(this.db, compiled),
11719
12070
  facets: params.facets.map((facet) => ({
11720
12071
  propertyId: facet.propertyId,
11721
12072
  buckets: readFacetBuckets(this.db, compiled, facet)
@@ -12496,9 +12847,7 @@ class SqliteObjectStorage {
12496
12847
  return this.reader.existsObjects(params);
12497
12848
  }
12498
12849
  async facetObjects(params) {
12499
- if (hasVectorProfile3(params.query))
12500
- return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
12501
- return this.reader.facetObjects(params);
12850
+ return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
12502
12851
  }
12503
12852
  async getByPrimaryId(params) {
12504
12853
  return this.reader.getByPrimaryId(params);
@@ -12547,13 +12896,14 @@ class SqliteObjectStorage {
12547
12896
  }
12548
12897
  async listByPrimaryIdPage(params) {
12549
12898
  assertReconciliationPageLimit(params.limit);
12899
+ const cursor = params.afterPrimaryId === undefined ? [] : [params.afterPrimaryId];
12550
12900
  const rows = this.db.query(`
12551
12901
  SELECT * FROM objects
12552
12902
  WHERE project_id = ? AND object_type_id = ?
12553
- AND (? IS NULL OR primary_id > ?)
12903
+ ${cursor.length > 0 ? "AND primary_id > ?" : ""}
12554
12904
  ORDER BY primary_id ASC
12555
12905
  LIMIT ?
12556
- `).all(params.projectId, params.objectTypeId, params.afterPrimaryId ?? null, params.afterPrimaryId ?? null, params.limit + 1);
12906
+ `).all(params.projectId, params.objectTypeId, ...cursor, params.limit + 1);
12557
12907
  const hasMore = rows.length > params.limit;
12558
12908
  const objects = rows.slice(0, params.limit).map((row) => rowToObject(row));
12559
12909
  const last = objects.at(-1);
@@ -12646,10 +12996,6 @@ function commitRecord(row) {
12646
12996
  requestHash: row.request_hash,
12647
12997
  executionId: row.execution_id,
12648
12998
  origin: parseJson2(row.origin),
12649
- ...row.requested_by === null ? {} : {
12650
- requestedBy: parseJson2(row.requested_by)
12651
- },
12652
- executor: parseJson2(row.executor),
12653
12999
  ontologyRevision: row.ontology_revision,
12654
13000
  ...row.projection_revision === null ? {} : { projectionRevision: row.projection_revision },
12655
13001
  ...row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash },
@@ -12659,9 +13005,15 @@ function commitRecord(row) {
12659
13005
  const result = parseJson2(row.result);
12660
13006
  return { ...base, intent, result };
12661
13007
  }
12662
- function outboxRecord(row) {
13008
+ function claimedOutboxDraft(row) {
13009
+ if (row.lease_id === null || row.lease_expires_at === null) {
13010
+ throw new MaterializationConflictError("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
13011
+ }
12663
13012
  return {
12664
- envelope: parseJson2(row.envelope),
13013
+ id: row.id,
13014
+ commitId: row.commit_id,
13015
+ commitOrdinal: row.commit_ordinal,
13016
+ event: parseJson2(row.event),
12665
13017
  availableAt: row.available_at,
12666
13018
  attempts: row.attempts,
12667
13019
  leaseId: row.lease_id,
@@ -12795,9 +13147,9 @@ import {
12795
13147
  linkRefKey as linkRefKey3,
12796
13148
  linkRefSortKey,
12797
13149
  linkScopeSortKey as linkScopeSortKey2,
12798
- MaterializationConflictError as MaterializationConflictError6,
12799
- MaterializationValidationError as MaterializationValidationError2,
12800
- objectRefKey as objectRefKey3,
13150
+ MaterializationConflictError as MaterializationConflictError7,
13151
+ MaterializationValidationError as MaterializationValidationError3,
13152
+ objectRefKey as objectRefKey4,
12801
13153
  objectRefSortKey,
12802
13154
  telemetryPointKey as telemetryPointKey2,
12803
13155
  telemetryPointSortKey as telemetryPointSortKey2
@@ -12808,24 +13160,24 @@ import {
12808
13160
  assertExpectedObjectRevision,
12809
13161
  assertMaterializationFinalizationCorrelation,
12810
13162
  assertMaterializationHeader,
12811
- assertMaterializationLaneCompletion,
12812
- assertPageRows,
12813
- assertPlanChunkCorrelations,
13163
+ assertPageRows as assertPageRows2,
12814
13164
  assertSourceActivationCorrelation,
13165
+ beginMaterializationApply,
13166
+ beginMaterializationVectorChanges,
13167
+ createCardinalityValidator,
12815
13168
  invalidCorrelation as invalidCorrelation2,
13169
+ materializationOutboxWrite,
13170
+ materializationPlanChunk,
12816
13171
  sameNonnegativeCounts as sameCounts,
12817
13172
  uniqueSorted
12818
13173
  } from "@sixb/core/internal/ontology-storage-provider";
12819
13174
 
12820
13175
  // src/ontology-storage/materialization-session.ts
12821
13176
  import {
12822
- MaterializationConflictError as MaterializationConflictError4
13177
+ MaterializationConflictError as MaterializationConflictError5
12823
13178
  } from "@sixb/core/internal/materialization";
12824
13179
  import {
12825
- correlateMaterializationChunk,
12826
13180
  duplicateMaterializationWork as duplicateWork,
12827
- invalidCorrelation,
12828
- materializationPlanItems,
12829
13181
  ProviderMaterializationSessionState,
12830
13182
  prepareMaterializationWork
12831
13183
  } from "@sixb/core/internal/ontology-storage-provider";
@@ -12936,18 +13288,35 @@ function stageSourceRoots(db, manifest, input) {
12936
13288
  }
12937
13289
  }
12938
13290
  }
12939
- function activateSourceRoots(db, projectId, candidate, activation) {
13291
+ function activateSourceRoots(db, projectId, commitId, candidate, activation) {
12940
13292
  if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
12941
13293
  throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
12942
13294
  if (candidate.base_materialization_id !== null && candidate.root_count === 0)
12943
13295
  return;
12944
13296
  const delta = candidate.base_materialization_id !== null;
12945
- db.query(`UPDATE ontology_source_roots SET retired_at = ?
12946
- WHERE retired_at IS NULL AND deleted = 0
12947
- AND version_id IN (SELECT version_id FROM ontology_sources
12948
- WHERE project_id = ? AND source_id = ? AND status IN ${PUBLISHED})
12949
- ${delta ? `AND project_id = ? AND root_key IN (
12950
- SELECT root_key FROM ontology_source_roots WHERE version_id = ?)` : ""}`).run(activation.updatedAt, projectId, activation.source.projectionId, ...delta ? [projectId, candidate.version_id] : []);
13297
+ const replaced = `retired_at IS NULL AND deleted = 0
13298
+ AND version_id IN (SELECT version_id FROM ontology_sources
13299
+ WHERE project_id = ? AND source_id = ? AND status IN ${PUBLISHED})
13300
+ ${delta ? `AND project_id = ? AND root_key IN (
13301
+ SELECT root_key FROM ontology_source_roots WHERE version_id = ?)` : ""}`;
13302
+ const replacedParams = [
13303
+ projectId,
13304
+ activation.source.projectionId,
13305
+ ...delta ? [projectId, candidate.version_id] : []
13306
+ ];
13307
+ db.query(`INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
13308
+ SELECT DISTINCT ?, ?, 'link', json_array(
13309
+ rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
13310
+ rows.target_primary_id
13311
+ )
13312
+ FROM (
13313
+ SELECT id FROM ontology_source_roots WHERE ${replaced}
13314
+ UNION ALL
13315
+ SELECT id FROM ontology_source_roots WHERE version_id = ?
13316
+ ) AS changed
13317
+ CROSS JOIN ontology_source_rows AS rows ON rows.root_id = changed.id
13318
+ WHERE rows.entity_kind = 'link'`).run(projectId, commitId, ...replacedParams, candidate.version_id);
13319
+ db.query(`UPDATE ontology_source_roots SET retired_at = ? WHERE ${replaced}`).run(activation.updatedAt, ...replacedParams);
12951
13320
  db.query(`UPDATE ontology_source_roots SET retired_at = ?
12952
13321
  WHERE version_id = ? AND deleted = 1`).run(activation.updatedAt, candidate.version_id);
12953
13322
  }
@@ -13000,7 +13369,6 @@ function replacementAssertionRows(db, input) {
13000
13369
 
13001
13370
  // src/ontology-storage/materialization-state.ts
13002
13371
  var SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
13003
- var SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
13004
13372
  function linkScopeRequestJson(scopes) {
13005
13373
  return canonicalJson(scopes.map(({ source, linkId }) => ({
13006
13374
  scopeSortKey: linkScopeSortKey(source, linkId),
@@ -13328,20 +13696,6 @@ class SqliteMaterializationStateReader {
13328
13696
  cursor = rows[rows.length - 1]?.sort_key ?? null;
13329
13697
  }
13330
13698
  }
13331
- *replacementIdentities(input) {
13332
- this.prepareReplacementIdentities(input);
13333
- let cursor = null;
13334
- while (true) {
13335
- const rows = this.replacementRows(input, cursor);
13336
- if (rows.length === 0)
13337
- break;
13338
- yield rows.map(replacementIdentity);
13339
- if (rows.length < input.pageRows)
13340
- break;
13341
- const last = rows[rows.length - 1];
13342
- cursor = { sortKey: last.sort_key, identityKey: last.identity_key };
13343
- }
13344
- }
13345
13699
  replacementObjectStates(sourceId, candidateMaterializationId, refs) {
13346
13700
  const base = this.objectStates(refs);
13347
13701
  const candidates = this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
@@ -13407,174 +13761,6 @@ class SqliteMaterializationStateReader {
13407
13761
  }
13408
13762
  return { owned, byMaterialization };
13409
13763
  }
13410
- prepareReplacementIdentities(input) {
13411
- if (input.kind === "object") {
13412
- this.prepareReplacementObjects(input);
13413
- return;
13414
- }
13415
- this.prepareReplacementLinks(input);
13416
- }
13417
- prepareReplacementObjects(input) {
13418
- const rows = replacementSourceRows(this.db, {
13419
- projectId: this.projectId,
13420
- sourceId: input.sourceId,
13421
- materializationId: input.candidateMaterializationId,
13422
- incremental: input.incremental ?? false
13423
- });
13424
- this.db.query(`
13425
- INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
13426
- session_id, entity_kind, identity_key, sort_key, diff_required
13427
- )
13428
- SELECT ?, 'object', json_array(object_type_id, primary_id),
13429
- ${objectSortExpression()}, 1
13430
- FROM (${rows.sql}) AS rows
13431
- WHERE entity_kind = 'object'
13432
- GROUP BY object_type_id, primary_id
13433
- `).run(input.sessionId, ...rows.values);
13434
- }
13435
- prepareReplacementLinks(input) {
13436
- const rows = replacementSourceRows(this.db, {
13437
- projectId: this.projectId,
13438
- sourceId: input.sourceId,
13439
- materializationId: input.candidateMaterializationId,
13440
- incremental: input.incremental ?? false
13441
- });
13442
- this.db.query(`
13443
- WITH incident_objects AS (
13444
- SELECT DISTINCT
13445
- json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
13446
- json_extract(payload, '$.ref.primaryId') AS primary_id
13447
- FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
13448
- WHERE session_id = ? AND kind = 'incident-object'
13449
- ), replacement_links AS (
13450
- SELECT source_type_id, source_primary_id AS source_id, link_id,
13451
- target_type_id, target_primary_id AS target_id
13452
- FROM (${rows.sql}) AS rows
13453
- WHERE entity_kind = 'link'
13454
- ), incident_links AS (
13455
- SELECT links.source_type_id, links.source_id, links.link_id,
13456
- links.target_type_id, links.target_id
13457
- FROM links
13458
- JOIN incident_objects
13459
- ON incident_objects.object_type_id = links.source_type_id
13460
- AND incident_objects.primary_id = links.source_id
13461
- WHERE links.project_id = ?
13462
- UNION
13463
- SELECT links.source_type_id, links.source_id, links.link_id,
13464
- links.target_type_id, links.target_id
13465
- FROM links
13466
- JOIN incident_objects
13467
- ON incident_objects.object_type_id = links.target_type_id
13468
- AND incident_objects.primary_id = links.target_id
13469
- WHERE links.project_id = ?
13470
- UNION
13471
- SELECT overrides.source_type_id,
13472
- overrides.source_primary_id AS source_id,
13473
- overrides.link_id, overrides.target_type_id,
13474
- overrides.target_primary_id AS target_id
13475
- FROM ontology_link_overrides AS overrides
13476
- JOIN incident_objects
13477
- ON incident_objects.object_type_id = overrides.source_type_id
13478
- AND incident_objects.primary_id = overrides.source_primary_id
13479
- WHERE overrides.project_id = ?
13480
- UNION
13481
- SELECT overrides.source_type_id,
13482
- overrides.source_primary_id AS source_id,
13483
- overrides.link_id, overrides.target_type_id,
13484
- overrides.target_primary_id AS target_id
13485
- FROM ontology_link_overrides AS overrides
13486
- JOIN incident_objects
13487
- ON incident_objects.object_type_id = overrides.target_type_id
13488
- AND incident_objects.primary_id = overrides.target_primary_id
13489
- WHERE overrides.project_id = ?
13490
- UNION
13491
- SELECT rows.source_type_id, rows.source_primary_id AS source_id,
13492
- rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
13493
- FROM incident_objects
13494
- CROSS JOIN ontology_source_rows AS rows
13495
- ON rows.entity_kind = 'link'
13496
- AND rows.source_type_id = incident_objects.object_type_id
13497
- AND rows.source_primary_id = incident_objects.primary_id
13498
- ${LIVE_SOURCE_ROWS_JOIN}
13499
- UNION
13500
- SELECT rows.source_type_id, rows.source_primary_id AS source_id,
13501
- rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
13502
- FROM incident_objects
13503
- CROSS JOIN ontology_source_rows AS rows
13504
- ON rows.entity_kind = 'link'
13505
- AND rows.target_type_id = incident_objects.object_type_id
13506
- AND rows.target_primary_id = incident_objects.primary_id
13507
- ${LIVE_SOURCE_ROWS_JOIN}
13508
- ), diff_links AS (
13509
- SELECT * FROM replacement_links
13510
- UNION SELECT * FROM incident_links
13511
- ), affected_scopes AS (
13512
- SELECT DISTINCT source_type_id, source_id, link_id FROM diff_links
13513
- ), all_links AS (
13514
- SELECT *, 1 AS diff_required FROM diff_links
13515
- UNION ALL
13516
- SELECT links.source_type_id, links.source_id, links.link_id,
13517
- links.target_type_id, links.target_id, 0 AS diff_required
13518
- FROM links
13519
- JOIN affected_scopes USING (source_type_id, source_id, link_id)
13520
- WHERE links.project_id = ?
13521
- UNION ALL
13522
- SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
13523
- overrides.link_id, overrides.target_type_id,
13524
- overrides.target_primary_id AS target_id, 0 AS diff_required
13525
- FROM ontology_link_overrides AS overrides
13526
- JOIN affected_scopes
13527
- ON affected_scopes.source_type_id = overrides.source_type_id
13528
- AND affected_scopes.source_id = overrides.source_primary_id
13529
- AND affected_scopes.link_id = overrides.link_id
13530
- WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
13531
- ), selected AS (
13532
- SELECT source_type_id, source_id, link_id, target_type_id, target_id,
13533
- ${linkSortExpression("all_links")} AS sort_key,
13534
- MAX(diff_required) AS diff_required
13535
- FROM all_links
13536
- GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
13537
- )
13538
- INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
13539
- session_id, entity_kind, identity_key, sort_key, diff_required
13540
- )
13541
- SELECT ?, 'link',
13542
- json_array(source_type_id, source_id, link_id, target_type_id, target_id),
13543
- sort_key, diff_required
13544
- FROM selected
13545
- `).run(input.sessionId, ...rows.values, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, input.sessionId);
13546
- }
13547
- replacementRows(input, cursor) {
13548
- return this.db.query(`
13549
- SELECT entity_kind, identity_key, sort_key, diff_required
13550
- FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
13551
- WHERE session_id = ? AND entity_kind = ?
13552
- AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
13553
- ORDER BY sort_key, identity_key
13554
- LIMIT ?
13555
- `).all(input.sessionId, input.kind, cursor?.sortKey ?? null, cursor?.sortKey ?? null, cursor?.identityKey ?? null, input.pageRows);
13556
- }
13557
- }
13558
- function replacementIdentity(row) {
13559
- const parts = parseJson2(row.identity_key);
13560
- if (row.entity_kind === "object") {
13561
- return {
13562
- kind: "object",
13563
- ref: { objectTypeId: parts[0], primaryId: parts[1] },
13564
- sortKey: row.sort_key,
13565
- diffRequired: true
13566
- };
13567
- }
13568
- return {
13569
- kind: "link",
13570
- ref: {
13571
- source: { objectTypeId: parts[0], primaryId: parts[1] },
13572
- linkId: parts[2],
13573
- target: { objectTypeId: parts[3], primaryId: parts[4] }
13574
- },
13575
- sortKey: row.sort_key,
13576
- diffRequired: row.diff_required === 1
13577
- };
13578
13764
  }
13579
13765
  function activeSourceMap(rows) {
13580
13766
  const result = new Map;
@@ -13702,20 +13888,709 @@ function storedPoint(row) {
13702
13888
  lastCommitId: row.last_commit_id
13703
13889
  };
13704
13890
  }
13705
- function objectSortExpression(alias) {
13706
- const prefix = alias ? `${alias}.` : "";
13707
- return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`;
13708
- }
13709
13891
  function linkSortExpression(alias) {
13710
13892
  const prefix = alias ? `${alias}.` : "";
13711
13893
  return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`;
13712
13894
  }
13713
13895
 
13896
+ // src/ontology-storage/replacement-plans.ts
13897
+ import {
13898
+ MaterializationConflictError as MaterializationConflictError4,
13899
+ MaterializationValidationError as MaterializationValidationError2,
13900
+ objectRefKey as objectRefKey2
13901
+ } from "@sixb/core/internal/materialization";
13902
+ import {
13903
+ assertNonblank as assertNonblank2,
13904
+ assertPageRows,
13905
+ assertPositiveInteger as assertPositiveInteger2,
13906
+ assertReplacementPlanCommit,
13907
+ canonicalJson as canonicalJson2,
13908
+ duplicateMaterializationWork,
13909
+ invalidCorrelation,
13910
+ materializationWorkColumns,
13911
+ prepareReplacementIdentities,
13912
+ workUniquenessKey
13913
+ } from "@sixb/core/internal/ontology-storage-provider";
13914
+ var SQLITE_PLAN_WORK_TABLE = "ontology_replacement_plan_work";
13915
+ var IDENTITIES = "ontology_replacement_plan_identities";
13916
+ var SORT_KEY = "LOWER(HEX(CAST(identity_key AS BLOB)))";
13917
+ function objectRevision(type, id) {
13918
+ const series = (alias) => `${alias}.project_id = $projectId AND ${alias}.object_type_id = ${type} AND ${alias}.object_id = ${id}`;
13919
+ return `json_array(
13920
+ (SELECT json_array(version, last_commit_id) FROM objects
13921
+ WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}),
13922
+ (SELECT last_commit_id FROM ontology_object_overrides
13923
+ WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}),
13924
+ (WITH RECURSIVE properties(property_id) AS (
13925
+ SELECT MIN(first.property_id) FROM timeseries AS first WHERE ${series("first")}
13926
+ UNION ALL
13927
+ SELECT (SELECT MIN(next.property_id) FROM timeseries AS next
13928
+ WHERE ${series("next")} AND next.property_id > properties.property_id)
13929
+ FROM properties WHERE properties.property_id IS NOT NULL
13930
+ )
13931
+ SELECT json_group_array(json((
13932
+ SELECT json_array(points.property_id, points.at, points.last_commit_id)
13933
+ FROM timeseries AS points
13934
+ WHERE ${series("points")} AND points.property_id = properties.property_id
13935
+ ORDER BY points.at DESC LIMIT 1
13936
+ ))) FROM properties WHERE properties.property_id IS NOT NULL)
13937
+ )`;
13938
+ }
13939
+ function endpointRevision(type, id) {
13940
+ return `CASE WHEN EXISTS (
13941
+ SELECT 1 FROM ${IDENTITIES} AS endpoint
13942
+ WHERE endpoint.version_id = $versionId AND endpoint.entity_kind = 'object'
13943
+ AND endpoint.identity_key = json_array(${type}, ${id})
13944
+ ) THEN ${objectRevision(type, id)}
13945
+ ELSE json(CASE WHEN EXISTS (
13946
+ SELECT 1 FROM objects
13947
+ WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}
13948
+ ) THEN 'true' ELSE 'false' END)
13949
+ END`;
13950
+ }
13951
+ function linkRevision(key) {
13952
+ const part = (index) => `json_extract(${key}, '$[${index}]')`;
13953
+ return `json_array(
13954
+ (SELECT last_commit_id FROM links
13955
+ WHERE project_id = $projectId AND source_type_id = ${part(0)} AND source_id = ${part(1)}
13956
+ AND link_id = ${part(2)} AND target_type_id = ${part(3)} AND target_id = ${part(4)}),
13957
+ (SELECT overrides.last_commit_id FROM ontology_link_overrides AS overrides
13958
+ WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'edge'
13959
+ AND overrides.identity_key = ${key}),
13960
+ (SELECT overrides.last_commit_id FROM ontology_link_overrides AS overrides
13961
+ WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
13962
+ AND overrides.identity_key = json_array(${part(0)}, ${part(1)}, ${part(2)})),
13963
+ (SELECT MAX(roots.id) FROM ontology_source_roots AS roots
13964
+ CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
13965
+ AND versions.status IN ${PUBLISHED}
13966
+ WHERE roots.project_id = $projectId AND roots.retired_at IS NULL AND roots.deleted = 0
13967
+ AND roots.root_key IN (
13968
+ '["link",' || substr(${key}, 2),
13969
+ '["object",' || substr(json_array(${part(0)}, ${part(1)}), 2)
13970
+ )
13971
+ AND EXISTS (
13972
+ -- \`+\` keeps the lookup on the root's few rows, not on every row of a hub source.
13973
+ SELECT 1 FROM ontology_source_rows AS rows
13974
+ WHERE rows.root_id = roots.id AND +rows.entity_kind = 'link'
13975
+ AND +rows.source_type_id = ${part(0)} AND +rows.source_primary_id = ${part(1)}
13976
+ AND +rows.link_id = ${part(2)} AND +rows.target_type_id = ${part(3)}
13977
+ AND +rows.target_primary_id = ${part(4)}
13978
+ )),
13979
+ ${endpointRevision(part(0), part(1))},
13980
+ ${endpointRevision(part(3), part(4))}
13981
+ )`;
13982
+ }
13983
+ function identityRevision(kind) {
13984
+ const key = "identities.identity_key";
13985
+ return kind === "object" ? objectRevision(`json_extract(${key}, '$[0]')`, `json_extract(${key}, '$[1]')`) : linkRevision(key);
13986
+ }
13987
+ function linkKeys(relation) {
13988
+ return `SELECT json_array(source_type_id, source_id, link_id, target_type_id, target_id)
13989
+ AS identity_key, json_array(source_type_id, source_id) AS source_key,
13990
+ json_array(target_type_id, target_id) AS target_key
13991
+ FROM ${relation}`;
13992
+ }
13993
+ var NEXT_EXPANSION = `(
13994
+ SELECT expansions FROM ontology_replacement_plans WHERE version_id = $versionId
13995
+ )`;
13996
+ var INSERT_INCIDENT_LINKS = `
13997
+ INSERT INTO ${IDENTITIES} (
13998
+ version_id, entity_kind, identity_key, sort_key, diff_required, expand_at, source_key,
13999
+ target_key
14000
+ )
14001
+ SELECT $versionId, 'link', identity_key, ${SORT_KEY}, 1, ${NEXT_EXPANSION}, source_key,
14002
+ target_key
14003
+ FROM (${linkKeys("incident_links")}) WHERE true
14004
+ ON CONFLICT (version_id, entity_kind, identity_key) DO UPDATE
14005
+ SET diff_required = 1, read_revision = NULL, planned_revision = NULL, classified = 0,
14006
+ change = NULL, expand_at = excluded.expand_at
14007
+ WHERE ${IDENTITIES}.diff_required = 0`;
14008
+ var INSERT_MEMBER_LINKS = `
14009
+ INSERT INTO ${IDENTITIES} (
14010
+ version_id, entity_kind, identity_key, sort_key, diff_required, source_key, target_key
14011
+ )
14012
+ SELECT $versionId, 'link', identity_key, ${SORT_KEY}, 0, source_key, target_key
14013
+ FROM (${linkKeys("members")}) WHERE true
14014
+ ON CONFLICT (version_id, entity_kind, identity_key) DO NOTHING`;
14015
+ var LIVE_ROWS = `CROSS JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
14016
+ AND roots.project_id = $projectId AND roots.retired_at IS NULL AND roots.deleted = 0
14017
+ CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
14018
+ AND versions.status IN ${PUBLISHED}`;
14019
+ var EXPAND_LINK_STATEMENTS = {
14020
+ incident: `WITH incident_objects AS MATERIALIZED (
14021
+ SELECT json_extract(work.payload, '$.ref.objectTypeId') AS object_type_id,
14022
+ json_extract(work.payload, '$.ref.primaryId') AS primary_id
14023
+ FROM ${IDENTITIES} AS objects
14024
+ CROSS JOIN ${SQLITE_PLAN_WORK_TABLE} AS work ON work.work_id = $versionId
14025
+ AND work.unique_key = 'incident-object:' || objects.identity_key
14026
+ WHERE objects.version_id = $versionId AND objects.entity_kind = 'object'
14027
+ AND objects.expand_at = ${NEXT_EXPANSION}
14028
+ ), incident_links AS (
14029
+ SELECT links.source_type_id, links.source_id, links.link_id,
14030
+ links.target_type_id, links.target_id
14031
+ FROM incident_objects CROSS JOIN links
14032
+ ON links.project_id = $projectId
14033
+ AND links.source_type_id = incident_objects.object_type_id
14034
+ AND links.source_id = incident_objects.primary_id
14035
+ UNION
14036
+ SELECT links.source_type_id, links.source_id, links.link_id,
14037
+ links.target_type_id, links.target_id
14038
+ FROM incident_objects CROSS JOIN links INDEXED BY idx_links_target
14039
+ ON links.project_id = $projectId
14040
+ AND links.target_type_id = incident_objects.object_type_id
14041
+ AND links.target_id = incident_objects.primary_id
14042
+ UNION
14043
+ SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
14044
+ overrides.target_type_id, overrides.target_primary_id
14045
+ FROM incident_objects CROSS JOIN ontology_link_overrides AS overrides
14046
+ ON overrides.project_id = $projectId AND overrides.identity_kind IN ('edge', 'slot')
14047
+ AND overrides.source_type_id = incident_objects.object_type_id
14048
+ AND overrides.source_primary_id = incident_objects.primary_id
14049
+ UNION
14050
+ SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
14051
+ overrides.target_type_id, overrides.target_primary_id
14052
+ FROM incident_objects
14053
+ CROSS JOIN ontology_link_overrides AS overrides INDEXED BY idx_ontology_link_overrides_target
14054
+ ON overrides.project_id = $projectId
14055
+ AND overrides.target_type_id = incident_objects.object_type_id
14056
+ AND overrides.target_primary_id = incident_objects.primary_id
14057
+ UNION
14058
+ SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
14059
+ rows.target_type_id, rows.target_primary_id
14060
+ FROM incident_objects CROSS JOIN ontology_source_rows AS rows
14061
+ ON rows.entity_kind = 'link'
14062
+ AND rows.source_type_id = incident_objects.object_type_id
14063
+ AND rows.source_primary_id = incident_objects.primary_id
14064
+ ${LIVE_ROWS}
14065
+ UNION
14066
+ SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
14067
+ rows.target_type_id, rows.target_primary_id
14068
+ FROM incident_objects CROSS JOIN ontology_source_rows AS rows
14069
+ ON rows.entity_kind = 'link'
14070
+ AND rows.target_type_id = incident_objects.object_type_id
14071
+ AND rows.target_primary_id = incident_objects.primary_id
14072
+ ${LIVE_ROWS}
14073
+ )
14074
+ ${INSERT_INCIDENT_LINKS}`,
14075
+ members: `WITH scopes AS (
14076
+ SELECT DISTINCT json_extract(identity_key, '$[0]') AS source_type_id,
14077
+ json_extract(identity_key, '$[1]') AS source_id,
14078
+ json_extract(identity_key, '$[2]') AS link_id
14079
+ FROM ${IDENTITIES}
14080
+ WHERE version_id = $versionId AND entity_kind = 'link' AND diff_required = 1
14081
+ AND expand_at = ${NEXT_EXPANSION}
14082
+ ), members AS (
14083
+ SELECT links.source_type_id, links.source_id, links.link_id,
14084
+ links.target_type_id, links.target_id
14085
+ FROM scopes CROSS JOIN links
14086
+ ON links.project_id = $projectId
14087
+ AND links.source_type_id = scopes.source_type_id
14088
+ AND links.source_id = scopes.source_id AND links.link_id = scopes.link_id
14089
+ UNION
14090
+ SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
14091
+ overrides.target_type_id, overrides.target_primary_id
14092
+ FROM scopes CROSS JOIN ontology_link_overrides AS overrides
14093
+ ON overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
14094
+ AND overrides.source_type_id = scopes.source_type_id
14095
+ AND overrides.source_primary_id = scopes.source_id
14096
+ AND overrides.link_id = scopes.link_id
14097
+ )
14098
+ ${INSERT_MEMBER_LINKS}`
14099
+ };
14100
+ var TOUCHED = `touched AS MATERIALIZED (
14101
+ SELECT DISTINCT touches.entity_kind, touches.identity_key
14102
+ FROM ontology_commits AS commits
14103
+ CROSS JOIN ontology_commit_touches AS touches
14104
+ ON touches.project_id = commits.project_id AND touches.commit_id = commits.id
14105
+ WHERE commits.rowid > (
14106
+ SELECT watermark FROM ontology_replacement_plans WHERE version_id = $versionId
14107
+ ) AND +commits.project_id = $projectId
14108
+ )`;
14109
+ var TOUCHED_REFRESH_STATEMENTS = {
14110
+ object: `WITH ${TOUCHED}
14111
+ UPDATE ${IDENTITIES} AS identities
14112
+ SET read_revision = NULL, planned_revision = NULL, classified = 0, change = NULL
14113
+ WHERE identities.version_id = $versionId AND identities.entity_kind = 'object'
14114
+ AND identities.identity_key IN (SELECT identity_key FROM touched WHERE entity_kind = 'object')
14115
+ AND identities.planned_revision IS NOT ${identityRevision("object")}`,
14116
+ link: `WITH ${TOUCHED}, candidates AS (
14117
+ SELECT identity_key FROM touched WHERE entity_kind = 'link'
14118
+ UNION
14119
+ SELECT links.identity_key FROM touched CROSS JOIN ${IDENTITIES} AS links
14120
+ ON links.version_id = $versionId
14121
+ AND links.source_key = json_array(
14122
+ json_extract(touched.identity_key, '$[0]'), json_extract(touched.identity_key, '$[1]')
14123
+ )
14124
+ WHERE touched.entity_kind IN ('object', 'scope')
14125
+ UNION
14126
+ SELECT links.identity_key FROM touched CROSS JOIN ${IDENTITIES} AS links
14127
+ ON links.version_id = $versionId AND links.target_key = touched.identity_key
14128
+ WHERE touched.entity_kind = 'object'
14129
+ )
14130
+ UPDATE ${IDENTITIES} AS identities
14131
+ SET read_revision = NULL, planned_revision = NULL, classified = 0, change = NULL
14132
+ WHERE identities.version_id = $versionId AND identities.entity_kind = 'link'
14133
+ AND identities.identity_key IN (SELECT identity_key FROM candidates)
14134
+ AND identities.planned_revision IS NOT ${identityRevision("link")}`
14135
+ };
14136
+ var TOUCHED_LINKS = `touched_links AS (
14137
+ SELECT identity_key, json_extract(identity_key, '$[0]') AS source_type_id,
14138
+ json_extract(identity_key, '$[1]') AS source_id, json_extract(identity_key, '$[2]') AS link_id,
14139
+ json_extract(identity_key, '$[3]') AS target_type_id,
14140
+ json_extract(identity_key, '$[4]') AS target_id
14141
+ FROM touched WHERE entity_kind = 'link'
14142
+ ), slot_targets AS (
14143
+ SELECT overrides.source_type_id, overrides.source_primary_id AS source_id, overrides.link_id,
14144
+ overrides.target_type_id, overrides.target_primary_id AS target_id
14145
+ FROM touched CROSS JOIN ontology_link_overrides AS overrides
14146
+ ON overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
14147
+ AND overrides.identity_key = touched.identity_key
14148
+ WHERE touched.entity_kind = 'scope'
14149
+ )`;
14150
+ var LINK_COLUMNS = "source_type_id, source_id, link_id, target_type_id, target_id";
14151
+ var EFFECTIVE_TOUCHED_LINK = `EXISTS (
14152
+ SELECT 1 FROM links AS stored
14153
+ WHERE stored.project_id = $projectId AND stored.source_type_id = touched_links.source_type_id
14154
+ AND stored.source_id = touched_links.source_id AND stored.link_id = touched_links.link_id
14155
+ AND stored.target_type_id = touched_links.target_type_id
14156
+ AND stored.target_id = touched_links.target_id
14157
+ )`;
14158
+ var EXPAND_TOUCHED_STATEMENTS = {
14159
+ incident: `WITH ${TOUCHED}, ${TOUCHED_LINKS}, incident_links AS (
14160
+ SELECT candidates.* FROM (
14161
+ SELECT ${LINK_COLUMNS} FROM touched_links
14162
+ WHERE ${EFFECTIVE_TOUCHED_LINK} OR EXISTS (
14163
+ SELECT 1 FROM ontology_link_overrides AS overrides
14164
+ WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'edge'
14165
+ AND overrides.identity_key = touched_links.identity_key
14166
+ ) OR EXISTS (
14167
+ SELECT 1 FROM ontology_source_rows AS rows ${LIVE_ROWS}
14168
+ WHERE rows.entity_kind = 'link' AND rows.source_type_id = touched_links.source_type_id
14169
+ AND rows.source_primary_id = touched_links.source_id
14170
+ AND rows.link_id = touched_links.link_id
14171
+ AND rows.target_type_id = touched_links.target_type_id
14172
+ AND rows.target_primary_id = touched_links.target_id
14173
+ )
14174
+ UNION
14175
+ SELECT ${LINK_COLUMNS} FROM slot_targets
14176
+ ) AS candidates
14177
+ WHERE EXISTS (
14178
+ SELECT 1 FROM ${SQLITE_PLAN_WORK_TABLE} AS work
14179
+ WHERE work.work_id = $versionId AND work.unique_key IN (
14180
+ 'incident-object:' || json_array(candidates.source_type_id, candidates.source_id),
14181
+ 'incident-object:' || json_array(candidates.target_type_id, candidates.target_id)
14182
+ )
14183
+ )
14184
+ )
14185
+ ${INSERT_INCIDENT_LINKS}`,
14186
+ members: `WITH ${TOUCHED}, ${TOUCHED_LINKS}, members AS (
14187
+ SELECT candidates.* FROM (
14188
+ SELECT ${LINK_COLUMNS} FROM touched_links WHERE ${EFFECTIVE_TOUCHED_LINK}
14189
+ UNION
14190
+ SELECT ${LINK_COLUMNS} FROM slot_targets
14191
+ ) AS candidates
14192
+ WHERE EXISTS (
14193
+ SELECT 1 FROM ${IDENTITIES} AS changed
14194
+ WHERE changed.version_id = $versionId
14195
+ AND changed.source_key = json_array(candidates.source_type_id, candidates.source_id)
14196
+ AND changed.diff_required = 1
14197
+ AND json_extract(changed.identity_key, '$[2]') = candidates.link_id
14198
+ )
14199
+ )
14200
+ ${INSERT_MEMBER_LINKS}`
14201
+ };
14202
+ var PURGE_TOUCHES_STATEMENT = `DELETE FROM ontology_commit_touches WHERE rowid IN (
14203
+ SELECT touches.rowid FROM ontology_commit_touches AS touches
14204
+ CROSS JOIN ontology_commits AS commits
14205
+ ON commits.project_id = touches.project_id AND commits.id = touches.commit_id
14206
+ WHERE touches.project_id = $projectId AND commits.rowid <= COALESCE(
14207
+ (SELECT MIN(plans.watermark) FROM ontology_replacement_plans AS plans
14208
+ JOIN ontology_sources AS versions USING (version_id)
14209
+ WHERE plans.project_id = $projectId AND versions.status = 'ready'),
14210
+ (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits)
14211
+ )
14212
+ LIMIT $limit
14213
+ )`;
14214
+
14215
+ class SqliteOntologyReplacementPlanStorage {
14216
+ db;
14217
+ runRootOperation;
14218
+ constructor(db, runRootOperation) {
14219
+ this.db = db;
14220
+ this.runRootOperation = runRootOperation;
14221
+ }
14222
+ async open(input) {
14223
+ return this.runRootOperation(() => {
14224
+ const { commit } = input;
14225
+ assertReplacementPlanCommit(commit);
14226
+ const candidate = this.requireCandidate(input);
14227
+ const active = this.db.query(`SELECT materialization_id, last_commit_id FROM ontology_sources
14228
+ WHERE project_id = ? AND source_id = ? AND status = 'active'`).get(input.projectId, input.source.projectionId);
14229
+ const replacedMaterializationId = active?.materialization_id ?? null;
14230
+ const replacedLastCommitId = active?.last_commit_id ?? null;
14231
+ const existing = planRow(this.db, candidate.version_id);
14232
+ if (existing?.replaced_materialization_id === replacedMaterializationId && existing.replaced_last_commit_id === replacedLastCommitId) {
14233
+ return { committedAt: existing.committed_at };
14234
+ }
14235
+ if (existing)
14236
+ deletePlan(this.db, candidate.version_id);
14237
+ this.db.query(`INSERT INTO ontology_replacement_plans (
14238
+ version_id, project_id, commit_id, committed_at, replaced_materialization_id,
14239
+ replaced_last_commit_id, watermark
14240
+ ) VALUES (?, ?, ?, ?, ?, ?, (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits))`).run(candidate.version_id, input.projectId, commit.id, commit.committedAt, replacedMaterializationId, replacedLastCommitId);
14241
+ const rows = replacementSourceRows(this.db, {
14242
+ projectId: input.projectId,
14243
+ sourceId: input.source.projectionId,
14244
+ materializationId: input.materializationId,
14245
+ incremental: candidate.base_materialization_id !== null
14246
+ });
14247
+ this.db.query(`INSERT INTO ${IDENTITIES} (
14248
+ version_id, entity_kind, identity_key, sort_key, diff_required, expand_at,
14249
+ source_key, target_key
14250
+ )
14251
+ SELECT ?, entity_kind, identity_key, ${SORT_KEY}, 1,
14252
+ CASE entity_kind WHEN 'link' THEN 0 END, source_key, target_key
14253
+ FROM (
14254
+ SELECT entity_kind, CASE entity_kind
14255
+ WHEN 'object' THEN json_array(object_type_id, primary_id)
14256
+ ELSE json_array(
14257
+ source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
14258
+ )
14259
+ END AS identity_key,
14260
+ CASE entity_kind
14261
+ WHEN 'link' THEN json_array(source_type_id, source_primary_id)
14262
+ END AS source_key,
14263
+ CASE entity_kind
14264
+ WHEN 'link' THEN json_array(target_type_id, target_primary_id)
14265
+ END AS target_key
14266
+ FROM (${rows.sql}) AS rows
14267
+ )
14268
+ GROUP BY entity_kind, identity_key`).run(candidate.version_id, ...rows.values);
14269
+ return { committedAt: commit.committedAt };
14270
+ });
14271
+ }
14272
+ async* streamState(input) {
14273
+ assertPageRows(input.pageRows);
14274
+ await this.runRootOperation(() => {
14275
+ const { candidate } = this.requirePlan(input);
14276
+ if (input.entityKind === "object" && candidate.projection_kind !== "object") {
14277
+ throw new MaterializationConflictError4("source-materialization", "Link projection replacement cannot stream object state.");
14278
+ }
14279
+ if (input.entityKind === "link") {
14280
+ if (unplannedCount(this.db, candidate.version_id, "object") > 0) {
14281
+ throw new MaterializationConflictError4("effective-state", "Object projection replacement must plan every object before its links.");
14282
+ }
14283
+ this.expandLinks(candidate.version_id, input.projectId);
14284
+ }
14285
+ });
14286
+ let after = "";
14287
+ while (true) {
14288
+ const page = await this.runRootOperation(() => {
14289
+ const { candidate } = this.requirePlan(input);
14290
+ const rows = this.db.query(`SELECT identity_key, sort_key, diff_required FROM ${IDENTITIES}
14291
+ WHERE version_id = ? AND entity_kind = ? AND planned_revision IS NULL AND sort_key > ?
14292
+ ORDER BY sort_key
14293
+ LIMIT ?`).all(candidate.version_id, input.entityKind, after, input.pageRows);
14294
+ if (rows.length === 0)
14295
+ return null;
14296
+ after = rows[rows.length - 1].sort_key;
14297
+ const state = this.readPage(input, candidate, rows);
14298
+ this.db.query(`UPDATE ${IDENTITIES} AS identities SET read_revision = ${identityRevision(input.entityKind)}
14299
+ WHERE identities.version_id = $versionId AND identities.entity_kind = $entityKind
14300
+ AND identities.identity_key IN (SELECT value FROM json_each($keys))`).run({
14301
+ $projectId: input.projectId,
14302
+ $versionId: candidate.version_id,
14303
+ $entityKind: input.entityKind,
14304
+ $keys: JSON.stringify(rows.map((row) => row.identity_key))
14305
+ });
14306
+ return state;
14307
+ });
14308
+ if (!page)
14309
+ return;
14310
+ yield page;
14311
+ }
14312
+ }
14313
+ async stage(input) {
14314
+ await this.runRootOperation(() => {
14315
+ const { plan, candidate } = this.requirePlan(input);
14316
+ const identities = prepareReplacementIdentities(input, {
14317
+ commitId: plan.commit_id,
14318
+ committedAt: plan.committed_at
14319
+ });
14320
+ const versionId = candidate.version_id;
14321
+ this.db.run("SAVEPOINT sixb_replacement_plan_stage");
14322
+ let recordKey;
14323
+ try {
14324
+ const known = this.db.query(`SELECT read_revision, planned_revision FROM ${IDENTITIES}
14325
+ WHERE version_id = ? AND entity_kind = ? AND identity_key = ?`);
14326
+ const insert = this.db.query(`INSERT INTO ${SQLITE_PLAN_WORK_TABLE} (
14327
+ work_id, entity_kind, identity_key, record_key, unique_key, kind, lane,
14328
+ major_order, minor_order, sort_one, sort_two,
14329
+ classification_entity_kind, classification_identity_key,
14330
+ cardinality_view, cardinality_occupied, cardinality_source_type_id,
14331
+ cardinality_source_primary_id, cardinality_link_id,
14332
+ cardinality_target_type_id, cardinality_target_primary_id, payload
14333
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, json(?))`);
14334
+ const planned = this.db.query(`UPDATE ${IDENTITIES}
14335
+ SET planned_revision = read_revision, classified = ?, change = ?,
14336
+ expand_at = CASE entity_kind
14337
+ WHEN 'object' THEN (
14338
+ SELECT expansions FROM ontology_replacement_plans WHERE version_id = ?
14339
+ )
14340
+ ELSE expand_at
14341
+ END
14342
+ WHERE version_id = ? AND entity_kind = ? AND identity_key = ?`);
14343
+ for (const identity of identities) {
14344
+ const row = known.get(versionId, identity.kind, identity.key);
14345
+ if (row?.read_revision == null || row.planned_revision !== null) {
14346
+ throw new MaterializationValidationError2(`Replacement identity ${identity.entityKey} is not streamed and still to plan.`);
14347
+ }
14348
+ for (const record of identity.records) {
14349
+ recordKey = record.recordKey;
14350
+ const columns = materializationWorkColumns(record);
14351
+ const classification = record.kind === "classification" ? record : null;
14352
+ const cardinality = record.kind === "cardinality" ? record : null;
14353
+ insert.run(versionId, identity.kind, identity.key, record.recordKey, workUniquenessKey(record), record.kind, columns.lane, columns.majorOrder, columns.minorOrder, columns.sortOne, columns.sortTwo, classification?.entityKind ?? null, classification?.identityKey ?? null, cardinality?.view ?? null, cardinality ? Number(cardinality.occupied) : null, cardinality?.ref.source.objectTypeId ?? null, cardinality?.ref.source.primaryId ?? null, cardinality?.ref.linkId ?? null, cardinality?.ref.target.objectTypeId ?? null, cardinality?.ref.target.primaryId ?? null, canonicalJson2(record));
14354
+ }
14355
+ planned.run(Number(identity.classified), identity.change, versionId, versionId, identity.kind, identity.key);
14356
+ }
14357
+ this.db.run("RELEASE SAVEPOINT sixb_replacement_plan_stage");
14358
+ } catch (error) {
14359
+ this.db.run("ROLLBACK TO SAVEPOINT sixb_replacement_plan_stage");
14360
+ this.db.run("RELEASE SAVEPOINT sixb_replacement_plan_stage");
14361
+ if (isSqliteConstraintError(error))
14362
+ throw duplicateMaterializationWork(recordKey);
14363
+ throw error;
14364
+ }
14365
+ });
14366
+ }
14367
+ async refresh(input) {
14368
+ return this.runRootOperation(() => {
14369
+ const { plan, candidate } = this.requirePlan(input);
14370
+ const versionId = candidate.version_id;
14371
+ const pending = unplannedCount(this.db, versionId);
14372
+ if (pending > 0)
14373
+ return { fresh: false, unplanned: pending };
14374
+ if (!committedSince(this.db, plan, input.projectId)) {
14375
+ return { fresh: true, counts: replacementPlanCounts(this.db, versionId) };
14376
+ }
14377
+ const params = { $projectId: input.projectId, $versionId: versionId };
14378
+ let unplanned = 0;
14379
+ for (const kind of ["object", "link"]) {
14380
+ const stale = this.db.query(TOUCHED_REFRESH_STATEMENTS[kind]).run(params).changes;
14381
+ if (stale === 0)
14382
+ continue;
14383
+ dropUnplannedWork(this.db, versionId, kind);
14384
+ unplanned += stale;
14385
+ }
14386
+ unplanned += this.expandTouched(versionId, input.projectId);
14387
+ this.db.query(`UPDATE ontology_replacement_plans
14388
+ SET watermark = (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits)
14389
+ WHERE version_id = ?`).run(versionId);
14390
+ if (unplanned > 0)
14391
+ return { fresh: false, unplanned };
14392
+ return { fresh: true, counts: replacementPlanCounts(this.db, versionId) };
14393
+ });
14394
+ }
14395
+ async purge(input) {
14396
+ return this.runRootOperation(() => {
14397
+ assertNonblank2(input.projectId, "Replacement plan purge project id");
14398
+ assertPositiveInteger2(input.limit, "Replacement plan purge limit");
14399
+ const spentPlan = this.db.query(`SELECT plans.version_id FROM ontology_replacement_plans AS plans
14400
+ JOIN ontology_sources AS versions USING (version_id)
14401
+ WHERE plans.project_id = ? AND versions.status <> 'ready'
14402
+ ORDER BY plans.version_id
14403
+ LIMIT 1`);
14404
+ let deleted = 0;
14405
+ while (deleted < input.limit) {
14406
+ const spent = spentPlan.get(input.projectId);
14407
+ if (!spent)
14408
+ break;
14409
+ for (const [table, column] of [
14410
+ [SQLITE_PLAN_WORK_TABLE, "work_id"],
14411
+ [IDENTITIES, "version_id"]
14412
+ ]) {
14413
+ deleted += this.db.query(`DELETE FROM ${table} WHERE rowid IN (
14414
+ SELECT rowid FROM ${table} WHERE ${column} = ? LIMIT ?
14415
+ )`).run(spent.version_id, input.limit - deleted).changes;
14416
+ if (deleted >= input.limit)
14417
+ return deleted;
14418
+ }
14419
+ this.db.query(`DELETE FROM ontology_replacement_plans WHERE version_id = ?`).run(spent.version_id);
14420
+ deleted += 1;
14421
+ }
14422
+ if (deleted >= input.limit)
14423
+ return deleted;
14424
+ return deleted + this.db.query(PURGE_TOUCHES_STATEMENT).run({ $projectId: input.projectId, $limit: input.limit - deleted }).changes;
14425
+ });
14426
+ }
14427
+ readPage(input, candidate, rows) {
14428
+ const reader = new SqliteMaterializationStateReader(this.db, input.projectId);
14429
+ if (input.entityKind === "object") {
14430
+ const refs = rows.map((row) => {
14431
+ const [objectTypeId, primaryId] = parseJson2(row.identity_key);
14432
+ return { objectTypeId, primaryId };
14433
+ });
14434
+ return {
14435
+ objects: reader.replacementObjectStates(input.source.projectionId, input.materializationId, refs),
14436
+ links: [],
14437
+ endpoints: []
14438
+ };
14439
+ }
14440
+ const identities = rows.map((row) => {
14441
+ const parts = parseJson2(row.identity_key);
14442
+ const ref = {
14443
+ source: { objectTypeId: parts[0], primaryId: parts[1] },
14444
+ linkId: parts[2],
14445
+ target: { objectTypeId: parts[3], primaryId: parts[4] }
14446
+ };
14447
+ return {
14448
+ ref,
14449
+ sortKey: row.sort_key,
14450
+ diffRequired: row.diff_required === 1
14451
+ };
14452
+ });
14453
+ const previous = this.db.query(`SELECT materialization_id FROM ontology_sources
14454
+ WHERE project_id = ? AND source_id = ? AND status = 'active'`).get(input.projectId, input.source.projectionId);
14455
+ const links = reader.replacementLinkStates(input.source.projectionId, input.materializationId, [input.materializationId, ...previous ? [previous.materialization_id] : []], identities, candidate.base_materialization_id !== null);
14456
+ return {
14457
+ objects: [],
14458
+ links,
14459
+ endpoints: this.endpoints(candidate.version_id, input.projectId, identities.flatMap(({ ref }) => [ref.source, ref.target]))
14460
+ };
14461
+ }
14462
+ endpoints(versionId, projectId, refs) {
14463
+ const unique = [...new Map(refs.map((ref) => [objectRefKey2(ref), ref])).values()];
14464
+ const rows = this.db.query(`SELECT requested.value AS identity_key, (
14465
+ SELECT json_extract(work.payload, '$.exists') FROM ${SQLITE_PLAN_WORK_TABLE} AS work
14466
+ WHERE work.work_id = ? AND work.unique_key = 'object-existence:' || requested.value
14467
+ ) AS planned, EXISTS (
14468
+ SELECT 1 FROM objects
14469
+ WHERE project_id = ? AND object_type_id = json_extract(requested.value, '$[0]')
14470
+ AND primary_id = json_extract(requested.value, '$[1]')
14471
+ ) AS effective
14472
+ FROM json_each(?) AS requested`).all(versionId, projectId, JSON.stringify(unique.map((ref) => objectRefKey2(ref))));
14473
+ const existence = new Map(rows.map((row) => [row.identity_key, (row.planned ?? row.effective) === 1]));
14474
+ return unique.map((ref) => ({ ref, exists: existence.get(objectRefKey2(ref)) ?? false }));
14475
+ }
14476
+ expandLinks(versionId, projectId) {
14477
+ const added = this.expand(versionId, projectId, EXPAND_LINK_STATEMENTS);
14478
+ this.db.query(`UPDATE ontology_replacement_plans SET expansions = expansions + 1 WHERE version_id = ?`).run(versionId);
14479
+ return added;
14480
+ }
14481
+ expandTouched(versionId, projectId) {
14482
+ return this.expand(versionId, projectId, EXPAND_TOUCHED_STATEMENTS);
14483
+ }
14484
+ expand(versionId, projectId, statements) {
14485
+ const run = (statement) => this.db.query(statement).run({ $projectId: projectId, $versionId: versionId }).changes;
14486
+ const diffLinks = run(statements.incident);
14487
+ if (diffLinks > 0)
14488
+ dropUnplannedWork(this.db, versionId, "link");
14489
+ return diffLinks + run(statements.members);
14490
+ }
14491
+ requireCandidate(input) {
14492
+ const candidate = sourceRow(this.db, input.projectId, input.source.projectionId, input.materializationId);
14493
+ if (!candidate || candidate.status !== "ready" || candidate.projection_run_id !== input.execution.projectionRunId || candidate.execution_token !== input.execution.executionToken) {
14494
+ throw new MaterializationConflictError4("source-materialization", `Candidate source materialization '${input.materializationId}' is missing, not ready, or owned by another execution.`);
14495
+ }
14496
+ assertProjectionExecution(this.db, {
14497
+ projectId: input.projectId,
14498
+ sourceId: input.source.projectionId,
14499
+ projectionRunId: input.execution.projectionRunId,
14500
+ executionToken: input.execution.executionToken
14501
+ });
14502
+ return candidate;
14503
+ }
14504
+ requirePlan(input) {
14505
+ const candidate = this.requireCandidate(input);
14506
+ const plan = planRow(this.db, candidate.version_id);
14507
+ if (!plan) {
14508
+ throw new MaterializationConflictError4("source-materialization", `Candidate source materialization '${input.materializationId}' has no open plan.`);
14509
+ }
14510
+ return { plan, candidate };
14511
+ }
14512
+ }
14513
+ function assertReplacementPlanCoverage(db, plan) {
14514
+ const mismatch = db.query(`SELECT 1 FROM ${IDENTITIES}
14515
+ WHERE version_id = ?
14516
+ AND classified <> (diff_required AND (entity_kind = 'link' OR ? = 1))
14517
+ LIMIT 1`).get(plan.versionId, plan.projectionKind === "object" ? 1 : 0);
14518
+ if (mismatch) {
14519
+ invalidCorrelation("Projection replacement classification coverage does not match its plan.");
14520
+ }
14521
+ }
14522
+ function replacementPlanCounts(db, versionId) {
14523
+ const rows = db.query(`SELECT entity_kind, COALESCE(change, 'unchanged') AS change, COUNT(*) AS count
14524
+ FROM ${IDENTITIES}
14525
+ WHERE version_id = ? AND classified = 1
14526
+ GROUP BY entity_kind, COALESCE(change, 'unchanged')`).all(versionId);
14527
+ const count = (kind, change) => rows.find((row) => row.entity_kind === kind && row.change === change)?.count ?? 0;
14528
+ return {
14529
+ objectsCreated: count("object", "created"),
14530
+ objectsUpdated: count("object", "updated"),
14531
+ objectsDeleted: count("object", "deleted"),
14532
+ objectsUnchanged: count("object", "unchanged"),
14533
+ linksCreated: count("link", "created"),
14534
+ linksUpdated: count("link", "updated"),
14535
+ linksDeleted: count("link", "deleted"),
14536
+ linksUnchanged: count("link", "unchanged")
14537
+ };
14538
+ }
14539
+ function unplannedCount(db, versionId, kind) {
14540
+ const row = db.query(`SELECT COUNT(*) AS count FROM ${IDENTITIES}
14541
+ WHERE version_id = ? AND planned_revision IS NULL AND (? IS NULL OR entity_kind = ?)`).get(versionId, kind ?? null, kind ?? null);
14542
+ return row.count;
14543
+ }
14544
+ function committedSince(db, plan, projectId) {
14545
+ return db.query(`SELECT 1 FROM ontology_commits WHERE rowid > ? AND +project_id = ? LIMIT 1`).get(plan.watermark, projectId) !== null;
14546
+ }
14547
+ function dropUnplannedWork(db, versionId, kind) {
14548
+ db.query(`DELETE FROM ${SQLITE_PLAN_WORK_TABLE}
14549
+ WHERE work_id = ? AND entity_kind = ? AND identity_key IN (
14550
+ SELECT identity_key FROM ${IDENTITIES}
14551
+ WHERE version_id = ? AND entity_kind = ? AND planned_revision IS NULL
14552
+ )`).run(versionId, kind, versionId, kind);
14553
+ }
14554
+ function deletePlan(db, versionId) {
14555
+ db.query(`DELETE FROM ${SQLITE_PLAN_WORK_TABLE} WHERE work_id = ?`).run(versionId);
14556
+ db.query(`DELETE FROM ${IDENTITIES} WHERE version_id = ?`).run(versionId);
14557
+ db.query(`DELETE FROM ontology_replacement_plans WHERE version_id = ?`).run(versionId);
14558
+ }
14559
+ function planRow(db, versionId) {
14560
+ return db.query(`SELECT * FROM ontology_replacement_plans WHERE version_id = ?`).get(versionId);
14561
+ }
14562
+ function sourceRow(db, projectId, sourceId, materializationId) {
14563
+ return db.query(`SELECT * FROM ontology_sources
14564
+ WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).get(projectId, sourceId, materializationId);
14565
+ }
14566
+ function boundReplacementPlan(db, header) {
14567
+ const ref = header.plan;
14568
+ const candidate = ref ? sourceRow(db, header.commit.projectId, ref.source.projectionId, ref.materializationId) : null;
14569
+ const plan = candidate ? planRow(db, candidate.version_id) : null;
14570
+ if (!ref || !candidate || !plan || header.commit.intent.kind !== "projection") {
14571
+ throw new MaterializationConflictError4("source-materialization", "A plan-bound session needs the open plan of a projection candidate.");
14572
+ }
14573
+ if (plan.commit_id !== header.commit.id || plan.committed_at !== header.commit.committedAt) {
14574
+ throw new MaterializationValidationError2("A plan-bound session must begin with the commit its plan carries.");
14575
+ }
14576
+ if (committedSince(db, plan, header.commit.projectId) || unplannedCount(db, candidate.version_id) > 0) {
14577
+ throw new MaterializationValidationError2("A replacement plan applies only fully planned and just refreshed.");
14578
+ }
14579
+ return { versionId: candidate.version_id, projectionKind: candidate.projection_kind };
14580
+ }
14581
+
13714
14582
  // src/ontology-storage/materialization-session.ts
13715
14583
  class SqliteMaterializationSessionState extends ProviderMaterializationSessionState {
13716
- appliedPlanCursor = null;
13717
- appliedEventCursor = null;
14584
+ plan;
13718
14585
  summary = emptyWorkSummary();
14586
+ workTable;
14587
+ workId;
14588
+ constructor(header, transactionId, plan) {
14589
+ super(header, transactionId);
14590
+ this.plan = plan;
14591
+ this.workTable = plan ? SQLITE_PLAN_WORK_TABLE : SQLITE_MATERIALIZATION_WORK_TABLE;
14592
+ this.workId = plan ? plan.versionId : this.id;
14593
+ }
13719
14594
  }
13720
14595
 
13721
14596
  class SqliteMaterializationSessions {
@@ -13727,12 +14602,12 @@ class SqliteMaterializationSessions {
13727
14602
  this.db = db;
13728
14603
  this.context = context;
13729
14604
  }
13730
- create(header) {
14605
+ create(header, plan) {
13731
14606
  if (!this.context?.active) {
13732
- throw new MaterializationConflictError4("effective-state", "Materialization sessions require an active storage transaction.");
14607
+ throw new MaterializationConflictError5("effective-state", "Materialization sessions require an active storage transaction.");
13733
14608
  }
13734
14609
  this.ensureTables();
13735
- const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id);
14610
+ const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id, plan);
13736
14611
  this.sessions.set(session.providerToken, session);
13737
14612
  this.live.add(session);
13738
14613
  this.context.materializations.register(session.providerToken);
@@ -13741,7 +14616,7 @@ class SqliteMaterializationSessions {
13741
14616
  require(session) {
13742
14617
  const value = this.sessions.get(session.providerToken);
13743
14618
  if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
13744
- throw new MaterializationConflictError4("effective-state", "Materialization session is inactive.");
14619
+ throw new MaterializationConflictError5("effective-state", "Materialization session is inactive.");
13745
14620
  }
13746
14621
  return value;
13747
14622
  }
@@ -13749,11 +14624,10 @@ class SqliteMaterializationSessions {
13749
14624
  if (!session.active)
13750
14625
  return;
13751
14626
  session.active = false;
13752
- session.replacement = null;
13753
14627
  this.live.delete(session);
13754
14628
  if (this.live.size === 0)
13755
14629
  this.dropWorkTables();
13756
- else
14630
+ else if (!session.plan)
13757
14631
  this.deleteWork(session.id);
13758
14632
  this.context?.materializations.complete(session.providerToken);
13759
14633
  }
@@ -13768,7 +14642,7 @@ class SqliteMaterializationSessions {
13768
14642
  return;
13769
14643
  const insert = this.db.query(`
13770
14644
  INSERT INTO ${SQLITE_MATERIALIZATION_WORK_TABLE} (
13771
- session_id, record_key, unique_key, kind, lane,
14645
+ work_id, record_key, unique_key, kind, lane,
13772
14646
  major_order, minor_order, sort_one, sort_two,
13773
14647
  classification_entity_kind, classification_identity_key,
13774
14648
  cardinality_view, cardinality_occupied, cardinality_source_type_id,
@@ -13795,74 +14669,62 @@ class SqliteMaterializationSessions {
13795
14669
  throw error;
13796
14670
  }
13797
14671
  for (const prepared of records)
13798
- recordSummary(session.summary, prepared.record, prepared.columns);
14672
+ recordSummary(session.summary, prepared.record);
13799
14673
  }
13800
- async* stream(input) {
13801
- const session = this.require(input.session);
13802
- const stream = session.workStreams[input.order];
13803
- if (stream.started) {
13804
- throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
13805
- }
13806
- stream.started = true;
13807
- session.workSealed = true;
14674
+ *lanePages(session, lane, pageRows) {
13808
14675
  let cursor = null;
13809
14676
  while (true) {
13810
- this.require(input.session);
13811
- const rows = this.readLane(input.order, session.id, cursor, input.pageRows);
14677
+ this.require(session.publicSession());
14678
+ const rows = this.readLane(session, lane, cursor, pageRows);
13812
14679
  if (rows.length === 0)
13813
- break;
13814
- const records = rows.map((row) => parseJson2(row.payload));
14680
+ return;
13815
14681
  cursor = workCursor(rows[rows.length - 1]);
13816
- stream.emittedCount += records.length;
13817
- yield { records };
14682
+ yield rows.map((row) => parseJson2(row.payload));
13818
14683
  }
13819
- this.require(input.session);
13820
- stream.completed = true;
13821
14684
  }
13822
- readObjectExistence(session, refs) {
13823
- const lookup = this.db.query(`
13824
- SELECT payload FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
13825
- WHERE session_id = ? AND unique_key = ? AND kind = 'object-existence'
14685
+ *vectorChangePages(session, objectTypeIds, pageRows) {
14686
+ if (objectTypeIds.length === 0 && !this.db.query(`SELECT 1 FROM object_vectors WHERE project_id = ? LIMIT 1`).get(session.header.commit.projectId)) {
14687
+ return;
14688
+ }
14689
+ const query = this.db.query(`
14690
+ WITH changes AS (
14691
+ SELECT record_key, payload,
14692
+ coalesce(
14693
+ json_extract(payload, '$.item.value.row.ref.objectTypeId'),
14694
+ json_extract(payload, '$.item.value.ref.objectTypeId')
14695
+ ) AS object_type_id,
14696
+ coalesce(
14697
+ json_extract(payload, '$.item.value.row.ref.primaryId'),
14698
+ json_extract(payload, '$.item.value.ref.primaryId')
14699
+ ) AS primary_id
14700
+ FROM ${session.workTable}
14701
+ WHERE work_id = ? AND lane = 'apply' AND kind = 'plan'
14702
+ AND json_extract(payload, '$.item.kind') IN ('object-upsert', 'object-delete')
14703
+ AND record_key > ?
14704
+ )
14705
+ SELECT record_key, payload FROM changes
14706
+ WHERE object_type_id IN (SELECT value FROM json_each(?))
14707
+ OR EXISTS (
14708
+ SELECT 1 FROM object_vectors
14709
+ WHERE project_id = ? AND object_type_id = changes.object_type_id
14710
+ AND primary_id = changes.primary_id
14711
+ )
14712
+ ORDER BY record_key
14713
+ LIMIT ?
13826
14714
  `);
13827
- return refs.flatMap((ref) => {
13828
- const key = `object-existence:${JSON.stringify([ref.objectTypeId, ref.primaryId])}`;
13829
- const row = lookup.get(session.id, key);
13830
- if (!row)
13831
- return [];
13832
- const record = parseJson2(row.payload);
13833
- return [{ ref: record.ref, exists: record.exists }];
13834
- });
13835
- }
13836
- prepareChunkSequence(session, chunk) {
13837
- const items = materializationPlanItems(chunk);
13838
- const planRows = this.readLane("apply", session.id, session.appliedPlanCursor, items.length);
13839
- const expectedItems = planRows.map((row) => {
13840
- const record = parseJson2(row.payload);
13841
- return record.item;
13842
- });
13843
- const eventRows = this.readLane("event", session.id, session.appliedEventCursor, chunk.outbox.length);
13844
- const expectedEvents = eventRows.map((row) => parseJson2(row.payload));
13845
- const progress = correlateMaterializationChunk(session, chunk, items, expectedItems, expectedEvents);
13846
- return {
13847
- ...progress,
13848
- appliedPlanCursor: planRows.length === 0 ? session.appliedPlanCursor : workCursor(planRows[planRows.length - 1]),
13849
- appliedEventCursor: eventRows.length === 0 ? session.appliedEventCursor : workCursor(eventRows[eventRows.length - 1])
13850
- };
13851
- }
13852
- commitChunkSequence(session, progress) {
13853
- session.appliedPlanCount = progress.appliedPlanCount;
13854
- session.appliedOutboxCount = progress.appliedOutboxCount;
13855
- session.appliedPlanCursor = progress.appliedPlanCursor;
13856
- session.appliedEventCursor = progress.appliedEventCursor;
13857
- }
13858
- laneCounts(session) {
13859
- return {
13860
- apply: session.summary.apply,
13861
- cardinality: session.summary.cardinality,
13862
- event: session.summary.event
13863
- };
14715
+ let after = "";
14716
+ while (true) {
14717
+ this.require(session.publicSession());
14718
+ const rows = query.all(session.workId, after, JSON.stringify(objectTypeIds), session.header.commit.projectId, pageRows);
14719
+ if (rows.length === 0)
14720
+ return;
14721
+ after = rows[rows.length - 1].record_key;
14722
+ yield rows.map((row) => parseJson2(row.payload).item);
14723
+ }
13864
14724
  }
13865
14725
  projectionCounts(session) {
14726
+ if (session.plan)
14727
+ return replacementPlanCounts(this.db, session.plan.versionId);
13866
14728
  const summary = session.summary;
13867
14729
  return {
13868
14730
  objectsCreated: summary.objectsCreated,
@@ -13888,67 +14750,29 @@ class SqliteMaterializationSessions {
13888
14750
  };
13889
14751
  }
13890
14752
  assertClassificationCoverage(session) {
13891
- if (!session.replacement)
13892
- return;
13893
- const expectedObject = session.replacement.projectionKind === "object" ? 1 : 0;
13894
- const mismatch = this.db.query(`
13895
- WITH invalid AS (
13896
- SELECT 1
13897
- FROM ${SQLITE_REPLACEMENT_WORK_TABLE} AS expected
13898
- WHERE expected.session_id = ? AND expected.diff_required = 1
13899
- AND (expected.entity_kind = 'link' OR ? = 1)
13900
- AND NOT EXISTS (
13901
- SELECT 1 FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} AS actual
13902
- WHERE actual.session_id = expected.session_id
13903
- AND actual.unique_key = 'classification:' || expected.entity_kind || ':'
13904
- || expected.identity_key
13905
- )
13906
- UNION ALL
13907
- SELECT 1
13908
- FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} AS actual
13909
- WHERE actual.session_id = ?
13910
- -- A bounded range uses the existing (session_id, unique_key) unique index and scans
13911
- -- classification entries only, instead of sorting the complete JSON work spool.
13912
- AND actual.unique_key >= 'classification:'
13913
- AND actual.unique_key < 'classification;'
13914
- AND (
13915
- actual.classification_entity_kind = 'point'
13916
- OR NOT EXISTS (
13917
- SELECT 1 FROM ${SQLITE_REPLACEMENT_WORK_TABLE} AS expected
13918
- WHERE expected.session_id = actual.session_id
13919
- AND expected.entity_kind = actual.classification_entity_kind
13920
- AND expected.identity_key = actual.classification_identity_key
13921
- AND expected.diff_required = 1
13922
- AND (expected.entity_kind = 'link' OR ? = 1)
13923
- )
13924
- )
13925
- )
13926
- SELECT 1 FROM invalid LIMIT 1
13927
- `).get(session.id, expectedObject, session.id, expectedObject);
13928
- if (mismatch) {
13929
- invalidCorrelation("Projection replacement classification coverage does not match its streamed state.");
13930
- }
14753
+ if (session.plan)
14754
+ assertReplacementPlanCoverage(this.db, session.plan);
13931
14755
  }
13932
- readLane(lane, sessionId, cursor, limit) {
14756
+ readLane(session, lane, cursor, limit) {
13933
14757
  if (limit === 0)
13934
14758
  return [];
13935
14759
  const selected = `record_key, major_order, minor_order, sort_one, sort_two, payload`;
13936
14760
  if (!cursor) {
13937
- return this.db.query(`SELECT ${selected} FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
13938
- WHERE session_id = ? AND lane = ?
14761
+ return this.db.query(`SELECT ${selected} FROM ${session.workTable}
14762
+ WHERE work_id = ? AND lane = ?
13939
14763
  ORDER BY major_order, minor_order, sort_one, sort_two, record_key
13940
- LIMIT ?`).all(sessionId, lane, limit);
14764
+ LIMIT ?`).all(session.workId, lane, limit);
13941
14765
  }
13942
- return this.db.query(`SELECT ${selected} FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
13943
- WHERE session_id = ? AND lane = ?
14766
+ return this.db.query(`SELECT ${selected} FROM ${session.workTable}
14767
+ WHERE work_id = ? AND lane = ?
13944
14768
  AND (major_order, minor_order, sort_one, sort_two, record_key) > (?, ?, ?, ?, ?)
13945
14769
  ORDER BY major_order, minor_order, sort_one, sort_two, record_key
13946
- LIMIT ?`).all(sessionId, lane, cursor.majorOrder, cursor.minorOrder, cursor.sortOne, cursor.sortTwo, cursor.recordKey, limit);
14770
+ LIMIT ?`).all(session.workId, lane, cursor.majorOrder, cursor.minorOrder, cursor.sortOne, cursor.sortTwo, cursor.recordKey, limit);
13947
14771
  }
13948
14772
  ensureTables() {
13949
14773
  this.db.run(`
13950
14774
  CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE} (
13951
- session_id TEXT NOT NULL,
14775
+ work_id TEXT NOT NULL,
13952
14776
  record_key TEXT NOT NULL,
13953
14777
  unique_key TEXT NOT NULL,
13954
14778
  kind TEXT NOT NULL,
@@ -13967,35 +14791,21 @@ class SqliteMaterializationSessions {
13967
14791
  cardinality_target_type_id TEXT,
13968
14792
  cardinality_target_primary_id TEXT,
13969
14793
  payload TEXT NOT NULL CHECK (json_valid(payload)),
13970
- PRIMARY KEY (session_id, record_key),
13971
- UNIQUE (session_id, unique_key)
14794
+ PRIMARY KEY (work_id, record_key),
14795
+ UNIQUE (work_id, unique_key)
13972
14796
  );
13973
14797
  CREATE INDEX IF NOT EXISTS idx_ontology_materialization_work_lane
13974
14798
  ON ${SQLITE_MATERIALIZATION_WORK_TABLE}(
13975
- session_id, lane, major_order, minor_order, sort_one, sort_two, record_key
13976
- );
13977
- CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE} (
13978
- session_id TEXT NOT NULL,
13979
- entity_kind TEXT NOT NULL,
13980
- identity_key TEXT NOT NULL,
13981
- sort_key TEXT NOT NULL,
13982
- diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
13983
- PRIMARY KEY (session_id, entity_kind, identity_key)
13984
- );
13985
- CREATE INDEX IF NOT EXISTS idx_ontology_replacement_work_order
13986
- ON ${SQLITE_REPLACEMENT_WORK_TABLE}(
13987
- session_id, entity_kind, sort_key, identity_key
14799
+ work_id, lane, major_order, minor_order, sort_one, sort_two, record_key
13988
14800
  );
13989
14801
  `);
13990
14802
  }
13991
14803
  deleteWork(sessionId) {
13992
- this.db.query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE session_id = ?`).run(sessionId);
13993
- this.db.query(`DELETE FROM ${SQLITE_REPLACEMENT_WORK_TABLE} WHERE session_id = ?`).run(sessionId);
14804
+ this.db.query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE work_id = ?`).run(sessionId);
13994
14805
  }
13995
14806
  dropWorkTables() {
13996
14807
  this.db.run(`
13997
14808
  DROP TABLE IF EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE};
13998
- DROP TABLE IF EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE};
13999
14809
  `);
14000
14810
  }
14001
14811
  }
@@ -14010,9 +14820,6 @@ function workCursor(row) {
14010
14820
  }
14011
14821
  function emptyWorkSummary() {
14012
14822
  return {
14013
- apply: 0,
14014
- cardinality: 0,
14015
- event: 0,
14016
14823
  objectClassifications: 0,
14017
14824
  linkClassifications: 0,
14018
14825
  pointClassifications: 0,
@@ -14027,9 +14834,7 @@ function emptyWorkSummary() {
14027
14834
  latestObjectsChanged: 0
14028
14835
  };
14029
14836
  }
14030
- function recordSummary(summary, record, columns) {
14031
- if (columns.lane !== "none")
14032
- summary[columns.lane] += 1;
14837
+ function recordSummary(summary, record) {
14033
14838
  if (record.kind === "classification") {
14034
14839
  if (record.entityKind === "object")
14035
14840
  summary.objectClassifications += 1;
@@ -14078,11 +14883,13 @@ function recordSummary(summary, record, columns) {
14078
14883
  // src/ontology-storage/materialization-writer.ts
14079
14884
  import {
14080
14885
  linkRefKey as linkRefKey2,
14081
- MaterializationConflictError as MaterializationConflictError5,
14082
- objectRefKey as objectRefKey2,
14886
+ MaterializationConflictError as MaterializationConflictError6,
14887
+ objectRefKey as objectRefKey3,
14083
14888
  telemetryPointKey
14084
14889
  } from "@sixb/core/internal/materialization";
14085
- import { effectiveConflict } from "@sixb/core/internal/ontology-storage-provider";
14890
+ import {
14891
+ effectiveConflict
14892
+ } from "@sixb/core/internal/ontology-storage-provider";
14086
14893
  function linkOverrideConflictMessage(identityKind) {
14087
14894
  return identityKind === "edge" ? "Expected link edge override changed." : "Expected link slot override changed.";
14088
14895
  }
@@ -14178,7 +14985,7 @@ class SqliteMaterializationWriter {
14178
14985
  for (const item of chunk.effective.objectDeletes) {
14179
14986
  requireChanges(this.db.query(`DELETE FROM objects
14180
14987
  WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
14181
- AND version = ? AND last_commit_id = ?`).run(projectId, item.ref.objectTypeId, item.ref.primaryId, item.expected.version, item.expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey2(item.ref)} changed.`);
14988
+ AND version = ? AND last_commit_id = ?`).run(projectId, item.ref.objectTypeId, item.ref.primaryId, item.expected.version, item.expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey3(item.ref)} changed.`);
14182
14989
  }
14183
14990
  for (const item of chunk.effective.objectUpserts)
14184
14991
  this.applyObject(projectId, item);
@@ -14197,7 +15004,7 @@ class SqliteMaterializationWriter {
14197
15004
  `).run(projectId, row.ref.objectTypeId, row.ref.primaryId, canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId);
14198
15005
  } catch (error) {
14199
15006
  if (isSqliteConstraintError(error)) {
14200
- throw effectiveConflict(`Expected object ${objectRefKey2(row.ref)} to be absent.`);
15007
+ throw effectiveConflict(`Expected object ${objectRefKey3(row.ref)} to be absent.`);
14201
15008
  }
14202
15009
  throw error;
14203
15010
  }
@@ -14209,7 +15016,7 @@ class SqliteMaterializationWriter {
14209
15016
  last_commit_id = ?
14210
15017
  WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
14211
15018
  AND version = ? AND last_commit_id = ?
14212
- `).run(canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId, projectId, row.ref.objectTypeId, row.ref.primaryId, expected.version, expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey2(row.ref)} changed.`);
15019
+ `).run(canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId, projectId, row.ref.objectTypeId, row.ref.primaryId, expected.version, expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey3(row.ref)} changed.`);
14213
15020
  }
14214
15021
  applyLink(projectId, item) {
14215
15022
  const { row, expected } = item;
@@ -14250,7 +15057,7 @@ class SqliteMaterializationWriter {
14250
15057
  `).run(projectId, point.series.object.objectTypeId, point.series.object.primaryId, point.series.propertyId, canonicalJson(point.value), point.unit ?? null, point.at, point.lastCommitId);
14251
15058
  } catch (error) {
14252
15059
  if (isSqliteConstraintError(error)) {
14253
- throw new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey(point.series, point.at)} changed.`);
15060
+ throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey(point.series, point.at)} changed.`);
14254
15061
  }
14255
15062
  throw error;
14256
15063
  }
@@ -14267,19 +15074,18 @@ class SqliteMaterializationWriter {
14267
15074
  applyOutbox(projectId, commitId, chunk) {
14268
15075
  const insert = this.db.query(`
14269
15076
  INSERT INTO ontology_outbox (
14270
- project_id, id, commit_id, commit_ordinal, envelope,
15077
+ project_id, id, commit_id, commit_ordinal, event,
14271
15078
  available_at, attempts, lease_id, lease_expires_at,
14272
15079
  published_at, last_failure, created_at
14273
15080
  ) VALUES (?, ?, ?, ?, json(?), ?, 0, NULL, NULL, NULL, NULL, ?)
14274
15081
  `);
14275
15082
  for (const item of chunk.outbox) {
14276
- assertTimestamp(item.availableAt, "Outbox availableAt");
14277
15083
  assertTimestamp(item.createdAt, "Outbox createdAt");
14278
15084
  try {
14279
- insert.run(projectId, item.envelope.id, commitId, item.envelope.commitOrdinal, canonicalJson(item.envelope), item.availableAt, item.createdAt);
15085
+ insert.run(projectId, item.id, commitId, item.commitOrdinal, canonicalJson(item.event), item.createdAt, item.createdAt);
14280
15086
  } catch (error) {
14281
15087
  if (isSqliteConstraintError(error)) {
14282
- throw effectiveConflict(`Duplicate outbox event '${item.envelope.id}'.`);
15088
+ throw effectiveConflict(`Duplicate outbox event '${item.id}'.`);
14283
15089
  }
14284
15090
  throw error;
14285
15091
  }
@@ -14288,6 +15094,48 @@ class SqliteMaterializationWriter {
14288
15094
  }
14289
15095
 
14290
15096
  // src/ontology-storage/materializations.ts
15097
+ var APPLY_PAGE_ROWS = 1000;
15098
+ function touchAppliedWorkStatement(workTable) {
15099
+ return `
15100
+ INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
15101
+ SELECT DISTINCT $projectId, $commitId, kind, CASE kind
15102
+ WHEN 'object' THEN
15103
+ json_array(json_extract(ref, '$.objectTypeId'), json_extract(ref, '$.primaryId'))
15104
+ WHEN 'scope' THEN json_array(
15105
+ json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
15106
+ json_extract(ref, '$.linkId')
15107
+ )
15108
+ ELSE json_array(
15109
+ json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
15110
+ json_extract(ref, '$.linkId'), json_extract(ref, '$.target.objectTypeId'),
15111
+ json_extract(ref, '$.target.primaryId')
15112
+ )
15113
+ END
15114
+ FROM (
15115
+ SELECT CASE json_extract(payload, '$.item.kind')
15116
+ WHEN 'object-upsert' THEN 'object'
15117
+ WHEN 'object-delete' THEN 'object'
15118
+ WHEN 'object-override-upsert' THEN 'object'
15119
+ WHEN 'object-override-delete' THEN 'object'
15120
+ WHEN 'point-upsert' THEN 'object'
15121
+ WHEN 'link-upsert' THEN 'link'
15122
+ WHEN 'link-delete' THEN 'link'
15123
+ WHEN 'link-override-upsert' THEN 'link'
15124
+ WHEN 'link-override-delete' THEN 'link'
15125
+ WHEN 'link-slot-override-upsert' THEN 'scope'
15126
+ WHEN 'link-slot-override-delete' THEN 'scope'
15127
+ END AS kind,
15128
+ CASE json_extract(payload, '$.item.kind')
15129
+ WHEN 'object-upsert' THEN json_extract(payload, '$.item.value.row.ref')
15130
+ WHEN 'link-upsert' THEN json_extract(payload, '$.item.value.row.ref')
15131
+ WHEN 'point-upsert' THEN json_extract(payload, '$.item.value.point.series.object')
15132
+ ELSE json_extract(payload, '$.item.value.ref')
15133
+ END AS ref
15134
+ FROM ${workTable}
15135
+ WHERE work_id = $workId AND lane = 'apply'
15136
+ )`;
15137
+ }
15138
+
14291
15139
  class SqliteOntologyMaterializationStorage {
14292
15140
  db;
14293
15141
  sessions;
@@ -14300,12 +15148,12 @@ class SqliteOntologyMaterializationStorage {
14300
15148
  assertVectorSession(session, projectId, commitId) {
14301
15149
  const active = this.sessions.require(session);
14302
15150
  if (active.header.commit.projectId !== projectId || commitId !== undefined && active.header.commit.id !== commitId) {
14303
- throw new MaterializationValidationError2("Vector mutation does not belong to this materialization session.");
15151
+ throw new MaterializationValidationError3("Vector mutation does not belong to this materialization session.");
14304
15152
  }
14305
15153
  }
14306
15154
  async begin(input) {
14307
15155
  assertMaterializationHeader(input);
14308
- const session = this.sessions.create(input);
15156
+ const session = this.sessions.create(input, input.plan ? boundReplacementPlan(this.db, input) : null);
14309
15157
  try {
14310
15158
  this.assertCommitAbsent(input);
14311
15159
  const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId);
@@ -14324,7 +15172,7 @@ class SqliteOntologyMaterializationStorage {
14324
15172
  for (const expected of input.expected.points) {
14325
15173
  const point = reader.exactPoint(expected.series, expected.at);
14326
15174
  if ((point?.lastCommitId ?? null) !== expected.lastCommitId) {
14327
- throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey2(expected.series, expected.at)} changed.`);
15175
+ throw new MaterializationConflictError7("timeseries-point", `Telemetry point ${telemetryPointKey2(expected.series, expected.at)} changed.`);
14328
15176
  }
14329
15177
  }
14330
15178
  return session.publicSession();
@@ -14335,11 +15183,11 @@ class SqliteOntologyMaterializationStorage {
14335
15183
  }
14336
15184
  async* streamState(input) {
14337
15185
  const session = this.sessions.require(input.session);
14338
- assertPageRows(input.pageRows);
15186
+ assertPageRows2(input.pageRows);
14339
15187
  const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
14340
15188
  for await (const request of input.requests) {
14341
15189
  this.sessions.require(input.session);
14342
- const objects = uniqueSorted(request.objects, objectRefKey3, objectRefSortKey);
15190
+ const objects = uniqueSorted(request.objects, objectRefKey4, objectRefSortKey);
14343
15191
  for (let offset = 0;offset < objects.length; offset += input.pageRows) {
14344
15192
  const page = reader.objectStates(objects.slice(offset, offset + input.pageRows));
14345
15193
  yield { objects: page, links: [], linkScopes: [], points: [] };
@@ -14375,103 +15223,50 @@ class SqliteOntologyMaterializationStorage {
14375
15223
  }
14376
15224
  }
14377
15225
  }
14378
- async* streamSourceReplacementState(input) {
14379
- const session = this.sessions.require(input.session);
14380
- assertPageRows(input.pageRows);
14381
- const replacement = this.requireReplacement(session, input);
14382
- const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
14383
- if (input.entityKind === "object") {
14384
- if (replacement.projectionKind !== "object") {
14385
- throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
14386
- }
14387
- if (replacement.objectStreamStarted) {
14388
- throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
14389
- }
14390
- replacement.objectStreamStarted = true;
14391
- for (const identities of reader.replacementIdentities({
14392
- sessionId: session.id,
14393
- sourceId: replacement.sourceId,
14394
- candidateMaterializationId: replacement.candidateMaterializationId,
14395
- previousMaterializationId: replacement.previousMaterializationId,
14396
- incremental: replacement.incremental,
14397
- kind: "object",
14398
- pageRows: input.pageRows
14399
- })) {
14400
- this.sessions.require(input.session);
14401
- const refs = identities.map((identity) => {
14402
- if (identity.kind !== "object") {
14403
- invalidCorrelation2("Object replacement returned a link identity.");
14404
- }
14405
- return identity.ref;
14406
- });
14407
- const objects = reader.replacementObjectStates(replacement.sourceId, replacement.candidateMaterializationId, refs);
14408
- yield { objects, links: [] };
14409
- }
14410
- this.sessions.require(input.session);
14411
- replacement.objectStreamCompleted = true;
14412
- return;
14413
- }
14414
- if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
14415
- throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
14416
- }
14417
- if (replacement.linkStreamStarted) {
14418
- throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
14419
- }
14420
- replacement.linkStreamStarted = true;
14421
- const materializationIds = [
14422
- replacement.candidateMaterializationId,
14423
- ...replacement.previousMaterializationId ? [replacement.previousMaterializationId] : []
14424
- ];
14425
- for (const identities of reader.replacementIdentities({
14426
- sessionId: session.id,
14427
- sourceId: replacement.sourceId,
14428
- candidateMaterializationId: replacement.candidateMaterializationId,
14429
- previousMaterializationId: replacement.previousMaterializationId,
14430
- incremental: replacement.incremental,
14431
- kind: "link",
14432
- pageRows: input.pageRows
14433
- })) {
14434
- this.sessions.require(input.session);
14435
- const linkIdentities = identities.map((identity) => {
14436
- if (identity.kind !== "link") {
14437
- invalidCorrelation2("Link replacement returned an object identity.");
14438
- }
14439
- return identity;
14440
- });
14441
- const links = reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
14442
- yield { objects: [], links };
14443
- }
14444
- this.sessions.require(input.session);
14445
- replacement.linkStreamCompleted = true;
14446
- }
14447
15226
  async stageWork(input) {
14448
15227
  this.sessions.stage(input);
14449
15228
  await yieldSqliteEventLoop();
14450
15229
  }
14451
- streamWork(input) {
14452
- assertPageRows(input.pageRows);
14453
- return this.sessions.stream(input);
14454
- }
14455
- async readObjectExistence(input) {
15230
+ async* streamVectorChanges(input) {
14456
15231
  const session = this.sessions.require(input.session);
14457
- return this.sessions.readObjectExistence(session, input.refs);
15232
+ assertPageRows2(input.pageRows);
15233
+ beginMaterializationVectorChanges(session);
15234
+ for (const items of this.sessions.vectorChangePages(session, input.objectTypeIds, input.pageRows)) {
15235
+ yield { items };
15236
+ }
14458
15237
  }
14459
- async applyChunk(input) {
15238
+ async apply(input) {
14460
15239
  const session = this.sessions.require(input.session);
15240
+ beginMaterializationApply(session);
14461
15241
  const { commit } = session.header;
14462
- assertPlanChunkCorrelations(input.chunk, commit);
14463
- const sequence = this.sessions.prepareChunkSequence(session, input.chunk);
14464
- this.db.run("SAVEPOINT sixb_ontology_apply_chunk");
15242
+ const cardinality = createCardinalityValidator();
15243
+ for (const records of this.sessions.lanePages(session, "cardinality", APPLY_PAGE_ROWS)) {
15244
+ for (const record of records)
15245
+ cardinality.accept(record);
15246
+ }
15247
+ let eventCount = 0;
15248
+ this.db.run("SAVEPOINT sixb_ontology_apply");
14465
15249
  try {
14466
- this.writer.apply(commit.projectId, commit.id, input.chunk);
14467
- this.db.run("RELEASE SAVEPOINT sixb_ontology_apply_chunk");
14468
- this.sessions.commitChunkSequence(session, sequence);
15250
+ for (const records of this.sessions.lanePages(session, "apply", APPLY_PAGE_ROWS)) {
15251
+ for (const phase of splitByPhase(records)) {
15252
+ this.writer.apply(commit.projectId, commit.id, materializationPlanChunk(phase.map((record) => record.item)));
15253
+ }
15254
+ await yieldSqliteEventLoop();
15255
+ }
15256
+ for (const records of this.sessions.lanePages(session, "event", APPLY_PAGE_ROWS)) {
15257
+ const outbox = records.map((record) => materializationOutboxWrite(commit, record.draft, eventCount++));
15258
+ this.writer.apply(commit.projectId, commit.id, materializationPlanChunk([], outbox));
15259
+ await yieldSqliteEventLoop();
15260
+ }
15261
+ this.db.query(touchAppliedWorkStatement(session.workTable)).run({ $projectId: commit.projectId, $commitId: commit.id, $workId: session.workId });
15262
+ this.db.run("RELEASE SAVEPOINT sixb_ontology_apply");
14469
15263
  } catch (error) {
14470
- this.db.run("ROLLBACK TO SAVEPOINT sixb_ontology_apply_chunk");
14471
- this.db.run("RELEASE SAVEPOINT sixb_ontology_apply_chunk");
15264
+ this.db.run("ROLLBACK TO SAVEPOINT sixb_ontology_apply");
15265
+ this.db.run("RELEASE SAVEPOINT sixb_ontology_apply");
14472
15266
  throw error;
14473
15267
  }
14474
- await yieldSqliteEventLoop();
15268
+ session.appliedEventCount = eventCount;
15269
+ return { eventCount };
14475
15270
  }
14476
15271
  async finalize(input) {
14477
15272
  const session = this.sessions.require(input.session);
@@ -14487,41 +15282,6 @@ class SqliteOntologyMaterializationStorage {
14487
15282
  deactivateSessions() {
14488
15283
  this.sessions.deactivateAll();
14489
15284
  }
14490
- requireReplacement(session, input) {
14491
- if (session.replacement) {
14492
- if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
14493
- throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
14494
- }
14495
- return session.replacement;
14496
- }
14497
- const projectId = session.header.commit.projectId;
14498
- const candidate = this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId);
14499
- if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
14500
- throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
14501
- }
14502
- assertProjectionExecution(this.db, {
14503
- projectId,
14504
- sourceId: input.source.projectionId,
14505
- projectionRunId: candidate.projection_run_id,
14506
- executionToken: candidate.execution_token
14507
- });
14508
- const previous = this.getActiveSource(projectId, input.source.projectionId);
14509
- if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
14510
- throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
14511
- }
14512
- session.replacement = {
14513
- sourceId: input.source.projectionId,
14514
- candidateMaterializationId: input.candidateMaterializationId,
14515
- previousMaterializationId: previous?.materialization_id ?? null,
14516
- incremental: candidate.base_materialization_id !== null,
14517
- projectionKind: candidate.projection_kind,
14518
- objectStreamStarted: false,
14519
- objectStreamCompleted: false,
14520
- linkStreamStarted: false,
14521
- linkStreamCompleted: false
14522
- };
14523
- return session.replacement;
14524
- }
14525
15285
  assertCommitAbsent(header) {
14526
15286
  const origin = originColumns(header.commit.origin);
14527
15287
  const duplicate = this.db.query(`
@@ -14542,25 +15302,23 @@ class SqliteOntologyMaterializationStorage {
14542
15302
  if (!duplicate)
14543
15303
  return;
14544
15304
  if (duplicate.duplicate === "origin") {
14545
- throw new MaterializationConflictError6("run-correlation", "Ontology commit origin already has an authoritative commit.");
15305
+ throw new MaterializationConflictError7("run-correlation", "Ontology commit origin already has an authoritative commit.");
14546
15306
  }
14547
- throw new MaterializationConflictError6("idempotency", duplicate.duplicate === "id" ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
15307
+ throw new MaterializationConflictError7("idempotency", duplicate.duplicate === "id" ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
14548
15308
  }
14549
15309
  assertSource(expected, projectId) {
14550
15310
  const active = this.getActiveSource(projectId, expected.source.projectionId);
14551
15311
  if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
14552
- throw new MaterializationConflictError6("projection-fence", `Source '${expected.source.projectionId}' changed.`);
15312
+ throw new MaterializationConflictError7("projection-fence", `Source '${expected.source.projectionId}' changed.`);
14553
15313
  }
14554
15314
  }
14555
15315
  async assertFinalization(session, input) {
14556
15316
  const { commit } = session.header;
14557
15317
  const { result } = input.finalization;
14558
15318
  assertMaterializationFinalizationCorrelation(session, input);
14559
- const laneCounts = this.sessions.laneCounts(session);
14560
- assertMaterializationLaneCompletion(session, laneCounts);
14561
15319
  this.assertFinalCardinality(session);
14562
15320
  await yieldSqliteEventLoop();
14563
- const eventCount = laneCounts.event;
15321
+ const eventCount = session.appliedEventCount;
14564
15322
  const outbox = this.db.query(`
14565
15323
  SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum, MAX(commit_ordinal) AS maximum
14566
15324
  FROM ontology_outbox WHERE project_id = ? AND commit_id = ?
@@ -14603,8 +15361,8 @@ class SqliteOntologyMaterializationStorage {
14603
15361
  THEN cardinality_target_type_id END) AS expected_target_type_id,
14604
15362
  MAX(CASE WHEN cardinality_occupied = 1
14605
15363
  THEN cardinality_target_primary_id END) AS expected_target_id
14606
- FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
14607
- WHERE session_id = ? AND kind = 'cardinality'
15364
+ FROM ${session.workTable}
15365
+ WHERE work_id = ? AND kind = 'cardinality'
14608
15366
  AND cardinality_view = 'effective'
14609
15367
  GROUP BY sort_one, cardinality_source_type_id,
14610
15368
  cardinality_source_primary_id, cardinality_link_id
@@ -14624,16 +15382,11 @@ class SqliteOntologyMaterializationStorage {
14624
15382
  GROUP BY scopes.scope_sort_key, scopes.expected_count,
14625
15383
  scopes.expected_target_type_id, scopes.expected_target_id
14626
15384
  )
14627
- SELECT CASE WHEN expected_count > 1 THEN 'duplicate' ELSE 'mismatch' END AS reason
15385
+ SELECT 1 AS mismatch
14628
15386
  FROM validated
14629
- WHERE expected_count > 1
14630
- OR actual_count <> expected_count
14631
- OR matching_count <> expected_count
15387
+ WHERE actual_count <> expected_count OR matching_count <> expected_count
14632
15388
  LIMIT 1
14633
- `).get(session.id, session.header.commit.projectId);
14634
- if (violation?.reason === "duplicate") {
14635
- invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
14636
- }
15389
+ `).get(session.workId, session.header.commit.projectId);
14637
15390
  if (violation) {
14638
15391
  invalidCorrelation2("Materialization cardinality work does not match the final effective link scope.");
14639
15392
  }
@@ -14643,7 +15396,7 @@ class SqliteOntologyMaterializationStorage {
14643
15396
  assertSourceActivationCorrelation(session, activation);
14644
15397
  const candidate = this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId);
14645
15398
  if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
14646
- throw new MaterializationConflictError6("source-materialization", "Source activation candidate is missing or is not ready.");
15399
+ throw new MaterializationConflictError7("source-materialization", "Source activation candidate is missing or is not ready.");
14647
15400
  }
14648
15401
  assertProjectionExecution(this.db, {
14649
15402
  projectId: commit.projectId,
@@ -14655,7 +15408,7 @@ class SqliteOntologyMaterializationStorage {
14655
15408
  invalidCorrelation2("Source activation does not match its ready candidate identity.");
14656
15409
  }
14657
15410
  this.assertSource(activation.expected, commit.projectId);
14658
- activateSourceRoots(this.db, commit.projectId, candidate, activation);
15411
+ activateSourceRoots(this.db, commit.projectId, commit.id, candidate, activation);
14659
15412
  const previous = this.getActiveSource(commit.projectId, activation.source.projectionId);
14660
15413
  if (previous) {
14661
15414
  assertPinnedDatasetWatermark({
@@ -14688,16 +15441,14 @@ class SqliteOntologyMaterializationStorage {
14688
15441
  this.db.query(`
14689
15442
  INSERT INTO ontology_commits (
14690
15443
  project_id, id, idempotency_key, request_hash, execution_id,
14691
- origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
15444
+ origin_kind, origin_run_id, origin_batch_ordinal, origin,
14692
15445
  ontology_revision, projection_revision, ownership_hash,
14693
15446
  intent, result, committed_at
14694
- ) VALUES (
14695
- ?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), json(?), ?, ?, ?, json(?), json(?), ?
14696
- )
14697
- `).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy), canonicalJson(commit.executor), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
15447
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, json(?), ?, ?, ?, json(?), json(?), ?)
15448
+ `).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
14698
15449
  } catch (error) {
14699
15450
  if (isSqliteConstraintError(error)) {
14700
- throw new MaterializationConflictError6("idempotency", "Ontology commit identity already exists.");
15451
+ throw new MaterializationConflictError7("idempotency", "Ontology commit identity already exists.");
14701
15452
  }
14702
15453
  throw error;
14703
15454
  }
@@ -14713,13 +15464,27 @@ class SqliteOntologyMaterializationStorage {
14713
15464
  WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).get(projectId, sourceId, materializationId);
14714
15465
  }
14715
15466
  }
15467
+ function splitByPhase(records) {
15468
+ const phases = [];
15469
+ for (const record of records) {
15470
+ const current = phases[phases.length - 1];
15471
+ if (current && current[0].applyPhase === record.applyPhase)
15472
+ current.push(record);
15473
+ else
15474
+ phases.push([record]);
15475
+ }
15476
+ return phases;
15477
+ }
14716
15478
 
14717
15479
  // src/ontology-storage/outbox.ts
14718
15480
  import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
14719
15481
  import {
14720
- MaterializationConflictError as MaterializationConflictError7,
14721
- MaterializationValidationError as MaterializationValidationError3
15482
+ MaterializationConflictError as MaterializationConflictError8,
15483
+ MaterializationValidationError as MaterializationValidationError4
14722
15484
  } from "@sixb/core/internal/materialization";
15485
+ import {
15486
+ claimedOutboxRows
15487
+ } from "@sixb/core/internal/ontology-storage-provider";
14723
15488
  import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
14724
15489
  class SqliteOntologyOutboxStorage {
14725
15490
  db;
@@ -14736,7 +15501,7 @@ class SqliteOntologyOutboxStorage {
14736
15501
  const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
14737
15502
  assertNonblank(input.leaseId, "Ontology outbox lease id");
14738
15503
  if (leaseExpiry <= now) {
14739
- throw new MaterializationValidationError3("Ontology outbox lease expiry must be later than the claim time.");
15504
+ throw new MaterializationValidationError4("Ontology outbox lease expiry must be later than the claim time.");
14740
15505
  }
14741
15506
  const eligible = this.db.query(`SELECT CAST(rowid AS TEXT) AS rowId
14742
15507
  FROM ontology_outbox INDEXED BY idx_ontology_outbox_claim
@@ -14755,15 +15520,19 @@ class SqliteOntologyOutboxStorage {
14755
15520
  UPDATE ontology_outbox
14756
15521
  SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
14757
15522
  WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
14758
- RETURNING envelope, available_at, attempts, lease_id, lease_expires_at,
14759
- published_at, last_failure, created_at
15523
+ RETURNING id, commit_id, commit_ordinal, event, available_at, attempts, lease_id,
15524
+ lease_expires_at, published_at, last_failure, created_at
14760
15525
  `).all(input.leaseId, input.leaseExpiresAt, JSON.stringify(selected.map((row) => row.rowId)));
14761
- return rows.map((row) => outboxRecord(row)).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.envelope.id.localeCompare(right.envelope.id)).map((record) => {
14762
- if (record.leaseId === null || record.leaseExpiresAt === null) {
14763
- throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
14764
- }
14765
- return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt };
14766
- });
15526
+ const contexts = this.db.query(`SELECT commits.id AS commit_id, commits.committed_at AS commit_committed_at,
15527
+ commits.origin AS commit_origin, executions.*
15528
+ FROM json_each(?) AS requested
15529
+ CROSS JOIN ontology_commits AS commits
15530
+ ON commits.project_id = ? AND commits.id = requested.value
15531
+ CROSS JOIN executions
15532
+ ON executions.project_id = commits.project_id AND executions.id = commits.execution_id`).all(JSON.stringify([...new Set(rows.map((row) => row.commit_id))]), input.projectId);
15533
+ const contextOf = new Map(contexts.map((row) => [row.commit_id, outboxEventContext(input.projectId, row)]));
15534
+ const drafts = rows.map(claimedOutboxDraft).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
15535
+ return claimedOutboxRows(drafts, (commitId) => contextOf.get(commitId));
14767
15536
  });
14768
15537
  }
14769
15538
  async markPublished(input) {
@@ -14856,12 +15625,23 @@ class SqliteOntologyOutboxStorage {
14856
15625
  });
14857
15626
  }
14858
15627
  }
15628
+ function outboxEventContext(projectId, row) {
15629
+ return {
15630
+ commit: {
15631
+ projectId,
15632
+ id: row.commit_id,
15633
+ committedAt: row.commit_committed_at,
15634
+ origin: parseJson2(row.commit_origin)
15635
+ },
15636
+ execution: sqliteExecutionRecord(row)
15637
+ };
15638
+ }
14859
15639
  function validateLeaseIds(ids) {
14860
15640
  const seen = new Set;
14861
15641
  for (const id of ids) {
14862
15642
  assertNonblank(id, "Ontology outbox event id");
14863
15643
  if (seen.has(id)) {
14864
- throw new MaterializationValidationError3(`Ontology outbox lease batch repeats event '${id}'.`);
15644
+ throw new MaterializationValidationError4(`Ontology outbox lease batch repeats event '${id}'.`);
14865
15645
  }
14866
15646
  seen.add(id);
14867
15647
  }
@@ -14872,11 +15652,11 @@ function assertLeaseInput(input) {
14872
15652
  assertNonblank(input.leaseId, "Ontology outbox lease id");
14873
15653
  }
14874
15654
  function leaseConflict() {
14875
- return new MaterializationConflictError7("outbox-lease", "Ontology outbox lease does not match.");
15655
+ return new MaterializationConflictError8("outbox-lease", "Ontology outbox lease does not match.");
14876
15656
  }
14877
15657
 
14878
15658
  // src/ontology-storage/sources.ts
14879
- import { MaterializationValidationError as MaterializationValidationError4 } from "@sixb/core/internal/materialization";
15659
+ import { MaterializationValidationError as MaterializationValidationError5 } from "@sixb/core/internal/materialization";
14880
15660
  import {
14881
15661
  assertSourceBeginInput as assertBeginInput,
14882
15662
  assertSourceExecutionIdentity as assertExecutionIdentity,
@@ -14894,13 +15674,16 @@ import {
14894
15674
  } from "@sixb/core/internal/ontology-storage-provider";
14895
15675
 
14896
15676
  // src/ontology-storage/source-cleanup.ts
15677
+ var PLAN_PURGED = `NOT EXISTS (
15678
+ SELECT 1 FROM ontology_replacement_plans AS plans WHERE plans.version_id = versions.version_id
15679
+ )`;
14897
15680
  function purgeAbandonedSourceVersions(db, input) {
14898
15681
  let remaining = input.limit;
14899
15682
  let rowsDeleted = 0;
14900
15683
  let materializationsDeleted = 0;
14901
15684
  const nextCandidate = db.query(`
14902
- SELECT version_id FROM ontology_sources
14903
- WHERE project_id = ? AND status = 'abandoned'
15685
+ SELECT version_id FROM ontology_sources AS versions
15686
+ WHERE project_id = ? AND status = 'abandoned' AND ${PLAN_PURGED}
14904
15687
  ORDER BY terminal_at, source_id, materialization_id LIMIT 1
14905
15688
  `);
14906
15689
  const deleteRows = db.query(`DELETE FROM ontology_source_rows WHERE rowid IN (
@@ -14951,15 +15734,20 @@ function cleanupSourceVersions(db, input) {
14951
15734
  WHERE ${retired}
14952
15735
  AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
14953
15736
  ORDER BY roots.retired_at, roots.id LIMIT ?
14954
- )`).run(input.projectId, input.terminalBefore, remaining).changes;
14955
- rowsDeleted += removed;
14956
- remaining -= removed;
15737
+ ) RETURNING version_id`).all(input.projectId, input.terminalBefore, remaining);
15738
+ const counts = new Map;
15739
+ for (const root of removed)
15740
+ counts.set(root.version_id, (counts.get(root.version_id) ?? 0) + 1);
15741
+ const drain = db.query("UPDATE ontology_sources SET root_count = root_count - ? WHERE version_id = ?");
15742
+ for (const [versionId, count] of counts)
15743
+ drain.run(count, versionId);
15744
+ rowsDeleted += removed.length;
15745
+ remaining -= removed.length;
14957
15746
  }
14958
15747
  const materializationsDeleted = remaining === 0 ? 0 : db.query(`DELETE FROM ontology_sources WHERE version_id IN (
14959
15748
  SELECT versions.version_id FROM ontology_sources AS versions
14960
- WHERE project_id = ? AND status = 'superseded' AND terminal_at < ?
14961
- AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
14962
- WHERE roots.version_id = versions.version_id)
15749
+ WHERE project_id = ? AND status = 'superseded' AND root_count = 0 AND terminal_at < ?
15750
+ AND ${PLAN_PURGED}
14963
15751
  ORDER BY terminal_at, source_id, materialization_id LIMIT ?
14964
15752
  )`).run(input.projectId, input.terminalBefore, remaining).changes;
14965
15753
  return { rowsDeleted, materializationsDeleted };
@@ -15045,7 +15833,7 @@ class SqliteOntologySourceStorage {
15045
15833
  throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
15046
15834
  }
15047
15835
  if (input.readyAt < manifest.created_at) {
15048
- throw new MaterializationValidationError4("Source readyAt cannot precede source createdAt.");
15836
+ throw new MaterializationValidationError5("Source readyAt cannot precede source createdAt.");
15049
15837
  }
15050
15838
  this.assertReadyState(manifest, input.rootCount, input.assertionCount);
15051
15839
  const result = this.db.query(`
@@ -15121,20 +15909,6 @@ class SqliteOntologySourceStorage {
15121
15909
  return purgeAbandonedSourceVersions(this.db, input);
15122
15910
  });
15123
15911
  }
15124
- async summarizeTerminal(input) {
15125
- return this.runRootOperation(() => {
15126
- assertNonblank(input.projectId, "Terminal source summary project id");
15127
- const summary = this.db.query(`
15128
- SELECT COUNT(*) AS terminal_count, MIN(terminal_at) AS oldest_terminal_at
15129
- FROM ontology_sources
15130
- WHERE project_id = ? AND status IN ('superseded', 'abandoned')
15131
- `).get(input.projectId);
15132
- return {
15133
- count: summary.terminal_count,
15134
- oldestTerminalAt: summary.oldest_terminal_at
15135
- };
15136
- });
15137
- }
15138
15912
  abandonCandidate(input) {
15139
15913
  const manifest = this.requireManifest(input.projectId, input.source.projectionId, input.materializationId);
15140
15914
  if (manifest.status === "abandoned") {
@@ -15163,7 +15937,7 @@ class SqliteOntologySourceStorage {
15163
15937
  }
15164
15938
  transitionToAbandoned(manifest, abandonedAt) {
15165
15939
  if (abandonedAt < manifest.created_at || manifest.ready_at !== null && abandonedAt < manifest.ready_at) {
15166
- throw new MaterializationValidationError4("Source abandonedAt cannot precede source creation or readiness.");
15940
+ throw new MaterializationValidationError5("Source abandonedAt cannot precede source creation or readiness.");
15167
15941
  }
15168
15942
  const changed = this.db.query(`
15169
15943
  UPDATE ontology_sources
@@ -15187,7 +15961,7 @@ class SqliteOntologySourceStorage {
15187
15961
  WHERE version_id = ?1
15188
15962
  `).get(manifest.version_id);
15189
15963
  if (counts.assertions !== assertionCount || counts.roots !== rootCount || counts.ordinals !== rootCount || rootCount > 0 && (counts.min_ordinal !== 0 || counts.max_ordinal !== rootCount - 1)) {
15190
- throw new MaterializationValidationError4("Source ready counts do not match the staged roots and assertions.");
15964
+ throw new MaterializationValidationError5("Source ready counts do not match the staged roots and assertions.");
15191
15965
  }
15192
15966
  assertSourceRootCoverage(this.db, manifest);
15193
15967
  const invalid = this.db.query(manifest.projection_kind === "link" ? `
@@ -15230,7 +16004,7 @@ class SqliteOntologySourceStorage {
15230
16004
  LIMIT 1
15231
16005
  `).get(manifest.version_id);
15232
16006
  if (invalid) {
15233
- throw new MaterializationValidationError4(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
16007
+ throw new MaterializationValidationError5(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
15234
16008
  }
15235
16009
  }
15236
16010
  findStageRows(manifest, rows) {
@@ -15414,7 +16188,7 @@ function decode(row) {
15414
16188
  }
15415
16189
 
15416
16190
  // src/ontology-storage/vectors.ts
15417
- import { MaterializationConflictError as MaterializationConflictError8 } from "@sixb/core/internal/materialization";
16191
+ import { MaterializationConflictError as MaterializationConflictError9 } from "@sixb/core/internal/materialization";
15418
16192
  class SqliteOntologyVectorStorage {
15419
16193
  db;
15420
16194
  runRootOperation;
@@ -15520,7 +16294,7 @@ class SqliteOntologyVectorStorage {
15520
16294
  }
15521
16295
  }
15522
16296
  function vectorConflict() {
15523
- return new MaterializationConflictError8("effective-state", "Vector changed since it was prepared.");
16297
+ return new MaterializationConflictError9("effective-state", "Vector changed since it was prepared.");
15524
16298
  }
15525
16299
 
15526
16300
  // src/ontology-storage/index.ts
@@ -15528,6 +16302,7 @@ class SqliteOntologyStorage {
15528
16302
  commits;
15529
16303
  sources;
15530
16304
  materializations;
16305
+ replacementPlans;
15531
16306
  vectorIndexing;
15532
16307
  vectors;
15533
16308
  outbox;
@@ -15535,6 +16310,7 @@ class SqliteOntologyStorage {
15535
16310
  this.commits = new SqliteOntologyCommitStorage(input.db, input.runRootOperation);
15536
16311
  this.sources = new SqliteOntologySourceStorage(input.db, input.runRootOperation);
15537
16312
  this.materializations = new SqliteOntologyMaterializationStorage(input.db, input.transactionContext);
16313
+ this.replacementPlans = new SqliteOntologyReplacementPlanStorage(input.db, input.runRootOperation);
15538
16314
  this.vectorIndexing = new SqliteVectorIndexingStorage(input.db, input.runRootOperation, (session, projectId) => this.materializations.assertVectorSession(session, projectId));
15539
16315
  this.vectors = new SqliteOntologyVectorStorage(input.db, input.runRootOperation, (session, projectId, commitId) => this.materializations.assertVectorSession(session, projectId, commitId));
15540
16316
  this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation);
@@ -16137,7 +16913,7 @@ class SqliteProjectionRunStorage {
16137
16913
  }
16138
16914
  async list(input) {
16139
16915
  assertProjectionRunNonEmpty(input.projectId, "projectId");
16140
- if (input.statuses && input.statuses.length === 0) {
16916
+ if (input.statuses?.length === 0 || input.errorCodes?.length === 0) {
16141
16917
  return { runs: [], hasMore: false, total: 0 };
16142
16918
  }
16143
16919
  const whereClauses = ["project_id = ?"];
@@ -16170,6 +16946,10 @@ class SqliteProjectionRunStorage {
16170
16946
  whereClauses.push(`status IN (${input.statuses.map(() => "?").join(", ")})`);
16171
16947
  args.push(...input.statuses);
16172
16948
  }
16949
+ if (input.errorCodes) {
16950
+ whereClauses.push(`json_extract(error, '$.code') IN (${input.errorCodes.map(() => "?").join(", ")})`);
16951
+ args.push(...input.errorCodes);
16952
+ }
16173
16953
  if (input.startedAfter) {
16174
16954
  whereClauses.push("COALESCE(started_at, queued_at) >= ?");
16175
16955
  args.push(input.startedAfter.toISOString());
@@ -16479,17 +17259,16 @@ class SqliteRulesStorage {
16479
17259
  }
16480
17260
  async listReconciliationPage(input) {
16481
17261
  assertPositiveLimit(input.limit);
17262
+ const { after } = input;
17263
+ const cursor = after ? [after.ruleId, after.objectTypeId, after.primaryId] : [];
16482
17264
  const rows = this.db.query(`
16483
17265
  SELECT project_id, rule_id, subject_kind, object_type_id, primary_id, triggered_at
16484
17266
  FROM rule_states
16485
- WHERE project_id = ? AND subject_kind = 'object'
16486
- AND (
16487
- ? IS NULL OR
16488
- (rule_id, object_type_id, primary_id) > (?, ?, ?)
16489
- )
16490
- ORDER BY rule_id ASC, object_type_id ASC, primary_id ASC
17267
+ WHERE project_id = ?
17268
+ ${cursor.length > 0 ? "AND (rule_id, subject_kind, object_type_id, primary_id) > (?, 'object', ?, ?)" : ""}
17269
+ ORDER BY rule_id ASC, subject_kind ASC, object_type_id ASC, primary_id ASC
16491
17270
  LIMIT ?
16492
- `).all(input.projectId, input.after?.ruleId ?? null, input.after?.ruleId ?? null, input.after?.objectTypeId ?? null, input.after?.primaryId ?? null, input.limit + 1);
17271
+ `).all(input.projectId, ...cursor, input.limit + 1);
16493
17272
  return reconciliationPage(rows, input.limit);
16494
17273
  }
16495
17274
  async applyTriggered(event) {
@@ -17080,10 +17859,8 @@ function isUniqueConstraintError4(error) {
17080
17859
 
17081
17860
  // src/testing.ts
17082
17861
  var adapters = new WeakMap;
17083
- function registerSqliteStorageTestingAdapter(storage, advanceConnectorConnectionTime) {
17084
- adapters.set(storage, {
17085
- advanceConnectorConnectionTime
17086
- });
17862
+ function registerSqliteStorageTestingAdapter(storage, adapter) {
17863
+ adapters.set(storage, adapter);
17087
17864
  }
17088
17865
 
17089
17866
  // src/timeseries-storage.ts
@@ -18239,6 +19016,7 @@ class SqliteStorage {
18239
19016
  shareSessions;
18240
19017
  connectorConnections;
18241
19018
  fileUploadSessions;
19019
+ fileDownloadGrants;
18242
19020
  migrators;
18243
19021
  connection;
18244
19022
  readConnection;
@@ -18290,8 +19068,12 @@ class SqliteStorage {
18290
19068
  this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope);
18291
19069
  this.connectorConnections = createOperationScopedFacade(stores.connectorConnections, scope);
18292
19070
  this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope);
19071
+ this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope);
18293
19072
  this.migrators = options.path ? createSqliteStorageMigrators(options.path) : [];
18294
- registerSqliteStorageTestingAdapter(this, (durationMs) => stores.connectorConnections.advanceTimeForTesting(durationMs));
19073
+ registerSqliteStorageTestingAdapter(this, {
19074
+ advanceConnectorConnectionTime: (durationMs) => stores.connectorConnections.advanceTimeForTesting(durationMs),
19075
+ db: this.connection.db
19076
+ });
18295
19077
  }
18296
19078
  async ping() {
18297
19079
  this.assertRootOperationAvailable();
@@ -18415,7 +19197,8 @@ function createSqliteStores(connection, options) {
18415
19197
  shareGrants,
18416
19198
  shareSessions,
18417
19199
  connectorConnections: new SqliteConnectorConnectionStorage(connection),
18418
- fileUploadSessions: new SqliteFileUploadSessionStorage(connection)
19200
+ fileUploadSessions: new SqliteFileUploadSessionStorage(connection),
19201
+ fileDownloadGrants: new SqliteFileDownloadGrantStorage(connection)
18419
19202
  };
18420
19203
  }
18421
19204
  export {
@@ -18423,4 +19206,4 @@ export {
18423
19206
  migrateSqliteStorage
18424
19207
  };
18425
19208
 
18426
- //# debugId=A2CEAFAB428734DF64756E2164756E21
19209
+ //# debugId=0E043C316BC5C57264756E2164756E21