@sixb/pg 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/index.js +471 -227
  2. package/dist/index.js.map +13 -11
  3. package/dist/migrations.d.ts.map +1 -1
  4. package/dist/ontology-storage/materialization-session.d.ts +3 -0
  5. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  6. package/dist/ontology-storage/materialization-state.d.ts +2 -1
  7. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  8. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  9. package/dist/ontology-storage/shared.d.ts +2 -0
  10. package/dist/ontology-storage/shared.d.ts.map +1 -1
  11. package/dist/ontology-storage/source-cleanup.d.ts +4 -0
  12. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -0
  13. package/dist/ontology-storage/source-roots.d.ts +24 -0
  14. package/dist/ontology-storage/source-roots.d.ts.map +1 -0
  15. package/dist/ontology-storage/sources.d.ts.map +1 -1
  16. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  17. package/package.json +3 -3
  18. package/src/connector-connection-storage.ts +3 -3
  19. package/src/migrations/038-outbox-publication-order.sql +5 -0
  20. package/src/migrations/039-projection-source-roots.sql +46 -0
  21. package/src/migrations/040-connector-optional-pkce.sql +3 -0
  22. package/src/migrations.ts +13 -0
  23. package/src/ontology-storage/materialization-session.ts +24 -8
  24. package/src/ontology-storage/materialization-state.ts +65 -86
  25. package/src/ontology-storage/materializations.ts +39 -13
  26. package/src/ontology-storage/shared.ts +10 -0
  27. package/src/ontology-storage/source-cleanup.ts +61 -0
  28. package/src/ontology-storage/source-roots.ts +251 -0
  29. package/src/ontology-storage/sources.ts +21 -57
  30. package/src/pg-projection-run-storage.ts +8 -2
package/dist/index.js CHANGED
@@ -3448,7 +3448,7 @@ class PgConnectorConnectionPersistence {
3448
3448
  ) VALUES (
3449
3449
  ${record.projectId}, ${record.connectorId}, ${record.id}, ${record.slot},
3450
3450
  ${record.initiatedByExecutionId}, ${record.stateHash},
3451
- ${jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
3451
+ ${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
3452
3452
  ${record.connectionRunId ?? null}, ${record.returnTo ?? null},
3453
3453
  ${record.callbackBindingHash ?? null}, ${record.reauthorizationId ?? null},
3454
3454
  ${record.reauthorizationRevision ?? null},
@@ -3741,7 +3741,7 @@ function authorizationAttemptFromRow(row) {
3741
3741
  slot: row.slot,
3742
3742
  initiatedByExecutionId: row.initiated_by_execution_id,
3743
3743
  stateHash: row.state_hash,
3744
- codeVerifier: row.code_verifier,
3744
+ ...row.code_verifier === null ? {} : { codeVerifier: row.code_verifier },
3745
3745
  redirectUri: row.redirect_uri,
3746
3746
  ...row.connection_run_id === null ? {} : { connectionRunId: row.connection_run_id },
3747
3747
  ...row.return_to === null ? {} : { returnTo: row.return_to },
@@ -7081,6 +7081,55 @@ ALTER TABLE executions
7081
7081
  ON DELETE RESTRICT;
7082
7082
  `;
7083
7083
 
7084
+ // src/migrations/039-projection-source-roots.sql
7085
+ var _039_projection_source_roots_default = `ALTER TABLE ontology_sources ADD COLUMN base_materialization_id TEXT;
7086
+ ALTER TABLE ontology_sources ADD COLUMN base_commit_id TEXT;
7087
+ ALTER TABLE ontology_sources ADD CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL));
7088
+
7089
+ -- A root selects one complete set of assertions. Unchanged roots keep their original version;
7090
+ -- activation only retires/replaces affected references, under the source commit fence.
7091
+ CREATE TABLE ontology_source_roots (
7092
+ project_id TEXT NOT NULL,
7093
+ source_id TEXT NOT NULL,
7094
+ materialization_id TEXT NOT NULL,
7095
+ root_sort_key TEXT NOT NULL,
7096
+ root_kind TEXT NOT NULL CHECK (root_kind IN ('object', 'link')),
7097
+ root_key JSONB NOT NULL,
7098
+ root JSONB NOT NULL,
7099
+ staging_ordinal BIGINT NOT NULL CHECK (staging_ordinal >= 0),
7100
+ deleted BOOLEAN NOT NULL DEFAULT FALSE,
7101
+ active BOOLEAN NOT NULL DEFAULT FALSE,
7102
+ retired_at TIMESTAMPTZ,
7103
+ PRIMARY KEY (project_id, source_id, materialization_id, root_sort_key),
7104
+ UNIQUE (project_id, source_id, materialization_id, staging_ordinal),
7105
+ FOREIGN KEY (project_id, source_id, materialization_id)
7106
+ REFERENCES ontology_sources(project_id, source_id, materialization_id) ON DELETE RESTRICT,
7107
+ CHECK (NOT active OR (NOT deleted AND retired_at IS NULL))
7108
+ );
7109
+ CREATE UNIQUE INDEX idx_ontology_source_roots_active
7110
+ ON ontology_source_roots (project_id, source_id, root_sort_key) WHERE active;
7111
+ CREATE INDEX idx_ontology_source_roots_lookup
7112
+ ON ontology_source_roots (project_id, root_sort_key, source_id) WHERE active;
7113
+ CREATE INDEX idx_ontology_source_roots_cleanup
7114
+ ON ontology_source_roots (project_id, retired_at, source_id, materialization_id, root_sort_key)
7115
+ WHERE NOT active AND retired_at IS NOT NULL;
7116
+
7117
+ INSERT INTO ontology_source_roots (
7118
+ project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root,
7119
+ staging_ordinal, active, retired_at
7120
+ )
7121
+ SELECT DISTINCT rows.project_id, rows.source_id, rows.materialization_id,
7122
+ rows.root_sort_key, rows.root_kind, rows.root_key, rows.root, rows.staging_ordinal,
7123
+ sources.status = 'active', sources.terminal_at
7124
+ FROM ontology_source_rows AS rows
7125
+ JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id);
7126
+
7127
+ ANALYZE ontology_source_roots;
7128
+
7129
+ ALTER TABLE projection_runs ADD COLUMN source_changes_read BIGINT
7130
+ CHECK (source_changes_read IS NULL OR source_changes_read >= 0);
7131
+ `;
7132
+
7084
7133
  // src/migrations/037-execution-requester-groups.sql
7085
7134
  var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
7086
7135
 
@@ -7105,6 +7154,20 @@ ALTER TABLE workflow_runs DROP COLUMN requester_group_ids;
7105
7154
  ALTER TABLE agent_runs DROP COLUMN requester_group_ids;
7106
7155
  `;
7107
7156
 
7157
+ // src/migrations/038-outbox-publication-order.sql
7158
+ var _038_outbox_publication_order_default = `-- Preserve the claim order without sorting every pending event for each batch.
7159
+ -- Keep the availability index for selective delayed/leased workloads.
7160
+ CREATE INDEX idx_ontology_outbox_publication_order
7161
+ ON ontology_outbox (project_id, created_at, commit_id, commit_ordinal)
7162
+ WHERE published_at IS NULL;
7163
+ `;
7164
+
7165
+ // src/migrations/040-connector-optional-pkce.sql
7166
+ var _040_connector_optional_pkce_default = `-- Existing attempts retain their encrypted S256 verifier. NULL denotes disabled PKCE.
7167
+ ALTER TABLE connector_authorization_attempts
7168
+ ALTER COLUMN code_verifier DROP NOT NULL;
7169
+ `;
7170
+
7108
7171
  // src/migrations.ts
7109
7172
  var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
7110
7173
  function pgSql(id, sqlText) {
@@ -7316,7 +7379,10 @@ var postgresStorageMigrations = defineMigrations({
7316
7379
  pgSql("034-device-authorizations", _034_device_authorizations_default),
7317
7380
  pgSql("035-share-grants", _035_share_grants_default),
7318
7381
  pgSql("036-share-sessions", _036_share_sessions_default),
7319
- pgSql("037-execution-requester-groups", _037_execution_requester_groups_default)
7382
+ pgSql("037-execution-requester-groups", _037_execution_requester_groups_default),
7383
+ pgSql("038-outbox-publication-order", _038_outbox_publication_order_default),
7384
+ pgSql("039-projection-source-roots", _039_projection_source_roots_default),
7385
+ pgSql("040-connector-optional-pkce", _040_connector_optional_pkce_default)
7320
7386
  ]
7321
7387
  });
7322
7388
 
@@ -9515,6 +9581,12 @@ function sourceRecord(row) {
9515
9581
  projectionRevision: row.projection_revision,
9516
9582
  ownershipHash: row.ownership_hash,
9517
9583
  ontologyRevision: row.ontology_revision,
9584
+ ...row.base_materialization_id === null ? {} : {
9585
+ base: {
9586
+ materializationId: row.base_materialization_id,
9587
+ lastCommitId: row.base_commit_id
9588
+ }
9589
+ },
9518
9590
  rootCount: row.root_count === null ? null : databaseSafeInteger(row.root_count, "Source root count"),
9519
9591
  assertionCount: row.assertion_count === null ? null : databaseSafeInteger(row.assertion_count, "Source assertion count"),
9520
9592
  createdAt: toIsoString2(row.created_at),
@@ -9683,7 +9755,7 @@ import {
9683
9755
  linkRefKey as linkRefKey3,
9684
9756
  linkRefSortKey,
9685
9757
  linkScopeSortKey as linkScopeSortKey2,
9686
- MaterializationConflictError as MaterializationConflictError5,
9758
+ MaterializationConflictError as MaterializationConflictError6,
9687
9759
  objectRefKey as objectRefKey3,
9688
9760
  objectRefSortKey,
9689
9761
  telemetryPointKey as telemetryPointKey3,
@@ -9702,7 +9774,7 @@ import {
9702
9774
  } from "@sixb/core/internal/ontology-storage-provider";
9703
9775
 
9704
9776
  // src/ontology-storage/materialization-session.ts
9705
- import { MaterializationConflictError as MaterializationConflictError3 } from "@sixb/core/internal/materialization";
9777
+ import { MaterializationConflictError as MaterializationConflictError4 } from "@sixb/core/internal/materialization";
9706
9778
  import {
9707
9779
  correlateMaterializationChunk,
9708
9780
  duplicateMaterializationWork as duplicateWork,
@@ -9716,9 +9788,9 @@ import {
9716
9788
  import {
9717
9789
  linkRefKey,
9718
9790
  linkScopeSortKey,
9719
- MaterializationConflictError as MaterializationConflictError2,
9791
+ MaterializationConflictError as MaterializationConflictError3,
9720
9792
  objectRefKey,
9721
- projectionEntityKey,
9793
+ projectionEntityKey as projectionEntityKey2,
9722
9794
  telemetryPointKey,
9723
9795
  telemetryPointSortKey
9724
9796
  } from "@sixb/core/internal/materialization";
@@ -9728,6 +9800,178 @@ import {
9728
9800
  linkSlotOverrideValue,
9729
9801
  startScopeAccumulator
9730
9802
  } from "@sixb/core/internal/ontology-storage-provider";
9803
+
9804
+ // src/ontology-storage/source-roots.ts
9805
+ import {
9806
+ MaterializationConflictError as MaterializationConflictError2,
9807
+ MaterializationValidationError,
9808
+ projectionEntityKey
9809
+ } from "@sixb/core/internal/materialization";
9810
+ import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider";
9811
+ function replacementSourceRows(sql, input) {
9812
+ const roots = input.incremental ? sql`
9813
+ SELECT roots.* FROM ontology_source_roots AS changed
9814
+ CROSS JOIN LATERAL (
9815
+ SELECT roots.* FROM ontology_source_roots AS roots
9816
+ WHERE roots.project_id = changed.project_id AND roots.source_id = changed.source_id
9817
+ AND roots.root_sort_key = changed.root_sort_key AND roots.active OFFSET 0
9818
+ ) AS roots
9819
+ WHERE changed.project_id = ${input.projectId} AND changed.source_id = ${input.sourceId}
9820
+ AND changed.materialization_id = ${input.materializationId}
9821
+ ` : sql`SELECT * FROM ontology_source_roots WHERE project_id = ${input.projectId} AND source_id = ${input.sourceId} AND active`;
9822
+ return sql`
9823
+ SELECT * FROM ontology_source_rows WHERE project_id = ${input.projectId}
9824
+ AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId}
9825
+ UNION ALL
9826
+ SELECT rows.* FROM (${roots}) AS roots
9827
+ CROSS JOIN LATERAL (
9828
+ SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.project_id = roots.project_id
9829
+ AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
9830
+ AND rows.root_sort_key = roots.root_sort_key OFFSET 0
9831
+ ) AS rows
9832
+ `;
9833
+ }
9834
+ async function stageSourceRoots(sql, manifest, input) {
9835
+ const roots = sourceStageRoots(manifest.projection_kind, input);
9836
+ if (roots.length === 0)
9837
+ return;
9838
+ if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
9839
+ throw new MaterializationValidationError("Root deletions require a source delta base.");
9840
+ }
9841
+ const values = jsonParameter2(sql, roots.map((root) => ({
9842
+ root_sort_key: root.sortKey,
9843
+ root_kind: root.root.kind,
9844
+ root_key: JSON.parse(root.rootKey),
9845
+ root: root.root,
9846
+ staging_ordinal: root.stagingOrdinal,
9847
+ deleted: root.deleted
9848
+ })));
9849
+ const inserted = await sql`
9850
+ INSERT INTO ontology_source_roots (
9851
+ project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
9852
+ ) SELECT ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
9853
+ root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
9854
+ FROM jsonb_to_recordset(${values}) AS input(
9855
+ root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
9856
+ ON CONFLICT DO NOTHING RETURNING root_sort_key
9857
+ `;
9858
+ if (inserted.length === roots.length)
9859
+ return;
9860
+ const insertedKeys = new Set(inserted.map((row) => row.root_sort_key));
9861
+ const skipped = roots.filter((root) => !insertedKeys.has(root.sortKey));
9862
+ const stored = await sql`
9863
+ SELECT root_sort_key, staging_ordinal, deleted FROM ontology_source_roots
9864
+ WHERE project_id = ${input.projectId} AND source_id = ${input.source.projectionId}
9865
+ AND materialization_id = ${input.materializationId}
9866
+ AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
9867
+ `;
9868
+ const existing = new Map(stored.map((root) => [root.root_sort_key, root]));
9869
+ if (skipped.some((root) => {
9870
+ const previous = existing.get(root.sortKey);
9871
+ return !previous || Number(previous.staging_ordinal) !== root.stagingOrdinal || previous.deleted !== root.deleted;
9872
+ }))
9873
+ throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
9874
+ }
9875
+ async function activateSourceRoots(sql, projectId, candidate, activation) {
9876
+ if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
9877
+ throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
9878
+ if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0)
9879
+ return;
9880
+ const affected = candidate.base_materialization_id === null ? sql`` : sql`AND root_sort_key IN (
9881
+ SELECT root_sort_key FROM ontology_source_roots
9882
+ WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
9883
+ AND materialization_id = ${activation.materializationId}
9884
+ )`;
9885
+ await sql`
9886
+ UPDATE ontology_source_roots AS roots SET active = FALSE, retired_at = ${activation.updatedAt}
9887
+ WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId} AND active
9888
+ ${affected}
9889
+ `;
9890
+ await sql`
9891
+ UPDATE ontology_source_roots SET active = NOT deleted,
9892
+ retired_at = CASE WHEN deleted THEN ${activation.updatedAt}::timestamptz ELSE NULL END
9893
+ WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
9894
+ AND materialization_id = ${activation.materializationId}
9895
+ `;
9896
+ }
9897
+ async function assertSourceRootCoverage(sql, manifest) {
9898
+ const [invalid] = await sql`
9899
+ SELECT 1 FROM ontology_source_roots AS roots
9900
+ WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
9901
+ AND materialization_id = ${manifest.materialization_id}
9902
+ AND roots.deleted = EXISTS (
9903
+ SELECT 1 FROM ontology_source_rows AS rows
9904
+ WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
9905
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
9906
+ ) LIMIT 1
9907
+ `;
9908
+ if (invalid)
9909
+ throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
9910
+ }
9911
+ function activeSourceRows(sql, projectId, refs) {
9912
+ const keys = refs.flatMap((entity) => {
9913
+ const entitySortKey = utf8SortKey(projectionEntityKey(entity));
9914
+ const roots = entity.kind === "object" ? [entitySortKey] : [
9915
+ entitySortKey,
9916
+ utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
9917
+ ];
9918
+ return roots.map((rootSortKey) => ({
9919
+ root_sort_key: rootSortKey,
9920
+ entity_sort_key: entitySortKey
9921
+ }));
9922
+ });
9923
+ return sql`
9924
+ SELECT rows.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)}) AS requested(root_sort_key TEXT, entity_sort_key TEXT)
9925
+ JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
9926
+ AND roots.active AND roots.root_sort_key = requested.root_sort_key
9927
+ JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
9928
+ AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
9929
+ AND rows.root_sort_key = roots.root_sort_key
9930
+ `;
9931
+ }
9932
+ function replacementAssertionRows(sql, input) {
9933
+ const keys = input.refs.map((ref) => ({
9934
+ entity_kind: ref.kind,
9935
+ entity_key: JSON.parse(projectionEntityKey(ref)),
9936
+ root_keys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
9937
+ utf8SortKey(projectionEntityKey(ref)),
9938
+ utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
9939
+ ]
9940
+ }));
9941
+ const previous = !input.includePrevious ? sql`` : sql`UNION ALL
9942
+ SELECT rows.* FROM (
9943
+ SELECT roots.* FROM ontology_source_roots AS roots
9944
+ WHERE roots.project_id = ${input.projectId} AND roots.source_id = ${input.sourceId}
9945
+ AND roots.active AND roots.root_sort_key = ANY(requested.root_keys)
9946
+ ${!input.incremental ? sql`` : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
9947
+ WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
9948
+ AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`}
9949
+
9950
+ OFFSET 0
9951
+ ) AS roots
9952
+ CROSS JOIN LATERAL (
9953
+ SELECT rows.* FROM ontology_source_rows AS rows
9954
+ WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
9955
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
9956
+ AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
9957
+ OFFSET 0
9958
+ ) AS rows
9959
+ `;
9960
+ return sql`
9961
+ SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)})
9962
+ AS requested(entity_kind TEXT, entity_key JSONB, root_keys TEXT[])
9963
+ CROSS JOIN LATERAL (
9964
+ SELECT rows.* FROM ontology_source_rows AS rows
9965
+ WHERE rows.project_id = ${input.projectId} AND rows.source_id = ${input.sourceId}
9966
+ AND rows.materialization_id = ${input.materializationId}
9967
+ AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
9968
+ ${previous}
9969
+ OFFSET 0
9970
+ ) AS selected
9971
+ `;
9972
+ }
9973
+
9974
+ // src/ontology-storage/materialization-state.ts
9731
9975
  var PG_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
9732
9976
  var PG_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
9733
9977
 
@@ -9760,22 +10004,7 @@ class PgMaterializationStateReader {
9760
10004
  JOIN requested USING (object_type_id, primary_id)
9761
10005
  WHERE objects.project_id = ${this.projectId}
9762
10006
  `,
9763
- this.sql`
9764
- WITH requested AS (
9765
- SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
9766
- AS requested_values(object_type_id TEXT, primary_id TEXT)
9767
- )
9768
- SELECT rows.*
9769
- FROM ontology_source_rows AS rows
9770
- JOIN requested USING (object_type_id, primary_id)
9771
- JOIN ontology_sources AS sources
9772
- ON sources.project_id = rows.project_id
9773
- AND sources.source_id = rows.source_id
9774
- AND sources.materialization_id = rows.materialization_id
9775
- WHERE rows.project_id = ${this.projectId}
9776
- AND rows.entity_kind = 'object'
9777
- AND sources.status = 'active'
9778
- `,
10007
+ activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "object", ref }))),
9779
10008
  this.sql`
9780
10009
  WITH requested AS (
9781
10010
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
@@ -9816,7 +10045,7 @@ class PgMaterializationStateReader {
9816
10045
  const effectiveRow = effective.get(key);
9817
10046
  return {
9818
10047
  ref: structuredClone(ref),
9819
- source: sourceObject(sources.get(projectionEntityKey({ kind: "object", ref }))),
10048
+ source: sourceObject(sources.get(projectionEntityKey2({ kind: "object", ref }))),
9820
10049
  override: overrides.get(key) ?? null,
9821
10050
  effective: effectiveRow ? effectiveObjectSnapshot(effectiveRow) : null,
9822
10051
  latestTelemetry: telemetry.get(key) ?? []
@@ -9852,26 +10081,7 @@ class PgMaterializationStateReader {
9852
10081
  JOIN requested USING (source_type_id, source_id, link_id, target_type_id, target_id)
9853
10082
  WHERE links.project_id = ${this.projectId}
9854
10083
  `,
9855
- this.sql`
9856
- WITH requested AS (
9857
- SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
9858
- source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
9859
- target_type_id TEXT, target_primary_id TEXT
9860
- )
9861
- )
9862
- SELECT rows.*
9863
- FROM ontology_source_rows AS rows
9864
- JOIN requested USING (
9865
- source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
9866
- )
9867
- JOIN ontology_sources AS sources
9868
- ON sources.project_id = rows.project_id
9869
- AND sources.source_id = rows.source_id
9870
- AND sources.materialization_id = rows.materialization_id
9871
- WHERE rows.project_id = ${this.projectId}
9872
- AND rows.entity_kind = 'link'
9873
- AND sources.status = 'active'
9874
- `,
10084
+ activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "link", ref }))),
9875
10085
  this.sql`
9876
10086
  WITH requested AS (
9877
10087
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
@@ -9910,7 +10120,7 @@ class PgMaterializationStateReader {
9910
10120
  const effectiveRow = effective.get(key);
9911
10121
  return {
9912
10122
  ref: structuredClone(ref),
9913
- source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
10123
+ source: sourceLink(sources.get(projectionEntityKey2({ kind: "link", ref }))),
9914
10124
  override: overrides.get(key) ?? null,
9915
10125
  slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
9916
10126
  effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null
@@ -9979,13 +10189,14 @@ class PgMaterializationStateReader {
9979
10189
  ON requested.source_type_id = rows.source_type_id
9980
10190
  AND requested.source_id = rows.source_primary_id
9981
10191
  AND requested.link_id = rows.link_id
9982
- JOIN ontology_sources AS sources
10192
+ JOIN ontology_source_roots AS sources
9983
10193
  ON sources.project_id = rows.project_id
9984
10194
  AND sources.source_id = rows.source_id
9985
10195
  AND sources.materialization_id = rows.materialization_id
10196
+ AND sources.root_sort_key = rows.root_sort_key
9986
10197
  WHERE rows.project_id = ${this.projectId}
9987
10198
  AND rows.entity_kind = 'link'
9988
- AND sources.status = 'active'
10199
+ AND sources.active
9989
10200
  `,
9990
10201
  this.sql`
9991
10202
  WITH requested AS (
@@ -10058,7 +10269,7 @@ class PgMaterializationStateReader {
10058
10269
  for (const row of rows) {
10059
10270
  const accumulator = accumulators.get(row.scope_sort_key);
10060
10271
  if (!accumulator) {
10061
- throw new MaterializationConflictError2("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
10272
+ throw new MaterializationConflictError3("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
10062
10273
  }
10063
10274
  const snapshot = effectiveLinkSnapshot(row);
10064
10275
  appendScopeSnapshot(accumulator, snapshot);
@@ -10127,30 +10338,32 @@ class PgMaterializationStateReader {
10127
10338
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10128
10339
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10129
10340
  FROM ontology_source_rows AS rows
10130
- JOIN ontology_sources AS sources
10341
+ JOIN ontology_source_roots AS sources
10131
10342
  ON sources.project_id = rows.project_id
10132
10343
  AND sources.source_id = rows.source_id
10133
10344
  AND sources.materialization_id = rows.materialization_id
10345
+ AND sources.root_sort_key = rows.root_sort_key
10134
10346
  JOIN requested
10135
10347
  ON requested.object_type_id = rows.source_type_id
10136
10348
  AND requested.primary_id = rows.source_primary_id
10137
10349
  WHERE rows.project_id = ${this.projectId}
10138
10350
  AND rows.entity_kind = 'link'
10139
- AND sources.status = 'active'
10351
+ AND sources.active
10140
10352
  UNION
10141
10353
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10142
10354
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10143
10355
  FROM ontology_source_rows AS rows
10144
- JOIN ontology_sources AS sources
10356
+ JOIN ontology_source_roots AS sources
10145
10357
  ON sources.project_id = rows.project_id
10146
10358
  AND sources.source_id = rows.source_id
10147
10359
  AND sources.materialization_id = rows.materialization_id
10360
+ AND sources.root_sort_key = rows.root_sort_key
10148
10361
  JOIN requested
10149
10362
  ON requested.object_type_id = rows.target_type_id
10150
10363
  AND requested.primary_id = rows.target_primary_id
10151
10364
  WHERE rows.project_id = ${this.projectId}
10152
10365
  AND rows.entity_kind = 'link'
10153
- AND sources.status = 'active'
10366
+ AND sources.active
10154
10367
  ), selected AS (
10155
10368
  SELECT DISTINCT links.*,
10156
10369
  ${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
@@ -10169,13 +10382,15 @@ class PgMaterializationStateReader {
10169
10382
  }
10170
10383
  async* replacementIdentities(input) {
10171
10384
  await this.prepareReplacementIdentities(input);
10385
+ await this.sql`ANALYZE ${this.sql(PG_REPLACEMENT_WORK_TABLE)}`;
10172
10386
  let cursor = null;
10173
10387
  while (true) {
10388
+ const after = cursor === null ? this.sql`` : this.sql`AND sort_key > ${cursor}`;
10174
10389
  const rows = await this.sql`
10175
10390
  SELECT entity_kind, identity_key, sort_key, diff_required
10176
10391
  FROM ${this.sql(PG_REPLACEMENT_WORK_TABLE)}
10177
10392
  WHERE session_id = ${input.sessionId} AND entity_kind = ${input.kind}
10178
- AND (${cursor}::text IS NULL OR sort_key > ${cursor})
10393
+ ${after}
10179
10394
  ORDER BY sort_key
10180
10395
  LIMIT ${input.pageRows}
10181
10396
  `;
@@ -10190,19 +10405,19 @@ class PgMaterializationStateReader {
10190
10405
  const candidates = await this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
10191
10406
  return base.map((state) => ({
10192
10407
  ref: state.ref,
10193
- candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey({ kind: "object", ref: state.ref }))),
10408
+ candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey2({ kind: "object", ref: state.ref }))),
10194
10409
  override: state.override,
10195
10410
  effective: state.effective,
10196
10411
  latestTelemetry: state.latestTelemetry
10197
10412
  }));
10198
10413
  }
10199
- async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities) {
10414
+ async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities, incremental = false) {
10200
10415
  const refs = identities.map((identity) => identity.ref);
10201
10416
  const base = await this.linkStates(refs);
10202
- const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })));
10417
+ const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })), incremental);
10203
10418
  const candidate = replacements.byMaterialization.get(candidateMaterializationId);
10204
10419
  return base.map((state, index) => {
10205
- const key = projectionEntityKey({ kind: "link", ref: state.ref });
10420
+ const key = projectionEntityKey2({ kind: "link", ref: state.ref });
10206
10421
  return {
10207
10422
  ref: state.ref,
10208
10423
  candidateSource: replacements.owned.has(key) ? sourceLink(candidate?.get(key)) : state.source,
@@ -10269,33 +10484,23 @@ class PgMaterializationStateReader {
10269
10484
  `;
10270
10485
  return new Map(rows.map((row) => [linkRefKey(linkRefFromColumns(row)), row.last_commit_id]));
10271
10486
  }
10272
- async replacementSources(sourceId, materializationIds, refs) {
10487
+ async replacementSources(sourceId, materializationIds, refs, incremental = false) {
10273
10488
  if (materializationIds.length === 0 || refs.length === 0) {
10274
10489
  return { owned: new Set, byMaterialization: new Map };
10275
10490
  }
10276
- const keys = refs.map((ref) => ({
10277
- entity_kind: ref.kind,
10278
- entity_key: JSON.parse(projectionEntityKey(ref))
10279
- }));
10280
- const rows = await this.sql`
10281
- WITH requested AS (
10282
- SELECT *
10283
- FROM jsonb_to_recordset(${jsonParameter2(this.sql, keys)})
10284
- AS requested_values(entity_kind TEXT, entity_key JSONB)
10285
- )
10286
- SELECT rows.* FROM ontology_source_rows AS rows
10287
- JOIN requested USING (entity_kind, entity_key)
10288
- WHERE rows.project_id = ${this.projectId}
10289
- AND rows.source_id = ${sourceId}
10290
- AND rows.materialization_id = ANY(
10291
- ${this.sql.array([...new Set(materializationIds)])}::text[]
10292
- )
10293
- `;
10491
+ const rows = await replacementAssertionRows(this.sql, {
10492
+ projectId: this.projectId,
10493
+ sourceId,
10494
+ materializationId: materializationIds[0],
10495
+ incremental,
10496
+ includePrevious: materializationIds.length > 1,
10497
+ refs
10498
+ });
10294
10499
  const owned = new Set;
10295
10500
  const byMaterialization = new Map;
10296
10501
  for (const row of rows) {
10297
10502
  const source = storedSource(row);
10298
- const key = projectionEntityKey(source.assertion);
10503
+ const key = projectionEntityKey2(source.assertion);
10299
10504
  owned.add(key);
10300
10505
  const materialization = byMaterialization.get(row.materialization_id) ?? new Map;
10301
10506
  materialization.set(key, source);
@@ -10311,20 +10516,19 @@ class PgMaterializationStateReader {
10311
10516
  await this.prepareReplacementLinks(input);
10312
10517
  }
10313
10518
  async prepareReplacementObjects(input) {
10314
- const materializationIds = [
10315
- input.candidateMaterializationId,
10316
- ...input.previousMaterializationId ? [input.previousMaterializationId] : []
10317
- ];
10519
+ const rows = replacementSourceRows(this.sql, {
10520
+ projectId: this.projectId,
10521
+ sourceId: input.sourceId,
10522
+ materializationId: input.candidateMaterializationId,
10523
+ incremental: input.incremental ?? false
10524
+ });
10318
10525
  await this.sql`
10319
10526
  WITH selected AS (
10320
10527
  SELECT DISTINCT object_type_id, primary_id,
10321
10528
  ${this.sql.unsafe(objectKeyExpression("rows"))} AS identity_key,
10322
10529
  ${this.sql.unsafe(objectSortExpression("rows"))} AS sort_key
10323
- FROM ontology_source_rows AS rows
10324
- WHERE project_id = ${this.projectId}
10325
- AND source_id = ${input.sourceId}
10326
- AND entity_kind = 'object'
10327
- AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
10530
+ FROM (${rows}) AS rows
10531
+ WHERE entity_kind = 'object'
10328
10532
  )
10329
10533
  INSERT INTO ${this.sql(PG_REPLACEMENT_WORK_TABLE)} (
10330
10534
  session_id, entity_kind, identity_key, sort_key, diff_required
@@ -10337,10 +10541,13 @@ class PgMaterializationStateReader {
10337
10541
  `;
10338
10542
  }
10339
10543
  async prepareReplacementLinks(input) {
10340
- const materializationIds = [
10341
- input.candidateMaterializationId,
10342
- ...input.previousMaterializationId ? [input.previousMaterializationId] : []
10343
- ];
10544
+ await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
10545
+ const rows = replacementSourceRows(this.sql, {
10546
+ projectId: this.projectId,
10547
+ sourceId: input.sourceId,
10548
+ materializationId: input.candidateMaterializationId,
10549
+ incremental: input.incremental ?? false
10550
+ });
10344
10551
  await this.sql`
10345
10552
  WITH incident_objects AS (
10346
10553
  SELECT DISTINCT payload->'ref'->>'objectTypeId' AS object_type_id,
@@ -10350,11 +10557,8 @@ class PgMaterializationStateReader {
10350
10557
  ), replacement_links AS (
10351
10558
  SELECT source_type_id, source_primary_id AS source_id, link_id,
10352
10559
  target_type_id, target_primary_id AS target_id
10353
- FROM ontology_source_rows
10354
- WHERE project_id = ${this.projectId}
10355
- AND source_id = ${input.sourceId}
10356
- AND entity_kind = 'link'
10357
- AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
10560
+ FROM (${rows}) AS rows
10561
+ WHERE entity_kind = 'link'
10358
10562
  ), incident_links AS (
10359
10563
  SELECT links.source_type_id, links.source_id, links.link_id,
10360
10564
  links.target_type_id, links.target_id
@@ -10393,30 +10597,32 @@ class PgMaterializationStateReader {
10393
10597
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10394
10598
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10395
10599
  FROM ontology_source_rows AS rows
10396
- JOIN ontology_sources AS sources
10600
+ JOIN ontology_source_roots AS sources
10397
10601
  ON sources.project_id = rows.project_id
10398
10602
  AND sources.source_id = rows.source_id
10399
10603
  AND sources.materialization_id = rows.materialization_id
10604
+ AND sources.root_sort_key = rows.root_sort_key
10400
10605
  JOIN incident_objects
10401
10606
  ON incident_objects.object_type_id = rows.source_type_id
10402
10607
  AND incident_objects.primary_id = rows.source_primary_id
10403
10608
  WHERE rows.project_id = ${this.projectId}
10404
10609
  AND rows.entity_kind = 'link'
10405
- AND sources.status = 'active'
10610
+ AND sources.active
10406
10611
  UNION
10407
10612
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10408
10613
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10409
10614
  FROM ontology_source_rows AS rows
10410
- JOIN ontology_sources AS sources
10615
+ JOIN ontology_source_roots AS sources
10411
10616
  ON sources.project_id = rows.project_id
10412
10617
  AND sources.source_id = rows.source_id
10413
10618
  AND sources.materialization_id = rows.materialization_id
10619
+ AND sources.root_sort_key = rows.root_sort_key
10414
10620
  JOIN incident_objects
10415
10621
  ON incident_objects.object_type_id = rows.target_type_id
10416
10622
  AND incident_objects.primary_id = rows.target_primary_id
10417
10623
  WHERE rows.project_id = ${this.projectId}
10418
10624
  AND rows.entity_kind = 'link'
10419
- AND sources.status = 'active'
10625
+ AND sources.active
10420
10626
  ), diff_links AS (
10421
10627
  SELECT * FROM replacement_links
10422
10628
  UNION SELECT * FROM incident_links
@@ -10484,9 +10690,9 @@ function activeSourceMap(rows) {
10484
10690
  const result = new Map;
10485
10691
  for (const row of rows) {
10486
10692
  const source = storedSource(row);
10487
- const key = projectionEntityKey(source.assertion);
10693
+ const key = projectionEntityKey2(source.assertion);
10488
10694
  if (result.has(key)) {
10489
- throw new MaterializationConflictError2("source-materialization", `Multiple active sources assert ${key}.`);
10695
+ throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert ${key}.`);
10490
10696
  }
10491
10697
  result.set(key, source);
10492
10698
  }
@@ -10501,7 +10707,7 @@ function activeScopeSourceMap(rows) {
10501
10707
  const ref = source.assertion.ref;
10502
10708
  const key = linkScopeSortKey(ref.source, ref.linkId);
10503
10709
  if (result.has(key)) {
10504
- throw new MaterializationConflictError2("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
10710
+ throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
10505
10711
  }
10506
10712
  result.set(key, source);
10507
10713
  }
@@ -10565,7 +10771,7 @@ function storedSource(row) {
10565
10771
  }
10566
10772
  function effectiveObjectSnapshot(row) {
10567
10773
  if (!row.last_commit_id) {
10568
- throw new MaterializationConflictError2("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
10774
+ throw new MaterializationConflictError3("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
10569
10775
  }
10570
10776
  return {
10571
10777
  ref: objectRefFromColumns(row),
@@ -10578,7 +10784,7 @@ function effectiveObjectSnapshot(row) {
10578
10784
  }
10579
10785
  function effectiveLinkSnapshot(row) {
10580
10786
  if (!row.last_commit_id) {
10581
- throw new MaterializationConflictError2("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
10787
+ throw new MaterializationConflictError3("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
10582
10788
  }
10583
10789
  return {
10584
10790
  ref: linkRefFromColumns(row),
@@ -10593,7 +10799,7 @@ function effectiveLinkSnapshot(row) {
10593
10799
  function storedPoint(row) {
10594
10800
  const at = toIsoString2(row.at);
10595
10801
  if (!row.last_commit_id) {
10596
- throw new MaterializationConflictError2("timeseries-point", `Telemetry point ${telemetryPointSortKey({
10802
+ throw new MaterializationConflictError3("timeseries-point", `Telemetry point ${telemetryPointSortKey({
10597
10803
  object: { objectTypeId: row.object_type_id, primaryId: row.object_id },
10598
10804
  propertyId: row.property_id
10599
10805
  }, at)} lacks materializer provenance.`);
@@ -10652,6 +10858,9 @@ function jsonTupleExpression(parts) {
10652
10858
 
10653
10859
  // src/ontology-storage/materialization-session.ts
10654
10860
  class PgMaterializationSessionState extends ProviderMaterializationSessionState {
10861
+ stagedWorkCount = 0;
10862
+ changedObjects = 0;
10863
+ changedLinks = 0;
10655
10864
  appliedPlanCursor = null;
10656
10865
  appliedEventCursor = null;
10657
10866
  }
@@ -10668,7 +10877,7 @@ class PgMaterializationSessions {
10668
10877
  }
10669
10878
  async create(header) {
10670
10879
  if (!this.context?.active) {
10671
- throw new MaterializationConflictError3("effective-state", "Materialization sessions require an active storage transaction.");
10880
+ throw new MaterializationConflictError4("effective-state", "Materialization sessions require an active storage transaction.");
10672
10881
  }
10673
10882
  await this.ensureTables();
10674
10883
  const session = new PgMaterializationSessionState(structuredClone(header), this.context.id);
@@ -10680,7 +10889,7 @@ class PgMaterializationSessions {
10680
10889
  require(session) {
10681
10890
  const value = this.sessions.get(session.providerToken);
10682
10891
  if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
10683
- throw new MaterializationConflictError3("effective-state", "Materialization session is inactive.");
10892
+ throw new MaterializationConflictError4("effective-state", "Materialization session is inactive.");
10684
10893
  }
10685
10894
  return value;
10686
10895
  }
@@ -10738,14 +10947,18 @@ class PgMaterializationSessions {
10738
10947
  }
10739
10948
  throw error;
10740
10949
  }
10950
+ session.stagedWorkCount += records.length;
10741
10951
  }
10742
10952
  async* stream(input) {
10743
10953
  const session = this.require(input.session);
10744
10954
  const stream = session.workStreams[input.order];
10745
10955
  if (stream.started) {
10746
- throw new MaterializationConflictError3("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
10956
+ throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
10747
10957
  }
10748
10958
  stream.started = true;
10959
+ if (!session.workSealed && session.stagedWorkCount >= 1e4) {
10960
+ await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
10961
+ }
10749
10962
  session.workSealed = true;
10750
10963
  let cursor = null;
10751
10964
  while (true) {
@@ -10766,10 +10979,13 @@ class PgMaterializationSessions {
10766
10979
  return [];
10767
10980
  const keys = refs.map((ref) => `object-existence:${JSON.stringify([ref.objectTypeId, ref.primaryId])}`);
10768
10981
  const rows = await this.sql`
10769
- SELECT unique_key, payload FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
10770
- WHERE session_id = ${session.id}
10771
- AND unique_key = ANY(${this.sql.array(keys)}::text[])
10772
- AND kind = 'object-existence'
10982
+ SELECT work.unique_key, work.payload
10983
+ FROM unnest(${this.sql.array(keys)}::text[]) AS requested(unique_key)
10984
+ CROSS JOIN LATERAL (
10985
+ SELECT unique_key, payload FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
10986
+ WHERE session_id = ${session.id} AND unique_key = requested.unique_key
10987
+ OFFSET 0
10988
+ ) AS work
10773
10989
  `;
10774
10990
  const found = new Map(rows.map((row) => [row.unique_key, row.payload]));
10775
10991
  return keys.flatMap((key) => {
@@ -10777,6 +10993,9 @@ class PgMaterializationSessions {
10777
10993
  if (!payload)
10778
10994
  return [];
10779
10995
  const record = structuredClone(payload);
10996
+ if (record.kind !== "object-existence") {
10997
+ invalidCorrelation("Object existence work has an invalid kind.");
10998
+ }
10780
10999
  return [{ ref: record.ref, exists: record.exists }];
10781
11000
  });
10782
11001
  }
@@ -10931,7 +11150,7 @@ function workCursor(row) {
10931
11150
  // src/ontology-storage/materialization-writer.ts
10932
11151
  import {
10933
11152
  linkRefKey as linkRefKey2,
10934
- MaterializationConflictError as MaterializationConflictError4,
11153
+ MaterializationConflictError as MaterializationConflictError5,
10935
11154
  objectRefKey as objectRefKey2,
10936
11155
  telemetryPointKey as telemetryPointKey2
10937
11156
  } from "@sixb/core/internal/materialization";
@@ -11447,7 +11666,7 @@ function effectiveLinkConflict(item) {
11447
11666
  return effectiveConflict(`Expected link ${linkIdentityKey(item)} changed.`);
11448
11667
  }
11449
11668
  function pointConflict(item) {
11450
- return new MaterializationConflictError4("timeseries-point", `Telemetry point ${telemetryPointKey2({
11669
+ return new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey2({
11451
11670
  object: { objectTypeId: item.objectTypeId, primaryId: item.objectId },
11452
11671
  propertyId: item.propertyId
11453
11672
  }, item.at)} changed.`);
@@ -11482,14 +11701,14 @@ class PgOntologyMaterializationStorage {
11482
11701
  const linkScopeRevisions = await reader.linkScopeRevisions(input.expected.linkScopes);
11483
11702
  for (const [index, expected] of input.expected.linkScopes.entries()) {
11484
11703
  if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
11485
- throw new MaterializationConflictError5("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
11704
+ throw new MaterializationConflictError6("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
11486
11705
  }
11487
11706
  }
11488
11707
  const points = await reader.exactPoints(input.expected.points, true);
11489
11708
  const pointRevisions = new Map(points.map((point) => [telemetryPointKey3(point.series, point.at), point.lastCommitId]));
11490
11709
  for (const expected of input.expected.points) {
11491
11710
  if ((pointRevisions.get(telemetryPointKey3(expected.series, expected.at)) ?? null) !== expected.lastCommitId) {
11492
- throw new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey3(expected.series, expected.at)} changed.`);
11711
+ throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey3(expected.series, expected.at)} changed.`);
11493
11712
  }
11494
11713
  }
11495
11714
  return session.publicSession();
@@ -11548,10 +11767,10 @@ class PgOntologyMaterializationStorage {
11548
11767
  const reader = new PgMaterializationStateReader(this.sql, session.header.commit.projectId);
11549
11768
  if (input.entityKind === "object") {
11550
11769
  if (replacement.projectionKind !== "object") {
11551
- throw new MaterializationConflictError5("source-materialization", "Link projection replacement cannot stream object state.");
11770
+ throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
11552
11771
  }
11553
11772
  if (replacement.objectStreamStarted) {
11554
- throw new MaterializationConflictError5("effective-state", "Replacement object state may only be streamed once per session.");
11773
+ throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
11555
11774
  }
11556
11775
  replacement.objectStreamStarted = true;
11557
11776
  for await (const identities of reader.replacementIdentities({
@@ -11559,6 +11778,7 @@ class PgOntologyMaterializationStorage {
11559
11778
  sourceId: replacement.sourceId,
11560
11779
  candidateMaterializationId: replacement.candidateMaterializationId,
11561
11780
  previousMaterializationId: replacement.previousMaterializationId,
11781
+ incremental: replacement.incremental,
11562
11782
  kind: "object",
11563
11783
  pageRows: input.pageRows
11564
11784
  })) {
@@ -11577,10 +11797,10 @@ class PgOntologyMaterializationStorage {
11577
11797
  return;
11578
11798
  }
11579
11799
  if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
11580
- throw new MaterializationConflictError5("effective-state", "Object projection replacement must fully stream object state before link state.");
11800
+ throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
11581
11801
  }
11582
11802
  if (replacement.linkStreamStarted) {
11583
- throw new MaterializationConflictError5("effective-state", "Replacement link state may only be streamed once per session.");
11803
+ throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
11584
11804
  }
11585
11805
  replacement.linkStreamStarted = true;
11586
11806
  const materializationIds = [
@@ -11592,6 +11812,7 @@ class PgOntologyMaterializationStorage {
11592
11812
  sourceId: replacement.sourceId,
11593
11813
  candidateMaterializationId: replacement.candidateMaterializationId,
11594
11814
  previousMaterializationId: replacement.previousMaterializationId,
11815
+ incremental: replacement.incremental,
11595
11816
  kind: "link",
11596
11817
  pageRows: input.pageRows
11597
11818
  })) {
@@ -11602,7 +11823,7 @@ class PgOntologyMaterializationStorage {
11602
11823
  }
11603
11824
  return identity;
11604
11825
  });
11605
- const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities);
11826
+ const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
11606
11827
  yield { objects: [], links };
11607
11828
  }
11608
11829
  this.sessions.require(input.session);
@@ -11625,6 +11846,8 @@ class PgOntologyMaterializationStorage {
11625
11846
  assertPlanChunkCorrelations(input.chunk, commit);
11626
11847
  const progress = await this.sessions.prepareChunkSequence(session, input.chunk);
11627
11848
  await this.writer.apply(commit.projectId, commit.id, input.chunk);
11849
+ session.changedObjects += input.chunk.effective.objectUpserts.length + input.chunk.effective.objectDeletes.length;
11850
+ session.changedLinks += input.chunk.effective.linkUpserts.length + input.chunk.effective.linkDeletes.length;
11628
11851
  this.sessions.commitChunkSequence(session, progress);
11629
11852
  }
11630
11853
  async finalize(input) {
@@ -11634,6 +11857,10 @@ class PgOntologyMaterializationStorage {
11634
11857
  for (const activation of input.finalization.sourceActivations) {
11635
11858
  await this.activateSource(session, activation);
11636
11859
  }
11860
+ if (session.changedObjects >= 1000)
11861
+ await this.sql`ANALYZE objects`;
11862
+ if (session.changedLinks >= 1000)
11863
+ await this.sql`ANALYZE links`;
11637
11864
  const record = await this.insertCommit(session.header, input);
11638
11865
  await this.sessions.release(session);
11639
11866
  return { commit: record };
@@ -11644,14 +11871,14 @@ class PgOntologyMaterializationStorage {
11644
11871
  async requireReplacement(session, input) {
11645
11872
  if (session.replacement) {
11646
11873
  if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
11647
- throw new MaterializationConflictError5("source-materialization", "Materialization session already owns another replacement union.");
11874
+ throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
11648
11875
  }
11649
11876
  return session.replacement;
11650
11877
  }
11651
11878
  const projectId = session.header.commit.projectId;
11652
11879
  const candidate = await this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId, true);
11653
11880
  if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
11654
- throw new MaterializationConflictError5("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
11881
+ throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
11655
11882
  }
11656
11883
  await assertProjectionExecution(this.sql, {
11657
11884
  projectId,
@@ -11661,12 +11888,13 @@ class PgOntologyMaterializationStorage {
11661
11888
  });
11662
11889
  const previous = await this.getActiveSource(projectId, input.source.projectionId, true);
11663
11890
  if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
11664
- throw new MaterializationConflictError5("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
11891
+ throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
11665
11892
  }
11666
11893
  session.replacement = {
11667
11894
  sourceId: input.source.projectionId,
11668
11895
  candidateMaterializationId: input.candidateMaterializationId,
11669
11896
  previousMaterializationId: previous?.materialization_id ?? null,
11897
+ incremental: candidate.base_materialization_id !== null,
11670
11898
  projectionKind: candidate.projection_kind,
11671
11899
  objectStreamStarted: false,
11672
11900
  objectStreamCompleted: false,
@@ -11686,7 +11914,7 @@ class PgOntologyMaterializationStorage {
11686
11914
  FOR UPDATE
11687
11915
  `;
11688
11916
  if (duplicateIdentity) {
11689
- throw new MaterializationConflictError5("idempotency", duplicateIdentity.id === header.commit.id ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
11917
+ throw new MaterializationConflictError6("idempotency", duplicateIdentity.id === header.commit.id ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
11690
11918
  }
11691
11919
  const origin = originColumns(header.commit.origin);
11692
11920
  if (origin.runId === null)
@@ -11709,7 +11937,7 @@ class PgOntologyMaterializationStorage {
11709
11937
  FOR UPDATE
11710
11938
  `;
11711
11939
  if (duplicateOrigin) {
11712
- throw new MaterializationConflictError5("run-correlation", "Ontology commit origin already has an authoritative commit.");
11940
+ throw new MaterializationConflictError6("run-correlation", "Ontology commit origin already has an authoritative commit.");
11713
11941
  }
11714
11942
  }
11715
11943
  async assertSources(expectedSources, projectId) {
@@ -11731,29 +11959,29 @@ class PgOntologyMaterializationStorage {
11731
11959
  }
11732
11960
  assertSourceRow(expected, active) {
11733
11961
  if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
11734
- throw new MaterializationConflictError5("projection-fence", `Source '${expected.source.projectionId}' changed.`);
11962
+ throw new MaterializationConflictError6("projection-fence", `Source '${expected.source.projectionId}' changed.`);
11735
11963
  }
11736
11964
  }
11737
11965
  assertObject(row, expected) {
11738
11966
  if (!expected.exists) {
11739
11967
  if (row) {
11740
- throw new MaterializationConflictError5("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
11968
+ throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
11741
11969
  }
11742
11970
  return;
11743
11971
  }
11744
11972
  if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
11745
- throw new MaterializationConflictError5("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
11973
+ throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
11746
11974
  }
11747
11975
  }
11748
11976
  assertLink(lastCommitId, expected) {
11749
11977
  if (!expected.exists) {
11750
11978
  if (lastCommitId !== undefined) {
11751
- throw new MaterializationConflictError5("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
11979
+ throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
11752
11980
  }
11753
11981
  return;
11754
11982
  }
11755
11983
  if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
11756
- throw new MaterializationConflictError5("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
11984
+ throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
11757
11985
  }
11758
11986
  }
11759
11987
  async assertFinalization(session, input) {
@@ -11762,7 +11990,8 @@ class PgOntologyMaterializationStorage {
11762
11990
  assertMaterializationFinalizationCorrelation(session, input);
11763
11991
  const laneCounts = await this.sessions.laneCounts(session);
11764
11992
  assertMaterializationLaneCompletion(session, laneCounts);
11765
- await this.assertFinalCardinality(session);
11993
+ if (laneCounts.cardinality > 0)
11994
+ await this.assertFinalCardinality(session);
11766
11995
  const eventCount = laneCounts.event;
11767
11996
  const [outbox] = await this.sql`
11768
11997
  SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum,
@@ -11795,17 +12024,18 @@ class PgOntologyMaterializationStorage {
11795
12024
  }
11796
12025
  }
11797
12026
  async assertFinalCardinality(session) {
11798
- const [violation] = await this.sql`
11799
- WITH work AS (
11800
- SELECT payload->>'scopeSortKey' AS scope_sort_key,
11801
- payload->>'linkSortKey' AS link_sort_key,
12027
+ const table = `ontology_cardinality_${session.id.replaceAll("-", "")}`;
12028
+ await this.sql.unsafe(`CREATE TEMP TABLE ${table} ON COMMIT DROP AS
12029
+ SELECT sort_one AS scope_sort_key, sort_two AS link_sort_key,
11802
12030
  (payload->>'occupied')::boolean AS occupied,
11803
12031
  payload->'ref'->'source'->>'objectTypeId' AS source_type_id,
11804
12032
  payload->'ref'->'source'->>'primaryId' AS source_id,
11805
12033
  payload->'ref'->>'linkId' AS link_id
11806
- FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
11807
- WHERE session_id = ${session.id} AND kind = 'cardinality'
11808
- AND payload->>'view' = 'effective'
12034
+ FROM ${PG_MATERIALIZATION_WORK_TABLE}
12035
+ WHERE session_id = $1 AND lane = 'cardinality' AND major_order = 0`, [session.id]);
12036
+ await this.sql.unsafe(`ANALYZE ${table}`);
12037
+ const [violation] = await this.sql.unsafe(`WITH work AS (
12038
+ SELECT * FROM ${table}
11809
12039
  ), duplicate AS (
11810
12040
  SELECT scope_sort_key FROM work WHERE occupied
11811
12041
  GROUP BY scope_sort_key HAVING COUNT(*) > 1
@@ -11815,10 +12045,10 @@ class PgOntologyMaterializationStorage {
11815
12045
  SELECT scope_sort_key, link_sort_key FROM work WHERE occupied
11816
12046
  ), actual AS (
11817
12047
  SELECT scopes.scope_sort_key,
11818
- ${this.sql.unsafe(linkSortExpression("links"))} AS link_sort_key
12048
+ ${linkSortExpression("links")} AS link_sort_key
11819
12049
  FROM scopes
11820
12050
  JOIN links USING (source_type_id, source_id, link_id)
11821
- WHERE links.project_id = ${session.header.commit.projectId}
12051
+ WHERE links.project_id = $1
11822
12052
  ), differences AS (
11823
12053
  (SELECT * FROM expected EXCEPT SELECT * FROM actual)
11824
12054
  UNION ALL
@@ -11827,8 +12057,8 @@ class PgOntologyMaterializationStorage {
11827
12057
  SELECT 'duplicate'::text AS reason FROM duplicate
11828
12058
  UNION ALL
11829
12059
  SELECT 'mismatch'::text AS reason FROM differences
11830
- LIMIT 1
11831
- `;
12060
+ LIMIT 1`, [session.header.commit.projectId]);
12061
+ await this.sql.unsafe(`DROP TABLE ${table}`);
11832
12062
  if (violation?.reason === "duplicate") {
11833
12063
  invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
11834
12064
  }
@@ -11924,7 +12154,7 @@ class PgOntologyMaterializationStorage {
11924
12154
  assertSourceActivationCorrelation(session, activation);
11925
12155
  const candidate = await this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId, true);
11926
12156
  if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
11927
- throw new MaterializationConflictError5("source-materialization", "Source activation candidate is missing or is not ready.");
12157
+ throw new MaterializationConflictError6("source-materialization", "Source activation candidate is missing or is not ready.");
11928
12158
  }
11929
12159
  await assertProjectionExecution(this.sql, {
11930
12160
  projectId: commit.projectId,
@@ -11937,6 +12167,7 @@ class PgOntologyMaterializationStorage {
11937
12167
  }
11938
12168
  const previous = await this.getActiveSource(commit.projectId, activation.source.projectionId, true);
11939
12169
  this.assertSourceRow(activation.expected, previous);
12170
+ await activateSourceRoots(this.sql, commit.projectId, candidate, activation);
11940
12171
  if (previous) {
11941
12172
  assertPinnedDatasetWatermark({
11942
12173
  datasetId: previous.dataset_id,
@@ -11958,7 +12189,7 @@ class PgOntologyMaterializationStorage {
11958
12189
  RETURNING materialization_id
11959
12190
  `;
11960
12191
  if (superseded.length !== 1) {
11961
- throw new MaterializationConflictError5("projection-fence", `Source '${activation.source.projectionId}' changed.`);
12192
+ throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
11962
12193
  }
11963
12194
  }
11964
12195
  let activated;
@@ -11977,12 +12208,12 @@ class PgOntologyMaterializationStorage {
11977
12208
  `;
11978
12209
  } catch (error) {
11979
12210
  if (isUniqueViolation(error)) {
11980
- throw new MaterializationConflictError5("projection-fence", `Source '${activation.source.projectionId}' changed.`);
12211
+ throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
11981
12212
  }
11982
12213
  throw error;
11983
12214
  }
11984
12215
  if (activated.length !== 1) {
11985
- throw new MaterializationConflictError5("source-materialization", "Source activation candidate changed.");
12216
+ throw new MaterializationConflictError6("source-materialization", "Source activation candidate changed.");
11986
12217
  }
11987
12218
  }
11988
12219
  async insertCommit(header, input) {
@@ -12009,7 +12240,7 @@ class PgOntologyMaterializationStorage {
12009
12240
  `;
12010
12241
  if (!rows[0]) {
12011
12242
  await this.assertCommitAbsent(header);
12012
- throw new MaterializationConflictError5("idempotency", "Ontology commit identity already exists.");
12243
+ throw new MaterializationConflictError6("idempotency", "Ontology commit identity already exists.");
12013
12244
  }
12014
12245
  return commitRecord(rows[0]);
12015
12246
  }
@@ -12057,8 +12288,8 @@ function databaseCount(value) {
12057
12288
  // src/ontology-storage/outbox.ts
12058
12289
  import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
12059
12290
  import {
12060
- MaterializationConflictError as MaterializationConflictError6,
12061
- MaterializationValidationError
12291
+ MaterializationConflictError as MaterializationConflictError7,
12292
+ MaterializationValidationError as MaterializationValidationError2
12062
12293
  } from "@sixb/core/internal/materialization";
12063
12294
  import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
12064
12295
  class PgOntologyOutboxStorage {
@@ -12074,7 +12305,7 @@ class PgOntologyOutboxStorage {
12074
12305
  const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
12075
12306
  assertNonblank(input.leaseId, "Ontology outbox lease id");
12076
12307
  if (leaseExpiry <= now) {
12077
- throw new MaterializationValidationError("Ontology outbox lease expiry must be later than the claim time.");
12308
+ throw new MaterializationValidationError2("Ontology outbox lease expiry must be later than the claim time.");
12078
12309
  }
12079
12310
  const rows = await sql`
12080
12311
  WITH candidates AS (
@@ -12104,7 +12335,7 @@ class PgOntologyOutboxStorage {
12104
12335
  return rows.map((row) => {
12105
12336
  const record = outboxRecord(row);
12106
12337
  if (record.leaseId === null || record.leaseExpiresAt === null) {
12107
- throw new MaterializationConflictError6("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
12338
+ throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
12108
12339
  }
12109
12340
  return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt };
12110
12341
  });
@@ -12225,7 +12456,7 @@ function validateLeaseIds(ids) {
12225
12456
  for (const id of ids) {
12226
12457
  assertNonblank(id, "Ontology outbox event id");
12227
12458
  if (seen.has(id)) {
12228
- throw new MaterializationValidationError(`Ontology outbox lease batch repeats event '${id}'.`);
12459
+ throw new MaterializationValidationError2(`Ontology outbox lease batch repeats event '${id}'.`);
12229
12460
  }
12230
12461
  seen.add(id);
12231
12462
  }
@@ -12236,11 +12467,11 @@ function assertLeaseInput(input) {
12236
12467
  assertNonblank(input.leaseId, "Ontology outbox lease id");
12237
12468
  }
12238
12469
  function leaseConflict() {
12239
- return new MaterializationConflictError6("outbox-lease", "Ontology outbox lease does not match.");
12470
+ return new MaterializationConflictError7("outbox-lease", "Ontology outbox lease does not match.");
12240
12471
  }
12241
12472
 
12242
12473
  // src/ontology-storage/sources.ts
12243
- import { MaterializationValidationError as MaterializationValidationError2 } from "@sixb/core/internal/materialization";
12474
+ import { MaterializationValidationError as MaterializationValidationError3 } from "@sixb/core/internal/materialization";
12244
12475
  import {
12245
12476
  assertSourceBeginInput as assertBeginInput,
12246
12477
  assertSourceExecutionIdentity as assertExecutionIdentity,
@@ -12254,8 +12485,61 @@ import {
12254
12485
  sourceMaterializationIdentity,
12255
12486
  sourceStageRow,
12256
12487
  sourceStageRows,
12257
- utf8SortKey
12488
+ utf8SortKey as utf8SortKey2
12258
12489
  } from "@sixb/core/internal/ontology-storage-provider";
12490
+
12491
+ // src/ontology-storage/source-cleanup.ts
12492
+ async function cleanupSourceVersions(sql, input) {
12493
+ let remaining = input.limit;
12494
+ const removedRows = await sql`
12495
+ WITH selected AS (
12496
+ SELECT rows.ctid FROM ontology_source_roots AS roots
12497
+ JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
12498
+ JOIN ontology_source_rows AS rows
12499
+ ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
12500
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
12501
+ WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
12502
+ AND sources.status IN ('superseded', 'abandoned')
12503
+ ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
12504
+ LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
12505
+ ) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
12506
+ `;
12507
+ let rowsDeleted = removedRows.length;
12508
+ remaining -= rowsDeleted;
12509
+ if (remaining > 0) {
12510
+ const removed = await sql`
12511
+ WITH selected AS (
12512
+ SELECT roots.ctid FROM ontology_source_roots AS roots
12513
+ JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
12514
+ WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
12515
+ AND sources.status IN ('superseded', 'abandoned')
12516
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
12517
+ WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
12518
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
12519
+ ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
12520
+ LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
12521
+ ) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
12522
+ `;
12523
+ rowsDeleted += removed.length;
12524
+ remaining -= removed.length;
12525
+ }
12526
+ const removedManifests = remaining === 0 ? [] : await sql`
12527
+ WITH selected AS (
12528
+ SELECT sources.ctid FROM ontology_sources AS sources
12529
+ WHERE project_id = ${input.projectId} AND status IN ('superseded', 'abandoned') AND terminal_at < ${input.terminalBefore}
12530
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
12531
+ WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
12532
+ AND roots.materialization_id = sources.materialization_id)
12533
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
12534
+ WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
12535
+ AND rows.materialization_id = sources.materialization_id)
12536
+ ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
12537
+ ) DELETE FROM ontology_sources AS sources USING selected WHERE sources.ctid = selected.ctid RETURNING 1
12538
+ `;
12539
+ return { rowsDeleted, materializationsDeleted: removedManifests.length };
12540
+ }
12541
+
12542
+ // src/ontology-storage/sources.ts
12259
12543
  class PgOntologySourceStorage {
12260
12544
  runRootOperation;
12261
12545
  constructor(runRootOperation) {
@@ -12286,14 +12570,15 @@ class PgOntologySourceStorage {
12286
12570
  dataset_id, dataset_version_id, dataset_version_created_at,
12287
12571
  projection_revision, ownership_hash, ontology_revision,
12288
12572
  root_count, assertion_count, created_at, ready_at, activated_at,
12289
- terminal_at, last_commit_id, updated_at
12573
+ terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
12290
12574
  ) VALUES (
12291
12575
  ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
12292
12576
  ${input.execution.projectionRunId}, ${input.projectionKind}, 'replacement',
12293
12577
  'staging', ${input.execution.executionToken}, ${input.datasetVersion.datasetId},
12294
12578
  ${input.datasetVersion.versionId}, ${input.datasetVersion.createdAt},
12295
12579
  ${input.projectionRevision}, ${input.ownershipHash}, ${input.ontologyRevision},
12296
- NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt}
12580
+ NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt},
12581
+ ${input.base?.materializationId ?? null}, ${input.base?.lastCommitId ?? null}
12297
12582
  )
12298
12583
  ON CONFLICT DO NOTHING
12299
12584
  RETURNING *
@@ -12322,6 +12607,7 @@ class PgOntologySourceStorage {
12322
12607
  const rows = sourceStageRows(manifest.projection_kind, input.rows);
12323
12608
  const existing = await this.findStageRows(sql, input, rows);
12324
12609
  const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
12610
+ await stageSourceRoots(sql, manifest, input);
12325
12611
  await this.insertStageRows(sql, input, pending);
12326
12612
  return { inserted: pending.length, unchanged };
12327
12613
  });
@@ -12345,7 +12631,10 @@ class PgOntologySourceStorage {
12345
12631
  throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
12346
12632
  }
12347
12633
  if (input.readyAt < toIsoString2(manifest.created_at)) {
12348
- throw new MaterializationValidationError2("Source readyAt cannot precede source createdAt.");
12634
+ throw new MaterializationValidationError3("Source readyAt cannot precede source createdAt.");
12635
+ }
12636
+ if (input.assertionCount >= 1000 || input.rootCount >= 1000) {
12637
+ await sql`ANALYZE ontology_source_rows, ontology_source_roots`;
12349
12638
  }
12350
12639
  await this.assertReadyState(sql, manifest, input.rootCount, input.assertionCount);
12351
12640
  const rows = await sql`
@@ -12392,61 +12681,7 @@ class PgOntologySourceStorage {
12392
12681
  assertNonblank(input.projectId, "Terminal source cleanup project id");
12393
12682
  assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true);
12394
12683
  assertPositiveInteger(input.limit, "Terminal source cleanup limit");
12395
- const manifests = await sql`
12396
- SELECT * FROM ontology_sources
12397
- WHERE project_id = ${input.projectId}
12398
- AND status IN ('superseded', 'abandoned')
12399
- AND terminal_at < ${input.terminalBefore}
12400
- ORDER BY terminal_at, source_id, materialization_id
12401
- LIMIT ${input.limit}
12402
- FOR UPDATE SKIP LOCKED
12403
- `;
12404
- let remaining = input.limit;
12405
- let rowsDeleted = 0;
12406
- let materializationsDeleted = 0;
12407
- for (const manifest of manifests) {
12408
- if (remaining === 0)
12409
- break;
12410
- const deletedRows = await sql`
12411
- WITH selected AS (
12412
- SELECT rows.ctid
12413
- FROM ontology_source_rows AS rows
12414
- WHERE rows.project_id = ${manifest.project_id}
12415
- AND rows.source_id = ${manifest.source_id}
12416
- AND rows.materialization_id = ${manifest.materialization_id}
12417
- ORDER BY rows.entity_sort_key
12418
- LIMIT ${remaining}
12419
- FOR UPDATE SKIP LOCKED
12420
- )
12421
- DELETE FROM ontology_source_rows AS rows
12422
- USING selected
12423
- WHERE rows.ctid = selected.ctid
12424
- RETURNING rows.entity_sort_key
12425
- `;
12426
- rowsDeleted += deletedRows.length;
12427
- remaining -= deletedRows.length;
12428
- if (remaining === 0)
12429
- break;
12430
- const removed = await sql`
12431
- DELETE FROM ontology_sources AS sources
12432
- WHERE sources.project_id = ${manifest.project_id}
12433
- AND sources.source_id = ${manifest.source_id}
12434
- AND sources.materialization_id = ${manifest.materialization_id}
12435
- AND sources.status IN ('superseded', 'abandoned')
12436
- AND sources.terminal_at = ${manifest.terminal_at}
12437
- AND sources.terminal_at < ${input.terminalBefore}
12438
- AND NOT EXISTS (
12439
- SELECT 1 FROM ontology_source_rows AS rows
12440
- WHERE rows.project_id = sources.project_id
12441
- AND rows.source_id = sources.source_id
12442
- AND rows.materialization_id = sources.materialization_id
12443
- )
12444
- RETURNING sources.materialization_id
12445
- `;
12446
- materializationsDeleted += removed.length;
12447
- remaining -= removed.length;
12448
- }
12449
- return { rowsDeleted, materializationsDeleted };
12684
+ return cleanupSourceVersions(sql, input);
12450
12685
  });
12451
12686
  }
12452
12687
  async summarizeTerminal(input) {
@@ -12494,7 +12729,7 @@ class PgOntologySourceStorage {
12494
12729
  }
12495
12730
  async transitionToAbandoned(sql, manifest, abandonedAt) {
12496
12731
  if (abandonedAt < toIsoString2(manifest.created_at) || manifest.ready_at !== null && abandonedAt < toIsoString2(manifest.ready_at)) {
12497
- throw new MaterializationValidationError2("Source abandonedAt cannot precede source creation or readiness.");
12732
+ throw new MaterializationValidationError3("Source abandonedAt cannot precede source creation or readiness.");
12498
12733
  }
12499
12734
  const rows = await sql`
12500
12735
  UPDATE ontology_sources
@@ -12509,14 +12744,19 @@ class PgOntologySourceStorage {
12509
12744
  if (!rows[0]) {
12510
12745
  throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
12511
12746
  }
12747
+ await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
12748
+ WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
12749
+ AND materialization_id = ${manifest.materialization_id} AND NOT active`;
12512
12750
  return sourceRecord(rows[0]);
12513
12751
  }
12514
12752
  async assertReadyState(sql, manifest, rootCount, assertionCount) {
12515
12753
  const [counts] = await sql`
12516
- SELECT COUNT(*) AS assertions, COUNT(DISTINCT root_key) AS roots,
12754
+ SELECT (SELECT COUNT(*) FROM ontology_source_rows
12755
+ WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
12756
+ AND materialization_id = ${manifest.materialization_id}) AS assertions, COUNT(*) AS roots,
12517
12757
  COUNT(DISTINCT staging_ordinal) AS ordinals,
12518
12758
  MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
12519
- FROM ontology_source_rows
12759
+ FROM ontology_source_roots
12520
12760
  WHERE project_id = ${manifest.project_id}
12521
12761
  AND source_id = ${manifest.source_id}
12522
12762
  AND materialization_id = ${manifest.materialization_id}
@@ -12527,8 +12767,9 @@ class PgOntologySourceStorage {
12527
12767
  const minimum = counts?.min_ordinal === null ? null : Number(counts?.min_ordinal);
12528
12768
  const maximum = counts?.max_ordinal === null ? null : Number(counts?.max_ordinal);
12529
12769
  if (assertions !== assertionCount || roots !== rootCount || ordinals !== rootCount || rootCount > 0 && (minimum !== 0 || maximum !== rootCount - 1)) {
12530
- throw new MaterializationValidationError2("Source ready counts do not match the staged roots and assertions.");
12770
+ throw new MaterializationValidationError3("Source ready counts do not match the staged roots and assertions.");
12531
12771
  }
12772
+ await assertSourceRootCoverage(sql, manifest);
12532
12773
  const [invalid] = manifest.projection_kind === "link" ? await sql`
12533
12774
  SELECT root_key
12534
12775
  FROM ontology_source_rows
@@ -12562,15 +12803,15 @@ class PgOntologySourceStorage {
12562
12803
  LIMIT 1
12563
12804
  `;
12564
12805
  if (invalid) {
12565
- throw new MaterializationValidationError2(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
12806
+ throw new MaterializationValidationError3(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
12566
12807
  }
12567
12808
  }
12568
12809
  async findStageRows(sql, input, rows) {
12569
12810
  if (rows.length === 0)
12570
12811
  return [];
12571
- const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey(row.rootKey)))];
12812
+ const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.rootKey)))];
12572
12813
  const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
12573
- const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey(row.entityKey)))];
12814
+ const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
12574
12815
  const existing = await sql`
12575
12816
  SELECT * FROM ontology_source_rows
12576
12817
  WHERE project_id = ${input.projectId}
@@ -12593,10 +12834,10 @@ class PgOntologySourceStorage {
12593
12834
  return {
12594
12835
  entityKind: row.assertion.kind,
12595
12836
  entityKey: JSON.parse(entityKey),
12596
- entitySortKey: utf8SortKey(entityKey),
12837
+ entitySortKey: utf8SortKey2(entityKey),
12597
12838
  rootKind: row.root.kind,
12598
12839
  rootKey: JSON.parse(rootKey),
12599
- rootSortKey: utf8SortKey(rootKey),
12840
+ rootSortKey: utf8SortKey2(rootKey),
12600
12841
  stagingOrdinal: row.stagingOrdinal,
12601
12842
  root: row.root,
12602
12843
  assertion: row.assertion,
@@ -15256,7 +15497,7 @@ class PgProjectionRunStorage {
15256
15497
  input_exhausted = ${input.fixedBatchSize === undefined ? null : false},
15257
15498
  missing_target_object_type_id = ${null}, missing_target_object_id = ${null},
15258
15499
  missing_target_batch_ordinal = ${null}, missing_target_first_seen_at = ${null},
15259
- source_rows_read = ${0}, source_rows_skipped = ${0}, error = ${null}
15500
+ source_rows_read = ${0}, source_rows_skipped = ${0}, source_changes_read = ${null}, error = ${null}
15260
15501
  WHERE project_id = ${input.projectId} AND id = ${input.id}
15261
15502
  AND status = ${"failed"} AND error->>'code' = ${"queue.enqueue_failed"}
15262
15503
  RETURNING *
@@ -15367,7 +15608,8 @@ class PgProjectionRunStorage {
15367
15608
  UPDATE projection_runs
15368
15609
  SET
15369
15610
  source_rows_read = ${progress.sourceRowsRead},
15370
- source_rows_skipped = ${progress.sourceRowsSkipped}
15611
+ source_rows_skipped = ${progress.sourceRowsSkipped},
15612
+ source_changes_read = ${progress.sourceChangesRead ?? null}
15371
15613
  WHERE project_id = ${input.projectId}
15372
15614
  AND id = ${input.id}
15373
15615
  AND status = ${"running"}
@@ -15392,6 +15634,7 @@ class PgProjectionRunStorage {
15392
15634
  execution_token = ${null},
15393
15635
  source_rows_read = ${plan.progress.sourceRowsRead},
15394
15636
  source_rows_skipped = ${plan.progress.sourceRowsSkipped},
15637
+ source_changes_read = ${plan.progress.sourceChangesRead ?? null},
15395
15638
  input_exhausted = ${plan.inputExhausted ?? existingRow.input_exhausted},
15396
15639
  error = ${plan.error === undefined ? null : serializeSixbFailure4(plan.error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
15397
15640
  WHERE project_id = ${input.projectId}
@@ -15634,7 +15877,8 @@ function restoreProjectionRunRow(row) {
15634
15877
  missingTargetFirstSeenAt: row.missing_target_first_seen_at ? new Date(row.missing_target_first_seen_at) : undefined,
15635
15878
  progress: {
15636
15879
  sourceRowsRead: databaseSafeInteger2(row.source_rows_read, "sourceRowsRead"),
15637
- sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped")
15880
+ sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped"),
15881
+ ...row.source_changes_read == null ? {} : { sourceChangesRead: databaseSafeInteger2(row.source_changes_read, "sourceChangesRead") }
15638
15882
  },
15639
15883
  error: row.error ?? undefined
15640
15884
  };
@@ -17866,4 +18110,4 @@ export {
17866
18110
  migratePostgresStorage
17867
18111
  };
17868
18112
 
17869
- //# debugId=E76835E14E96BFEC64756E2164756E21
18113
+ //# debugId=672C519862E8368164756E2164756E21