@sixb/pg 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/agents/rows.d.ts +0 -1
  2. package/dist/agents/rows.d.ts.map +1 -1
  3. package/dist/agents/runs.d.ts.map +1 -1
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +562 -299
  6. package/dist/index.js.map +24 -22
  7. package/dist/migrations.d.ts.map +1 -1
  8. package/dist/ontology-storage/materialization-session.d.ts +3 -0
  9. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  10. package/dist/ontology-storage/materialization-state.d.ts +2 -1
  11. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  12. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  13. package/dist/ontology-storage/shared.d.ts +2 -0
  14. package/dist/ontology-storage/shared.d.ts.map +1 -1
  15. package/dist/ontology-storage/source-cleanup.d.ts +4 -0
  16. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -0
  17. package/dist/ontology-storage/source-roots.d.ts +24 -0
  18. package/dist/ontology-storage/source-roots.d.ts.map +1 -0
  19. package/dist/ontology-storage/sources.d.ts.map +1 -1
  20. package/dist/pg-action-run-storage.d.ts.map +1 -1
  21. package/dist/pg-ai-cost-storage.d.ts.map +1 -1
  22. package/dist/pg-ai-usage-storage.d.ts.map +1 -1
  23. package/dist/pg-execution-storage.d.ts.map +1 -1
  24. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  25. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  26. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  27. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  28. package/package.json +4 -4
  29. package/src/agents/rows.ts +0 -5
  30. package/src/agents/runs.ts +1 -6
  31. package/src/ai-cost-groups.ts +3 -0
  32. package/src/connector-connection-storage.ts +3 -3
  33. package/src/index.ts +4 -2
  34. package/src/migrations/037-execution-requester-groups.sql +21 -0
  35. package/src/migrations/038-outbox-publication-order.sql +5 -0
  36. package/src/migrations/039-projection-source-roots.sql +46 -0
  37. package/src/migrations/040-connector-optional-pkce.sql +3 -0
  38. package/src/migrations.ts +18 -0
  39. package/src/ontology-storage/materialization-session.ts +24 -8
  40. package/src/ontology-storage/materialization-state.ts +65 -86
  41. package/src/ontology-storage/materializations.ts +39 -13
  42. package/src/ontology-storage/shared.ts +10 -0
  43. package/src/ontology-storage/source-cleanup.ts +61 -0
  44. package/src/ontology-storage/source-roots.ts +251 -0
  45. package/src/ontology-storage/sources.ts +21 -57
  46. package/src/pg-action-run-storage.ts +6 -7
  47. package/src/pg-ai-cost-storage.ts +11 -2
  48. package/src/pg-ai-usage-storage.ts +1 -2
  49. package/src/pg-execution-storage.ts +6 -4
  50. package/src/pg-pipeline-run-storage.ts +2 -5
  51. package/src/pg-projection-run-storage.ts +8 -2
  52. package/src/pg-sync-run-storage.ts +2 -5
  53. package/src/pg-workflow-run-storage.ts +0 -5
package/dist/index.js CHANGED
@@ -2,6 +2,10 @@
2
2
  // src/index.ts
3
3
  import { AsyncLocalStorage } from "async_hooks";
4
4
  import { ProviderMaterializationTransactionLifecycle } from "@sixb/core/internal/ontology-storage-provider";
5
+ import {
6
+ createTransactionStorageProxy,
7
+ throwNestedStorageTransaction
8
+ } from "@sixb/core/internal/storage";
5
9
  import {
6
10
  createAgentOperationScope,
7
11
  createAuthOperationScope,
@@ -12,9 +16,7 @@ import {
12
16
  createWorkflowRunOperationScope
13
17
  } from "@sixb/core/internal/storage-operation-scope";
14
18
  import {
15
- createTransactionStorageProxy,
16
- StorageTransactionError,
17
- throwNestedStorageTransaction
19
+ StorageTransactionError
18
20
  } from "@sixb/core/storage";
19
21
 
20
22
  // src/agents/checkpoints.ts
@@ -147,7 +149,6 @@ function rowToRunRecord(row) {
147
149
  id: row.id,
148
150
  projectId: row.project_id,
149
151
  executionId: row.execution_id,
150
- requesterGroupIds: typeof row.requester_group_ids === "string" ? JSON.parse(row.requester_group_ids) : row.requester_group_ids,
151
152
  status: row.status,
152
153
  modelId: row.model_id ?? undefined,
153
154
  finishReason: coerceAgentRunFinishReason(row.finish_reason),
@@ -580,7 +581,6 @@ import {
580
581
  assertSubagentRunResult,
581
582
  subagentRunMatchesCreateInput
582
583
  } from "@sixb/core/internal/agent-run-storage-provider";
583
- import { normalizeRequesterGroupIds } from "@sixb/core/internal/auth";
584
584
  import { serializeSixbFailure } from "@sixb/core/internal/errors";
585
585
  import {
586
586
  AGENT_RUN_FAILURE_CODES as AGENT_RUN_FAILURE_CODES2,
@@ -683,7 +683,6 @@ class PgAgentRunStore {
683
683
  thread_id,
684
684
  trigger_message_id,
685
685
  spec,
686
- requester_group_ids,
687
686
  status,
688
687
  attempt,
689
688
  created_at
@@ -695,7 +694,6 @@ class PgAgentRunStore {
695
694
  ${input.threadId},
696
695
  ${input.triggerMessageId},
697
696
  ${JSON.stringify(input.spec)}::text::jsonb,
698
- ${JSON.stringify(normalizeRequesterGroupIds(input.requesterGroupIds))}::text::jsonb,
699
697
  ${"queued"},
700
698
  ${0},
701
699
  ${createdAt}
@@ -756,7 +754,6 @@ class PgAgentRunStore {
756
754
  if (parent.execution_token !== input.parentExecutionToken) {
757
755
  throw new AgentStorageError3("execution_lost", `[SixbPg] Parent Agent run '${parent.id}' is no longer owned by this execution.`);
758
756
  }
759
- const parentRecord = rowToRunRecord(parent);
760
757
  const [countRow] = await tx`
761
758
  SELECT COUNT(*)::bigint AS count FROM agent_runs
762
759
  WHERE project_id = ${input.projectId}
@@ -770,12 +767,11 @@ class PgAgentRunStore {
770
767
  const [row] = await tx`
771
768
  INSERT INTO agent_runs (
772
769
  project_id, id, execution_id, kind, parent_run_id, spawn_key, spec,
773
- requester_group_ids, status, attempt, created_at
770
+ status, attempt, created_at
774
771
  ) VALUES (
775
772
  ${input.projectId}, ${input.id}, ${input.executionId}, ${"subagent"},
776
773
  ${input.parentRunId}, ${input.spawnKey},
777
774
  ${JSON.stringify(input.spec)}::text::jsonb,
778
- ${JSON.stringify(parentRecord.requesterGroupIds)}::text::jsonb,
779
775
  ${"queued"}, ${0}, ${input.createdAt ?? new Date}
780
776
  )
781
777
  RETURNING *
@@ -1871,13 +1867,13 @@ class PgAuthAccessTokenStore {
1871
1867
  }
1872
1868
  async assertSubjectExists(projectId, subjectType, subjectId) {
1873
1869
  if (subjectType === "user") {
1874
- const [row2] = await this.sql`
1870
+ const [row] = await this.sql`
1875
1871
  SELECT *
1876
1872
  FROM auth_users
1877
1873
  WHERE project_id = ${projectId}
1878
1874
  AND id = ${subjectId}
1879
1875
  `;
1880
- if (row2)
1876
+ if (row)
1881
1877
  return;
1882
1878
  throw new AuthStorageError2("missing_user", `[Sixb] User '${subjectId}' not found for project '${projectId}'.`);
1883
1879
  }
@@ -3452,7 +3448,7 @@ class PgConnectorConnectionPersistence {
3452
3448
  ) VALUES (
3453
3449
  ${record.projectId}, ${record.connectorId}, ${record.id}, ${record.slot},
3454
3450
  ${record.initiatedByExecutionId}, ${record.stateHash},
3455
- ${jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
3451
+ ${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
3456
3452
  ${record.connectionRunId ?? null}, ${record.returnTo ?? null},
3457
3453
  ${record.callbackBindingHash ?? null}, ${record.reauthorizationId ?? null},
3458
3454
  ${record.reauthorizationRevision ?? null},
@@ -3745,7 +3741,7 @@ function authorizationAttemptFromRow(row) {
3745
3741
  slot: row.slot,
3746
3742
  initiatedByExecutionId: row.initiated_by_execution_id,
3747
3743
  stateHash: row.state_hash,
3748
- codeVerifier: row.code_verifier,
3744
+ ...row.code_verifier === null ? {} : { codeVerifier: row.code_verifier },
3749
3745
  redirectUri: row.redirect_uri,
3750
3746
  ...row.connection_run_id === null ? {} : { connectionRunId: row.connection_run_id },
3751
3747
  ...row.return_to === null ? {} : { returnTo: row.return_to },
@@ -7085,6 +7081,93 @@ ALTER TABLE executions
7085
7081
  ON DELETE RESTRICT;
7086
7082
  `;
7087
7083
 
7084
+ // src/migrations/039-projection-source-roots.sql
7085
+ var _039_projection_source_roots_default = `ALTER TABLE ontology_sources ADD COLUMN base_materialization_id TEXT;
7086
+ ALTER TABLE ontology_sources ADD COLUMN base_commit_id TEXT;
7087
+ ALTER TABLE ontology_sources ADD CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL));
7088
+
7089
+ -- A root selects one complete set of assertions. Unchanged roots keep their original version;
7090
+ -- activation only retires/replaces affected references, under the source commit fence.
7091
+ CREATE TABLE ontology_source_roots (
7092
+ project_id TEXT NOT NULL,
7093
+ source_id TEXT NOT NULL,
7094
+ materialization_id TEXT NOT NULL,
7095
+ root_sort_key TEXT NOT NULL,
7096
+ root_kind TEXT NOT NULL CHECK (root_kind IN ('object', 'link')),
7097
+ root_key JSONB NOT NULL,
7098
+ root JSONB NOT NULL,
7099
+ staging_ordinal BIGINT NOT NULL CHECK (staging_ordinal >= 0),
7100
+ deleted BOOLEAN NOT NULL DEFAULT FALSE,
7101
+ active BOOLEAN NOT NULL DEFAULT FALSE,
7102
+ retired_at TIMESTAMPTZ,
7103
+ PRIMARY KEY (project_id, source_id, materialization_id, root_sort_key),
7104
+ UNIQUE (project_id, source_id, materialization_id, staging_ordinal),
7105
+ FOREIGN KEY (project_id, source_id, materialization_id)
7106
+ REFERENCES ontology_sources(project_id, source_id, materialization_id) ON DELETE RESTRICT,
7107
+ CHECK (NOT active OR (NOT deleted AND retired_at IS NULL))
7108
+ );
7109
+ CREATE UNIQUE INDEX idx_ontology_source_roots_active
7110
+ ON ontology_source_roots (project_id, source_id, root_sort_key) WHERE active;
7111
+ CREATE INDEX idx_ontology_source_roots_lookup
7112
+ ON ontology_source_roots (project_id, root_sort_key, source_id) WHERE active;
7113
+ CREATE INDEX idx_ontology_source_roots_cleanup
7114
+ ON ontology_source_roots (project_id, retired_at, source_id, materialization_id, root_sort_key)
7115
+ WHERE NOT active AND retired_at IS NOT NULL;
7116
+
7117
+ INSERT INTO ontology_source_roots (
7118
+ project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root,
7119
+ staging_ordinal, active, retired_at
7120
+ )
7121
+ SELECT DISTINCT rows.project_id, rows.source_id, rows.materialization_id,
7122
+ rows.root_sort_key, rows.root_kind, rows.root_key, rows.root, rows.staging_ordinal,
7123
+ sources.status = 'active', sources.terminal_at
7124
+ FROM ontology_source_rows AS rows
7125
+ JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id);
7126
+
7127
+ ANALYZE ontology_source_roots;
7128
+
7129
+ ALTER TABLE projection_runs ADD COLUMN source_changes_read BIGINT
7130
+ CHECK (source_changes_read IS NULL OR source_changes_read >= 0);
7131
+ `;
7132
+
7133
+ // src/migrations/037-execution-requester-groups.sql
7134
+ var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
7135
+
7136
+ -- Keep historical admission snapshots; never reconstruct them from current memberships.
7137
+ WITH RECURSIVE attribution(project_id, id, groups) AS (
7138
+ SELECT e.project_id, e.id, COALESCE(w.requester_group_ids, a.requester_group_ids, '[]'::jsonb)
7139
+ FROM executions e
7140
+ LEFT JOIN workflow_runs w ON w.project_id = e.project_id AND w.execution_id = e.id
7141
+ LEFT JOIN agent_runs a ON a.project_id = e.project_id AND a.execution_id = e.id
7142
+ WHERE e.parent_execution_id IS NULL
7143
+ UNION ALL
7144
+ SELECT e.project_id, e.id, COALESCE(w.requester_group_ids, a.requester_group_ids, p.groups)
7145
+ FROM executions e
7146
+ JOIN attribution p ON p.project_id = e.project_id AND p.id = e.parent_execution_id
7147
+ LEFT JOIN workflow_runs w ON w.project_id = e.project_id AND w.execution_id = e.id
7148
+ LEFT JOIN agent_runs a ON a.project_id = e.project_id AND a.execution_id = e.id
7149
+ )
7150
+ UPDATE executions e SET requester_group_ids = a.groups
7151
+ FROM attribution a WHERE a.project_id = e.project_id AND a.id = e.id;
7152
+
7153
+ ALTER TABLE workflow_runs DROP COLUMN requester_group_ids;
7154
+ ALTER TABLE agent_runs DROP COLUMN requester_group_ids;
7155
+ `;
7156
+
7157
+ // src/migrations/038-outbox-publication-order.sql
7158
+ var _038_outbox_publication_order_default = `-- Preserve the claim order without sorting every pending event for each batch.
7159
+ -- Keep the availability index for selective delayed/leased workloads.
7160
+ CREATE INDEX idx_ontology_outbox_publication_order
7161
+ ON ontology_outbox (project_id, created_at, commit_id, commit_ordinal)
7162
+ WHERE published_at IS NULL;
7163
+ `;
7164
+
7165
+ // src/migrations/040-connector-optional-pkce.sql
7166
+ var _040_connector_optional_pkce_default = `-- Existing attempts retain their encrypted S256 verifier. NULL denotes disabled PKCE.
7167
+ ALTER TABLE connector_authorization_attempts
7168
+ ALTER COLUMN code_verifier DROP NOT NULL;
7169
+ `;
7170
+
7088
7171
  // src/migrations.ts
7089
7172
  var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
7090
7173
  function pgSql(id, sqlText) {
@@ -7295,7 +7378,11 @@ var postgresStorageMigrations = defineMigrations({
7295
7378
  pgSql("033-retire-agent-definitions", _033_retire_agent_definitions_default),
7296
7379
  pgSql("034-device-authorizations", _034_device_authorizations_default),
7297
7380
  pgSql("035-share-grants", _035_share_grants_default),
7298
- pgSql("036-share-sessions", _036_share_sessions_default)
7381
+ pgSql("036-share-sessions", _036_share_sessions_default),
7382
+ pgSql("037-execution-requester-groups", _037_execution_requester_groups_default),
7383
+ pgSql("038-outbox-publication-order", _038_outbox_publication_order_default),
7384
+ pgSql("039-projection-source-roots", _039_projection_source_roots_default),
7385
+ pgSql("040-connector-optional-pkce", _040_connector_optional_pkce_default)
7299
7386
  ]
7300
7387
  });
7301
7388
 
@@ -7744,18 +7831,18 @@ function compileExpansionOrder(expansion, target, neighborType, neighborId) {
7744
7831
  }
7745
7832
  function compileSet(projectId, op, inputs, ctx) {
7746
7833
  if (inputs.length === 0) {
7747
- const order2 = compileOrder(identityOrderFields());
7834
+ const order = compileOrder(identityOrderFields());
7748
7835
  return {
7749
7836
  sql: `
7750
7837
  SELECT *, properties AS _cursor_properties
7751
7838
  FROM ${ctx.source.objectsTable}
7752
7839
  WHERE 1 = 0
7753
- ORDER BY ${order2.sql}
7840
+ ORDER BY ${order.sql}
7754
7841
  `,
7755
- args: order2.args,
7842
+ args: order.args,
7756
7843
  totalSql: "SELECT 0::bigint AS total",
7757
7844
  totalArgs: [],
7758
- order: order2,
7845
+ order,
7759
7846
  hasMore: () => false,
7760
7847
  trimRows: identityRows,
7761
7848
  nextPageToken: () => {
@@ -9428,7 +9515,7 @@ function valuesJoin(sql, select, columns, tuples, where, whereParams) {
9428
9515
  const colWidth = columns.length;
9429
9516
  const base = whereParams.length;
9430
9517
  const valuePlaceholders = tuples.map((_, i) => {
9431
- const cols = columns.map((_2, j) => `$${base + i * colWidth + j + 1}`);
9518
+ const cols = columns.map((_, j) => `$${base + i * colWidth + j + 1}`);
9432
9519
  return `(${cols.join(",")})`;
9433
9520
  }).join(",");
9434
9521
  const onClause = columns.map((c) => {
@@ -9494,6 +9581,12 @@ function sourceRecord(row) {
9494
9581
  projectionRevision: row.projection_revision,
9495
9582
  ownershipHash: row.ownership_hash,
9496
9583
  ontologyRevision: row.ontology_revision,
9584
+ ...row.base_materialization_id === null ? {} : {
9585
+ base: {
9586
+ materializationId: row.base_materialization_id,
9587
+ lastCommitId: row.base_commit_id
9588
+ }
9589
+ },
9497
9590
  rootCount: row.root_count === null ? null : databaseSafeInteger(row.root_count, "Source root count"),
9498
9591
  assertionCount: row.assertion_count === null ? null : databaseSafeInteger(row.assertion_count, "Source assertion count"),
9499
9592
  createdAt: toIsoString2(row.created_at),
@@ -9662,7 +9755,7 @@ import {
9662
9755
  linkRefKey as linkRefKey3,
9663
9756
  linkRefSortKey,
9664
9757
  linkScopeSortKey as linkScopeSortKey2,
9665
- MaterializationConflictError as MaterializationConflictError5,
9758
+ MaterializationConflictError as MaterializationConflictError6,
9666
9759
  objectRefKey as objectRefKey3,
9667
9760
  objectRefSortKey,
9668
9761
  telemetryPointKey as telemetryPointKey3,
@@ -9681,7 +9774,7 @@ import {
9681
9774
  } from "@sixb/core/internal/ontology-storage-provider";
9682
9775
 
9683
9776
  // src/ontology-storage/materialization-session.ts
9684
- import { MaterializationConflictError as MaterializationConflictError3 } from "@sixb/core/internal/materialization";
9777
+ import { MaterializationConflictError as MaterializationConflictError4 } from "@sixb/core/internal/materialization";
9685
9778
  import {
9686
9779
  correlateMaterializationChunk,
9687
9780
  duplicateMaterializationWork as duplicateWork,
@@ -9695,9 +9788,9 @@ import {
9695
9788
  import {
9696
9789
  linkRefKey,
9697
9790
  linkScopeSortKey,
9698
- MaterializationConflictError as MaterializationConflictError2,
9791
+ MaterializationConflictError as MaterializationConflictError3,
9699
9792
  objectRefKey,
9700
- projectionEntityKey,
9793
+ projectionEntityKey as projectionEntityKey2,
9701
9794
  telemetryPointKey,
9702
9795
  telemetryPointSortKey
9703
9796
  } from "@sixb/core/internal/materialization";
@@ -9707,6 +9800,178 @@ import {
9707
9800
  linkSlotOverrideValue,
9708
9801
  startScopeAccumulator
9709
9802
  } from "@sixb/core/internal/ontology-storage-provider";
9803
+
9804
+ // src/ontology-storage/source-roots.ts
9805
+ import {
9806
+ MaterializationConflictError as MaterializationConflictError2,
9807
+ MaterializationValidationError,
9808
+ projectionEntityKey
9809
+ } from "@sixb/core/internal/materialization";
9810
+ import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider";
9811
+ function replacementSourceRows(sql, input) {
9812
+ const roots = input.incremental ? sql`
9813
+ SELECT roots.* FROM ontology_source_roots AS changed
9814
+ CROSS JOIN LATERAL (
9815
+ SELECT roots.* FROM ontology_source_roots AS roots
9816
+ WHERE roots.project_id = changed.project_id AND roots.source_id = changed.source_id
9817
+ AND roots.root_sort_key = changed.root_sort_key AND roots.active OFFSET 0
9818
+ ) AS roots
9819
+ WHERE changed.project_id = ${input.projectId} AND changed.source_id = ${input.sourceId}
9820
+ AND changed.materialization_id = ${input.materializationId}
9821
+ ` : sql`SELECT * FROM ontology_source_roots WHERE project_id = ${input.projectId} AND source_id = ${input.sourceId} AND active`;
9822
+ return sql`
9823
+ SELECT * FROM ontology_source_rows WHERE project_id = ${input.projectId}
9824
+ AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId}
9825
+ UNION ALL
9826
+ SELECT rows.* FROM (${roots}) AS roots
9827
+ CROSS JOIN LATERAL (
9828
+ SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.project_id = roots.project_id
9829
+ AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
9830
+ AND rows.root_sort_key = roots.root_sort_key OFFSET 0
9831
+ ) AS rows
9832
+ `;
9833
+ }
9834
+ async function stageSourceRoots(sql, manifest, input) {
9835
+ const roots = sourceStageRoots(manifest.projection_kind, input);
9836
+ if (roots.length === 0)
9837
+ return;
9838
+ if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
9839
+ throw new MaterializationValidationError("Root deletions require a source delta base.");
9840
+ }
9841
+ const values = jsonParameter2(sql, roots.map((root) => ({
9842
+ root_sort_key: root.sortKey,
9843
+ root_kind: root.root.kind,
9844
+ root_key: JSON.parse(root.rootKey),
9845
+ root: root.root,
9846
+ staging_ordinal: root.stagingOrdinal,
9847
+ deleted: root.deleted
9848
+ })));
9849
+ const inserted = await sql`
9850
+ INSERT INTO ontology_source_roots (
9851
+ project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
9852
+ ) SELECT ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
9853
+ root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
9854
+ FROM jsonb_to_recordset(${values}) AS input(
9855
+ root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
9856
+ ON CONFLICT DO NOTHING RETURNING root_sort_key
9857
+ `;
9858
+ if (inserted.length === roots.length)
9859
+ return;
9860
+ const insertedKeys = new Set(inserted.map((row) => row.root_sort_key));
9861
+ const skipped = roots.filter((root) => !insertedKeys.has(root.sortKey));
9862
+ const stored = await sql`
9863
+ SELECT root_sort_key, staging_ordinal, deleted FROM ontology_source_roots
9864
+ WHERE project_id = ${input.projectId} AND source_id = ${input.source.projectionId}
9865
+ AND materialization_id = ${input.materializationId}
9866
+ AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
9867
+ `;
9868
+ const existing = new Map(stored.map((root) => [root.root_sort_key, root]));
9869
+ if (skipped.some((root) => {
9870
+ const previous = existing.get(root.sortKey);
9871
+ return !previous || Number(previous.staging_ordinal) !== root.stagingOrdinal || previous.deleted !== root.deleted;
9872
+ }))
9873
+ throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
9874
+ }
9875
+ async function activateSourceRoots(sql, projectId, candidate, activation) {
9876
+ if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
9877
+ throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
9878
+ if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0)
9879
+ return;
9880
+ const affected = candidate.base_materialization_id === null ? sql`` : sql`AND root_sort_key IN (
9881
+ SELECT root_sort_key FROM ontology_source_roots
9882
+ WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
9883
+ AND materialization_id = ${activation.materializationId}
9884
+ )`;
9885
+ await sql`
9886
+ UPDATE ontology_source_roots AS roots SET active = FALSE, retired_at = ${activation.updatedAt}
9887
+ WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId} AND active
9888
+ ${affected}
9889
+ `;
9890
+ await sql`
9891
+ UPDATE ontology_source_roots SET active = NOT deleted,
9892
+ retired_at = CASE WHEN deleted THEN ${activation.updatedAt}::timestamptz ELSE NULL END
9893
+ WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
9894
+ AND materialization_id = ${activation.materializationId}
9895
+ `;
9896
+ }
9897
+ async function assertSourceRootCoverage(sql, manifest) {
9898
+ const [invalid] = await sql`
9899
+ SELECT 1 FROM ontology_source_roots AS roots
9900
+ WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
9901
+ AND materialization_id = ${manifest.materialization_id}
9902
+ AND roots.deleted = EXISTS (
9903
+ SELECT 1 FROM ontology_source_rows AS rows
9904
+ WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
9905
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
9906
+ ) LIMIT 1
9907
+ `;
9908
+ if (invalid)
9909
+ throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
9910
+ }
9911
+ function activeSourceRows(sql, projectId, refs) {
9912
+ const keys = refs.flatMap((entity) => {
9913
+ const entitySortKey = utf8SortKey(projectionEntityKey(entity));
9914
+ const roots = entity.kind === "object" ? [entitySortKey] : [
9915
+ entitySortKey,
9916
+ utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
9917
+ ];
9918
+ return roots.map((rootSortKey) => ({
9919
+ root_sort_key: rootSortKey,
9920
+ entity_sort_key: entitySortKey
9921
+ }));
9922
+ });
9923
+ return sql`
9924
+ SELECT rows.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)}) AS requested(root_sort_key TEXT, entity_sort_key TEXT)
9925
+ JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
9926
+ AND roots.active AND roots.root_sort_key = requested.root_sort_key
9927
+ JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
9928
+ AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
9929
+ AND rows.root_sort_key = roots.root_sort_key
9930
+ `;
9931
+ }
9932
+ function replacementAssertionRows(sql, input) {
9933
+ const keys = input.refs.map((ref) => ({
9934
+ entity_kind: ref.kind,
9935
+ entity_key: JSON.parse(projectionEntityKey(ref)),
9936
+ root_keys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
9937
+ utf8SortKey(projectionEntityKey(ref)),
9938
+ utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
9939
+ ]
9940
+ }));
9941
+ const previous = !input.includePrevious ? sql`` : sql`UNION ALL
9942
+ SELECT rows.* FROM (
9943
+ SELECT roots.* FROM ontology_source_roots AS roots
9944
+ WHERE roots.project_id = ${input.projectId} AND roots.source_id = ${input.sourceId}
9945
+ AND roots.active AND roots.root_sort_key = ANY(requested.root_keys)
9946
+ ${!input.incremental ? sql`` : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
9947
+ WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
9948
+ AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`}
9949
+
9950
+ OFFSET 0
9951
+ ) AS roots
9952
+ CROSS JOIN LATERAL (
9953
+ SELECT rows.* FROM ontology_source_rows AS rows
9954
+ WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
9955
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
9956
+ AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
9957
+ OFFSET 0
9958
+ ) AS rows
9959
+ `;
9960
+ return sql`
9961
+ SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)})
9962
+ AS requested(entity_kind TEXT, entity_key JSONB, root_keys TEXT[])
9963
+ CROSS JOIN LATERAL (
9964
+ SELECT rows.* FROM ontology_source_rows AS rows
9965
+ WHERE rows.project_id = ${input.projectId} AND rows.source_id = ${input.sourceId}
9966
+ AND rows.materialization_id = ${input.materializationId}
9967
+ AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
9968
+ ${previous}
9969
+ OFFSET 0
9970
+ ) AS selected
9971
+ `;
9972
+ }
9973
+
9974
+ // src/ontology-storage/materialization-state.ts
9710
9975
  var PG_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
9711
9976
  var PG_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
9712
9977
 
@@ -9739,22 +10004,7 @@ class PgMaterializationStateReader {
9739
10004
  JOIN requested USING (object_type_id, primary_id)
9740
10005
  WHERE objects.project_id = ${this.projectId}
9741
10006
  `,
9742
- this.sql`
9743
- WITH requested AS (
9744
- SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
9745
- AS requested_values(object_type_id TEXT, primary_id TEXT)
9746
- )
9747
- SELECT rows.*
9748
- FROM ontology_source_rows AS rows
9749
- JOIN requested USING (object_type_id, primary_id)
9750
- JOIN ontology_sources AS sources
9751
- ON sources.project_id = rows.project_id
9752
- AND sources.source_id = rows.source_id
9753
- AND sources.materialization_id = rows.materialization_id
9754
- WHERE rows.project_id = ${this.projectId}
9755
- AND rows.entity_kind = 'object'
9756
- AND sources.status = 'active'
9757
- `,
10007
+ activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "object", ref }))),
9758
10008
  this.sql`
9759
10009
  WITH requested AS (
9760
10010
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
@@ -9795,7 +10045,7 @@ class PgMaterializationStateReader {
9795
10045
  const effectiveRow = effective.get(key);
9796
10046
  return {
9797
10047
  ref: structuredClone(ref),
9798
- source: sourceObject(sources.get(projectionEntityKey({ kind: "object", ref }))),
10048
+ source: sourceObject(sources.get(projectionEntityKey2({ kind: "object", ref }))),
9799
10049
  override: overrides.get(key) ?? null,
9800
10050
  effective: effectiveRow ? effectiveObjectSnapshot(effectiveRow) : null,
9801
10051
  latestTelemetry: telemetry.get(key) ?? []
@@ -9831,26 +10081,7 @@ class PgMaterializationStateReader {
9831
10081
  JOIN requested USING (source_type_id, source_id, link_id, target_type_id, target_id)
9832
10082
  WHERE links.project_id = ${this.projectId}
9833
10083
  `,
9834
- this.sql`
9835
- WITH requested AS (
9836
- SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
9837
- source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
9838
- target_type_id TEXT, target_primary_id TEXT
9839
- )
9840
- )
9841
- SELECT rows.*
9842
- FROM ontology_source_rows AS rows
9843
- JOIN requested USING (
9844
- source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
9845
- )
9846
- JOIN ontology_sources AS sources
9847
- ON sources.project_id = rows.project_id
9848
- AND sources.source_id = rows.source_id
9849
- AND sources.materialization_id = rows.materialization_id
9850
- WHERE rows.project_id = ${this.projectId}
9851
- AND rows.entity_kind = 'link'
9852
- AND sources.status = 'active'
9853
- `,
10084
+ activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "link", ref }))),
9854
10085
  this.sql`
9855
10086
  WITH requested AS (
9856
10087
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
@@ -9889,7 +10120,7 @@ class PgMaterializationStateReader {
9889
10120
  const effectiveRow = effective.get(key);
9890
10121
  return {
9891
10122
  ref: structuredClone(ref),
9892
- source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
10123
+ source: sourceLink(sources.get(projectionEntityKey2({ kind: "link", ref }))),
9893
10124
  override: overrides.get(key) ?? null,
9894
10125
  slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
9895
10126
  effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null
@@ -9958,13 +10189,14 @@ class PgMaterializationStateReader {
9958
10189
  ON requested.source_type_id = rows.source_type_id
9959
10190
  AND requested.source_id = rows.source_primary_id
9960
10191
  AND requested.link_id = rows.link_id
9961
- JOIN ontology_sources AS sources
10192
+ JOIN ontology_source_roots AS sources
9962
10193
  ON sources.project_id = rows.project_id
9963
10194
  AND sources.source_id = rows.source_id
9964
10195
  AND sources.materialization_id = rows.materialization_id
10196
+ AND sources.root_sort_key = rows.root_sort_key
9965
10197
  WHERE rows.project_id = ${this.projectId}
9966
10198
  AND rows.entity_kind = 'link'
9967
- AND sources.status = 'active'
10199
+ AND sources.active
9968
10200
  `,
9969
10201
  this.sql`
9970
10202
  WITH requested AS (
@@ -10037,7 +10269,7 @@ class PgMaterializationStateReader {
10037
10269
  for (const row of rows) {
10038
10270
  const accumulator = accumulators.get(row.scope_sort_key);
10039
10271
  if (!accumulator) {
10040
- throw new MaterializationConflictError2("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
10272
+ throw new MaterializationConflictError3("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
10041
10273
  }
10042
10274
  const snapshot = effectiveLinkSnapshot(row);
10043
10275
  appendScopeSnapshot(accumulator, snapshot);
@@ -10106,30 +10338,32 @@ class PgMaterializationStateReader {
10106
10338
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10107
10339
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10108
10340
  FROM ontology_source_rows AS rows
10109
- JOIN ontology_sources AS sources
10341
+ JOIN ontology_source_roots AS sources
10110
10342
  ON sources.project_id = rows.project_id
10111
10343
  AND sources.source_id = rows.source_id
10112
10344
  AND sources.materialization_id = rows.materialization_id
10345
+ AND sources.root_sort_key = rows.root_sort_key
10113
10346
  JOIN requested
10114
10347
  ON requested.object_type_id = rows.source_type_id
10115
10348
  AND requested.primary_id = rows.source_primary_id
10116
10349
  WHERE rows.project_id = ${this.projectId}
10117
10350
  AND rows.entity_kind = 'link'
10118
- AND sources.status = 'active'
10351
+ AND sources.active
10119
10352
  UNION
10120
10353
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10121
10354
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10122
10355
  FROM ontology_source_rows AS rows
10123
- JOIN ontology_sources AS sources
10356
+ JOIN ontology_source_roots AS sources
10124
10357
  ON sources.project_id = rows.project_id
10125
10358
  AND sources.source_id = rows.source_id
10126
10359
  AND sources.materialization_id = rows.materialization_id
10360
+ AND sources.root_sort_key = rows.root_sort_key
10127
10361
  JOIN requested
10128
10362
  ON requested.object_type_id = rows.target_type_id
10129
10363
  AND requested.primary_id = rows.target_primary_id
10130
10364
  WHERE rows.project_id = ${this.projectId}
10131
10365
  AND rows.entity_kind = 'link'
10132
- AND sources.status = 'active'
10366
+ AND sources.active
10133
10367
  ), selected AS (
10134
10368
  SELECT DISTINCT links.*,
10135
10369
  ${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
@@ -10148,13 +10382,15 @@ class PgMaterializationStateReader {
10148
10382
  }
10149
10383
  async* replacementIdentities(input) {
10150
10384
  await this.prepareReplacementIdentities(input);
10385
+ await this.sql`ANALYZE ${this.sql(PG_REPLACEMENT_WORK_TABLE)}`;
10151
10386
  let cursor = null;
10152
10387
  while (true) {
10388
+ const after = cursor === null ? this.sql`` : this.sql`AND sort_key > ${cursor}`;
10153
10389
  const rows = await this.sql`
10154
10390
  SELECT entity_kind, identity_key, sort_key, diff_required
10155
10391
  FROM ${this.sql(PG_REPLACEMENT_WORK_TABLE)}
10156
10392
  WHERE session_id = ${input.sessionId} AND entity_kind = ${input.kind}
10157
- AND (${cursor}::text IS NULL OR sort_key > ${cursor})
10393
+ ${after}
10158
10394
  ORDER BY sort_key
10159
10395
  LIMIT ${input.pageRows}
10160
10396
  `;
@@ -10169,19 +10405,19 @@ class PgMaterializationStateReader {
10169
10405
  const candidates = await this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
10170
10406
  return base.map((state) => ({
10171
10407
  ref: state.ref,
10172
- candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey({ kind: "object", ref: state.ref }))),
10408
+ candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey2({ kind: "object", ref: state.ref }))),
10173
10409
  override: state.override,
10174
10410
  effective: state.effective,
10175
10411
  latestTelemetry: state.latestTelemetry
10176
10412
  }));
10177
10413
  }
10178
- async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities) {
10414
+ async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities, incremental = false) {
10179
10415
  const refs = identities.map((identity) => identity.ref);
10180
10416
  const base = await this.linkStates(refs);
10181
- const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })));
10417
+ const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })), incremental);
10182
10418
  const candidate = replacements.byMaterialization.get(candidateMaterializationId);
10183
10419
  return base.map((state, index) => {
10184
- const key = projectionEntityKey({ kind: "link", ref: state.ref });
10420
+ const key = projectionEntityKey2({ kind: "link", ref: state.ref });
10185
10421
  return {
10186
10422
  ref: state.ref,
10187
10423
  candidateSource: replacements.owned.has(key) ? sourceLink(candidate?.get(key)) : state.source,
@@ -10248,33 +10484,23 @@ class PgMaterializationStateReader {
10248
10484
  `;
10249
10485
  return new Map(rows.map((row) => [linkRefKey(linkRefFromColumns(row)), row.last_commit_id]));
10250
10486
  }
10251
- async replacementSources(sourceId, materializationIds, refs) {
10487
+ async replacementSources(sourceId, materializationIds, refs, incremental = false) {
10252
10488
  if (materializationIds.length === 0 || refs.length === 0) {
10253
10489
  return { owned: new Set, byMaterialization: new Map };
10254
10490
  }
10255
- const keys = refs.map((ref) => ({
10256
- entity_kind: ref.kind,
10257
- entity_key: JSON.parse(projectionEntityKey(ref))
10258
- }));
10259
- const rows = await this.sql`
10260
- WITH requested AS (
10261
- SELECT *
10262
- FROM jsonb_to_recordset(${jsonParameter2(this.sql, keys)})
10263
- AS requested_values(entity_kind TEXT, entity_key JSONB)
10264
- )
10265
- SELECT rows.* FROM ontology_source_rows AS rows
10266
- JOIN requested USING (entity_kind, entity_key)
10267
- WHERE rows.project_id = ${this.projectId}
10268
- AND rows.source_id = ${sourceId}
10269
- AND rows.materialization_id = ANY(
10270
- ${this.sql.array([...new Set(materializationIds)])}::text[]
10271
- )
10272
- `;
10491
+ const rows = await replacementAssertionRows(this.sql, {
10492
+ projectId: this.projectId,
10493
+ sourceId,
10494
+ materializationId: materializationIds[0],
10495
+ incremental,
10496
+ includePrevious: materializationIds.length > 1,
10497
+ refs
10498
+ });
10273
10499
  const owned = new Set;
10274
10500
  const byMaterialization = new Map;
10275
10501
  for (const row of rows) {
10276
10502
  const source = storedSource(row);
10277
- const key = projectionEntityKey(source.assertion);
10503
+ const key = projectionEntityKey2(source.assertion);
10278
10504
  owned.add(key);
10279
10505
  const materialization = byMaterialization.get(row.materialization_id) ?? new Map;
10280
10506
  materialization.set(key, source);
@@ -10290,20 +10516,19 @@ class PgMaterializationStateReader {
10290
10516
  await this.prepareReplacementLinks(input);
10291
10517
  }
10292
10518
  async prepareReplacementObjects(input) {
10293
- const materializationIds = [
10294
- input.candidateMaterializationId,
10295
- ...input.previousMaterializationId ? [input.previousMaterializationId] : []
10296
- ];
10519
+ const rows = replacementSourceRows(this.sql, {
10520
+ projectId: this.projectId,
10521
+ sourceId: input.sourceId,
10522
+ materializationId: input.candidateMaterializationId,
10523
+ incremental: input.incremental ?? false
10524
+ });
10297
10525
  await this.sql`
10298
10526
  WITH selected AS (
10299
10527
  SELECT DISTINCT object_type_id, primary_id,
10300
10528
  ${this.sql.unsafe(objectKeyExpression("rows"))} AS identity_key,
10301
10529
  ${this.sql.unsafe(objectSortExpression("rows"))} AS sort_key
10302
- FROM ontology_source_rows AS rows
10303
- WHERE project_id = ${this.projectId}
10304
- AND source_id = ${input.sourceId}
10305
- AND entity_kind = 'object'
10306
- AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
10530
+ FROM (${rows}) AS rows
10531
+ WHERE entity_kind = 'object'
10307
10532
  )
10308
10533
  INSERT INTO ${this.sql(PG_REPLACEMENT_WORK_TABLE)} (
10309
10534
  session_id, entity_kind, identity_key, sort_key, diff_required
@@ -10316,10 +10541,13 @@ class PgMaterializationStateReader {
10316
10541
  `;
10317
10542
  }
10318
10543
  async prepareReplacementLinks(input) {
10319
- const materializationIds = [
10320
- input.candidateMaterializationId,
10321
- ...input.previousMaterializationId ? [input.previousMaterializationId] : []
10322
- ];
10544
+ await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
10545
+ const rows = replacementSourceRows(this.sql, {
10546
+ projectId: this.projectId,
10547
+ sourceId: input.sourceId,
10548
+ materializationId: input.candidateMaterializationId,
10549
+ incremental: input.incremental ?? false
10550
+ });
10323
10551
  await this.sql`
10324
10552
  WITH incident_objects AS (
10325
10553
  SELECT DISTINCT payload->'ref'->>'objectTypeId' AS object_type_id,
@@ -10329,11 +10557,8 @@ class PgMaterializationStateReader {
10329
10557
  ), replacement_links AS (
10330
10558
  SELECT source_type_id, source_primary_id AS source_id, link_id,
10331
10559
  target_type_id, target_primary_id AS target_id
10332
- FROM ontology_source_rows
10333
- WHERE project_id = ${this.projectId}
10334
- AND source_id = ${input.sourceId}
10335
- AND entity_kind = 'link'
10336
- AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
10560
+ FROM (${rows}) AS rows
10561
+ WHERE entity_kind = 'link'
10337
10562
  ), incident_links AS (
10338
10563
  SELECT links.source_type_id, links.source_id, links.link_id,
10339
10564
  links.target_type_id, links.target_id
@@ -10372,30 +10597,32 @@ class PgMaterializationStateReader {
10372
10597
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10373
10598
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10374
10599
  FROM ontology_source_rows AS rows
10375
- JOIN ontology_sources AS sources
10600
+ JOIN ontology_source_roots AS sources
10376
10601
  ON sources.project_id = rows.project_id
10377
10602
  AND sources.source_id = rows.source_id
10378
10603
  AND sources.materialization_id = rows.materialization_id
10604
+ AND sources.root_sort_key = rows.root_sort_key
10379
10605
  JOIN incident_objects
10380
10606
  ON incident_objects.object_type_id = rows.source_type_id
10381
10607
  AND incident_objects.primary_id = rows.source_primary_id
10382
10608
  WHERE rows.project_id = ${this.projectId}
10383
10609
  AND rows.entity_kind = 'link'
10384
- AND sources.status = 'active'
10610
+ AND sources.active
10385
10611
  UNION
10386
10612
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10387
10613
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10388
10614
  FROM ontology_source_rows AS rows
10389
- JOIN ontology_sources AS sources
10615
+ JOIN ontology_source_roots AS sources
10390
10616
  ON sources.project_id = rows.project_id
10391
10617
  AND sources.source_id = rows.source_id
10392
10618
  AND sources.materialization_id = rows.materialization_id
10619
+ AND sources.root_sort_key = rows.root_sort_key
10393
10620
  JOIN incident_objects
10394
10621
  ON incident_objects.object_type_id = rows.target_type_id
10395
10622
  AND incident_objects.primary_id = rows.target_primary_id
10396
10623
  WHERE rows.project_id = ${this.projectId}
10397
10624
  AND rows.entity_kind = 'link'
10398
- AND sources.status = 'active'
10625
+ AND sources.active
10399
10626
  ), diff_links AS (
10400
10627
  SELECT * FROM replacement_links
10401
10628
  UNION SELECT * FROM incident_links
@@ -10463,9 +10690,9 @@ function activeSourceMap(rows) {
10463
10690
  const result = new Map;
10464
10691
  for (const row of rows) {
10465
10692
  const source = storedSource(row);
10466
- const key = projectionEntityKey(source.assertion);
10693
+ const key = projectionEntityKey2(source.assertion);
10467
10694
  if (result.has(key)) {
10468
- throw new MaterializationConflictError2("source-materialization", `Multiple active sources assert ${key}.`);
10695
+ throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert ${key}.`);
10469
10696
  }
10470
10697
  result.set(key, source);
10471
10698
  }
@@ -10480,7 +10707,7 @@ function activeScopeSourceMap(rows) {
10480
10707
  const ref = source.assertion.ref;
10481
10708
  const key = linkScopeSortKey(ref.source, ref.linkId);
10482
10709
  if (result.has(key)) {
10483
- throw new MaterializationConflictError2("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
10710
+ throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
10484
10711
  }
10485
10712
  result.set(key, source);
10486
10713
  }
@@ -10544,7 +10771,7 @@ function storedSource(row) {
10544
10771
  }
10545
10772
  function effectiveObjectSnapshot(row) {
10546
10773
  if (!row.last_commit_id) {
10547
- throw new MaterializationConflictError2("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
10774
+ throw new MaterializationConflictError3("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
10548
10775
  }
10549
10776
  return {
10550
10777
  ref: objectRefFromColumns(row),
@@ -10557,7 +10784,7 @@ function effectiveObjectSnapshot(row) {
10557
10784
  }
10558
10785
  function effectiveLinkSnapshot(row) {
10559
10786
  if (!row.last_commit_id) {
10560
- throw new MaterializationConflictError2("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
10787
+ throw new MaterializationConflictError3("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
10561
10788
  }
10562
10789
  return {
10563
10790
  ref: linkRefFromColumns(row),
@@ -10572,7 +10799,7 @@ function effectiveLinkSnapshot(row) {
10572
10799
  function storedPoint(row) {
10573
10800
  const at = toIsoString2(row.at);
10574
10801
  if (!row.last_commit_id) {
10575
- throw new MaterializationConflictError2("timeseries-point", `Telemetry point ${telemetryPointSortKey({
10802
+ throw new MaterializationConflictError3("timeseries-point", `Telemetry point ${telemetryPointSortKey({
10576
10803
  object: { objectTypeId: row.object_type_id, primaryId: row.object_id },
10577
10804
  propertyId: row.property_id
10578
10805
  }, at)} lacks materializer provenance.`);
@@ -10631,6 +10858,9 @@ function jsonTupleExpression(parts) {
10631
10858
 
10632
10859
  // src/ontology-storage/materialization-session.ts
10633
10860
  class PgMaterializationSessionState extends ProviderMaterializationSessionState {
10861
+ stagedWorkCount = 0;
10862
+ changedObjects = 0;
10863
+ changedLinks = 0;
10634
10864
  appliedPlanCursor = null;
10635
10865
  appliedEventCursor = null;
10636
10866
  }
@@ -10647,7 +10877,7 @@ class PgMaterializationSessions {
10647
10877
  }
10648
10878
  async create(header) {
10649
10879
  if (!this.context?.active) {
10650
- throw new MaterializationConflictError3("effective-state", "Materialization sessions require an active storage transaction.");
10880
+ throw new MaterializationConflictError4("effective-state", "Materialization sessions require an active storage transaction.");
10651
10881
  }
10652
10882
  await this.ensureTables();
10653
10883
  const session = new PgMaterializationSessionState(structuredClone(header), this.context.id);
@@ -10659,7 +10889,7 @@ class PgMaterializationSessions {
10659
10889
  require(session) {
10660
10890
  const value = this.sessions.get(session.providerToken);
10661
10891
  if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
10662
- throw new MaterializationConflictError3("effective-state", "Materialization session is inactive.");
10892
+ throw new MaterializationConflictError4("effective-state", "Materialization session is inactive.");
10663
10893
  }
10664
10894
  return value;
10665
10895
  }
@@ -10717,14 +10947,18 @@ class PgMaterializationSessions {
10717
10947
  }
10718
10948
  throw error;
10719
10949
  }
10950
+ session.stagedWorkCount += records.length;
10720
10951
  }
10721
10952
  async* stream(input) {
10722
10953
  const session = this.require(input.session);
10723
10954
  const stream = session.workStreams[input.order];
10724
10955
  if (stream.started) {
10725
- throw new MaterializationConflictError3("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
10956
+ throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
10726
10957
  }
10727
10958
  stream.started = true;
10959
+ if (!session.workSealed && session.stagedWorkCount >= 1e4) {
10960
+ await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
10961
+ }
10728
10962
  session.workSealed = true;
10729
10963
  let cursor = null;
10730
10964
  while (true) {
@@ -10745,10 +10979,13 @@ class PgMaterializationSessions {
10745
10979
  return [];
10746
10980
  const keys = refs.map((ref) => `object-existence:${JSON.stringify([ref.objectTypeId, ref.primaryId])}`);
10747
10981
  const rows = await this.sql`
10748
- SELECT unique_key, payload FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
10749
- WHERE session_id = ${session.id}
10750
- AND unique_key = ANY(${this.sql.array(keys)}::text[])
10751
- AND kind = 'object-existence'
10982
+ SELECT work.unique_key, work.payload
10983
+ FROM unnest(${this.sql.array(keys)}::text[]) AS requested(unique_key)
10984
+ CROSS JOIN LATERAL (
10985
+ SELECT unique_key, payload FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
10986
+ WHERE session_id = ${session.id} AND unique_key = requested.unique_key
10987
+ OFFSET 0
10988
+ ) AS work
10752
10989
  `;
10753
10990
  const found = new Map(rows.map((row) => [row.unique_key, row.payload]));
10754
10991
  return keys.flatMap((key) => {
@@ -10756,6 +10993,9 @@ class PgMaterializationSessions {
10756
10993
  if (!payload)
10757
10994
  return [];
10758
10995
  const record = structuredClone(payload);
10996
+ if (record.kind !== "object-existence") {
10997
+ invalidCorrelation("Object existence work has an invalid kind.");
10998
+ }
10759
10999
  return [{ ref: record.ref, exists: record.exists }];
10760
11000
  });
10761
11001
  }
@@ -10910,7 +11150,7 @@ function workCursor(row) {
10910
11150
  // src/ontology-storage/materialization-writer.ts
10911
11151
  import {
10912
11152
  linkRefKey as linkRefKey2,
10913
- MaterializationConflictError as MaterializationConflictError4,
11153
+ MaterializationConflictError as MaterializationConflictError5,
10914
11154
  objectRefKey as objectRefKey2,
10915
11155
  telemetryPointKey as telemetryPointKey2
10916
11156
  } from "@sixb/core/internal/materialization";
@@ -11426,7 +11666,7 @@ function effectiveLinkConflict(item) {
11426
11666
  return effectiveConflict(`Expected link ${linkIdentityKey(item)} changed.`);
11427
11667
  }
11428
11668
  function pointConflict(item) {
11429
- return new MaterializationConflictError4("timeseries-point", `Telemetry point ${telemetryPointKey2({
11669
+ return new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey2({
11430
11670
  object: { objectTypeId: item.objectTypeId, primaryId: item.objectId },
11431
11671
  propertyId: item.propertyId
11432
11672
  }, item.at)} changed.`);
@@ -11461,14 +11701,14 @@ class PgOntologyMaterializationStorage {
11461
11701
  const linkScopeRevisions = await reader.linkScopeRevisions(input.expected.linkScopes);
11462
11702
  for (const [index, expected] of input.expected.linkScopes.entries()) {
11463
11703
  if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
11464
- throw new MaterializationConflictError5("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
11704
+ throw new MaterializationConflictError6("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
11465
11705
  }
11466
11706
  }
11467
11707
  const points = await reader.exactPoints(input.expected.points, true);
11468
11708
  const pointRevisions = new Map(points.map((point) => [telemetryPointKey3(point.series, point.at), point.lastCommitId]));
11469
11709
  for (const expected of input.expected.points) {
11470
11710
  if ((pointRevisions.get(telemetryPointKey3(expected.series, expected.at)) ?? null) !== expected.lastCommitId) {
11471
- throw new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey3(expected.series, expected.at)} changed.`);
11711
+ throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey3(expected.series, expected.at)} changed.`);
11472
11712
  }
11473
11713
  }
11474
11714
  return session.publicSession();
@@ -11527,10 +11767,10 @@ class PgOntologyMaterializationStorage {
11527
11767
  const reader = new PgMaterializationStateReader(this.sql, session.header.commit.projectId);
11528
11768
  if (input.entityKind === "object") {
11529
11769
  if (replacement.projectionKind !== "object") {
11530
- throw new MaterializationConflictError5("source-materialization", "Link projection replacement cannot stream object state.");
11770
+ throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
11531
11771
  }
11532
11772
  if (replacement.objectStreamStarted) {
11533
- throw new MaterializationConflictError5("effective-state", "Replacement object state may only be streamed once per session.");
11773
+ throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
11534
11774
  }
11535
11775
  replacement.objectStreamStarted = true;
11536
11776
  for await (const identities of reader.replacementIdentities({
@@ -11538,6 +11778,7 @@ class PgOntologyMaterializationStorage {
11538
11778
  sourceId: replacement.sourceId,
11539
11779
  candidateMaterializationId: replacement.candidateMaterializationId,
11540
11780
  previousMaterializationId: replacement.previousMaterializationId,
11781
+ incremental: replacement.incremental,
11541
11782
  kind: "object",
11542
11783
  pageRows: input.pageRows
11543
11784
  })) {
@@ -11556,10 +11797,10 @@ class PgOntologyMaterializationStorage {
11556
11797
  return;
11557
11798
  }
11558
11799
  if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
11559
- throw new MaterializationConflictError5("effective-state", "Object projection replacement must fully stream object state before link state.");
11800
+ throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
11560
11801
  }
11561
11802
  if (replacement.linkStreamStarted) {
11562
- throw new MaterializationConflictError5("effective-state", "Replacement link state may only be streamed once per session.");
11803
+ throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
11563
11804
  }
11564
11805
  replacement.linkStreamStarted = true;
11565
11806
  const materializationIds = [
@@ -11571,6 +11812,7 @@ class PgOntologyMaterializationStorage {
11571
11812
  sourceId: replacement.sourceId,
11572
11813
  candidateMaterializationId: replacement.candidateMaterializationId,
11573
11814
  previousMaterializationId: replacement.previousMaterializationId,
11815
+ incremental: replacement.incremental,
11574
11816
  kind: "link",
11575
11817
  pageRows: input.pageRows
11576
11818
  })) {
@@ -11581,7 +11823,7 @@ class PgOntologyMaterializationStorage {
11581
11823
  }
11582
11824
  return identity;
11583
11825
  });
11584
- const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities);
11826
+ const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
11585
11827
  yield { objects: [], links };
11586
11828
  }
11587
11829
  this.sessions.require(input.session);
@@ -11604,6 +11846,8 @@ class PgOntologyMaterializationStorage {
11604
11846
  assertPlanChunkCorrelations(input.chunk, commit);
11605
11847
  const progress = await this.sessions.prepareChunkSequence(session, input.chunk);
11606
11848
  await this.writer.apply(commit.projectId, commit.id, input.chunk);
11849
+ session.changedObjects += input.chunk.effective.objectUpserts.length + input.chunk.effective.objectDeletes.length;
11850
+ session.changedLinks += input.chunk.effective.linkUpserts.length + input.chunk.effective.linkDeletes.length;
11607
11851
  this.sessions.commitChunkSequence(session, progress);
11608
11852
  }
11609
11853
  async finalize(input) {
@@ -11613,6 +11857,10 @@ class PgOntologyMaterializationStorage {
11613
11857
  for (const activation of input.finalization.sourceActivations) {
11614
11858
  await this.activateSource(session, activation);
11615
11859
  }
11860
+ if (session.changedObjects >= 1000)
11861
+ await this.sql`ANALYZE objects`;
11862
+ if (session.changedLinks >= 1000)
11863
+ await this.sql`ANALYZE links`;
11616
11864
  const record = await this.insertCommit(session.header, input);
11617
11865
  await this.sessions.release(session);
11618
11866
  return { commit: record };
@@ -11623,14 +11871,14 @@ class PgOntologyMaterializationStorage {
11623
11871
  async requireReplacement(session, input) {
11624
11872
  if (session.replacement) {
11625
11873
  if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
11626
- throw new MaterializationConflictError5("source-materialization", "Materialization session already owns another replacement union.");
11874
+ throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
11627
11875
  }
11628
11876
  return session.replacement;
11629
11877
  }
11630
11878
  const projectId = session.header.commit.projectId;
11631
11879
  const candidate = await this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId, true);
11632
11880
  if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
11633
- throw new MaterializationConflictError5("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
11881
+ throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
11634
11882
  }
11635
11883
  await assertProjectionExecution(this.sql, {
11636
11884
  projectId,
@@ -11640,12 +11888,13 @@ class PgOntologyMaterializationStorage {
11640
11888
  });
11641
11889
  const previous = await this.getActiveSource(projectId, input.source.projectionId, true);
11642
11890
  if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
11643
- throw new MaterializationConflictError5("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
11891
+ throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
11644
11892
  }
11645
11893
  session.replacement = {
11646
11894
  sourceId: input.source.projectionId,
11647
11895
  candidateMaterializationId: input.candidateMaterializationId,
11648
11896
  previousMaterializationId: previous?.materialization_id ?? null,
11897
+ incremental: candidate.base_materialization_id !== null,
11649
11898
  projectionKind: candidate.projection_kind,
11650
11899
  objectStreamStarted: false,
11651
11900
  objectStreamCompleted: false,
@@ -11665,7 +11914,7 @@ class PgOntologyMaterializationStorage {
11665
11914
  FOR UPDATE
11666
11915
  `;
11667
11916
  if (duplicateIdentity) {
11668
- throw new MaterializationConflictError5("idempotency", duplicateIdentity.id === header.commit.id ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
11917
+ throw new MaterializationConflictError6("idempotency", duplicateIdentity.id === header.commit.id ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
11669
11918
  }
11670
11919
  const origin = originColumns(header.commit.origin);
11671
11920
  if (origin.runId === null)
@@ -11688,7 +11937,7 @@ class PgOntologyMaterializationStorage {
11688
11937
  FOR UPDATE
11689
11938
  `;
11690
11939
  if (duplicateOrigin) {
11691
- throw new MaterializationConflictError5("run-correlation", "Ontology commit origin already has an authoritative commit.");
11940
+ throw new MaterializationConflictError6("run-correlation", "Ontology commit origin already has an authoritative commit.");
11692
11941
  }
11693
11942
  }
11694
11943
  async assertSources(expectedSources, projectId) {
@@ -11710,29 +11959,29 @@ class PgOntologyMaterializationStorage {
11710
11959
  }
11711
11960
  assertSourceRow(expected, active) {
11712
11961
  if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
11713
- throw new MaterializationConflictError5("projection-fence", `Source '${expected.source.projectionId}' changed.`);
11962
+ throw new MaterializationConflictError6("projection-fence", `Source '${expected.source.projectionId}' changed.`);
11714
11963
  }
11715
11964
  }
11716
11965
  assertObject(row, expected) {
11717
11966
  if (!expected.exists) {
11718
11967
  if (row) {
11719
- throw new MaterializationConflictError5("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
11968
+ throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
11720
11969
  }
11721
11970
  return;
11722
11971
  }
11723
11972
  if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
11724
- throw new MaterializationConflictError5("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
11973
+ throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
11725
11974
  }
11726
11975
  }
11727
11976
  assertLink(lastCommitId, expected) {
11728
11977
  if (!expected.exists) {
11729
11978
  if (lastCommitId !== undefined) {
11730
- throw new MaterializationConflictError5("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
11979
+ throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
11731
11980
  }
11732
11981
  return;
11733
11982
  }
11734
11983
  if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
11735
- throw new MaterializationConflictError5("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
11984
+ throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
11736
11985
  }
11737
11986
  }
11738
11987
  async assertFinalization(session, input) {
@@ -11741,7 +11990,8 @@ class PgOntologyMaterializationStorage {
11741
11990
  assertMaterializationFinalizationCorrelation(session, input);
11742
11991
  const laneCounts = await this.sessions.laneCounts(session);
11743
11992
  assertMaterializationLaneCompletion(session, laneCounts);
11744
- await this.assertFinalCardinality(session);
11993
+ if (laneCounts.cardinality > 0)
11994
+ await this.assertFinalCardinality(session);
11745
11995
  const eventCount = laneCounts.event;
11746
11996
  const [outbox] = await this.sql`
11747
11997
  SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum,
@@ -11774,17 +12024,18 @@ class PgOntologyMaterializationStorage {
11774
12024
  }
11775
12025
  }
11776
12026
  async assertFinalCardinality(session) {
11777
- const [violation] = await this.sql`
11778
- WITH work AS (
11779
- SELECT payload->>'scopeSortKey' AS scope_sort_key,
11780
- payload->>'linkSortKey' AS link_sort_key,
12027
+ const table = `ontology_cardinality_${session.id.replaceAll("-", "")}`;
12028
+ await this.sql.unsafe(`CREATE TEMP TABLE ${table} ON COMMIT DROP AS
12029
+ SELECT sort_one AS scope_sort_key, sort_two AS link_sort_key,
11781
12030
  (payload->>'occupied')::boolean AS occupied,
11782
12031
  payload->'ref'->'source'->>'objectTypeId' AS source_type_id,
11783
12032
  payload->'ref'->'source'->>'primaryId' AS source_id,
11784
12033
  payload->'ref'->>'linkId' AS link_id
11785
- FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
11786
- WHERE session_id = ${session.id} AND kind = 'cardinality'
11787
- AND payload->>'view' = 'effective'
12034
+ FROM ${PG_MATERIALIZATION_WORK_TABLE}
12035
+ WHERE session_id = $1 AND lane = 'cardinality' AND major_order = 0`, [session.id]);
12036
+ await this.sql.unsafe(`ANALYZE ${table}`);
12037
+ const [violation] = await this.sql.unsafe(`WITH work AS (
12038
+ SELECT * FROM ${table}
11788
12039
  ), duplicate AS (
11789
12040
  SELECT scope_sort_key FROM work WHERE occupied
11790
12041
  GROUP BY scope_sort_key HAVING COUNT(*) > 1
@@ -11794,10 +12045,10 @@ class PgOntologyMaterializationStorage {
11794
12045
  SELECT scope_sort_key, link_sort_key FROM work WHERE occupied
11795
12046
  ), actual AS (
11796
12047
  SELECT scopes.scope_sort_key,
11797
- ${this.sql.unsafe(linkSortExpression("links"))} AS link_sort_key
12048
+ ${linkSortExpression("links")} AS link_sort_key
11798
12049
  FROM scopes
11799
12050
  JOIN links USING (source_type_id, source_id, link_id)
11800
- WHERE links.project_id = ${session.header.commit.projectId}
12051
+ WHERE links.project_id = $1
11801
12052
  ), differences AS (
11802
12053
  (SELECT * FROM expected EXCEPT SELECT * FROM actual)
11803
12054
  UNION ALL
@@ -11806,8 +12057,8 @@ class PgOntologyMaterializationStorage {
11806
12057
  SELECT 'duplicate'::text AS reason FROM duplicate
11807
12058
  UNION ALL
11808
12059
  SELECT 'mismatch'::text AS reason FROM differences
11809
- LIMIT 1
11810
- `;
12060
+ LIMIT 1`, [session.header.commit.projectId]);
12061
+ await this.sql.unsafe(`DROP TABLE ${table}`);
11811
12062
  if (violation?.reason === "duplicate") {
11812
12063
  invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
11813
12064
  }
@@ -11903,7 +12154,7 @@ class PgOntologyMaterializationStorage {
11903
12154
  assertSourceActivationCorrelation(session, activation);
11904
12155
  const candidate = await this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId, true);
11905
12156
  if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
11906
- throw new MaterializationConflictError5("source-materialization", "Source activation candidate is missing or is not ready.");
12157
+ throw new MaterializationConflictError6("source-materialization", "Source activation candidate is missing or is not ready.");
11907
12158
  }
11908
12159
  await assertProjectionExecution(this.sql, {
11909
12160
  projectId: commit.projectId,
@@ -11916,6 +12167,7 @@ class PgOntologyMaterializationStorage {
11916
12167
  }
11917
12168
  const previous = await this.getActiveSource(commit.projectId, activation.source.projectionId, true);
11918
12169
  this.assertSourceRow(activation.expected, previous);
12170
+ await activateSourceRoots(this.sql, commit.projectId, candidate, activation);
11919
12171
  if (previous) {
11920
12172
  assertPinnedDatasetWatermark({
11921
12173
  datasetId: previous.dataset_id,
@@ -11937,7 +12189,7 @@ class PgOntologyMaterializationStorage {
11937
12189
  RETURNING materialization_id
11938
12190
  `;
11939
12191
  if (superseded.length !== 1) {
11940
- throw new MaterializationConflictError5("projection-fence", `Source '${activation.source.projectionId}' changed.`);
12192
+ throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
11941
12193
  }
11942
12194
  }
11943
12195
  let activated;
@@ -11956,12 +12208,12 @@ class PgOntologyMaterializationStorage {
11956
12208
  `;
11957
12209
  } catch (error) {
11958
12210
  if (isUniqueViolation(error)) {
11959
- throw new MaterializationConflictError5("projection-fence", `Source '${activation.source.projectionId}' changed.`);
12211
+ throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
11960
12212
  }
11961
12213
  throw error;
11962
12214
  }
11963
12215
  if (activated.length !== 1) {
11964
- throw new MaterializationConflictError5("source-materialization", "Source activation candidate changed.");
12216
+ throw new MaterializationConflictError6("source-materialization", "Source activation candidate changed.");
11965
12217
  }
11966
12218
  }
11967
12219
  async insertCommit(header, input) {
@@ -11988,7 +12240,7 @@ class PgOntologyMaterializationStorage {
11988
12240
  `;
11989
12241
  if (!rows[0]) {
11990
12242
  await this.assertCommitAbsent(header);
11991
- throw new MaterializationConflictError5("idempotency", "Ontology commit identity already exists.");
12243
+ throw new MaterializationConflictError6("idempotency", "Ontology commit identity already exists.");
11992
12244
  }
11993
12245
  return commitRecord(rows[0]);
11994
12246
  }
@@ -12036,8 +12288,8 @@ function databaseCount(value) {
12036
12288
  // src/ontology-storage/outbox.ts
12037
12289
  import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
12038
12290
  import {
12039
- MaterializationConflictError as MaterializationConflictError6,
12040
- MaterializationValidationError
12291
+ MaterializationConflictError as MaterializationConflictError7,
12292
+ MaterializationValidationError as MaterializationValidationError2
12041
12293
  } from "@sixb/core/internal/materialization";
12042
12294
  import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
12043
12295
  class PgOntologyOutboxStorage {
@@ -12053,7 +12305,7 @@ class PgOntologyOutboxStorage {
12053
12305
  const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
12054
12306
  assertNonblank(input.leaseId, "Ontology outbox lease id");
12055
12307
  if (leaseExpiry <= now) {
12056
- throw new MaterializationValidationError("Ontology outbox lease expiry must be later than the claim time.");
12308
+ throw new MaterializationValidationError2("Ontology outbox lease expiry must be later than the claim time.");
12057
12309
  }
12058
12310
  const rows = await sql`
12059
12311
  WITH candidates AS (
@@ -12083,7 +12335,7 @@ class PgOntologyOutboxStorage {
12083
12335
  return rows.map((row) => {
12084
12336
  const record = outboxRecord(row);
12085
12337
  if (record.leaseId === null || record.leaseExpiresAt === null) {
12086
- throw new MaterializationConflictError6("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
12338
+ throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
12087
12339
  }
12088
12340
  return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt };
12089
12341
  });
@@ -12204,7 +12456,7 @@ function validateLeaseIds(ids) {
12204
12456
  for (const id of ids) {
12205
12457
  assertNonblank(id, "Ontology outbox event id");
12206
12458
  if (seen.has(id)) {
12207
- throw new MaterializationValidationError(`Ontology outbox lease batch repeats event '${id}'.`);
12459
+ throw new MaterializationValidationError2(`Ontology outbox lease batch repeats event '${id}'.`);
12208
12460
  }
12209
12461
  seen.add(id);
12210
12462
  }
@@ -12215,11 +12467,11 @@ function assertLeaseInput(input) {
12215
12467
  assertNonblank(input.leaseId, "Ontology outbox lease id");
12216
12468
  }
12217
12469
  function leaseConflict() {
12218
- return new MaterializationConflictError6("outbox-lease", "Ontology outbox lease does not match.");
12470
+ return new MaterializationConflictError7("outbox-lease", "Ontology outbox lease does not match.");
12219
12471
  }
12220
12472
 
12221
12473
  // src/ontology-storage/sources.ts
12222
- import { MaterializationValidationError as MaterializationValidationError2 } from "@sixb/core/internal/materialization";
12474
+ import { MaterializationValidationError as MaterializationValidationError3 } from "@sixb/core/internal/materialization";
12223
12475
  import {
12224
12476
  assertSourceBeginInput as assertBeginInput,
12225
12477
  assertSourceExecutionIdentity as assertExecutionIdentity,
@@ -12233,8 +12485,61 @@ import {
12233
12485
  sourceMaterializationIdentity,
12234
12486
  sourceStageRow,
12235
12487
  sourceStageRows,
12236
- utf8SortKey
12488
+ utf8SortKey as utf8SortKey2
12237
12489
  } from "@sixb/core/internal/ontology-storage-provider";
12490
+
12491
+ // src/ontology-storage/source-cleanup.ts
12492
+ async function cleanupSourceVersions(sql, input) {
12493
+ let remaining = input.limit;
12494
+ const removedRows = await sql`
12495
+ WITH selected AS (
12496
+ SELECT rows.ctid FROM ontology_source_roots AS roots
12497
+ JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
12498
+ JOIN ontology_source_rows AS rows
12499
+ ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
12500
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
12501
+ WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
12502
+ AND sources.status IN ('superseded', 'abandoned')
12503
+ ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
12504
+ LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
12505
+ ) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
12506
+ `;
12507
+ let rowsDeleted = removedRows.length;
12508
+ remaining -= rowsDeleted;
12509
+ if (remaining > 0) {
12510
+ const removed = await sql`
12511
+ WITH selected AS (
12512
+ SELECT roots.ctid FROM ontology_source_roots AS roots
12513
+ JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
12514
+ WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
12515
+ AND sources.status IN ('superseded', 'abandoned')
12516
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
12517
+ WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
12518
+ AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
12519
+ ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
12520
+ LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
12521
+ ) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
12522
+ `;
12523
+ rowsDeleted += removed.length;
12524
+ remaining -= removed.length;
12525
+ }
12526
+ const removedManifests = remaining === 0 ? [] : await sql`
12527
+ WITH selected AS (
12528
+ SELECT sources.ctid FROM ontology_sources AS sources
12529
+ WHERE project_id = ${input.projectId} AND status IN ('superseded', 'abandoned') AND terminal_at < ${input.terminalBefore}
12530
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
12531
+ WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
12532
+ AND roots.materialization_id = sources.materialization_id)
12533
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
12534
+ WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
12535
+ AND rows.materialization_id = sources.materialization_id)
12536
+ ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
12537
+ ) DELETE FROM ontology_sources AS sources USING selected WHERE sources.ctid = selected.ctid RETURNING 1
12538
+ `;
12539
+ return { rowsDeleted, materializationsDeleted: removedManifests.length };
12540
+ }
12541
+
12542
+ // src/ontology-storage/sources.ts
12238
12543
  class PgOntologySourceStorage {
12239
12544
  runRootOperation;
12240
12545
  constructor(runRootOperation) {
@@ -12265,14 +12570,15 @@ class PgOntologySourceStorage {
12265
12570
  dataset_id, dataset_version_id, dataset_version_created_at,
12266
12571
  projection_revision, ownership_hash, ontology_revision,
12267
12572
  root_count, assertion_count, created_at, ready_at, activated_at,
12268
- terminal_at, last_commit_id, updated_at
12573
+ terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
12269
12574
  ) VALUES (
12270
12575
  ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
12271
12576
  ${input.execution.projectionRunId}, ${input.projectionKind}, 'replacement',
12272
12577
  'staging', ${input.execution.executionToken}, ${input.datasetVersion.datasetId},
12273
12578
  ${input.datasetVersion.versionId}, ${input.datasetVersion.createdAt},
12274
12579
  ${input.projectionRevision}, ${input.ownershipHash}, ${input.ontologyRevision},
12275
- NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt}
12580
+ NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt},
12581
+ ${input.base?.materializationId ?? null}, ${input.base?.lastCommitId ?? null}
12276
12582
  )
12277
12583
  ON CONFLICT DO NOTHING
12278
12584
  RETURNING *
@@ -12301,6 +12607,7 @@ class PgOntologySourceStorage {
12301
12607
  const rows = sourceStageRows(manifest.projection_kind, input.rows);
12302
12608
  const existing = await this.findStageRows(sql, input, rows);
12303
12609
  const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
12610
+ await stageSourceRoots(sql, manifest, input);
12304
12611
  await this.insertStageRows(sql, input, pending);
12305
12612
  return { inserted: pending.length, unchanged };
12306
12613
  });
@@ -12324,7 +12631,10 @@ class PgOntologySourceStorage {
12324
12631
  throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
12325
12632
  }
12326
12633
  if (input.readyAt < toIsoString2(manifest.created_at)) {
12327
- throw new MaterializationValidationError2("Source readyAt cannot precede source createdAt.");
12634
+ throw new MaterializationValidationError3("Source readyAt cannot precede source createdAt.");
12635
+ }
12636
+ if (input.assertionCount >= 1000 || input.rootCount >= 1000) {
12637
+ await sql`ANALYZE ontology_source_rows, ontology_source_roots`;
12328
12638
  }
12329
12639
  await this.assertReadyState(sql, manifest, input.rootCount, input.assertionCount);
12330
12640
  const rows = await sql`
@@ -12371,61 +12681,7 @@ class PgOntologySourceStorage {
12371
12681
  assertNonblank(input.projectId, "Terminal source cleanup project id");
12372
12682
  assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true);
12373
12683
  assertPositiveInteger(input.limit, "Terminal source cleanup limit");
12374
- const manifests = await sql`
12375
- SELECT * FROM ontology_sources
12376
- WHERE project_id = ${input.projectId}
12377
- AND status IN ('superseded', 'abandoned')
12378
- AND terminal_at < ${input.terminalBefore}
12379
- ORDER BY terminal_at, source_id, materialization_id
12380
- LIMIT ${input.limit}
12381
- FOR UPDATE SKIP LOCKED
12382
- `;
12383
- let remaining = input.limit;
12384
- let rowsDeleted = 0;
12385
- let materializationsDeleted = 0;
12386
- for (const manifest of manifests) {
12387
- if (remaining === 0)
12388
- break;
12389
- const deletedRows = await sql`
12390
- WITH selected AS (
12391
- SELECT rows.ctid
12392
- FROM ontology_source_rows AS rows
12393
- WHERE rows.project_id = ${manifest.project_id}
12394
- AND rows.source_id = ${manifest.source_id}
12395
- AND rows.materialization_id = ${manifest.materialization_id}
12396
- ORDER BY rows.entity_sort_key
12397
- LIMIT ${remaining}
12398
- FOR UPDATE SKIP LOCKED
12399
- )
12400
- DELETE FROM ontology_source_rows AS rows
12401
- USING selected
12402
- WHERE rows.ctid = selected.ctid
12403
- RETURNING rows.entity_sort_key
12404
- `;
12405
- rowsDeleted += deletedRows.length;
12406
- remaining -= deletedRows.length;
12407
- if (remaining === 0)
12408
- break;
12409
- const removed = await sql`
12410
- DELETE FROM ontology_sources AS sources
12411
- WHERE sources.project_id = ${manifest.project_id}
12412
- AND sources.source_id = ${manifest.source_id}
12413
- AND sources.materialization_id = ${manifest.materialization_id}
12414
- AND sources.status IN ('superseded', 'abandoned')
12415
- AND sources.terminal_at = ${manifest.terminal_at}
12416
- AND sources.terminal_at < ${input.terminalBefore}
12417
- AND NOT EXISTS (
12418
- SELECT 1 FROM ontology_source_rows AS rows
12419
- WHERE rows.project_id = sources.project_id
12420
- AND rows.source_id = sources.source_id
12421
- AND rows.materialization_id = sources.materialization_id
12422
- )
12423
- RETURNING sources.materialization_id
12424
- `;
12425
- materializationsDeleted += removed.length;
12426
- remaining -= removed.length;
12427
- }
12428
- return { rowsDeleted, materializationsDeleted };
12684
+ return cleanupSourceVersions(sql, input);
12429
12685
  });
12430
12686
  }
12431
12687
  async summarizeTerminal(input) {
@@ -12473,7 +12729,7 @@ class PgOntologySourceStorage {
12473
12729
  }
12474
12730
  async transitionToAbandoned(sql, manifest, abandonedAt) {
12475
12731
  if (abandonedAt < toIsoString2(manifest.created_at) || manifest.ready_at !== null && abandonedAt < toIsoString2(manifest.ready_at)) {
12476
- throw new MaterializationValidationError2("Source abandonedAt cannot precede source creation or readiness.");
12732
+ throw new MaterializationValidationError3("Source abandonedAt cannot precede source creation or readiness.");
12477
12733
  }
12478
12734
  const rows = await sql`
12479
12735
  UPDATE ontology_sources
@@ -12488,14 +12744,19 @@ class PgOntologySourceStorage {
12488
12744
  if (!rows[0]) {
12489
12745
  throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
12490
12746
  }
12747
+ await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
12748
+ WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
12749
+ AND materialization_id = ${manifest.materialization_id} AND NOT active`;
12491
12750
  return sourceRecord(rows[0]);
12492
12751
  }
12493
12752
  async assertReadyState(sql, manifest, rootCount, assertionCount) {
12494
12753
  const [counts] = await sql`
12495
- SELECT COUNT(*) AS assertions, COUNT(DISTINCT root_key) AS roots,
12754
+ SELECT (SELECT COUNT(*) FROM ontology_source_rows
12755
+ WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
12756
+ AND materialization_id = ${manifest.materialization_id}) AS assertions, COUNT(*) AS roots,
12496
12757
  COUNT(DISTINCT staging_ordinal) AS ordinals,
12497
12758
  MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
12498
- FROM ontology_source_rows
12759
+ FROM ontology_source_roots
12499
12760
  WHERE project_id = ${manifest.project_id}
12500
12761
  AND source_id = ${manifest.source_id}
12501
12762
  AND materialization_id = ${manifest.materialization_id}
@@ -12506,8 +12767,9 @@ class PgOntologySourceStorage {
12506
12767
  const minimum = counts?.min_ordinal === null ? null : Number(counts?.min_ordinal);
12507
12768
  const maximum = counts?.max_ordinal === null ? null : Number(counts?.max_ordinal);
12508
12769
  if (assertions !== assertionCount || roots !== rootCount || ordinals !== rootCount || rootCount > 0 && (minimum !== 0 || maximum !== rootCount - 1)) {
12509
- throw new MaterializationValidationError2("Source ready counts do not match the staged roots and assertions.");
12770
+ throw new MaterializationValidationError3("Source ready counts do not match the staged roots and assertions.");
12510
12771
  }
12772
+ await assertSourceRootCoverage(sql, manifest);
12511
12773
  const [invalid] = manifest.projection_kind === "link" ? await sql`
12512
12774
  SELECT root_key
12513
12775
  FROM ontology_source_rows
@@ -12541,15 +12803,15 @@ class PgOntologySourceStorage {
12541
12803
  LIMIT 1
12542
12804
  `;
12543
12805
  if (invalid) {
12544
- throw new MaterializationValidationError2(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
12806
+ throw new MaterializationValidationError3(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
12545
12807
  }
12546
12808
  }
12547
12809
  async findStageRows(sql, input, rows) {
12548
12810
  if (rows.length === 0)
12549
12811
  return [];
12550
- const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey(row.rootKey)))];
12812
+ const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.rootKey)))];
12551
12813
  const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
12552
- const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey(row.entityKey)))];
12814
+ const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
12553
12815
  const existing = await sql`
12554
12816
  SELECT * FROM ontology_source_rows
12555
12817
  WHERE project_id = ${input.projectId}
@@ -12572,10 +12834,10 @@ class PgOntologySourceStorage {
12572
12834
  return {
12573
12835
  entityKind: row.assertion.kind,
12574
12836
  entityKey: JSON.parse(entityKey),
12575
- entitySortKey: utf8SortKey(entityKey),
12837
+ entitySortKey: utf8SortKey2(entityKey),
12576
12838
  rootKind: row.root.kind,
12577
12839
  rootKey: JSON.parse(rootKey),
12578
- rootSortKey: utf8SortKey(rootKey),
12840
+ rootSortKey: utf8SortKey2(rootKey),
12579
12841
  stagingOrdinal: row.stagingOrdinal,
12580
12842
  root: row.root,
12581
12843
  assertion: row.assertion,
@@ -12697,12 +12959,11 @@ import {
12697
12959
  } from "@sixb/core/internal/action-run-storage";
12698
12960
  import { assertActionRunExecution } from "@sixb/core/internal/action-run-storage-provider";
12699
12961
  import {
12700
- ActionRunError,
12701
12962
  actionRunPhaseRecordsEqual,
12702
12963
  canRequeueActionRunAfterEnqueueFailure,
12703
- finishActionRunPhase,
12704
- isTerminalActionRun
12705
- } from "@sixb/core/storage";
12964
+ finishActionRunPhase
12965
+ } from "@sixb/core/internal/storage";
12966
+ import { ActionRunError, isTerminalActionRun } from "@sixb/core/storage";
12706
12967
  class PgActionRunStorage {
12707
12968
  sql;
12708
12969
  executions;
@@ -12960,8 +13221,8 @@ class PgActionRunStorage {
12960
13221
  params.push(input.actionId);
12961
13222
  }
12962
13223
  if (input.actionIds) {
12963
- const placeholders2 = input.actionIds.map(() => `$${index++}`);
12964
- whereClauses.push(`action_id IN (${placeholders2.join(", ")})`);
13224
+ const placeholders = input.actionIds.map(() => `$${index++}`);
13225
+ whereClauses.push(`action_id IN (${placeholders.join(", ")})`);
12965
13226
  params.push(...input.actionIds);
12966
13227
  }
12967
13228
  if (input.objectTypeIds) {
@@ -12970,8 +13231,8 @@ class PgActionRunStorage {
12970
13231
  params.push("object");
12971
13232
  } else {
12972
13233
  const subjectKindIndex = index++;
12973
- const placeholders2 = input.objectTypeIds.map(() => `$${index++}`);
12974
- whereClauses.push(`(subject_kind <> $${subjectKindIndex} OR object_type_id IN (${placeholders2.join(", ")}))`);
13234
+ const placeholders = input.objectTypeIds.map(() => `$${index++}`);
13235
+ whereClauses.push(`(subject_kind <> $${subjectKindIndex} OR object_type_id IN (${placeholders.join(", ")}))`);
12975
13236
  params.push("object", ...input.objectTypeIds);
12976
13237
  }
12977
13238
  }
@@ -12998,8 +13259,8 @@ class PgActionRunStorage {
12998
13259
  }
12999
13260
  }
13000
13261
  if (input.statuses) {
13001
- const placeholders2 = input.statuses.map(() => `$${index++}`);
13002
- whereClauses.push(`status IN (${placeholders2.join(", ")})`);
13262
+ const placeholders = input.statuses.map(() => `$${index++}`);
13263
+ whereClauses.push(`status IN (${placeholders.join(", ")})`);
13003
13264
  params.push(...input.statuses);
13004
13265
  }
13005
13266
  if (input.startedAfter) {
@@ -13157,6 +13418,7 @@ import {
13157
13418
  aiModelCallCostDetails,
13158
13419
  aiModelCallCostMatchesUsage,
13159
13420
  buildAiAccountingOverviewFromFragments,
13421
+ directModelCallAttribution,
13160
13422
  normalizeAiAccountingQuery,
13161
13423
  normalizeAiModelCallAccountingQuery as normalizeAiModelCallAccountingQuery2,
13162
13424
  normalizeAiModelCallCostRecord,
@@ -13201,6 +13463,7 @@ WITH filtered AS (
13201
13463
  SELECT COALESCE(root_agent.execution_id, usage.execution_id) AS root_execution_id,
13202
13464
  usage.execution_id, usage.provider_id, usage.requested_model_id, usage.occurred_at,
13203
13465
  usage.total_tokens,
13466
+ execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
13204
13467
  COALESCE(cost.status, 'unvalued') AS valuation_status,
13205
13468
  cost.currency AS amount_currency, cost.amount_nanos,
13206
13469
  root_agent.id AS root_agent_run_id,
@@ -13213,6 +13476,7 @@ WITH filtered AS (
13213
13476
  workflow_node.workflow_id AS workflow_id,
13214
13477
  workflow_node.workflow_run_id AS workflow_run_id
13215
13478
  FROM ai_model_call_usage AS usage
13479
+ JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
13216
13480
  LEFT JOIN agent_runs AS direct_agent
13217
13481
  ON direct_agent.project_id = usage.project_id AND direct_agent.execution_id = usage.execution_id
13218
13482
  LEFT JOIN agent_runs AS root_agent
@@ -13250,6 +13514,7 @@ SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id
13250
13514
  CAST(SUM((amount_nanos / 1000000) % 1000000) AS TEXT) AS amount_middle,
13251
13515
  CAST(SUM(amount_nanos % 1000000) AS TEXT) AS amount_low,
13252
13516
  MAX(root_agent_run_id) AS root_agent_run_id,
13517
+ MAX(executor_kind) AS executor_kind, MAX(executor_id) AS executor_id, MAX(primitive_id) AS primitive_id,
13253
13518
  MAX(root_thread_id) AS root_thread_id,
13254
13519
  MAX(direct_kind) AS direct_kind,
13255
13520
  MAX(direct_run_id) AS direct_run_id,
@@ -13264,12 +13529,11 @@ GROUP BY filtered.root_execution_id, execution_id, provider_id, requested_model_
13264
13529
  `;
13265
13530
 
13266
13531
  // src/pg-ai-usage-storage.ts
13532
+ import { assertAiUsageExecutionId, normalizeAiModelCallRecord } from "@sixb/core/internal/storage";
13267
13533
  import { isModelReasoning, normalizeModelProviderIds } from "@sixb/core/models";
13268
13534
  import {
13269
13535
  AiUsageStorageError,
13270
- aggregateAiModelCallUsage,
13271
- assertAiUsageExecutionId,
13272
- normalizeAiModelCallRecord
13536
+ aggregateAiModelCallUsage
13273
13537
  } from "@sixb/core/storage";
13274
13538
  class PgAiUsageStorage {
13275
13539
  sql;
@@ -13581,11 +13845,14 @@ class PgAiCostStorage {
13581
13845
  ELSE NULL
13582
13846
  END AS attribution_agent_kind,
13583
13847
  workflow_node.node_id AS attribution_agent_step_id,
13584
- workflow_node.workflow_id AS attribution_workflow_id,
13848
+ COALESCE(workflow_node.workflow_id,
13849
+ CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
13850
+ ) AS attribution_workflow_id,
13585
13851
  cost.status AS cost_status,
13586
13852
  cost.currency AS cost_currency,
13587
13853
  cost.amount_nanos AS cost_amount_nanos
13588
13854
  FROM ai_model_call_usage AS usage
13855
+ JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
13589
13856
  LEFT JOIN agent_runs AS direct_agent
13590
13857
  ON direct_agent.project_id = usage.project_id
13591
13858
  AND direct_agent.execution_id = usage.execution_id
@@ -13712,6 +13979,7 @@ class PgAiCostStorage {
13712
13979
  workflow_agent.node_run_id AS attribution_node_run_id,
13713
13980
  workflow_node.workflow_id AS attribution_workflow_id,
13714
13981
  workflow_node.workflow_run_id AS attribution_workflow_run_id,
13982
+ execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
13715
13983
  COALESCE((
13716
13984
  SELECT jsonb_agg(groups.group_id ORDER BY groups.group_id)
13717
13985
  FROM ai_model_call_usage_groups AS groups
@@ -13726,6 +13994,7 @@ class PgAiCostStorage {
13726
13994
  cost.details AS cost_details,
13727
13995
  cost.rated_at AS cost_rated_at
13728
13996
  FROM ai_model_call_usage AS usage
13997
+ JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
13729
13998
  LEFT JOIN agent_runs AS direct_agent
13730
13999
  ON direct_agent.project_id = usage.project_id
13731
14000
  AND direct_agent.execution_id = usage.execution_id
@@ -13814,7 +14083,7 @@ function meterFragment(presentCallCount, total) {
13814
14083
  function accountingItemFromRow(row) {
13815
14084
  const usage = usageFromRow2(row, parseStringArray(row.requester_group_ids));
13816
14085
  const cost = row.cost_status === null ? undefined : costFromJoinedRow(row);
13817
- const attribution = row.attribution_kind === null ? undefined : row.attribution_kind === "agent" ? {
14086
+ const attribution = row.attribution_kind === null ? directModelCallAttribution(row.executor_kind, row.executor_id, row.primitive_id) : row.attribution_kind === "agent" ? {
13818
14087
  kind: "agent",
13819
14088
  agentRunId: required2(row.attribution_agent_run_id),
13820
14089
  threadId: required2(row.attribution_thread_id)
@@ -14649,8 +14918,7 @@ function createPgClient(options) {
14649
14918
  import {
14650
14919
  executionRecordFromStorageRow,
14651
14920
  executionRecordToStorageRow,
14652
- normalizeExecutionRecord,
14653
- validateExecutionRecordReferences
14921
+ prepareExecutionRecord
14654
14922
  } from "@sixb/core/internal/execution-storage";
14655
14923
  import { ExecutionStorageError } from "@sixb/core/storage";
14656
14924
  class PgExecutionStorage {
@@ -14663,8 +14931,7 @@ class PgExecutionStorage {
14663
14931
  this.shareSessions = shareSessions;
14664
14932
  }
14665
14933
  async create(input) {
14666
- const record = normalizeExecutionRecord(input);
14667
- await validateExecutionRecordReferences(record, {
14934
+ const record = await prepareExecutionRecord(input, {
14668
14935
  auth: this.auth,
14669
14936
  getExecution: (params) => this.getById(params),
14670
14937
  getShareSession: (params) => this.shareSessions.getById(params)
@@ -14681,6 +14948,7 @@ class PgExecutionStorage {
14681
14948
  source_id,
14682
14949
  requested_by_user_id,
14683
14950
  requested_by_service_account_id,
14951
+ requester_group_ids,
14684
14952
  correlation_id,
14685
14953
  parent_execution_id,
14686
14954
  authority_kind,
@@ -14704,6 +14972,7 @@ class PgExecutionStorage {
14704
14972
  ${row.sourceId},
14705
14973
  ${row.requestedByUserId},
14706
14974
  ${row.requestedByServiceAccountId},
14975
+ ${JSON.stringify(row.requesterGroupIds)}::text::jsonb,
14707
14976
  ${row.correlationId},
14708
14977
  ${row.parentExecutionId},
14709
14978
  ${row.authorityKind},
@@ -14751,6 +15020,7 @@ function toStorageRow(row) {
14751
15020
  sourceId: row.source_id,
14752
15021
  requestedByUserId: row.requested_by_user_id,
14753
15022
  requestedByServiceAccountId: row.requested_by_service_account_id,
15023
+ requesterGroupIds: row.requester_group_ids,
14754
15024
  correlationId: row.correlation_id,
14755
15025
  parentExecutionId: row.parent_execution_id,
14756
15026
  authorityKind: row.authority_kind,
@@ -14771,11 +15041,8 @@ function toStorageRow(row) {
14771
15041
  // src/pg-pipeline-run-storage.ts
14772
15042
  import { parseSixbFailure as parseSixbFailure3, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
14773
15043
  import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
14774
- import {
14775
- canRequeuePipelineRunAfterEnqueueFailure,
14776
- PIPELINE_RUN_FAILURE_CODES,
14777
- PipelineRunError
14778
- } from "@sixb/core/storage";
15044
+ import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
15045
+ import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
14779
15046
 
14780
15047
  // src/latest-run-query.ts
14781
15048
  async function queryLatestRunsByOwnerId(input) {
@@ -14783,12 +15050,12 @@ async function queryLatestRunsByOwnerId(input) {
14783
15050
  if (ownerIds.length === 0) {
14784
15051
  return [];
14785
15052
  }
14786
- const placeholders2 = ownerIds.map((_, index) => `$${index + 2}`).join(", ");
15053
+ const placeholders = ownerIds.map((_, index) => `$${index + 2}`).join(", ");
14787
15054
  const orderColumn = input.tableName === "sync_runs" || input.tableName === "pipeline_runs" || input.tableName === "projection_runs" ? "COALESCE(started_at, queued_at)" : "started_at";
14788
15055
  const rows = await input.sql.unsafe(`
14789
15056
  SELECT DISTINCT ON (${input.ownerColumn}) ${input.selectList ?? "*"}
14790
15057
  FROM ${input.tableName}
14791
- WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders2})
15058
+ WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})
14792
15059
  ORDER BY ${input.ownerColumn}, ${orderColumn} DESC, id DESC
14793
15060
  `, [input.projectId, ...ownerIds]);
14794
15061
  const latestByOwnerId = new Map(rows.map((row) => [input.ownerIdFor(row), row]));
@@ -15031,8 +15298,8 @@ class PgPipelineRunStorage {
15031
15298
  params.push(input.pipelineId);
15032
15299
  }
15033
15300
  if (input.pipelineIds) {
15034
- const placeholders2 = input.pipelineIds.map(() => `$${index++}`).join(", ");
15035
- whereClauses.push(`pipeline_id IN (${placeholders2})`);
15301
+ const placeholders = input.pipelineIds.map(() => `$${index++}`).join(", ");
15302
+ whereClauses.push(`pipeline_id IN (${placeholders})`);
15036
15303
  params.push(...input.pipelineIds);
15037
15304
  }
15038
15305
  index = appendRunListFilters(whereClauses, params, index, input, "COALESCE(started_at, queued_at)");
@@ -15230,7 +15497,7 @@ class PgProjectionRunStorage {
15230
15497
  input_exhausted = ${input.fixedBatchSize === undefined ? null : false},
15231
15498
  missing_target_object_type_id = ${null}, missing_target_object_id = ${null},
15232
15499
  missing_target_batch_ordinal = ${null}, missing_target_first_seen_at = ${null},
15233
- source_rows_read = ${0}, source_rows_skipped = ${0}, error = ${null}
15500
+ source_rows_read = ${0}, source_rows_skipped = ${0}, source_changes_read = ${null}, error = ${null}
15234
15501
  WHERE project_id = ${input.projectId} AND id = ${input.id}
15235
15502
  AND status = ${"failed"} AND error->>'code' = ${"queue.enqueue_failed"}
15236
15503
  RETURNING *
@@ -15341,7 +15608,8 @@ class PgProjectionRunStorage {
15341
15608
  UPDATE projection_runs
15342
15609
  SET
15343
15610
  source_rows_read = ${progress.sourceRowsRead},
15344
- source_rows_skipped = ${progress.sourceRowsSkipped}
15611
+ source_rows_skipped = ${progress.sourceRowsSkipped},
15612
+ source_changes_read = ${progress.sourceChangesRead ?? null}
15345
15613
  WHERE project_id = ${input.projectId}
15346
15614
  AND id = ${input.id}
15347
15615
  AND status = ${"running"}
@@ -15366,6 +15634,7 @@ class PgProjectionRunStorage {
15366
15634
  execution_token = ${null},
15367
15635
  source_rows_read = ${plan.progress.sourceRowsRead},
15368
15636
  source_rows_skipped = ${plan.progress.sourceRowsSkipped},
15637
+ source_changes_read = ${plan.progress.sourceChangesRead ?? null},
15369
15638
  input_exhausted = ${plan.inputExhausted ?? existingRow.input_exhausted},
15370
15639
  error = ${plan.error === undefined ? null : serializeSixbFailure4(plan.error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
15371
15640
  WHERE project_id = ${input.projectId}
@@ -15467,16 +15736,16 @@ class PgProjectionRunStorage {
15467
15736
  if (input.objectTypeIds.length === 0) {
15468
15737
  return { runs: [], hasMore: false, total: 0 };
15469
15738
  }
15470
- const placeholders2 = input.objectTypeIds.map(() => `$${index++}`);
15471
- const list = placeholders2.join(", ");
15739
+ const placeholders = input.objectTypeIds.map(() => `$${index++}`);
15740
+ const list = placeholders.join(", ");
15472
15741
  const sourceList = input.objectTypeIds.map(() => `$${index++}`).join(", ");
15473
15742
  const targetList = input.objectTypeIds.map(() => `$${index++}`).join(", ");
15474
15743
  whereClauses.push(`(object_type_id IN (${list})` + ` OR (source_object_type_id IN (${sourceList})` + ` AND target_object_type_id IN (${targetList})))`);
15475
15744
  params.push(...input.objectTypeIds, ...input.objectTypeIds, ...input.objectTypeIds);
15476
15745
  }
15477
15746
  if (input.statuses) {
15478
- const placeholders2 = input.statuses.map(() => `$${index++}`);
15479
- whereClauses.push(`status IN (${placeholders2.join(", ")})`);
15747
+ const placeholders = input.statuses.map(() => `$${index++}`);
15748
+ whereClauses.push(`status IN (${placeholders.join(", ")})`);
15480
15749
  params.push(...input.statuses);
15481
15750
  }
15482
15751
  if (input.startedAfter) {
@@ -15608,7 +15877,8 @@ function restoreProjectionRunRow(row) {
15608
15877
  missingTargetFirstSeenAt: row.missing_target_first_seen_at ? new Date(row.missing_target_first_seen_at) : undefined,
15609
15878
  progress: {
15610
15879
  sourceRowsRead: databaseSafeInteger2(row.source_rows_read, "sourceRowsRead"),
15611
- sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped")
15880
+ sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped"),
15881
+ ...row.source_changes_read == null ? {} : { sourceChangesRead: databaseSafeInteger2(row.source_changes_read, "sourceChangesRead") }
15612
15882
  },
15613
15883
  error: row.error ?? undefined
15614
15884
  };
@@ -15694,8 +15964,8 @@ class PgRulesStorage {
15694
15964
  params.push(input.objectTypeId);
15695
15965
  }
15696
15966
  if (input.objectTypeIds) {
15697
- const placeholders2 = input.objectTypeIds.map(() => `$${index++}`);
15698
- whereClauses.push(`object_type_id IN (${placeholders2.join(", ")})`);
15967
+ const placeholders = input.objectTypeIds.map(() => `$${index++}`);
15968
+ whereClauses.push(`object_type_id IN (${placeholders.join(", ")})`);
15699
15969
  params.push(...input.objectTypeIds);
15700
15970
  }
15701
15971
  if (input.primaryId) {
@@ -15871,12 +16141,9 @@ function toIsoString3(value) {
15871
16141
 
15872
16142
  // src/pg-sync-run-storage.ts
15873
16143
  import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
16144
+ import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
15874
16145
  import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
15875
- import {
15876
- canRequeueSyncRunAfterEnqueueFailure,
15877
- SYNC_RUN_FAILURE_CODES,
15878
- SyncRunError
15879
- } from "@sixb/core/storage";
16146
+ import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
15880
16147
  class PgSyncRunStorage {
15881
16148
  sql;
15882
16149
  executions;
@@ -16036,8 +16303,8 @@ class PgSyncRunStorage {
16036
16303
  params.push(input.syncId);
16037
16304
  }
16038
16305
  if (input.syncIds) {
16039
- const placeholders2 = input.syncIds.map(() => `$${index++}`);
16040
- whereClauses.push(`sync_id IN (${placeholders2.join(", ")})`);
16306
+ const placeholders = input.syncIds.map(() => `$${index++}`);
16307
+ whereClauses.push(`sync_id IN (${placeholders.join(", ")})`);
16041
16308
  params.push(...input.syncIds);
16042
16309
  }
16043
16310
  if (input.datasetId) {
@@ -16139,14 +16406,14 @@ class PgTimeseriesStorage {
16139
16406
  const params = [];
16140
16407
  let nextIndex = 1;
16141
16408
  const values = input.series.map((series, seriesIndex) => {
16142
- const placeholders2 = [
16409
+ const placeholders = [
16143
16410
  `$${nextIndex++}`,
16144
16411
  `$${nextIndex++}`,
16145
16412
  `$${nextIndex++}`,
16146
16413
  `$${nextIndex++}`
16147
16414
  ];
16148
16415
  params.push(seriesIndex, series.objectTypeId, series.objectId, series.propertyId);
16149
- return `(${placeholders2.join(", ")})`;
16416
+ return `(${placeholders.join(", ")})`;
16150
16417
  }).join(", ");
16151
16418
  const projectIdPlaceholder = `$${nextIndex++}`;
16152
16419
  params.push(input.projectId);
@@ -16498,8 +16765,8 @@ class PgWorkflowInterventionStorage {
16498
16765
  const params = [input.projectId];
16499
16766
  let index = 2;
16500
16767
  if (input.statuses) {
16501
- const placeholders2 = input.statuses.map(() => `$${index++}`);
16502
- whereClauses.push(`status IN (${placeholders2.join(", ")})`);
16768
+ const placeholders = input.statuses.map(() => `$${index++}`);
16769
+ whereClauses.push(`status IN (${placeholders.join(", ")})`);
16503
16770
  params.push(...input.statuses);
16504
16771
  }
16505
16772
  if (input.workflowId) {
@@ -16507,8 +16774,8 @@ class PgWorkflowInterventionStorage {
16507
16774
  params.push(input.workflowId);
16508
16775
  }
16509
16776
  if (input.workflowIds) {
16510
- const placeholders2 = input.workflowIds.map(() => `$${index++}`);
16511
- whereClauses.push(`workflow_id IN (${placeholders2.join(", ")})`);
16777
+ const placeholders = input.workflowIds.map(() => `$${index++}`);
16778
+ whereClauses.push(`workflow_id IN (${placeholders.join(", ")})`);
16512
16779
  params.push(...input.workflowIds);
16513
16780
  }
16514
16781
  if (input.workflowRunId) {
@@ -16622,7 +16889,6 @@ function assertNonNegativeInteger(value, fieldName) {
16622
16889
  }
16623
16890
 
16624
16891
  // src/pg-workflow-run-storage.ts
16625
- import { normalizeRequesterGroupIds as normalizeRequesterGroupIds2 } from "@sixb/core/internal/auth";
16626
16892
  import { parseSixbFailure as parseSixbFailure6, serializeSixbFailure as serializeSixbFailure7 } from "@sixb/core/internal/errors";
16627
16893
  import {
16628
16894
  assertWorkflowAgentNodeRunExecution,
@@ -16664,7 +16930,6 @@ class PgWorkflowRunStorage {
16664
16930
  input,
16665
16931
  queued_at,
16666
16932
  started_at,
16667
- requester_group_ids,
16668
16933
  attempt
16669
16934
  ) VALUES (
16670
16935
  ${input.projectId},
@@ -16675,7 +16940,6 @@ class PgWorkflowRunStorage {
16675
16940
  ${serializeRecord2(input.input)}::text::jsonb,
16676
16941
  ${queuedAt},
16677
16942
  ${queuedAt},
16678
- ${JSON.stringify(normalizeRequesterGroupIds2(input.requesterGroupIds))}::text::jsonb,
16679
16943
  ${0}
16680
16944
  )
16681
16945
  RETURNING *
@@ -16861,8 +17125,8 @@ class PgWorkflowRunStorage {
16861
17125
  params.push(input.workflowId);
16862
17126
  }
16863
17127
  if (input.workflowIds) {
16864
- const placeholders2 = input.workflowIds.map(() => `$${index++}`);
16865
- whereClauses.push(`workflow_id IN (${placeholders2.join(", ")})`);
17128
+ const placeholders = input.workflowIds.map(() => `$${index++}`);
17129
+ whereClauses.push(`workflow_id IN (${placeholders.join(", ")})`);
16866
17130
  params.push(...input.workflowIds);
16867
17131
  }
16868
17132
  index = appendRunListFilters(whereClauses, params, index, input);
@@ -17268,7 +17532,6 @@ function rowToWorkflowRunRecord(row) {
17268
17532
  startedAt: new Date(row.started_at),
17269
17533
  finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
17270
17534
  error: row.error === null ? undefined : parseSixbFailure6(row.error, WORKFLOW_RUN_FAILURE_CODES),
17271
- requesterGroupIds: parseJson(row.requester_group_ids),
17272
17535
  attempt: Number(row.attempt),
17273
17536
  ...row.execution_token && row.execution_queue_lease_expires_at ? {
17274
17537
  execution: {
@@ -17843,8 +18106,8 @@ function resolveTimeoutMillis(value, label) {
17843
18106
  return Math.trunc(value);
17844
18107
  }
17845
18108
  export {
17846
- migratePostgresStorage,
17847
- PostgresStorage
18109
+ PostgresStorage,
18110
+ migratePostgresStorage
17848
18111
  };
17849
18112
 
17850
- //# debugId=66A7E573B9EBA9B464756E2164756E21
18113
+ //# debugId=672C519862E8368164756E2164756E21