@sixb/pg 0.1.14 → 0.1.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/index.d.ts +8 -1
  2. package/dist/index.d.ts.map +1 -1
  3. package/dist/index.js +742 -312
  4. package/dist/index.js.map +14 -14
  5. package/dist/latest-run-query.d.ts +2 -0
  6. package/dist/latest-run-query.d.ts.map +1 -1
  7. package/dist/migrations.d.ts.map +1 -1
  8. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  9. package/dist/ontology-storage/shared.d.ts +12 -9
  10. package/dist/ontology-storage/shared.d.ts.map +1 -1
  11. package/dist/ontology-storage/source-cleanup.d.ts +7 -1
  12. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
  13. package/dist/ontology-storage/source-roots.d.ts +8 -0
  14. package/dist/ontology-storage/source-roots.d.ts.map +1 -1
  15. package/dist/ontology-storage/sources.d.ts +8 -3
  16. package/dist/ontology-storage/sources.d.ts.map +1 -1
  17. package/dist/pg-client.d.ts +2 -0
  18. package/dist/pg-client.d.ts.map +1 -1
  19. package/dist/pg-projection-run-storage.d.ts +3 -1
  20. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  21. package/dist/storage-errors.d.ts +13 -3
  22. package/dist/storage-errors.d.ts.map +1 -1
  23. package/dist/transactions.d.ts +23 -1
  24. package/dist/transactions.d.ts.map +1 -1
  25. package/package.json +3 -3
  26. package/src/index.ts +8 -1
  27. package/src/latest-run-query.ts +10 -2
  28. package/src/migrations/051-projection-run-supersession.sql +43 -0
  29. package/src/migrations/052-compact-source-storage.sql +118 -0
  30. package/src/migrations.ts +26 -23
  31. package/src/ontology-storage/materialization-state.ts +20 -46
  32. package/src/ontology-storage/shared.ts +18 -12
  33. package/src/ontology-storage/source-cleanup.ts +66 -22
  34. package/src/ontology-storage/source-roots.ts +164 -102
  35. package/src/ontology-storage/sources.ts +150 -120
  36. package/src/pg-client.ts +132 -2
  37. package/src/pg-projection-run-storage.ts +56 -2
  38. package/src/storage-errors.ts +101 -15
  39. package/src/transactions.ts +113 -20
package/dist/index.js CHANGED
@@ -32,20 +32,34 @@ import {
32
32
  } from "@sixb/core/storage";
33
33
 
34
34
  // src/storage-errors.ts
35
+ import { createSixbError } from "@sixb/core/internal/errors";
35
36
  var UNIQUE_VIOLATION = "23505";
36
37
  var FOREIGN_KEY_VIOLATION = "23503";
37
38
  var RETRYABLE_TRANSACTION_CONFLICT_CODES = new Set(["40001", "40P01"]);
38
- var CONNECTION_ERROR_CODES = new Set([
39
- "CONNECT_TIMEOUT",
39
+ var CONNECTION_LOST_CODES = new Set([
40
40
  "CONNECTION_CLOSED",
41
41
  "CONNECTION_DESTROYED",
42
- "CONNECTION_ENDED",
43
- "ECONNREFUSED",
44
42
  "ECONNRESET",
45
- "ETIMEDOUT",
43
+ "EPIPE",
44
+ "ETIMEDOUT"
45
+ ]);
46
+ var CONNECT_FAILED_CODES = new Set([
47
+ "CONNECT_TIMEOUT",
48
+ "ECONNREFUSED",
46
49
  "ENOTFOUND",
47
- "EAI_AGAIN"
50
+ "EAI_AGAIN",
51
+ "EHOSTUNREACH",
52
+ "ENETUNREACH"
48
53
  ]);
54
+ var LOCK_TABLES = [
55
+ {
56
+ setting: "max_pred_locks_per_transaction",
57
+ table: "predicate lock table",
58
+ note: " A long serializable transaction also keeps the predicate locks of every serializable" + " transaction that commits while it is open."
59
+ },
60
+ { setting: "max_locks_per_transaction", table: "lock table", note: "" }
61
+ ];
62
+ var OUT_OF_MEMORY = "53200";
49
63
  function pgErrorCode(error) {
50
64
  if (error instanceof Error) {
51
65
  const code = error.code;
@@ -65,30 +79,101 @@ function isRetryableTransactionConflict(error) {
65
79
  const code = pgErrorCode(error);
66
80
  return code !== undefined && RETRYABLE_TRANSACTION_CONFLICT_CODES.has(code);
67
81
  }
82
+ function isConnectionLost(error) {
83
+ const code = pgErrorCode(error);
84
+ if (code !== undefined && CONNECTION_LOST_CODES.has(code)) {
85
+ return true;
86
+ }
87
+ const severity = error instanceof Error ? error.severity : null;
88
+ return severity === "FATAL" || severity === "PANIC";
89
+ }
90
+ function explainPgError(error) {
91
+ if (isStorageUnavailable(error)) {
92
+ return storageUnavailable(error);
93
+ }
94
+ if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
95
+ return error;
96
+ }
97
+ const hint = error.hint;
98
+ const lockTable = typeof hint === "string" ? LOCK_TABLES.find(({ setting }) => hint.includes(setting)) : undefined;
99
+ if (!lockTable) {
100
+ return error;
101
+ }
102
+ const { setting, table, note } = lockTable;
103
+ return new Error(`[SixbPg] PostgreSQL aborted the transaction because its shared ${table} is full ` + `(SQLSTATE 53200 "${error.message}"). The table holds ${setting} \xD7 (max_connections + ` + `max_prepared_transactions) entries shared by all sessions.${note} Raise ${setting} ` + "(applied at server restart) or shorten the transactions that overlap.", { cause: error });
104
+ }
105
+ function isStorageUnavailable(error) {
106
+ const code = pgErrorCode(error);
107
+ return isConnectionLost(error) || code !== undefined && CONNECT_FAILED_CODES.has(code);
108
+ }
109
+ function storageUnavailable(error) {
110
+ const code = pgErrorCode(error);
111
+ const reason = error instanceof Error ? error.message : String(error);
112
+ return createSixbError("storage.unavailable", `[SixbPg] PostgreSQL is unavailable: the connection was lost or could not be opened (${code ?? "no code"}: ${reason}).`, { cause: error, ...code === undefined ? {} : { details: { code } } });
113
+ }
68
114
 
69
115
  // src/transactions.ts
70
116
  import { createHash } from "crypto";
117
+ var BEGIN_STATEMENTS = {
118
+ unverifiedDefault: "BEGIN",
119
+ repeatableRead: "BEGIN ISOLATION LEVEL REPEATABLE READ",
120
+ serializable: "BEGIN ISOLATION LEVEL SERIALIZABLE"
121
+ };
122
+ var pgTransactionClients = new WeakSet;
71
123
  var activePgTransactions = new WeakMap;
72
124
  async function runPgTransaction(sql, run, options = {}) {
73
125
  if (!canStartPgTransaction(sql)) {
74
126
  return run(sql);
75
127
  }
76
128
  const isolation = options.isolation ?? "unverifiedDefault";
77
- const trackedRun = async (tx) => {
78
- activePgTransactions.set(tx, isolation);
79
- try {
80
- return await run(tx);
81
- } finally {
82
- activePgTransactions.delete(tx);
129
+ try {
130
+ return await withReservedPgConnection(sql, (tx) => {
131
+ pgTransactionClients.add(tx);
132
+ return runPgTransactionOn(tx, async () => {
133
+ activePgTransactions.set(tx, isolation);
134
+ try {
135
+ return await run(tx);
136
+ } finally {
137
+ activePgTransactions.delete(tx);
138
+ }
139
+ }, isolation);
140
+ });
141
+ } catch (error) {
142
+ throw explainPgError(error);
143
+ }
144
+ }
145
+ async function withReservedPgConnection(sql, run) {
146
+ const connection = await sql.reserve();
147
+ let usable = true;
148
+ try {
149
+ return await run(connection);
150
+ } catch (error) {
151
+ usable = !isConnectionLost(error);
152
+ throw error;
153
+ } finally {
154
+ if (usable) {
155
+ connection.release();
83
156
  }
84
- };
85
- if (isolation === "serializable") {
86
- return sql.begin("isolation level serializable", trackedRun);
87
157
  }
88
- if (isolation === "repeatableRead") {
89
- return sql.begin("isolation level repeatable read", trackedRun);
158
+ }
159
+ async function runPgTransactionOn(connection, run, isolation = "unverifiedDefault") {
160
+ await connection.unsafe(BEGIN_STATEMENTS[isolation]);
161
+ const result = await undoOnFailure(run, () => connection.unsafe("ROLLBACK"));
162
+ const commit = await connection.unsafe("COMMIT");
163
+ if (commit.command !== "COMMIT") {
164
+ throw new Error("[SixbPg] PostgreSQL rolled the transaction back instead of committing it: a statement in it" + " failed and its error was caught without being rethrown. A failed statement aborts the" + " whole transaction, so let its error propagate out of the transaction callback.");
165
+ }
166
+ return result;
167
+ }
168
+ async function undoOnFailure(run, undo) {
169
+ try {
170
+ return await run();
171
+ } catch (error) {
172
+ if (!isConnectionLost(error)) {
173
+ await undo();
174
+ }
175
+ throw error;
90
176
  }
91
- return sql.begin(trackedRun);
92
177
  }
93
178
  async function runPgRepeatableReadTransaction(sql, run) {
94
179
  if (canStartPgTransaction(sql)) {
@@ -121,7 +206,7 @@ function advisoryLockParts(key) {
121
206
  return [hash.readInt32BE(0), hash.readInt32BE(4)];
122
207
  }
123
208
  function canStartPgTransaction(sql) {
124
- return typeof sql.begin === "function";
209
+ return !pgTransactionClients.has(sql) && typeof sql.reserve === "function";
125
210
  }
126
211
 
127
212
  // src/agents/rows.ts
@@ -7379,6 +7464,173 @@ ALTER TABLE auth_group_memberships ADD CONSTRAINT auth_group_memberships_source_
7379
7464
  CHECK (source IN ('invitation', 'manual', 'agent', 'directory'));
7380
7465
  `;
7381
7466
 
7467
+ // src/migrations/051-projection-run-supersession.sql
7468
+ var _051_projection_run_supersession_default = `-- Projection runs gain the terminal status \`superseded\`, and a running run may carry the failure
7469
+ -- of its last attempt in \`error\` while the queue retries it.
7470
+
7471
+ -- The status checks were unnamed. Drop exactly the two that enumerate the terminal statuses, then
7472
+ -- replace them with named ones that admit \`superseded\`.
7473
+ DO $$
7474
+ DECLARE status_constraint RECORD;
7475
+ BEGIN
7476
+ FOR status_constraint IN
7477
+ SELECT conname
7478
+ FROM pg_constraint
7479
+ WHERE conrelid = 'projection_runs'::regclass
7480
+ AND contype = 'c'
7481
+ AND pg_get_constraintdef(oid) LIKE '%''cancelled''%'
7482
+ LOOP
7483
+ EXECUTE format('ALTER TABLE projection_runs DROP CONSTRAINT %I', status_constraint.conname);
7484
+ END LOOP;
7485
+ END $$;
7486
+
7487
+ ALTER TABLE projection_runs
7488
+ ADD CONSTRAINT projection_runs_status_check CHECK (
7489
+ status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled', 'superseded')
7490
+ ),
7491
+ ADD CONSTRAINT projection_runs_lifecycle_check CHECK (
7492
+ (status = 'queued' AND attempt = 0 AND started_at IS NULL AND finished_at IS NULL
7493
+ AND error IS NULL)
7494
+ OR (status = 'running' AND attempt >= 1 AND started_at IS NOT NULL AND finished_at IS NULL)
7495
+ OR (
7496
+ status IN ('succeeded', 'failed', 'cancelled', 'superseded')
7497
+ AND finished_at IS NOT NULL
7498
+ AND (
7499
+ (error->>'code' = 'queue.enqueue_failed'
7500
+ AND status = 'failed' AND attempt = 0 AND started_at IS NULL)
7501
+ OR (COALESCE(error->>'code', '') != 'queue.enqueue_failed'
7502
+ AND attempt >= 1 AND started_at IS NOT NULL)
7503
+ )
7504
+ )
7505
+ ),
7506
+ -- A run is superseded instead of materialized, so it carries no failure, and only replacement
7507
+ -- runs can be: telemetry appends readings rather than replace a source.
7508
+ ADD CONSTRAINT projection_runs_superseded_check CHECK (
7509
+ status != 'superseded' OR (error IS NULL AND materialization_protocol = 'replacement')
7510
+ );
7511
+ `;
7512
+
7513
+ // src/migrations/052-compact-source-storage.sql
7514
+ var _052_compact_source_storage_default = `-- Source versions get a compact surrogate key; roots and rows stop repeating the text triple.
7515
+ ALTER TABLE ontology_sources ADD COLUMN version_id BIGINT GENERATED ALWAYS AS IDENTITY;
7516
+ ALTER TABLE ontology_sources ADD CONSTRAINT ontology_sources_version_id_key UNIQUE (version_id);
7517
+
7518
+ ALTER TABLE ontology_source_rows RENAME TO legacy_ontology_source_rows;
7519
+ ALTER TABLE ontology_source_roots RENAME TO legacy_ontology_source_roots;
7520
+ -- Index names are schema-wide: free them for the new tables.
7521
+ DO $$
7522
+ DECLARE
7523
+ legacy record;
7524
+ BEGIN
7525
+ FOR legacy IN
7526
+ SELECT index_class.relname AS name
7527
+ FROM pg_index
7528
+ JOIN pg_class AS index_class ON index_class.oid = pg_index.indexrelid
7529
+ JOIN pg_class AS table_class ON table_class.oid = pg_index.indrelid
7530
+ WHERE table_class.relnamespace = current_schema()::regnamespace
7531
+ AND table_class.relname IN ('legacy_ontology_source_rows', 'legacy_ontology_source_roots')
7532
+ LOOP
7533
+ EXECUTE format('ALTER INDEX %I RENAME TO %I', legacy.name, left('legacy_' || legacy.name, 63));
7534
+ END LOOP;
7535
+ END $$;
7536
+
7537
+ -- A root is live while its version is published (active or superseded), it is not retired and
7538
+ -- it is not a deletion. Activation therefore writes only the roots it replaces, never its own.
7539
+ -- \`root_key\` is the canonical key; "C" orders it by its UTF-8 bytes.
7540
+ CREATE TABLE ontology_source_roots (
7541
+ id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
7542
+ version_id BIGINT NOT NULL REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
7543
+ project_id TEXT NOT NULL,
7544
+ root_key TEXT COLLATE "C" NOT NULL,
7545
+ staging_ordinal BIGINT NOT NULL CHECK (staging_ordinal >= 0),
7546
+ deleted BOOLEAN NOT NULL DEFAULT FALSE,
7547
+ retired_at TIMESTAMPTZ,
7548
+ CONSTRAINT ontology_source_roots_key UNIQUE (version_id, root_key),
7549
+ CONSTRAINT ontology_source_roots_ordinal UNIQUE (version_id, staging_ordinal)
7550
+ );
7551
+ CREATE INDEX idx_ontology_source_roots_live
7552
+ ON ontology_source_roots (project_id, root_key) WHERE retired_at IS NULL AND NOT deleted;
7553
+ CREATE INDEX idx_ontology_source_roots_retired
7554
+ ON ontology_source_roots (project_id, retired_at) WHERE retired_at IS NOT NULL;
7555
+
7556
+ -- A row's identity lives in its typed columns; \`payload\` is the assertion without kind and ref.
7557
+ CREATE TABLE ontology_source_rows (
7558
+ root_id BIGINT NOT NULL REFERENCES ontology_source_roots (id) ON DELETE RESTRICT,
7559
+ entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
7560
+ object_type_id TEXT,
7561
+ primary_id TEXT,
7562
+ source_type_id TEXT,
7563
+ source_primary_id TEXT,
7564
+ link_id TEXT,
7565
+ target_type_id TEXT,
7566
+ target_primary_id TEXT,
7567
+ payload JSONB,
7568
+ CHECK (
7569
+ (
7570
+ entity_kind = 'object'
7571
+ AND object_type_id IS NOT NULL AND primary_id IS NOT NULL
7572
+ AND source_type_id IS NULL AND source_primary_id IS NULL AND link_id IS NULL
7573
+ AND target_type_id IS NULL AND target_primary_id IS NULL
7574
+ )
7575
+ OR
7576
+ (
7577
+ entity_kind = 'link'
7578
+ AND object_type_id IS NULL AND primary_id IS NULL
7579
+ AND source_type_id IS NOT NULL AND source_primary_id IS NOT NULL AND link_id IS NOT NULL
7580
+ AND target_type_id IS NOT NULL AND target_primary_id IS NOT NULL
7581
+ )
7582
+ )
7583
+ );
7584
+ CREATE INDEX idx_ontology_source_rows_root ON ontology_source_rows (root_id);
7585
+ CREATE INDEX idx_ontology_source_rows_link_source
7586
+ ON ontology_source_rows (source_type_id, source_primary_id, link_id)
7587
+ WHERE entity_kind = 'link';
7588
+ CREATE INDEX idx_ontology_source_rows_link_target
7589
+ ON ontology_source_rows (target_type_id, target_primary_id)
7590
+ WHERE entity_kind = 'link';
7591
+
7592
+ -- A published root that was not active is retired, so liveness stays exactly what it was.
7593
+ INSERT INTO ontology_source_roots (
7594
+ version_id, project_id, root_key, staging_ordinal, deleted, retired_at
7595
+ )
7596
+ SELECT versions.version_id, roots.project_id,
7597
+ convert_from(decode(roots.root_sort_key, 'hex'), 'UTF8'), roots.staging_ordinal, roots.deleted,
7598
+ CASE
7599
+ WHEN roots.active THEN NULL
7600
+ WHEN roots.retired_at IS NOT NULL THEN roots.retired_at
7601
+ WHEN versions.status IN ('active', 'superseded') THEN versions.updated_at
7602
+ END
7603
+ FROM legacy_ontology_source_roots AS roots
7604
+ JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id);
7605
+
7606
+ INSERT INTO ontology_source_rows (
7607
+ root_id, entity_kind, object_type_id, primary_id,
7608
+ source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
7609
+ )
7610
+ SELECT roots.id, rows.entity_kind, rows.object_type_id, rows.primary_id,
7611
+ rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
7612
+ rows.target_primary_id, NULLIF(rows.assertion - 'kind' - 'ref', '{}'::jsonb)
7613
+ FROM legacy_ontology_source_rows AS rows
7614
+ JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id)
7615
+ JOIN ontology_source_roots AS roots
7616
+ ON roots.version_id = versions.version_id
7617
+ AND roots.root_key = convert_from(decode(rows.root_sort_key, 'hex'), 'UTF8');
7618
+
7619
+ DO $$
7620
+ BEGIN
7621
+ IF (SELECT count(*) FROM legacy_ontology_source_roots)
7622
+ <> (SELECT count(*) FROM ontology_source_roots)
7623
+ OR (SELECT count(*) FROM legacy_ontology_source_rows)
7624
+ <> (SELECT count(*) FROM ontology_source_rows) THEN
7625
+ RAISE EXCEPTION 'Source roots or rows were lost while compacting source storage.';
7626
+ END IF;
7627
+ END $$;
7628
+
7629
+ DROP TABLE legacy_ontology_source_rows;
7630
+ DROP TABLE legacy_ontology_source_roots;
7631
+ ANALYZE ontology_sources, ontology_source_roots, ontology_source_rows;
7632
+ `;
7633
+
7382
7634
  // src/migrations/037-execution-requester-groups.sql
7383
7635
  var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
7384
7636
 
@@ -7564,15 +7816,11 @@ function postgresMigrationSession(sql, schemaName) {
7564
7816
  async transaction(run) {
7565
7817
  const previous = active;
7566
7818
  active = sql;
7567
- await sql.unsafe("BEGIN");
7568
7819
  try {
7569
- await sql.unsafe(`SET LOCAL search_path TO ${schema}`);
7570
- const result = await run();
7571
- await sql.unsafe("COMMIT");
7572
- return result;
7573
- } catch (error) {
7574
- await sql.unsafe("ROLLBACK");
7575
- throw error;
7820
+ return await runPgTransactionOn(sql, async () => {
7821
+ await sql.unsafe(`SET LOCAL search_path TO ${schema}`);
7822
+ return run();
7823
+ });
7576
7824
  } finally {
7577
7825
  active = previous;
7578
7826
  }
@@ -7592,18 +7840,14 @@ async function readPostgresHistory(sql, schemaName, adapterId) {
7592
7840
  return rows.map(rowToMigrationRecord);
7593
7841
  }
7594
7842
  async function withPostgresMigrationLock(params, run) {
7595
- const sql = await params.sql.reserve();
7596
7843
  const [first, second] = advisoryLockParts2(`storage:migration:${params.schemaName}:${params.migrations.adapterId}`);
7597
- try {
7844
+ return withReservedPgConnection(params.sql, async (sql) => {
7845
+ const unlock = () => sql`SELECT pg_advisory_unlock(${first}, ${second})`;
7598
7846
  await sql`SELECT pg_advisory_lock(${first}, ${second})`;
7599
- try {
7600
- return await run(sql);
7601
- } finally {
7602
- await sql`SELECT pg_advisory_unlock(${first}, ${second})`;
7603
- }
7604
- } finally {
7605
- sql.release();
7606
- }
7847
+ const result = await undoOnFailure(() => run(sql), unlock);
7848
+ await unlock();
7849
+ return result;
7850
+ });
7607
7851
  }
7608
7852
  function advisoryLockParts2(key) {
7609
7853
  const hash = createHash2("sha256").update(key).digest();
@@ -7678,7 +7922,9 @@ var postgresStorageMigrations = defineMigrations({
7678
7922
  pgSql("047-ontology-commit-attribution", _047_ontology_commit_attribution_default),
7679
7923
  pgSql("048-file-upload-sessions", _048_file_upload_sessions_default),
7680
7924
  pgSql("049-native-sessions", _049_native_sessions_default),
7681
- pgSql("050-directory-group-memberships", _050_directory_group_memberships_default)
7925
+ pgSql("050-directory-group-memberships", _050_directory_group_memberships_default),
7926
+ pgSql("051-projection-run-supersession", _051_projection_run_supersession_default),
7927
+ pgSql("052-compact-source-storage", _052_compact_source_storage_default)
7682
7928
  ]
7683
7929
  });
7684
7930
 
@@ -9928,6 +10174,10 @@ function assertReconciliationPageLimit(limit) {
9928
10174
  // src/ontology-storage/shared.ts
9929
10175
  import { parseSixbFailure as parseSixbFailure2 } from "@sixb/core/internal/errors";
9930
10176
  import { MaterializationConflictError } from "@sixb/core/internal/materialization";
10177
+ import {
10178
+ sourceAssertionFromColumns,
10179
+ sourceEntityFromKey
10180
+ } from "@sixb/core/internal/ontology-storage-provider";
9931
10181
  import { ONTOLOGY_OUTBOX_FAILURE_CODES } from "@sixb/core/storage";
9932
10182
  import {
9933
10183
  assertNonblank,
@@ -9993,8 +10243,8 @@ function sourceRecord(row) {
9993
10243
  }
9994
10244
  function sourceAssertion(row) {
9995
10245
  return {
9996
- root: structuredClone(row.root),
9997
- assertion: structuredClone(row.assertion),
10246
+ root: sourceEntityFromKey(row.root_key),
10247
+ assertion: sourceAssertionFromColumns(row, row.payload),
9998
10248
  stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal")
9999
10249
  };
10000
10250
  }
@@ -10206,27 +10456,80 @@ import {
10206
10456
  MaterializationValidationError,
10207
10457
  projectionEntityKey
10208
10458
  } from "@sixb/core/internal/materialization";
10209
- import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider";
10459
+ import {
10460
+ sourceEntityColumns,
10461
+ sourceStageRoots
10462
+ } from "@sixb/core/internal/ontology-storage-provider";
10463
+ var PUBLISHED = "('active', 'superseded')";
10464
+ function sourceAssertionColumns(sql) {
10465
+ return sql.unsafe(`versions.source_id, versions.materialization_id, roots.root_key,
10466
+ roots.staging_ordinal, rows.entity_kind, rows.object_type_id, rows.primary_id,
10467
+ rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
10468
+ rows.target_primary_id, rows.payload`);
10469
+ }
10470
+ function liveSourceRowsJoin(sql, projectId) {
10471
+ return sql`
10472
+ JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
10473
+ AND roots.project_id = ${projectId} AND roots.retired_at IS NULL AND NOT roots.deleted
10474
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
10475
+ AND versions.status IN ${sql.unsafe(PUBLISHED)}
10476
+ `;
10477
+ }
10478
+ function candidateVersion(sql, input) {
10479
+ return sql`(SELECT version_id FROM ontology_sources WHERE project_id = ${input.projectId}
10480
+ AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId})`;
10481
+ }
10482
+ var ENTITY_MATCH = `rows.entity_kind = requested.entity_kind
10483
+ AND rows.object_type_id IS NOT DISTINCT FROM requested.object_type_id
10484
+ AND rows.primary_id IS NOT DISTINCT FROM requested.primary_id
10485
+ AND rows.source_type_id IS NOT DISTINCT FROM requested.source_type_id
10486
+ AND rows.source_primary_id IS NOT DISTINCT FROM requested.source_primary_id
10487
+ AND rows.link_id IS NOT DISTINCT FROM requested.link_id
10488
+ AND rows.target_type_id IS NOT DISTINCT FROM requested.target_type_id
10489
+ AND rows.target_primary_id IS NOT DISTINCT FROM requested.target_primary_id`;
10490
+ var REQUESTED_ENTITY = `entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
10491
+ source_type_id TEXT, source_primary_id TEXT, link_id TEXT, target_type_id TEXT,
10492
+ target_primary_id TEXT`;
10493
+ function requestedEntity(ref) {
10494
+ const columns = sourceEntityColumns(ref);
10495
+ const key = projectionEntityKey(ref);
10496
+ return {
10497
+ entity_kind: ref.kind,
10498
+ object_type_id: columns.objectTypeId,
10499
+ primary_id: columns.primaryId,
10500
+ source_type_id: columns.sourceTypeId,
10501
+ source_primary_id: columns.sourcePrimaryId,
10502
+ link_id: columns.linkId,
10503
+ target_type_id: columns.targetTypeId,
10504
+ target_primary_id: columns.targetPrimaryId,
10505
+ root_keys: ref.kind === "object" ? [key] : [key, projectionEntityKey({ kind: "object", ref: ref.ref.source })]
10506
+ };
10507
+ }
10210
10508
  function replacementSourceRows(sql, input) {
10509
+ const live = sql`roots.retired_at IS NULL AND NOT roots.deleted
10510
+ AND versions.source_id = ${input.sourceId} AND versions.status IN ${sql.unsafe(PUBLISHED)}`;
10211
10511
  const roots = input.incremental ? sql`
10212
- SELECT roots.* FROM ontology_source_roots AS changed
10512
+ SELECT live.id FROM ontology_source_roots AS changed
10213
10513
  CROSS JOIN LATERAL (
10214
- SELECT roots.* FROM ontology_source_roots AS roots
10215
- WHERE roots.project_id = changed.project_id AND roots.source_id = changed.source_id
10216
- AND roots.root_sort_key = changed.root_sort_key AND roots.active OFFSET 0
10217
- ) AS roots
10218
- WHERE changed.project_id = ${input.projectId} AND changed.source_id = ${input.sourceId}
10219
- AND changed.materialization_id = ${input.materializationId}
10220
- ` : sql`SELECT * FROM ontology_source_roots WHERE project_id = ${input.projectId} AND source_id = ${input.sourceId} AND active`;
10514
+ SELECT roots.id FROM ontology_source_roots AS roots
10515
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
10516
+ WHERE roots.project_id = ${input.projectId} AND roots.root_key = changed.root_key AND ${live}
10517
+ OFFSET 0
10518
+ ) AS live
10519
+ WHERE changed.version_id = ${candidateVersion(sql, input)}
10520
+ ` : sql`
10521
+ SELECT roots.id FROM ontology_sources AS versions
10522
+ JOIN ontology_source_roots AS roots ON roots.version_id = versions.version_id
10523
+ WHERE versions.project_id = ${input.projectId} AND ${live}
10524
+ `;
10221
10525
  return sql`
10222
- SELECT * FROM ontology_source_rows WHERE project_id = ${input.projectId}
10223
- AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId}
10526
+ SELECT rows.* FROM ontology_source_roots AS roots
10527
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
10528
+ WHERE roots.version_id = ${candidateVersion(sql, input)}
10224
10529
  UNION ALL
10225
10530
  SELECT rows.* FROM (${roots}) AS roots
10226
10531
  CROSS JOIN LATERAL (
10227
- SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.project_id = roots.project_id
10228
- AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
10229
- AND rows.root_sort_key = roots.root_sort_key OFFSET 0
10532
+ SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id OFFSET 0
10230
10533
  ) AS rows
10231
10534
  `;
10232
10535
  }
@@ -10238,35 +10541,28 @@ async function stageSourceRoots(sql, manifest, input) {
10238
10541
  throw new MaterializationValidationError("Root deletions require a source delta base.");
10239
10542
  }
10240
10543
  const values = jsonParameter2(sql, roots.map((root) => ({
10241
- root_sort_key: root.sortKey,
10242
- root_kind: root.root.kind,
10243
- root_key: JSON.parse(root.rootKey),
10244
- root: root.root,
10544
+ root_key: root.rootKey,
10245
10545
  staging_ordinal: root.stagingOrdinal,
10246
10546
  deleted: root.deleted
10247
10547
  })));
10248
10548
  const inserted = await sql`
10249
- INSERT INTO ontology_source_roots (
10250
- project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
10251
- ) SELECT ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
10252
- root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
10253
- FROM jsonb_to_recordset(${values}) AS input(
10254
- root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
10255
- ON CONFLICT DO NOTHING RETURNING root_sort_key
10549
+ INSERT INTO ontology_source_roots (version_id, project_id, root_key, staging_ordinal, deleted)
10550
+ SELECT ${manifest.version_id}, ${input.projectId}, root_key, staging_ordinal, deleted
10551
+ FROM jsonb_to_recordset(${values}) AS input(root_key TEXT, staging_ordinal BIGINT, deleted BOOLEAN)
10552
+ ON CONFLICT DO NOTHING RETURNING root_key
10256
10553
  `;
10257
10554
  if (inserted.length === roots.length)
10258
10555
  return;
10259
- const insertedKeys = new Set(inserted.map((row) => row.root_sort_key));
10260
- const skipped = roots.filter((root) => !insertedKeys.has(root.sortKey));
10556
+ const insertedKeys = new Set(inserted.map((row) => row.root_key));
10557
+ const skipped = roots.filter((root) => !insertedKeys.has(root.rootKey));
10261
10558
  const stored = await sql`
10262
- SELECT root_sort_key, staging_ordinal, deleted FROM ontology_source_roots
10263
- WHERE project_id = ${input.projectId} AND source_id = ${input.source.projectionId}
10264
- AND materialization_id = ${input.materializationId}
10265
- AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
10559
+ SELECT root_key, staging_ordinal, deleted FROM ontology_source_roots
10560
+ WHERE version_id = ${manifest.version_id}
10561
+ AND root_key = ANY(${sql.array(skipped.map((root) => root.rootKey))}::text[])
10266
10562
  `;
10267
- const existing = new Map(stored.map((root) => [root.root_sort_key, root]));
10563
+ const existing = new Map(stored.map((root) => [root.root_key, root]));
10268
10564
  if (skipped.some((root) => {
10269
- const previous = existing.get(root.sortKey);
10565
+ const previous = existing.get(root.rootKey);
10270
10566
  return !previous || Number(previous.staging_ordinal) !== root.stagingOrdinal || previous.deleted !== root.deleted;
10271
10567
  }))
10272
10568
  throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
@@ -10276,94 +10572,84 @@ async function activateSourceRoots(sql, projectId, candidate, activation) {
10276
10572
  throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
10277
10573
  if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0)
10278
10574
  return;
10279
- const affected = candidate.base_materialization_id === null ? sql`` : sql`AND root_sort_key IN (
10280
- SELECT root_sort_key FROM ontology_source_roots
10281
- WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
10282
- AND materialization_id = ${activation.materializationId}
10283
- )`;
10284
- await sql`
10285
- UPDATE ontology_source_roots AS roots SET active = FALSE, retired_at = ${activation.updatedAt}
10286
- WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId} AND active
10287
- ${affected}
10288
- `;
10575
+ const live = sql`roots.retired_at IS NULL AND NOT roots.deleted
10576
+ AND versions.version_id = roots.version_id AND versions.project_id = ${projectId}
10577
+ AND versions.source_id = ${activation.source.projectionId}
10578
+ AND versions.status IN ${sql.unsafe(PUBLISHED)}`;
10579
+ if (candidate.base_materialization_id === null) {
10580
+ await sql`
10581
+ UPDATE ontology_source_roots AS roots SET retired_at = ${activation.updatedAt}
10582
+ FROM ontology_sources AS versions WHERE ${live}
10583
+ `;
10584
+ } else {
10585
+ await sql`
10586
+ UPDATE ontology_source_roots AS roots SET retired_at = ${activation.updatedAt}
10587
+ FROM ontology_source_roots AS changed, ontology_sources AS versions
10588
+ WHERE changed.version_id = ${candidate.version_id} AND roots.project_id = ${projectId}
10589
+ AND roots.root_key = changed.root_key AND ${live}
10590
+ `;
10591
+ }
10289
10592
  await sql`
10290
- UPDATE ontology_source_roots SET active = NOT deleted,
10291
- retired_at = CASE WHEN deleted THEN ${activation.updatedAt}::timestamptz ELSE NULL END
10292
- WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
10293
- AND materialization_id = ${activation.materializationId}
10593
+ UPDATE ontology_source_roots SET retired_at = ${activation.updatedAt}
10594
+ WHERE version_id = ${candidate.version_id} AND deleted
10294
10595
  `;
10295
10596
  }
10296
10597
  async function assertSourceRootCoverage(sql, manifest) {
10297
10598
  const [invalid] = await sql`
10298
10599
  SELECT 1 FROM ontology_source_roots AS roots
10299
- WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
10300
- AND materialization_id = ${manifest.materialization_id}
10600
+ WHERE roots.version_id = ${manifest.version_id}
10301
10601
  AND roots.deleted = EXISTS (
10302
- SELECT 1 FROM ontology_source_rows AS rows
10303
- WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
10304
- AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
10305
- ) LIMIT 1
10602
+ SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id
10603
+ )
10604
+ LIMIT 1
10306
10605
  `;
10307
10606
  if (invalid)
10308
10607
  throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
10309
10608
  }
10310
10609
  function activeSourceRows(sql, projectId, refs) {
10311
- const keys = refs.flatMap((entity) => {
10312
- const entitySortKey = utf8SortKey(projectionEntityKey(entity));
10313
- const roots = entity.kind === "object" ? [entitySortKey] : [
10314
- entitySortKey,
10315
- utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
10316
- ];
10317
- return roots.map((rootSortKey) => ({
10318
- root_sort_key: rootSortKey,
10319
- entity_sort_key: entitySortKey
10320
- }));
10610
+ const requested = refs.flatMap((ref) => {
10611
+ const { root_keys, ...entity } = requestedEntity(ref);
10612
+ return root_keys.map((root_key) => ({ ...entity, root_key }));
10321
10613
  });
10322
10614
  return sql`
10323
- SELECT rows.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)}) AS requested(root_sort_key TEXT, entity_sort_key TEXT)
10615
+ SELECT ${sourceAssertionColumns(sql)}
10616
+ FROM jsonb_to_recordset(${jsonParameter2(sql, requested)})
10617
+ AS requested(root_key TEXT, ${sql.unsafe(REQUESTED_ENTITY)})
10324
10618
  JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
10325
- AND roots.active AND roots.root_sort_key = requested.root_sort_key
10326
- JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
10327
- AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
10328
- AND rows.root_sort_key = roots.root_sort_key
10619
+ AND roots.root_key = requested.root_key AND roots.retired_at IS NULL AND NOT roots.deleted
10620
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
10621
+ AND versions.status IN ${sql.unsafe(PUBLISHED)}
10622
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
10329
10623
  `;
10330
10624
  }
10331
10625
  function replacementAssertionRows(sql, input) {
10332
- const keys = input.refs.map((ref) => ({
10333
- entity_kind: ref.kind,
10334
- entity_key: JSON.parse(projectionEntityKey(ref)),
10335
- root_keys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
10336
- utf8SortKey(projectionEntityKey(ref)),
10337
- utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
10338
- ]
10339
- }));
10626
+ const version = candidateVersion(sql, input);
10340
10627
  const previous = !input.includePrevious ? sql`` : sql`UNION ALL
10341
- SELECT rows.* FROM (
10628
+ SELECT ${sourceAssertionColumns(sql)} FROM (
10342
10629
  SELECT roots.* FROM ontology_source_roots AS roots
10343
- WHERE roots.project_id = ${input.projectId} AND roots.source_id = ${input.sourceId}
10344
- AND roots.active AND roots.root_sort_key = ANY(requested.root_keys)
10630
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
10631
+ WHERE roots.project_id = ${input.projectId} AND roots.root_key = ANY(requested.root_keys)
10632
+ AND roots.retired_at IS NULL AND NOT roots.deleted
10633
+ AND versions.source_id = ${input.sourceId} AND versions.status IN ${sql.unsafe(PUBLISHED)}
10345
10634
  ${!input.incremental ? sql`` : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
10346
- WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
10347
- AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`}
10348
-
10635
+ WHERE changed.version_id = ${version} AND changed.root_key = roots.root_key)`}
10349
10636
  OFFSET 0
10350
10637
  ) AS roots
10638
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
10351
10639
  CROSS JOIN LATERAL (
10352
10640
  SELECT rows.* FROM ontology_source_rows AS rows
10353
- WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
10354
- AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
10355
- AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
10641
+ WHERE rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
10356
10642
  OFFSET 0
10357
10643
  ) AS rows
10358
10644
  `;
10359
10645
  return sql`
10360
- SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)})
10361
- AS requested(entity_kind TEXT, entity_key JSONB, root_keys TEXT[])
10646
+ SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, input.refs.map(requestedEntity))})
10647
+ AS requested(${sql.unsafe(REQUESTED_ENTITY)}, root_keys TEXT[])
10362
10648
  CROSS JOIN LATERAL (
10363
- SELECT rows.* FROM ontology_source_rows AS rows
10364
- WHERE rows.project_id = ${input.projectId} AND rows.source_id = ${input.sourceId}
10365
- AND rows.materialization_id = ${input.materializationId}
10366
- AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
10649
+ SELECT ${sourceAssertionColumns(sql)} FROM ontology_source_roots AS roots
10650
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
10651
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
10652
+ WHERE roots.version_id = ${version} AND roots.root_key = ANY(requested.root_keys)
10367
10653
  ${previous}
10368
10654
  OFFSET 0
10369
10655
  ) AS selected
@@ -10582,20 +10868,14 @@ class PgMaterializationStateReader {
10582
10868
  scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
10583
10869
  )
10584
10870
  )
10585
- SELECT rows.*
10871
+ SELECT ${sourceAssertionColumns(this.sql)}
10586
10872
  FROM ontology_source_rows AS rows
10587
10873
  JOIN requested
10588
10874
  ON requested.source_type_id = rows.source_type_id
10589
10875
  AND requested.source_id = rows.source_primary_id
10590
10876
  AND requested.link_id = rows.link_id
10591
- JOIN ontology_source_roots AS sources
10592
- ON sources.project_id = rows.project_id
10593
- AND sources.source_id = rows.source_id
10594
- AND sources.materialization_id = rows.materialization_id
10595
- AND sources.root_sort_key = rows.root_sort_key
10596
- WHERE rows.project_id = ${this.projectId}
10597
- AND rows.entity_kind = 'link'
10598
- AND sources.active
10877
+ ${liveSourceRowsJoin(this.sql, this.projectId)}
10878
+ WHERE rows.entity_kind = 'link'
10599
10879
  `,
10600
10880
  this.sql`
10601
10881
  WITH requested AS (
@@ -10737,32 +11017,20 @@ class PgMaterializationStateReader {
10737
11017
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10738
11018
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10739
11019
  FROM ontology_source_rows AS rows
10740
- JOIN ontology_source_roots AS sources
10741
- ON sources.project_id = rows.project_id
10742
- AND sources.source_id = rows.source_id
10743
- AND sources.materialization_id = rows.materialization_id
10744
- AND sources.root_sort_key = rows.root_sort_key
11020
+ ${liveSourceRowsJoin(this.sql, this.projectId)}
10745
11021
  JOIN requested
10746
11022
  ON requested.object_type_id = rows.source_type_id
10747
11023
  AND requested.primary_id = rows.source_primary_id
10748
- WHERE rows.project_id = ${this.projectId}
10749
- AND rows.entity_kind = 'link'
10750
- AND sources.active
11024
+ WHERE rows.entity_kind = 'link'
10751
11025
  UNION
10752
11026
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10753
11027
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10754
11028
  FROM ontology_source_rows AS rows
10755
- JOIN ontology_source_roots AS sources
10756
- ON sources.project_id = rows.project_id
10757
- AND sources.source_id = rows.source_id
10758
- AND sources.materialization_id = rows.materialization_id
10759
- AND sources.root_sort_key = rows.root_sort_key
11029
+ ${liveSourceRowsJoin(this.sql, this.projectId)}
10760
11030
  JOIN requested
10761
11031
  ON requested.object_type_id = rows.target_type_id
10762
11032
  AND requested.primary_id = rows.target_primary_id
10763
- WHERE rows.project_id = ${this.projectId}
10764
- AND rows.entity_kind = 'link'
10765
- AND sources.active
11033
+ WHERE rows.entity_kind = 'link'
10766
11034
  ), selected AS (
10767
11035
  SELECT DISTINCT links.*,
10768
11036
  ${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
@@ -10996,32 +11264,20 @@ class PgMaterializationStateReader {
10996
11264
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
10997
11265
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
10998
11266
  FROM ontology_source_rows AS rows
10999
- JOIN ontology_source_roots AS sources
11000
- ON sources.project_id = rows.project_id
11001
- AND sources.source_id = rows.source_id
11002
- AND sources.materialization_id = rows.materialization_id
11003
- AND sources.root_sort_key = rows.root_sort_key
11267
+ ${liveSourceRowsJoin(this.sql, this.projectId)}
11004
11268
  JOIN incident_objects
11005
11269
  ON incident_objects.object_type_id = rows.source_type_id
11006
11270
  AND incident_objects.primary_id = rows.source_primary_id
11007
- WHERE rows.project_id = ${this.projectId}
11008
- AND rows.entity_kind = 'link'
11009
- AND sources.active
11271
+ WHERE rows.entity_kind = 'link'
11010
11272
  UNION
11011
11273
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
11012
11274
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
11013
11275
  FROM ontology_source_rows AS rows
11014
- JOIN ontology_source_roots AS sources
11015
- ON sources.project_id = rows.project_id
11016
- AND sources.source_id = rows.source_id
11017
- AND sources.materialization_id = rows.materialization_id
11018
- AND sources.root_sort_key = rows.root_sort_key
11276
+ ${liveSourceRowsJoin(this.sql, this.projectId)}
11019
11277
  JOIN incident_objects
11020
11278
  ON incident_objects.object_type_id = rows.target_type_id
11021
11279
  AND incident_objects.primary_id = rows.target_primary_id
11022
- WHERE rows.project_id = ${this.projectId}
11023
- AND rows.entity_kind = 'link'
11024
- AND sources.active
11280
+ WHERE rows.entity_kind = 'link'
11025
11281
  ), diff_links AS (
11026
11282
  SELECT * FROM replacement_links
11027
11283
  UNION SELECT * FROM incident_links
@@ -11163,9 +11419,7 @@ function storedSource(row) {
11163
11419
  return {
11164
11420
  source: { projectionId: row.source_id },
11165
11421
  materializationId: row.materialization_id,
11166
- root: structuredClone(row.root),
11167
- assertion: structuredClone(row.assertion),
11168
- stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal")
11422
+ ...sourceAssertion(row)
11169
11423
  };
11170
11424
  }
11171
11425
  function effectiveObjectSnapshot(row) {
@@ -12863,26 +13117,70 @@ import {
12863
13117
  sourceEntityColumns as entityColumns,
12864
13118
  isExactStagingManifest,
12865
13119
  reconcileSourceStageRows,
13120
+ sourceAssertionPayload,
12866
13121
  sourceConflict,
12867
13122
  sourceMaterializationIdentity,
12868
13123
  sourceStageRow,
12869
- sourceStageRows,
12870
- utf8SortKey as utf8SortKey2
13124
+ sourceStageRows
12871
13125
  } from "@sixb/core/internal/ontology-storage-provider";
12872
13126
 
12873
13127
  // src/ontology-storage/source-cleanup.ts
13128
+ async function purgeAbandonedSourceVersions(sql, input) {
13129
+ let remaining = input.limit;
13130
+ let rowsDeleted = 0;
13131
+ let materializationsDeleted = 0;
13132
+ while (remaining > 0) {
13133
+ const [candidate] = await sql`
13134
+ SELECT version_id FROM ontology_sources
13135
+ WHERE project_id = ${input.projectId} AND status = 'abandoned'
13136
+ ORDER BY terminal_at, source_id, materialization_id
13137
+ LIMIT 1 FOR UPDATE SKIP LOCKED
13138
+ `;
13139
+ if (!candidate)
13140
+ break;
13141
+ const rows = await sql`
13142
+ DELETE FROM ontology_source_rows WHERE ctid IN (
13143
+ SELECT rows.ctid FROM ontology_source_roots AS roots
13144
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
13145
+ WHERE roots.version_id = ${candidate.version_id} LIMIT ${remaining}
13146
+ )
13147
+ `;
13148
+ rowsDeleted += rows.count;
13149
+ remaining -= rows.count;
13150
+ if (remaining === 0)
13151
+ break;
13152
+ const roots = await sql`
13153
+ DELETE FROM ontology_source_roots WHERE ctid IN (
13154
+ SELECT ctid FROM ontology_source_roots
13155
+ WHERE version_id = ${candidate.version_id} LIMIT ${remaining}
13156
+ )
13157
+ `;
13158
+ rowsDeleted += roots.count;
13159
+ remaining -= roots.count;
13160
+ if (remaining === 0)
13161
+ break;
13162
+ const manifest = await sql`
13163
+ DELETE FROM ontology_sources
13164
+ WHERE version_id = ${candidate.version_id} AND status = 'abandoned'
13165
+ `;
13166
+ if (manifest.count === 0)
13167
+ break;
13168
+ materializationsDeleted += manifest.count;
13169
+ remaining -= manifest.count;
13170
+ }
13171
+ return { rowsDeleted, materializationsDeleted };
13172
+ }
12874
13173
  async function cleanupSourceVersions(sql, input) {
12875
13174
  let remaining = input.limit;
13175
+ const retired = sql`roots.project_id = ${input.projectId}
13176
+ AND roots.retired_at < ${input.terminalBefore} AND versions.status = 'superseded'`;
12876
13177
  const removedRows = await sql`
12877
13178
  WITH selected AS (
12878
13179
  SELECT rows.ctid FROM ontology_source_roots AS roots
12879
- JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
12880
- JOIN ontology_source_rows AS rows
12881
- ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
12882
- AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
12883
- WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
12884
- AND sources.status IN ('superseded', 'abandoned')
12885
- ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
13180
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
13181
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
13182
+ WHERE ${retired}
13183
+ ORDER BY roots.retired_at, roots.id
12886
13184
  LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
12887
13185
  ) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
12888
13186
  `;
@@ -12892,13 +13190,10 @@ async function cleanupSourceVersions(sql, input) {
12892
13190
  const removed = await sql`
12893
13191
  WITH selected AS (
12894
13192
  SELECT roots.ctid FROM ontology_source_roots AS roots
12895
- JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
12896
- WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
12897
- AND sources.status IN ('superseded', 'abandoned')
12898
- AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
12899
- WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
12900
- AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
12901
- ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
13193
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
13194
+ WHERE ${retired}
13195
+ AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
13196
+ ORDER BY roots.retired_at, roots.id
12902
13197
  LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
12903
13198
  ) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
12904
13199
  `;
@@ -12907,16 +13202,13 @@ async function cleanupSourceVersions(sql, input) {
12907
13202
  }
12908
13203
  const removedManifests = remaining === 0 ? [] : await sql`
12909
13204
  WITH selected AS (
12910
- SELECT sources.ctid FROM ontology_sources AS sources
12911
- WHERE project_id = ${input.projectId} AND status IN ('superseded', 'abandoned') AND terminal_at < ${input.terminalBefore}
13205
+ SELECT versions.ctid FROM ontology_sources AS versions
13206
+ WHERE project_id = ${input.projectId} AND status = 'superseded'
13207
+ AND terminal_at < ${input.terminalBefore}
12912
13208
  AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
12913
- WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
12914
- AND roots.materialization_id = sources.materialization_id)
12915
- AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
12916
- WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
12917
- AND rows.materialization_id = sources.materialization_id)
13209
+ WHERE roots.version_id = versions.version_id)
12918
13210
  ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
12919
- ) DELETE FROM ontology_sources AS sources USING selected WHERE sources.ctid = selected.ctid RETURNING 1
13211
+ ) DELETE FROM ontology_sources AS versions USING selected WHERE versions.ctid = selected.ctid RETURNING 1
12920
13212
  `;
12921
13213
  return { rowsDeleted, materializationsDeleted: removedManifests.length };
12922
13214
  }
@@ -12943,7 +13235,7 @@ class PgOntologySourceStorage {
12943
13235
  }
12944
13236
  const candidate = await this.getRunCandidate(sql, input);
12945
13237
  if (candidate) {
12946
- throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; reclaim it before beginning another.`);
13238
+ throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`);
12947
13239
  }
12948
13240
  const rows = await sql`
12949
13241
  INSERT INTO ontology_sources (
@@ -12972,7 +13264,7 @@ class PgOntologySourceStorage {
12972
13264
  return sourceRecord(racedManifest);
12973
13265
  }
12974
13266
  if (await this.getRunCandidate(sql, input)) {
12975
- throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; reclaim it before beginning another.`);
13267
+ throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`);
12976
13268
  }
12977
13269
  throw sourceConflict(`Source materialization '${input.materializationId}' already exists with different identity or state.`);
12978
13270
  });
@@ -12987,10 +13279,10 @@ class PgOntologySourceStorage {
12987
13279
  throw sourceConflict(`Source materialization '${input.materializationId}' is '${manifest.status}' and cannot accept rows.`);
12988
13280
  }
12989
13281
  const rows = sourceStageRows(manifest.projection_kind, input.rows);
12990
- const existing = await this.findStageRows(sql, input, rows);
13282
+ const existing = await this.findStageRows(sql, manifest, rows);
12991
13283
  const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
12992
13284
  await stageSourceRoots(sql, manifest, input);
12993
- await this.insertStageRows(sql, input, pending);
13285
+ await this.insertStageRows(sql, manifest, pending);
12994
13286
  return { inserted: pending.length, unchanged };
12995
13287
  });
12996
13288
  }
@@ -13049,13 +13341,43 @@ class PgOntologySourceStorage {
13049
13341
  return row ? sourceRecord(row) : null;
13050
13342
  });
13051
13343
  }
13344
+ async adopt(input) {
13345
+ return this.runRootOperation(async (sql) => {
13346
+ assertProjectAndSource(input);
13347
+ assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken);
13348
+ assertTimestamp(input.adoptedAt, "Source adoptedAt", true);
13349
+ await this.assertExecution(sql, input);
13350
+ const candidate = await this.lockRunCandidate(sql, input);
13351
+ if (!candidate)
13352
+ return null;
13353
+ const [adopted] = await sql`
13354
+ UPDATE ontology_sources
13355
+ SET execution_token = ${input.execution.executionToken},
13356
+ updated_at = GREATEST(updated_at, ${input.adoptedAt}::timestamptz)
13357
+ WHERE project_id = ${candidate.project_id} AND source_id = ${candidate.source_id}
13358
+ AND materialization_id = ${candidate.materialization_id}
13359
+ AND status IN ('staging', 'ready')
13360
+ RETURNING *
13361
+ `;
13362
+ if (!adopted) {
13363
+ throw sourceConflict(`Source materialization '${candidate.materialization_id}' changed.`);
13364
+ }
13365
+ return {
13366
+ record: sourceRecord(adopted),
13367
+ resumeStagingOrdinal: adopted.status === "ready" ? Number(adopted.root_count) : await this.lastStagedOrdinal(sql, adopted)
13368
+ };
13369
+ });
13370
+ }
13052
13371
  async abandon(input) {
13053
13372
  return this.runRootOperation(async (sql) => {
13054
13373
  assertProjectAndSource(input);
13055
13374
  assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken);
13056
13375
  assertTimestamp(input.abandonedAt, "Source abandonedAt", true);
13057
13376
  await this.assertExecution(sql, input);
13058
- return input.kind === "candidate" ? this.abandonCandidate(sql, input) : this.abandonForReclaim(sql, input);
13377
+ if (input.kind === "candidate")
13378
+ return this.abandonCandidate(sql, input);
13379
+ const candidate = await this.lockRunCandidate(sql, input);
13380
+ return candidate ? this.transitionToAbandoned(sql, candidate, input.abandonedAt) : null;
13059
13381
  });
13060
13382
  }
13061
13383
  async cleanupTerminal(input) {
@@ -13066,6 +13388,13 @@ class PgOntologySourceStorage {
13066
13388
  return cleanupSourceVersions(sql, input);
13067
13389
  });
13068
13390
  }
13391
+ async purgeAbandoned(input) {
13392
+ return this.runRootOperation(async (sql) => {
13393
+ assertNonblank(input.projectId, "Abandoned source purge project id");
13394
+ assertPositiveInteger(input.limit, "Abandoned source purge limit");
13395
+ return purgeAbandonedSourceVersions(sql, input);
13396
+ });
13397
+ }
13069
13398
  async summarizeTerminal(input) {
13070
13399
  return this.runRootOperation(async (sql) => {
13071
13400
  assertNonblank(input.projectId, "Terminal source summary project id");
@@ -13092,7 +13421,7 @@ class PgOntologySourceStorage {
13092
13421
  assertSourceCandidateOwner(sourceRecord(manifest), input.execution);
13093
13422
  return this.transitionToAbandoned(sql, manifest, input.abandonedAt);
13094
13423
  }
13095
- async abandonForReclaim(sql, input) {
13424
+ async lockRunCandidate(sql, input) {
13096
13425
  const candidates = await sql`
13097
13426
  SELECT * FROM ontology_sources
13098
13427
  WHERE project_id = ${input.projectId}
@@ -13101,13 +13430,17 @@ class PgOntologySourceStorage {
13101
13430
  AND status IN ('staging', 'ready')
13102
13431
  FOR UPDATE
13103
13432
  `;
13104
- if (candidates.some((candidate) => candidate.execution_token === input.execution.executionToken)) {
13105
- throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`);
13106
- }
13107
13433
  if (candidates.length > 1) {
13108
13434
  throw sourceConflict(`Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`);
13109
13435
  }
13110
- return candidates[0] ? this.transitionToAbandoned(sql, candidates[0], input.abandonedAt) : null;
13436
+ return candidates[0] ?? null;
13437
+ }
13438
+ async lastStagedOrdinal(sql, manifest) {
13439
+ const [last] = await sql`
13440
+ SELECT MAX(staging_ordinal) AS ordinal FROM ontology_source_roots
13441
+ WHERE version_id = ${manifest.version_id}
13442
+ `;
13443
+ return last?.ordinal == null ? 0 : Number(last.ordinal);
13111
13444
  }
13112
13445
  async transitionToAbandoned(sql, manifest, abandonedAt) {
13113
13446
  if (abandonedAt < toIsoString2(manifest.created_at) || manifest.ready_at !== null && abandonedAt < toIsoString2(manifest.ready_at)) {
@@ -13126,22 +13459,18 @@ class PgOntologySourceStorage {
13126
13459
  if (!rows[0]) {
13127
13460
  throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
13128
13461
  }
13129
- await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
13130
- WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
13131
- AND materialization_id = ${manifest.materialization_id} AND NOT active`;
13132
13462
  return sourceRecord(rows[0]);
13133
13463
  }
13134
13464
  async assertReadyState(sql, manifest, rootCount, assertionCount) {
13465
+ const version = manifest.version_id;
13135
13466
  const [counts] = await sql`
13136
- SELECT (SELECT COUNT(*) FROM ontology_source_rows
13137
- WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
13138
- AND materialization_id = ${manifest.materialization_id}) AS assertions, COUNT(*) AS roots,
13139
- COUNT(DISTINCT staging_ordinal) AS ordinals,
13467
+ SELECT (SELECT COUNT(*) FROM ontology_source_roots AS roots
13468
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
13469
+ WHERE roots.version_id = ${version}) AS assertions,
13470
+ COUNT(*) AS roots, COUNT(DISTINCT staging_ordinal) AS ordinals,
13140
13471
  MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
13141
13472
  FROM ontology_source_roots
13142
- WHERE project_id = ${manifest.project_id}
13143
- AND source_id = ${manifest.source_id}
13144
- AND materialization_id = ${manifest.materialization_id}
13473
+ WHERE version_id = ${version}
13145
13474
  `;
13146
13475
  const assertions = Number(counts?.assertions ?? 0);
13147
13476
  const roots = Number(counts?.roots ?? 0);
@@ -13153,122 +13482,101 @@ class PgOntologySourceStorage {
13153
13482
  }
13154
13483
  await assertSourceRootCoverage(sql, manifest);
13155
13484
  const [invalid] = manifest.projection_kind === "link" ? await sql`
13156
- SELECT root_key
13157
- FROM ontology_source_rows
13158
- WHERE project_id = ${manifest.project_id}
13159
- AND source_id = ${manifest.source_id}
13160
- AND materialization_id = ${manifest.materialization_id}
13161
- GROUP BY root_key
13485
+ SELECT roots.root_key
13486
+ FROM ontology_source_roots AS roots
13487
+ CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
13488
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
13489
+ WHERE roots.version_id = ${version}
13490
+ GROUP BY roots.id, roots.root_key
13162
13491
  HAVING COUNT(*) <> 1
13163
- OR BOOL_OR(root_kind <> 'link')
13164
- OR BOOL_OR(entity_kind <> 'link')
13165
- OR BOOL_OR(root_key <> entity_key)
13492
+ OR BOOL_OR(root.parts->>0 <> 'link')
13493
+ OR BOOL_OR(rows.entity_kind <> 'link')
13494
+ OR BOOL_OR(rows.source_type_id <> root.parts->>1
13495
+ OR rows.source_primary_id <> root.parts->>2 OR rows.link_id <> root.parts->>3
13496
+ OR rows.target_type_id <> root.parts->>4 OR rows.target_primary_id <> root.parts->>5)
13166
13497
  LIMIT 1
13167
13498
  ` : await sql`
13168
- SELECT root_key
13169
- FROM ontology_source_rows
13170
- WHERE project_id = ${manifest.project_id}
13171
- AND source_id = ${manifest.source_id}
13172
- AND materialization_id = ${manifest.materialization_id}
13173
- GROUP BY root_key
13174
- HAVING BOOL_OR(root_kind <> 'object')
13499
+ SELECT roots.root_key
13500
+ FROM ontology_source_roots AS roots
13501
+ CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
13502
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
13503
+ WHERE roots.version_id = ${version}
13504
+ GROUP BY roots.id, roots.root_key
13505
+ HAVING BOOL_OR(root.parts->>0 <> 'object')
13175
13506
  OR COUNT(*) FILTER (
13176
- WHERE entity_kind = 'object' AND entity_key = root_key
13507
+ WHERE rows.entity_kind = 'object' AND rows.object_type_id = root.parts->>1
13508
+ AND rows.primary_id = root.parts->>2
13177
13509
  ) <> 1
13178
13510
  OR COUNT(*) FILTER (
13179
- WHERE (entity_kind = 'object' AND entity_key <> root_key)
13180
- OR (entity_kind = 'link' AND (
13181
- source_type_id <> root_object_type_id
13182
- OR source_primary_id <> root_primary_id
13183
- ))
13511
+ WHERE (rows.entity_kind = 'object' AND (rows.object_type_id <> root.parts->>1
13512
+ OR rows.primary_id <> root.parts->>2))
13513
+ OR (rows.entity_kind = 'link' AND (rows.source_type_id <> root.parts->>1
13514
+ OR rows.source_primary_id <> root.parts->>2))
13184
13515
  ) <> 0
13516
+ OR COUNT(*) FILTER (WHERE rows.entity_kind = 'link') <> COUNT(DISTINCT (
13517
+ rows.link_id, rows.target_type_id, rows.target_primary_id
13518
+ )) FILTER (WHERE rows.entity_kind = 'link')
13185
13519
  LIMIT 1
13186
13520
  `;
13187
13521
  if (invalid) {
13188
13522
  throw new MaterializationValidationError4(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
13189
13523
  }
13190
13524
  }
13191
- async findStageRows(sql, input, rows) {
13525
+ async findStageRows(sql, manifest, rows) {
13192
13526
  if (rows.length === 0)
13193
13527
  return [];
13194
- const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.rootKey)))];
13528
+ const rootKeys = [...new Set(rows.map((row) => row.rootKey))];
13195
13529
  const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
13196
- const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
13197
13530
  const existing = await sql`
13198
- SELECT * FROM ontology_source_rows
13199
- WHERE project_id = ${input.projectId}
13200
- AND source_id = ${input.source.projectionId}
13201
- AND materialization_id = ${input.materializationId}
13531
+ SELECT ${sourceAssertionColumns(sql)}
13532
+ FROM ontology_source_roots AS roots
13533
+ JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
13534
+ JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
13535
+ WHERE roots.version_id = ${manifest.version_id}
13202
13536
  AND (
13203
- root_sort_key = ANY(${sql.array(rootSortKeys)}::text[])
13204
- OR staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
13205
- OR entity_sort_key = ANY(${sql.array(entitySortKeys)}::text[])
13537
+ roots.root_key = ANY(${sql.array(rootKeys)}::text[])
13538
+ OR roots.staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
13206
13539
  )
13207
13540
  `;
13208
13541
  return existing.map((row) => sourceStageRow(sourceAssertion(row)));
13209
13542
  }
13210
- async insertStageRows(sql, input, rows) {
13543
+ async insertStageRows(sql, manifest, rows) {
13211
13544
  if (rows.length === 0)
13212
13545
  return;
13213
- const payload = rows.map(({ entityKey, rootKey, row }) => {
13546
+ const payload = rows.map(({ rootKey, row }) => {
13214
13547
  const entity = entityColumns(row.assertion);
13215
- const root = entityColumns(row.root);
13216
13548
  return {
13217
- entityKind: row.assertion.kind,
13218
- entityKey: JSON.parse(entityKey),
13219
- entitySortKey: utf8SortKey2(entityKey),
13220
- rootKind: row.root.kind,
13221
- rootKey: JSON.parse(rootKey),
13222
- rootSortKey: utf8SortKey2(rootKey),
13223
- stagingOrdinal: row.stagingOrdinal,
13224
- root: row.root,
13225
- assertion: row.assertion,
13226
- objectTypeId: entity.objectTypeId,
13227
- primaryId: entity.primaryId,
13228
- sourceTypeId: entity.sourceTypeId,
13229
- sourcePrimaryId: entity.sourcePrimaryId,
13230
- linkId: entity.linkId,
13231
- targetTypeId: entity.targetTypeId,
13232
- targetPrimaryId: entity.targetPrimaryId,
13233
- rootObjectTypeId: root.objectTypeId,
13234
- rootPrimaryId: root.primaryId,
13235
- rootSourceTypeId: root.sourceTypeId,
13236
- rootSourcePrimaryId: root.sourcePrimaryId,
13237
- rootLinkId: root.linkId,
13238
- rootTargetTypeId: root.targetTypeId,
13239
- rootTargetPrimaryId: root.targetPrimaryId
13549
+ root_key: rootKey,
13550
+ entity_kind: row.assertion.kind,
13551
+ object_type_id: entity.objectTypeId,
13552
+ primary_id: entity.primaryId,
13553
+ source_type_id: entity.sourceTypeId,
13554
+ source_primary_id: entity.sourcePrimaryId,
13555
+ link_id: entity.linkId,
13556
+ target_type_id: entity.targetTypeId,
13557
+ target_primary_id: entity.targetPrimaryId,
13558
+ payload: sourceAssertionPayload(row.assertion)
13240
13559
  };
13241
13560
  });
13242
- await sql`
13243
- WITH staged AS (
13244
- SELECT value
13245
- FROM jsonb_array_elements(${jsonParameter2(sql, payload)}::jsonb)
13246
- )
13561
+ const inserted = await sql`
13247
13562
  INSERT INTO ontology_source_rows (
13248
- project_id, source_id, materialization_id,
13249
- entity_kind, entity_key, entity_sort_key,
13250
- root_kind, root_key, root_sort_key, staging_ordinal,
13251
- root, assertion,
13252
- object_type_id, primary_id,
13253
- source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
13254
- root_object_type_id, root_primary_id,
13255
- root_source_type_id, root_source_primary_id, root_link_id,
13256
- root_target_type_id, root_target_primary_id
13563
+ root_id, entity_kind, object_type_id, primary_id,
13564
+ source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
13257
13565
  )
13258
- SELECT
13259
- ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
13260
- value->>'entityKind', value->'entityKey', value->>'entitySortKey',
13261
- value->>'rootKind', value->'rootKey', value->>'rootSortKey',
13262
- (value->>'stagingOrdinal')::bigint,
13263
- value->'root', value->'assertion',
13264
- value->>'objectTypeId', value->>'primaryId',
13265
- value->>'sourceTypeId', value->>'sourcePrimaryId', value->>'linkId',
13266
- value->>'targetTypeId', value->>'targetPrimaryId',
13267
- value->>'rootObjectTypeId', value->>'rootPrimaryId',
13268
- value->>'rootSourceTypeId', value->>'rootSourcePrimaryId', value->>'rootLinkId',
13269
- value->>'rootTargetTypeId', value->>'rootTargetPrimaryId'
13270
- FROM staged
13566
+ SELECT roots.id, staged.entity_kind, staged.object_type_id, staged.primary_id,
13567
+ staged.source_type_id, staged.source_primary_id, staged.link_id,
13568
+ staged.target_type_id, staged.target_primary_id, staged.payload
13569
+ FROM jsonb_to_recordset(${jsonParameter2(sql, payload)}) AS staged(
13570
+ root_key TEXT, entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
13571
+ source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
13572
+ target_type_id TEXT, target_primary_id TEXT, payload JSONB
13573
+ )
13574
+ JOIN ontology_source_roots AS roots
13575
+ ON roots.version_id = ${manifest.version_id} AND roots.root_key = staged.root_key
13271
13576
  `;
13577
+ if (inserted.count !== rows.length) {
13578
+ throw sourceConflict("Staged source rows do not all belong to a staged root.");
13579
+ }
13272
13580
  }
13273
13581
  assertExecution(sql, input, identity) {
13274
13582
  return assertProjectionExecution(sql, {
@@ -15545,6 +15853,8 @@ function invalidState(reservation, operation) {
15545
15853
  }
15546
15854
 
15547
15855
  // src/pg-client.ts
15856
+ import { isIP } from "net";
15857
+ import { connect as connectTls } from "tls";
15548
15858
  import postgres from "postgres";
15549
15859
  var DEFAULT_IDLE_TIMEOUT_SECONDS = 30;
15550
15860
  var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
@@ -15567,10 +15877,10 @@ function createPgClient(options) {
15567
15877
  connection,
15568
15878
  onnotice: () => {},
15569
15879
  ...options.prepare !== undefined ? { prepare: options.prepare } : {},
15570
- ...options.ssl ? { ssl: options.ssl } : {}
15880
+ ...tlsOptions(options)
15571
15881
  };
15572
15882
  if (options.connectionString) {
15573
- return postgres(options.connectionString, base);
15883
+ return assertDirectTlsTarget(postgres(options.connectionString, base), base);
15574
15884
  }
15575
15885
  return postgres({
15576
15886
  ...base,
@@ -15581,6 +15891,78 @@ function createPgClient(options) {
15581
15891
  ...options.password !== undefined ? { password: options.password } : {}
15582
15892
  });
15583
15893
  }
15894
+ function tlsOptions(options) {
15895
+ const verification = directTlsVerification(options);
15896
+ if (verification === null) {
15897
+ return options.ssl ? { ssl: options.ssl } : {};
15898
+ }
15899
+ return { ssl: false, sslnegotiation: null, socket: directTlsSocket(verification) };
15900
+ }
15901
+ function directTlsVerification(options) {
15902
+ const params = connectionStringParams(options.connectionString);
15903
+ if (params.get("sslnegotiation") !== "direct") {
15904
+ return null;
15905
+ }
15906
+ const sslmode = options.ssl === true ? "verify-full" : options.ssl || (params.get("sslrootcert") === "system" ? "verify-full" : params.get("sslmode") ?? params.get("ssl"));
15907
+ switch (sslmode) {
15908
+ case "require":
15909
+ return "none";
15910
+ case "verify-ca":
15911
+ return "ca";
15912
+ case "verify-full":
15913
+ case "true":
15914
+ return "full";
15915
+ default:
15916
+ throw new Error(`[SixbPg] sslnegotiation=direct requires sslmode=require, verify-ca or verify-full, got ${sslmode ? `"${sslmode}"` : "no sslmode"}: a direct TLS connection has no plaintext fallback.`);
15917
+ }
15918
+ }
15919
+ function connectionStringParams(connectionString) {
15920
+ return new URLSearchParams(connectionString?.split("?")[1] ?? "");
15921
+ }
15922
+ function directTlsSocket(verification) {
15923
+ let reportedRefusal = false;
15924
+ return ({ host: [host = "localhost"], port: [port = 5432] }) => {
15925
+ const socket = connectTls({
15926
+ host,
15927
+ port,
15928
+ servername: isIP(host) ? undefined : host,
15929
+ ALPNProtocols: ["postgresql"],
15930
+ rejectUnauthorized: verification !== "none",
15931
+ ...verification === "ca" ? { checkServerIdentity: () => {
15932
+ return;
15933
+ } } : {}
15934
+ });
15935
+ let reachedServer = false;
15936
+ let secured = false;
15937
+ socket.once("connect", () => {
15938
+ reachedServer = true;
15939
+ });
15940
+ socket.once("secureConnect", () => {
15941
+ secured = true;
15942
+ if (socket.alpnProtocol !== "postgresql") {
15943
+ socket.destroy(new Error(`[SixbPg] ${host}:${port} accepted TLS without selecting the "postgresql" protocol` + " (ALPN), so it is not a PostgreSQL server accepting sslnegotiation=direct."));
15944
+ }
15945
+ });
15946
+ socket.once("error", (error) => {
15947
+ if (reportedRefusal || !reachedServer || secured)
15948
+ return;
15949
+ if (!["ECONNRESET", "EPIPE", "ERR_SOCKET_CLOSED"].includes(pgErrorCode(error) ?? ""))
15950
+ return;
15951
+ reportedRefusal = true;
15952
+ console.error(`[SixbPg] ${host}:${port} closed the connection during a direct TLS handshake` + " (sslnegotiation=direct). Direct TLS needs PostgreSQL 17 or later, reached without a" + " proxy or pooler that expects an SSLRequest; remove sslnegotiation=direct otherwise.");
15953
+ });
15954
+ return socket;
15955
+ };
15956
+ }
15957
+ function assertDirectTlsTarget(sql, options) {
15958
+ if (!("socket" in options)) {
15959
+ return sql;
15960
+ }
15961
+ if (sql.options.host.length !== 1 || sql.options.path) {
15962
+ throw new Error("[SixbPg] sslnegotiation=direct supports one TCP host:port in the connection string, not" + " several hosts or a Unix socket path.");
15963
+ }
15964
+ return sql;
15965
+ }
15584
15966
 
15585
15967
  // src/pg-execution-storage.ts
15586
15968
  import {
@@ -15845,13 +16227,19 @@ async function queryLatestRunsByOwnerId(input) {
15845
16227
  return [];
15846
16228
  }
15847
16229
  const placeholders = ownerIds.map((_, index) => `$${index + 2}`).join(", ");
16230
+ const parameters = [input.projectId, ...ownerIds];
16231
+ let statusFilter = "";
16232
+ if (input.excludedStatus !== undefined) {
16233
+ parameters.push(input.excludedStatus);
16234
+ statusFilter = ` AND status <> $${parameters.length}`;
16235
+ }
15848
16236
  const orderColumn = input.tableName === "sync_runs" || input.tableName === "pipeline_runs" || input.tableName === "projection_runs" ? "COALESCE(started_at, queued_at)" : "started_at";
15849
16237
  const rows = await input.sql.unsafe(`
15850
16238
  SELECT DISTINCT ON (${input.ownerColumn}) ${input.selectList ?? "*"}
15851
16239
  FROM ${input.tableName}
15852
- WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})
16240
+ WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})${statusFilter}
15853
16241
  ORDER BY ${input.ownerColumn}, ${orderColumn} DESC, id DESC
15854
- `, [input.projectId, ...ownerIds]);
16242
+ `, parameters);
15855
16243
  const latestByOwnerId = new Map(rows.map((row) => [input.ownerIdFor(row), row]));
15856
16244
  return ownerIds.flatMap((ownerId) => {
15857
16245
  const row = latestByOwnerId.get(ownerId);
@@ -16240,12 +16628,14 @@ import {
16240
16628
  failProjectionRunEnqueue,
16241
16629
  immutableDatasetVersionConflict,
16242
16630
  mergeProjectionRunProgress,
16631
+ parseProjectionAttemptFailure,
16243
16632
  planProjectionRunFinish,
16244
16633
  planProjectionRunReclaim,
16245
16634
  projectionRunNotFound,
16246
16635
  publicProjectionRunRecord,
16247
16636
  requireTelemetryProjectionRun,
16248
16637
  restoreProjectionRun,
16638
+ SUPERSEDING_PROJECTION_RUN_STATUSES,
16249
16639
  staleProjectionRunExecution
16250
16640
  } from "@sixb/core/internal/projection-run-storage-provider";
16251
16641
  import { PROJECTION_RUN_FAILURE_CODES, ProjectionRunError } from "@sixb/core/storage";
@@ -16347,8 +16737,7 @@ class PgProjectionRunStorage {
16347
16737
  const [updated] = await tx`
16348
16738
  UPDATE projection_runs
16349
16739
  SET status = ${"running"}, started_at = COALESCE(started_at, ${input.startedAt ?? new Date}),
16350
- finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken},
16351
- error = ${null}
16740
+ finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
16352
16741
  WHERE project_id = ${input.projectId} AND id = ${input.id}
16353
16742
  AND (
16354
16743
  (status = ${"queued"} AND execution_token IS NULL)
@@ -16415,6 +16804,24 @@ class PgProjectionRunStorage {
16415
16804
  return rowToProjectionRunRecord(updated);
16416
16805
  });
16417
16806
  }
16807
+ async recordAttemptFailure(input) {
16808
+ return runPgTransaction(this.sql, async (tx) => {
16809
+ const existingRow = await requireMaterializationExecution(tx, input);
16810
+ const error = parseProjectionAttemptFailure(restoreProjectionRunRow(existingRow), input.error);
16811
+ const [updated] = await tx`
16812
+ UPDATE projection_runs
16813
+ SET error = ${serializeSixbFailure4(error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
16814
+ WHERE project_id = ${input.projectId}
16815
+ AND id = ${input.id}
16816
+ AND status = ${"running"}
16817
+ AND execution_token = ${input.executionToken}
16818
+ RETURNING *
16819
+ `;
16820
+ if (!updated)
16821
+ throw staleProjectionRunExecution(input.id);
16822
+ return rowToProjectionRunRecord(updated);
16823
+ });
16824
+ }
16418
16825
  async finish(input) {
16419
16826
  return runPgTransaction(this.sql, async (tx) => {
16420
16827
  const existingRow = await requireMaterializationExecution(tx, input);
@@ -16574,6 +16981,28 @@ class PgProjectionRunStorage {
16574
16981
  const runs = rows.map(rowToProjectionRunRecord);
16575
16982
  return { runs, hasMore: offset + runs.length < total, total };
16576
16983
  }
16984
+ async findSupersedingRun(input) {
16985
+ const [run] = await this.sql`
16986
+ SELECT projection_id, materialization_protocol, dataset_version_created_at
16987
+ FROM projection_runs
16988
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
16989
+ `;
16990
+ if (!run)
16991
+ throw projectionRunNotFound(input.projectId, input.id);
16992
+ if (run.materialization_protocol !== "replacement")
16993
+ return null;
16994
+ const [newer] = await this.sql`
16995
+ SELECT * FROM projection_runs
16996
+ WHERE project_id = ${input.projectId}
16997
+ AND projection_id = ${run.projection_id}
16998
+ AND materialization_protocol = ${"replacement"}
16999
+ AND status = ANY(${this.sql.array([...SUPERSEDING_PROJECTION_RUN_STATUSES])}::text[])
17000
+ AND dataset_version_created_at COLLATE "C" > ${run.dataset_version_created_at} COLLATE "C"
17001
+ ORDER BY dataset_version_created_at COLLATE "C" DESC, id DESC
17002
+ LIMIT 1
17003
+ `;
17004
+ return newer ? rowToProjectionRunRecord(newer) : null;
17005
+ }
16577
17006
  async listLatestByProjectionIds(input) {
16578
17007
  const rows = await queryLatestRunsByOwnerId({
16579
17008
  sql: this.sql,
@@ -16581,7 +17010,8 @@ class PgProjectionRunStorage {
16581
17010
  ownerColumn: "projection_id",
16582
17011
  ownerIds: input.projectionIds,
16583
17012
  projectId: input.projectId,
16584
- ownerIdFor: (row) => row.projection_id
17013
+ ownerIdFor: (row) => row.projection_id,
17014
+ excludedStatus: "superseded"
16585
17015
  });
16586
17016
  return { runs: rows.map(rowToProjectionRunRecord) };
16587
17017
  }
@@ -18907,4 +19337,4 @@ export {
18907
19337
  migratePostgresStorage
18908
19338
  };
18909
19339
 
18910
- //# debugId=5427C0EF9727207C64756E2164756E21
19340
+ //# debugId=4748AE3972DA287E64756E2164756E21