@sixb/pg 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +742 -312
- package/dist/index.js.map +14 -14
- package/dist/latest-run-query.d.ts +2 -0
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +12 -9
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts +7 -1
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
- package/dist/ontology-storage/source-roots.d.ts +8 -0
- package/dist/ontology-storage/source-roots.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +8 -3
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pg-client.d.ts +2 -0
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +3 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/storage-errors.d.ts +13 -3
- package/dist/storage-errors.d.ts.map +1 -1
- package/dist/transactions.d.ts +23 -1
- package/dist/transactions.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/index.ts +8 -1
- package/src/latest-run-query.ts +10 -2
- package/src/migrations/051-projection-run-supersession.sql +43 -0
- package/src/migrations/052-compact-source-storage.sql +118 -0
- package/src/migrations.ts +26 -23
- package/src/ontology-storage/materialization-state.ts +20 -46
- package/src/ontology-storage/shared.ts +18 -12
- package/src/ontology-storage/source-cleanup.ts +66 -22
- package/src/ontology-storage/source-roots.ts +164 -102
- package/src/ontology-storage/sources.ts +150 -120
- package/src/pg-client.ts +132 -2
- package/src/pg-projection-run-storage.ts +56 -2
- package/src/storage-errors.ts +101 -15
- package/src/transactions.ts +113 -20
package/dist/index.js
CHANGED
|
@@ -32,20 +32,34 @@ import {
|
|
|
32
32
|
} from "@sixb/core/storage";
|
|
33
33
|
|
|
34
34
|
// src/storage-errors.ts
|
|
35
|
+
import { createSixbError } from "@sixb/core/internal/errors";
|
|
35
36
|
var UNIQUE_VIOLATION = "23505";
|
|
36
37
|
var FOREIGN_KEY_VIOLATION = "23503";
|
|
37
38
|
var RETRYABLE_TRANSACTION_CONFLICT_CODES = new Set(["40001", "40P01"]);
|
|
38
|
-
var
|
|
39
|
-
"CONNECT_TIMEOUT",
|
|
39
|
+
var CONNECTION_LOST_CODES = new Set([
|
|
40
40
|
"CONNECTION_CLOSED",
|
|
41
41
|
"CONNECTION_DESTROYED",
|
|
42
|
-
"CONNECTION_ENDED",
|
|
43
|
-
"ECONNREFUSED",
|
|
44
42
|
"ECONNRESET",
|
|
45
|
-
"
|
|
43
|
+
"EPIPE",
|
|
44
|
+
"ETIMEDOUT"
|
|
45
|
+
]);
|
|
46
|
+
var CONNECT_FAILED_CODES = new Set([
|
|
47
|
+
"CONNECT_TIMEOUT",
|
|
48
|
+
"ECONNREFUSED",
|
|
46
49
|
"ENOTFOUND",
|
|
47
|
-
"EAI_AGAIN"
|
|
50
|
+
"EAI_AGAIN",
|
|
51
|
+
"EHOSTUNREACH",
|
|
52
|
+
"ENETUNREACH"
|
|
48
53
|
]);
|
|
54
|
+
var LOCK_TABLES = [
|
|
55
|
+
{
|
|
56
|
+
setting: "max_pred_locks_per_transaction",
|
|
57
|
+
table: "predicate lock table",
|
|
58
|
+
note: " A long serializable transaction also keeps the predicate locks of every serializable" + " transaction that commits while it is open."
|
|
59
|
+
},
|
|
60
|
+
{ setting: "max_locks_per_transaction", table: "lock table", note: "" }
|
|
61
|
+
];
|
|
62
|
+
var OUT_OF_MEMORY = "53200";
|
|
49
63
|
function pgErrorCode(error) {
|
|
50
64
|
if (error instanceof Error) {
|
|
51
65
|
const code = error.code;
|
|
@@ -65,30 +79,101 @@ function isRetryableTransactionConflict(error) {
|
|
|
65
79
|
const code = pgErrorCode(error);
|
|
66
80
|
return code !== undefined && RETRYABLE_TRANSACTION_CONFLICT_CODES.has(code);
|
|
67
81
|
}
|
|
82
|
+
function isConnectionLost(error) {
|
|
83
|
+
const code = pgErrorCode(error);
|
|
84
|
+
if (code !== undefined && CONNECTION_LOST_CODES.has(code)) {
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
const severity = error instanceof Error ? error.severity : null;
|
|
88
|
+
return severity === "FATAL" || severity === "PANIC";
|
|
89
|
+
}
|
|
90
|
+
function explainPgError(error) {
|
|
91
|
+
if (isStorageUnavailable(error)) {
|
|
92
|
+
return storageUnavailable(error);
|
|
93
|
+
}
|
|
94
|
+
if (!(error instanceof Error) || pgErrorCode(error) !== OUT_OF_MEMORY) {
|
|
95
|
+
return error;
|
|
96
|
+
}
|
|
97
|
+
const hint = error.hint;
|
|
98
|
+
const lockTable = typeof hint === "string" ? LOCK_TABLES.find(({ setting }) => hint.includes(setting)) : undefined;
|
|
99
|
+
if (!lockTable) {
|
|
100
|
+
return error;
|
|
101
|
+
}
|
|
102
|
+
const { setting, table, note } = lockTable;
|
|
103
|
+
return new Error(`[SixbPg] PostgreSQL aborted the transaction because its shared ${table} is full ` + `(SQLSTATE 53200 "${error.message}"). The table holds ${setting} \xD7 (max_connections + ` + `max_prepared_transactions) entries shared by all sessions.${note} Raise ${setting} ` + "(applied at server restart) or shorten the transactions that overlap.", { cause: error });
|
|
104
|
+
}
|
|
105
|
+
function isStorageUnavailable(error) {
|
|
106
|
+
const code = pgErrorCode(error);
|
|
107
|
+
return isConnectionLost(error) || code !== undefined && CONNECT_FAILED_CODES.has(code);
|
|
108
|
+
}
|
|
109
|
+
function storageUnavailable(error) {
|
|
110
|
+
const code = pgErrorCode(error);
|
|
111
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
112
|
+
return createSixbError("storage.unavailable", `[SixbPg] PostgreSQL is unavailable: the connection was lost or could not be opened (${code ?? "no code"}: ${reason}).`, { cause: error, ...code === undefined ? {} : { details: { code } } });
|
|
113
|
+
}
|
|
68
114
|
|
|
69
115
|
// src/transactions.ts
|
|
70
116
|
import { createHash } from "crypto";
|
|
117
|
+
var BEGIN_STATEMENTS = {
|
|
118
|
+
unverifiedDefault: "BEGIN",
|
|
119
|
+
repeatableRead: "BEGIN ISOLATION LEVEL REPEATABLE READ",
|
|
120
|
+
serializable: "BEGIN ISOLATION LEVEL SERIALIZABLE"
|
|
121
|
+
};
|
|
122
|
+
var pgTransactionClients = new WeakSet;
|
|
71
123
|
var activePgTransactions = new WeakMap;
|
|
72
124
|
async function runPgTransaction(sql, run, options = {}) {
|
|
73
125
|
if (!canStartPgTransaction(sql)) {
|
|
74
126
|
return run(sql);
|
|
75
127
|
}
|
|
76
128
|
const isolation = options.isolation ?? "unverifiedDefault";
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
return
|
|
81
|
-
|
|
82
|
-
|
|
129
|
+
try {
|
|
130
|
+
return await withReservedPgConnection(sql, (tx) => {
|
|
131
|
+
pgTransactionClients.add(tx);
|
|
132
|
+
return runPgTransactionOn(tx, async () => {
|
|
133
|
+
activePgTransactions.set(tx, isolation);
|
|
134
|
+
try {
|
|
135
|
+
return await run(tx);
|
|
136
|
+
} finally {
|
|
137
|
+
activePgTransactions.delete(tx);
|
|
138
|
+
}
|
|
139
|
+
}, isolation);
|
|
140
|
+
});
|
|
141
|
+
} catch (error) {
|
|
142
|
+
throw explainPgError(error);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
async function withReservedPgConnection(sql, run) {
|
|
146
|
+
const connection = await sql.reserve();
|
|
147
|
+
let usable = true;
|
|
148
|
+
try {
|
|
149
|
+
return await run(connection);
|
|
150
|
+
} catch (error) {
|
|
151
|
+
usable = !isConnectionLost(error);
|
|
152
|
+
throw error;
|
|
153
|
+
} finally {
|
|
154
|
+
if (usable) {
|
|
155
|
+
connection.release();
|
|
83
156
|
}
|
|
84
|
-
};
|
|
85
|
-
if (isolation === "serializable") {
|
|
86
|
-
return sql.begin("isolation level serializable", trackedRun);
|
|
87
157
|
}
|
|
88
|
-
|
|
89
|
-
|
|
158
|
+
}
|
|
159
|
+
async function runPgTransactionOn(connection, run, isolation = "unverifiedDefault") {
|
|
160
|
+
await connection.unsafe(BEGIN_STATEMENTS[isolation]);
|
|
161
|
+
const result = await undoOnFailure(run, () => connection.unsafe("ROLLBACK"));
|
|
162
|
+
const commit = await connection.unsafe("COMMIT");
|
|
163
|
+
if (commit.command !== "COMMIT") {
|
|
164
|
+
throw new Error("[SixbPg] PostgreSQL rolled the transaction back instead of committing it: a statement in it" + " failed and its error was caught without being rethrown. A failed statement aborts the" + " whole transaction, so let its error propagate out of the transaction callback.");
|
|
165
|
+
}
|
|
166
|
+
return result;
|
|
167
|
+
}
|
|
168
|
+
async function undoOnFailure(run, undo) {
|
|
169
|
+
try {
|
|
170
|
+
return await run();
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (!isConnectionLost(error)) {
|
|
173
|
+
await undo();
|
|
174
|
+
}
|
|
175
|
+
throw error;
|
|
90
176
|
}
|
|
91
|
-
return sql.begin(trackedRun);
|
|
92
177
|
}
|
|
93
178
|
async function runPgRepeatableReadTransaction(sql, run) {
|
|
94
179
|
if (canStartPgTransaction(sql)) {
|
|
@@ -121,7 +206,7 @@ function advisoryLockParts(key) {
|
|
|
121
206
|
return [hash.readInt32BE(0), hash.readInt32BE(4)];
|
|
122
207
|
}
|
|
123
208
|
function canStartPgTransaction(sql) {
|
|
124
|
-
return typeof sql.
|
|
209
|
+
return !pgTransactionClients.has(sql) && typeof sql.reserve === "function";
|
|
125
210
|
}
|
|
126
211
|
|
|
127
212
|
// src/agents/rows.ts
|
|
@@ -7379,6 +7464,173 @@ ALTER TABLE auth_group_memberships ADD CONSTRAINT auth_group_memberships_source_
|
|
|
7379
7464
|
CHECK (source IN ('invitation', 'manual', 'agent', 'directory'));
|
|
7380
7465
|
`;
|
|
7381
7466
|
|
|
7467
|
+
// src/migrations/051-projection-run-supersession.sql
|
|
7468
|
+
var _051_projection_run_supersession_default = `-- Projection runs gain the terminal status \`superseded\`, and a running run may carry the failure
|
|
7469
|
+
-- of its last attempt in \`error\` while the queue retries it.
|
|
7470
|
+
|
|
7471
|
+
-- The status checks were unnamed. Drop exactly the two that enumerate the terminal statuses, then
|
|
7472
|
+
-- replace them with named ones that admit \`superseded\`.
|
|
7473
|
+
DO $$
|
|
7474
|
+
DECLARE status_constraint RECORD;
|
|
7475
|
+
BEGIN
|
|
7476
|
+
FOR status_constraint IN
|
|
7477
|
+
SELECT conname
|
|
7478
|
+
FROM pg_constraint
|
|
7479
|
+
WHERE conrelid = 'projection_runs'::regclass
|
|
7480
|
+
AND contype = 'c'
|
|
7481
|
+
AND pg_get_constraintdef(oid) LIKE '%''cancelled''%'
|
|
7482
|
+
LOOP
|
|
7483
|
+
EXECUTE format('ALTER TABLE projection_runs DROP CONSTRAINT %I', status_constraint.conname);
|
|
7484
|
+
END LOOP;
|
|
7485
|
+
END $$;
|
|
7486
|
+
|
|
7487
|
+
ALTER TABLE projection_runs
|
|
7488
|
+
ADD CONSTRAINT projection_runs_status_check CHECK (
|
|
7489
|
+
status IN ('queued', 'running', 'succeeded', 'failed', 'cancelled', 'superseded')
|
|
7490
|
+
),
|
|
7491
|
+
ADD CONSTRAINT projection_runs_lifecycle_check CHECK (
|
|
7492
|
+
(status = 'queued' AND attempt = 0 AND started_at IS NULL AND finished_at IS NULL
|
|
7493
|
+
AND error IS NULL)
|
|
7494
|
+
OR (status = 'running' AND attempt >= 1 AND started_at IS NOT NULL AND finished_at IS NULL)
|
|
7495
|
+
OR (
|
|
7496
|
+
status IN ('succeeded', 'failed', 'cancelled', 'superseded')
|
|
7497
|
+
AND finished_at IS NOT NULL
|
|
7498
|
+
AND (
|
|
7499
|
+
(error->>'code' = 'queue.enqueue_failed'
|
|
7500
|
+
AND status = 'failed' AND attempt = 0 AND started_at IS NULL)
|
|
7501
|
+
OR (COALESCE(error->>'code', '') != 'queue.enqueue_failed'
|
|
7502
|
+
AND attempt >= 1 AND started_at IS NOT NULL)
|
|
7503
|
+
)
|
|
7504
|
+
)
|
|
7505
|
+
),
|
|
7506
|
+
-- A run is superseded instead of materialized, so it carries no failure, and only replacement
|
|
7507
|
+
-- runs can be: telemetry appends readings rather than replace a source.
|
|
7508
|
+
ADD CONSTRAINT projection_runs_superseded_check CHECK (
|
|
7509
|
+
status != 'superseded' OR (error IS NULL AND materialization_protocol = 'replacement')
|
|
7510
|
+
);
|
|
7511
|
+
`;
|
|
7512
|
+
|
|
7513
|
+
// src/migrations/052-compact-source-storage.sql
|
|
7514
|
+
var _052_compact_source_storage_default = `-- Source versions get a compact surrogate key; roots and rows stop repeating the text triple.
|
|
7515
|
+
ALTER TABLE ontology_sources ADD COLUMN version_id BIGINT GENERATED ALWAYS AS IDENTITY;
|
|
7516
|
+
ALTER TABLE ontology_sources ADD CONSTRAINT ontology_sources_version_id_key UNIQUE (version_id);
|
|
7517
|
+
|
|
7518
|
+
ALTER TABLE ontology_source_rows RENAME TO legacy_ontology_source_rows;
|
|
7519
|
+
ALTER TABLE ontology_source_roots RENAME TO legacy_ontology_source_roots;
|
|
7520
|
+
-- Index names are schema-wide: free them for the new tables.
|
|
7521
|
+
DO $$
|
|
7522
|
+
DECLARE
|
|
7523
|
+
legacy record;
|
|
7524
|
+
BEGIN
|
|
7525
|
+
FOR legacy IN
|
|
7526
|
+
SELECT index_class.relname AS name
|
|
7527
|
+
FROM pg_index
|
|
7528
|
+
JOIN pg_class AS index_class ON index_class.oid = pg_index.indexrelid
|
|
7529
|
+
JOIN pg_class AS table_class ON table_class.oid = pg_index.indrelid
|
|
7530
|
+
WHERE table_class.relnamespace = current_schema()::regnamespace
|
|
7531
|
+
AND table_class.relname IN ('legacy_ontology_source_rows', 'legacy_ontology_source_roots')
|
|
7532
|
+
LOOP
|
|
7533
|
+
EXECUTE format('ALTER INDEX %I RENAME TO %I', legacy.name, left('legacy_' || legacy.name, 63));
|
|
7534
|
+
END LOOP;
|
|
7535
|
+
END $$;
|
|
7536
|
+
|
|
7537
|
+
-- A root is live while its version is published (active or superseded), it is not retired and
|
|
7538
|
+
-- it is not a deletion. Activation therefore writes only the roots it replaces, never its own.
|
|
7539
|
+
-- \`root_key\` is the canonical key; "C" orders it by its UTF-8 bytes.
|
|
7540
|
+
CREATE TABLE ontology_source_roots (
|
|
7541
|
+
id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
7542
|
+
version_id BIGINT NOT NULL REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
|
|
7543
|
+
project_id TEXT NOT NULL,
|
|
7544
|
+
root_key TEXT COLLATE "C" NOT NULL,
|
|
7545
|
+
staging_ordinal BIGINT NOT NULL CHECK (staging_ordinal >= 0),
|
|
7546
|
+
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
|
7547
|
+
retired_at TIMESTAMPTZ,
|
|
7548
|
+
CONSTRAINT ontology_source_roots_key UNIQUE (version_id, root_key),
|
|
7549
|
+
CONSTRAINT ontology_source_roots_ordinal UNIQUE (version_id, staging_ordinal)
|
|
7550
|
+
);
|
|
7551
|
+
CREATE INDEX idx_ontology_source_roots_live
|
|
7552
|
+
ON ontology_source_roots (project_id, root_key) WHERE retired_at IS NULL AND NOT deleted;
|
|
7553
|
+
CREATE INDEX idx_ontology_source_roots_retired
|
|
7554
|
+
ON ontology_source_roots (project_id, retired_at) WHERE retired_at IS NOT NULL;
|
|
7555
|
+
|
|
7556
|
+
-- A row's identity lives in its typed columns; \`payload\` is the assertion without kind and ref.
|
|
7557
|
+
CREATE TABLE ontology_source_rows (
|
|
7558
|
+
root_id BIGINT NOT NULL REFERENCES ontology_source_roots (id) ON DELETE RESTRICT,
|
|
7559
|
+
entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
|
|
7560
|
+
object_type_id TEXT,
|
|
7561
|
+
primary_id TEXT,
|
|
7562
|
+
source_type_id TEXT,
|
|
7563
|
+
source_primary_id TEXT,
|
|
7564
|
+
link_id TEXT,
|
|
7565
|
+
target_type_id TEXT,
|
|
7566
|
+
target_primary_id TEXT,
|
|
7567
|
+
payload JSONB,
|
|
7568
|
+
CHECK (
|
|
7569
|
+
(
|
|
7570
|
+
entity_kind = 'object'
|
|
7571
|
+
AND object_type_id IS NOT NULL AND primary_id IS NOT NULL
|
|
7572
|
+
AND source_type_id IS NULL AND source_primary_id IS NULL AND link_id IS NULL
|
|
7573
|
+
AND target_type_id IS NULL AND target_primary_id IS NULL
|
|
7574
|
+
)
|
|
7575
|
+
OR
|
|
7576
|
+
(
|
|
7577
|
+
entity_kind = 'link'
|
|
7578
|
+
AND object_type_id IS NULL AND primary_id IS NULL
|
|
7579
|
+
AND source_type_id IS NOT NULL AND source_primary_id IS NOT NULL AND link_id IS NOT NULL
|
|
7580
|
+
AND target_type_id IS NOT NULL AND target_primary_id IS NOT NULL
|
|
7581
|
+
)
|
|
7582
|
+
)
|
|
7583
|
+
);
|
|
7584
|
+
CREATE INDEX idx_ontology_source_rows_root ON ontology_source_rows (root_id);
|
|
7585
|
+
CREATE INDEX idx_ontology_source_rows_link_source
|
|
7586
|
+
ON ontology_source_rows (source_type_id, source_primary_id, link_id)
|
|
7587
|
+
WHERE entity_kind = 'link';
|
|
7588
|
+
CREATE INDEX idx_ontology_source_rows_link_target
|
|
7589
|
+
ON ontology_source_rows (target_type_id, target_primary_id)
|
|
7590
|
+
WHERE entity_kind = 'link';
|
|
7591
|
+
|
|
7592
|
+
-- A published root that was not active is retired, so liveness stays exactly what it was.
|
|
7593
|
+
INSERT INTO ontology_source_roots (
|
|
7594
|
+
version_id, project_id, root_key, staging_ordinal, deleted, retired_at
|
|
7595
|
+
)
|
|
7596
|
+
SELECT versions.version_id, roots.project_id,
|
|
7597
|
+
convert_from(decode(roots.root_sort_key, 'hex'), 'UTF8'), roots.staging_ordinal, roots.deleted,
|
|
7598
|
+
CASE
|
|
7599
|
+
WHEN roots.active THEN NULL
|
|
7600
|
+
WHEN roots.retired_at IS NOT NULL THEN roots.retired_at
|
|
7601
|
+
WHEN versions.status IN ('active', 'superseded') THEN versions.updated_at
|
|
7602
|
+
END
|
|
7603
|
+
FROM legacy_ontology_source_roots AS roots
|
|
7604
|
+
JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id);
|
|
7605
|
+
|
|
7606
|
+
INSERT INTO ontology_source_rows (
|
|
7607
|
+
root_id, entity_kind, object_type_id, primary_id,
|
|
7608
|
+
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
|
|
7609
|
+
)
|
|
7610
|
+
SELECT roots.id, rows.entity_kind, rows.object_type_id, rows.primary_id,
|
|
7611
|
+
rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
|
|
7612
|
+
rows.target_primary_id, NULLIF(rows.assertion - 'kind' - 'ref', '{}'::jsonb)
|
|
7613
|
+
FROM legacy_ontology_source_rows AS rows
|
|
7614
|
+
JOIN ontology_sources AS versions USING (project_id, source_id, materialization_id)
|
|
7615
|
+
JOIN ontology_source_roots AS roots
|
|
7616
|
+
ON roots.version_id = versions.version_id
|
|
7617
|
+
AND roots.root_key = convert_from(decode(rows.root_sort_key, 'hex'), 'UTF8');
|
|
7618
|
+
|
|
7619
|
+
DO $$
|
|
7620
|
+
BEGIN
|
|
7621
|
+
IF (SELECT count(*) FROM legacy_ontology_source_roots)
|
|
7622
|
+
<> (SELECT count(*) FROM ontology_source_roots)
|
|
7623
|
+
OR (SELECT count(*) FROM legacy_ontology_source_rows)
|
|
7624
|
+
<> (SELECT count(*) FROM ontology_source_rows) THEN
|
|
7625
|
+
RAISE EXCEPTION 'Source roots or rows were lost while compacting source storage.';
|
|
7626
|
+
END IF;
|
|
7627
|
+
END $$;
|
|
7628
|
+
|
|
7629
|
+
DROP TABLE legacy_ontology_source_rows;
|
|
7630
|
+
DROP TABLE legacy_ontology_source_roots;
|
|
7631
|
+
ANALYZE ontology_sources, ontology_source_roots, ontology_source_rows;
|
|
7632
|
+
`;
|
|
7633
|
+
|
|
7382
7634
|
// src/migrations/037-execution-requester-groups.sql
|
|
7383
7635
|
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
|
|
7384
7636
|
|
|
@@ -7564,15 +7816,11 @@ function postgresMigrationSession(sql, schemaName) {
|
|
|
7564
7816
|
async transaction(run) {
|
|
7565
7817
|
const previous = active;
|
|
7566
7818
|
active = sql;
|
|
7567
|
-
await sql.unsafe("BEGIN");
|
|
7568
7819
|
try {
|
|
7569
|
-
await sql
|
|
7570
|
-
|
|
7571
|
-
|
|
7572
|
-
|
|
7573
|
-
} catch (error) {
|
|
7574
|
-
await sql.unsafe("ROLLBACK");
|
|
7575
|
-
throw error;
|
|
7820
|
+
return await runPgTransactionOn(sql, async () => {
|
|
7821
|
+
await sql.unsafe(`SET LOCAL search_path TO ${schema}`);
|
|
7822
|
+
return run();
|
|
7823
|
+
});
|
|
7576
7824
|
} finally {
|
|
7577
7825
|
active = previous;
|
|
7578
7826
|
}
|
|
@@ -7592,18 +7840,14 @@ async function readPostgresHistory(sql, schemaName, adapterId) {
|
|
|
7592
7840
|
return rows.map(rowToMigrationRecord);
|
|
7593
7841
|
}
|
|
7594
7842
|
async function withPostgresMigrationLock(params, run) {
|
|
7595
|
-
const sql = await params.sql.reserve();
|
|
7596
7843
|
const [first, second] = advisoryLockParts2(`storage:migration:${params.schemaName}:${params.migrations.adapterId}`);
|
|
7597
|
-
|
|
7844
|
+
return withReservedPgConnection(params.sql, async (sql) => {
|
|
7845
|
+
const unlock = () => sql`SELECT pg_advisory_unlock(${first}, ${second})`;
|
|
7598
7846
|
await sql`SELECT pg_advisory_lock(${first}, ${second})`;
|
|
7599
|
-
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
}
|
|
7604
|
-
} finally {
|
|
7605
|
-
sql.release();
|
|
7606
|
-
}
|
|
7847
|
+
const result = await undoOnFailure(() => run(sql), unlock);
|
|
7848
|
+
await unlock();
|
|
7849
|
+
return result;
|
|
7850
|
+
});
|
|
7607
7851
|
}
|
|
7608
7852
|
function advisoryLockParts2(key) {
|
|
7609
7853
|
const hash = createHash2("sha256").update(key).digest();
|
|
@@ -7678,7 +7922,9 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
7678
7922
|
pgSql("047-ontology-commit-attribution", _047_ontology_commit_attribution_default),
|
|
7679
7923
|
pgSql("048-file-upload-sessions", _048_file_upload_sessions_default),
|
|
7680
7924
|
pgSql("049-native-sessions", _049_native_sessions_default),
|
|
7681
|
-
pgSql("050-directory-group-memberships", _050_directory_group_memberships_default)
|
|
7925
|
+
pgSql("050-directory-group-memberships", _050_directory_group_memberships_default),
|
|
7926
|
+
pgSql("051-projection-run-supersession", _051_projection_run_supersession_default),
|
|
7927
|
+
pgSql("052-compact-source-storage", _052_compact_source_storage_default)
|
|
7682
7928
|
]
|
|
7683
7929
|
});
|
|
7684
7930
|
|
|
@@ -9928,6 +10174,10 @@ function assertReconciliationPageLimit(limit) {
|
|
|
9928
10174
|
// src/ontology-storage/shared.ts
|
|
9929
10175
|
import { parseSixbFailure as parseSixbFailure2 } from "@sixb/core/internal/errors";
|
|
9930
10176
|
import { MaterializationConflictError } from "@sixb/core/internal/materialization";
|
|
10177
|
+
import {
|
|
10178
|
+
sourceAssertionFromColumns,
|
|
10179
|
+
sourceEntityFromKey
|
|
10180
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
9931
10181
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES } from "@sixb/core/storage";
|
|
9932
10182
|
import {
|
|
9933
10183
|
assertNonblank,
|
|
@@ -9993,8 +10243,8 @@ function sourceRecord(row) {
|
|
|
9993
10243
|
}
|
|
9994
10244
|
function sourceAssertion(row) {
|
|
9995
10245
|
return {
|
|
9996
|
-
root:
|
|
9997
|
-
assertion:
|
|
10246
|
+
root: sourceEntityFromKey(row.root_key),
|
|
10247
|
+
assertion: sourceAssertionFromColumns(row, row.payload),
|
|
9998
10248
|
stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal")
|
|
9999
10249
|
};
|
|
10000
10250
|
}
|
|
@@ -10206,27 +10456,80 @@ import {
|
|
|
10206
10456
|
MaterializationValidationError,
|
|
10207
10457
|
projectionEntityKey
|
|
10208
10458
|
} from "@sixb/core/internal/materialization";
|
|
10209
|
-
import {
|
|
10459
|
+
import {
|
|
10460
|
+
sourceEntityColumns,
|
|
10461
|
+
sourceStageRoots
|
|
10462
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
10463
|
+
var PUBLISHED = "('active', 'superseded')";
|
|
10464
|
+
function sourceAssertionColumns(sql) {
|
|
10465
|
+
return sql.unsafe(`versions.source_id, versions.materialization_id, roots.root_key,
|
|
10466
|
+
roots.staging_ordinal, rows.entity_kind, rows.object_type_id, rows.primary_id,
|
|
10467
|
+
rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
|
|
10468
|
+
rows.target_primary_id, rows.payload`);
|
|
10469
|
+
}
|
|
10470
|
+
function liveSourceRowsJoin(sql, projectId) {
|
|
10471
|
+
return sql`
|
|
10472
|
+
JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
|
|
10473
|
+
AND roots.project_id = ${projectId} AND roots.retired_at IS NULL AND NOT roots.deleted
|
|
10474
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
10475
|
+
AND versions.status IN ${sql.unsafe(PUBLISHED)}
|
|
10476
|
+
`;
|
|
10477
|
+
}
|
|
10478
|
+
function candidateVersion(sql, input) {
|
|
10479
|
+
return sql`(SELECT version_id FROM ontology_sources WHERE project_id = ${input.projectId}
|
|
10480
|
+
AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId})`;
|
|
10481
|
+
}
|
|
10482
|
+
var ENTITY_MATCH = `rows.entity_kind = requested.entity_kind
|
|
10483
|
+
AND rows.object_type_id IS NOT DISTINCT FROM requested.object_type_id
|
|
10484
|
+
AND rows.primary_id IS NOT DISTINCT FROM requested.primary_id
|
|
10485
|
+
AND rows.source_type_id IS NOT DISTINCT FROM requested.source_type_id
|
|
10486
|
+
AND rows.source_primary_id IS NOT DISTINCT FROM requested.source_primary_id
|
|
10487
|
+
AND rows.link_id IS NOT DISTINCT FROM requested.link_id
|
|
10488
|
+
AND rows.target_type_id IS NOT DISTINCT FROM requested.target_type_id
|
|
10489
|
+
AND rows.target_primary_id IS NOT DISTINCT FROM requested.target_primary_id`;
|
|
10490
|
+
var REQUESTED_ENTITY = `entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
|
|
10491
|
+
source_type_id TEXT, source_primary_id TEXT, link_id TEXT, target_type_id TEXT,
|
|
10492
|
+
target_primary_id TEXT`;
|
|
10493
|
+
function requestedEntity(ref) {
|
|
10494
|
+
const columns = sourceEntityColumns(ref);
|
|
10495
|
+
const key = projectionEntityKey(ref);
|
|
10496
|
+
return {
|
|
10497
|
+
entity_kind: ref.kind,
|
|
10498
|
+
object_type_id: columns.objectTypeId,
|
|
10499
|
+
primary_id: columns.primaryId,
|
|
10500
|
+
source_type_id: columns.sourceTypeId,
|
|
10501
|
+
source_primary_id: columns.sourcePrimaryId,
|
|
10502
|
+
link_id: columns.linkId,
|
|
10503
|
+
target_type_id: columns.targetTypeId,
|
|
10504
|
+
target_primary_id: columns.targetPrimaryId,
|
|
10505
|
+
root_keys: ref.kind === "object" ? [key] : [key, projectionEntityKey({ kind: "object", ref: ref.ref.source })]
|
|
10506
|
+
};
|
|
10507
|
+
}
|
|
10210
10508
|
function replacementSourceRows(sql, input) {
|
|
10509
|
+
const live = sql`roots.retired_at IS NULL AND NOT roots.deleted
|
|
10510
|
+
AND versions.source_id = ${input.sourceId} AND versions.status IN ${sql.unsafe(PUBLISHED)}`;
|
|
10211
10511
|
const roots = input.incremental ? sql`
|
|
10212
|
-
SELECT
|
|
10512
|
+
SELECT live.id FROM ontology_source_roots AS changed
|
|
10213
10513
|
CROSS JOIN LATERAL (
|
|
10214
|
-
SELECT roots
|
|
10215
|
-
|
|
10216
|
-
|
|
10217
|
-
|
|
10218
|
-
|
|
10219
|
-
|
|
10220
|
-
` : sql`
|
|
10514
|
+
SELECT roots.id FROM ontology_source_roots AS roots
|
|
10515
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
10516
|
+
WHERE roots.project_id = ${input.projectId} AND roots.root_key = changed.root_key AND ${live}
|
|
10517
|
+
OFFSET 0
|
|
10518
|
+
) AS live
|
|
10519
|
+
WHERE changed.version_id = ${candidateVersion(sql, input)}
|
|
10520
|
+
` : sql`
|
|
10521
|
+
SELECT roots.id FROM ontology_sources AS versions
|
|
10522
|
+
JOIN ontology_source_roots AS roots ON roots.version_id = versions.version_id
|
|
10523
|
+
WHERE versions.project_id = ${input.projectId} AND ${live}
|
|
10524
|
+
`;
|
|
10221
10525
|
return sql`
|
|
10222
|
-
SELECT
|
|
10223
|
-
|
|
10526
|
+
SELECT rows.* FROM ontology_source_roots AS roots
|
|
10527
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
10528
|
+
WHERE roots.version_id = ${candidateVersion(sql, input)}
|
|
10224
10529
|
UNION ALL
|
|
10225
10530
|
SELECT rows.* FROM (${roots}) AS roots
|
|
10226
10531
|
CROSS JOIN LATERAL (
|
|
10227
|
-
SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.
|
|
10228
|
-
AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
|
|
10229
|
-
AND rows.root_sort_key = roots.root_sort_key OFFSET 0
|
|
10532
|
+
SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id OFFSET 0
|
|
10230
10533
|
) AS rows
|
|
10231
10534
|
`;
|
|
10232
10535
|
}
|
|
@@ -10238,35 +10541,28 @@ async function stageSourceRoots(sql, manifest, input) {
|
|
|
10238
10541
|
throw new MaterializationValidationError("Root deletions require a source delta base.");
|
|
10239
10542
|
}
|
|
10240
10543
|
const values = jsonParameter2(sql, roots.map((root) => ({
|
|
10241
|
-
|
|
10242
|
-
root_kind: root.root.kind,
|
|
10243
|
-
root_key: JSON.parse(root.rootKey),
|
|
10244
|
-
root: root.root,
|
|
10544
|
+
root_key: root.rootKey,
|
|
10245
10545
|
staging_ordinal: root.stagingOrdinal,
|
|
10246
10546
|
deleted: root.deleted
|
|
10247
10547
|
})));
|
|
10248
10548
|
const inserted = await sql`
|
|
10249
|
-
INSERT INTO ontology_source_roots (
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
FROM jsonb_to_recordset(${values}) AS input(
|
|
10254
|
-
root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
|
|
10255
|
-
ON CONFLICT DO NOTHING RETURNING root_sort_key
|
|
10549
|
+
INSERT INTO ontology_source_roots (version_id, project_id, root_key, staging_ordinal, deleted)
|
|
10550
|
+
SELECT ${manifest.version_id}, ${input.projectId}, root_key, staging_ordinal, deleted
|
|
10551
|
+
FROM jsonb_to_recordset(${values}) AS input(root_key TEXT, staging_ordinal BIGINT, deleted BOOLEAN)
|
|
10552
|
+
ON CONFLICT DO NOTHING RETURNING root_key
|
|
10256
10553
|
`;
|
|
10257
10554
|
if (inserted.length === roots.length)
|
|
10258
10555
|
return;
|
|
10259
|
-
const insertedKeys = new Set(inserted.map((row) => row.
|
|
10260
|
-
const skipped = roots.filter((root) => !insertedKeys.has(root.
|
|
10556
|
+
const insertedKeys = new Set(inserted.map((row) => row.root_key));
|
|
10557
|
+
const skipped = roots.filter((root) => !insertedKeys.has(root.rootKey));
|
|
10261
10558
|
const stored = await sql`
|
|
10262
|
-
SELECT
|
|
10263
|
-
WHERE
|
|
10264
|
-
AND
|
|
10265
|
-
AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
|
|
10559
|
+
SELECT root_key, staging_ordinal, deleted FROM ontology_source_roots
|
|
10560
|
+
WHERE version_id = ${manifest.version_id}
|
|
10561
|
+
AND root_key = ANY(${sql.array(skipped.map((root) => root.rootKey))}::text[])
|
|
10266
10562
|
`;
|
|
10267
|
-
const existing = new Map(stored.map((root) => [root.
|
|
10563
|
+
const existing = new Map(stored.map((root) => [root.root_key, root]));
|
|
10268
10564
|
if (skipped.some((root) => {
|
|
10269
|
-
const previous = existing.get(root.
|
|
10565
|
+
const previous = existing.get(root.rootKey);
|
|
10270
10566
|
return !previous || Number(previous.staging_ordinal) !== root.stagingOrdinal || previous.deleted !== root.deleted;
|
|
10271
10567
|
}))
|
|
10272
10568
|
throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
|
|
@@ -10276,94 +10572,84 @@ async function activateSourceRoots(sql, projectId, candidate, activation) {
|
|
|
10276
10572
|
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
10277
10573
|
if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0)
|
|
10278
10574
|
return;
|
|
10279
|
-
const
|
|
10280
|
-
|
|
10281
|
-
|
|
10282
|
-
|
|
10283
|
-
)
|
|
10284
|
-
|
|
10285
|
-
|
|
10286
|
-
|
|
10287
|
-
|
|
10288
|
-
|
|
10575
|
+
const live = sql`roots.retired_at IS NULL AND NOT roots.deleted
|
|
10576
|
+
AND versions.version_id = roots.version_id AND versions.project_id = ${projectId}
|
|
10577
|
+
AND versions.source_id = ${activation.source.projectionId}
|
|
10578
|
+
AND versions.status IN ${sql.unsafe(PUBLISHED)}`;
|
|
10579
|
+
if (candidate.base_materialization_id === null) {
|
|
10580
|
+
await sql`
|
|
10581
|
+
UPDATE ontology_source_roots AS roots SET retired_at = ${activation.updatedAt}
|
|
10582
|
+
FROM ontology_sources AS versions WHERE ${live}
|
|
10583
|
+
`;
|
|
10584
|
+
} else {
|
|
10585
|
+
await sql`
|
|
10586
|
+
UPDATE ontology_source_roots AS roots SET retired_at = ${activation.updatedAt}
|
|
10587
|
+
FROM ontology_source_roots AS changed, ontology_sources AS versions
|
|
10588
|
+
WHERE changed.version_id = ${candidate.version_id} AND roots.project_id = ${projectId}
|
|
10589
|
+
AND roots.root_key = changed.root_key AND ${live}
|
|
10590
|
+
`;
|
|
10591
|
+
}
|
|
10289
10592
|
await sql`
|
|
10290
|
-
UPDATE ontology_source_roots SET
|
|
10291
|
-
|
|
10292
|
-
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
|
|
10293
|
-
AND materialization_id = ${activation.materializationId}
|
|
10593
|
+
UPDATE ontology_source_roots SET retired_at = ${activation.updatedAt}
|
|
10594
|
+
WHERE version_id = ${candidate.version_id} AND deleted
|
|
10294
10595
|
`;
|
|
10295
10596
|
}
|
|
10296
10597
|
async function assertSourceRootCoverage(sql, manifest) {
|
|
10297
10598
|
const [invalid] = await sql`
|
|
10298
10599
|
SELECT 1 FROM ontology_source_roots AS roots
|
|
10299
|
-
WHERE
|
|
10300
|
-
AND materialization_id = ${manifest.materialization_id}
|
|
10600
|
+
WHERE roots.version_id = ${manifest.version_id}
|
|
10301
10601
|
AND roots.deleted = EXISTS (
|
|
10302
|
-
SELECT 1 FROM ontology_source_rows AS rows
|
|
10303
|
-
|
|
10304
|
-
|
|
10305
|
-
) LIMIT 1
|
|
10602
|
+
SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id
|
|
10603
|
+
)
|
|
10604
|
+
LIMIT 1
|
|
10306
10605
|
`;
|
|
10307
10606
|
if (invalid)
|
|
10308
10607
|
throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
|
|
10309
10608
|
}
|
|
10310
10609
|
function activeSourceRows(sql, projectId, refs) {
|
|
10311
|
-
const
|
|
10312
|
-
const
|
|
10313
|
-
|
|
10314
|
-
entitySortKey,
|
|
10315
|
-
utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
|
|
10316
|
-
];
|
|
10317
|
-
return roots.map((rootSortKey) => ({
|
|
10318
|
-
root_sort_key: rootSortKey,
|
|
10319
|
-
entity_sort_key: entitySortKey
|
|
10320
|
-
}));
|
|
10610
|
+
const requested = refs.flatMap((ref) => {
|
|
10611
|
+
const { root_keys, ...entity } = requestedEntity(ref);
|
|
10612
|
+
return root_keys.map((root_key) => ({ ...entity, root_key }));
|
|
10321
10613
|
});
|
|
10322
10614
|
return sql`
|
|
10323
|
-
SELECT
|
|
10615
|
+
SELECT ${sourceAssertionColumns(sql)}
|
|
10616
|
+
FROM jsonb_to_recordset(${jsonParameter2(sql, requested)})
|
|
10617
|
+
AS requested(root_key TEXT, ${sql.unsafe(REQUESTED_ENTITY)})
|
|
10324
10618
|
JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
|
|
10325
|
-
AND roots.
|
|
10326
|
-
JOIN
|
|
10327
|
-
AND
|
|
10328
|
-
|
|
10619
|
+
AND roots.root_key = requested.root_key AND roots.retired_at IS NULL AND NOT roots.deleted
|
|
10620
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
10621
|
+
AND versions.status IN ${sql.unsafe(PUBLISHED)}
|
|
10622
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
|
|
10329
10623
|
`;
|
|
10330
10624
|
}
|
|
10331
10625
|
function replacementAssertionRows(sql, input) {
|
|
10332
|
-
const
|
|
10333
|
-
entity_kind: ref.kind,
|
|
10334
|
-
entity_key: JSON.parse(projectionEntityKey(ref)),
|
|
10335
|
-
root_keys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
|
|
10336
|
-
utf8SortKey(projectionEntityKey(ref)),
|
|
10337
|
-
utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
|
|
10338
|
-
]
|
|
10339
|
-
}));
|
|
10626
|
+
const version = candidateVersion(sql, input);
|
|
10340
10627
|
const previous = !input.includePrevious ? sql`` : sql`UNION ALL
|
|
10341
|
-
SELECT
|
|
10628
|
+
SELECT ${sourceAssertionColumns(sql)} FROM (
|
|
10342
10629
|
SELECT roots.* FROM ontology_source_roots AS roots
|
|
10343
|
-
|
|
10344
|
-
|
|
10630
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
10631
|
+
WHERE roots.project_id = ${input.projectId} AND roots.root_key = ANY(requested.root_keys)
|
|
10632
|
+
AND roots.retired_at IS NULL AND NOT roots.deleted
|
|
10633
|
+
AND versions.source_id = ${input.sourceId} AND versions.status IN ${sql.unsafe(PUBLISHED)}
|
|
10345
10634
|
${!input.incremental ? sql`` : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
|
|
10346
|
-
WHERE changed.
|
|
10347
|
-
AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`}
|
|
10348
|
-
|
|
10635
|
+
WHERE changed.version_id = ${version} AND changed.root_key = roots.root_key)`}
|
|
10349
10636
|
OFFSET 0
|
|
10350
10637
|
) AS roots
|
|
10638
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
10351
10639
|
CROSS JOIN LATERAL (
|
|
10352
10640
|
SELECT rows.* FROM ontology_source_rows AS rows
|
|
10353
|
-
WHERE rows.
|
|
10354
|
-
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
10355
|
-
AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
10641
|
+
WHERE rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
|
|
10356
10642
|
OFFSET 0
|
|
10357
10643
|
) AS rows
|
|
10358
10644
|
`;
|
|
10359
10645
|
return sql`
|
|
10360
|
-
SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql,
|
|
10361
|
-
AS requested(
|
|
10646
|
+
SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, input.refs.map(requestedEntity))})
|
|
10647
|
+
AS requested(${sql.unsafe(REQUESTED_ENTITY)}, root_keys TEXT[])
|
|
10362
10648
|
CROSS JOIN LATERAL (
|
|
10363
|
-
SELECT
|
|
10364
|
-
|
|
10365
|
-
|
|
10366
|
-
|
|
10649
|
+
SELECT ${sourceAssertionColumns(sql)} FROM ontology_source_roots AS roots
|
|
10650
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
10651
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id AND ${sql.unsafe(ENTITY_MATCH)}
|
|
10652
|
+
WHERE roots.version_id = ${version} AND roots.root_key = ANY(requested.root_keys)
|
|
10367
10653
|
${previous}
|
|
10368
10654
|
OFFSET 0
|
|
10369
10655
|
) AS selected
|
|
@@ -10582,20 +10868,14 @@ class PgMaterializationStateReader {
|
|
|
10582
10868
|
scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
|
|
10583
10869
|
)
|
|
10584
10870
|
)
|
|
10585
|
-
SELECT
|
|
10871
|
+
SELECT ${sourceAssertionColumns(this.sql)}
|
|
10586
10872
|
FROM ontology_source_rows AS rows
|
|
10587
10873
|
JOIN requested
|
|
10588
10874
|
ON requested.source_type_id = rows.source_type_id
|
|
10589
10875
|
AND requested.source_id = rows.source_primary_id
|
|
10590
10876
|
AND requested.link_id = rows.link_id
|
|
10591
|
-
|
|
10592
|
-
|
|
10593
|
-
AND sources.source_id = rows.source_id
|
|
10594
|
-
AND sources.materialization_id = rows.materialization_id
|
|
10595
|
-
AND sources.root_sort_key = rows.root_sort_key
|
|
10596
|
-
WHERE rows.project_id = ${this.projectId}
|
|
10597
|
-
AND rows.entity_kind = 'link'
|
|
10598
|
-
AND sources.active
|
|
10877
|
+
${liveSourceRowsJoin(this.sql, this.projectId)}
|
|
10878
|
+
WHERE rows.entity_kind = 'link'
|
|
10599
10879
|
`,
|
|
10600
10880
|
this.sql`
|
|
10601
10881
|
WITH requested AS (
|
|
@@ -10737,32 +11017,20 @@ class PgMaterializationStateReader {
|
|
|
10737
11017
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10738
11018
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10739
11019
|
FROM ontology_source_rows AS rows
|
|
10740
|
-
|
|
10741
|
-
ON sources.project_id = rows.project_id
|
|
10742
|
-
AND sources.source_id = rows.source_id
|
|
10743
|
-
AND sources.materialization_id = rows.materialization_id
|
|
10744
|
-
AND sources.root_sort_key = rows.root_sort_key
|
|
11020
|
+
${liveSourceRowsJoin(this.sql, this.projectId)}
|
|
10745
11021
|
JOIN requested
|
|
10746
11022
|
ON requested.object_type_id = rows.source_type_id
|
|
10747
11023
|
AND requested.primary_id = rows.source_primary_id
|
|
10748
|
-
WHERE rows.
|
|
10749
|
-
AND rows.entity_kind = 'link'
|
|
10750
|
-
AND sources.active
|
|
11024
|
+
WHERE rows.entity_kind = 'link'
|
|
10751
11025
|
UNION
|
|
10752
11026
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10753
11027
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10754
11028
|
FROM ontology_source_rows AS rows
|
|
10755
|
-
|
|
10756
|
-
ON sources.project_id = rows.project_id
|
|
10757
|
-
AND sources.source_id = rows.source_id
|
|
10758
|
-
AND sources.materialization_id = rows.materialization_id
|
|
10759
|
-
AND sources.root_sort_key = rows.root_sort_key
|
|
11029
|
+
${liveSourceRowsJoin(this.sql, this.projectId)}
|
|
10760
11030
|
JOIN requested
|
|
10761
11031
|
ON requested.object_type_id = rows.target_type_id
|
|
10762
11032
|
AND requested.primary_id = rows.target_primary_id
|
|
10763
|
-
WHERE rows.
|
|
10764
|
-
AND rows.entity_kind = 'link'
|
|
10765
|
-
AND sources.active
|
|
11033
|
+
WHERE rows.entity_kind = 'link'
|
|
10766
11034
|
), selected AS (
|
|
10767
11035
|
SELECT DISTINCT links.*,
|
|
10768
11036
|
${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
|
|
@@ -10996,32 +11264,20 @@ class PgMaterializationStateReader {
|
|
|
10996
11264
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10997
11265
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10998
11266
|
FROM ontology_source_rows AS rows
|
|
10999
|
-
|
|
11000
|
-
ON sources.project_id = rows.project_id
|
|
11001
|
-
AND sources.source_id = rows.source_id
|
|
11002
|
-
AND sources.materialization_id = rows.materialization_id
|
|
11003
|
-
AND sources.root_sort_key = rows.root_sort_key
|
|
11267
|
+
${liveSourceRowsJoin(this.sql, this.projectId)}
|
|
11004
11268
|
JOIN incident_objects
|
|
11005
11269
|
ON incident_objects.object_type_id = rows.source_type_id
|
|
11006
11270
|
AND incident_objects.primary_id = rows.source_primary_id
|
|
11007
|
-
WHERE rows.
|
|
11008
|
-
AND rows.entity_kind = 'link'
|
|
11009
|
-
AND sources.active
|
|
11271
|
+
WHERE rows.entity_kind = 'link'
|
|
11010
11272
|
UNION
|
|
11011
11273
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
11012
11274
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
11013
11275
|
FROM ontology_source_rows AS rows
|
|
11014
|
-
|
|
11015
|
-
ON sources.project_id = rows.project_id
|
|
11016
|
-
AND sources.source_id = rows.source_id
|
|
11017
|
-
AND sources.materialization_id = rows.materialization_id
|
|
11018
|
-
AND sources.root_sort_key = rows.root_sort_key
|
|
11276
|
+
${liveSourceRowsJoin(this.sql, this.projectId)}
|
|
11019
11277
|
JOIN incident_objects
|
|
11020
11278
|
ON incident_objects.object_type_id = rows.target_type_id
|
|
11021
11279
|
AND incident_objects.primary_id = rows.target_primary_id
|
|
11022
|
-
WHERE rows.
|
|
11023
|
-
AND rows.entity_kind = 'link'
|
|
11024
|
-
AND sources.active
|
|
11280
|
+
WHERE rows.entity_kind = 'link'
|
|
11025
11281
|
), diff_links AS (
|
|
11026
11282
|
SELECT * FROM replacement_links
|
|
11027
11283
|
UNION SELECT * FROM incident_links
|
|
@@ -11163,9 +11419,7 @@ function storedSource(row) {
|
|
|
11163
11419
|
return {
|
|
11164
11420
|
source: { projectionId: row.source_id },
|
|
11165
11421
|
materializationId: row.materialization_id,
|
|
11166
|
-
|
|
11167
|
-
assertion: structuredClone(row.assertion),
|
|
11168
|
-
stagingOrdinal: databaseSafeInteger(row.staging_ordinal, "Source staging ordinal")
|
|
11422
|
+
...sourceAssertion(row)
|
|
11169
11423
|
};
|
|
11170
11424
|
}
|
|
11171
11425
|
function effectiveObjectSnapshot(row) {
|
|
@@ -12863,26 +13117,70 @@ import {
|
|
|
12863
13117
|
sourceEntityColumns as entityColumns,
|
|
12864
13118
|
isExactStagingManifest,
|
|
12865
13119
|
reconcileSourceStageRows,
|
|
13120
|
+
sourceAssertionPayload,
|
|
12866
13121
|
sourceConflict,
|
|
12867
13122
|
sourceMaterializationIdentity,
|
|
12868
13123
|
sourceStageRow,
|
|
12869
|
-
sourceStageRows
|
|
12870
|
-
utf8SortKey as utf8SortKey2
|
|
13124
|
+
sourceStageRows
|
|
12871
13125
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
12872
13126
|
|
|
12873
13127
|
// src/ontology-storage/source-cleanup.ts
|
|
13128
|
+
async function purgeAbandonedSourceVersions(sql, input) {
|
|
13129
|
+
let remaining = input.limit;
|
|
13130
|
+
let rowsDeleted = 0;
|
|
13131
|
+
let materializationsDeleted = 0;
|
|
13132
|
+
while (remaining > 0) {
|
|
13133
|
+
const [candidate] = await sql`
|
|
13134
|
+
SELECT version_id FROM ontology_sources
|
|
13135
|
+
WHERE project_id = ${input.projectId} AND status = 'abandoned'
|
|
13136
|
+
ORDER BY terminal_at, source_id, materialization_id
|
|
13137
|
+
LIMIT 1 FOR UPDATE SKIP LOCKED
|
|
13138
|
+
`;
|
|
13139
|
+
if (!candidate)
|
|
13140
|
+
break;
|
|
13141
|
+
const rows = await sql`
|
|
13142
|
+
DELETE FROM ontology_source_rows WHERE ctid IN (
|
|
13143
|
+
SELECT rows.ctid FROM ontology_source_roots AS roots
|
|
13144
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
13145
|
+
WHERE roots.version_id = ${candidate.version_id} LIMIT ${remaining}
|
|
13146
|
+
)
|
|
13147
|
+
`;
|
|
13148
|
+
rowsDeleted += rows.count;
|
|
13149
|
+
remaining -= rows.count;
|
|
13150
|
+
if (remaining === 0)
|
|
13151
|
+
break;
|
|
13152
|
+
const roots = await sql`
|
|
13153
|
+
DELETE FROM ontology_source_roots WHERE ctid IN (
|
|
13154
|
+
SELECT ctid FROM ontology_source_roots
|
|
13155
|
+
WHERE version_id = ${candidate.version_id} LIMIT ${remaining}
|
|
13156
|
+
)
|
|
13157
|
+
`;
|
|
13158
|
+
rowsDeleted += roots.count;
|
|
13159
|
+
remaining -= roots.count;
|
|
13160
|
+
if (remaining === 0)
|
|
13161
|
+
break;
|
|
13162
|
+
const manifest = await sql`
|
|
13163
|
+
DELETE FROM ontology_sources
|
|
13164
|
+
WHERE version_id = ${candidate.version_id} AND status = 'abandoned'
|
|
13165
|
+
`;
|
|
13166
|
+
if (manifest.count === 0)
|
|
13167
|
+
break;
|
|
13168
|
+
materializationsDeleted += manifest.count;
|
|
13169
|
+
remaining -= manifest.count;
|
|
13170
|
+
}
|
|
13171
|
+
return { rowsDeleted, materializationsDeleted };
|
|
13172
|
+
}
|
|
12874
13173
|
async function cleanupSourceVersions(sql, input) {
|
|
12875
13174
|
let remaining = input.limit;
|
|
13175
|
+
const retired = sql`roots.project_id = ${input.projectId}
|
|
13176
|
+
AND roots.retired_at < ${input.terminalBefore} AND versions.status = 'superseded'`;
|
|
12876
13177
|
const removedRows = await sql`
|
|
12877
13178
|
WITH selected AS (
|
|
12878
13179
|
SELECT rows.ctid FROM ontology_source_roots AS roots
|
|
12879
|
-
JOIN ontology_sources AS
|
|
12880
|
-
JOIN ontology_source_rows AS rows
|
|
12881
|
-
|
|
12882
|
-
|
|
12883
|
-
WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
|
|
12884
|
-
AND sources.status IN ('superseded', 'abandoned')
|
|
12885
|
-
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
|
|
13180
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
13181
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
13182
|
+
WHERE ${retired}
|
|
13183
|
+
ORDER BY roots.retired_at, roots.id
|
|
12886
13184
|
LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
|
|
12887
13185
|
) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
|
|
12888
13186
|
`;
|
|
@@ -12892,13 +13190,10 @@ async function cleanupSourceVersions(sql, input) {
|
|
|
12892
13190
|
const removed = await sql`
|
|
12893
13191
|
WITH selected AS (
|
|
12894
13192
|
SELECT roots.ctid FROM ontology_source_roots AS roots
|
|
12895
|
-
JOIN ontology_sources AS
|
|
12896
|
-
WHERE
|
|
12897
|
-
AND
|
|
12898
|
-
|
|
12899
|
-
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
12900
|
-
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
|
|
12901
|
-
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
|
|
13193
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
13194
|
+
WHERE ${retired}
|
|
13195
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
|
|
13196
|
+
ORDER BY roots.retired_at, roots.id
|
|
12902
13197
|
LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
|
|
12903
13198
|
) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
|
|
12904
13199
|
`;
|
|
@@ -12907,16 +13202,13 @@ async function cleanupSourceVersions(sql, input) {
|
|
|
12907
13202
|
}
|
|
12908
13203
|
const removedManifests = remaining === 0 ? [] : await sql`
|
|
12909
13204
|
WITH selected AS (
|
|
12910
|
-
SELECT
|
|
12911
|
-
WHERE project_id = ${input.projectId} AND status
|
|
13205
|
+
SELECT versions.ctid FROM ontology_sources AS versions
|
|
13206
|
+
WHERE project_id = ${input.projectId} AND status = 'superseded'
|
|
13207
|
+
AND terminal_at < ${input.terminalBefore}
|
|
12912
13208
|
AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
|
|
12913
|
-
WHERE roots.
|
|
12914
|
-
AND roots.materialization_id = sources.materialization_id)
|
|
12915
|
-
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
12916
|
-
WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
|
|
12917
|
-
AND rows.materialization_id = sources.materialization_id)
|
|
13209
|
+
WHERE roots.version_id = versions.version_id)
|
|
12918
13210
|
ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
|
|
12919
|
-
) DELETE FROM ontology_sources AS
|
|
13211
|
+
) DELETE FROM ontology_sources AS versions USING selected WHERE versions.ctid = selected.ctid RETURNING 1
|
|
12920
13212
|
`;
|
|
12921
13213
|
return { rowsDeleted, materializationsDeleted: removedManifests.length };
|
|
12922
13214
|
}
|
|
@@ -12943,7 +13235,7 @@ class PgOntologySourceStorage {
|
|
|
12943
13235
|
}
|
|
12944
13236
|
const candidate = await this.getRunCandidate(sql, input);
|
|
12945
13237
|
if (candidate) {
|
|
12946
|
-
throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization;
|
|
13238
|
+
throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`);
|
|
12947
13239
|
}
|
|
12948
13240
|
const rows = await sql`
|
|
12949
13241
|
INSERT INTO ontology_sources (
|
|
@@ -12972,7 +13264,7 @@ class PgOntologySourceStorage {
|
|
|
12972
13264
|
return sourceRecord(racedManifest);
|
|
12973
13265
|
}
|
|
12974
13266
|
if (await this.getRunCandidate(sql, input)) {
|
|
12975
|
-
throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization;
|
|
13267
|
+
throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a nonterminal source materialization; adopt it before beginning another.`);
|
|
12976
13268
|
}
|
|
12977
13269
|
throw sourceConflict(`Source materialization '${input.materializationId}' already exists with different identity or state.`);
|
|
12978
13270
|
});
|
|
@@ -12987,10 +13279,10 @@ class PgOntologySourceStorage {
|
|
|
12987
13279
|
throw sourceConflict(`Source materialization '${input.materializationId}' is '${manifest.status}' and cannot accept rows.`);
|
|
12988
13280
|
}
|
|
12989
13281
|
const rows = sourceStageRows(manifest.projection_kind, input.rows);
|
|
12990
|
-
const existing = await this.findStageRows(sql,
|
|
13282
|
+
const existing = await this.findStageRows(sql, manifest, rows);
|
|
12991
13283
|
const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
|
|
12992
13284
|
await stageSourceRoots(sql, manifest, input);
|
|
12993
|
-
await this.insertStageRows(sql,
|
|
13285
|
+
await this.insertStageRows(sql, manifest, pending);
|
|
12994
13286
|
return { inserted: pending.length, unchanged };
|
|
12995
13287
|
});
|
|
12996
13288
|
}
|
|
@@ -13049,13 +13341,43 @@ class PgOntologySourceStorage {
|
|
|
13049
13341
|
return row ? sourceRecord(row) : null;
|
|
13050
13342
|
});
|
|
13051
13343
|
}
|
|
13344
|
+
async adopt(input) {
|
|
13345
|
+
return this.runRootOperation(async (sql) => {
|
|
13346
|
+
assertProjectAndSource(input);
|
|
13347
|
+
assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken);
|
|
13348
|
+
assertTimestamp(input.adoptedAt, "Source adoptedAt", true);
|
|
13349
|
+
await this.assertExecution(sql, input);
|
|
13350
|
+
const candidate = await this.lockRunCandidate(sql, input);
|
|
13351
|
+
if (!candidate)
|
|
13352
|
+
return null;
|
|
13353
|
+
const [adopted] = await sql`
|
|
13354
|
+
UPDATE ontology_sources
|
|
13355
|
+
SET execution_token = ${input.execution.executionToken},
|
|
13356
|
+
updated_at = GREATEST(updated_at, ${input.adoptedAt}::timestamptz)
|
|
13357
|
+
WHERE project_id = ${candidate.project_id} AND source_id = ${candidate.source_id}
|
|
13358
|
+
AND materialization_id = ${candidate.materialization_id}
|
|
13359
|
+
AND status IN ('staging', 'ready')
|
|
13360
|
+
RETURNING *
|
|
13361
|
+
`;
|
|
13362
|
+
if (!adopted) {
|
|
13363
|
+
throw sourceConflict(`Source materialization '${candidate.materialization_id}' changed.`);
|
|
13364
|
+
}
|
|
13365
|
+
return {
|
|
13366
|
+
record: sourceRecord(adopted),
|
|
13367
|
+
resumeStagingOrdinal: adopted.status === "ready" ? Number(adopted.root_count) : await this.lastStagedOrdinal(sql, adopted)
|
|
13368
|
+
};
|
|
13369
|
+
});
|
|
13370
|
+
}
|
|
13052
13371
|
async abandon(input) {
|
|
13053
13372
|
return this.runRootOperation(async (sql) => {
|
|
13054
13373
|
assertProjectAndSource(input);
|
|
13055
13374
|
assertExecutionIdentity(input.execution.projectionRunId, input.execution.executionToken);
|
|
13056
13375
|
assertTimestamp(input.abandonedAt, "Source abandonedAt", true);
|
|
13057
13376
|
await this.assertExecution(sql, input);
|
|
13058
|
-
|
|
13377
|
+
if (input.kind === "candidate")
|
|
13378
|
+
return this.abandonCandidate(sql, input);
|
|
13379
|
+
const candidate = await this.lockRunCandidate(sql, input);
|
|
13380
|
+
return candidate ? this.transitionToAbandoned(sql, candidate, input.abandonedAt) : null;
|
|
13059
13381
|
});
|
|
13060
13382
|
}
|
|
13061
13383
|
async cleanupTerminal(input) {
|
|
@@ -13066,6 +13388,13 @@ class PgOntologySourceStorage {
|
|
|
13066
13388
|
return cleanupSourceVersions(sql, input);
|
|
13067
13389
|
});
|
|
13068
13390
|
}
|
|
13391
|
+
async purgeAbandoned(input) {
|
|
13392
|
+
return this.runRootOperation(async (sql) => {
|
|
13393
|
+
assertNonblank(input.projectId, "Abandoned source purge project id");
|
|
13394
|
+
assertPositiveInteger(input.limit, "Abandoned source purge limit");
|
|
13395
|
+
return purgeAbandonedSourceVersions(sql, input);
|
|
13396
|
+
});
|
|
13397
|
+
}
|
|
13069
13398
|
async summarizeTerminal(input) {
|
|
13070
13399
|
return this.runRootOperation(async (sql) => {
|
|
13071
13400
|
assertNonblank(input.projectId, "Terminal source summary project id");
|
|
@@ -13092,7 +13421,7 @@ class PgOntologySourceStorage {
|
|
|
13092
13421
|
assertSourceCandidateOwner(sourceRecord(manifest), input.execution);
|
|
13093
13422
|
return this.transitionToAbandoned(sql, manifest, input.abandonedAt);
|
|
13094
13423
|
}
|
|
13095
|
-
async
|
|
13424
|
+
async lockRunCandidate(sql, input) {
|
|
13096
13425
|
const candidates = await sql`
|
|
13097
13426
|
SELECT * FROM ontology_sources
|
|
13098
13427
|
WHERE project_id = ${input.projectId}
|
|
@@ -13101,13 +13430,17 @@ class PgOntologySourceStorage {
|
|
|
13101
13430
|
AND status IN ('staging', 'ready')
|
|
13102
13431
|
FOR UPDATE
|
|
13103
13432
|
`;
|
|
13104
|
-
if (candidates.some((candidate) => candidate.execution_token === input.execution.executionToken)) {
|
|
13105
|
-
throw sourceConflict(`Projection run '${input.execution.projectionRunId}' already has a source materialization owned by the current execution.`);
|
|
13106
|
-
}
|
|
13107
13433
|
if (candidates.length > 1) {
|
|
13108
13434
|
throw sourceConflict(`Projection run '${input.execution.projectionRunId}' has multiple nonterminal source materializations.`);
|
|
13109
13435
|
}
|
|
13110
|
-
return candidates[0]
|
|
13436
|
+
return candidates[0] ?? null;
|
|
13437
|
+
}
|
|
13438
|
+
async lastStagedOrdinal(sql, manifest) {
|
|
13439
|
+
const [last] = await sql`
|
|
13440
|
+
SELECT MAX(staging_ordinal) AS ordinal FROM ontology_source_roots
|
|
13441
|
+
WHERE version_id = ${manifest.version_id}
|
|
13442
|
+
`;
|
|
13443
|
+
return last?.ordinal == null ? 0 : Number(last.ordinal);
|
|
13111
13444
|
}
|
|
13112
13445
|
async transitionToAbandoned(sql, manifest, abandonedAt) {
|
|
13113
13446
|
if (abandonedAt < toIsoString2(manifest.created_at) || manifest.ready_at !== null && abandonedAt < toIsoString2(manifest.ready_at)) {
|
|
@@ -13126,22 +13459,18 @@ class PgOntologySourceStorage {
|
|
|
13126
13459
|
if (!rows[0]) {
|
|
13127
13460
|
throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
|
|
13128
13461
|
}
|
|
13129
|
-
await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
|
|
13130
|
-
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
13131
|
-
AND materialization_id = ${manifest.materialization_id} AND NOT active`;
|
|
13132
13462
|
return sourceRecord(rows[0]);
|
|
13133
13463
|
}
|
|
13134
13464
|
async assertReadyState(sql, manifest, rootCount, assertionCount) {
|
|
13465
|
+
const version = manifest.version_id;
|
|
13135
13466
|
const [counts] = await sql`
|
|
13136
|
-
SELECT (SELECT COUNT(*) FROM
|
|
13137
|
-
|
|
13138
|
-
|
|
13139
|
-
COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
13467
|
+
SELECT (SELECT COUNT(*) FROM ontology_source_roots AS roots
|
|
13468
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
13469
|
+
WHERE roots.version_id = ${version}) AS assertions,
|
|
13470
|
+
COUNT(*) AS roots, COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
13140
13471
|
MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
|
|
13141
13472
|
FROM ontology_source_roots
|
|
13142
|
-
WHERE
|
|
13143
|
-
AND source_id = ${manifest.source_id}
|
|
13144
|
-
AND materialization_id = ${manifest.materialization_id}
|
|
13473
|
+
WHERE version_id = ${version}
|
|
13145
13474
|
`;
|
|
13146
13475
|
const assertions = Number(counts?.assertions ?? 0);
|
|
13147
13476
|
const roots = Number(counts?.roots ?? 0);
|
|
@@ -13153,122 +13482,101 @@ class PgOntologySourceStorage {
|
|
|
13153
13482
|
}
|
|
13154
13483
|
await assertSourceRootCoverage(sql, manifest);
|
|
13155
13484
|
const [invalid] = manifest.projection_kind === "link" ? await sql`
|
|
13156
|
-
SELECT root_key
|
|
13157
|
-
FROM
|
|
13158
|
-
|
|
13159
|
-
|
|
13160
|
-
|
|
13161
|
-
GROUP BY root_key
|
|
13485
|
+
SELECT roots.root_key
|
|
13486
|
+
FROM ontology_source_roots AS roots
|
|
13487
|
+
CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
|
|
13488
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
13489
|
+
WHERE roots.version_id = ${version}
|
|
13490
|
+
GROUP BY roots.id, roots.root_key
|
|
13162
13491
|
HAVING COUNT(*) <> 1
|
|
13163
|
-
OR BOOL_OR(
|
|
13164
|
-
OR BOOL_OR(entity_kind <> 'link')
|
|
13165
|
-
OR BOOL_OR(
|
|
13492
|
+
OR BOOL_OR(root.parts->>0 <> 'link')
|
|
13493
|
+
OR BOOL_OR(rows.entity_kind <> 'link')
|
|
13494
|
+
OR BOOL_OR(rows.source_type_id <> root.parts->>1
|
|
13495
|
+
OR rows.source_primary_id <> root.parts->>2 OR rows.link_id <> root.parts->>3
|
|
13496
|
+
OR rows.target_type_id <> root.parts->>4 OR rows.target_primary_id <> root.parts->>5)
|
|
13166
13497
|
LIMIT 1
|
|
13167
13498
|
` : await sql`
|
|
13168
|
-
SELECT root_key
|
|
13169
|
-
FROM
|
|
13170
|
-
|
|
13171
|
-
|
|
13172
|
-
|
|
13173
|
-
GROUP BY root_key
|
|
13174
|
-
HAVING BOOL_OR(
|
|
13499
|
+
SELECT roots.root_key
|
|
13500
|
+
FROM ontology_source_roots AS roots
|
|
13501
|
+
CROSS JOIN LATERAL (SELECT roots.root_key::jsonb AS parts) AS root
|
|
13502
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
13503
|
+
WHERE roots.version_id = ${version}
|
|
13504
|
+
GROUP BY roots.id, roots.root_key
|
|
13505
|
+
HAVING BOOL_OR(root.parts->>0 <> 'object')
|
|
13175
13506
|
OR COUNT(*) FILTER (
|
|
13176
|
-
WHERE entity_kind = 'object' AND
|
|
13507
|
+
WHERE rows.entity_kind = 'object' AND rows.object_type_id = root.parts->>1
|
|
13508
|
+
AND rows.primary_id = root.parts->>2
|
|
13177
13509
|
) <> 1
|
|
13178
13510
|
OR COUNT(*) FILTER (
|
|
13179
|
-
WHERE (entity_kind = 'object' AND
|
|
13180
|
-
|
|
13181
|
-
|
|
13182
|
-
|
|
13183
|
-
))
|
|
13511
|
+
WHERE (rows.entity_kind = 'object' AND (rows.object_type_id <> root.parts->>1
|
|
13512
|
+
OR rows.primary_id <> root.parts->>2))
|
|
13513
|
+
OR (rows.entity_kind = 'link' AND (rows.source_type_id <> root.parts->>1
|
|
13514
|
+
OR rows.source_primary_id <> root.parts->>2))
|
|
13184
13515
|
) <> 0
|
|
13516
|
+
OR COUNT(*) FILTER (WHERE rows.entity_kind = 'link') <> COUNT(DISTINCT (
|
|
13517
|
+
rows.link_id, rows.target_type_id, rows.target_primary_id
|
|
13518
|
+
)) FILTER (WHERE rows.entity_kind = 'link')
|
|
13185
13519
|
LIMIT 1
|
|
13186
13520
|
`;
|
|
13187
13521
|
if (invalid) {
|
|
13188
13522
|
throw new MaterializationValidationError4(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
13189
13523
|
}
|
|
13190
13524
|
}
|
|
13191
|
-
async findStageRows(sql,
|
|
13525
|
+
async findStageRows(sql, manifest, rows) {
|
|
13192
13526
|
if (rows.length === 0)
|
|
13193
13527
|
return [];
|
|
13194
|
-
const
|
|
13528
|
+
const rootKeys = [...new Set(rows.map((row) => row.rootKey))];
|
|
13195
13529
|
const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
|
|
13196
|
-
const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
|
|
13197
13530
|
const existing = await sql`
|
|
13198
|
-
SELECT
|
|
13199
|
-
|
|
13200
|
-
|
|
13201
|
-
|
|
13531
|
+
SELECT ${sourceAssertionColumns(sql)}
|
|
13532
|
+
FROM ontology_source_roots AS roots
|
|
13533
|
+
JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
13534
|
+
JOIN ontology_source_rows AS rows ON rows.root_id = roots.id
|
|
13535
|
+
WHERE roots.version_id = ${manifest.version_id}
|
|
13202
13536
|
AND (
|
|
13203
|
-
|
|
13204
|
-
OR staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
|
|
13205
|
-
OR entity_sort_key = ANY(${sql.array(entitySortKeys)}::text[])
|
|
13537
|
+
roots.root_key = ANY(${sql.array(rootKeys)}::text[])
|
|
13538
|
+
OR roots.staging_ordinal = ANY(${sql.array(ordinals)}::bigint[])
|
|
13206
13539
|
)
|
|
13207
13540
|
`;
|
|
13208
13541
|
return existing.map((row) => sourceStageRow(sourceAssertion(row)));
|
|
13209
13542
|
}
|
|
13210
|
-
async insertStageRows(sql,
|
|
13543
|
+
async insertStageRows(sql, manifest, rows) {
|
|
13211
13544
|
if (rows.length === 0)
|
|
13212
13545
|
return;
|
|
13213
|
-
const payload = rows.map(({
|
|
13546
|
+
const payload = rows.map(({ rootKey, row }) => {
|
|
13214
13547
|
const entity = entityColumns(row.assertion);
|
|
13215
|
-
const root = entityColumns(row.root);
|
|
13216
13548
|
return {
|
|
13217
|
-
|
|
13218
|
-
|
|
13219
|
-
|
|
13220
|
-
|
|
13221
|
-
|
|
13222
|
-
|
|
13223
|
-
|
|
13224
|
-
|
|
13225
|
-
|
|
13226
|
-
|
|
13227
|
-
primaryId: entity.primaryId,
|
|
13228
|
-
sourceTypeId: entity.sourceTypeId,
|
|
13229
|
-
sourcePrimaryId: entity.sourcePrimaryId,
|
|
13230
|
-
linkId: entity.linkId,
|
|
13231
|
-
targetTypeId: entity.targetTypeId,
|
|
13232
|
-
targetPrimaryId: entity.targetPrimaryId,
|
|
13233
|
-
rootObjectTypeId: root.objectTypeId,
|
|
13234
|
-
rootPrimaryId: root.primaryId,
|
|
13235
|
-
rootSourceTypeId: root.sourceTypeId,
|
|
13236
|
-
rootSourcePrimaryId: root.sourcePrimaryId,
|
|
13237
|
-
rootLinkId: root.linkId,
|
|
13238
|
-
rootTargetTypeId: root.targetTypeId,
|
|
13239
|
-
rootTargetPrimaryId: root.targetPrimaryId
|
|
13549
|
+
root_key: rootKey,
|
|
13550
|
+
entity_kind: row.assertion.kind,
|
|
13551
|
+
object_type_id: entity.objectTypeId,
|
|
13552
|
+
primary_id: entity.primaryId,
|
|
13553
|
+
source_type_id: entity.sourceTypeId,
|
|
13554
|
+
source_primary_id: entity.sourcePrimaryId,
|
|
13555
|
+
link_id: entity.linkId,
|
|
13556
|
+
target_type_id: entity.targetTypeId,
|
|
13557
|
+
target_primary_id: entity.targetPrimaryId,
|
|
13558
|
+
payload: sourceAssertionPayload(row.assertion)
|
|
13240
13559
|
};
|
|
13241
13560
|
});
|
|
13242
|
-
await sql`
|
|
13243
|
-
WITH staged AS (
|
|
13244
|
-
SELECT value
|
|
13245
|
-
FROM jsonb_array_elements(${jsonParameter2(sql, payload)}::jsonb)
|
|
13246
|
-
)
|
|
13561
|
+
const inserted = await sql`
|
|
13247
13562
|
INSERT INTO ontology_source_rows (
|
|
13248
|
-
|
|
13249
|
-
|
|
13250
|
-
root_kind, root_key, root_sort_key, staging_ordinal,
|
|
13251
|
-
root, assertion,
|
|
13252
|
-
object_type_id, primary_id,
|
|
13253
|
-
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
|
|
13254
|
-
root_object_type_id, root_primary_id,
|
|
13255
|
-
root_source_type_id, root_source_primary_id, root_link_id,
|
|
13256
|
-
root_target_type_id, root_target_primary_id
|
|
13563
|
+
root_id, entity_kind, object_type_id, primary_id,
|
|
13564
|
+
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id, payload
|
|
13257
13565
|
)
|
|
13258
|
-
SELECT
|
|
13259
|
-
|
|
13260
|
-
|
|
13261
|
-
|
|
13262
|
-
|
|
13263
|
-
|
|
13264
|
-
|
|
13265
|
-
|
|
13266
|
-
|
|
13267
|
-
|
|
13268
|
-
value->>'rootSourceTypeId', value->>'rootSourcePrimaryId', value->>'rootLinkId',
|
|
13269
|
-
value->>'rootTargetTypeId', value->>'rootTargetPrimaryId'
|
|
13270
|
-
FROM staged
|
|
13566
|
+
SELECT roots.id, staged.entity_kind, staged.object_type_id, staged.primary_id,
|
|
13567
|
+
staged.source_type_id, staged.source_primary_id, staged.link_id,
|
|
13568
|
+
staged.target_type_id, staged.target_primary_id, staged.payload
|
|
13569
|
+
FROM jsonb_to_recordset(${jsonParameter2(sql, payload)}) AS staged(
|
|
13570
|
+
root_key TEXT, entity_kind TEXT, object_type_id TEXT, primary_id TEXT,
|
|
13571
|
+
source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
|
|
13572
|
+
target_type_id TEXT, target_primary_id TEXT, payload JSONB
|
|
13573
|
+
)
|
|
13574
|
+
JOIN ontology_source_roots AS roots
|
|
13575
|
+
ON roots.version_id = ${manifest.version_id} AND roots.root_key = staged.root_key
|
|
13271
13576
|
`;
|
|
13577
|
+
if (inserted.count !== rows.length) {
|
|
13578
|
+
throw sourceConflict("Staged source rows do not all belong to a staged root.");
|
|
13579
|
+
}
|
|
13272
13580
|
}
|
|
13273
13581
|
assertExecution(sql, input, identity) {
|
|
13274
13582
|
return assertProjectionExecution(sql, {
|
|
@@ -15545,6 +15853,8 @@ function invalidState(reservation, operation) {
|
|
|
15545
15853
|
}
|
|
15546
15854
|
|
|
15547
15855
|
// src/pg-client.ts
|
|
15856
|
+
import { isIP } from "net";
|
|
15857
|
+
import { connect as connectTls } from "tls";
|
|
15548
15858
|
import postgres from "postgres";
|
|
15549
15859
|
var DEFAULT_IDLE_TIMEOUT_SECONDS = 30;
|
|
15550
15860
|
var DEFAULT_CONNECT_TIMEOUT_SECONDS = 10;
|
|
@@ -15567,10 +15877,10 @@ function createPgClient(options) {
|
|
|
15567
15877
|
connection,
|
|
15568
15878
|
onnotice: () => {},
|
|
15569
15879
|
...options.prepare !== undefined ? { prepare: options.prepare } : {},
|
|
15570
|
-
...options
|
|
15880
|
+
...tlsOptions(options)
|
|
15571
15881
|
};
|
|
15572
15882
|
if (options.connectionString) {
|
|
15573
|
-
return postgres(options.connectionString, base);
|
|
15883
|
+
return assertDirectTlsTarget(postgres(options.connectionString, base), base);
|
|
15574
15884
|
}
|
|
15575
15885
|
return postgres({
|
|
15576
15886
|
...base,
|
|
@@ -15581,6 +15891,78 @@ function createPgClient(options) {
|
|
|
15581
15891
|
...options.password !== undefined ? { password: options.password } : {}
|
|
15582
15892
|
});
|
|
15583
15893
|
}
|
|
15894
|
+
function tlsOptions(options) {
|
|
15895
|
+
const verification = directTlsVerification(options);
|
|
15896
|
+
if (verification === null) {
|
|
15897
|
+
return options.ssl ? { ssl: options.ssl } : {};
|
|
15898
|
+
}
|
|
15899
|
+
return { ssl: false, sslnegotiation: null, socket: directTlsSocket(verification) };
|
|
15900
|
+
}
|
|
15901
|
+
function directTlsVerification(options) {
|
|
15902
|
+
const params = connectionStringParams(options.connectionString);
|
|
15903
|
+
if (params.get("sslnegotiation") !== "direct") {
|
|
15904
|
+
return null;
|
|
15905
|
+
}
|
|
15906
|
+
const sslmode = options.ssl === true ? "verify-full" : options.ssl || (params.get("sslrootcert") === "system" ? "verify-full" : params.get("sslmode") ?? params.get("ssl"));
|
|
15907
|
+
switch (sslmode) {
|
|
15908
|
+
case "require":
|
|
15909
|
+
return "none";
|
|
15910
|
+
case "verify-ca":
|
|
15911
|
+
return "ca";
|
|
15912
|
+
case "verify-full":
|
|
15913
|
+
case "true":
|
|
15914
|
+
return "full";
|
|
15915
|
+
default:
|
|
15916
|
+
throw new Error(`[SixbPg] sslnegotiation=direct requires sslmode=require, verify-ca or verify-full, got ${sslmode ? `"${sslmode}"` : "no sslmode"}: a direct TLS connection has no plaintext fallback.`);
|
|
15917
|
+
}
|
|
15918
|
+
}
|
|
15919
|
+
function connectionStringParams(connectionString) {
|
|
15920
|
+
return new URLSearchParams(connectionString?.split("?")[1] ?? "");
|
|
15921
|
+
}
|
|
15922
|
+
function directTlsSocket(verification) {
|
|
15923
|
+
let reportedRefusal = false;
|
|
15924
|
+
return ({ host: [host = "localhost"], port: [port = 5432] }) => {
|
|
15925
|
+
const socket = connectTls({
|
|
15926
|
+
host,
|
|
15927
|
+
port,
|
|
15928
|
+
servername: isIP(host) ? undefined : host,
|
|
15929
|
+
ALPNProtocols: ["postgresql"],
|
|
15930
|
+
rejectUnauthorized: verification !== "none",
|
|
15931
|
+
...verification === "ca" ? { checkServerIdentity: () => {
|
|
15932
|
+
return;
|
|
15933
|
+
} } : {}
|
|
15934
|
+
});
|
|
15935
|
+
let reachedServer = false;
|
|
15936
|
+
let secured = false;
|
|
15937
|
+
socket.once("connect", () => {
|
|
15938
|
+
reachedServer = true;
|
|
15939
|
+
});
|
|
15940
|
+
socket.once("secureConnect", () => {
|
|
15941
|
+
secured = true;
|
|
15942
|
+
if (socket.alpnProtocol !== "postgresql") {
|
|
15943
|
+
socket.destroy(new Error(`[SixbPg] ${host}:${port} accepted TLS without selecting the "postgresql" protocol` + " (ALPN), so it is not a PostgreSQL server accepting sslnegotiation=direct."));
|
|
15944
|
+
}
|
|
15945
|
+
});
|
|
15946
|
+
socket.once("error", (error) => {
|
|
15947
|
+
if (reportedRefusal || !reachedServer || secured)
|
|
15948
|
+
return;
|
|
15949
|
+
if (!["ECONNRESET", "EPIPE", "ERR_SOCKET_CLOSED"].includes(pgErrorCode(error) ?? ""))
|
|
15950
|
+
return;
|
|
15951
|
+
reportedRefusal = true;
|
|
15952
|
+
console.error(`[SixbPg] ${host}:${port} closed the connection during a direct TLS handshake` + " (sslnegotiation=direct). Direct TLS needs PostgreSQL 17 or later, reached without a" + " proxy or pooler that expects an SSLRequest; remove sslnegotiation=direct otherwise.");
|
|
15953
|
+
});
|
|
15954
|
+
return socket;
|
|
15955
|
+
};
|
|
15956
|
+
}
|
|
15957
|
+
function assertDirectTlsTarget(sql, options) {
|
|
15958
|
+
if (!("socket" in options)) {
|
|
15959
|
+
return sql;
|
|
15960
|
+
}
|
|
15961
|
+
if (sql.options.host.length !== 1 || sql.options.path) {
|
|
15962
|
+
throw new Error("[SixbPg] sslnegotiation=direct supports one TCP host:port in the connection string, not" + " several hosts or a Unix socket path.");
|
|
15963
|
+
}
|
|
15964
|
+
return sql;
|
|
15965
|
+
}
|
|
15584
15966
|
|
|
15585
15967
|
// src/pg-execution-storage.ts
|
|
15586
15968
|
import {
|
|
@@ -15845,13 +16227,19 @@ async function queryLatestRunsByOwnerId(input) {
|
|
|
15845
16227
|
return [];
|
|
15846
16228
|
}
|
|
15847
16229
|
const placeholders = ownerIds.map((_, index) => `$${index + 2}`).join(", ");
|
|
16230
|
+
const parameters = [input.projectId, ...ownerIds];
|
|
16231
|
+
let statusFilter = "";
|
|
16232
|
+
if (input.excludedStatus !== undefined) {
|
|
16233
|
+
parameters.push(input.excludedStatus);
|
|
16234
|
+
statusFilter = ` AND status <> $${parameters.length}`;
|
|
16235
|
+
}
|
|
15848
16236
|
const orderColumn = input.tableName === "sync_runs" || input.tableName === "pipeline_runs" || input.tableName === "projection_runs" ? "COALESCE(started_at, queued_at)" : "started_at";
|
|
15849
16237
|
const rows = await input.sql.unsafe(`
|
|
15850
16238
|
SELECT DISTINCT ON (${input.ownerColumn}) ${input.selectList ?? "*"}
|
|
15851
16239
|
FROM ${input.tableName}
|
|
15852
|
-
WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})
|
|
16240
|
+
WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})${statusFilter}
|
|
15853
16241
|
ORDER BY ${input.ownerColumn}, ${orderColumn} DESC, id DESC
|
|
15854
|
-
`,
|
|
16242
|
+
`, parameters);
|
|
15855
16243
|
const latestByOwnerId = new Map(rows.map((row) => [input.ownerIdFor(row), row]));
|
|
15856
16244
|
return ownerIds.flatMap((ownerId) => {
|
|
15857
16245
|
const row = latestByOwnerId.get(ownerId);
|
|
@@ -16240,12 +16628,14 @@ import {
|
|
|
16240
16628
|
failProjectionRunEnqueue,
|
|
16241
16629
|
immutableDatasetVersionConflict,
|
|
16242
16630
|
mergeProjectionRunProgress,
|
|
16631
|
+
parseProjectionAttemptFailure,
|
|
16243
16632
|
planProjectionRunFinish,
|
|
16244
16633
|
planProjectionRunReclaim,
|
|
16245
16634
|
projectionRunNotFound,
|
|
16246
16635
|
publicProjectionRunRecord,
|
|
16247
16636
|
requireTelemetryProjectionRun,
|
|
16248
16637
|
restoreProjectionRun,
|
|
16638
|
+
SUPERSEDING_PROJECTION_RUN_STATUSES,
|
|
16249
16639
|
staleProjectionRunExecution
|
|
16250
16640
|
} from "@sixb/core/internal/projection-run-storage-provider";
|
|
16251
16641
|
import { PROJECTION_RUN_FAILURE_CODES, ProjectionRunError } from "@sixb/core/storage";
|
|
@@ -16347,8 +16737,7 @@ class PgProjectionRunStorage {
|
|
|
16347
16737
|
const [updated] = await tx`
|
|
16348
16738
|
UPDATE projection_runs
|
|
16349
16739
|
SET status = ${"running"}, started_at = COALESCE(started_at, ${input.startedAt ?? new Date}),
|
|
16350
|
-
finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
|
|
16351
|
-
error = ${null}
|
|
16740
|
+
finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
|
|
16352
16741
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
16353
16742
|
AND (
|
|
16354
16743
|
(status = ${"queued"} AND execution_token IS NULL)
|
|
@@ -16415,6 +16804,24 @@ class PgProjectionRunStorage {
|
|
|
16415
16804
|
return rowToProjectionRunRecord(updated);
|
|
16416
16805
|
});
|
|
16417
16806
|
}
|
|
16807
|
+
async recordAttemptFailure(input) {
|
|
16808
|
+
return runPgTransaction(this.sql, async (tx) => {
|
|
16809
|
+
const existingRow = await requireMaterializationExecution(tx, input);
|
|
16810
|
+
const error = parseProjectionAttemptFailure(restoreProjectionRunRow(existingRow), input.error);
|
|
16811
|
+
const [updated] = await tx`
|
|
16812
|
+
UPDATE projection_runs
|
|
16813
|
+
SET error = ${serializeSixbFailure4(error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
|
|
16814
|
+
WHERE project_id = ${input.projectId}
|
|
16815
|
+
AND id = ${input.id}
|
|
16816
|
+
AND status = ${"running"}
|
|
16817
|
+
AND execution_token = ${input.executionToken}
|
|
16818
|
+
RETURNING *
|
|
16819
|
+
`;
|
|
16820
|
+
if (!updated)
|
|
16821
|
+
throw staleProjectionRunExecution(input.id);
|
|
16822
|
+
return rowToProjectionRunRecord(updated);
|
|
16823
|
+
});
|
|
16824
|
+
}
|
|
16418
16825
|
async finish(input) {
|
|
16419
16826
|
return runPgTransaction(this.sql, async (tx) => {
|
|
16420
16827
|
const existingRow = await requireMaterializationExecution(tx, input);
|
|
@@ -16574,6 +16981,28 @@ class PgProjectionRunStorage {
|
|
|
16574
16981
|
const runs = rows.map(rowToProjectionRunRecord);
|
|
16575
16982
|
return { runs, hasMore: offset + runs.length < total, total };
|
|
16576
16983
|
}
|
|
16984
|
+
async findSupersedingRun(input) {
|
|
16985
|
+
const [run] = await this.sql`
|
|
16986
|
+
SELECT projection_id, materialization_protocol, dataset_version_created_at
|
|
16987
|
+
FROM projection_runs
|
|
16988
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
16989
|
+
`;
|
|
16990
|
+
if (!run)
|
|
16991
|
+
throw projectionRunNotFound(input.projectId, input.id);
|
|
16992
|
+
if (run.materialization_protocol !== "replacement")
|
|
16993
|
+
return null;
|
|
16994
|
+
const [newer] = await this.sql`
|
|
16995
|
+
SELECT * FROM projection_runs
|
|
16996
|
+
WHERE project_id = ${input.projectId}
|
|
16997
|
+
AND projection_id = ${run.projection_id}
|
|
16998
|
+
AND materialization_protocol = ${"replacement"}
|
|
16999
|
+
AND status = ANY(${this.sql.array([...SUPERSEDING_PROJECTION_RUN_STATUSES])}::text[])
|
|
17000
|
+
AND dataset_version_created_at COLLATE "C" > ${run.dataset_version_created_at} COLLATE "C"
|
|
17001
|
+
ORDER BY dataset_version_created_at COLLATE "C" DESC, id DESC
|
|
17002
|
+
LIMIT 1
|
|
17003
|
+
`;
|
|
17004
|
+
return newer ? rowToProjectionRunRecord(newer) : null;
|
|
17005
|
+
}
|
|
16577
17006
|
async listLatestByProjectionIds(input) {
|
|
16578
17007
|
const rows = await queryLatestRunsByOwnerId({
|
|
16579
17008
|
sql: this.sql,
|
|
@@ -16581,7 +17010,8 @@ class PgProjectionRunStorage {
|
|
|
16581
17010
|
ownerColumn: "projection_id",
|
|
16582
17011
|
ownerIds: input.projectionIds,
|
|
16583
17012
|
projectId: input.projectId,
|
|
16584
|
-
ownerIdFor: (row) => row.projection_id
|
|
17013
|
+
ownerIdFor: (row) => row.projection_id,
|
|
17014
|
+
excludedStatus: "superseded"
|
|
16585
17015
|
});
|
|
16586
17016
|
return { runs: rows.map(rowToProjectionRunRecord) };
|
|
16587
17017
|
}
|
|
@@ -18907,4 +19337,4 @@ export {
|
|
|
18907
19337
|
migratePostgresStorage
|
|
18908
19338
|
};
|
|
18909
19339
|
|
|
18910
|
-
//# debugId=
|
|
19340
|
+
//# debugId=4748AE3972DA287E64756E2164756E21
|