@sixb/pg 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +471 -227
- package/dist/index.js.map +13 -11
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +3 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +2 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -0
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts +4 -0
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -0
- package/dist/ontology-storage/source-roots.d.ts +24 -0
- package/dist/ontology-storage/source-roots.d.ts.map +1 -0
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/connector-connection-storage.ts +3 -3
- package/src/migrations/038-outbox-publication-order.sql +5 -0
- package/src/migrations/039-projection-source-roots.sql +46 -0
- package/src/migrations/040-connector-optional-pkce.sql +3 -0
- package/src/migrations.ts +13 -0
- package/src/ontology-storage/materialization-session.ts +24 -8
- package/src/ontology-storage/materialization-state.ts +65 -86
- package/src/ontology-storage/materializations.ts +39 -13
- package/src/ontology-storage/shared.ts +10 -0
- package/src/ontology-storage/source-cleanup.ts +61 -0
- package/src/ontology-storage/source-roots.ts +251 -0
- package/src/ontology-storage/sources.ts +21 -57
- package/src/pg-projection-run-storage.ts +8 -2
package/dist/index.js
CHANGED
|
@@ -3448,7 +3448,7 @@ class PgConnectorConnectionPersistence {
|
|
|
3448
3448
|
) VALUES (
|
|
3449
3449
|
${record.projectId}, ${record.connectorId}, ${record.id}, ${record.slot},
|
|
3450
3450
|
${record.initiatedByExecutionId}, ${record.stateHash},
|
|
3451
|
-
${jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
|
|
3451
|
+
${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
|
|
3452
3452
|
${record.connectionRunId ?? null}, ${record.returnTo ?? null},
|
|
3453
3453
|
${record.callbackBindingHash ?? null}, ${record.reauthorizationId ?? null},
|
|
3454
3454
|
${record.reauthorizationRevision ?? null},
|
|
@@ -3741,7 +3741,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
3741
3741
|
slot: row.slot,
|
|
3742
3742
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
3743
3743
|
stateHash: row.state_hash,
|
|
3744
|
-
codeVerifier: row.code_verifier,
|
|
3744
|
+
...row.code_verifier === null ? {} : { codeVerifier: row.code_verifier },
|
|
3745
3745
|
redirectUri: row.redirect_uri,
|
|
3746
3746
|
...row.connection_run_id === null ? {} : { connectionRunId: row.connection_run_id },
|
|
3747
3747
|
...row.return_to === null ? {} : { returnTo: row.return_to },
|
|
@@ -7081,6 +7081,55 @@ ALTER TABLE executions
|
|
|
7081
7081
|
ON DELETE RESTRICT;
|
|
7082
7082
|
`;
|
|
7083
7083
|
|
|
7084
|
+
// src/migrations/039-projection-source-roots.sql
|
|
7085
|
+
var _039_projection_source_roots_default = `ALTER TABLE ontology_sources ADD COLUMN base_materialization_id TEXT;
|
|
7086
|
+
ALTER TABLE ontology_sources ADD COLUMN base_commit_id TEXT;
|
|
7087
|
+
ALTER TABLE ontology_sources ADD CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL));
|
|
7088
|
+
|
|
7089
|
+
-- A root selects one complete set of assertions. Unchanged roots keep their original version;
|
|
7090
|
+
-- activation only retires/replaces affected references, under the source commit fence.
|
|
7091
|
+
CREATE TABLE ontology_source_roots (
|
|
7092
|
+
project_id TEXT NOT NULL,
|
|
7093
|
+
source_id TEXT NOT NULL,
|
|
7094
|
+
materialization_id TEXT NOT NULL,
|
|
7095
|
+
root_sort_key TEXT NOT NULL,
|
|
7096
|
+
root_kind TEXT NOT NULL CHECK (root_kind IN ('object', 'link')),
|
|
7097
|
+
root_key JSONB NOT NULL,
|
|
7098
|
+
root JSONB NOT NULL,
|
|
7099
|
+
staging_ordinal BIGINT NOT NULL CHECK (staging_ordinal >= 0),
|
|
7100
|
+
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
|
7101
|
+
active BOOLEAN NOT NULL DEFAULT FALSE,
|
|
7102
|
+
retired_at TIMESTAMPTZ,
|
|
7103
|
+
PRIMARY KEY (project_id, source_id, materialization_id, root_sort_key),
|
|
7104
|
+
UNIQUE (project_id, source_id, materialization_id, staging_ordinal),
|
|
7105
|
+
FOREIGN KEY (project_id, source_id, materialization_id)
|
|
7106
|
+
REFERENCES ontology_sources(project_id, source_id, materialization_id) ON DELETE RESTRICT,
|
|
7107
|
+
CHECK (NOT active OR (NOT deleted AND retired_at IS NULL))
|
|
7108
|
+
);
|
|
7109
|
+
CREATE UNIQUE INDEX idx_ontology_source_roots_active
|
|
7110
|
+
ON ontology_source_roots (project_id, source_id, root_sort_key) WHERE active;
|
|
7111
|
+
CREATE INDEX idx_ontology_source_roots_lookup
|
|
7112
|
+
ON ontology_source_roots (project_id, root_sort_key, source_id) WHERE active;
|
|
7113
|
+
CREATE INDEX idx_ontology_source_roots_cleanup
|
|
7114
|
+
ON ontology_source_roots (project_id, retired_at, source_id, materialization_id, root_sort_key)
|
|
7115
|
+
WHERE NOT active AND retired_at IS NOT NULL;
|
|
7116
|
+
|
|
7117
|
+
INSERT INTO ontology_source_roots (
|
|
7118
|
+
project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root,
|
|
7119
|
+
staging_ordinal, active, retired_at
|
|
7120
|
+
)
|
|
7121
|
+
SELECT DISTINCT rows.project_id, rows.source_id, rows.materialization_id,
|
|
7122
|
+
rows.root_sort_key, rows.root_kind, rows.root_key, rows.root, rows.staging_ordinal,
|
|
7123
|
+
sources.status = 'active', sources.terminal_at
|
|
7124
|
+
FROM ontology_source_rows AS rows
|
|
7125
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id);
|
|
7126
|
+
|
|
7127
|
+
ANALYZE ontology_source_roots;
|
|
7128
|
+
|
|
7129
|
+
ALTER TABLE projection_runs ADD COLUMN source_changes_read BIGINT
|
|
7130
|
+
CHECK (source_changes_read IS NULL OR source_changes_read >= 0);
|
|
7131
|
+
`;
|
|
7132
|
+
|
|
7084
7133
|
// src/migrations/037-execution-requester-groups.sql
|
|
7085
7134
|
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
|
|
7086
7135
|
|
|
@@ -7105,6 +7154,20 @@ ALTER TABLE workflow_runs DROP COLUMN requester_group_ids;
|
|
|
7105
7154
|
ALTER TABLE agent_runs DROP COLUMN requester_group_ids;
|
|
7106
7155
|
`;
|
|
7107
7156
|
|
|
7157
|
+
// src/migrations/038-outbox-publication-order.sql
|
|
7158
|
+
var _038_outbox_publication_order_default = `-- Preserve the claim order without sorting every pending event for each batch.
|
|
7159
|
+
-- Keep the availability index for selective delayed/leased workloads.
|
|
7160
|
+
CREATE INDEX idx_ontology_outbox_publication_order
|
|
7161
|
+
ON ontology_outbox (project_id, created_at, commit_id, commit_ordinal)
|
|
7162
|
+
WHERE published_at IS NULL;
|
|
7163
|
+
`;
|
|
7164
|
+
|
|
7165
|
+
// src/migrations/040-connector-optional-pkce.sql
|
|
7166
|
+
var _040_connector_optional_pkce_default = `-- Existing attempts retain their encrypted S256 verifier. NULL denotes disabled PKCE.
|
|
7167
|
+
ALTER TABLE connector_authorization_attempts
|
|
7168
|
+
ALTER COLUMN code_verifier DROP NOT NULL;
|
|
7169
|
+
`;
|
|
7170
|
+
|
|
7108
7171
|
// src/migrations.ts
|
|
7109
7172
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
7110
7173
|
function pgSql(id, sqlText) {
|
|
@@ -7316,7 +7379,10 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
7316
7379
|
pgSql("034-device-authorizations", _034_device_authorizations_default),
|
|
7317
7380
|
pgSql("035-share-grants", _035_share_grants_default),
|
|
7318
7381
|
pgSql("036-share-sessions", _036_share_sessions_default),
|
|
7319
|
-
pgSql("037-execution-requester-groups", _037_execution_requester_groups_default)
|
|
7382
|
+
pgSql("037-execution-requester-groups", _037_execution_requester_groups_default),
|
|
7383
|
+
pgSql("038-outbox-publication-order", _038_outbox_publication_order_default),
|
|
7384
|
+
pgSql("039-projection-source-roots", _039_projection_source_roots_default),
|
|
7385
|
+
pgSql("040-connector-optional-pkce", _040_connector_optional_pkce_default)
|
|
7320
7386
|
]
|
|
7321
7387
|
});
|
|
7322
7388
|
|
|
@@ -9515,6 +9581,12 @@ function sourceRecord(row) {
|
|
|
9515
9581
|
projectionRevision: row.projection_revision,
|
|
9516
9582
|
ownershipHash: row.ownership_hash,
|
|
9517
9583
|
ontologyRevision: row.ontology_revision,
|
|
9584
|
+
...row.base_materialization_id === null ? {} : {
|
|
9585
|
+
base: {
|
|
9586
|
+
materializationId: row.base_materialization_id,
|
|
9587
|
+
lastCommitId: row.base_commit_id
|
|
9588
|
+
}
|
|
9589
|
+
},
|
|
9518
9590
|
rootCount: row.root_count === null ? null : databaseSafeInteger(row.root_count, "Source root count"),
|
|
9519
9591
|
assertionCount: row.assertion_count === null ? null : databaseSafeInteger(row.assertion_count, "Source assertion count"),
|
|
9520
9592
|
createdAt: toIsoString2(row.created_at),
|
|
@@ -9683,7 +9755,7 @@ import {
|
|
|
9683
9755
|
linkRefKey as linkRefKey3,
|
|
9684
9756
|
linkRefSortKey,
|
|
9685
9757
|
linkScopeSortKey as linkScopeSortKey2,
|
|
9686
|
-
MaterializationConflictError as
|
|
9758
|
+
MaterializationConflictError as MaterializationConflictError6,
|
|
9687
9759
|
objectRefKey as objectRefKey3,
|
|
9688
9760
|
objectRefSortKey,
|
|
9689
9761
|
telemetryPointKey as telemetryPointKey3,
|
|
@@ -9702,7 +9774,7 @@ import {
|
|
|
9702
9774
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
9703
9775
|
|
|
9704
9776
|
// src/ontology-storage/materialization-session.ts
|
|
9705
|
-
import { MaterializationConflictError as
|
|
9777
|
+
import { MaterializationConflictError as MaterializationConflictError4 } from "@sixb/core/internal/materialization";
|
|
9706
9778
|
import {
|
|
9707
9779
|
correlateMaterializationChunk,
|
|
9708
9780
|
duplicateMaterializationWork as duplicateWork,
|
|
@@ -9716,9 +9788,9 @@ import {
|
|
|
9716
9788
|
import {
|
|
9717
9789
|
linkRefKey,
|
|
9718
9790
|
linkScopeSortKey,
|
|
9719
|
-
MaterializationConflictError as
|
|
9791
|
+
MaterializationConflictError as MaterializationConflictError3,
|
|
9720
9792
|
objectRefKey,
|
|
9721
|
-
projectionEntityKey,
|
|
9793
|
+
projectionEntityKey as projectionEntityKey2,
|
|
9722
9794
|
telemetryPointKey,
|
|
9723
9795
|
telemetryPointSortKey
|
|
9724
9796
|
} from "@sixb/core/internal/materialization";
|
|
@@ -9728,6 +9800,178 @@ import {
|
|
|
9728
9800
|
linkSlotOverrideValue,
|
|
9729
9801
|
startScopeAccumulator
|
|
9730
9802
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
9803
|
+
|
|
9804
|
+
// src/ontology-storage/source-roots.ts
|
|
9805
|
+
import {
|
|
9806
|
+
MaterializationConflictError as MaterializationConflictError2,
|
|
9807
|
+
MaterializationValidationError,
|
|
9808
|
+
projectionEntityKey
|
|
9809
|
+
} from "@sixb/core/internal/materialization";
|
|
9810
|
+
import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider";
|
|
9811
|
+
function replacementSourceRows(sql, input) {
|
|
9812
|
+
const roots = input.incremental ? sql`
|
|
9813
|
+
SELECT roots.* FROM ontology_source_roots AS changed
|
|
9814
|
+
CROSS JOIN LATERAL (
|
|
9815
|
+
SELECT roots.* FROM ontology_source_roots AS roots
|
|
9816
|
+
WHERE roots.project_id = changed.project_id AND roots.source_id = changed.source_id
|
|
9817
|
+
AND roots.root_sort_key = changed.root_sort_key AND roots.active OFFSET 0
|
|
9818
|
+
) AS roots
|
|
9819
|
+
WHERE changed.project_id = ${input.projectId} AND changed.source_id = ${input.sourceId}
|
|
9820
|
+
AND changed.materialization_id = ${input.materializationId}
|
|
9821
|
+
` : sql`SELECT * FROM ontology_source_roots WHERE project_id = ${input.projectId} AND source_id = ${input.sourceId} AND active`;
|
|
9822
|
+
return sql`
|
|
9823
|
+
SELECT * FROM ontology_source_rows WHERE project_id = ${input.projectId}
|
|
9824
|
+
AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId}
|
|
9825
|
+
UNION ALL
|
|
9826
|
+
SELECT rows.* FROM (${roots}) AS roots
|
|
9827
|
+
CROSS JOIN LATERAL (
|
|
9828
|
+
SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.project_id = roots.project_id
|
|
9829
|
+
AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
|
|
9830
|
+
AND rows.root_sort_key = roots.root_sort_key OFFSET 0
|
|
9831
|
+
) AS rows
|
|
9832
|
+
`;
|
|
9833
|
+
}
|
|
9834
|
+
async function stageSourceRoots(sql, manifest, input) {
|
|
9835
|
+
const roots = sourceStageRoots(manifest.projection_kind, input);
|
|
9836
|
+
if (roots.length === 0)
|
|
9837
|
+
return;
|
|
9838
|
+
if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
|
|
9839
|
+
throw new MaterializationValidationError("Root deletions require a source delta base.");
|
|
9840
|
+
}
|
|
9841
|
+
const values = jsonParameter2(sql, roots.map((root) => ({
|
|
9842
|
+
root_sort_key: root.sortKey,
|
|
9843
|
+
root_kind: root.root.kind,
|
|
9844
|
+
root_key: JSON.parse(root.rootKey),
|
|
9845
|
+
root: root.root,
|
|
9846
|
+
staging_ordinal: root.stagingOrdinal,
|
|
9847
|
+
deleted: root.deleted
|
|
9848
|
+
})));
|
|
9849
|
+
const inserted = await sql`
|
|
9850
|
+
INSERT INTO ontology_source_roots (
|
|
9851
|
+
project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
|
|
9852
|
+
) SELECT ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
|
|
9853
|
+
root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
|
|
9854
|
+
FROM jsonb_to_recordset(${values}) AS input(
|
|
9855
|
+
root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
|
|
9856
|
+
ON CONFLICT DO NOTHING RETURNING root_sort_key
|
|
9857
|
+
`;
|
|
9858
|
+
if (inserted.length === roots.length)
|
|
9859
|
+
return;
|
|
9860
|
+
const insertedKeys = new Set(inserted.map((row) => row.root_sort_key));
|
|
9861
|
+
const skipped = roots.filter((root) => !insertedKeys.has(root.sortKey));
|
|
9862
|
+
const stored = await sql`
|
|
9863
|
+
SELECT root_sort_key, staging_ordinal, deleted FROM ontology_source_roots
|
|
9864
|
+
WHERE project_id = ${input.projectId} AND source_id = ${input.source.projectionId}
|
|
9865
|
+
AND materialization_id = ${input.materializationId}
|
|
9866
|
+
AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
|
|
9867
|
+
`;
|
|
9868
|
+
const existing = new Map(stored.map((root) => [root.root_sort_key, root]));
|
|
9869
|
+
if (skipped.some((root) => {
|
|
9870
|
+
const previous = existing.get(root.sortKey);
|
|
9871
|
+
return !previous || Number(previous.staging_ordinal) !== root.stagingOrdinal || previous.deleted !== root.deleted;
|
|
9872
|
+
}))
|
|
9873
|
+
throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
|
|
9874
|
+
}
|
|
9875
|
+
async function activateSourceRoots(sql, projectId, candidate, activation) {
|
|
9876
|
+
if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
|
|
9877
|
+
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
9878
|
+
if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0)
|
|
9879
|
+
return;
|
|
9880
|
+
const affected = candidate.base_materialization_id === null ? sql`` : sql`AND root_sort_key IN (
|
|
9881
|
+
SELECT root_sort_key FROM ontology_source_roots
|
|
9882
|
+
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
|
|
9883
|
+
AND materialization_id = ${activation.materializationId}
|
|
9884
|
+
)`;
|
|
9885
|
+
await sql`
|
|
9886
|
+
UPDATE ontology_source_roots AS roots SET active = FALSE, retired_at = ${activation.updatedAt}
|
|
9887
|
+
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId} AND active
|
|
9888
|
+
${affected}
|
|
9889
|
+
`;
|
|
9890
|
+
await sql`
|
|
9891
|
+
UPDATE ontology_source_roots SET active = NOT deleted,
|
|
9892
|
+
retired_at = CASE WHEN deleted THEN ${activation.updatedAt}::timestamptz ELSE NULL END
|
|
9893
|
+
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
|
|
9894
|
+
AND materialization_id = ${activation.materializationId}
|
|
9895
|
+
`;
|
|
9896
|
+
}
|
|
9897
|
+
async function assertSourceRootCoverage(sql, manifest) {
|
|
9898
|
+
const [invalid] = await sql`
|
|
9899
|
+
SELECT 1 FROM ontology_source_roots AS roots
|
|
9900
|
+
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
9901
|
+
AND materialization_id = ${manifest.materialization_id}
|
|
9902
|
+
AND roots.deleted = EXISTS (
|
|
9903
|
+
SELECT 1 FROM ontology_source_rows AS rows
|
|
9904
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
9905
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
9906
|
+
) LIMIT 1
|
|
9907
|
+
`;
|
|
9908
|
+
if (invalid)
|
|
9909
|
+
throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
|
|
9910
|
+
}
|
|
9911
|
+
function activeSourceRows(sql, projectId, refs) {
|
|
9912
|
+
const keys = refs.flatMap((entity) => {
|
|
9913
|
+
const entitySortKey = utf8SortKey(projectionEntityKey(entity));
|
|
9914
|
+
const roots = entity.kind === "object" ? [entitySortKey] : [
|
|
9915
|
+
entitySortKey,
|
|
9916
|
+
utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
|
|
9917
|
+
];
|
|
9918
|
+
return roots.map((rootSortKey) => ({
|
|
9919
|
+
root_sort_key: rootSortKey,
|
|
9920
|
+
entity_sort_key: entitySortKey
|
|
9921
|
+
}));
|
|
9922
|
+
});
|
|
9923
|
+
return sql`
|
|
9924
|
+
SELECT rows.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)}) AS requested(root_sort_key TEXT, entity_sort_key TEXT)
|
|
9925
|
+
JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
|
|
9926
|
+
AND roots.active AND roots.root_sort_key = requested.root_sort_key
|
|
9927
|
+
JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
9928
|
+
AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
|
|
9929
|
+
AND rows.root_sort_key = roots.root_sort_key
|
|
9930
|
+
`;
|
|
9931
|
+
}
|
|
9932
|
+
function replacementAssertionRows(sql, input) {
|
|
9933
|
+
const keys = input.refs.map((ref) => ({
|
|
9934
|
+
entity_kind: ref.kind,
|
|
9935
|
+
entity_key: JSON.parse(projectionEntityKey(ref)),
|
|
9936
|
+
root_keys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
|
|
9937
|
+
utf8SortKey(projectionEntityKey(ref)),
|
|
9938
|
+
utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
|
|
9939
|
+
]
|
|
9940
|
+
}));
|
|
9941
|
+
const previous = !input.includePrevious ? sql`` : sql`UNION ALL
|
|
9942
|
+
SELECT rows.* FROM (
|
|
9943
|
+
SELECT roots.* FROM ontology_source_roots AS roots
|
|
9944
|
+
WHERE roots.project_id = ${input.projectId} AND roots.source_id = ${input.sourceId}
|
|
9945
|
+
AND roots.active AND roots.root_sort_key = ANY(requested.root_keys)
|
|
9946
|
+
${!input.incremental ? sql`` : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
|
|
9947
|
+
WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
|
|
9948
|
+
AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`}
|
|
9949
|
+
|
|
9950
|
+
OFFSET 0
|
|
9951
|
+
) AS roots
|
|
9952
|
+
CROSS JOIN LATERAL (
|
|
9953
|
+
SELECT rows.* FROM ontology_source_rows AS rows
|
|
9954
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
9955
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
9956
|
+
AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
9957
|
+
OFFSET 0
|
|
9958
|
+
) AS rows
|
|
9959
|
+
`;
|
|
9960
|
+
return sql`
|
|
9961
|
+
SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)})
|
|
9962
|
+
AS requested(entity_kind TEXT, entity_key JSONB, root_keys TEXT[])
|
|
9963
|
+
CROSS JOIN LATERAL (
|
|
9964
|
+
SELECT rows.* FROM ontology_source_rows AS rows
|
|
9965
|
+
WHERE rows.project_id = ${input.projectId} AND rows.source_id = ${input.sourceId}
|
|
9966
|
+
AND rows.materialization_id = ${input.materializationId}
|
|
9967
|
+
AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
9968
|
+
${previous}
|
|
9969
|
+
OFFSET 0
|
|
9970
|
+
) AS selected
|
|
9971
|
+
`;
|
|
9972
|
+
}
|
|
9973
|
+
|
|
9974
|
+
// src/ontology-storage/materialization-state.ts
|
|
9731
9975
|
var PG_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
|
|
9732
9976
|
var PG_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
|
|
9733
9977
|
|
|
@@ -9760,22 +10004,7 @@ class PgMaterializationStateReader {
|
|
|
9760
10004
|
JOIN requested USING (object_type_id, primary_id)
|
|
9761
10005
|
WHERE objects.project_id = ${this.projectId}
|
|
9762
10006
|
`,
|
|
9763
|
-
this.sql
|
|
9764
|
-
WITH requested AS (
|
|
9765
|
-
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
9766
|
-
AS requested_values(object_type_id TEXT, primary_id TEXT)
|
|
9767
|
-
)
|
|
9768
|
-
SELECT rows.*
|
|
9769
|
-
FROM ontology_source_rows AS rows
|
|
9770
|
-
JOIN requested USING (object_type_id, primary_id)
|
|
9771
|
-
JOIN ontology_sources AS sources
|
|
9772
|
-
ON sources.project_id = rows.project_id
|
|
9773
|
-
AND sources.source_id = rows.source_id
|
|
9774
|
-
AND sources.materialization_id = rows.materialization_id
|
|
9775
|
-
WHERE rows.project_id = ${this.projectId}
|
|
9776
|
-
AND rows.entity_kind = 'object'
|
|
9777
|
-
AND sources.status = 'active'
|
|
9778
|
-
`,
|
|
10007
|
+
activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "object", ref }))),
|
|
9779
10008
|
this.sql`
|
|
9780
10009
|
WITH requested AS (
|
|
9781
10010
|
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
@@ -9816,7 +10045,7 @@ class PgMaterializationStateReader {
|
|
|
9816
10045
|
const effectiveRow = effective.get(key);
|
|
9817
10046
|
return {
|
|
9818
10047
|
ref: structuredClone(ref),
|
|
9819
|
-
source: sourceObject(sources.get(
|
|
10048
|
+
source: sourceObject(sources.get(projectionEntityKey2({ kind: "object", ref }))),
|
|
9820
10049
|
override: overrides.get(key) ?? null,
|
|
9821
10050
|
effective: effectiveRow ? effectiveObjectSnapshot(effectiveRow) : null,
|
|
9822
10051
|
latestTelemetry: telemetry.get(key) ?? []
|
|
@@ -9852,26 +10081,7 @@ class PgMaterializationStateReader {
|
|
|
9852
10081
|
JOIN requested USING (source_type_id, source_id, link_id, target_type_id, target_id)
|
|
9853
10082
|
WHERE links.project_id = ${this.projectId}
|
|
9854
10083
|
`,
|
|
9855
|
-
this.sql
|
|
9856
|
-
WITH requested AS (
|
|
9857
|
-
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
|
|
9858
|
-
source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
|
|
9859
|
-
target_type_id TEXT, target_primary_id TEXT
|
|
9860
|
-
)
|
|
9861
|
-
)
|
|
9862
|
-
SELECT rows.*
|
|
9863
|
-
FROM ontology_source_rows AS rows
|
|
9864
|
-
JOIN requested USING (
|
|
9865
|
-
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
9866
|
-
)
|
|
9867
|
-
JOIN ontology_sources AS sources
|
|
9868
|
-
ON sources.project_id = rows.project_id
|
|
9869
|
-
AND sources.source_id = rows.source_id
|
|
9870
|
-
AND sources.materialization_id = rows.materialization_id
|
|
9871
|
-
WHERE rows.project_id = ${this.projectId}
|
|
9872
|
-
AND rows.entity_kind = 'link'
|
|
9873
|
-
AND sources.status = 'active'
|
|
9874
|
-
`,
|
|
10084
|
+
activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "link", ref }))),
|
|
9875
10085
|
this.sql`
|
|
9876
10086
|
WITH requested AS (
|
|
9877
10087
|
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
|
|
@@ -9910,7 +10120,7 @@ class PgMaterializationStateReader {
|
|
|
9910
10120
|
const effectiveRow = effective.get(key);
|
|
9911
10121
|
return {
|
|
9912
10122
|
ref: structuredClone(ref),
|
|
9913
|
-
source: sourceLink(sources.get(
|
|
10123
|
+
source: sourceLink(sources.get(projectionEntityKey2({ kind: "link", ref }))),
|
|
9914
10124
|
override: overrides.get(key) ?? null,
|
|
9915
10125
|
slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
|
|
9916
10126
|
effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null
|
|
@@ -9979,13 +10189,14 @@ class PgMaterializationStateReader {
|
|
|
9979
10189
|
ON requested.source_type_id = rows.source_type_id
|
|
9980
10190
|
AND requested.source_id = rows.source_primary_id
|
|
9981
10191
|
AND requested.link_id = rows.link_id
|
|
9982
|
-
JOIN
|
|
10192
|
+
JOIN ontology_source_roots AS sources
|
|
9983
10193
|
ON sources.project_id = rows.project_id
|
|
9984
10194
|
AND sources.source_id = rows.source_id
|
|
9985
10195
|
AND sources.materialization_id = rows.materialization_id
|
|
10196
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
9986
10197
|
WHERE rows.project_id = ${this.projectId}
|
|
9987
10198
|
AND rows.entity_kind = 'link'
|
|
9988
|
-
AND sources.
|
|
10199
|
+
AND sources.active
|
|
9989
10200
|
`,
|
|
9990
10201
|
this.sql`
|
|
9991
10202
|
WITH requested AS (
|
|
@@ -10058,7 +10269,7 @@ class PgMaterializationStateReader {
|
|
|
10058
10269
|
for (const row of rows) {
|
|
10059
10270
|
const accumulator = accumulators.get(row.scope_sort_key);
|
|
10060
10271
|
if (!accumulator) {
|
|
10061
|
-
throw new
|
|
10272
|
+
throw new MaterializationConflictError3("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
|
|
10062
10273
|
}
|
|
10063
10274
|
const snapshot = effectiveLinkSnapshot(row);
|
|
10064
10275
|
appendScopeSnapshot(accumulator, snapshot);
|
|
@@ -10127,30 +10338,32 @@ class PgMaterializationStateReader {
|
|
|
10127
10338
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10128
10339
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10129
10340
|
FROM ontology_source_rows AS rows
|
|
10130
|
-
JOIN
|
|
10341
|
+
JOIN ontology_source_roots AS sources
|
|
10131
10342
|
ON sources.project_id = rows.project_id
|
|
10132
10343
|
AND sources.source_id = rows.source_id
|
|
10133
10344
|
AND sources.materialization_id = rows.materialization_id
|
|
10345
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10134
10346
|
JOIN requested
|
|
10135
10347
|
ON requested.object_type_id = rows.source_type_id
|
|
10136
10348
|
AND requested.primary_id = rows.source_primary_id
|
|
10137
10349
|
WHERE rows.project_id = ${this.projectId}
|
|
10138
10350
|
AND rows.entity_kind = 'link'
|
|
10139
|
-
AND sources.
|
|
10351
|
+
AND sources.active
|
|
10140
10352
|
UNION
|
|
10141
10353
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10142
10354
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10143
10355
|
FROM ontology_source_rows AS rows
|
|
10144
|
-
JOIN
|
|
10356
|
+
JOIN ontology_source_roots AS sources
|
|
10145
10357
|
ON sources.project_id = rows.project_id
|
|
10146
10358
|
AND sources.source_id = rows.source_id
|
|
10147
10359
|
AND sources.materialization_id = rows.materialization_id
|
|
10360
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10148
10361
|
JOIN requested
|
|
10149
10362
|
ON requested.object_type_id = rows.target_type_id
|
|
10150
10363
|
AND requested.primary_id = rows.target_primary_id
|
|
10151
10364
|
WHERE rows.project_id = ${this.projectId}
|
|
10152
10365
|
AND rows.entity_kind = 'link'
|
|
10153
|
-
AND sources.
|
|
10366
|
+
AND sources.active
|
|
10154
10367
|
), selected AS (
|
|
10155
10368
|
SELECT DISTINCT links.*,
|
|
10156
10369
|
${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
|
|
@@ -10169,13 +10382,15 @@ class PgMaterializationStateReader {
|
|
|
10169
10382
|
}
|
|
10170
10383
|
async* replacementIdentities(input) {
|
|
10171
10384
|
await this.prepareReplacementIdentities(input);
|
|
10385
|
+
await this.sql`ANALYZE ${this.sql(PG_REPLACEMENT_WORK_TABLE)}`;
|
|
10172
10386
|
let cursor = null;
|
|
10173
10387
|
while (true) {
|
|
10388
|
+
const after = cursor === null ? this.sql`` : this.sql`AND sort_key > ${cursor}`;
|
|
10174
10389
|
const rows = await this.sql`
|
|
10175
10390
|
SELECT entity_kind, identity_key, sort_key, diff_required
|
|
10176
10391
|
FROM ${this.sql(PG_REPLACEMENT_WORK_TABLE)}
|
|
10177
10392
|
WHERE session_id = ${input.sessionId} AND entity_kind = ${input.kind}
|
|
10178
|
-
|
|
10393
|
+
${after}
|
|
10179
10394
|
ORDER BY sort_key
|
|
10180
10395
|
LIMIT ${input.pageRows}
|
|
10181
10396
|
`;
|
|
@@ -10190,19 +10405,19 @@ class PgMaterializationStateReader {
|
|
|
10190
10405
|
const candidates = await this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
|
|
10191
10406
|
return base.map((state) => ({
|
|
10192
10407
|
ref: state.ref,
|
|
10193
|
-
candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(
|
|
10408
|
+
candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey2({ kind: "object", ref: state.ref }))),
|
|
10194
10409
|
override: state.override,
|
|
10195
10410
|
effective: state.effective,
|
|
10196
10411
|
latestTelemetry: state.latestTelemetry
|
|
10197
10412
|
}));
|
|
10198
10413
|
}
|
|
10199
|
-
async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities) {
|
|
10414
|
+
async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities, incremental = false) {
|
|
10200
10415
|
const refs = identities.map((identity) => identity.ref);
|
|
10201
10416
|
const base = await this.linkStates(refs);
|
|
10202
|
-
const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })));
|
|
10417
|
+
const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })), incremental);
|
|
10203
10418
|
const candidate = replacements.byMaterialization.get(candidateMaterializationId);
|
|
10204
10419
|
return base.map((state, index) => {
|
|
10205
|
-
const key =
|
|
10420
|
+
const key = projectionEntityKey2({ kind: "link", ref: state.ref });
|
|
10206
10421
|
return {
|
|
10207
10422
|
ref: state.ref,
|
|
10208
10423
|
candidateSource: replacements.owned.has(key) ? sourceLink(candidate?.get(key)) : state.source,
|
|
@@ -10269,33 +10484,23 @@ class PgMaterializationStateReader {
|
|
|
10269
10484
|
`;
|
|
10270
10485
|
return new Map(rows.map((row) => [linkRefKey(linkRefFromColumns(row)), row.last_commit_id]));
|
|
10271
10486
|
}
|
|
10272
|
-
async replacementSources(sourceId, materializationIds, refs) {
|
|
10487
|
+
async replacementSources(sourceId, materializationIds, refs, incremental = false) {
|
|
10273
10488
|
if (materializationIds.length === 0 || refs.length === 0) {
|
|
10274
10489
|
return { owned: new Set, byMaterialization: new Map };
|
|
10275
10490
|
}
|
|
10276
|
-
const
|
|
10277
|
-
|
|
10278
|
-
|
|
10279
|
-
|
|
10280
|
-
|
|
10281
|
-
|
|
10282
|
-
|
|
10283
|
-
|
|
10284
|
-
AS requested_values(entity_kind TEXT, entity_key JSONB)
|
|
10285
|
-
)
|
|
10286
|
-
SELECT rows.* FROM ontology_source_rows AS rows
|
|
10287
|
-
JOIN requested USING (entity_kind, entity_key)
|
|
10288
|
-
WHERE rows.project_id = ${this.projectId}
|
|
10289
|
-
AND rows.source_id = ${sourceId}
|
|
10290
|
-
AND rows.materialization_id = ANY(
|
|
10291
|
-
${this.sql.array([...new Set(materializationIds)])}::text[]
|
|
10292
|
-
)
|
|
10293
|
-
`;
|
|
10491
|
+
const rows = await replacementAssertionRows(this.sql, {
|
|
10492
|
+
projectId: this.projectId,
|
|
10493
|
+
sourceId,
|
|
10494
|
+
materializationId: materializationIds[0],
|
|
10495
|
+
incremental,
|
|
10496
|
+
includePrevious: materializationIds.length > 1,
|
|
10497
|
+
refs
|
|
10498
|
+
});
|
|
10294
10499
|
const owned = new Set;
|
|
10295
10500
|
const byMaterialization = new Map;
|
|
10296
10501
|
for (const row of rows) {
|
|
10297
10502
|
const source = storedSource(row);
|
|
10298
|
-
const key =
|
|
10503
|
+
const key = projectionEntityKey2(source.assertion);
|
|
10299
10504
|
owned.add(key);
|
|
10300
10505
|
const materialization = byMaterialization.get(row.materialization_id) ?? new Map;
|
|
10301
10506
|
materialization.set(key, source);
|
|
@@ -10311,20 +10516,19 @@ class PgMaterializationStateReader {
|
|
|
10311
10516
|
await this.prepareReplacementLinks(input);
|
|
10312
10517
|
}
|
|
10313
10518
|
async prepareReplacementObjects(input) {
|
|
10314
|
-
const
|
|
10315
|
-
|
|
10316
|
-
|
|
10317
|
-
|
|
10519
|
+
const rows = replacementSourceRows(this.sql, {
|
|
10520
|
+
projectId: this.projectId,
|
|
10521
|
+
sourceId: input.sourceId,
|
|
10522
|
+
materializationId: input.candidateMaterializationId,
|
|
10523
|
+
incremental: input.incremental ?? false
|
|
10524
|
+
});
|
|
10318
10525
|
await this.sql`
|
|
10319
10526
|
WITH selected AS (
|
|
10320
10527
|
SELECT DISTINCT object_type_id, primary_id,
|
|
10321
10528
|
${this.sql.unsafe(objectKeyExpression("rows"))} AS identity_key,
|
|
10322
10529
|
${this.sql.unsafe(objectSortExpression("rows"))} AS sort_key
|
|
10323
|
-
FROM
|
|
10324
|
-
WHERE
|
|
10325
|
-
AND source_id = ${input.sourceId}
|
|
10326
|
-
AND entity_kind = 'object'
|
|
10327
|
-
AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
|
|
10530
|
+
FROM (${rows}) AS rows
|
|
10531
|
+
WHERE entity_kind = 'object'
|
|
10328
10532
|
)
|
|
10329
10533
|
INSERT INTO ${this.sql(PG_REPLACEMENT_WORK_TABLE)} (
|
|
10330
10534
|
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
@@ -10337,10 +10541,13 @@ class PgMaterializationStateReader {
|
|
|
10337
10541
|
`;
|
|
10338
10542
|
}
|
|
10339
10543
|
async prepareReplacementLinks(input) {
|
|
10340
|
-
|
|
10341
|
-
|
|
10342
|
-
|
|
10343
|
-
|
|
10544
|
+
await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
|
|
10545
|
+
const rows = replacementSourceRows(this.sql, {
|
|
10546
|
+
projectId: this.projectId,
|
|
10547
|
+
sourceId: input.sourceId,
|
|
10548
|
+
materializationId: input.candidateMaterializationId,
|
|
10549
|
+
incremental: input.incremental ?? false
|
|
10550
|
+
});
|
|
10344
10551
|
await this.sql`
|
|
10345
10552
|
WITH incident_objects AS (
|
|
10346
10553
|
SELECT DISTINCT payload->'ref'->>'objectTypeId' AS object_type_id,
|
|
@@ -10350,11 +10557,8 @@ class PgMaterializationStateReader {
|
|
|
10350
10557
|
), replacement_links AS (
|
|
10351
10558
|
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
10352
10559
|
target_type_id, target_primary_id AS target_id
|
|
10353
|
-
FROM
|
|
10354
|
-
WHERE
|
|
10355
|
-
AND source_id = ${input.sourceId}
|
|
10356
|
-
AND entity_kind = 'link'
|
|
10357
|
-
AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
|
|
10560
|
+
FROM (${rows}) AS rows
|
|
10561
|
+
WHERE entity_kind = 'link'
|
|
10358
10562
|
), incident_links AS (
|
|
10359
10563
|
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
10360
10564
|
links.target_type_id, links.target_id
|
|
@@ -10393,30 +10597,32 @@ class PgMaterializationStateReader {
|
|
|
10393
10597
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10394
10598
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10395
10599
|
FROM ontology_source_rows AS rows
|
|
10396
|
-
JOIN
|
|
10600
|
+
JOIN ontology_source_roots AS sources
|
|
10397
10601
|
ON sources.project_id = rows.project_id
|
|
10398
10602
|
AND sources.source_id = rows.source_id
|
|
10399
10603
|
AND sources.materialization_id = rows.materialization_id
|
|
10604
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10400
10605
|
JOIN incident_objects
|
|
10401
10606
|
ON incident_objects.object_type_id = rows.source_type_id
|
|
10402
10607
|
AND incident_objects.primary_id = rows.source_primary_id
|
|
10403
10608
|
WHERE rows.project_id = ${this.projectId}
|
|
10404
10609
|
AND rows.entity_kind = 'link'
|
|
10405
|
-
AND sources.
|
|
10610
|
+
AND sources.active
|
|
10406
10611
|
UNION
|
|
10407
10612
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10408
10613
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10409
10614
|
FROM ontology_source_rows AS rows
|
|
10410
|
-
JOIN
|
|
10615
|
+
JOIN ontology_source_roots AS sources
|
|
10411
10616
|
ON sources.project_id = rows.project_id
|
|
10412
10617
|
AND sources.source_id = rows.source_id
|
|
10413
10618
|
AND sources.materialization_id = rows.materialization_id
|
|
10619
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10414
10620
|
JOIN incident_objects
|
|
10415
10621
|
ON incident_objects.object_type_id = rows.target_type_id
|
|
10416
10622
|
AND incident_objects.primary_id = rows.target_primary_id
|
|
10417
10623
|
WHERE rows.project_id = ${this.projectId}
|
|
10418
10624
|
AND rows.entity_kind = 'link'
|
|
10419
|
-
AND sources.
|
|
10625
|
+
AND sources.active
|
|
10420
10626
|
), diff_links AS (
|
|
10421
10627
|
SELECT * FROM replacement_links
|
|
10422
10628
|
UNION SELECT * FROM incident_links
|
|
@@ -10484,9 +10690,9 @@ function activeSourceMap(rows) {
|
|
|
10484
10690
|
const result = new Map;
|
|
10485
10691
|
for (const row of rows) {
|
|
10486
10692
|
const source = storedSource(row);
|
|
10487
|
-
const key =
|
|
10693
|
+
const key = projectionEntityKey2(source.assertion);
|
|
10488
10694
|
if (result.has(key)) {
|
|
10489
|
-
throw new
|
|
10695
|
+
throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert ${key}.`);
|
|
10490
10696
|
}
|
|
10491
10697
|
result.set(key, source);
|
|
10492
10698
|
}
|
|
@@ -10501,7 +10707,7 @@ function activeScopeSourceMap(rows) {
|
|
|
10501
10707
|
const ref = source.assertion.ref;
|
|
10502
10708
|
const key = linkScopeSortKey(ref.source, ref.linkId);
|
|
10503
10709
|
if (result.has(key)) {
|
|
10504
|
-
throw new
|
|
10710
|
+
throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
|
|
10505
10711
|
}
|
|
10506
10712
|
result.set(key, source);
|
|
10507
10713
|
}
|
|
@@ -10565,7 +10771,7 @@ function storedSource(row) {
|
|
|
10565
10771
|
}
|
|
10566
10772
|
function effectiveObjectSnapshot(row) {
|
|
10567
10773
|
if (!row.last_commit_id) {
|
|
10568
|
-
throw new
|
|
10774
|
+
throw new MaterializationConflictError3("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
|
|
10569
10775
|
}
|
|
10570
10776
|
return {
|
|
10571
10777
|
ref: objectRefFromColumns(row),
|
|
@@ -10578,7 +10784,7 @@ function effectiveObjectSnapshot(row) {
|
|
|
10578
10784
|
}
|
|
10579
10785
|
function effectiveLinkSnapshot(row) {
|
|
10580
10786
|
if (!row.last_commit_id) {
|
|
10581
|
-
throw new
|
|
10787
|
+
throw new MaterializationConflictError3("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
|
|
10582
10788
|
}
|
|
10583
10789
|
return {
|
|
10584
10790
|
ref: linkRefFromColumns(row),
|
|
@@ -10593,7 +10799,7 @@ function effectiveLinkSnapshot(row) {
|
|
|
10593
10799
|
function storedPoint(row) {
|
|
10594
10800
|
const at = toIsoString2(row.at);
|
|
10595
10801
|
if (!row.last_commit_id) {
|
|
10596
|
-
throw new
|
|
10802
|
+
throw new MaterializationConflictError3("timeseries-point", `Telemetry point ${telemetryPointSortKey({
|
|
10597
10803
|
object: { objectTypeId: row.object_type_id, primaryId: row.object_id },
|
|
10598
10804
|
propertyId: row.property_id
|
|
10599
10805
|
}, at)} lacks materializer provenance.`);
|
|
@@ -10652,6 +10858,9 @@ function jsonTupleExpression(parts) {
|
|
|
10652
10858
|
|
|
10653
10859
|
// src/ontology-storage/materialization-session.ts
|
|
10654
10860
|
class PgMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
10861
|
+
stagedWorkCount = 0;
|
|
10862
|
+
changedObjects = 0;
|
|
10863
|
+
changedLinks = 0;
|
|
10655
10864
|
appliedPlanCursor = null;
|
|
10656
10865
|
appliedEventCursor = null;
|
|
10657
10866
|
}
|
|
@@ -10668,7 +10877,7 @@ class PgMaterializationSessions {
|
|
|
10668
10877
|
}
|
|
10669
10878
|
async create(header) {
|
|
10670
10879
|
if (!this.context?.active) {
|
|
10671
|
-
throw new
|
|
10880
|
+
throw new MaterializationConflictError4("effective-state", "Materialization sessions require an active storage transaction.");
|
|
10672
10881
|
}
|
|
10673
10882
|
await this.ensureTables();
|
|
10674
10883
|
const session = new PgMaterializationSessionState(structuredClone(header), this.context.id);
|
|
@@ -10680,7 +10889,7 @@ class PgMaterializationSessions {
|
|
|
10680
10889
|
require(session) {
|
|
10681
10890
|
const value = this.sessions.get(session.providerToken);
|
|
10682
10891
|
if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
|
|
10683
|
-
throw new
|
|
10892
|
+
throw new MaterializationConflictError4("effective-state", "Materialization session is inactive.");
|
|
10684
10893
|
}
|
|
10685
10894
|
return value;
|
|
10686
10895
|
}
|
|
@@ -10738,14 +10947,18 @@ class PgMaterializationSessions {
|
|
|
10738
10947
|
}
|
|
10739
10948
|
throw error;
|
|
10740
10949
|
}
|
|
10950
|
+
session.stagedWorkCount += records.length;
|
|
10741
10951
|
}
|
|
10742
10952
|
async* stream(input) {
|
|
10743
10953
|
const session = this.require(input.session);
|
|
10744
10954
|
const stream = session.workStreams[input.order];
|
|
10745
10955
|
if (stream.started) {
|
|
10746
|
-
throw new
|
|
10956
|
+
throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
|
|
10747
10957
|
}
|
|
10748
10958
|
stream.started = true;
|
|
10959
|
+
if (!session.workSealed && session.stagedWorkCount >= 1e4) {
|
|
10960
|
+
await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
|
|
10961
|
+
}
|
|
10749
10962
|
session.workSealed = true;
|
|
10750
10963
|
let cursor = null;
|
|
10751
10964
|
while (true) {
|
|
@@ -10766,10 +10979,13 @@ class PgMaterializationSessions {
|
|
|
10766
10979
|
return [];
|
|
10767
10980
|
const keys = refs.map((ref) => `object-existence:${JSON.stringify([ref.objectTypeId, ref.primaryId])}`);
|
|
10768
10981
|
const rows = await this.sql`
|
|
10769
|
-
SELECT unique_key, payload
|
|
10770
|
-
|
|
10771
|
-
|
|
10772
|
-
|
|
10982
|
+
SELECT work.unique_key, work.payload
|
|
10983
|
+
FROM unnest(${this.sql.array(keys)}::text[]) AS requested(unique_key)
|
|
10984
|
+
CROSS JOIN LATERAL (
|
|
10985
|
+
SELECT unique_key, payload FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
|
|
10986
|
+
WHERE session_id = ${session.id} AND unique_key = requested.unique_key
|
|
10987
|
+
OFFSET 0
|
|
10988
|
+
) AS work
|
|
10773
10989
|
`;
|
|
10774
10990
|
const found = new Map(rows.map((row) => [row.unique_key, row.payload]));
|
|
10775
10991
|
return keys.flatMap((key) => {
|
|
@@ -10777,6 +10993,9 @@ class PgMaterializationSessions {
|
|
|
10777
10993
|
if (!payload)
|
|
10778
10994
|
return [];
|
|
10779
10995
|
const record = structuredClone(payload);
|
|
10996
|
+
if (record.kind !== "object-existence") {
|
|
10997
|
+
invalidCorrelation("Object existence work has an invalid kind.");
|
|
10998
|
+
}
|
|
10780
10999
|
return [{ ref: record.ref, exists: record.exists }];
|
|
10781
11000
|
});
|
|
10782
11001
|
}
|
|
@@ -10931,7 +11150,7 @@ function workCursor(row) {
|
|
|
10931
11150
|
// src/ontology-storage/materialization-writer.ts
|
|
10932
11151
|
import {
|
|
10933
11152
|
linkRefKey as linkRefKey2,
|
|
10934
|
-
MaterializationConflictError as
|
|
11153
|
+
MaterializationConflictError as MaterializationConflictError5,
|
|
10935
11154
|
objectRefKey as objectRefKey2,
|
|
10936
11155
|
telemetryPointKey as telemetryPointKey2
|
|
10937
11156
|
} from "@sixb/core/internal/materialization";
|
|
@@ -11447,7 +11666,7 @@ function effectiveLinkConflict(item) {
|
|
|
11447
11666
|
return effectiveConflict(`Expected link ${linkIdentityKey(item)} changed.`);
|
|
11448
11667
|
}
|
|
11449
11668
|
function pointConflict(item) {
|
|
11450
|
-
return new
|
|
11669
|
+
return new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey2({
|
|
11451
11670
|
object: { objectTypeId: item.objectTypeId, primaryId: item.objectId },
|
|
11452
11671
|
propertyId: item.propertyId
|
|
11453
11672
|
}, item.at)} changed.`);
|
|
@@ -11482,14 +11701,14 @@ class PgOntologyMaterializationStorage {
|
|
|
11482
11701
|
const linkScopeRevisions = await reader.linkScopeRevisions(input.expected.linkScopes);
|
|
11483
11702
|
for (const [index, expected] of input.expected.linkScopes.entries()) {
|
|
11484
11703
|
if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
|
|
11485
|
-
throw new
|
|
11704
|
+
throw new MaterializationConflictError6("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
|
|
11486
11705
|
}
|
|
11487
11706
|
}
|
|
11488
11707
|
const points = await reader.exactPoints(input.expected.points, true);
|
|
11489
11708
|
const pointRevisions = new Map(points.map((point) => [telemetryPointKey3(point.series, point.at), point.lastCommitId]));
|
|
11490
11709
|
for (const expected of input.expected.points) {
|
|
11491
11710
|
if ((pointRevisions.get(telemetryPointKey3(expected.series, expected.at)) ?? null) !== expected.lastCommitId) {
|
|
11492
|
-
throw new
|
|
11711
|
+
throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey3(expected.series, expected.at)} changed.`);
|
|
11493
11712
|
}
|
|
11494
11713
|
}
|
|
11495
11714
|
return session.publicSession();
|
|
@@ -11548,10 +11767,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11548
11767
|
const reader = new PgMaterializationStateReader(this.sql, session.header.commit.projectId);
|
|
11549
11768
|
if (input.entityKind === "object") {
|
|
11550
11769
|
if (replacement.projectionKind !== "object") {
|
|
11551
|
-
throw new
|
|
11770
|
+
throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
|
|
11552
11771
|
}
|
|
11553
11772
|
if (replacement.objectStreamStarted) {
|
|
11554
|
-
throw new
|
|
11773
|
+
throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
|
|
11555
11774
|
}
|
|
11556
11775
|
replacement.objectStreamStarted = true;
|
|
11557
11776
|
for await (const identities of reader.replacementIdentities({
|
|
@@ -11559,6 +11778,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11559
11778
|
sourceId: replacement.sourceId,
|
|
11560
11779
|
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
11561
11780
|
previousMaterializationId: replacement.previousMaterializationId,
|
|
11781
|
+
incremental: replacement.incremental,
|
|
11562
11782
|
kind: "object",
|
|
11563
11783
|
pageRows: input.pageRows
|
|
11564
11784
|
})) {
|
|
@@ -11577,10 +11797,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11577
11797
|
return;
|
|
11578
11798
|
}
|
|
11579
11799
|
if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
|
|
11580
|
-
throw new
|
|
11800
|
+
throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
|
|
11581
11801
|
}
|
|
11582
11802
|
if (replacement.linkStreamStarted) {
|
|
11583
|
-
throw new
|
|
11803
|
+
throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
|
|
11584
11804
|
}
|
|
11585
11805
|
replacement.linkStreamStarted = true;
|
|
11586
11806
|
const materializationIds = [
|
|
@@ -11592,6 +11812,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11592
11812
|
sourceId: replacement.sourceId,
|
|
11593
11813
|
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
11594
11814
|
previousMaterializationId: replacement.previousMaterializationId,
|
|
11815
|
+
incremental: replacement.incremental,
|
|
11595
11816
|
kind: "link",
|
|
11596
11817
|
pageRows: input.pageRows
|
|
11597
11818
|
})) {
|
|
@@ -11602,7 +11823,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11602
11823
|
}
|
|
11603
11824
|
return identity;
|
|
11604
11825
|
});
|
|
11605
|
-
const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities);
|
|
11826
|
+
const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
|
|
11606
11827
|
yield { objects: [], links };
|
|
11607
11828
|
}
|
|
11608
11829
|
this.sessions.require(input.session);
|
|
@@ -11625,6 +11846,8 @@ class PgOntologyMaterializationStorage {
|
|
|
11625
11846
|
assertPlanChunkCorrelations(input.chunk, commit);
|
|
11626
11847
|
const progress = await this.sessions.prepareChunkSequence(session, input.chunk);
|
|
11627
11848
|
await this.writer.apply(commit.projectId, commit.id, input.chunk);
|
|
11849
|
+
session.changedObjects += input.chunk.effective.objectUpserts.length + input.chunk.effective.objectDeletes.length;
|
|
11850
|
+
session.changedLinks += input.chunk.effective.linkUpserts.length + input.chunk.effective.linkDeletes.length;
|
|
11628
11851
|
this.sessions.commitChunkSequence(session, progress);
|
|
11629
11852
|
}
|
|
11630
11853
|
async finalize(input) {
|
|
@@ -11634,6 +11857,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11634
11857
|
for (const activation of input.finalization.sourceActivations) {
|
|
11635
11858
|
await this.activateSource(session, activation);
|
|
11636
11859
|
}
|
|
11860
|
+
if (session.changedObjects >= 1000)
|
|
11861
|
+
await this.sql`ANALYZE objects`;
|
|
11862
|
+
if (session.changedLinks >= 1000)
|
|
11863
|
+
await this.sql`ANALYZE links`;
|
|
11637
11864
|
const record = await this.insertCommit(session.header, input);
|
|
11638
11865
|
await this.sessions.release(session);
|
|
11639
11866
|
return { commit: record };
|
|
@@ -11644,14 +11871,14 @@ class PgOntologyMaterializationStorage {
|
|
|
11644
11871
|
async requireReplacement(session, input) {
|
|
11645
11872
|
if (session.replacement) {
|
|
11646
11873
|
if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
|
|
11647
|
-
throw new
|
|
11874
|
+
throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
|
|
11648
11875
|
}
|
|
11649
11876
|
return session.replacement;
|
|
11650
11877
|
}
|
|
11651
11878
|
const projectId = session.header.commit.projectId;
|
|
11652
11879
|
const candidate = await this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId, true);
|
|
11653
11880
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
11654
|
-
throw new
|
|
11881
|
+
throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
|
|
11655
11882
|
}
|
|
11656
11883
|
await assertProjectionExecution(this.sql, {
|
|
11657
11884
|
projectId,
|
|
@@ -11661,12 +11888,13 @@ class PgOntologyMaterializationStorage {
|
|
|
11661
11888
|
});
|
|
11662
11889
|
const previous = await this.getActiveSource(projectId, input.source.projectionId, true);
|
|
11663
11890
|
if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
|
|
11664
|
-
throw new
|
|
11891
|
+
throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
|
|
11665
11892
|
}
|
|
11666
11893
|
session.replacement = {
|
|
11667
11894
|
sourceId: input.source.projectionId,
|
|
11668
11895
|
candidateMaterializationId: input.candidateMaterializationId,
|
|
11669
11896
|
previousMaterializationId: previous?.materialization_id ?? null,
|
|
11897
|
+
incremental: candidate.base_materialization_id !== null,
|
|
11670
11898
|
projectionKind: candidate.projection_kind,
|
|
11671
11899
|
objectStreamStarted: false,
|
|
11672
11900
|
objectStreamCompleted: false,
|
|
@@ -11686,7 +11914,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11686
11914
|
FOR UPDATE
|
|
11687
11915
|
`;
|
|
11688
11916
|
if (duplicateIdentity) {
|
|
11689
|
-
throw new
|
|
11917
|
+
throw new MaterializationConflictError6("idempotency", duplicateIdentity.id === header.commit.id ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
|
|
11690
11918
|
}
|
|
11691
11919
|
const origin = originColumns(header.commit.origin);
|
|
11692
11920
|
if (origin.runId === null)
|
|
@@ -11709,7 +11937,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11709
11937
|
FOR UPDATE
|
|
11710
11938
|
`;
|
|
11711
11939
|
if (duplicateOrigin) {
|
|
11712
|
-
throw new
|
|
11940
|
+
throw new MaterializationConflictError6("run-correlation", "Ontology commit origin already has an authoritative commit.");
|
|
11713
11941
|
}
|
|
11714
11942
|
}
|
|
11715
11943
|
async assertSources(expectedSources, projectId) {
|
|
@@ -11731,29 +11959,29 @@ class PgOntologyMaterializationStorage {
|
|
|
11731
11959
|
}
|
|
11732
11960
|
assertSourceRow(expected, active) {
|
|
11733
11961
|
if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
|
|
11734
|
-
throw new
|
|
11962
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${expected.source.projectionId}' changed.`);
|
|
11735
11963
|
}
|
|
11736
11964
|
}
|
|
11737
11965
|
assertObject(row, expected) {
|
|
11738
11966
|
if (!expected.exists) {
|
|
11739
11967
|
if (row) {
|
|
11740
|
-
throw new
|
|
11968
|
+
throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
|
|
11741
11969
|
}
|
|
11742
11970
|
return;
|
|
11743
11971
|
}
|
|
11744
11972
|
if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
|
|
11745
|
-
throw new
|
|
11973
|
+
throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
|
|
11746
11974
|
}
|
|
11747
11975
|
}
|
|
11748
11976
|
assertLink(lastCommitId, expected) {
|
|
11749
11977
|
if (!expected.exists) {
|
|
11750
11978
|
if (lastCommitId !== undefined) {
|
|
11751
|
-
throw new
|
|
11979
|
+
throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
|
|
11752
11980
|
}
|
|
11753
11981
|
return;
|
|
11754
11982
|
}
|
|
11755
11983
|
if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
|
|
11756
|
-
throw new
|
|
11984
|
+
throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
|
|
11757
11985
|
}
|
|
11758
11986
|
}
|
|
11759
11987
|
async assertFinalization(session, input) {
|
|
@@ -11762,7 +11990,8 @@ class PgOntologyMaterializationStorage {
|
|
|
11762
11990
|
assertMaterializationFinalizationCorrelation(session, input);
|
|
11763
11991
|
const laneCounts = await this.sessions.laneCounts(session);
|
|
11764
11992
|
assertMaterializationLaneCompletion(session, laneCounts);
|
|
11765
|
-
|
|
11993
|
+
if (laneCounts.cardinality > 0)
|
|
11994
|
+
await this.assertFinalCardinality(session);
|
|
11766
11995
|
const eventCount = laneCounts.event;
|
|
11767
11996
|
const [outbox] = await this.sql`
|
|
11768
11997
|
SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum,
|
|
@@ -11795,17 +12024,18 @@ class PgOntologyMaterializationStorage {
|
|
|
11795
12024
|
}
|
|
11796
12025
|
}
|
|
11797
12026
|
async assertFinalCardinality(session) {
|
|
11798
|
-
const
|
|
11799
|
-
|
|
11800
|
-
SELECT
|
|
11801
|
-
payload->>'linkSortKey' AS link_sort_key,
|
|
12027
|
+
const table = `ontology_cardinality_${session.id.replaceAll("-", "")}`;
|
|
12028
|
+
await this.sql.unsafe(`CREATE TEMP TABLE ${table} ON COMMIT DROP AS
|
|
12029
|
+
SELECT sort_one AS scope_sort_key, sort_two AS link_sort_key,
|
|
11802
12030
|
(payload->>'occupied')::boolean AS occupied,
|
|
11803
12031
|
payload->'ref'->'source'->>'objectTypeId' AS source_type_id,
|
|
11804
12032
|
payload->'ref'->'source'->>'primaryId' AS source_id,
|
|
11805
12033
|
payload->'ref'->>'linkId' AS link_id
|
|
11806
|
-
FROM ${
|
|
11807
|
-
WHERE session_id = $
|
|
11808
|
-
|
|
12034
|
+
FROM ${PG_MATERIALIZATION_WORK_TABLE}
|
|
12035
|
+
WHERE session_id = $1 AND lane = 'cardinality' AND major_order = 0`, [session.id]);
|
|
12036
|
+
await this.sql.unsafe(`ANALYZE ${table}`);
|
|
12037
|
+
const [violation] = await this.sql.unsafe(`WITH work AS (
|
|
12038
|
+
SELECT * FROM ${table}
|
|
11809
12039
|
), duplicate AS (
|
|
11810
12040
|
SELECT scope_sort_key FROM work WHERE occupied
|
|
11811
12041
|
GROUP BY scope_sort_key HAVING COUNT(*) > 1
|
|
@@ -11815,10 +12045,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11815
12045
|
SELECT scope_sort_key, link_sort_key FROM work WHERE occupied
|
|
11816
12046
|
), actual AS (
|
|
11817
12047
|
SELECT scopes.scope_sort_key,
|
|
11818
|
-
${
|
|
12048
|
+
${linkSortExpression("links")} AS link_sort_key
|
|
11819
12049
|
FROM scopes
|
|
11820
12050
|
JOIN links USING (source_type_id, source_id, link_id)
|
|
11821
|
-
WHERE links.project_id = $
|
|
12051
|
+
WHERE links.project_id = $1
|
|
11822
12052
|
), differences AS (
|
|
11823
12053
|
(SELECT * FROM expected EXCEPT SELECT * FROM actual)
|
|
11824
12054
|
UNION ALL
|
|
@@ -11827,8 +12057,8 @@ class PgOntologyMaterializationStorage {
|
|
|
11827
12057
|
SELECT 'duplicate'::text AS reason FROM duplicate
|
|
11828
12058
|
UNION ALL
|
|
11829
12059
|
SELECT 'mismatch'::text AS reason FROM differences
|
|
11830
|
-
LIMIT 1
|
|
11831
|
-
|
|
12060
|
+
LIMIT 1`, [session.header.commit.projectId]);
|
|
12061
|
+
await this.sql.unsafe(`DROP TABLE ${table}`);
|
|
11832
12062
|
if (violation?.reason === "duplicate") {
|
|
11833
12063
|
invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
|
|
11834
12064
|
}
|
|
@@ -11924,7 +12154,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11924
12154
|
assertSourceActivationCorrelation(session, activation);
|
|
11925
12155
|
const candidate = await this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId, true);
|
|
11926
12156
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
11927
|
-
throw new
|
|
12157
|
+
throw new MaterializationConflictError6("source-materialization", "Source activation candidate is missing or is not ready.");
|
|
11928
12158
|
}
|
|
11929
12159
|
await assertProjectionExecution(this.sql, {
|
|
11930
12160
|
projectId: commit.projectId,
|
|
@@ -11937,6 +12167,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11937
12167
|
}
|
|
11938
12168
|
const previous = await this.getActiveSource(commit.projectId, activation.source.projectionId, true);
|
|
11939
12169
|
this.assertSourceRow(activation.expected, previous);
|
|
12170
|
+
await activateSourceRoots(this.sql, commit.projectId, candidate, activation);
|
|
11940
12171
|
if (previous) {
|
|
11941
12172
|
assertPinnedDatasetWatermark({
|
|
11942
12173
|
datasetId: previous.dataset_id,
|
|
@@ -11958,7 +12189,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11958
12189
|
RETURNING materialization_id
|
|
11959
12190
|
`;
|
|
11960
12191
|
if (superseded.length !== 1) {
|
|
11961
|
-
throw new
|
|
12192
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
|
|
11962
12193
|
}
|
|
11963
12194
|
}
|
|
11964
12195
|
let activated;
|
|
@@ -11977,12 +12208,12 @@ class PgOntologyMaterializationStorage {
|
|
|
11977
12208
|
`;
|
|
11978
12209
|
} catch (error) {
|
|
11979
12210
|
if (isUniqueViolation(error)) {
|
|
11980
|
-
throw new
|
|
12211
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
|
|
11981
12212
|
}
|
|
11982
12213
|
throw error;
|
|
11983
12214
|
}
|
|
11984
12215
|
if (activated.length !== 1) {
|
|
11985
|
-
throw new
|
|
12216
|
+
throw new MaterializationConflictError6("source-materialization", "Source activation candidate changed.");
|
|
11986
12217
|
}
|
|
11987
12218
|
}
|
|
11988
12219
|
async insertCommit(header, input) {
|
|
@@ -12009,7 +12240,7 @@ class PgOntologyMaterializationStorage {
|
|
|
12009
12240
|
`;
|
|
12010
12241
|
if (!rows[0]) {
|
|
12011
12242
|
await this.assertCommitAbsent(header);
|
|
12012
|
-
throw new
|
|
12243
|
+
throw new MaterializationConflictError6("idempotency", "Ontology commit identity already exists.");
|
|
12013
12244
|
}
|
|
12014
12245
|
return commitRecord(rows[0]);
|
|
12015
12246
|
}
|
|
@@ -12057,8 +12288,8 @@ function databaseCount(value) {
|
|
|
12057
12288
|
// src/ontology-storage/outbox.ts
|
|
12058
12289
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
12059
12290
|
import {
|
|
12060
|
-
MaterializationConflictError as
|
|
12061
|
-
MaterializationValidationError
|
|
12291
|
+
MaterializationConflictError as MaterializationConflictError7,
|
|
12292
|
+
MaterializationValidationError as MaterializationValidationError2
|
|
12062
12293
|
} from "@sixb/core/internal/materialization";
|
|
12063
12294
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
|
|
12064
12295
|
class PgOntologyOutboxStorage {
|
|
@@ -12074,7 +12305,7 @@ class PgOntologyOutboxStorage {
|
|
|
12074
12305
|
const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
|
|
12075
12306
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
12076
12307
|
if (leaseExpiry <= now) {
|
|
12077
|
-
throw new
|
|
12308
|
+
throw new MaterializationValidationError2("Ontology outbox lease expiry must be later than the claim time.");
|
|
12078
12309
|
}
|
|
12079
12310
|
const rows = await sql`
|
|
12080
12311
|
WITH candidates AS (
|
|
@@ -12104,7 +12335,7 @@ class PgOntologyOutboxStorage {
|
|
|
12104
12335
|
return rows.map((row) => {
|
|
12105
12336
|
const record = outboxRecord(row);
|
|
12106
12337
|
if (record.leaseId === null || record.leaseExpiresAt === null) {
|
|
12107
|
-
throw new
|
|
12338
|
+
throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
|
|
12108
12339
|
}
|
|
12109
12340
|
return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt };
|
|
12110
12341
|
});
|
|
@@ -12225,7 +12456,7 @@ function validateLeaseIds(ids) {
|
|
|
12225
12456
|
for (const id of ids) {
|
|
12226
12457
|
assertNonblank(id, "Ontology outbox event id");
|
|
12227
12458
|
if (seen.has(id)) {
|
|
12228
|
-
throw new
|
|
12459
|
+
throw new MaterializationValidationError2(`Ontology outbox lease batch repeats event '${id}'.`);
|
|
12229
12460
|
}
|
|
12230
12461
|
seen.add(id);
|
|
12231
12462
|
}
|
|
@@ -12236,11 +12467,11 @@ function assertLeaseInput(input) {
|
|
|
12236
12467
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
12237
12468
|
}
|
|
12238
12469
|
function leaseConflict() {
|
|
12239
|
-
return new
|
|
12470
|
+
return new MaterializationConflictError7("outbox-lease", "Ontology outbox lease does not match.");
|
|
12240
12471
|
}
|
|
12241
12472
|
|
|
12242
12473
|
// src/ontology-storage/sources.ts
|
|
12243
|
-
import { MaterializationValidationError as
|
|
12474
|
+
import { MaterializationValidationError as MaterializationValidationError3 } from "@sixb/core/internal/materialization";
|
|
12244
12475
|
import {
|
|
12245
12476
|
assertSourceBeginInput as assertBeginInput,
|
|
12246
12477
|
assertSourceExecutionIdentity as assertExecutionIdentity,
|
|
@@ -12254,8 +12485,61 @@ import {
|
|
|
12254
12485
|
sourceMaterializationIdentity,
|
|
12255
12486
|
sourceStageRow,
|
|
12256
12487
|
sourceStageRows,
|
|
12257
|
-
utf8SortKey
|
|
12488
|
+
utf8SortKey as utf8SortKey2
|
|
12258
12489
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
12490
|
+
|
|
12491
|
+
// src/ontology-storage/source-cleanup.ts
|
|
12492
|
+
async function cleanupSourceVersions(sql, input) {
|
|
12493
|
+
let remaining = input.limit;
|
|
12494
|
+
const removedRows = await sql`
|
|
12495
|
+
WITH selected AS (
|
|
12496
|
+
SELECT rows.ctid FROM ontology_source_roots AS roots
|
|
12497
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
|
|
12498
|
+
JOIN ontology_source_rows AS rows
|
|
12499
|
+
ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
12500
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
12501
|
+
WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
|
|
12502
|
+
AND sources.status IN ('superseded', 'abandoned')
|
|
12503
|
+
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
|
|
12504
|
+
LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
|
|
12505
|
+
) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
|
|
12506
|
+
`;
|
|
12507
|
+
let rowsDeleted = removedRows.length;
|
|
12508
|
+
remaining -= rowsDeleted;
|
|
12509
|
+
if (remaining > 0) {
|
|
12510
|
+
const removed = await sql`
|
|
12511
|
+
WITH selected AS (
|
|
12512
|
+
SELECT roots.ctid FROM ontology_source_roots AS roots
|
|
12513
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
|
|
12514
|
+
WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
|
|
12515
|
+
AND sources.status IN ('superseded', 'abandoned')
|
|
12516
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
12517
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
12518
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
|
|
12519
|
+
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
|
|
12520
|
+
LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
|
|
12521
|
+
) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
|
|
12522
|
+
`;
|
|
12523
|
+
rowsDeleted += removed.length;
|
|
12524
|
+
remaining -= removed.length;
|
|
12525
|
+
}
|
|
12526
|
+
const removedManifests = remaining === 0 ? [] : await sql`
|
|
12527
|
+
WITH selected AS (
|
|
12528
|
+
SELECT sources.ctid FROM ontology_sources AS sources
|
|
12529
|
+
WHERE project_id = ${input.projectId} AND status IN ('superseded', 'abandoned') AND terminal_at < ${input.terminalBefore}
|
|
12530
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
|
|
12531
|
+
WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
|
|
12532
|
+
AND roots.materialization_id = sources.materialization_id)
|
|
12533
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
12534
|
+
WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
|
|
12535
|
+
AND rows.materialization_id = sources.materialization_id)
|
|
12536
|
+
ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
|
|
12537
|
+
) DELETE FROM ontology_sources AS sources USING selected WHERE sources.ctid = selected.ctid RETURNING 1
|
|
12538
|
+
`;
|
|
12539
|
+
return { rowsDeleted, materializationsDeleted: removedManifests.length };
|
|
12540
|
+
}
|
|
12541
|
+
|
|
12542
|
+
// src/ontology-storage/sources.ts
|
|
12259
12543
|
class PgOntologySourceStorage {
|
|
12260
12544
|
runRootOperation;
|
|
12261
12545
|
constructor(runRootOperation) {
|
|
@@ -12286,14 +12570,15 @@ class PgOntologySourceStorage {
|
|
|
12286
12570
|
dataset_id, dataset_version_id, dataset_version_created_at,
|
|
12287
12571
|
projection_revision, ownership_hash, ontology_revision,
|
|
12288
12572
|
root_count, assertion_count, created_at, ready_at, activated_at,
|
|
12289
|
-
terminal_at, last_commit_id, updated_at
|
|
12573
|
+
terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
|
|
12290
12574
|
) VALUES (
|
|
12291
12575
|
${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
|
|
12292
12576
|
${input.execution.projectionRunId}, ${input.projectionKind}, 'replacement',
|
|
12293
12577
|
'staging', ${input.execution.executionToken}, ${input.datasetVersion.datasetId},
|
|
12294
12578
|
${input.datasetVersion.versionId}, ${input.datasetVersion.createdAt},
|
|
12295
12579
|
${input.projectionRevision}, ${input.ownershipHash}, ${input.ontologyRevision},
|
|
12296
|
-
NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt}
|
|
12580
|
+
NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt},
|
|
12581
|
+
${input.base?.materializationId ?? null}, ${input.base?.lastCommitId ?? null}
|
|
12297
12582
|
)
|
|
12298
12583
|
ON CONFLICT DO NOTHING
|
|
12299
12584
|
RETURNING *
|
|
@@ -12322,6 +12607,7 @@ class PgOntologySourceStorage {
|
|
|
12322
12607
|
const rows = sourceStageRows(manifest.projection_kind, input.rows);
|
|
12323
12608
|
const existing = await this.findStageRows(sql, input, rows);
|
|
12324
12609
|
const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
|
|
12610
|
+
await stageSourceRoots(sql, manifest, input);
|
|
12325
12611
|
await this.insertStageRows(sql, input, pending);
|
|
12326
12612
|
return { inserted: pending.length, unchanged };
|
|
12327
12613
|
});
|
|
@@ -12345,7 +12631,10 @@ class PgOntologySourceStorage {
|
|
|
12345
12631
|
throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
|
|
12346
12632
|
}
|
|
12347
12633
|
if (input.readyAt < toIsoString2(manifest.created_at)) {
|
|
12348
|
-
throw new
|
|
12634
|
+
throw new MaterializationValidationError3("Source readyAt cannot precede source createdAt.");
|
|
12635
|
+
}
|
|
12636
|
+
if (input.assertionCount >= 1000 || input.rootCount >= 1000) {
|
|
12637
|
+
await sql`ANALYZE ontology_source_rows, ontology_source_roots`;
|
|
12349
12638
|
}
|
|
12350
12639
|
await this.assertReadyState(sql, manifest, input.rootCount, input.assertionCount);
|
|
12351
12640
|
const rows = await sql`
|
|
@@ -12392,61 +12681,7 @@ class PgOntologySourceStorage {
|
|
|
12392
12681
|
assertNonblank(input.projectId, "Terminal source cleanup project id");
|
|
12393
12682
|
assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true);
|
|
12394
12683
|
assertPositiveInteger(input.limit, "Terminal source cleanup limit");
|
|
12395
|
-
|
|
12396
|
-
SELECT * FROM ontology_sources
|
|
12397
|
-
WHERE project_id = ${input.projectId}
|
|
12398
|
-
AND status IN ('superseded', 'abandoned')
|
|
12399
|
-
AND terminal_at < ${input.terminalBefore}
|
|
12400
|
-
ORDER BY terminal_at, source_id, materialization_id
|
|
12401
|
-
LIMIT ${input.limit}
|
|
12402
|
-
FOR UPDATE SKIP LOCKED
|
|
12403
|
-
`;
|
|
12404
|
-
let remaining = input.limit;
|
|
12405
|
-
let rowsDeleted = 0;
|
|
12406
|
-
let materializationsDeleted = 0;
|
|
12407
|
-
for (const manifest of manifests) {
|
|
12408
|
-
if (remaining === 0)
|
|
12409
|
-
break;
|
|
12410
|
-
const deletedRows = await sql`
|
|
12411
|
-
WITH selected AS (
|
|
12412
|
-
SELECT rows.ctid
|
|
12413
|
-
FROM ontology_source_rows AS rows
|
|
12414
|
-
WHERE rows.project_id = ${manifest.project_id}
|
|
12415
|
-
AND rows.source_id = ${manifest.source_id}
|
|
12416
|
-
AND rows.materialization_id = ${manifest.materialization_id}
|
|
12417
|
-
ORDER BY rows.entity_sort_key
|
|
12418
|
-
LIMIT ${remaining}
|
|
12419
|
-
FOR UPDATE SKIP LOCKED
|
|
12420
|
-
)
|
|
12421
|
-
DELETE FROM ontology_source_rows AS rows
|
|
12422
|
-
USING selected
|
|
12423
|
-
WHERE rows.ctid = selected.ctid
|
|
12424
|
-
RETURNING rows.entity_sort_key
|
|
12425
|
-
`;
|
|
12426
|
-
rowsDeleted += deletedRows.length;
|
|
12427
|
-
remaining -= deletedRows.length;
|
|
12428
|
-
if (remaining === 0)
|
|
12429
|
-
break;
|
|
12430
|
-
const removed = await sql`
|
|
12431
|
-
DELETE FROM ontology_sources AS sources
|
|
12432
|
-
WHERE sources.project_id = ${manifest.project_id}
|
|
12433
|
-
AND sources.source_id = ${manifest.source_id}
|
|
12434
|
-
AND sources.materialization_id = ${manifest.materialization_id}
|
|
12435
|
-
AND sources.status IN ('superseded', 'abandoned')
|
|
12436
|
-
AND sources.terminal_at = ${manifest.terminal_at}
|
|
12437
|
-
AND sources.terminal_at < ${input.terminalBefore}
|
|
12438
|
-
AND NOT EXISTS (
|
|
12439
|
-
SELECT 1 FROM ontology_source_rows AS rows
|
|
12440
|
-
WHERE rows.project_id = sources.project_id
|
|
12441
|
-
AND rows.source_id = sources.source_id
|
|
12442
|
-
AND rows.materialization_id = sources.materialization_id
|
|
12443
|
-
)
|
|
12444
|
-
RETURNING sources.materialization_id
|
|
12445
|
-
`;
|
|
12446
|
-
materializationsDeleted += removed.length;
|
|
12447
|
-
remaining -= removed.length;
|
|
12448
|
-
}
|
|
12449
|
-
return { rowsDeleted, materializationsDeleted };
|
|
12684
|
+
return cleanupSourceVersions(sql, input);
|
|
12450
12685
|
});
|
|
12451
12686
|
}
|
|
12452
12687
|
async summarizeTerminal(input) {
|
|
@@ -12494,7 +12729,7 @@ class PgOntologySourceStorage {
|
|
|
12494
12729
|
}
|
|
12495
12730
|
async transitionToAbandoned(sql, manifest, abandonedAt) {
|
|
12496
12731
|
if (abandonedAt < toIsoString2(manifest.created_at) || manifest.ready_at !== null && abandonedAt < toIsoString2(manifest.ready_at)) {
|
|
12497
|
-
throw new
|
|
12732
|
+
throw new MaterializationValidationError3("Source abandonedAt cannot precede source creation or readiness.");
|
|
12498
12733
|
}
|
|
12499
12734
|
const rows = await sql`
|
|
12500
12735
|
UPDATE ontology_sources
|
|
@@ -12509,14 +12744,19 @@ class PgOntologySourceStorage {
|
|
|
12509
12744
|
if (!rows[0]) {
|
|
12510
12745
|
throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
|
|
12511
12746
|
}
|
|
12747
|
+
await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
|
|
12748
|
+
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
12749
|
+
AND materialization_id = ${manifest.materialization_id} AND NOT active`;
|
|
12512
12750
|
return sourceRecord(rows[0]);
|
|
12513
12751
|
}
|
|
12514
12752
|
async assertReadyState(sql, manifest, rootCount, assertionCount) {
|
|
12515
12753
|
const [counts] = await sql`
|
|
12516
|
-
SELECT
|
|
12754
|
+
SELECT (SELECT COUNT(*) FROM ontology_source_rows
|
|
12755
|
+
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
12756
|
+
AND materialization_id = ${manifest.materialization_id}) AS assertions, COUNT(*) AS roots,
|
|
12517
12757
|
COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
12518
12758
|
MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
|
|
12519
|
-
FROM
|
|
12759
|
+
FROM ontology_source_roots
|
|
12520
12760
|
WHERE project_id = ${manifest.project_id}
|
|
12521
12761
|
AND source_id = ${manifest.source_id}
|
|
12522
12762
|
AND materialization_id = ${manifest.materialization_id}
|
|
@@ -12527,8 +12767,9 @@ class PgOntologySourceStorage {
|
|
|
12527
12767
|
const minimum = counts?.min_ordinal === null ? null : Number(counts?.min_ordinal);
|
|
12528
12768
|
const maximum = counts?.max_ordinal === null ? null : Number(counts?.max_ordinal);
|
|
12529
12769
|
if (assertions !== assertionCount || roots !== rootCount || ordinals !== rootCount || rootCount > 0 && (minimum !== 0 || maximum !== rootCount - 1)) {
|
|
12530
|
-
throw new
|
|
12770
|
+
throw new MaterializationValidationError3("Source ready counts do not match the staged roots and assertions.");
|
|
12531
12771
|
}
|
|
12772
|
+
await assertSourceRootCoverage(sql, manifest);
|
|
12532
12773
|
const [invalid] = manifest.projection_kind === "link" ? await sql`
|
|
12533
12774
|
SELECT root_key
|
|
12534
12775
|
FROM ontology_source_rows
|
|
@@ -12562,15 +12803,15 @@ class PgOntologySourceStorage {
|
|
|
12562
12803
|
LIMIT 1
|
|
12563
12804
|
`;
|
|
12564
12805
|
if (invalid) {
|
|
12565
|
-
throw new
|
|
12806
|
+
throw new MaterializationValidationError3(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
12566
12807
|
}
|
|
12567
12808
|
}
|
|
12568
12809
|
async findStageRows(sql, input, rows) {
|
|
12569
12810
|
if (rows.length === 0)
|
|
12570
12811
|
return [];
|
|
12571
|
-
const rootSortKeys = [...new Set(rows.map((row) =>
|
|
12812
|
+
const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.rootKey)))];
|
|
12572
12813
|
const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
|
|
12573
|
-
const entitySortKeys = [...new Set(rows.map((row) =>
|
|
12814
|
+
const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
|
|
12574
12815
|
const existing = await sql`
|
|
12575
12816
|
SELECT * FROM ontology_source_rows
|
|
12576
12817
|
WHERE project_id = ${input.projectId}
|
|
@@ -12593,10 +12834,10 @@ class PgOntologySourceStorage {
|
|
|
12593
12834
|
return {
|
|
12594
12835
|
entityKind: row.assertion.kind,
|
|
12595
12836
|
entityKey: JSON.parse(entityKey),
|
|
12596
|
-
entitySortKey:
|
|
12837
|
+
entitySortKey: utf8SortKey2(entityKey),
|
|
12597
12838
|
rootKind: row.root.kind,
|
|
12598
12839
|
rootKey: JSON.parse(rootKey),
|
|
12599
|
-
rootSortKey:
|
|
12840
|
+
rootSortKey: utf8SortKey2(rootKey),
|
|
12600
12841
|
stagingOrdinal: row.stagingOrdinal,
|
|
12601
12842
|
root: row.root,
|
|
12602
12843
|
assertion: row.assertion,
|
|
@@ -15256,7 +15497,7 @@ class PgProjectionRunStorage {
|
|
|
15256
15497
|
input_exhausted = ${input.fixedBatchSize === undefined ? null : false},
|
|
15257
15498
|
missing_target_object_type_id = ${null}, missing_target_object_id = ${null},
|
|
15258
15499
|
missing_target_batch_ordinal = ${null}, missing_target_first_seen_at = ${null},
|
|
15259
|
-
source_rows_read = ${0}, source_rows_skipped = ${0}, error = ${null}
|
|
15500
|
+
source_rows_read = ${0}, source_rows_skipped = ${0}, source_changes_read = ${null}, error = ${null}
|
|
15260
15501
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
15261
15502
|
AND status = ${"failed"} AND error->>'code' = ${"queue.enqueue_failed"}
|
|
15262
15503
|
RETURNING *
|
|
@@ -15367,7 +15608,8 @@ class PgProjectionRunStorage {
|
|
|
15367
15608
|
UPDATE projection_runs
|
|
15368
15609
|
SET
|
|
15369
15610
|
source_rows_read = ${progress.sourceRowsRead},
|
|
15370
|
-
source_rows_skipped = ${progress.sourceRowsSkipped}
|
|
15611
|
+
source_rows_skipped = ${progress.sourceRowsSkipped},
|
|
15612
|
+
source_changes_read = ${progress.sourceChangesRead ?? null}
|
|
15371
15613
|
WHERE project_id = ${input.projectId}
|
|
15372
15614
|
AND id = ${input.id}
|
|
15373
15615
|
AND status = ${"running"}
|
|
@@ -15392,6 +15634,7 @@ class PgProjectionRunStorage {
|
|
|
15392
15634
|
execution_token = ${null},
|
|
15393
15635
|
source_rows_read = ${plan.progress.sourceRowsRead},
|
|
15394
15636
|
source_rows_skipped = ${plan.progress.sourceRowsSkipped},
|
|
15637
|
+
source_changes_read = ${plan.progress.sourceChangesRead ?? null},
|
|
15395
15638
|
input_exhausted = ${plan.inputExhausted ?? existingRow.input_exhausted},
|
|
15396
15639
|
error = ${plan.error === undefined ? null : serializeSixbFailure4(plan.error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
|
|
15397
15640
|
WHERE project_id = ${input.projectId}
|
|
@@ -15634,7 +15877,8 @@ function restoreProjectionRunRow(row) {
|
|
|
15634
15877
|
missingTargetFirstSeenAt: row.missing_target_first_seen_at ? new Date(row.missing_target_first_seen_at) : undefined,
|
|
15635
15878
|
progress: {
|
|
15636
15879
|
sourceRowsRead: databaseSafeInteger2(row.source_rows_read, "sourceRowsRead"),
|
|
15637
|
-
sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped")
|
|
15880
|
+
sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped"),
|
|
15881
|
+
...row.source_changes_read == null ? {} : { sourceChangesRead: databaseSafeInteger2(row.source_changes_read, "sourceChangesRead") }
|
|
15638
15882
|
},
|
|
15639
15883
|
error: row.error ?? undefined
|
|
15640
15884
|
};
|
|
@@ -17866,4 +18110,4 @@ export {
|
|
|
17866
18110
|
migratePostgresStorage
|
|
17867
18111
|
};
|
|
17868
18112
|
|
|
17869
|
-
//# debugId=
|
|
18113
|
+
//# debugId=672C519862E8368164756E2164756E21
|