@sixb/pg 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/rows.d.ts +0 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +562 -299
- package/dist/index.js.map +24 -22
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +3 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +2 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -0
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts +4 -0
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -0
- package/dist/ontology-storage/source-roots.d.ts +24 -0
- package/dist/ontology-storage/source-roots.d.ts.map +1 -0
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pg-action-run-storage.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts.map +1 -1
- package/dist/pg-execution-storage.d.ts.map +1 -1
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/agents/rows.ts +0 -5
- package/src/agents/runs.ts +1 -6
- package/src/ai-cost-groups.ts +3 -0
- package/src/connector-connection-storage.ts +3 -3
- package/src/index.ts +4 -2
- package/src/migrations/037-execution-requester-groups.sql +21 -0
- package/src/migrations/038-outbox-publication-order.sql +5 -0
- package/src/migrations/039-projection-source-roots.sql +46 -0
- package/src/migrations/040-connector-optional-pkce.sql +3 -0
- package/src/migrations.ts +18 -0
- package/src/ontology-storage/materialization-session.ts +24 -8
- package/src/ontology-storage/materialization-state.ts +65 -86
- package/src/ontology-storage/materializations.ts +39 -13
- package/src/ontology-storage/shared.ts +10 -0
- package/src/ontology-storage/source-cleanup.ts +61 -0
- package/src/ontology-storage/source-roots.ts +251 -0
- package/src/ontology-storage/sources.ts +21 -57
- package/src/pg-action-run-storage.ts +6 -7
- package/src/pg-ai-cost-storage.ts +11 -2
- package/src/pg-ai-usage-storage.ts +1 -2
- package/src/pg-execution-storage.ts +6 -4
- package/src/pg-pipeline-run-storage.ts +2 -5
- package/src/pg-projection-run-storage.ts +8 -2
- package/src/pg-sync-run-storage.ts +2 -5
- package/src/pg-workflow-run-storage.ts +0 -5
package/dist/index.js
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
// src/index.ts
|
|
3
3
|
import { AsyncLocalStorage } from "async_hooks";
|
|
4
4
|
import { ProviderMaterializationTransactionLifecycle } from "@sixb/core/internal/ontology-storage-provider";
|
|
5
|
+
import {
|
|
6
|
+
createTransactionStorageProxy,
|
|
7
|
+
throwNestedStorageTransaction
|
|
8
|
+
} from "@sixb/core/internal/storage";
|
|
5
9
|
import {
|
|
6
10
|
createAgentOperationScope,
|
|
7
11
|
createAuthOperationScope,
|
|
@@ -12,9 +16,7 @@ import {
|
|
|
12
16
|
createWorkflowRunOperationScope
|
|
13
17
|
} from "@sixb/core/internal/storage-operation-scope";
|
|
14
18
|
import {
|
|
15
|
-
|
|
16
|
-
StorageTransactionError,
|
|
17
|
-
throwNestedStorageTransaction
|
|
19
|
+
StorageTransactionError
|
|
18
20
|
} from "@sixb/core/storage";
|
|
19
21
|
|
|
20
22
|
// src/agents/checkpoints.ts
|
|
@@ -147,7 +149,6 @@ function rowToRunRecord(row) {
|
|
|
147
149
|
id: row.id,
|
|
148
150
|
projectId: row.project_id,
|
|
149
151
|
executionId: row.execution_id,
|
|
150
|
-
requesterGroupIds: typeof row.requester_group_ids === "string" ? JSON.parse(row.requester_group_ids) : row.requester_group_ids,
|
|
151
152
|
status: row.status,
|
|
152
153
|
modelId: row.model_id ?? undefined,
|
|
153
154
|
finishReason: coerceAgentRunFinishReason(row.finish_reason),
|
|
@@ -580,7 +581,6 @@ import {
|
|
|
580
581
|
assertSubagentRunResult,
|
|
581
582
|
subagentRunMatchesCreateInput
|
|
582
583
|
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
583
|
-
import { normalizeRequesterGroupIds } from "@sixb/core/internal/auth";
|
|
584
584
|
import { serializeSixbFailure } from "@sixb/core/internal/errors";
|
|
585
585
|
import {
|
|
586
586
|
AGENT_RUN_FAILURE_CODES as AGENT_RUN_FAILURE_CODES2,
|
|
@@ -683,7 +683,6 @@ class PgAgentRunStore {
|
|
|
683
683
|
thread_id,
|
|
684
684
|
trigger_message_id,
|
|
685
685
|
spec,
|
|
686
|
-
requester_group_ids,
|
|
687
686
|
status,
|
|
688
687
|
attempt,
|
|
689
688
|
created_at
|
|
@@ -695,7 +694,6 @@ class PgAgentRunStore {
|
|
|
695
694
|
${input.threadId},
|
|
696
695
|
${input.triggerMessageId},
|
|
697
696
|
${JSON.stringify(input.spec)}::text::jsonb,
|
|
698
|
-
${JSON.stringify(normalizeRequesterGroupIds(input.requesterGroupIds))}::text::jsonb,
|
|
699
697
|
${"queued"},
|
|
700
698
|
${0},
|
|
701
699
|
${createdAt}
|
|
@@ -756,7 +754,6 @@ class PgAgentRunStore {
|
|
|
756
754
|
if (parent.execution_token !== input.parentExecutionToken) {
|
|
757
755
|
throw new AgentStorageError3("execution_lost", `[SixbPg] Parent Agent run '${parent.id}' is no longer owned by this execution.`);
|
|
758
756
|
}
|
|
759
|
-
const parentRecord = rowToRunRecord(parent);
|
|
760
757
|
const [countRow] = await tx`
|
|
761
758
|
SELECT COUNT(*)::bigint AS count FROM agent_runs
|
|
762
759
|
WHERE project_id = ${input.projectId}
|
|
@@ -770,12 +767,11 @@ class PgAgentRunStore {
|
|
|
770
767
|
const [row] = await tx`
|
|
771
768
|
INSERT INTO agent_runs (
|
|
772
769
|
project_id, id, execution_id, kind, parent_run_id, spawn_key, spec,
|
|
773
|
-
|
|
770
|
+
status, attempt, created_at
|
|
774
771
|
) VALUES (
|
|
775
772
|
${input.projectId}, ${input.id}, ${input.executionId}, ${"subagent"},
|
|
776
773
|
${input.parentRunId}, ${input.spawnKey},
|
|
777
774
|
${JSON.stringify(input.spec)}::text::jsonb,
|
|
778
|
-
${JSON.stringify(parentRecord.requesterGroupIds)}::text::jsonb,
|
|
779
775
|
${"queued"}, ${0}, ${input.createdAt ?? new Date}
|
|
780
776
|
)
|
|
781
777
|
RETURNING *
|
|
@@ -1871,13 +1867,13 @@ class PgAuthAccessTokenStore {
|
|
|
1871
1867
|
}
|
|
1872
1868
|
async assertSubjectExists(projectId, subjectType, subjectId) {
|
|
1873
1869
|
if (subjectType === "user") {
|
|
1874
|
-
const [
|
|
1870
|
+
const [row] = await this.sql`
|
|
1875
1871
|
SELECT *
|
|
1876
1872
|
FROM auth_users
|
|
1877
1873
|
WHERE project_id = ${projectId}
|
|
1878
1874
|
AND id = ${subjectId}
|
|
1879
1875
|
`;
|
|
1880
|
-
if (
|
|
1876
|
+
if (row)
|
|
1881
1877
|
return;
|
|
1882
1878
|
throw new AuthStorageError2("missing_user", `[Sixb] User '${subjectId}' not found for project '${projectId}'.`);
|
|
1883
1879
|
}
|
|
@@ -3452,7 +3448,7 @@ class PgConnectorConnectionPersistence {
|
|
|
3452
3448
|
) VALUES (
|
|
3453
3449
|
${record.projectId}, ${record.connectorId}, ${record.id}, ${record.slot},
|
|
3454
3450
|
${record.initiatedByExecutionId}, ${record.stateHash},
|
|
3455
|
-
${jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
|
|
3451
|
+
${record.codeVerifier === undefined ? null : jsonParameter(this.sql, record.codeVerifier)}, ${record.redirectUri},
|
|
3456
3452
|
${record.connectionRunId ?? null}, ${record.returnTo ?? null},
|
|
3457
3453
|
${record.callbackBindingHash ?? null}, ${record.reauthorizationId ?? null},
|
|
3458
3454
|
${record.reauthorizationRevision ?? null},
|
|
@@ -3745,7 +3741,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
3745
3741
|
slot: row.slot,
|
|
3746
3742
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
3747
3743
|
stateHash: row.state_hash,
|
|
3748
|
-
codeVerifier: row.code_verifier,
|
|
3744
|
+
...row.code_verifier === null ? {} : { codeVerifier: row.code_verifier },
|
|
3749
3745
|
redirectUri: row.redirect_uri,
|
|
3750
3746
|
...row.connection_run_id === null ? {} : { connectionRunId: row.connection_run_id },
|
|
3751
3747
|
...row.return_to === null ? {} : { returnTo: row.return_to },
|
|
@@ -7085,6 +7081,93 @@ ALTER TABLE executions
|
|
|
7085
7081
|
ON DELETE RESTRICT;
|
|
7086
7082
|
`;
|
|
7087
7083
|
|
|
7084
|
+
// src/migrations/039-projection-source-roots.sql
|
|
7085
|
+
var _039_projection_source_roots_default = `ALTER TABLE ontology_sources ADD COLUMN base_materialization_id TEXT;
|
|
7086
|
+
ALTER TABLE ontology_sources ADD COLUMN base_commit_id TEXT;
|
|
7087
|
+
ALTER TABLE ontology_sources ADD CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL));
|
|
7088
|
+
|
|
7089
|
+
-- A root selects one complete set of assertions. Unchanged roots keep their original version;
|
|
7090
|
+
-- activation only retires/replaces affected references, under the source commit fence.
|
|
7091
|
+
CREATE TABLE ontology_source_roots (
|
|
7092
|
+
project_id TEXT NOT NULL,
|
|
7093
|
+
source_id TEXT NOT NULL,
|
|
7094
|
+
materialization_id TEXT NOT NULL,
|
|
7095
|
+
root_sort_key TEXT NOT NULL,
|
|
7096
|
+
root_kind TEXT NOT NULL CHECK (root_kind IN ('object', 'link')),
|
|
7097
|
+
root_key JSONB NOT NULL,
|
|
7098
|
+
root JSONB NOT NULL,
|
|
7099
|
+
staging_ordinal BIGINT NOT NULL CHECK (staging_ordinal >= 0),
|
|
7100
|
+
deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
|
7101
|
+
active BOOLEAN NOT NULL DEFAULT FALSE,
|
|
7102
|
+
retired_at TIMESTAMPTZ,
|
|
7103
|
+
PRIMARY KEY (project_id, source_id, materialization_id, root_sort_key),
|
|
7104
|
+
UNIQUE (project_id, source_id, materialization_id, staging_ordinal),
|
|
7105
|
+
FOREIGN KEY (project_id, source_id, materialization_id)
|
|
7106
|
+
REFERENCES ontology_sources(project_id, source_id, materialization_id) ON DELETE RESTRICT,
|
|
7107
|
+
CHECK (NOT active OR (NOT deleted AND retired_at IS NULL))
|
|
7108
|
+
);
|
|
7109
|
+
CREATE UNIQUE INDEX idx_ontology_source_roots_active
|
|
7110
|
+
ON ontology_source_roots (project_id, source_id, root_sort_key) WHERE active;
|
|
7111
|
+
CREATE INDEX idx_ontology_source_roots_lookup
|
|
7112
|
+
ON ontology_source_roots (project_id, root_sort_key, source_id) WHERE active;
|
|
7113
|
+
CREATE INDEX idx_ontology_source_roots_cleanup
|
|
7114
|
+
ON ontology_source_roots (project_id, retired_at, source_id, materialization_id, root_sort_key)
|
|
7115
|
+
WHERE NOT active AND retired_at IS NOT NULL;
|
|
7116
|
+
|
|
7117
|
+
INSERT INTO ontology_source_roots (
|
|
7118
|
+
project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root,
|
|
7119
|
+
staging_ordinal, active, retired_at
|
|
7120
|
+
)
|
|
7121
|
+
SELECT DISTINCT rows.project_id, rows.source_id, rows.materialization_id,
|
|
7122
|
+
rows.root_sort_key, rows.root_kind, rows.root_key, rows.root, rows.staging_ordinal,
|
|
7123
|
+
sources.status = 'active', sources.terminal_at
|
|
7124
|
+
FROM ontology_source_rows AS rows
|
|
7125
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id);
|
|
7126
|
+
|
|
7127
|
+
ANALYZE ontology_source_roots;
|
|
7128
|
+
|
|
7129
|
+
ALTER TABLE projection_runs ADD COLUMN source_changes_read BIGINT
|
|
7130
|
+
CHECK (source_changes_read IS NULL OR source_changes_read >= 0);
|
|
7131
|
+
`;
|
|
7132
|
+
|
|
7133
|
+
// src/migrations/037-execution-requester-groups.sql
|
|
7134
|
+
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
|
|
7135
|
+
|
|
7136
|
+
-- Keep historical admission snapshots; never reconstruct them from current memberships.
|
|
7137
|
+
WITH RECURSIVE attribution(project_id, id, groups) AS (
|
|
7138
|
+
SELECT e.project_id, e.id, COALESCE(w.requester_group_ids, a.requester_group_ids, '[]'::jsonb)
|
|
7139
|
+
FROM executions e
|
|
7140
|
+
LEFT JOIN workflow_runs w ON w.project_id = e.project_id AND w.execution_id = e.id
|
|
7141
|
+
LEFT JOIN agent_runs a ON a.project_id = e.project_id AND a.execution_id = e.id
|
|
7142
|
+
WHERE e.parent_execution_id IS NULL
|
|
7143
|
+
UNION ALL
|
|
7144
|
+
SELECT e.project_id, e.id, COALESCE(w.requester_group_ids, a.requester_group_ids, p.groups)
|
|
7145
|
+
FROM executions e
|
|
7146
|
+
JOIN attribution p ON p.project_id = e.project_id AND p.id = e.parent_execution_id
|
|
7147
|
+
LEFT JOIN workflow_runs w ON w.project_id = e.project_id AND w.execution_id = e.id
|
|
7148
|
+
LEFT JOIN agent_runs a ON a.project_id = e.project_id AND a.execution_id = e.id
|
|
7149
|
+
)
|
|
7150
|
+
UPDATE executions e SET requester_group_ids = a.groups
|
|
7151
|
+
FROM attribution a WHERE a.project_id = e.project_id AND a.id = e.id;
|
|
7152
|
+
|
|
7153
|
+
ALTER TABLE workflow_runs DROP COLUMN requester_group_ids;
|
|
7154
|
+
ALTER TABLE agent_runs DROP COLUMN requester_group_ids;
|
|
7155
|
+
`;
|
|
7156
|
+
|
|
7157
|
+
// src/migrations/038-outbox-publication-order.sql
|
|
7158
|
+
var _038_outbox_publication_order_default = `-- Preserve the claim order without sorting every pending event for each batch.
|
|
7159
|
+
-- Keep the availability index for selective delayed/leased workloads.
|
|
7160
|
+
CREATE INDEX idx_ontology_outbox_publication_order
|
|
7161
|
+
ON ontology_outbox (project_id, created_at, commit_id, commit_ordinal)
|
|
7162
|
+
WHERE published_at IS NULL;
|
|
7163
|
+
`;
|
|
7164
|
+
|
|
7165
|
+
// src/migrations/040-connector-optional-pkce.sql
|
|
7166
|
+
var _040_connector_optional_pkce_default = `-- Existing attempts retain their encrypted S256 verifier. NULL denotes disabled PKCE.
|
|
7167
|
+
ALTER TABLE connector_authorization_attempts
|
|
7168
|
+
ALTER COLUMN code_verifier DROP NOT NULL;
|
|
7169
|
+
`;
|
|
7170
|
+
|
|
7088
7171
|
// src/migrations.ts
|
|
7089
7172
|
var POSTGRES_STORAGE_ADAPTER_ID = "SixbPostgresStorage";
|
|
7090
7173
|
function pgSql(id, sqlText) {
|
|
@@ -7295,7 +7378,11 @@ var postgresStorageMigrations = defineMigrations({
|
|
|
7295
7378
|
pgSql("033-retire-agent-definitions", _033_retire_agent_definitions_default),
|
|
7296
7379
|
pgSql("034-device-authorizations", _034_device_authorizations_default),
|
|
7297
7380
|
pgSql("035-share-grants", _035_share_grants_default),
|
|
7298
|
-
pgSql("036-share-sessions", _036_share_sessions_default)
|
|
7381
|
+
pgSql("036-share-sessions", _036_share_sessions_default),
|
|
7382
|
+
pgSql("037-execution-requester-groups", _037_execution_requester_groups_default),
|
|
7383
|
+
pgSql("038-outbox-publication-order", _038_outbox_publication_order_default),
|
|
7384
|
+
pgSql("039-projection-source-roots", _039_projection_source_roots_default),
|
|
7385
|
+
pgSql("040-connector-optional-pkce", _040_connector_optional_pkce_default)
|
|
7299
7386
|
]
|
|
7300
7387
|
});
|
|
7301
7388
|
|
|
@@ -7744,18 +7831,18 @@ function compileExpansionOrder(expansion, target, neighborType, neighborId) {
|
|
|
7744
7831
|
}
|
|
7745
7832
|
function compileSet(projectId, op, inputs, ctx) {
|
|
7746
7833
|
if (inputs.length === 0) {
|
|
7747
|
-
const
|
|
7834
|
+
const order = compileOrder(identityOrderFields());
|
|
7748
7835
|
return {
|
|
7749
7836
|
sql: `
|
|
7750
7837
|
SELECT *, properties AS _cursor_properties
|
|
7751
7838
|
FROM ${ctx.source.objectsTable}
|
|
7752
7839
|
WHERE 1 = 0
|
|
7753
|
-
ORDER BY ${
|
|
7840
|
+
ORDER BY ${order.sql}
|
|
7754
7841
|
`,
|
|
7755
|
-
args:
|
|
7842
|
+
args: order.args,
|
|
7756
7843
|
totalSql: "SELECT 0::bigint AS total",
|
|
7757
7844
|
totalArgs: [],
|
|
7758
|
-
order
|
|
7845
|
+
order,
|
|
7759
7846
|
hasMore: () => false,
|
|
7760
7847
|
trimRows: identityRows,
|
|
7761
7848
|
nextPageToken: () => {
|
|
@@ -9428,7 +9515,7 @@ function valuesJoin(sql, select, columns, tuples, where, whereParams) {
|
|
|
9428
9515
|
const colWidth = columns.length;
|
|
9429
9516
|
const base = whereParams.length;
|
|
9430
9517
|
const valuePlaceholders = tuples.map((_, i) => {
|
|
9431
|
-
const cols = columns.map((
|
|
9518
|
+
const cols = columns.map((_, j) => `$${base + i * colWidth + j + 1}`);
|
|
9432
9519
|
return `(${cols.join(",")})`;
|
|
9433
9520
|
}).join(",");
|
|
9434
9521
|
const onClause = columns.map((c) => {
|
|
@@ -9494,6 +9581,12 @@ function sourceRecord(row) {
|
|
|
9494
9581
|
projectionRevision: row.projection_revision,
|
|
9495
9582
|
ownershipHash: row.ownership_hash,
|
|
9496
9583
|
ontologyRevision: row.ontology_revision,
|
|
9584
|
+
...row.base_materialization_id === null ? {} : {
|
|
9585
|
+
base: {
|
|
9586
|
+
materializationId: row.base_materialization_id,
|
|
9587
|
+
lastCommitId: row.base_commit_id
|
|
9588
|
+
}
|
|
9589
|
+
},
|
|
9497
9590
|
rootCount: row.root_count === null ? null : databaseSafeInteger(row.root_count, "Source root count"),
|
|
9498
9591
|
assertionCount: row.assertion_count === null ? null : databaseSafeInteger(row.assertion_count, "Source assertion count"),
|
|
9499
9592
|
createdAt: toIsoString2(row.created_at),
|
|
@@ -9662,7 +9755,7 @@ import {
|
|
|
9662
9755
|
linkRefKey as linkRefKey3,
|
|
9663
9756
|
linkRefSortKey,
|
|
9664
9757
|
linkScopeSortKey as linkScopeSortKey2,
|
|
9665
|
-
MaterializationConflictError as
|
|
9758
|
+
MaterializationConflictError as MaterializationConflictError6,
|
|
9666
9759
|
objectRefKey as objectRefKey3,
|
|
9667
9760
|
objectRefSortKey,
|
|
9668
9761
|
telemetryPointKey as telemetryPointKey3,
|
|
@@ -9681,7 +9774,7 @@ import {
|
|
|
9681
9774
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
9682
9775
|
|
|
9683
9776
|
// src/ontology-storage/materialization-session.ts
|
|
9684
|
-
import { MaterializationConflictError as
|
|
9777
|
+
import { MaterializationConflictError as MaterializationConflictError4 } from "@sixb/core/internal/materialization";
|
|
9685
9778
|
import {
|
|
9686
9779
|
correlateMaterializationChunk,
|
|
9687
9780
|
duplicateMaterializationWork as duplicateWork,
|
|
@@ -9695,9 +9788,9 @@ import {
|
|
|
9695
9788
|
import {
|
|
9696
9789
|
linkRefKey,
|
|
9697
9790
|
linkScopeSortKey,
|
|
9698
|
-
MaterializationConflictError as
|
|
9791
|
+
MaterializationConflictError as MaterializationConflictError3,
|
|
9699
9792
|
objectRefKey,
|
|
9700
|
-
projectionEntityKey,
|
|
9793
|
+
projectionEntityKey as projectionEntityKey2,
|
|
9701
9794
|
telemetryPointKey,
|
|
9702
9795
|
telemetryPointSortKey
|
|
9703
9796
|
} from "@sixb/core/internal/materialization";
|
|
@@ -9707,6 +9800,178 @@ import {
|
|
|
9707
9800
|
linkSlotOverrideValue,
|
|
9708
9801
|
startScopeAccumulator
|
|
9709
9802
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
9803
|
+
|
|
9804
|
+
// src/ontology-storage/source-roots.ts
|
|
9805
|
+
import {
|
|
9806
|
+
MaterializationConflictError as MaterializationConflictError2,
|
|
9807
|
+
MaterializationValidationError,
|
|
9808
|
+
projectionEntityKey
|
|
9809
|
+
} from "@sixb/core/internal/materialization";
|
|
9810
|
+
import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider";
|
|
9811
|
+
function replacementSourceRows(sql, input) {
|
|
9812
|
+
const roots = input.incremental ? sql`
|
|
9813
|
+
SELECT roots.* FROM ontology_source_roots AS changed
|
|
9814
|
+
CROSS JOIN LATERAL (
|
|
9815
|
+
SELECT roots.* FROM ontology_source_roots AS roots
|
|
9816
|
+
WHERE roots.project_id = changed.project_id AND roots.source_id = changed.source_id
|
|
9817
|
+
AND roots.root_sort_key = changed.root_sort_key AND roots.active OFFSET 0
|
|
9818
|
+
) AS roots
|
|
9819
|
+
WHERE changed.project_id = ${input.projectId} AND changed.source_id = ${input.sourceId}
|
|
9820
|
+
AND changed.materialization_id = ${input.materializationId}
|
|
9821
|
+
` : sql`SELECT * FROM ontology_source_roots WHERE project_id = ${input.projectId} AND source_id = ${input.sourceId} AND active`;
|
|
9822
|
+
return sql`
|
|
9823
|
+
SELECT * FROM ontology_source_rows WHERE project_id = ${input.projectId}
|
|
9824
|
+
AND source_id = ${input.sourceId} AND materialization_id = ${input.materializationId}
|
|
9825
|
+
UNION ALL
|
|
9826
|
+
SELECT rows.* FROM (${roots}) AS roots
|
|
9827
|
+
CROSS JOIN LATERAL (
|
|
9828
|
+
SELECT rows.* FROM ontology_source_rows AS rows WHERE rows.project_id = roots.project_id
|
|
9829
|
+
AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
|
|
9830
|
+
AND rows.root_sort_key = roots.root_sort_key OFFSET 0
|
|
9831
|
+
) AS rows
|
|
9832
|
+
`;
|
|
9833
|
+
}
|
|
9834
|
+
async function stageSourceRoots(sql, manifest, input) {
|
|
9835
|
+
const roots = sourceStageRoots(manifest.projection_kind, input);
|
|
9836
|
+
if (roots.length === 0)
|
|
9837
|
+
return;
|
|
9838
|
+
if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
|
|
9839
|
+
throw new MaterializationValidationError("Root deletions require a source delta base.");
|
|
9840
|
+
}
|
|
9841
|
+
const values = jsonParameter2(sql, roots.map((root) => ({
|
|
9842
|
+
root_sort_key: root.sortKey,
|
|
9843
|
+
root_kind: root.root.kind,
|
|
9844
|
+
root_key: JSON.parse(root.rootKey),
|
|
9845
|
+
root: root.root,
|
|
9846
|
+
staging_ordinal: root.stagingOrdinal,
|
|
9847
|
+
deleted: root.deleted
|
|
9848
|
+
})));
|
|
9849
|
+
const inserted = await sql`
|
|
9850
|
+
INSERT INTO ontology_source_roots (
|
|
9851
|
+
project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
|
|
9852
|
+
) SELECT ${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
|
|
9853
|
+
root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
|
|
9854
|
+
FROM jsonb_to_recordset(${values}) AS input(
|
|
9855
|
+
root_sort_key TEXT, root_kind TEXT, root_key JSONB, root JSONB, staging_ordinal BIGINT, deleted BOOLEAN)
|
|
9856
|
+
ON CONFLICT DO NOTHING RETURNING root_sort_key
|
|
9857
|
+
`;
|
|
9858
|
+
if (inserted.length === roots.length)
|
|
9859
|
+
return;
|
|
9860
|
+
const insertedKeys = new Set(inserted.map((row) => row.root_sort_key));
|
|
9861
|
+
const skipped = roots.filter((root) => !insertedKeys.has(root.sortKey));
|
|
9862
|
+
const stored = await sql`
|
|
9863
|
+
SELECT root_sort_key, staging_ordinal, deleted FROM ontology_source_roots
|
|
9864
|
+
WHERE project_id = ${input.projectId} AND source_id = ${input.source.projectionId}
|
|
9865
|
+
AND materialization_id = ${input.materializationId}
|
|
9866
|
+
AND root_sort_key = ANY(${sql.array(skipped.map((root) => root.sortKey))}::text[])
|
|
9867
|
+
`;
|
|
9868
|
+
const existing = new Map(stored.map((root) => [root.root_sort_key, root]));
|
|
9869
|
+
if (skipped.some((root) => {
|
|
9870
|
+
const previous = existing.get(root.sortKey);
|
|
9871
|
+
return !previous || Number(previous.staging_ordinal) !== root.stagingOrdinal || previous.deleted !== root.deleted;
|
|
9872
|
+
}))
|
|
9873
|
+
throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
|
|
9874
|
+
}
|
|
9875
|
+
async function activateSourceRoots(sql, projectId, candidate, activation) {
|
|
9876
|
+
if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
|
|
9877
|
+
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
9878
|
+
if (candidate.base_materialization_id !== null && Number(candidate.root_count) === 0)
|
|
9879
|
+
return;
|
|
9880
|
+
const affected = candidate.base_materialization_id === null ? sql`` : sql`AND root_sort_key IN (
|
|
9881
|
+
SELECT root_sort_key FROM ontology_source_roots
|
|
9882
|
+
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
|
|
9883
|
+
AND materialization_id = ${activation.materializationId}
|
|
9884
|
+
)`;
|
|
9885
|
+
await sql`
|
|
9886
|
+
UPDATE ontology_source_roots AS roots SET active = FALSE, retired_at = ${activation.updatedAt}
|
|
9887
|
+
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId} AND active
|
|
9888
|
+
${affected}
|
|
9889
|
+
`;
|
|
9890
|
+
await sql`
|
|
9891
|
+
UPDATE ontology_source_roots SET active = NOT deleted,
|
|
9892
|
+
retired_at = CASE WHEN deleted THEN ${activation.updatedAt}::timestamptz ELSE NULL END
|
|
9893
|
+
WHERE project_id = ${projectId} AND source_id = ${activation.source.projectionId}
|
|
9894
|
+
AND materialization_id = ${activation.materializationId}
|
|
9895
|
+
`;
|
|
9896
|
+
}
|
|
9897
|
+
async function assertSourceRootCoverage(sql, manifest) {
|
|
9898
|
+
const [invalid] = await sql`
|
|
9899
|
+
SELECT 1 FROM ontology_source_roots AS roots
|
|
9900
|
+
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
9901
|
+
AND materialization_id = ${manifest.materialization_id}
|
|
9902
|
+
AND roots.deleted = EXISTS (
|
|
9903
|
+
SELECT 1 FROM ontology_source_rows AS rows
|
|
9904
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
9905
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
9906
|
+
) LIMIT 1
|
|
9907
|
+
`;
|
|
9908
|
+
if (invalid)
|
|
9909
|
+
throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
|
|
9910
|
+
}
|
|
9911
|
+
function activeSourceRows(sql, projectId, refs) {
|
|
9912
|
+
const keys = refs.flatMap((entity) => {
|
|
9913
|
+
const entitySortKey = utf8SortKey(projectionEntityKey(entity));
|
|
9914
|
+
const roots = entity.kind === "object" ? [entitySortKey] : [
|
|
9915
|
+
entitySortKey,
|
|
9916
|
+
utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
|
|
9917
|
+
];
|
|
9918
|
+
return roots.map((rootSortKey) => ({
|
|
9919
|
+
root_sort_key: rootSortKey,
|
|
9920
|
+
entity_sort_key: entitySortKey
|
|
9921
|
+
}));
|
|
9922
|
+
});
|
|
9923
|
+
return sql`
|
|
9924
|
+
SELECT rows.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)}) AS requested(root_sort_key TEXT, entity_sort_key TEXT)
|
|
9925
|
+
JOIN ontology_source_roots AS roots ON roots.project_id = ${projectId}
|
|
9926
|
+
AND roots.active AND roots.root_sort_key = requested.root_sort_key
|
|
9927
|
+
JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
9928
|
+
AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
|
|
9929
|
+
AND rows.root_sort_key = roots.root_sort_key
|
|
9930
|
+
`;
|
|
9931
|
+
}
|
|
9932
|
+
function replacementAssertionRows(sql, input) {
|
|
9933
|
+
const keys = input.refs.map((ref) => ({
|
|
9934
|
+
entity_kind: ref.kind,
|
|
9935
|
+
entity_key: JSON.parse(projectionEntityKey(ref)),
|
|
9936
|
+
root_keys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
|
|
9937
|
+
utf8SortKey(projectionEntityKey(ref)),
|
|
9938
|
+
utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
|
|
9939
|
+
]
|
|
9940
|
+
}));
|
|
9941
|
+
const previous = !input.includePrevious ? sql`` : sql`UNION ALL
|
|
9942
|
+
SELECT rows.* FROM (
|
|
9943
|
+
SELECT roots.* FROM ontology_source_roots AS roots
|
|
9944
|
+
WHERE roots.project_id = ${input.projectId} AND roots.source_id = ${input.sourceId}
|
|
9945
|
+
AND roots.active AND roots.root_sort_key = ANY(requested.root_keys)
|
|
9946
|
+
${!input.incremental ? sql`` : sql`AND EXISTS (SELECT 1 FROM ontology_source_roots AS changed
|
|
9947
|
+
WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
|
|
9948
|
+
AND changed.materialization_id = ${input.materializationId} AND changed.root_sort_key = roots.root_sort_key)`}
|
|
9949
|
+
|
|
9950
|
+
OFFSET 0
|
|
9951
|
+
) AS roots
|
|
9952
|
+
CROSS JOIN LATERAL (
|
|
9953
|
+
SELECT rows.* FROM ontology_source_rows AS rows
|
|
9954
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
9955
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
9956
|
+
AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
9957
|
+
OFFSET 0
|
|
9958
|
+
) AS rows
|
|
9959
|
+
`;
|
|
9960
|
+
return sql`
|
|
9961
|
+
SELECT selected.* FROM jsonb_to_recordset(${jsonParameter2(sql, keys)})
|
|
9962
|
+
AS requested(entity_kind TEXT, entity_key JSONB, root_keys TEXT[])
|
|
9963
|
+
CROSS JOIN LATERAL (
|
|
9964
|
+
SELECT rows.* FROM ontology_source_rows AS rows
|
|
9965
|
+
WHERE rows.project_id = ${input.projectId} AND rows.source_id = ${input.sourceId}
|
|
9966
|
+
AND rows.materialization_id = ${input.materializationId}
|
|
9967
|
+
AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
9968
|
+
${previous}
|
|
9969
|
+
OFFSET 0
|
|
9970
|
+
) AS selected
|
|
9971
|
+
`;
|
|
9972
|
+
}
|
|
9973
|
+
|
|
9974
|
+
// src/ontology-storage/materialization-state.ts
|
|
9710
9975
|
var PG_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
|
|
9711
9976
|
var PG_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
|
|
9712
9977
|
|
|
@@ -9739,22 +10004,7 @@ class PgMaterializationStateReader {
|
|
|
9739
10004
|
JOIN requested USING (object_type_id, primary_id)
|
|
9740
10005
|
WHERE objects.project_id = ${this.projectId}
|
|
9741
10006
|
`,
|
|
9742
|
-
this.sql
|
|
9743
|
-
WITH requested AS (
|
|
9744
|
-
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
9745
|
-
AS requested_values(object_type_id TEXT, primary_id TEXT)
|
|
9746
|
-
)
|
|
9747
|
-
SELECT rows.*
|
|
9748
|
-
FROM ontology_source_rows AS rows
|
|
9749
|
-
JOIN requested USING (object_type_id, primary_id)
|
|
9750
|
-
JOIN ontology_sources AS sources
|
|
9751
|
-
ON sources.project_id = rows.project_id
|
|
9752
|
-
AND sources.source_id = rows.source_id
|
|
9753
|
-
AND sources.materialization_id = rows.materialization_id
|
|
9754
|
-
WHERE rows.project_id = ${this.projectId}
|
|
9755
|
-
AND rows.entity_kind = 'object'
|
|
9756
|
-
AND sources.status = 'active'
|
|
9757
|
-
`,
|
|
10007
|
+
activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "object", ref }))),
|
|
9758
10008
|
this.sql`
|
|
9759
10009
|
WITH requested AS (
|
|
9760
10010
|
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
|
|
@@ -9795,7 +10045,7 @@ class PgMaterializationStateReader {
|
|
|
9795
10045
|
const effectiveRow = effective.get(key);
|
|
9796
10046
|
return {
|
|
9797
10047
|
ref: structuredClone(ref),
|
|
9798
|
-
source: sourceObject(sources.get(
|
|
10048
|
+
source: sourceObject(sources.get(projectionEntityKey2({ kind: "object", ref }))),
|
|
9799
10049
|
override: overrides.get(key) ?? null,
|
|
9800
10050
|
effective: effectiveRow ? effectiveObjectSnapshot(effectiveRow) : null,
|
|
9801
10051
|
latestTelemetry: telemetry.get(key) ?? []
|
|
@@ -9831,26 +10081,7 @@ class PgMaterializationStateReader {
|
|
|
9831
10081
|
JOIN requested USING (source_type_id, source_id, link_id, target_type_id, target_id)
|
|
9832
10082
|
WHERE links.project_id = ${this.projectId}
|
|
9833
10083
|
`,
|
|
9834
|
-
this.sql
|
|
9835
|
-
WITH requested AS (
|
|
9836
|
-
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
|
|
9837
|
-
source_type_id TEXT, source_primary_id TEXT, link_id TEXT,
|
|
9838
|
-
target_type_id TEXT, target_primary_id TEXT
|
|
9839
|
-
)
|
|
9840
|
-
)
|
|
9841
|
-
SELECT rows.*
|
|
9842
|
-
FROM ontology_source_rows AS rows
|
|
9843
|
-
JOIN requested USING (
|
|
9844
|
-
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
9845
|
-
)
|
|
9846
|
-
JOIN ontology_sources AS sources
|
|
9847
|
-
ON sources.project_id = rows.project_id
|
|
9848
|
-
AND sources.source_id = rows.source_id
|
|
9849
|
-
AND sources.materialization_id = rows.materialization_id
|
|
9850
|
-
WHERE rows.project_id = ${this.projectId}
|
|
9851
|
-
AND rows.entity_kind = 'link'
|
|
9852
|
-
AND sources.status = 'active'
|
|
9853
|
-
`,
|
|
10084
|
+
activeSourceRows(this.sql, this.projectId, refs.map((ref) => ({ kind: "link", ref }))),
|
|
9854
10085
|
this.sql`
|
|
9855
10086
|
WITH requested AS (
|
|
9856
10087
|
SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
|
|
@@ -9889,7 +10120,7 @@ class PgMaterializationStateReader {
|
|
|
9889
10120
|
const effectiveRow = effective.get(key);
|
|
9890
10121
|
return {
|
|
9891
10122
|
ref: structuredClone(ref),
|
|
9892
|
-
source: sourceLink(sources.get(
|
|
10123
|
+
source: sourceLink(sources.get(projectionEntityKey2({ kind: "link", ref }))),
|
|
9893
10124
|
override: overrides.get(key) ?? null,
|
|
9894
10125
|
slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
|
|
9895
10126
|
effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null
|
|
@@ -9958,13 +10189,14 @@ class PgMaterializationStateReader {
|
|
|
9958
10189
|
ON requested.source_type_id = rows.source_type_id
|
|
9959
10190
|
AND requested.source_id = rows.source_primary_id
|
|
9960
10191
|
AND requested.link_id = rows.link_id
|
|
9961
|
-
JOIN
|
|
10192
|
+
JOIN ontology_source_roots AS sources
|
|
9962
10193
|
ON sources.project_id = rows.project_id
|
|
9963
10194
|
AND sources.source_id = rows.source_id
|
|
9964
10195
|
AND sources.materialization_id = rows.materialization_id
|
|
10196
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
9965
10197
|
WHERE rows.project_id = ${this.projectId}
|
|
9966
10198
|
AND rows.entity_kind = 'link'
|
|
9967
|
-
AND sources.
|
|
10199
|
+
AND sources.active
|
|
9968
10200
|
`,
|
|
9969
10201
|
this.sql`
|
|
9970
10202
|
WITH requested AS (
|
|
@@ -10037,7 +10269,7 @@ class PgMaterializationStateReader {
|
|
|
10037
10269
|
for (const row of rows) {
|
|
10038
10270
|
const accumulator = accumulators.get(row.scope_sort_key);
|
|
10039
10271
|
if (!accumulator) {
|
|
10040
|
-
throw new
|
|
10272
|
+
throw new MaterializationConflictError3("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
|
|
10041
10273
|
}
|
|
10042
10274
|
const snapshot = effectiveLinkSnapshot(row);
|
|
10043
10275
|
appendScopeSnapshot(accumulator, snapshot);
|
|
@@ -10106,30 +10338,32 @@ class PgMaterializationStateReader {
|
|
|
10106
10338
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10107
10339
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10108
10340
|
FROM ontology_source_rows AS rows
|
|
10109
|
-
JOIN
|
|
10341
|
+
JOIN ontology_source_roots AS sources
|
|
10110
10342
|
ON sources.project_id = rows.project_id
|
|
10111
10343
|
AND sources.source_id = rows.source_id
|
|
10112
10344
|
AND sources.materialization_id = rows.materialization_id
|
|
10345
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10113
10346
|
JOIN requested
|
|
10114
10347
|
ON requested.object_type_id = rows.source_type_id
|
|
10115
10348
|
AND requested.primary_id = rows.source_primary_id
|
|
10116
10349
|
WHERE rows.project_id = ${this.projectId}
|
|
10117
10350
|
AND rows.entity_kind = 'link'
|
|
10118
|
-
AND sources.
|
|
10351
|
+
AND sources.active
|
|
10119
10352
|
UNION
|
|
10120
10353
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10121
10354
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10122
10355
|
FROM ontology_source_rows AS rows
|
|
10123
|
-
JOIN
|
|
10356
|
+
JOIN ontology_source_roots AS sources
|
|
10124
10357
|
ON sources.project_id = rows.project_id
|
|
10125
10358
|
AND sources.source_id = rows.source_id
|
|
10126
10359
|
AND sources.materialization_id = rows.materialization_id
|
|
10360
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10127
10361
|
JOIN requested
|
|
10128
10362
|
ON requested.object_type_id = rows.target_type_id
|
|
10129
10363
|
AND requested.primary_id = rows.target_primary_id
|
|
10130
10364
|
WHERE rows.project_id = ${this.projectId}
|
|
10131
10365
|
AND rows.entity_kind = 'link'
|
|
10132
|
-
AND sources.
|
|
10366
|
+
AND sources.active
|
|
10133
10367
|
), selected AS (
|
|
10134
10368
|
SELECT DISTINCT links.*,
|
|
10135
10369
|
${this.sql.unsafe(linkSortExpression("links"))} AS sort_key
|
|
@@ -10148,13 +10382,15 @@ class PgMaterializationStateReader {
|
|
|
10148
10382
|
}
|
|
10149
10383
|
async* replacementIdentities(input) {
|
|
10150
10384
|
await this.prepareReplacementIdentities(input);
|
|
10385
|
+
await this.sql`ANALYZE ${this.sql(PG_REPLACEMENT_WORK_TABLE)}`;
|
|
10151
10386
|
let cursor = null;
|
|
10152
10387
|
while (true) {
|
|
10388
|
+
const after = cursor === null ? this.sql`` : this.sql`AND sort_key > ${cursor}`;
|
|
10153
10389
|
const rows = await this.sql`
|
|
10154
10390
|
SELECT entity_kind, identity_key, sort_key, diff_required
|
|
10155
10391
|
FROM ${this.sql(PG_REPLACEMENT_WORK_TABLE)}
|
|
10156
10392
|
WHERE session_id = ${input.sessionId} AND entity_kind = ${input.kind}
|
|
10157
|
-
|
|
10393
|
+
${after}
|
|
10158
10394
|
ORDER BY sort_key
|
|
10159
10395
|
LIMIT ${input.pageRows}
|
|
10160
10396
|
`;
|
|
@@ -10169,19 +10405,19 @@ class PgMaterializationStateReader {
|
|
|
10169
10405
|
const candidates = await this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
|
|
10170
10406
|
return base.map((state) => ({
|
|
10171
10407
|
ref: state.ref,
|
|
10172
|
-
candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(
|
|
10408
|
+
candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey2({ kind: "object", ref: state.ref }))),
|
|
10173
10409
|
override: state.override,
|
|
10174
10410
|
effective: state.effective,
|
|
10175
10411
|
latestTelemetry: state.latestTelemetry
|
|
10176
10412
|
}));
|
|
10177
10413
|
}
|
|
10178
|
-
async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities) {
|
|
10414
|
+
async replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities, incremental = false) {
|
|
10179
10415
|
const refs = identities.map((identity) => identity.ref);
|
|
10180
10416
|
const base = await this.linkStates(refs);
|
|
10181
|
-
const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })));
|
|
10417
|
+
const replacements = await this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })), incremental);
|
|
10182
10418
|
const candidate = replacements.byMaterialization.get(candidateMaterializationId);
|
|
10183
10419
|
return base.map((state, index) => {
|
|
10184
|
-
const key =
|
|
10420
|
+
const key = projectionEntityKey2({ kind: "link", ref: state.ref });
|
|
10185
10421
|
return {
|
|
10186
10422
|
ref: state.ref,
|
|
10187
10423
|
candidateSource: replacements.owned.has(key) ? sourceLink(candidate?.get(key)) : state.source,
|
|
@@ -10248,33 +10484,23 @@ class PgMaterializationStateReader {
|
|
|
10248
10484
|
`;
|
|
10249
10485
|
return new Map(rows.map((row) => [linkRefKey(linkRefFromColumns(row)), row.last_commit_id]));
|
|
10250
10486
|
}
|
|
10251
|
-
async replacementSources(sourceId, materializationIds, refs) {
|
|
10487
|
+
async replacementSources(sourceId, materializationIds, refs, incremental = false) {
|
|
10252
10488
|
if (materializationIds.length === 0 || refs.length === 0) {
|
|
10253
10489
|
return { owned: new Set, byMaterialization: new Map };
|
|
10254
10490
|
}
|
|
10255
|
-
const
|
|
10256
|
-
|
|
10257
|
-
|
|
10258
|
-
|
|
10259
|
-
|
|
10260
|
-
|
|
10261
|
-
|
|
10262
|
-
|
|
10263
|
-
AS requested_values(entity_kind TEXT, entity_key JSONB)
|
|
10264
|
-
)
|
|
10265
|
-
SELECT rows.* FROM ontology_source_rows AS rows
|
|
10266
|
-
JOIN requested USING (entity_kind, entity_key)
|
|
10267
|
-
WHERE rows.project_id = ${this.projectId}
|
|
10268
|
-
AND rows.source_id = ${sourceId}
|
|
10269
|
-
AND rows.materialization_id = ANY(
|
|
10270
|
-
${this.sql.array([...new Set(materializationIds)])}::text[]
|
|
10271
|
-
)
|
|
10272
|
-
`;
|
|
10491
|
+
const rows = await replacementAssertionRows(this.sql, {
|
|
10492
|
+
projectId: this.projectId,
|
|
10493
|
+
sourceId,
|
|
10494
|
+
materializationId: materializationIds[0],
|
|
10495
|
+
incremental,
|
|
10496
|
+
includePrevious: materializationIds.length > 1,
|
|
10497
|
+
refs
|
|
10498
|
+
});
|
|
10273
10499
|
const owned = new Set;
|
|
10274
10500
|
const byMaterialization = new Map;
|
|
10275
10501
|
for (const row of rows) {
|
|
10276
10502
|
const source = storedSource(row);
|
|
10277
|
-
const key =
|
|
10503
|
+
const key = projectionEntityKey2(source.assertion);
|
|
10278
10504
|
owned.add(key);
|
|
10279
10505
|
const materialization = byMaterialization.get(row.materialization_id) ?? new Map;
|
|
10280
10506
|
materialization.set(key, source);
|
|
@@ -10290,20 +10516,19 @@ class PgMaterializationStateReader {
|
|
|
10290
10516
|
await this.prepareReplacementLinks(input);
|
|
10291
10517
|
}
|
|
10292
10518
|
async prepareReplacementObjects(input) {
|
|
10293
|
-
const
|
|
10294
|
-
|
|
10295
|
-
|
|
10296
|
-
|
|
10519
|
+
const rows = replacementSourceRows(this.sql, {
|
|
10520
|
+
projectId: this.projectId,
|
|
10521
|
+
sourceId: input.sourceId,
|
|
10522
|
+
materializationId: input.candidateMaterializationId,
|
|
10523
|
+
incremental: input.incremental ?? false
|
|
10524
|
+
});
|
|
10297
10525
|
await this.sql`
|
|
10298
10526
|
WITH selected AS (
|
|
10299
10527
|
SELECT DISTINCT object_type_id, primary_id,
|
|
10300
10528
|
${this.sql.unsafe(objectKeyExpression("rows"))} AS identity_key,
|
|
10301
10529
|
${this.sql.unsafe(objectSortExpression("rows"))} AS sort_key
|
|
10302
|
-
FROM
|
|
10303
|
-
WHERE
|
|
10304
|
-
AND source_id = ${input.sourceId}
|
|
10305
|
-
AND entity_kind = 'object'
|
|
10306
|
-
AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
|
|
10530
|
+
FROM (${rows}) AS rows
|
|
10531
|
+
WHERE entity_kind = 'object'
|
|
10307
10532
|
)
|
|
10308
10533
|
INSERT INTO ${this.sql(PG_REPLACEMENT_WORK_TABLE)} (
|
|
10309
10534
|
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
@@ -10316,10 +10541,13 @@ class PgMaterializationStateReader {
|
|
|
10316
10541
|
`;
|
|
10317
10542
|
}
|
|
10318
10543
|
async prepareReplacementLinks(input) {
|
|
10319
|
-
|
|
10320
|
-
|
|
10321
|
-
|
|
10322
|
-
|
|
10544
|
+
await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
|
|
10545
|
+
const rows = replacementSourceRows(this.sql, {
|
|
10546
|
+
projectId: this.projectId,
|
|
10547
|
+
sourceId: input.sourceId,
|
|
10548
|
+
materializationId: input.candidateMaterializationId,
|
|
10549
|
+
incremental: input.incremental ?? false
|
|
10550
|
+
});
|
|
10323
10551
|
await this.sql`
|
|
10324
10552
|
WITH incident_objects AS (
|
|
10325
10553
|
SELECT DISTINCT payload->'ref'->>'objectTypeId' AS object_type_id,
|
|
@@ -10329,11 +10557,8 @@ class PgMaterializationStateReader {
|
|
|
10329
10557
|
), replacement_links AS (
|
|
10330
10558
|
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
10331
10559
|
target_type_id, target_primary_id AS target_id
|
|
10332
|
-
FROM
|
|
10333
|
-
WHERE
|
|
10334
|
-
AND source_id = ${input.sourceId}
|
|
10335
|
-
AND entity_kind = 'link'
|
|
10336
|
-
AND materialization_id = ANY(${this.sql.array(materializationIds)}::text[])
|
|
10560
|
+
FROM (${rows}) AS rows
|
|
10561
|
+
WHERE entity_kind = 'link'
|
|
10337
10562
|
), incident_links AS (
|
|
10338
10563
|
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
10339
10564
|
links.target_type_id, links.target_id
|
|
@@ -10372,30 +10597,32 @@ class PgMaterializationStateReader {
|
|
|
10372
10597
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10373
10598
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10374
10599
|
FROM ontology_source_rows AS rows
|
|
10375
|
-
JOIN
|
|
10600
|
+
JOIN ontology_source_roots AS sources
|
|
10376
10601
|
ON sources.project_id = rows.project_id
|
|
10377
10602
|
AND sources.source_id = rows.source_id
|
|
10378
10603
|
AND sources.materialization_id = rows.materialization_id
|
|
10604
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10379
10605
|
JOIN incident_objects
|
|
10380
10606
|
ON incident_objects.object_type_id = rows.source_type_id
|
|
10381
10607
|
AND incident_objects.primary_id = rows.source_primary_id
|
|
10382
10608
|
WHERE rows.project_id = ${this.projectId}
|
|
10383
10609
|
AND rows.entity_kind = 'link'
|
|
10384
|
-
AND sources.
|
|
10610
|
+
AND sources.active
|
|
10385
10611
|
UNION
|
|
10386
10612
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
10387
10613
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
10388
10614
|
FROM ontology_source_rows AS rows
|
|
10389
|
-
JOIN
|
|
10615
|
+
JOIN ontology_source_roots AS sources
|
|
10390
10616
|
ON sources.project_id = rows.project_id
|
|
10391
10617
|
AND sources.source_id = rows.source_id
|
|
10392
10618
|
AND sources.materialization_id = rows.materialization_id
|
|
10619
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
10393
10620
|
JOIN incident_objects
|
|
10394
10621
|
ON incident_objects.object_type_id = rows.target_type_id
|
|
10395
10622
|
AND incident_objects.primary_id = rows.target_primary_id
|
|
10396
10623
|
WHERE rows.project_id = ${this.projectId}
|
|
10397
10624
|
AND rows.entity_kind = 'link'
|
|
10398
|
-
AND sources.
|
|
10625
|
+
AND sources.active
|
|
10399
10626
|
), diff_links AS (
|
|
10400
10627
|
SELECT * FROM replacement_links
|
|
10401
10628
|
UNION SELECT * FROM incident_links
|
|
@@ -10463,9 +10690,9 @@ function activeSourceMap(rows) {
|
|
|
10463
10690
|
const result = new Map;
|
|
10464
10691
|
for (const row of rows) {
|
|
10465
10692
|
const source = storedSource(row);
|
|
10466
|
-
const key =
|
|
10693
|
+
const key = projectionEntityKey2(source.assertion);
|
|
10467
10694
|
if (result.has(key)) {
|
|
10468
|
-
throw new
|
|
10695
|
+
throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert ${key}.`);
|
|
10469
10696
|
}
|
|
10470
10697
|
result.set(key, source);
|
|
10471
10698
|
}
|
|
@@ -10480,7 +10707,7 @@ function activeScopeSourceMap(rows) {
|
|
|
10480
10707
|
const ref = source.assertion.ref;
|
|
10481
10708
|
const key = linkScopeSortKey(ref.source, ref.linkId);
|
|
10482
10709
|
if (result.has(key)) {
|
|
10483
|
-
throw new
|
|
10710
|
+
throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
|
|
10484
10711
|
}
|
|
10485
10712
|
result.set(key, source);
|
|
10486
10713
|
}
|
|
@@ -10544,7 +10771,7 @@ function storedSource(row) {
|
|
|
10544
10771
|
}
|
|
10545
10772
|
function effectiveObjectSnapshot(row) {
|
|
10546
10773
|
if (!row.last_commit_id) {
|
|
10547
|
-
throw new
|
|
10774
|
+
throw new MaterializationConflictError3("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
|
|
10548
10775
|
}
|
|
10549
10776
|
return {
|
|
10550
10777
|
ref: objectRefFromColumns(row),
|
|
@@ -10557,7 +10784,7 @@ function effectiveObjectSnapshot(row) {
|
|
|
10557
10784
|
}
|
|
10558
10785
|
function effectiveLinkSnapshot(row) {
|
|
10559
10786
|
if (!row.last_commit_id) {
|
|
10560
|
-
throw new
|
|
10787
|
+
throw new MaterializationConflictError3("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
|
|
10561
10788
|
}
|
|
10562
10789
|
return {
|
|
10563
10790
|
ref: linkRefFromColumns(row),
|
|
@@ -10572,7 +10799,7 @@ function effectiveLinkSnapshot(row) {
|
|
|
10572
10799
|
function storedPoint(row) {
|
|
10573
10800
|
const at = toIsoString2(row.at);
|
|
10574
10801
|
if (!row.last_commit_id) {
|
|
10575
|
-
throw new
|
|
10802
|
+
throw new MaterializationConflictError3("timeseries-point", `Telemetry point ${telemetryPointSortKey({
|
|
10576
10803
|
object: { objectTypeId: row.object_type_id, primaryId: row.object_id },
|
|
10577
10804
|
propertyId: row.property_id
|
|
10578
10805
|
}, at)} lacks materializer provenance.`);
|
|
@@ -10631,6 +10858,9 @@ function jsonTupleExpression(parts) {
|
|
|
10631
10858
|
|
|
10632
10859
|
// src/ontology-storage/materialization-session.ts
|
|
10633
10860
|
class PgMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
10861
|
+
stagedWorkCount = 0;
|
|
10862
|
+
changedObjects = 0;
|
|
10863
|
+
changedLinks = 0;
|
|
10634
10864
|
appliedPlanCursor = null;
|
|
10635
10865
|
appliedEventCursor = null;
|
|
10636
10866
|
}
|
|
@@ -10647,7 +10877,7 @@ class PgMaterializationSessions {
|
|
|
10647
10877
|
}
|
|
10648
10878
|
async create(header) {
|
|
10649
10879
|
if (!this.context?.active) {
|
|
10650
|
-
throw new
|
|
10880
|
+
throw new MaterializationConflictError4("effective-state", "Materialization sessions require an active storage transaction.");
|
|
10651
10881
|
}
|
|
10652
10882
|
await this.ensureTables();
|
|
10653
10883
|
const session = new PgMaterializationSessionState(structuredClone(header), this.context.id);
|
|
@@ -10659,7 +10889,7 @@ class PgMaterializationSessions {
|
|
|
10659
10889
|
require(session) {
|
|
10660
10890
|
const value = this.sessions.get(session.providerToken);
|
|
10661
10891
|
if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
|
|
10662
|
-
throw new
|
|
10892
|
+
throw new MaterializationConflictError4("effective-state", "Materialization session is inactive.");
|
|
10663
10893
|
}
|
|
10664
10894
|
return value;
|
|
10665
10895
|
}
|
|
@@ -10717,14 +10947,18 @@ class PgMaterializationSessions {
|
|
|
10717
10947
|
}
|
|
10718
10948
|
throw error;
|
|
10719
10949
|
}
|
|
10950
|
+
session.stagedWorkCount += records.length;
|
|
10720
10951
|
}
|
|
10721
10952
|
async* stream(input) {
|
|
10722
10953
|
const session = this.require(input.session);
|
|
10723
10954
|
const stream = session.workStreams[input.order];
|
|
10724
10955
|
if (stream.started) {
|
|
10725
|
-
throw new
|
|
10956
|
+
throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
|
|
10726
10957
|
}
|
|
10727
10958
|
stream.started = true;
|
|
10959
|
+
if (!session.workSealed && session.stagedWorkCount >= 1e4) {
|
|
10960
|
+
await this.sql`ANALYZE ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}`;
|
|
10961
|
+
}
|
|
10728
10962
|
session.workSealed = true;
|
|
10729
10963
|
let cursor = null;
|
|
10730
10964
|
while (true) {
|
|
@@ -10745,10 +10979,13 @@ class PgMaterializationSessions {
|
|
|
10745
10979
|
return [];
|
|
10746
10980
|
const keys = refs.map((ref) => `object-existence:${JSON.stringify([ref.objectTypeId, ref.primaryId])}`);
|
|
10747
10981
|
const rows = await this.sql`
|
|
10748
|
-
SELECT unique_key, payload
|
|
10749
|
-
|
|
10750
|
-
|
|
10751
|
-
|
|
10982
|
+
SELECT work.unique_key, work.payload
|
|
10983
|
+
FROM unnest(${this.sql.array(keys)}::text[]) AS requested(unique_key)
|
|
10984
|
+
CROSS JOIN LATERAL (
|
|
10985
|
+
SELECT unique_key, payload FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
|
|
10986
|
+
WHERE session_id = ${session.id} AND unique_key = requested.unique_key
|
|
10987
|
+
OFFSET 0
|
|
10988
|
+
) AS work
|
|
10752
10989
|
`;
|
|
10753
10990
|
const found = new Map(rows.map((row) => [row.unique_key, row.payload]));
|
|
10754
10991
|
return keys.flatMap((key) => {
|
|
@@ -10756,6 +10993,9 @@ class PgMaterializationSessions {
|
|
|
10756
10993
|
if (!payload)
|
|
10757
10994
|
return [];
|
|
10758
10995
|
const record = structuredClone(payload);
|
|
10996
|
+
if (record.kind !== "object-existence") {
|
|
10997
|
+
invalidCorrelation("Object existence work has an invalid kind.");
|
|
10998
|
+
}
|
|
10759
10999
|
return [{ ref: record.ref, exists: record.exists }];
|
|
10760
11000
|
});
|
|
10761
11001
|
}
|
|
@@ -10910,7 +11150,7 @@ function workCursor(row) {
|
|
|
10910
11150
|
// src/ontology-storage/materialization-writer.ts
|
|
10911
11151
|
import {
|
|
10912
11152
|
linkRefKey as linkRefKey2,
|
|
10913
|
-
MaterializationConflictError as
|
|
11153
|
+
MaterializationConflictError as MaterializationConflictError5,
|
|
10914
11154
|
objectRefKey as objectRefKey2,
|
|
10915
11155
|
telemetryPointKey as telemetryPointKey2
|
|
10916
11156
|
} from "@sixb/core/internal/materialization";
|
|
@@ -11426,7 +11666,7 @@ function effectiveLinkConflict(item) {
|
|
|
11426
11666
|
return effectiveConflict(`Expected link ${linkIdentityKey(item)} changed.`);
|
|
11427
11667
|
}
|
|
11428
11668
|
function pointConflict(item) {
|
|
11429
|
-
return new
|
|
11669
|
+
return new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey2({
|
|
11430
11670
|
object: { objectTypeId: item.objectTypeId, primaryId: item.objectId },
|
|
11431
11671
|
propertyId: item.propertyId
|
|
11432
11672
|
}, item.at)} changed.`);
|
|
@@ -11461,14 +11701,14 @@ class PgOntologyMaterializationStorage {
|
|
|
11461
11701
|
const linkScopeRevisions = await reader.linkScopeRevisions(input.expected.linkScopes);
|
|
11462
11702
|
for (const [index, expected] of input.expected.linkScopes.entries()) {
|
|
11463
11703
|
if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
|
|
11464
|
-
throw new
|
|
11704
|
+
throw new MaterializationConflictError6("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
|
|
11465
11705
|
}
|
|
11466
11706
|
}
|
|
11467
11707
|
const points = await reader.exactPoints(input.expected.points, true);
|
|
11468
11708
|
const pointRevisions = new Map(points.map((point) => [telemetryPointKey3(point.series, point.at), point.lastCommitId]));
|
|
11469
11709
|
for (const expected of input.expected.points) {
|
|
11470
11710
|
if ((pointRevisions.get(telemetryPointKey3(expected.series, expected.at)) ?? null) !== expected.lastCommitId) {
|
|
11471
|
-
throw new
|
|
11711
|
+
throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey3(expected.series, expected.at)} changed.`);
|
|
11472
11712
|
}
|
|
11473
11713
|
}
|
|
11474
11714
|
return session.publicSession();
|
|
@@ -11527,10 +11767,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11527
11767
|
const reader = new PgMaterializationStateReader(this.sql, session.header.commit.projectId);
|
|
11528
11768
|
if (input.entityKind === "object") {
|
|
11529
11769
|
if (replacement.projectionKind !== "object") {
|
|
11530
|
-
throw new
|
|
11770
|
+
throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
|
|
11531
11771
|
}
|
|
11532
11772
|
if (replacement.objectStreamStarted) {
|
|
11533
|
-
throw new
|
|
11773
|
+
throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
|
|
11534
11774
|
}
|
|
11535
11775
|
replacement.objectStreamStarted = true;
|
|
11536
11776
|
for await (const identities of reader.replacementIdentities({
|
|
@@ -11538,6 +11778,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11538
11778
|
sourceId: replacement.sourceId,
|
|
11539
11779
|
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
11540
11780
|
previousMaterializationId: replacement.previousMaterializationId,
|
|
11781
|
+
incremental: replacement.incremental,
|
|
11541
11782
|
kind: "object",
|
|
11542
11783
|
pageRows: input.pageRows
|
|
11543
11784
|
})) {
|
|
@@ -11556,10 +11797,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11556
11797
|
return;
|
|
11557
11798
|
}
|
|
11558
11799
|
if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
|
|
11559
|
-
throw new
|
|
11800
|
+
throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
|
|
11560
11801
|
}
|
|
11561
11802
|
if (replacement.linkStreamStarted) {
|
|
11562
|
-
throw new
|
|
11803
|
+
throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
|
|
11563
11804
|
}
|
|
11564
11805
|
replacement.linkStreamStarted = true;
|
|
11565
11806
|
const materializationIds = [
|
|
@@ -11571,6 +11812,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11571
11812
|
sourceId: replacement.sourceId,
|
|
11572
11813
|
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
11573
11814
|
previousMaterializationId: replacement.previousMaterializationId,
|
|
11815
|
+
incremental: replacement.incremental,
|
|
11574
11816
|
kind: "link",
|
|
11575
11817
|
pageRows: input.pageRows
|
|
11576
11818
|
})) {
|
|
@@ -11581,7 +11823,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11581
11823
|
}
|
|
11582
11824
|
return identity;
|
|
11583
11825
|
});
|
|
11584
|
-
const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities);
|
|
11826
|
+
const links = await reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
|
|
11585
11827
|
yield { objects: [], links };
|
|
11586
11828
|
}
|
|
11587
11829
|
this.sessions.require(input.session);
|
|
@@ -11604,6 +11846,8 @@ class PgOntologyMaterializationStorage {
|
|
|
11604
11846
|
assertPlanChunkCorrelations(input.chunk, commit);
|
|
11605
11847
|
const progress = await this.sessions.prepareChunkSequence(session, input.chunk);
|
|
11606
11848
|
await this.writer.apply(commit.projectId, commit.id, input.chunk);
|
|
11849
|
+
session.changedObjects += input.chunk.effective.objectUpserts.length + input.chunk.effective.objectDeletes.length;
|
|
11850
|
+
session.changedLinks += input.chunk.effective.linkUpserts.length + input.chunk.effective.linkDeletes.length;
|
|
11607
11851
|
this.sessions.commitChunkSequence(session, progress);
|
|
11608
11852
|
}
|
|
11609
11853
|
async finalize(input) {
|
|
@@ -11613,6 +11857,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11613
11857
|
for (const activation of input.finalization.sourceActivations) {
|
|
11614
11858
|
await this.activateSource(session, activation);
|
|
11615
11859
|
}
|
|
11860
|
+
if (session.changedObjects >= 1000)
|
|
11861
|
+
await this.sql`ANALYZE objects`;
|
|
11862
|
+
if (session.changedLinks >= 1000)
|
|
11863
|
+
await this.sql`ANALYZE links`;
|
|
11616
11864
|
const record = await this.insertCommit(session.header, input);
|
|
11617
11865
|
await this.sessions.release(session);
|
|
11618
11866
|
return { commit: record };
|
|
@@ -11623,14 +11871,14 @@ class PgOntologyMaterializationStorage {
|
|
|
11623
11871
|
async requireReplacement(session, input) {
|
|
11624
11872
|
if (session.replacement) {
|
|
11625
11873
|
if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
|
|
11626
|
-
throw new
|
|
11874
|
+
throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
|
|
11627
11875
|
}
|
|
11628
11876
|
return session.replacement;
|
|
11629
11877
|
}
|
|
11630
11878
|
const projectId = session.header.commit.projectId;
|
|
11631
11879
|
const candidate = await this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId, true);
|
|
11632
11880
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
11633
|
-
throw new
|
|
11881
|
+
throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
|
|
11634
11882
|
}
|
|
11635
11883
|
await assertProjectionExecution(this.sql, {
|
|
11636
11884
|
projectId,
|
|
@@ -11640,12 +11888,13 @@ class PgOntologyMaterializationStorage {
|
|
|
11640
11888
|
});
|
|
11641
11889
|
const previous = await this.getActiveSource(projectId, input.source.projectionId, true);
|
|
11642
11890
|
if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
|
|
11643
|
-
throw new
|
|
11891
|
+
throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
|
|
11644
11892
|
}
|
|
11645
11893
|
session.replacement = {
|
|
11646
11894
|
sourceId: input.source.projectionId,
|
|
11647
11895
|
candidateMaterializationId: input.candidateMaterializationId,
|
|
11648
11896
|
previousMaterializationId: previous?.materialization_id ?? null,
|
|
11897
|
+
incremental: candidate.base_materialization_id !== null,
|
|
11649
11898
|
projectionKind: candidate.projection_kind,
|
|
11650
11899
|
objectStreamStarted: false,
|
|
11651
11900
|
objectStreamCompleted: false,
|
|
@@ -11665,7 +11914,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11665
11914
|
FOR UPDATE
|
|
11666
11915
|
`;
|
|
11667
11916
|
if (duplicateIdentity) {
|
|
11668
|
-
throw new
|
|
11917
|
+
throw new MaterializationConflictError6("idempotency", duplicateIdentity.id === header.commit.id ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
|
|
11669
11918
|
}
|
|
11670
11919
|
const origin = originColumns(header.commit.origin);
|
|
11671
11920
|
if (origin.runId === null)
|
|
@@ -11688,7 +11937,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11688
11937
|
FOR UPDATE
|
|
11689
11938
|
`;
|
|
11690
11939
|
if (duplicateOrigin) {
|
|
11691
|
-
throw new
|
|
11940
|
+
throw new MaterializationConflictError6("run-correlation", "Ontology commit origin already has an authoritative commit.");
|
|
11692
11941
|
}
|
|
11693
11942
|
}
|
|
11694
11943
|
async assertSources(expectedSources, projectId) {
|
|
@@ -11710,29 +11959,29 @@ class PgOntologyMaterializationStorage {
|
|
|
11710
11959
|
}
|
|
11711
11960
|
assertSourceRow(expected, active) {
|
|
11712
11961
|
if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
|
|
11713
|
-
throw new
|
|
11962
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${expected.source.projectionId}' changed.`);
|
|
11714
11963
|
}
|
|
11715
11964
|
}
|
|
11716
11965
|
assertObject(row, expected) {
|
|
11717
11966
|
if (!expected.exists) {
|
|
11718
11967
|
if (row) {
|
|
11719
|
-
throw new
|
|
11968
|
+
throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
|
|
11720
11969
|
}
|
|
11721
11970
|
return;
|
|
11722
11971
|
}
|
|
11723
11972
|
if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
|
|
11724
|
-
throw new
|
|
11973
|
+
throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
|
|
11725
11974
|
}
|
|
11726
11975
|
}
|
|
11727
11976
|
assertLink(lastCommitId, expected) {
|
|
11728
11977
|
if (!expected.exists) {
|
|
11729
11978
|
if (lastCommitId !== undefined) {
|
|
11730
|
-
throw new
|
|
11979
|
+
throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
|
|
11731
11980
|
}
|
|
11732
11981
|
return;
|
|
11733
11982
|
}
|
|
11734
11983
|
if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
|
|
11735
|
-
throw new
|
|
11984
|
+
throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
|
|
11736
11985
|
}
|
|
11737
11986
|
}
|
|
11738
11987
|
async assertFinalization(session, input) {
|
|
@@ -11741,7 +11990,8 @@ class PgOntologyMaterializationStorage {
|
|
|
11741
11990
|
assertMaterializationFinalizationCorrelation(session, input);
|
|
11742
11991
|
const laneCounts = await this.sessions.laneCounts(session);
|
|
11743
11992
|
assertMaterializationLaneCompletion(session, laneCounts);
|
|
11744
|
-
|
|
11993
|
+
if (laneCounts.cardinality > 0)
|
|
11994
|
+
await this.assertFinalCardinality(session);
|
|
11745
11995
|
const eventCount = laneCounts.event;
|
|
11746
11996
|
const [outbox] = await this.sql`
|
|
11747
11997
|
SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum,
|
|
@@ -11774,17 +12024,18 @@ class PgOntologyMaterializationStorage {
|
|
|
11774
12024
|
}
|
|
11775
12025
|
}
|
|
11776
12026
|
async assertFinalCardinality(session) {
|
|
11777
|
-
const
|
|
11778
|
-
|
|
11779
|
-
SELECT
|
|
11780
|
-
payload->>'linkSortKey' AS link_sort_key,
|
|
12027
|
+
const table = `ontology_cardinality_${session.id.replaceAll("-", "")}`;
|
|
12028
|
+
await this.sql.unsafe(`CREATE TEMP TABLE ${table} ON COMMIT DROP AS
|
|
12029
|
+
SELECT sort_one AS scope_sort_key, sort_two AS link_sort_key,
|
|
11781
12030
|
(payload->>'occupied')::boolean AS occupied,
|
|
11782
12031
|
payload->'ref'->'source'->>'objectTypeId' AS source_type_id,
|
|
11783
12032
|
payload->'ref'->'source'->>'primaryId' AS source_id,
|
|
11784
12033
|
payload->'ref'->>'linkId' AS link_id
|
|
11785
|
-
FROM ${
|
|
11786
|
-
WHERE session_id = $
|
|
11787
|
-
|
|
12034
|
+
FROM ${PG_MATERIALIZATION_WORK_TABLE}
|
|
12035
|
+
WHERE session_id = $1 AND lane = 'cardinality' AND major_order = 0`, [session.id]);
|
|
12036
|
+
await this.sql.unsafe(`ANALYZE ${table}`);
|
|
12037
|
+
const [violation] = await this.sql.unsafe(`WITH work AS (
|
|
12038
|
+
SELECT * FROM ${table}
|
|
11788
12039
|
), duplicate AS (
|
|
11789
12040
|
SELECT scope_sort_key FROM work WHERE occupied
|
|
11790
12041
|
GROUP BY scope_sort_key HAVING COUNT(*) > 1
|
|
@@ -11794,10 +12045,10 @@ class PgOntologyMaterializationStorage {
|
|
|
11794
12045
|
SELECT scope_sort_key, link_sort_key FROM work WHERE occupied
|
|
11795
12046
|
), actual AS (
|
|
11796
12047
|
SELECT scopes.scope_sort_key,
|
|
11797
|
-
${
|
|
12048
|
+
${linkSortExpression("links")} AS link_sort_key
|
|
11798
12049
|
FROM scopes
|
|
11799
12050
|
JOIN links USING (source_type_id, source_id, link_id)
|
|
11800
|
-
WHERE links.project_id = $
|
|
12051
|
+
WHERE links.project_id = $1
|
|
11801
12052
|
), differences AS (
|
|
11802
12053
|
(SELECT * FROM expected EXCEPT SELECT * FROM actual)
|
|
11803
12054
|
UNION ALL
|
|
@@ -11806,8 +12057,8 @@ class PgOntologyMaterializationStorage {
|
|
|
11806
12057
|
SELECT 'duplicate'::text AS reason FROM duplicate
|
|
11807
12058
|
UNION ALL
|
|
11808
12059
|
SELECT 'mismatch'::text AS reason FROM differences
|
|
11809
|
-
LIMIT 1
|
|
11810
|
-
|
|
12060
|
+
LIMIT 1`, [session.header.commit.projectId]);
|
|
12061
|
+
await this.sql.unsafe(`DROP TABLE ${table}`);
|
|
11811
12062
|
if (violation?.reason === "duplicate") {
|
|
11812
12063
|
invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
|
|
11813
12064
|
}
|
|
@@ -11903,7 +12154,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11903
12154
|
assertSourceActivationCorrelation(session, activation);
|
|
11904
12155
|
const candidate = await this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId, true);
|
|
11905
12156
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
11906
|
-
throw new
|
|
12157
|
+
throw new MaterializationConflictError6("source-materialization", "Source activation candidate is missing or is not ready.");
|
|
11907
12158
|
}
|
|
11908
12159
|
await assertProjectionExecution(this.sql, {
|
|
11909
12160
|
projectId: commit.projectId,
|
|
@@ -11916,6 +12167,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11916
12167
|
}
|
|
11917
12168
|
const previous = await this.getActiveSource(commit.projectId, activation.source.projectionId, true);
|
|
11918
12169
|
this.assertSourceRow(activation.expected, previous);
|
|
12170
|
+
await activateSourceRoots(this.sql, commit.projectId, candidate, activation);
|
|
11919
12171
|
if (previous) {
|
|
11920
12172
|
assertPinnedDatasetWatermark({
|
|
11921
12173
|
datasetId: previous.dataset_id,
|
|
@@ -11937,7 +12189,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11937
12189
|
RETURNING materialization_id
|
|
11938
12190
|
`;
|
|
11939
12191
|
if (superseded.length !== 1) {
|
|
11940
|
-
throw new
|
|
12192
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
|
|
11941
12193
|
}
|
|
11942
12194
|
}
|
|
11943
12195
|
let activated;
|
|
@@ -11956,12 +12208,12 @@ class PgOntologyMaterializationStorage {
|
|
|
11956
12208
|
`;
|
|
11957
12209
|
} catch (error) {
|
|
11958
12210
|
if (isUniqueViolation(error)) {
|
|
11959
|
-
throw new
|
|
12211
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${activation.source.projectionId}' changed.`);
|
|
11960
12212
|
}
|
|
11961
12213
|
throw error;
|
|
11962
12214
|
}
|
|
11963
12215
|
if (activated.length !== 1) {
|
|
11964
|
-
throw new
|
|
12216
|
+
throw new MaterializationConflictError6("source-materialization", "Source activation candidate changed.");
|
|
11965
12217
|
}
|
|
11966
12218
|
}
|
|
11967
12219
|
async insertCommit(header, input) {
|
|
@@ -11988,7 +12240,7 @@ class PgOntologyMaterializationStorage {
|
|
|
11988
12240
|
`;
|
|
11989
12241
|
if (!rows[0]) {
|
|
11990
12242
|
await this.assertCommitAbsent(header);
|
|
11991
|
-
throw new
|
|
12243
|
+
throw new MaterializationConflictError6("idempotency", "Ontology commit identity already exists.");
|
|
11992
12244
|
}
|
|
11993
12245
|
return commitRecord(rows[0]);
|
|
11994
12246
|
}
|
|
@@ -12036,8 +12288,8 @@ function databaseCount(value) {
|
|
|
12036
12288
|
// src/ontology-storage/outbox.ts
|
|
12037
12289
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
12038
12290
|
import {
|
|
12039
|
-
MaterializationConflictError as
|
|
12040
|
-
MaterializationValidationError
|
|
12291
|
+
MaterializationConflictError as MaterializationConflictError7,
|
|
12292
|
+
MaterializationValidationError as MaterializationValidationError2
|
|
12041
12293
|
} from "@sixb/core/internal/materialization";
|
|
12042
12294
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
|
|
12043
12295
|
class PgOntologyOutboxStorage {
|
|
@@ -12053,7 +12305,7 @@ class PgOntologyOutboxStorage {
|
|
|
12053
12305
|
const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
|
|
12054
12306
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
12055
12307
|
if (leaseExpiry <= now) {
|
|
12056
|
-
throw new
|
|
12308
|
+
throw new MaterializationValidationError2("Ontology outbox lease expiry must be later than the claim time.");
|
|
12057
12309
|
}
|
|
12058
12310
|
const rows = await sql`
|
|
12059
12311
|
WITH candidates AS (
|
|
@@ -12083,7 +12335,7 @@ class PgOntologyOutboxStorage {
|
|
|
12083
12335
|
return rows.map((row) => {
|
|
12084
12336
|
const record = outboxRecord(row);
|
|
12085
12337
|
if (record.leaseId === null || record.leaseExpiresAt === null) {
|
|
12086
|
-
throw new
|
|
12338
|
+
throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
|
|
12087
12339
|
}
|
|
12088
12340
|
return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt };
|
|
12089
12341
|
});
|
|
@@ -12204,7 +12456,7 @@ function validateLeaseIds(ids) {
|
|
|
12204
12456
|
for (const id of ids) {
|
|
12205
12457
|
assertNonblank(id, "Ontology outbox event id");
|
|
12206
12458
|
if (seen.has(id)) {
|
|
12207
|
-
throw new
|
|
12459
|
+
throw new MaterializationValidationError2(`Ontology outbox lease batch repeats event '${id}'.`);
|
|
12208
12460
|
}
|
|
12209
12461
|
seen.add(id);
|
|
12210
12462
|
}
|
|
@@ -12215,11 +12467,11 @@ function assertLeaseInput(input) {
|
|
|
12215
12467
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
12216
12468
|
}
|
|
12217
12469
|
function leaseConflict() {
|
|
12218
|
-
return new
|
|
12470
|
+
return new MaterializationConflictError7("outbox-lease", "Ontology outbox lease does not match.");
|
|
12219
12471
|
}
|
|
12220
12472
|
|
|
12221
12473
|
// src/ontology-storage/sources.ts
|
|
12222
|
-
import { MaterializationValidationError as
|
|
12474
|
+
import { MaterializationValidationError as MaterializationValidationError3 } from "@sixb/core/internal/materialization";
|
|
12223
12475
|
import {
|
|
12224
12476
|
assertSourceBeginInput as assertBeginInput,
|
|
12225
12477
|
assertSourceExecutionIdentity as assertExecutionIdentity,
|
|
@@ -12233,8 +12485,61 @@ import {
|
|
|
12233
12485
|
sourceMaterializationIdentity,
|
|
12234
12486
|
sourceStageRow,
|
|
12235
12487
|
sourceStageRows,
|
|
12236
|
-
utf8SortKey
|
|
12488
|
+
utf8SortKey as utf8SortKey2
|
|
12237
12489
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
12490
|
+
|
|
12491
|
+
// src/ontology-storage/source-cleanup.ts
|
|
12492
|
+
async function cleanupSourceVersions(sql, input) {
|
|
12493
|
+
let remaining = input.limit;
|
|
12494
|
+
const removedRows = await sql`
|
|
12495
|
+
WITH selected AS (
|
|
12496
|
+
SELECT rows.ctid FROM ontology_source_roots AS roots
|
|
12497
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
|
|
12498
|
+
JOIN ontology_source_rows AS rows
|
|
12499
|
+
ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
12500
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
12501
|
+
WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
|
|
12502
|
+
AND sources.status IN ('superseded', 'abandoned')
|
|
12503
|
+
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
|
|
12504
|
+
LIMIT ${remaining} FOR UPDATE OF rows SKIP LOCKED
|
|
12505
|
+
) DELETE FROM ontology_source_rows AS rows USING selected WHERE rows.ctid = selected.ctid RETURNING 1
|
|
12506
|
+
`;
|
|
12507
|
+
let rowsDeleted = removedRows.length;
|
|
12508
|
+
remaining -= rowsDeleted;
|
|
12509
|
+
if (remaining > 0) {
|
|
12510
|
+
const removed = await sql`
|
|
12511
|
+
WITH selected AS (
|
|
12512
|
+
SELECT roots.ctid FROM ontology_source_roots AS roots
|
|
12513
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
|
|
12514
|
+
WHERE roots.project_id = ${input.projectId} AND NOT roots.active AND roots.retired_at < ${input.terminalBefore}
|
|
12515
|
+
AND sources.status IN ('superseded', 'abandoned')
|
|
12516
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
12517
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
12518
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
|
|
12519
|
+
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key
|
|
12520
|
+
LIMIT ${remaining} FOR UPDATE OF roots SKIP LOCKED
|
|
12521
|
+
) DELETE FROM ontology_source_roots AS roots USING selected WHERE roots.ctid = selected.ctid RETURNING 1
|
|
12522
|
+
`;
|
|
12523
|
+
rowsDeleted += removed.length;
|
|
12524
|
+
remaining -= removed.length;
|
|
12525
|
+
}
|
|
12526
|
+
const removedManifests = remaining === 0 ? [] : await sql`
|
|
12527
|
+
WITH selected AS (
|
|
12528
|
+
SELECT sources.ctid FROM ontology_sources AS sources
|
|
12529
|
+
WHERE project_id = ${input.projectId} AND status IN ('superseded', 'abandoned') AND terminal_at < ${input.terminalBefore}
|
|
12530
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
|
|
12531
|
+
WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
|
|
12532
|
+
AND roots.materialization_id = sources.materialization_id)
|
|
12533
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
12534
|
+
WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
|
|
12535
|
+
AND rows.materialization_id = sources.materialization_id)
|
|
12536
|
+
ORDER BY terminal_at, source_id, materialization_id LIMIT ${remaining} FOR UPDATE SKIP LOCKED
|
|
12537
|
+
) DELETE FROM ontology_sources AS sources USING selected WHERE sources.ctid = selected.ctid RETURNING 1
|
|
12538
|
+
`;
|
|
12539
|
+
return { rowsDeleted, materializationsDeleted: removedManifests.length };
|
|
12540
|
+
}
|
|
12541
|
+
|
|
12542
|
+
// src/ontology-storage/sources.ts
|
|
12238
12543
|
class PgOntologySourceStorage {
|
|
12239
12544
|
runRootOperation;
|
|
12240
12545
|
constructor(runRootOperation) {
|
|
@@ -12265,14 +12570,15 @@ class PgOntologySourceStorage {
|
|
|
12265
12570
|
dataset_id, dataset_version_id, dataset_version_created_at,
|
|
12266
12571
|
projection_revision, ownership_hash, ontology_revision,
|
|
12267
12572
|
root_count, assertion_count, created_at, ready_at, activated_at,
|
|
12268
|
-
terminal_at, last_commit_id, updated_at
|
|
12573
|
+
terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
|
|
12269
12574
|
) VALUES (
|
|
12270
12575
|
${input.projectId}, ${input.source.projectionId}, ${input.materializationId},
|
|
12271
12576
|
${input.execution.projectionRunId}, ${input.projectionKind}, 'replacement',
|
|
12272
12577
|
'staging', ${input.execution.executionToken}, ${input.datasetVersion.datasetId},
|
|
12273
12578
|
${input.datasetVersion.versionId}, ${input.datasetVersion.createdAt},
|
|
12274
12579
|
${input.projectionRevision}, ${input.ownershipHash}, ${input.ontologyRevision},
|
|
12275
|
-
NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt}
|
|
12580
|
+
NULL, NULL, ${input.createdAt}, NULL, NULL, NULL, NULL, ${input.createdAt},
|
|
12581
|
+
${input.base?.materializationId ?? null}, ${input.base?.lastCommitId ?? null}
|
|
12276
12582
|
)
|
|
12277
12583
|
ON CONFLICT DO NOTHING
|
|
12278
12584
|
RETURNING *
|
|
@@ -12301,6 +12607,7 @@ class PgOntologySourceStorage {
|
|
|
12301
12607
|
const rows = sourceStageRows(manifest.projection_kind, input.rows);
|
|
12302
12608
|
const existing = await this.findStageRows(sql, input, rows);
|
|
12303
12609
|
const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
|
|
12610
|
+
await stageSourceRoots(sql, manifest, input);
|
|
12304
12611
|
await this.insertStageRows(sql, input, pending);
|
|
12305
12612
|
return { inserted: pending.length, unchanged };
|
|
12306
12613
|
});
|
|
@@ -12324,7 +12631,10 @@ class PgOntologySourceStorage {
|
|
|
12324
12631
|
throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
|
|
12325
12632
|
}
|
|
12326
12633
|
if (input.readyAt < toIsoString2(manifest.created_at)) {
|
|
12327
|
-
throw new
|
|
12634
|
+
throw new MaterializationValidationError3("Source readyAt cannot precede source createdAt.");
|
|
12635
|
+
}
|
|
12636
|
+
if (input.assertionCount >= 1000 || input.rootCount >= 1000) {
|
|
12637
|
+
await sql`ANALYZE ontology_source_rows, ontology_source_roots`;
|
|
12328
12638
|
}
|
|
12329
12639
|
await this.assertReadyState(sql, manifest, input.rootCount, input.assertionCount);
|
|
12330
12640
|
const rows = await sql`
|
|
@@ -12371,61 +12681,7 @@ class PgOntologySourceStorage {
|
|
|
12371
12681
|
assertNonblank(input.projectId, "Terminal source cleanup project id");
|
|
12372
12682
|
assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true);
|
|
12373
12683
|
assertPositiveInteger(input.limit, "Terminal source cleanup limit");
|
|
12374
|
-
|
|
12375
|
-
SELECT * FROM ontology_sources
|
|
12376
|
-
WHERE project_id = ${input.projectId}
|
|
12377
|
-
AND status IN ('superseded', 'abandoned')
|
|
12378
|
-
AND terminal_at < ${input.terminalBefore}
|
|
12379
|
-
ORDER BY terminal_at, source_id, materialization_id
|
|
12380
|
-
LIMIT ${input.limit}
|
|
12381
|
-
FOR UPDATE SKIP LOCKED
|
|
12382
|
-
`;
|
|
12383
|
-
let remaining = input.limit;
|
|
12384
|
-
let rowsDeleted = 0;
|
|
12385
|
-
let materializationsDeleted = 0;
|
|
12386
|
-
for (const manifest of manifests) {
|
|
12387
|
-
if (remaining === 0)
|
|
12388
|
-
break;
|
|
12389
|
-
const deletedRows = await sql`
|
|
12390
|
-
WITH selected AS (
|
|
12391
|
-
SELECT rows.ctid
|
|
12392
|
-
FROM ontology_source_rows AS rows
|
|
12393
|
-
WHERE rows.project_id = ${manifest.project_id}
|
|
12394
|
-
AND rows.source_id = ${manifest.source_id}
|
|
12395
|
-
AND rows.materialization_id = ${manifest.materialization_id}
|
|
12396
|
-
ORDER BY rows.entity_sort_key
|
|
12397
|
-
LIMIT ${remaining}
|
|
12398
|
-
FOR UPDATE SKIP LOCKED
|
|
12399
|
-
)
|
|
12400
|
-
DELETE FROM ontology_source_rows AS rows
|
|
12401
|
-
USING selected
|
|
12402
|
-
WHERE rows.ctid = selected.ctid
|
|
12403
|
-
RETURNING rows.entity_sort_key
|
|
12404
|
-
`;
|
|
12405
|
-
rowsDeleted += deletedRows.length;
|
|
12406
|
-
remaining -= deletedRows.length;
|
|
12407
|
-
if (remaining === 0)
|
|
12408
|
-
break;
|
|
12409
|
-
const removed = await sql`
|
|
12410
|
-
DELETE FROM ontology_sources AS sources
|
|
12411
|
-
WHERE sources.project_id = ${manifest.project_id}
|
|
12412
|
-
AND sources.source_id = ${manifest.source_id}
|
|
12413
|
-
AND sources.materialization_id = ${manifest.materialization_id}
|
|
12414
|
-
AND sources.status IN ('superseded', 'abandoned')
|
|
12415
|
-
AND sources.terminal_at = ${manifest.terminal_at}
|
|
12416
|
-
AND sources.terminal_at < ${input.terminalBefore}
|
|
12417
|
-
AND NOT EXISTS (
|
|
12418
|
-
SELECT 1 FROM ontology_source_rows AS rows
|
|
12419
|
-
WHERE rows.project_id = sources.project_id
|
|
12420
|
-
AND rows.source_id = sources.source_id
|
|
12421
|
-
AND rows.materialization_id = sources.materialization_id
|
|
12422
|
-
)
|
|
12423
|
-
RETURNING sources.materialization_id
|
|
12424
|
-
`;
|
|
12425
|
-
materializationsDeleted += removed.length;
|
|
12426
|
-
remaining -= removed.length;
|
|
12427
|
-
}
|
|
12428
|
-
return { rowsDeleted, materializationsDeleted };
|
|
12684
|
+
return cleanupSourceVersions(sql, input);
|
|
12429
12685
|
});
|
|
12430
12686
|
}
|
|
12431
12687
|
async summarizeTerminal(input) {
|
|
@@ -12473,7 +12729,7 @@ class PgOntologySourceStorage {
|
|
|
12473
12729
|
}
|
|
12474
12730
|
async transitionToAbandoned(sql, manifest, abandonedAt) {
|
|
12475
12731
|
if (abandonedAt < toIsoString2(manifest.created_at) || manifest.ready_at !== null && abandonedAt < toIsoString2(manifest.ready_at)) {
|
|
12476
|
-
throw new
|
|
12732
|
+
throw new MaterializationValidationError3("Source abandonedAt cannot precede source creation or readiness.");
|
|
12477
12733
|
}
|
|
12478
12734
|
const rows = await sql`
|
|
12479
12735
|
UPDATE ontology_sources
|
|
@@ -12488,14 +12744,19 @@ class PgOntologySourceStorage {
|
|
|
12488
12744
|
if (!rows[0]) {
|
|
12489
12745
|
throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
|
|
12490
12746
|
}
|
|
12747
|
+
await sql`UPDATE ontology_source_roots SET retired_at = ${abandonedAt}
|
|
12748
|
+
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
12749
|
+
AND materialization_id = ${manifest.materialization_id} AND NOT active`;
|
|
12491
12750
|
return sourceRecord(rows[0]);
|
|
12492
12751
|
}
|
|
12493
12752
|
async assertReadyState(sql, manifest, rootCount, assertionCount) {
|
|
12494
12753
|
const [counts] = await sql`
|
|
12495
|
-
SELECT
|
|
12754
|
+
SELECT (SELECT COUNT(*) FROM ontology_source_rows
|
|
12755
|
+
WHERE project_id = ${manifest.project_id} AND source_id = ${manifest.source_id}
|
|
12756
|
+
AND materialization_id = ${manifest.materialization_id}) AS assertions, COUNT(*) AS roots,
|
|
12496
12757
|
COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
12497
12758
|
MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
|
|
12498
|
-
FROM
|
|
12759
|
+
FROM ontology_source_roots
|
|
12499
12760
|
WHERE project_id = ${manifest.project_id}
|
|
12500
12761
|
AND source_id = ${manifest.source_id}
|
|
12501
12762
|
AND materialization_id = ${manifest.materialization_id}
|
|
@@ -12506,8 +12767,9 @@ class PgOntologySourceStorage {
|
|
|
12506
12767
|
const minimum = counts?.min_ordinal === null ? null : Number(counts?.min_ordinal);
|
|
12507
12768
|
const maximum = counts?.max_ordinal === null ? null : Number(counts?.max_ordinal);
|
|
12508
12769
|
if (assertions !== assertionCount || roots !== rootCount || ordinals !== rootCount || rootCount > 0 && (minimum !== 0 || maximum !== rootCount - 1)) {
|
|
12509
|
-
throw new
|
|
12770
|
+
throw new MaterializationValidationError3("Source ready counts do not match the staged roots and assertions.");
|
|
12510
12771
|
}
|
|
12772
|
+
await assertSourceRootCoverage(sql, manifest);
|
|
12511
12773
|
const [invalid] = manifest.projection_kind === "link" ? await sql`
|
|
12512
12774
|
SELECT root_key
|
|
12513
12775
|
FROM ontology_source_rows
|
|
@@ -12541,15 +12803,15 @@ class PgOntologySourceStorage {
|
|
|
12541
12803
|
LIMIT 1
|
|
12542
12804
|
`;
|
|
12543
12805
|
if (invalid) {
|
|
12544
|
-
throw new
|
|
12806
|
+
throw new MaterializationValidationError3(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
12545
12807
|
}
|
|
12546
12808
|
}
|
|
12547
12809
|
async findStageRows(sql, input, rows) {
|
|
12548
12810
|
if (rows.length === 0)
|
|
12549
12811
|
return [];
|
|
12550
|
-
const rootSortKeys = [...new Set(rows.map((row) =>
|
|
12812
|
+
const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.rootKey)))];
|
|
12551
12813
|
const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
|
|
12552
|
-
const entitySortKeys = [...new Set(rows.map((row) =>
|
|
12814
|
+
const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
|
|
12553
12815
|
const existing = await sql`
|
|
12554
12816
|
SELECT * FROM ontology_source_rows
|
|
12555
12817
|
WHERE project_id = ${input.projectId}
|
|
@@ -12572,10 +12834,10 @@ class PgOntologySourceStorage {
|
|
|
12572
12834
|
return {
|
|
12573
12835
|
entityKind: row.assertion.kind,
|
|
12574
12836
|
entityKey: JSON.parse(entityKey),
|
|
12575
|
-
entitySortKey:
|
|
12837
|
+
entitySortKey: utf8SortKey2(entityKey),
|
|
12576
12838
|
rootKind: row.root.kind,
|
|
12577
12839
|
rootKey: JSON.parse(rootKey),
|
|
12578
|
-
rootSortKey:
|
|
12840
|
+
rootSortKey: utf8SortKey2(rootKey),
|
|
12579
12841
|
stagingOrdinal: row.stagingOrdinal,
|
|
12580
12842
|
root: row.root,
|
|
12581
12843
|
assertion: row.assertion,
|
|
@@ -12697,12 +12959,11 @@ import {
|
|
|
12697
12959
|
} from "@sixb/core/internal/action-run-storage";
|
|
12698
12960
|
import { assertActionRunExecution } from "@sixb/core/internal/action-run-storage-provider";
|
|
12699
12961
|
import {
|
|
12700
|
-
ActionRunError,
|
|
12701
12962
|
actionRunPhaseRecordsEqual,
|
|
12702
12963
|
canRequeueActionRunAfterEnqueueFailure,
|
|
12703
|
-
finishActionRunPhase
|
|
12704
|
-
|
|
12705
|
-
} from "@sixb/core/storage";
|
|
12964
|
+
finishActionRunPhase
|
|
12965
|
+
} from "@sixb/core/internal/storage";
|
|
12966
|
+
import { ActionRunError, isTerminalActionRun } from "@sixb/core/storage";
|
|
12706
12967
|
class PgActionRunStorage {
|
|
12707
12968
|
sql;
|
|
12708
12969
|
executions;
|
|
@@ -12960,8 +13221,8 @@ class PgActionRunStorage {
|
|
|
12960
13221
|
params.push(input.actionId);
|
|
12961
13222
|
}
|
|
12962
13223
|
if (input.actionIds) {
|
|
12963
|
-
const
|
|
12964
|
-
whereClauses.push(`action_id IN (${
|
|
13224
|
+
const placeholders = input.actionIds.map(() => `$${index++}`);
|
|
13225
|
+
whereClauses.push(`action_id IN (${placeholders.join(", ")})`);
|
|
12965
13226
|
params.push(...input.actionIds);
|
|
12966
13227
|
}
|
|
12967
13228
|
if (input.objectTypeIds) {
|
|
@@ -12970,8 +13231,8 @@ class PgActionRunStorage {
|
|
|
12970
13231
|
params.push("object");
|
|
12971
13232
|
} else {
|
|
12972
13233
|
const subjectKindIndex = index++;
|
|
12973
|
-
const
|
|
12974
|
-
whereClauses.push(`(subject_kind <> $${subjectKindIndex} OR object_type_id IN (${
|
|
13234
|
+
const placeholders = input.objectTypeIds.map(() => `$${index++}`);
|
|
13235
|
+
whereClauses.push(`(subject_kind <> $${subjectKindIndex} OR object_type_id IN (${placeholders.join(", ")}))`);
|
|
12975
13236
|
params.push("object", ...input.objectTypeIds);
|
|
12976
13237
|
}
|
|
12977
13238
|
}
|
|
@@ -12998,8 +13259,8 @@ class PgActionRunStorage {
|
|
|
12998
13259
|
}
|
|
12999
13260
|
}
|
|
13000
13261
|
if (input.statuses) {
|
|
13001
|
-
const
|
|
13002
|
-
whereClauses.push(`status IN (${
|
|
13262
|
+
const placeholders = input.statuses.map(() => `$${index++}`);
|
|
13263
|
+
whereClauses.push(`status IN (${placeholders.join(", ")})`);
|
|
13003
13264
|
params.push(...input.statuses);
|
|
13004
13265
|
}
|
|
13005
13266
|
if (input.startedAfter) {
|
|
@@ -13157,6 +13418,7 @@ import {
|
|
|
13157
13418
|
aiModelCallCostDetails,
|
|
13158
13419
|
aiModelCallCostMatchesUsage,
|
|
13159
13420
|
buildAiAccountingOverviewFromFragments,
|
|
13421
|
+
directModelCallAttribution,
|
|
13160
13422
|
normalizeAiAccountingQuery,
|
|
13161
13423
|
normalizeAiModelCallAccountingQuery as normalizeAiModelCallAccountingQuery2,
|
|
13162
13424
|
normalizeAiModelCallCostRecord,
|
|
@@ -13201,6 +13463,7 @@ WITH filtered AS (
|
|
|
13201
13463
|
SELECT COALESCE(root_agent.execution_id, usage.execution_id) AS root_execution_id,
|
|
13202
13464
|
usage.execution_id, usage.provider_id, usage.requested_model_id, usage.occurred_at,
|
|
13203
13465
|
usage.total_tokens,
|
|
13466
|
+
execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
|
|
13204
13467
|
COALESCE(cost.status, 'unvalued') AS valuation_status,
|
|
13205
13468
|
cost.currency AS amount_currency, cost.amount_nanos,
|
|
13206
13469
|
root_agent.id AS root_agent_run_id,
|
|
@@ -13213,6 +13476,7 @@ WITH filtered AS (
|
|
|
13213
13476
|
workflow_node.workflow_id AS workflow_id,
|
|
13214
13477
|
workflow_node.workflow_run_id AS workflow_run_id
|
|
13215
13478
|
FROM ai_model_call_usage AS usage
|
|
13479
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
13216
13480
|
LEFT JOIN agent_runs AS direct_agent
|
|
13217
13481
|
ON direct_agent.project_id = usage.project_id AND direct_agent.execution_id = usage.execution_id
|
|
13218
13482
|
LEFT JOIN agent_runs AS root_agent
|
|
@@ -13250,6 +13514,7 @@ SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id
|
|
|
13250
13514
|
CAST(SUM((amount_nanos / 1000000) % 1000000) AS TEXT) AS amount_middle,
|
|
13251
13515
|
CAST(SUM(amount_nanos % 1000000) AS TEXT) AS amount_low,
|
|
13252
13516
|
MAX(root_agent_run_id) AS root_agent_run_id,
|
|
13517
|
+
MAX(executor_kind) AS executor_kind, MAX(executor_id) AS executor_id, MAX(primitive_id) AS primitive_id,
|
|
13253
13518
|
MAX(root_thread_id) AS root_thread_id,
|
|
13254
13519
|
MAX(direct_kind) AS direct_kind,
|
|
13255
13520
|
MAX(direct_run_id) AS direct_run_id,
|
|
@@ -13264,12 +13529,11 @@ GROUP BY filtered.root_execution_id, execution_id, provider_id, requested_model_
|
|
|
13264
13529
|
`;
|
|
13265
13530
|
|
|
13266
13531
|
// src/pg-ai-usage-storage.ts
|
|
13532
|
+
import { assertAiUsageExecutionId, normalizeAiModelCallRecord } from "@sixb/core/internal/storage";
|
|
13267
13533
|
import { isModelReasoning, normalizeModelProviderIds } from "@sixb/core/models";
|
|
13268
13534
|
import {
|
|
13269
13535
|
AiUsageStorageError,
|
|
13270
|
-
aggregateAiModelCallUsage
|
|
13271
|
-
assertAiUsageExecutionId,
|
|
13272
|
-
normalizeAiModelCallRecord
|
|
13536
|
+
aggregateAiModelCallUsage
|
|
13273
13537
|
} from "@sixb/core/storage";
|
|
13274
13538
|
class PgAiUsageStorage {
|
|
13275
13539
|
sql;
|
|
@@ -13581,11 +13845,14 @@ class PgAiCostStorage {
|
|
|
13581
13845
|
ELSE NULL
|
|
13582
13846
|
END AS attribution_agent_kind,
|
|
13583
13847
|
workflow_node.node_id AS attribution_agent_step_id,
|
|
13584
|
-
workflow_node.workflow_id
|
|
13848
|
+
COALESCE(workflow_node.workflow_id,
|
|
13849
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
13850
|
+
) AS attribution_workflow_id,
|
|
13585
13851
|
cost.status AS cost_status,
|
|
13586
13852
|
cost.currency AS cost_currency,
|
|
13587
13853
|
cost.amount_nanos AS cost_amount_nanos
|
|
13588
13854
|
FROM ai_model_call_usage AS usage
|
|
13855
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
13589
13856
|
LEFT JOIN agent_runs AS direct_agent
|
|
13590
13857
|
ON direct_agent.project_id = usage.project_id
|
|
13591
13858
|
AND direct_agent.execution_id = usage.execution_id
|
|
@@ -13712,6 +13979,7 @@ class PgAiCostStorage {
|
|
|
13712
13979
|
workflow_agent.node_run_id AS attribution_node_run_id,
|
|
13713
13980
|
workflow_node.workflow_id AS attribution_workflow_id,
|
|
13714
13981
|
workflow_node.workflow_run_id AS attribution_workflow_run_id,
|
|
13982
|
+
execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
|
|
13715
13983
|
COALESCE((
|
|
13716
13984
|
SELECT jsonb_agg(groups.group_id ORDER BY groups.group_id)
|
|
13717
13985
|
FROM ai_model_call_usage_groups AS groups
|
|
@@ -13726,6 +13994,7 @@ class PgAiCostStorage {
|
|
|
13726
13994
|
cost.details AS cost_details,
|
|
13727
13995
|
cost.rated_at AS cost_rated_at
|
|
13728
13996
|
FROM ai_model_call_usage AS usage
|
|
13997
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
13729
13998
|
LEFT JOIN agent_runs AS direct_agent
|
|
13730
13999
|
ON direct_agent.project_id = usage.project_id
|
|
13731
14000
|
AND direct_agent.execution_id = usage.execution_id
|
|
@@ -13814,7 +14083,7 @@ function meterFragment(presentCallCount, total) {
|
|
|
13814
14083
|
function accountingItemFromRow(row) {
|
|
13815
14084
|
const usage = usageFromRow2(row, parseStringArray(row.requester_group_ids));
|
|
13816
14085
|
const cost = row.cost_status === null ? undefined : costFromJoinedRow(row);
|
|
13817
|
-
const attribution = row.attribution_kind === null ?
|
|
14086
|
+
const attribution = row.attribution_kind === null ? directModelCallAttribution(row.executor_kind, row.executor_id, row.primitive_id) : row.attribution_kind === "agent" ? {
|
|
13818
14087
|
kind: "agent",
|
|
13819
14088
|
agentRunId: required2(row.attribution_agent_run_id),
|
|
13820
14089
|
threadId: required2(row.attribution_thread_id)
|
|
@@ -14649,8 +14918,7 @@ function createPgClient(options) {
|
|
|
14649
14918
|
import {
|
|
14650
14919
|
executionRecordFromStorageRow,
|
|
14651
14920
|
executionRecordToStorageRow,
|
|
14652
|
-
|
|
14653
|
-
validateExecutionRecordReferences
|
|
14921
|
+
prepareExecutionRecord
|
|
14654
14922
|
} from "@sixb/core/internal/execution-storage";
|
|
14655
14923
|
import { ExecutionStorageError } from "@sixb/core/storage";
|
|
14656
14924
|
class PgExecutionStorage {
|
|
@@ -14663,8 +14931,7 @@ class PgExecutionStorage {
|
|
|
14663
14931
|
this.shareSessions = shareSessions;
|
|
14664
14932
|
}
|
|
14665
14933
|
async create(input) {
|
|
14666
|
-
const record =
|
|
14667
|
-
await validateExecutionRecordReferences(record, {
|
|
14934
|
+
const record = await prepareExecutionRecord(input, {
|
|
14668
14935
|
auth: this.auth,
|
|
14669
14936
|
getExecution: (params) => this.getById(params),
|
|
14670
14937
|
getShareSession: (params) => this.shareSessions.getById(params)
|
|
@@ -14681,6 +14948,7 @@ class PgExecutionStorage {
|
|
|
14681
14948
|
source_id,
|
|
14682
14949
|
requested_by_user_id,
|
|
14683
14950
|
requested_by_service_account_id,
|
|
14951
|
+
requester_group_ids,
|
|
14684
14952
|
correlation_id,
|
|
14685
14953
|
parent_execution_id,
|
|
14686
14954
|
authority_kind,
|
|
@@ -14704,6 +14972,7 @@ class PgExecutionStorage {
|
|
|
14704
14972
|
${row.sourceId},
|
|
14705
14973
|
${row.requestedByUserId},
|
|
14706
14974
|
${row.requestedByServiceAccountId},
|
|
14975
|
+
${JSON.stringify(row.requesterGroupIds)}::text::jsonb,
|
|
14707
14976
|
${row.correlationId},
|
|
14708
14977
|
${row.parentExecutionId},
|
|
14709
14978
|
${row.authorityKind},
|
|
@@ -14751,6 +15020,7 @@ function toStorageRow(row) {
|
|
|
14751
15020
|
sourceId: row.source_id,
|
|
14752
15021
|
requestedByUserId: row.requested_by_user_id,
|
|
14753
15022
|
requestedByServiceAccountId: row.requested_by_service_account_id,
|
|
15023
|
+
requesterGroupIds: row.requester_group_ids,
|
|
14754
15024
|
correlationId: row.correlation_id,
|
|
14755
15025
|
parentExecutionId: row.parent_execution_id,
|
|
14756
15026
|
authorityKind: row.authority_kind,
|
|
@@ -14771,11 +15041,8 @@ function toStorageRow(row) {
|
|
|
14771
15041
|
// src/pg-pipeline-run-storage.ts
|
|
14772
15042
|
import { parseSixbFailure as parseSixbFailure3, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
|
|
14773
15043
|
import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
|
|
14774
|
-
import {
|
|
14775
|
-
|
|
14776
|
-
PIPELINE_RUN_FAILURE_CODES,
|
|
14777
|
-
PipelineRunError
|
|
14778
|
-
} from "@sixb/core/storage";
|
|
15044
|
+
import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
|
|
15045
|
+
import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
|
|
14779
15046
|
|
|
14780
15047
|
// src/latest-run-query.ts
|
|
14781
15048
|
async function queryLatestRunsByOwnerId(input) {
|
|
@@ -14783,12 +15050,12 @@ async function queryLatestRunsByOwnerId(input) {
|
|
|
14783
15050
|
if (ownerIds.length === 0) {
|
|
14784
15051
|
return [];
|
|
14785
15052
|
}
|
|
14786
|
-
const
|
|
15053
|
+
const placeholders = ownerIds.map((_, index) => `$${index + 2}`).join(", ");
|
|
14787
15054
|
const orderColumn = input.tableName === "sync_runs" || input.tableName === "pipeline_runs" || input.tableName === "projection_runs" ? "COALESCE(started_at, queued_at)" : "started_at";
|
|
14788
15055
|
const rows = await input.sql.unsafe(`
|
|
14789
15056
|
SELECT DISTINCT ON (${input.ownerColumn}) ${input.selectList ?? "*"}
|
|
14790
15057
|
FROM ${input.tableName}
|
|
14791
|
-
WHERE project_id = $1 AND ${input.ownerColumn} IN (${
|
|
15058
|
+
WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})
|
|
14792
15059
|
ORDER BY ${input.ownerColumn}, ${orderColumn} DESC, id DESC
|
|
14793
15060
|
`, [input.projectId, ...ownerIds]);
|
|
14794
15061
|
const latestByOwnerId = new Map(rows.map((row) => [input.ownerIdFor(row), row]));
|
|
@@ -15031,8 +15298,8 @@ class PgPipelineRunStorage {
|
|
|
15031
15298
|
params.push(input.pipelineId);
|
|
15032
15299
|
}
|
|
15033
15300
|
if (input.pipelineIds) {
|
|
15034
|
-
const
|
|
15035
|
-
whereClauses.push(`pipeline_id IN (${
|
|
15301
|
+
const placeholders = input.pipelineIds.map(() => `$${index++}`).join(", ");
|
|
15302
|
+
whereClauses.push(`pipeline_id IN (${placeholders})`);
|
|
15036
15303
|
params.push(...input.pipelineIds);
|
|
15037
15304
|
}
|
|
15038
15305
|
index = appendRunListFilters(whereClauses, params, index, input, "COALESCE(started_at, queued_at)");
|
|
@@ -15230,7 +15497,7 @@ class PgProjectionRunStorage {
|
|
|
15230
15497
|
input_exhausted = ${input.fixedBatchSize === undefined ? null : false},
|
|
15231
15498
|
missing_target_object_type_id = ${null}, missing_target_object_id = ${null},
|
|
15232
15499
|
missing_target_batch_ordinal = ${null}, missing_target_first_seen_at = ${null},
|
|
15233
|
-
source_rows_read = ${0}, source_rows_skipped = ${0}, error = ${null}
|
|
15500
|
+
source_rows_read = ${0}, source_rows_skipped = ${0}, source_changes_read = ${null}, error = ${null}
|
|
15234
15501
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
15235
15502
|
AND status = ${"failed"} AND error->>'code' = ${"queue.enqueue_failed"}
|
|
15236
15503
|
RETURNING *
|
|
@@ -15341,7 +15608,8 @@ class PgProjectionRunStorage {
|
|
|
15341
15608
|
UPDATE projection_runs
|
|
15342
15609
|
SET
|
|
15343
15610
|
source_rows_read = ${progress.sourceRowsRead},
|
|
15344
|
-
source_rows_skipped = ${progress.sourceRowsSkipped}
|
|
15611
|
+
source_rows_skipped = ${progress.sourceRowsSkipped},
|
|
15612
|
+
source_changes_read = ${progress.sourceChangesRead ?? null}
|
|
15345
15613
|
WHERE project_id = ${input.projectId}
|
|
15346
15614
|
AND id = ${input.id}
|
|
15347
15615
|
AND status = ${"running"}
|
|
@@ -15366,6 +15634,7 @@ class PgProjectionRunStorage {
|
|
|
15366
15634
|
execution_token = ${null},
|
|
15367
15635
|
source_rows_read = ${plan.progress.sourceRowsRead},
|
|
15368
15636
|
source_rows_skipped = ${plan.progress.sourceRowsSkipped},
|
|
15637
|
+
source_changes_read = ${plan.progress.sourceChangesRead ?? null},
|
|
15369
15638
|
input_exhausted = ${plan.inputExhausted ?? existingRow.input_exhausted},
|
|
15370
15639
|
error = ${plan.error === undefined ? null : serializeSixbFailure4(plan.error, PROJECTION_RUN_FAILURE_CODES)}::text::jsonb
|
|
15371
15640
|
WHERE project_id = ${input.projectId}
|
|
@@ -15467,16 +15736,16 @@ class PgProjectionRunStorage {
|
|
|
15467
15736
|
if (input.objectTypeIds.length === 0) {
|
|
15468
15737
|
return { runs: [], hasMore: false, total: 0 };
|
|
15469
15738
|
}
|
|
15470
|
-
const
|
|
15471
|
-
const list =
|
|
15739
|
+
const placeholders = input.objectTypeIds.map(() => `$${index++}`);
|
|
15740
|
+
const list = placeholders.join(", ");
|
|
15472
15741
|
const sourceList = input.objectTypeIds.map(() => `$${index++}`).join(", ");
|
|
15473
15742
|
const targetList = input.objectTypeIds.map(() => `$${index++}`).join(", ");
|
|
15474
15743
|
whereClauses.push(`(object_type_id IN (${list})` + ` OR (source_object_type_id IN (${sourceList})` + ` AND target_object_type_id IN (${targetList})))`);
|
|
15475
15744
|
params.push(...input.objectTypeIds, ...input.objectTypeIds, ...input.objectTypeIds);
|
|
15476
15745
|
}
|
|
15477
15746
|
if (input.statuses) {
|
|
15478
|
-
const
|
|
15479
|
-
whereClauses.push(`status IN (${
|
|
15747
|
+
const placeholders = input.statuses.map(() => `$${index++}`);
|
|
15748
|
+
whereClauses.push(`status IN (${placeholders.join(", ")})`);
|
|
15480
15749
|
params.push(...input.statuses);
|
|
15481
15750
|
}
|
|
15482
15751
|
if (input.startedAfter) {
|
|
@@ -15608,7 +15877,8 @@ function restoreProjectionRunRow(row) {
|
|
|
15608
15877
|
missingTargetFirstSeenAt: row.missing_target_first_seen_at ? new Date(row.missing_target_first_seen_at) : undefined,
|
|
15609
15878
|
progress: {
|
|
15610
15879
|
sourceRowsRead: databaseSafeInteger2(row.source_rows_read, "sourceRowsRead"),
|
|
15611
|
-
sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped")
|
|
15880
|
+
sourceRowsSkipped: databaseSafeInteger2(row.source_rows_skipped, "sourceRowsSkipped"),
|
|
15881
|
+
...row.source_changes_read == null ? {} : { sourceChangesRead: databaseSafeInteger2(row.source_changes_read, "sourceChangesRead") }
|
|
15612
15882
|
},
|
|
15613
15883
|
error: row.error ?? undefined
|
|
15614
15884
|
};
|
|
@@ -15694,8 +15964,8 @@ class PgRulesStorage {
|
|
|
15694
15964
|
params.push(input.objectTypeId);
|
|
15695
15965
|
}
|
|
15696
15966
|
if (input.objectTypeIds) {
|
|
15697
|
-
const
|
|
15698
|
-
whereClauses.push(`object_type_id IN (${
|
|
15967
|
+
const placeholders = input.objectTypeIds.map(() => `$${index++}`);
|
|
15968
|
+
whereClauses.push(`object_type_id IN (${placeholders.join(", ")})`);
|
|
15699
15969
|
params.push(...input.objectTypeIds);
|
|
15700
15970
|
}
|
|
15701
15971
|
if (input.primaryId) {
|
|
@@ -15871,12 +16141,9 @@ function toIsoString3(value) {
|
|
|
15871
16141
|
|
|
15872
16142
|
// src/pg-sync-run-storage.ts
|
|
15873
16143
|
import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
|
|
16144
|
+
import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
|
|
15874
16145
|
import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
|
|
15875
|
-
import {
|
|
15876
|
-
canRequeueSyncRunAfterEnqueueFailure,
|
|
15877
|
-
SYNC_RUN_FAILURE_CODES,
|
|
15878
|
-
SyncRunError
|
|
15879
|
-
} from "@sixb/core/storage";
|
|
16146
|
+
import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
|
|
15880
16147
|
class PgSyncRunStorage {
|
|
15881
16148
|
sql;
|
|
15882
16149
|
executions;
|
|
@@ -16036,8 +16303,8 @@ class PgSyncRunStorage {
|
|
|
16036
16303
|
params.push(input.syncId);
|
|
16037
16304
|
}
|
|
16038
16305
|
if (input.syncIds) {
|
|
16039
|
-
const
|
|
16040
|
-
whereClauses.push(`sync_id IN (${
|
|
16306
|
+
const placeholders = input.syncIds.map(() => `$${index++}`);
|
|
16307
|
+
whereClauses.push(`sync_id IN (${placeholders.join(", ")})`);
|
|
16041
16308
|
params.push(...input.syncIds);
|
|
16042
16309
|
}
|
|
16043
16310
|
if (input.datasetId) {
|
|
@@ -16139,14 +16406,14 @@ class PgTimeseriesStorage {
|
|
|
16139
16406
|
const params = [];
|
|
16140
16407
|
let nextIndex = 1;
|
|
16141
16408
|
const values = input.series.map((series, seriesIndex) => {
|
|
16142
|
-
const
|
|
16409
|
+
const placeholders = [
|
|
16143
16410
|
`$${nextIndex++}`,
|
|
16144
16411
|
`$${nextIndex++}`,
|
|
16145
16412
|
`$${nextIndex++}`,
|
|
16146
16413
|
`$${nextIndex++}`
|
|
16147
16414
|
];
|
|
16148
16415
|
params.push(seriesIndex, series.objectTypeId, series.objectId, series.propertyId);
|
|
16149
|
-
return `(${
|
|
16416
|
+
return `(${placeholders.join(", ")})`;
|
|
16150
16417
|
}).join(", ");
|
|
16151
16418
|
const projectIdPlaceholder = `$${nextIndex++}`;
|
|
16152
16419
|
params.push(input.projectId);
|
|
@@ -16498,8 +16765,8 @@ class PgWorkflowInterventionStorage {
|
|
|
16498
16765
|
const params = [input.projectId];
|
|
16499
16766
|
let index = 2;
|
|
16500
16767
|
if (input.statuses) {
|
|
16501
|
-
const
|
|
16502
|
-
whereClauses.push(`status IN (${
|
|
16768
|
+
const placeholders = input.statuses.map(() => `$${index++}`);
|
|
16769
|
+
whereClauses.push(`status IN (${placeholders.join(", ")})`);
|
|
16503
16770
|
params.push(...input.statuses);
|
|
16504
16771
|
}
|
|
16505
16772
|
if (input.workflowId) {
|
|
@@ -16507,8 +16774,8 @@ class PgWorkflowInterventionStorage {
|
|
|
16507
16774
|
params.push(input.workflowId);
|
|
16508
16775
|
}
|
|
16509
16776
|
if (input.workflowIds) {
|
|
16510
|
-
const
|
|
16511
|
-
whereClauses.push(`workflow_id IN (${
|
|
16777
|
+
const placeholders = input.workflowIds.map(() => `$${index++}`);
|
|
16778
|
+
whereClauses.push(`workflow_id IN (${placeholders.join(", ")})`);
|
|
16512
16779
|
params.push(...input.workflowIds);
|
|
16513
16780
|
}
|
|
16514
16781
|
if (input.workflowRunId) {
|
|
@@ -16622,7 +16889,6 @@ function assertNonNegativeInteger(value, fieldName) {
|
|
|
16622
16889
|
}
|
|
16623
16890
|
|
|
16624
16891
|
// src/pg-workflow-run-storage.ts
|
|
16625
|
-
import { normalizeRequesterGroupIds as normalizeRequesterGroupIds2 } from "@sixb/core/internal/auth";
|
|
16626
16892
|
import { parseSixbFailure as parseSixbFailure6, serializeSixbFailure as serializeSixbFailure7 } from "@sixb/core/internal/errors";
|
|
16627
16893
|
import {
|
|
16628
16894
|
assertWorkflowAgentNodeRunExecution,
|
|
@@ -16664,7 +16930,6 @@ class PgWorkflowRunStorage {
|
|
|
16664
16930
|
input,
|
|
16665
16931
|
queued_at,
|
|
16666
16932
|
started_at,
|
|
16667
|
-
requester_group_ids,
|
|
16668
16933
|
attempt
|
|
16669
16934
|
) VALUES (
|
|
16670
16935
|
${input.projectId},
|
|
@@ -16675,7 +16940,6 @@ class PgWorkflowRunStorage {
|
|
|
16675
16940
|
${serializeRecord2(input.input)}::text::jsonb,
|
|
16676
16941
|
${queuedAt},
|
|
16677
16942
|
${queuedAt},
|
|
16678
|
-
${JSON.stringify(normalizeRequesterGroupIds2(input.requesterGroupIds))}::text::jsonb,
|
|
16679
16943
|
${0}
|
|
16680
16944
|
)
|
|
16681
16945
|
RETURNING *
|
|
@@ -16861,8 +17125,8 @@ class PgWorkflowRunStorage {
|
|
|
16861
17125
|
params.push(input.workflowId);
|
|
16862
17126
|
}
|
|
16863
17127
|
if (input.workflowIds) {
|
|
16864
|
-
const
|
|
16865
|
-
whereClauses.push(`workflow_id IN (${
|
|
17128
|
+
const placeholders = input.workflowIds.map(() => `$${index++}`);
|
|
17129
|
+
whereClauses.push(`workflow_id IN (${placeholders.join(", ")})`);
|
|
16866
17130
|
params.push(...input.workflowIds);
|
|
16867
17131
|
}
|
|
16868
17132
|
index = appendRunListFilters(whereClauses, params, index, input);
|
|
@@ -17268,7 +17532,6 @@ function rowToWorkflowRunRecord(row) {
|
|
|
17268
17532
|
startedAt: new Date(row.started_at),
|
|
17269
17533
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
17270
17534
|
error: row.error === null ? undefined : parseSixbFailure6(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
17271
|
-
requesterGroupIds: parseJson(row.requester_group_ids),
|
|
17272
17535
|
attempt: Number(row.attempt),
|
|
17273
17536
|
...row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
17274
17537
|
execution: {
|
|
@@ -17843,8 +18106,8 @@ function resolveTimeoutMillis(value, label) {
|
|
|
17843
18106
|
return Math.trunc(value);
|
|
17844
18107
|
}
|
|
17845
18108
|
export {
|
|
17846
|
-
|
|
17847
|
-
|
|
18109
|
+
PostgresStorage,
|
|
18110
|
+
migratePostgresStorage
|
|
17848
18111
|
};
|
|
17849
18112
|
|
|
17850
|
-
//# debugId=
|
|
18113
|
+
//# debugId=672C519862E8368164756E2164756E21
|