@sixb/sqlite 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action-run-storage.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +0 -1
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/ai-cost-storage.d.ts.map +1 -1
- package/dist/ai-usage-storage.d.ts.map +1 -1
- package/dist/execution-storage.d.ts.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +495 -244
- package/dist/index.js.map +23 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +2 -1
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +2 -0
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts +4 -0
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -0
- package/dist/ontology-storage/source-roots.d.ts +27 -0
- package/dist/ontology-storage/source-roots.d.ts.map +1 -0
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pipeline-run-storage.d.ts.map +1 -1
- package/dist/projection-run-storage.d.ts.map +1 -1
- package/dist/sync-run-storage.d.ts.map +1 -1
- package/dist/workflow-run-storage.d.ts.map +1 -1
- package/package.json +4 -4
- package/src/action-run-storage.ts +6 -7
- package/src/agents/rows.ts +0 -2
- package/src/agents/runs.ts +3 -7
- package/src/ai-cost-groups.ts +3 -0
- package/src/ai-cost-storage.ts +11 -2
- package/src/ai-usage-storage.ts +1 -2
- package/src/connector-connection-storage.ts +3 -3
- package/src/execution-storage.ts +7 -5
- package/src/index.ts +4 -2
- package/src/migrations/037-execution-requester-groups.sql +24 -0
- package/src/migrations/038-projection-source-roots.sql +44 -0
- package/src/migrations/039-connector-optional-pkce.sql +52 -0
- package/src/migrations.ts +14 -0
- package/src/ontology-storage/materialization-state.ts +56 -91
- package/src/ontology-storage/materializations.ts +7 -1
- package/src/ontology-storage/shared.ts +10 -0
- package/src/ontology-storage/source-cleanup.ts +59 -0
- package/src/ontology-storage/source-roots.ts +230 -0
- package/src/ontology-storage/sources.ts +25 -78
- package/src/pipeline-run-storage.ts +2 -5
- package/src/projection-run-storage.ts +11 -3
- package/src/sync-run-storage.ts +2 -5
- package/src/workflow-run-storage.ts +1 -6
package/dist/index.js
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
// src/index.ts
|
|
3
3
|
import { AsyncLocalStorage } from "async_hooks";
|
|
4
4
|
import { ProviderMaterializationTransactionLifecycle } from "@sixb/core/internal/ontology-storage-provider";
|
|
5
|
+
import {
|
|
6
|
+
createTransactionStorageProxy,
|
|
7
|
+
throwNestedStorageTransaction
|
|
8
|
+
} from "@sixb/core/internal/storage";
|
|
5
9
|
import {
|
|
6
10
|
createAgentOperationScope,
|
|
7
11
|
createAuthOperationScope,
|
|
@@ -12,9 +16,7 @@ import {
|
|
|
12
16
|
createWorkflowRunOperationScope
|
|
13
17
|
} from "@sixb/core/internal/storage-operation-scope";
|
|
14
18
|
import {
|
|
15
|
-
|
|
16
|
-
StorageTransactionError,
|
|
17
|
-
throwNestedStorageTransaction
|
|
19
|
+
StorageTransactionError
|
|
18
20
|
} from "@sixb/core/storage";
|
|
19
21
|
|
|
20
22
|
// src/action-run-storage.ts
|
|
@@ -24,12 +26,11 @@ import {
|
|
|
24
26
|
} from "@sixb/core/internal/action-run-storage";
|
|
25
27
|
import { assertActionRunExecution } from "@sixb/core/internal/action-run-storage-provider";
|
|
26
28
|
import {
|
|
27
|
-
ActionRunError,
|
|
28
29
|
actionRunPhaseRecordsEqual,
|
|
29
30
|
canRequeueActionRunAfterEnqueueFailure,
|
|
30
|
-
finishActionRunPhase
|
|
31
|
-
|
|
32
|
-
} from "@sixb/core/storage";
|
|
31
|
+
finishActionRunPhase
|
|
32
|
+
} from "@sixb/core/internal/storage";
|
|
33
|
+
import { ActionRunError, isTerminalActionRun } from "@sixb/core/storage";
|
|
33
34
|
|
|
34
35
|
// src/migrations.ts
|
|
35
36
|
import { Database } from "bun:sqlite";
|
|
@@ -3931,6 +3932,135 @@ DROP TABLE executions;
|
|
|
3931
3932
|
ALTER TABLE executions_v2 RENAME TO executions;
|
|
3932
3933
|
`;
|
|
3933
3934
|
|
|
3935
|
+
// src/migrations/038-projection-source-roots.sql
|
|
3936
|
+
var _038_projection_source_roots_default = `ALTER TABLE ontology_sources ADD COLUMN base_materialization_id TEXT;
|
|
3937
|
+
ALTER TABLE ontology_sources ADD COLUMN base_commit_id TEXT
|
|
3938
|
+
CHECK ((base_materialization_id IS NULL) = (base_commit_id IS NULL));
|
|
3939
|
+
|
|
3940
|
+
CREATE TABLE ontology_source_roots (
|
|
3941
|
+
project_id TEXT NOT NULL,
|
|
3942
|
+
source_id TEXT NOT NULL,
|
|
3943
|
+
materialization_id TEXT NOT NULL,
|
|
3944
|
+
root_sort_key TEXT NOT NULL,
|
|
3945
|
+
root_kind TEXT NOT NULL CHECK (root_kind IN ('object', 'link')),
|
|
3946
|
+
root_key TEXT NOT NULL CHECK (json_valid(root_key)),
|
|
3947
|
+
root TEXT NOT NULL CHECK (json_valid(root)),
|
|
3948
|
+
staging_ordinal INTEGER NOT NULL CHECK (staging_ordinal >= 0),
|
|
3949
|
+
deleted INTEGER NOT NULL DEFAULT 0 CHECK (deleted IN (0, 1)),
|
|
3950
|
+
active INTEGER NOT NULL DEFAULT 0 CHECK (active IN (0, 1)),
|
|
3951
|
+
retired_at TEXT,
|
|
3952
|
+
PRIMARY KEY (project_id, source_id, materialization_id, root_sort_key),
|
|
3953
|
+
UNIQUE (project_id, source_id, materialization_id, staging_ordinal),
|
|
3954
|
+
FOREIGN KEY (project_id, source_id, materialization_id)
|
|
3955
|
+
REFERENCES ontology_sources(project_id, source_id, materialization_id) ON DELETE RESTRICT,
|
|
3956
|
+
CHECK (active = 0 OR (deleted = 0 AND retired_at IS NULL))
|
|
3957
|
+
);
|
|
3958
|
+
CREATE UNIQUE INDEX idx_ontology_source_roots_active
|
|
3959
|
+
ON ontology_source_roots (project_id, source_id, root_sort_key) WHERE active = 1;
|
|
3960
|
+
CREATE INDEX idx_ontology_source_roots_lookup
|
|
3961
|
+
ON ontology_source_roots (project_id, root_sort_key, source_id) WHERE active = 1;
|
|
3962
|
+
CREATE INDEX idx_ontology_source_roots_cleanup
|
|
3963
|
+
ON ontology_source_roots (project_id, retired_at, source_id, materialization_id, root_sort_key)
|
|
3964
|
+
WHERE active = 0 AND retired_at IS NOT NULL;
|
|
3965
|
+
|
|
3966
|
+
INSERT INTO ontology_source_roots (
|
|
3967
|
+
project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root,
|
|
3968
|
+
staging_ordinal, active, retired_at
|
|
3969
|
+
)
|
|
3970
|
+
SELECT DISTINCT rows.project_id, rows.source_id, rows.materialization_id,
|
|
3971
|
+
rows.root_sort_key, rows.root_kind, rows.root_key, rows.root, rows.staging_ordinal,
|
|
3972
|
+
sources.status = 'active', sources.terminal_at
|
|
3973
|
+
FROM ontology_source_rows AS rows
|
|
3974
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id);
|
|
3975
|
+
|
|
3976
|
+
ANALYZE ontology_source_roots;
|
|
3977
|
+
|
|
3978
|
+
ALTER TABLE projection_runs ADD COLUMN source_changes_read INTEGER
|
|
3979
|
+
CHECK (source_changes_read IS NULL OR source_changes_read >= 0);
|
|
3980
|
+
`;
|
|
3981
|
+
|
|
3982
|
+
// src/migrations/037-execution-requester-groups.sql
|
|
3983
|
+
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids TEXT NOT NULL DEFAULT '[]';
|
|
3984
|
+
|
|
3985
|
+
-- Preserve each admitted snapshot, including an intentionally empty one. Executions without
|
|
3986
|
+
-- their own historical snapshot inherit from their parent; historical roots remain unknown/empty.
|
|
3987
|
+
WITH RECURSIVE attribution(project_id, id, groups) AS (
|
|
3988
|
+
SELECT e.project_id, e.id, COALESCE(w.requester_group_ids, a.requester_group_ids, '[]')
|
|
3989
|
+
FROM executions e
|
|
3990
|
+
LEFT JOIN workflow_runs w ON w.project_id = e.project_id AND w.execution_id = e.id
|
|
3991
|
+
LEFT JOIN agent_runs a ON a.project_id = e.project_id AND a.execution_id = e.id
|
|
3992
|
+
WHERE e.parent_execution_id IS NULL
|
|
3993
|
+
UNION ALL
|
|
3994
|
+
SELECT e.project_id, e.id, COALESCE(w.requester_group_ids, a.requester_group_ids, p.groups)
|
|
3995
|
+
FROM executions e
|
|
3996
|
+
JOIN attribution p ON p.project_id = e.project_id AND p.id = e.parent_execution_id
|
|
3997
|
+
LEFT JOIN workflow_runs w ON w.project_id = e.project_id AND w.execution_id = e.id
|
|
3998
|
+
LEFT JOIN agent_runs a ON a.project_id = e.project_id AND a.execution_id = e.id
|
|
3999
|
+
)
|
|
4000
|
+
UPDATE executions SET requester_group_ids = (
|
|
4001
|
+
SELECT groups FROM attribution
|
|
4002
|
+
WHERE attribution.project_id = executions.project_id AND attribution.id = executions.id
|
|
4003
|
+
);
|
|
4004
|
+
|
|
4005
|
+
ALTER TABLE workflow_runs DROP COLUMN requester_group_ids;
|
|
4006
|
+
ALTER TABLE agent_runs DROP COLUMN requester_group_ids;
|
|
4007
|
+
`;
|
|
4008
|
+
|
|
4009
|
+
// src/migrations/039-connector-optional-pkce.sql
|
|
4010
|
+
var _039_connector_optional_pkce_default = `-- Rebuild to remove NOT NULL while preserving existing attempts and their constraints.
|
|
4011
|
+
CREATE TABLE connector_authorization_attempts_optional_pkce (
|
|
4012
|
+
project_id TEXT NOT NULL,
|
|
4013
|
+
connector_id TEXT NOT NULL,
|
|
4014
|
+
id TEXT NOT NULL,
|
|
4015
|
+
slot TEXT NOT NULL,
|
|
4016
|
+
initiated_by_execution_id TEXT NOT NULL,
|
|
4017
|
+
state_hash TEXT NOT NULL,
|
|
4018
|
+
code_verifier TEXT CHECK (
|
|
4019
|
+
code_verifier IS NULL
|
|
4020
|
+
OR (json_valid(code_verifier) AND json_type(code_verifier) = 'object')
|
|
4021
|
+
),
|
|
4022
|
+
redirect_uri TEXT NOT NULL,
|
|
4023
|
+
connection_run_id TEXT,
|
|
4024
|
+
return_to TEXT,
|
|
4025
|
+
callback_binding_hash TEXT,
|
|
4026
|
+
reauthorization_id TEXT,
|
|
4027
|
+
reauthorization_revision INTEGER CHECK (
|
|
4028
|
+
reauthorization_revision IS NULL OR reauthorization_revision >= 0
|
|
4029
|
+
),
|
|
4030
|
+
reauthorization_connection_ids TEXT CHECK (
|
|
4031
|
+
reauthorization_connection_ids IS NULL
|
|
4032
|
+
OR (json_valid(reauthorization_connection_ids) AND json_type(reauthorization_connection_ids) = 'array')
|
|
4033
|
+
),
|
|
4034
|
+
created_at TEXT NOT NULL,
|
|
4035
|
+
expires_at TEXT NOT NULL,
|
|
4036
|
+
PRIMARY KEY (project_id, connector_id, id),
|
|
4037
|
+
UNIQUE (project_id, id),
|
|
4038
|
+
UNIQUE (project_id, connector_id, connection_run_id),
|
|
4039
|
+
FOREIGN KEY (project_id, connector_id, connection_run_id)
|
|
4040
|
+
REFERENCES connector_connection_runs (project_id, connector_id, id)
|
|
4041
|
+
ON DELETE RESTRICT,
|
|
4042
|
+
CHECK (
|
|
4043
|
+
(connection_run_id IS NULL AND return_to IS NULL AND callback_binding_hash IS NULL)
|
|
4044
|
+
OR
|
|
4045
|
+
(connection_run_id IS NOT NULL AND return_to IS NOT NULL AND callback_binding_hash IS NOT NULL)
|
|
4046
|
+
),
|
|
4047
|
+
CHECK (
|
|
4048
|
+
(reauthorization_id IS NULL AND reauthorization_revision IS NULL AND reauthorization_connection_ids IS NULL)
|
|
4049
|
+
OR
|
|
4050
|
+
(reauthorization_id IS NOT NULL AND reauthorization_revision IS NOT NULL AND reauthorization_connection_ids IS NOT NULL)
|
|
4051
|
+
)
|
|
4052
|
+
);
|
|
4053
|
+
|
|
4054
|
+
INSERT INTO connector_authorization_attempts_optional_pkce
|
|
4055
|
+
SELECT * FROM connector_authorization_attempts;
|
|
4056
|
+
|
|
4057
|
+
DROP TABLE connector_authorization_attempts;
|
|
4058
|
+
ALTER TABLE connector_authorization_attempts_optional_pkce RENAME TO connector_authorization_attempts;
|
|
4059
|
+
|
|
4060
|
+
CREATE INDEX idx_connector_attempts_expiry
|
|
4061
|
+
ON connector_authorization_attempts (project_id, connector_id, expires_at);
|
|
4062
|
+
`;
|
|
4063
|
+
|
|
3934
4064
|
// src/migrations.ts
|
|
3935
4065
|
var MIGRATIONS_TABLE_SQL = `
|
|
3936
4066
|
CREATE TABLE IF NOT EXISTS sixb_migrations (
|
|
@@ -4001,7 +4131,10 @@ var sqliteStorageMigrations = defineMigrations({
|
|
|
4001
4131
|
sqliteSql("033-retire-agent-definitions", _033_retire_agent_definitions_default),
|
|
4002
4132
|
sqliteSql("034-device-authorizations", _034_device_authorizations_default),
|
|
4003
4133
|
sqliteSql("035-share-grants", _035_share_grants_default),
|
|
4004
|
-
sqliteSql("036-share-sessions", _036_share_sessions_default)
|
|
4134
|
+
sqliteSql("036-share-sessions", _036_share_sessions_default),
|
|
4135
|
+
sqliteSql("037-execution-requester-groups", _037_execution_requester_groups_default),
|
|
4136
|
+
sqliteSql("038-projection-source-roots", _038_projection_source_roots_default),
|
|
4137
|
+
sqliteSql("039-connector-optional-pkce", _039_connector_optional_pkce_default)
|
|
4005
4138
|
]
|
|
4006
4139
|
});
|
|
4007
4140
|
function sqliteStoragePath(basePath) {
|
|
@@ -4710,7 +4843,6 @@ function rowToRunRecord(row) {
|
|
|
4710
4843
|
id: row.id,
|
|
4711
4844
|
projectId: row.project_id,
|
|
4712
4845
|
executionId: row.execution_id,
|
|
4713
|
-
requesterGroupIds: JSON.parse(row.requester_group_ids),
|
|
4714
4846
|
status: row.status,
|
|
4715
4847
|
modelId: row.model_id ?? undefined,
|
|
4716
4848
|
finishReason: coerceAgentRunFinishReason(row.finish_reason),
|
|
@@ -5056,7 +5188,6 @@ import {
|
|
|
5056
5188
|
assertSubagentRunResult,
|
|
5057
5189
|
subagentRunMatchesCreateInput
|
|
5058
5190
|
} from "@sixb/core/internal/agent-run-storage-provider";
|
|
5059
|
-
import { normalizeRequesterGroupIds } from "@sixb/core/internal/auth";
|
|
5060
5191
|
import { serializeSixbFailure } from "@sixb/core/internal/errors";
|
|
5061
5192
|
import {
|
|
5062
5193
|
AGENT_RUN_FAILURE_CODES as AGENT_RUN_FAILURE_CODES2,
|
|
@@ -5151,12 +5282,11 @@ class SqliteAgentRunStore {
|
|
|
5151
5282
|
thread_id,
|
|
5152
5283
|
trigger_message_id,
|
|
5153
5284
|
spec,
|
|
5154
|
-
requester_group_ids,
|
|
5155
5285
|
status,
|
|
5156
5286
|
attempt,
|
|
5157
5287
|
created_at
|
|
5158
|
-
) VALUES (?, ?, ?, 'conversation', ?, ?, ?,
|
|
5159
|
-
`).run(input.projectId, input.id, input.executionId, input.threadId, input.triggerMessageId, JSON.stringify(input.spec),
|
|
5288
|
+
) VALUES (?, ?, ?, 'conversation', ?, ?, ?, 'queued', 0, ?)
|
|
5289
|
+
`).run(input.projectId, input.id, input.executionId, input.threadId, input.triggerMessageId, JSON.stringify(input.spec), createdAt.toISOString());
|
|
5160
5290
|
} catch (error) {
|
|
5161
5291
|
if (isUniqueConstraintError2(error)) {
|
|
5162
5292
|
throw new AgentStorageError3("duplicate_id", `[SixbSqlite] Agent run '${input.id}' already exists for project '${input.projectId}'.`);
|
|
@@ -5209,9 +5339,9 @@ class SqliteAgentRunStore {
|
|
|
5209
5339
|
this.db.query(`
|
|
5210
5340
|
INSERT INTO agent_runs (
|
|
5211
5341
|
project_id, id, execution_id, kind, parent_run_id, spawn_key, spec,
|
|
5212
|
-
|
|
5213
|
-
) VALUES (?, ?, ?, 'subagent', ?, ?, ?,
|
|
5214
|
-
`).run(input.projectId, input.id, input.executionId, input.parentRunId, input.spawnKey, JSON.stringify(input.spec),
|
|
5342
|
+
status, attempt, created_at
|
|
5343
|
+
) VALUES (?, ?, ?, 'subagent', ?, ?, ?, 'queued', 0, ?)
|
|
5344
|
+
`).run(input.projectId, input.id, input.executionId, input.parentRunId, input.spawnKey, JSON.stringify(input.spec), (input.createdAt ?? new Date).toISOString());
|
|
5215
5345
|
} catch (error) {
|
|
5216
5346
|
if (isUniqueConstraintError2(error)) {
|
|
5217
5347
|
throw new AgentStorageError3("duplicate_id", `[SixbSqlite] Subagent run '${input.id}' conflicts with an existing durable spawn.`);
|
|
@@ -5499,6 +5629,7 @@ import {
|
|
|
5499
5629
|
aiModelCallCostDetails,
|
|
5500
5630
|
aiModelCallCostMatchesUsage,
|
|
5501
5631
|
buildAiAccountingOverviewFromFragments,
|
|
5632
|
+
directModelCallAttribution,
|
|
5502
5633
|
normalizeAiAccountingQuery,
|
|
5503
5634
|
normalizeAiModelCallAccountingQuery as normalizeAiModelCallAccountingQuery2,
|
|
5504
5635
|
normalizeAiModelCallCostRecord,
|
|
@@ -5539,6 +5670,7 @@ WITH filtered AS (
|
|
|
5539
5670
|
SELECT COALESCE(root_agent.execution_id, usage.execution_id) AS root_execution_id,
|
|
5540
5671
|
usage.execution_id, usage.provider_id, usage.requested_model_id, usage.occurred_at,
|
|
5541
5672
|
usage.total_tokens,
|
|
5673
|
+
execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
|
|
5542
5674
|
COALESCE(cost.status, 'unvalued') AS valuation_status,
|
|
5543
5675
|
cost.currency AS amount_currency, cost.amount_nanos,
|
|
5544
5676
|
root_agent.id AS root_agent_run_id,
|
|
@@ -5551,6 +5683,7 @@ WITH filtered AS (
|
|
|
5551
5683
|
workflow_node.workflow_id AS workflow_id,
|
|
5552
5684
|
workflow_node.workflow_run_id AS workflow_run_id
|
|
5553
5685
|
FROM ai_model_call_usage AS usage
|
|
5686
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
5554
5687
|
LEFT JOIN agent_runs AS direct_agent
|
|
5555
5688
|
ON direct_agent.project_id = usage.project_id AND direct_agent.execution_id = usage.execution_id
|
|
5556
5689
|
LEFT JOIN agent_runs AS root_agent
|
|
@@ -5588,6 +5721,7 @@ SELECT filtered.root_execution_id, execution_id, provider_id, requested_model_id
|
|
|
5588
5721
|
CAST(SUM((amount_nanos / 1000000) % 1000000) AS TEXT) AS amount_middle,
|
|
5589
5722
|
CAST(SUM(amount_nanos % 1000000) AS TEXT) AS amount_low,
|
|
5590
5723
|
MAX(root_agent_run_id) AS root_agent_run_id,
|
|
5724
|
+
MAX(executor_kind) AS executor_kind, MAX(executor_id) AS executor_id, MAX(primitive_id) AS primitive_id,
|
|
5591
5725
|
MAX(root_thread_id) AS root_thread_id,
|
|
5592
5726
|
MAX(direct_kind) AS direct_kind,
|
|
5593
5727
|
MAX(direct_run_id) AS direct_run_id,
|
|
@@ -5602,12 +5736,11 @@ GROUP BY filtered.root_execution_id, execution_id, provider_id, requested_model_
|
|
|
5602
5736
|
`;
|
|
5603
5737
|
|
|
5604
5738
|
// src/ai-usage-storage.ts
|
|
5739
|
+
import { assertAiUsageExecutionId, normalizeAiModelCallRecord } from "@sixb/core/internal/storage";
|
|
5605
5740
|
import { isModelReasoning, normalizeModelProviderIds } from "@sixb/core/models";
|
|
5606
5741
|
import {
|
|
5607
5742
|
AiUsageStorageError,
|
|
5608
|
-
aggregateAiModelCallUsage
|
|
5609
|
-
assertAiUsageExecutionId,
|
|
5610
|
-
normalizeAiModelCallRecord
|
|
5743
|
+
aggregateAiModelCallUsage
|
|
5611
5744
|
} from "@sixb/core/storage";
|
|
5612
5745
|
class SqliteAiUsageStorage {
|
|
5613
5746
|
connection;
|
|
@@ -5930,6 +6063,7 @@ var SQLITE_ACCOUNTING_LIST_PAGE_SQL = `
|
|
|
5930
6063
|
workflow_agent.node_run_id AS attribution_node_run_id,
|
|
5931
6064
|
workflow_node.workflow_id AS attribution_workflow_id,
|
|
5932
6065
|
workflow_node.workflow_run_id AS attribution_workflow_run_id,
|
|
6066
|
+
execution.executor_kind, execution.executor_id, execution.authority_primitive_id AS primitive_id,
|
|
5933
6067
|
COALESCE((
|
|
5934
6068
|
SELECT json_group_array(group_id) FROM (
|
|
5935
6069
|
SELECT group_id FROM ai_model_call_usage_groups
|
|
@@ -5946,6 +6080,7 @@ var SQLITE_ACCOUNTING_LIST_PAGE_SQL = `
|
|
|
5946
6080
|
cost.details AS cost_details,
|
|
5947
6081
|
cost.rated_at AS cost_rated_at
|
|
5948
6082
|
FROM ai_model_call_usage AS usage
|
|
6083
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
5949
6084
|
LEFT JOIN agent_runs AS direct_agent
|
|
5950
6085
|
ON direct_agent.project_id = usage.project_id
|
|
5951
6086
|
AND direct_agent.execution_id = usage.execution_id
|
|
@@ -5980,7 +6115,9 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
5980
6115
|
ELSE NULL
|
|
5981
6116
|
END AS attribution_agent_kind,
|
|
5982
6117
|
workflow_node.node_id AS attribution_agent_step_id,
|
|
5983
|
-
workflow_node.workflow_id
|
|
6118
|
+
COALESCE(workflow_node.workflow_id,
|
|
6119
|
+
CASE WHEN execution.executor_kind = 'workflow' THEN execution.authority_primitive_id END
|
|
6120
|
+
) AS attribution_workflow_id,
|
|
5984
6121
|
CASE
|
|
5985
6122
|
WHEN cost.status = 'rated' THEN 'rated'
|
|
5986
6123
|
WHEN cost.status = 'unpriceable' THEN 'unpriceable'
|
|
@@ -5989,6 +6126,7 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
5989
6126
|
CASE WHEN cost.status = 'rated' THEN cost.currency ELSE NULL END AS amount_currency,
|
|
5990
6127
|
CASE WHEN cost.status = 'rated' THEN cost.amount_nanos ELSE NULL END AS amount_nanos
|
|
5991
6128
|
FROM ai_model_call_usage AS usage
|
|
6129
|
+
JOIN executions AS execution ON execution.project_id = usage.project_id AND execution.id = usage.execution_id
|
|
5992
6130
|
LEFT JOIN agent_runs AS direct_agent
|
|
5993
6131
|
ON direct_agent.project_id = usage.project_id
|
|
5994
6132
|
AND direct_agent.execution_id = usage.execution_id
|
|
@@ -6120,7 +6258,7 @@ function sqliteAggregateAmount(row) {
|
|
|
6120
6258
|
function accountingItemFromRow(row) {
|
|
6121
6259
|
const usage = usageFromRow2(row, parseStringArray(row.requester_group_ids));
|
|
6122
6260
|
const cost = row.cost_status === null ? undefined : costFromJoinedRow(row);
|
|
6123
|
-
const attribution = row.attribution_kind === null ?
|
|
6261
|
+
const attribution = row.attribution_kind === null ? directModelCallAttribution(row.executor_kind, row.executor_id, row.primitive_id) : row.attribution_kind === "agent" ? {
|
|
6124
6262
|
kind: "agent",
|
|
6125
6263
|
agentRunId: required(row.attribution_agent_run_id),
|
|
6126
6264
|
threadId: required(row.attribution_thread_id)
|
|
@@ -7532,8 +7670,8 @@ class SqliteAuthAccessTokenStore {
|
|
|
7532
7670
|
}
|
|
7533
7671
|
function assertSubjectExists(db, projectId, subjectType, subjectId) {
|
|
7534
7672
|
if (subjectType === "user") {
|
|
7535
|
-
const
|
|
7536
|
-
if (
|
|
7673
|
+
const row = db.query("SELECT * FROM auth_users WHERE project_id = ? AND id = ?").get(projectId, subjectId);
|
|
7674
|
+
if (row)
|
|
7537
7675
|
return;
|
|
7538
7676
|
throw new AuthStorageError2("missing_user", `[Sixb] User '${subjectId}' not found for project '${projectId}'.`);
|
|
7539
7677
|
}
|
|
@@ -8664,7 +8802,7 @@ class SqliteAuthStorage {
|
|
|
8664
8802
|
}
|
|
8665
8803
|
this.consumeOidcAttempt(input, completedAt, projectId);
|
|
8666
8804
|
if (!userRow) {
|
|
8667
|
-
const
|
|
8805
|
+
const user = this.insertUser({
|
|
8668
8806
|
id: newUserId ?? assertNonEmpty(input.newUserId, "User id"),
|
|
8669
8807
|
projectId,
|
|
8670
8808
|
email,
|
|
@@ -8673,14 +8811,14 @@ class SqliteAuthStorage {
|
|
|
8673
8811
|
createdAt: completedAt
|
|
8674
8812
|
});
|
|
8675
8813
|
userRow = {
|
|
8676
|
-
project_id:
|
|
8677
|
-
id:
|
|
8678
|
-
email:
|
|
8679
|
-
display_name:
|
|
8680
|
-
avatar_url:
|
|
8681
|
-
status:
|
|
8682
|
-
created_at: toIso(
|
|
8683
|
-
updated_at: toIso(
|
|
8814
|
+
project_id: user.projectId,
|
|
8815
|
+
id: user.id,
|
|
8816
|
+
email: user.email,
|
|
8817
|
+
display_name: user.displayName ?? null,
|
|
8818
|
+
avatar_url: user.avatarUrl ?? null,
|
|
8819
|
+
status: user.status,
|
|
8820
|
+
created_at: toIso(user.createdAt),
|
|
8821
|
+
updated_at: toIso(user.updatedAt)
|
|
8684
8822
|
};
|
|
8685
8823
|
}
|
|
8686
8824
|
const user = rowToUserRecord(userRow);
|
|
@@ -8980,7 +9118,7 @@ class SqliteConnectorConnectionPersistence {
|
|
|
8980
9118
|
callback_binding_hash, reauthorization_id, reauthorization_revision,
|
|
8981
9119
|
reauthorization_connection_ids, created_at, expires_at
|
|
8982
9120
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
8983
|
-
`).run(record.projectId, record.connectorId, record.id, record.slot, record.initiatedByExecutionId, record.stateHash, json(record.codeVerifier), record.redirectUri, record.connectionRunId ?? null, record.returnTo ?? null, record.callbackBindingHash ?? null, record.reauthorizationId ?? null, record.reauthorizationRevision ?? null, record.reauthorizationConnectionIds ? json(record.reauthorizationConnectionIds) : null, iso(record.createdAt), iso(record.expiresAt));
|
|
9121
|
+
`).run(record.projectId, record.connectorId, record.id, record.slot, record.initiatedByExecutionId, record.stateHash, record.codeVerifier === undefined ? null : json(record.codeVerifier), record.redirectUri, record.connectionRunId ?? null, record.returnTo ?? null, record.callbackBindingHash ?? null, record.reauthorizationId ?? null, record.reauthorizationRevision ?? null, record.reauthorizationConnectionIds ? json(record.reauthorizationConnectionIds) : null, iso(record.createdAt), iso(record.expiresAt));
|
|
8984
9122
|
return true;
|
|
8985
9123
|
} catch (error) {
|
|
8986
9124
|
if (isUniqueConstraintError2(error))
|
|
@@ -9161,7 +9299,7 @@ function authorizationAttemptFromRow(row) {
|
|
|
9161
9299
|
slot: row.slot,
|
|
9162
9300
|
initiatedByExecutionId: row.initiated_by_execution_id,
|
|
9163
9301
|
stateHash: row.state_hash,
|
|
9164
|
-
codeVerifier: parseJson(row.code_verifier),
|
|
9302
|
+
...row.code_verifier === null ? {} : { codeVerifier: parseJson(row.code_verifier) },
|
|
9165
9303
|
redirectUri: row.redirect_uri,
|
|
9166
9304
|
...row.connection_run_id === null ? {} : { connectionRunId: row.connection_run_id },
|
|
9167
9305
|
...row.return_to === null ? {} : { returnTo: row.return_to },
|
|
@@ -9398,8 +9536,7 @@ function assertChanged(changes, label, id) {
|
|
|
9398
9536
|
import {
|
|
9399
9537
|
executionRecordFromStorageRow,
|
|
9400
9538
|
executionRecordToStorageRow,
|
|
9401
|
-
|
|
9402
|
-
validateExecutionRecordReferences
|
|
9539
|
+
prepareExecutionRecord
|
|
9403
9540
|
} from "@sixb/core/internal/execution-storage";
|
|
9404
9541
|
import { ExecutionStorageError } from "@sixb/core/storage";
|
|
9405
9542
|
|
|
@@ -9413,8 +9550,7 @@ class SqliteExecutionStorage {
|
|
|
9413
9550
|
this.shareSessions = shareSessions;
|
|
9414
9551
|
}
|
|
9415
9552
|
async create(input) {
|
|
9416
|
-
const record =
|
|
9417
|
-
await validateExecutionRecordReferences(record, {
|
|
9553
|
+
const record = await prepareExecutionRecord(input, {
|
|
9418
9554
|
auth: this.auth,
|
|
9419
9555
|
getExecution: (params) => this.getById(params),
|
|
9420
9556
|
getShareSession: (params) => this.shareSessions.getById(params)
|
|
@@ -9431,6 +9567,7 @@ class SqliteExecutionStorage {
|
|
|
9431
9567
|
source_id,
|
|
9432
9568
|
requested_by_user_id,
|
|
9433
9569
|
requested_by_service_account_id,
|
|
9570
|
+
requester_group_ids,
|
|
9434
9571
|
correlation_id,
|
|
9435
9572
|
parent_execution_id,
|
|
9436
9573
|
authority_kind,
|
|
@@ -9445,8 +9582,8 @@ class SqliteExecutionStorage {
|
|
|
9445
9582
|
authority_delegation_id,
|
|
9446
9583
|
authority_delegation_session_id,
|
|
9447
9584
|
created_at
|
|
9448
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
9449
|
-
`).run(row.projectId, row.id, row.executorKind, row.executorId, row.sourceKind, row.sourceId, row.requestedByUserId, row.requestedByServiceAccountId, row.correlationId, row.parentExecutionId, row.authorityKind, row.authorityUserId, row.authorityServiceAccountId, row.authoritySessionId, row.authorityAccessTokenId, row.authorityPrimitiveKind, row.authorityPrimitiveId, row.authorityKernelOperation, row.authorityDelegationKind, row.authorityDelegationId, row.authorityDelegationSessionId, row.createdAt.toISOString());
|
|
9585
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
9586
|
+
`).run(row.projectId, row.id, row.executorKind, row.executorId, row.sourceKind, row.sourceId, row.requestedByUserId, row.requestedByServiceAccountId, JSON.stringify(row.requesterGroupIds), row.correlationId, row.parentExecutionId, row.authorityKind, row.authorityUserId, row.authorityServiceAccountId, row.authoritySessionId, row.authorityAccessTokenId, row.authorityPrimitiveKind, row.authorityPrimitiveId, row.authorityKernelOperation, row.authorityDelegationKind, row.authorityDelegationId, row.authorityDelegationSessionId, row.createdAt.toISOString());
|
|
9450
9587
|
} catch (error) {
|
|
9451
9588
|
if (isDuplicateExecutionError(error)) {
|
|
9452
9589
|
throw new ExecutionStorageError("duplicate_execution", `[SixbSqlite] Execution '${record.id}' already exists in project '${record.projectId}'.`, { cause: error });
|
|
@@ -9474,6 +9611,7 @@ function toStorageRow(row) {
|
|
|
9474
9611
|
sourceId: row.source_id,
|
|
9475
9612
|
requestedByUserId: row.requested_by_user_id,
|
|
9476
9613
|
requestedByServiceAccountId: row.requested_by_service_account_id,
|
|
9614
|
+
requesterGroupIds: JSON.parse(row.requester_group_ids),
|
|
9477
9615
|
correlationId: row.correlation_id,
|
|
9478
9616
|
parentExecutionId: row.parent_execution_id,
|
|
9479
9617
|
authorityKind: row.authority_kind,
|
|
@@ -9901,18 +10039,18 @@ function compileExpansionOrder(expansion, target, neighborType, neighborId) {
|
|
|
9901
10039
|
}
|
|
9902
10040
|
function compileSet(projectId, op, inputs, ctx) {
|
|
9903
10041
|
if (inputs.length === 0) {
|
|
9904
|
-
const
|
|
10042
|
+
const order = compileOrder(identityOrderFields());
|
|
9905
10043
|
return {
|
|
9906
10044
|
sql: `
|
|
9907
10045
|
SELECT *
|
|
9908
10046
|
FROM ${ctx.source.objectsTable}
|
|
9909
10047
|
WHERE 1 = 0
|
|
9910
|
-
ORDER BY ${
|
|
10048
|
+
ORDER BY ${order.sql}
|
|
9911
10049
|
`,
|
|
9912
|
-
args:
|
|
10050
|
+
args: order.args,
|
|
9913
10051
|
totalSql: "SELECT 0 as total",
|
|
9914
10052
|
totalArgs: [],
|
|
9915
|
-
order
|
|
10053
|
+
order,
|
|
9916
10054
|
hasMore: () => false,
|
|
9917
10055
|
trimRows: identityRows,
|
|
9918
10056
|
nextPageToken: () => {
|
|
@@ -11353,6 +11491,12 @@ function sourceRecord(row) {
|
|
|
11353
11491
|
projectionRevision: row.projection_revision,
|
|
11354
11492
|
ownershipHash: row.ownership_hash,
|
|
11355
11493
|
ontologyRevision: row.ontology_revision,
|
|
11494
|
+
...row.base_materialization_id === null ? {} : {
|
|
11495
|
+
base: {
|
|
11496
|
+
materializationId: row.base_materialization_id,
|
|
11497
|
+
lastCommitId: row.base_commit_id
|
|
11498
|
+
}
|
|
11499
|
+
},
|
|
11356
11500
|
rootCount: row.root_count,
|
|
11357
11501
|
assertionCount: row.assertion_count,
|
|
11358
11502
|
createdAt: row.created_at,
|
|
@@ -11524,7 +11668,7 @@ import {
|
|
|
11524
11668
|
linkRefKey as linkRefKey3,
|
|
11525
11669
|
linkRefSortKey,
|
|
11526
11670
|
linkScopeSortKey as linkScopeSortKey2,
|
|
11527
|
-
MaterializationConflictError as
|
|
11671
|
+
MaterializationConflictError as MaterializationConflictError6,
|
|
11528
11672
|
objectRefKey as objectRefKey3,
|
|
11529
11673
|
objectRefSortKey,
|
|
11530
11674
|
telemetryPointKey as telemetryPointKey2,
|
|
@@ -11544,7 +11688,7 @@ import {
|
|
|
11544
11688
|
|
|
11545
11689
|
// src/ontology-storage/materialization-session.ts
|
|
11546
11690
|
import {
|
|
11547
|
-
MaterializationConflictError as
|
|
11691
|
+
MaterializationConflictError as MaterializationConflictError4
|
|
11548
11692
|
} from "@sixb/core/internal/materialization";
|
|
11549
11693
|
import {
|
|
11550
11694
|
correlateMaterializationChunk,
|
|
@@ -11559,9 +11703,9 @@ import {
|
|
|
11559
11703
|
import {
|
|
11560
11704
|
linkRefKey,
|
|
11561
11705
|
linkScopeSortKey,
|
|
11562
|
-
MaterializationConflictError as
|
|
11706
|
+
MaterializationConflictError as MaterializationConflictError3,
|
|
11563
11707
|
objectRefKey,
|
|
11564
|
-
projectionEntityKey,
|
|
11708
|
+
projectionEntityKey as projectionEntityKey2,
|
|
11565
11709
|
telemetryPointSortKey
|
|
11566
11710
|
} from "@sixb/core/internal/materialization";
|
|
11567
11711
|
import {
|
|
@@ -11570,6 +11714,132 @@ import {
|
|
|
11570
11714
|
linkSlotOverrideValue,
|
|
11571
11715
|
startScopeAccumulator
|
|
11572
11716
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
11717
|
+
|
|
11718
|
+
// src/ontology-storage/source-roots.ts
|
|
11719
|
+
import {
|
|
11720
|
+
MaterializationConflictError as MaterializationConflictError2,
|
|
11721
|
+
MaterializationValidationError,
|
|
11722
|
+
projectionEntityKey
|
|
11723
|
+
} from "@sixb/core/internal/materialization";
|
|
11724
|
+
import { sourceStageRoots, utf8SortKey } from "@sixb/core/internal/ontology-storage-provider";
|
|
11725
|
+
function replacementSourceRows(input) {
|
|
11726
|
+
const roots = input.incremental ? `ontology_source_roots AS changed
|
|
11727
|
+
CROSS JOIN ontology_source_roots AS roots ON roots.project_id = changed.project_id
|
|
11728
|
+
AND roots.source_id = changed.source_id AND roots.root_sort_key = changed.root_sort_key AND roots.active = 1` : "ontology_source_roots AS roots";
|
|
11729
|
+
const selected = input.incremental ? "changed" : "roots";
|
|
11730
|
+
return {
|
|
11731
|
+
sql: `SELECT * FROM ontology_source_rows WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
11732
|
+
UNION ALL
|
|
11733
|
+
SELECT rows.* FROM ${roots}
|
|
11734
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id
|
|
11735
|
+
AND rows.source_id = roots.source_id AND rows.materialization_id = roots.materialization_id
|
|
11736
|
+
AND rows.root_sort_key = roots.root_sort_key
|
|
11737
|
+
WHERE ${selected}.project_id = ? AND ${selected}.source_id = ?
|
|
11738
|
+
AND ${input.incremental ? "changed.materialization_id = ?" : "roots.active = 1"}`,
|
|
11739
|
+
values: [
|
|
11740
|
+
input.projectId,
|
|
11741
|
+
input.sourceId,
|
|
11742
|
+
input.materializationId,
|
|
11743
|
+
input.projectId,
|
|
11744
|
+
input.sourceId,
|
|
11745
|
+
...input.incremental ? [input.materializationId] : []
|
|
11746
|
+
]
|
|
11747
|
+
};
|
|
11748
|
+
}
|
|
11749
|
+
function stageSourceRoots(db, manifest, input) {
|
|
11750
|
+
const roots = sourceStageRoots(manifest.projection_kind, input);
|
|
11751
|
+
if (!manifest.base_materialization_id && roots.some((root) => root.deleted)) {
|
|
11752
|
+
throw new MaterializationValidationError("Root deletions require a source delta base.");
|
|
11753
|
+
}
|
|
11754
|
+
const existing = db.query(`SELECT staging_ordinal, deleted FROM ontology_source_roots
|
|
11755
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND root_sort_key = ?`);
|
|
11756
|
+
const insert = db.query(`INSERT INTO ontology_source_roots (
|
|
11757
|
+
project_id, source_id, materialization_id, root_sort_key, root_kind, root_key, root, staging_ordinal, deleted
|
|
11758
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING RETURNING root_sort_key`);
|
|
11759
|
+
for (const root of roots) {
|
|
11760
|
+
const inserted = insert.get(input.projectId, input.source.projectionId, input.materializationId, root.sortKey, root.root.kind, root.rootKey, canonicalJson(root.root), root.stagingOrdinal, Number(root.deleted));
|
|
11761
|
+
if (inserted)
|
|
11762
|
+
continue;
|
|
11763
|
+
const previous = existing.get(input.projectId, input.source.projectionId, input.materializationId, root.sortKey);
|
|
11764
|
+
if (!previous || previous.staging_ordinal !== root.stagingOrdinal || Boolean(previous.deleted) !== root.deleted) {
|
|
11765
|
+
throw new MaterializationValidationError("Source materialization repeats root or stream ordinal with different content.");
|
|
11766
|
+
}
|
|
11767
|
+
}
|
|
11768
|
+
}
|
|
11769
|
+
function activateSourceRoots(db, projectId, candidate, activation) {
|
|
11770
|
+
if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
|
|
11771
|
+
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
11772
|
+
if (candidate.base_materialization_id !== null && candidate.root_count === 0)
|
|
11773
|
+
return;
|
|
11774
|
+
const scope = [projectId, activation.source.projectionId, activation.materializationId];
|
|
11775
|
+
db.query(`UPDATE ontology_source_roots AS roots SET active = 0, retired_at = ?
|
|
11776
|
+
WHERE project_id = ? AND source_id = ? AND active = 1
|
|
11777
|
+
${candidate.base_materialization_id === null ? "" : `AND root_sort_key IN (
|
|
11778
|
+
SELECT root_sort_key FROM ontology_source_roots
|
|
11779
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?)`}`).run(activation.updatedAt, projectId, activation.source.projectionId, ...candidate.base_materialization_id === null ? [] : [...scope]);
|
|
11780
|
+
db.query(`UPDATE ontology_source_roots SET active = CASE WHEN deleted = 1 THEN 0 ELSE 1 END,
|
|
11781
|
+
retired_at = CASE WHEN deleted = 1 THEN ? ELSE NULL END
|
|
11782
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).run(activation.updatedAt, ...scope);
|
|
11783
|
+
}
|
|
11784
|
+
function assertSourceRootCoverage(db, manifest) {
|
|
11785
|
+
const invalid = db.query(`SELECT 1 FROM ontology_source_roots AS roots
|
|
11786
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
11787
|
+
AND roots.deleted = EXISTS (
|
|
11788
|
+
SELECT 1 FROM ontology_source_rows AS rows
|
|
11789
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
11790
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
11791
|
+
) LIMIT 1`).get(manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
11792
|
+
if (invalid)
|
|
11793
|
+
throw new MaterializationValidationError("Source roots must have complete assertions or an explicit deletion, never both.");
|
|
11794
|
+
}
|
|
11795
|
+
function activeSourceRows(db, projectId, refs) {
|
|
11796
|
+
const keys = refs.flatMap((entity) => {
|
|
11797
|
+
const entitySortKey = utf8SortKey(projectionEntityKey(entity));
|
|
11798
|
+
const roots = entity.kind === "object" ? [entitySortKey] : [
|
|
11799
|
+
entitySortKey,
|
|
11800
|
+
utf8SortKey(projectionEntityKey({ kind: "object", ref: entity.ref.source }))
|
|
11801
|
+
];
|
|
11802
|
+
return roots.map((rootSortKey) => ({ rootSortKey, entitySortKey }));
|
|
11803
|
+
});
|
|
11804
|
+
return db.query(`WITH requested AS (
|
|
11805
|
+
SELECT DISTINCT json_extract(value, '$.rootSortKey') AS root_sort_key,
|
|
11806
|
+
json_extract(value, '$.entitySortKey') AS entity_sort_key FROM json_each(?)
|
|
11807
|
+
) SELECT rows.* FROM requested
|
|
11808
|
+
CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ?
|
|
11809
|
+
AND roots.active = 1 AND roots.root_sort_key = requested.root_sort_key
|
|
11810
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
11811
|
+
AND rows.materialization_id = roots.materialization_id AND rows.entity_sort_key = requested.entity_sort_key
|
|
11812
|
+
AND rows.root_sort_key = roots.root_sort_key`).all(canonicalJson(keys), projectId);
|
|
11813
|
+
}
|
|
11814
|
+
function replacementAssertionRows(db, input) {
|
|
11815
|
+
const keys = input.refs.map((ref) => ({
|
|
11816
|
+
entityKind: ref.kind,
|
|
11817
|
+
entityKey: projectionEntityKey(ref),
|
|
11818
|
+
rootKeys: ref.kind === "object" ? [utf8SortKey(projectionEntityKey(ref))] : [
|
|
11819
|
+
utf8SortKey(projectionEntityKey(ref)),
|
|
11820
|
+
utf8SortKey(projectionEntityKey({ kind: "object", ref: ref.ref.source }))
|
|
11821
|
+
]
|
|
11822
|
+
}));
|
|
11823
|
+
const previous = !input.includePrevious ? "" : `UNION ALL
|
|
11824
|
+
SELECT rows.* FROM requested_entities AS requested
|
|
11825
|
+
CROSS JOIN ontology_source_roots AS roots ON roots.project_id = ? AND roots.source_id = ?
|
|
11826
|
+
AND roots.active = 1 AND roots.root_sort_key IN (SELECT value FROM json_each(requested.root_keys))
|
|
11827
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
11828
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
11829
|
+
AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
11830
|
+
${!input.incremental ? "" : `WHERE EXISTS (SELECT 1 FROM ontology_source_roots AS changed
|
|
11831
|
+
WHERE changed.project_id = roots.project_id AND changed.source_id = roots.source_id
|
|
11832
|
+
AND changed.materialization_id = ? AND changed.root_sort_key = roots.root_sort_key)`}`;
|
|
11833
|
+
return db.query(`WITH requested_entities AS (
|
|
11834
|
+
SELECT DISTINCT json_extract(value, '$.entityKind') AS entity_kind,
|
|
11835
|
+
json_extract(value, '$.entityKey') AS entity_key, json_extract(value, '$.rootKeys') AS root_keys FROM json_each(?)
|
|
11836
|
+
) SELECT rows.* FROM requested_entities AS requested
|
|
11837
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.project_id = ? AND rows.source_id = ?
|
|
11838
|
+
AND rows.materialization_id = ? AND rows.entity_kind = requested.entity_kind AND rows.entity_key = requested.entity_key
|
|
11839
|
+
${previous}`).all(canonicalJson(keys), input.projectId, input.sourceId, input.materializationId, ...input.includePrevious ? [input.projectId, input.sourceId, ...input.incremental ? [input.materializationId] : []] : []);
|
|
11840
|
+
}
|
|
11841
|
+
|
|
11842
|
+
// src/ontology-storage/materialization-state.ts
|
|
11573
11843
|
var SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
|
|
11574
11844
|
var SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
|
|
11575
11845
|
function linkScopeRequestJson(scopes) {
|
|
@@ -11606,18 +11876,7 @@ class SqliteMaterializationStateReader {
|
|
|
11606
11876
|
ON objects.project_id = ?
|
|
11607
11877
|
AND objects.object_type_id = requested.object_type_id
|
|
11608
11878
|
AND objects.primary_id = requested.primary_id`).all(requested, this.projectId);
|
|
11609
|
-
const sourceRows = this.db.
|
|
11610
|
-
SELECT rows.* FROM requested
|
|
11611
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
11612
|
-
ON rows.project_id = ?
|
|
11613
|
-
AND rows.entity_kind = 'object'
|
|
11614
|
-
AND rows.object_type_id = requested.object_type_id
|
|
11615
|
-
AND rows.primary_id = requested.primary_id
|
|
11616
|
-
JOIN ontology_sources AS sources
|
|
11617
|
-
ON sources.project_id = rows.project_id
|
|
11618
|
-
AND sources.source_id = rows.source_id
|
|
11619
|
-
AND sources.materialization_id = rows.materialization_id
|
|
11620
|
-
WHERE sources.status = 'active'`).all(requested, this.projectId);
|
|
11879
|
+
const sourceRows = activeSourceRows(this.db, this.projectId, refs.map((ref) => ({ kind: "object", ref })));
|
|
11621
11880
|
const overrideRows = this.db.query(`WITH requested AS (${requestedObjects})
|
|
11622
11881
|
SELECT overrides.* FROM requested
|
|
11623
11882
|
CROSS JOIN ontology_object_overrides AS overrides
|
|
@@ -11656,7 +11915,7 @@ class SqliteMaterializationStateReader {
|
|
|
11656
11915
|
const effectiveRow = effective.get(key);
|
|
11657
11916
|
return {
|
|
11658
11917
|
ref: structuredClone(ref),
|
|
11659
|
-
source: sourceObject(sources.get(
|
|
11918
|
+
source: sourceObject(sources.get(projectionEntityKey2({ kind: "object", ref }))),
|
|
11660
11919
|
override: overrides.get(key) ?? null,
|
|
11661
11920
|
effective: effectiveRow ? effectiveObjectSnapshot(effectiveRow) : null,
|
|
11662
11921
|
latestTelemetry: telemetry.get(key) ?? []
|
|
@@ -11701,21 +11960,7 @@ class SqliteMaterializationStateReader {
|
|
|
11701
11960
|
AND links.link_id = requested.link_id
|
|
11702
11961
|
AND links.target_type_id = requested.target_type_id
|
|
11703
11962
|
AND links.target_id = requested.target_id`).all(requested, this.projectId);
|
|
11704
|
-
const sourceRows = this.db.
|
|
11705
|
-
SELECT rows.* FROM requested
|
|
11706
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
11707
|
-
ON rows.project_id = ?
|
|
11708
|
-
AND rows.entity_kind = 'link'
|
|
11709
|
-
AND rows.source_type_id = requested.source_type_id
|
|
11710
|
-
AND rows.source_primary_id = requested.source_primary_id
|
|
11711
|
-
AND rows.link_id = requested.link_id
|
|
11712
|
-
AND rows.target_type_id = requested.target_type_id
|
|
11713
|
-
AND rows.target_primary_id = requested.target_primary_id
|
|
11714
|
-
JOIN ontology_sources AS sources
|
|
11715
|
-
ON sources.project_id = rows.project_id
|
|
11716
|
-
AND sources.source_id = rows.source_id
|
|
11717
|
-
AND sources.materialization_id = rows.materialization_id
|
|
11718
|
-
WHERE sources.status = 'active'`).all(requested, this.projectId);
|
|
11963
|
+
const sourceRows = activeSourceRows(this.db, this.projectId, refs.map((ref) => ({ kind: "link", ref })));
|
|
11719
11964
|
const overrideRows = this.db.query(`WITH requested AS (${sourceRequest})
|
|
11720
11965
|
SELECT overrides.* FROM requested
|
|
11721
11966
|
CROSS JOIN ontology_link_overrides AS overrides
|
|
@@ -11746,7 +11991,7 @@ class SqliteMaterializationStateReader {
|
|
|
11746
11991
|
const effectiveRow = effective.get(key);
|
|
11747
11992
|
return {
|
|
11748
11993
|
ref: structuredClone(ref),
|
|
11749
|
-
source: sourceLink(sources.get(
|
|
11994
|
+
source: sourceLink(sources.get(projectionEntityKey2({ kind: "link", ref }))),
|
|
11750
11995
|
override: overrides.get(key) ?? null,
|
|
11751
11996
|
slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
|
|
11752
11997
|
effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null
|
|
@@ -11811,11 +12056,12 @@ class SqliteMaterializationStateReader {
|
|
|
11811
12056
|
AND rows.source_type_id = requested.source_type_id
|
|
11812
12057
|
AND rows.source_primary_id = requested.source_primary_id
|
|
11813
12058
|
AND rows.link_id = requested.link_id
|
|
11814
|
-
JOIN
|
|
12059
|
+
JOIN ontology_source_roots AS sources
|
|
11815
12060
|
ON sources.project_id = rows.project_id
|
|
11816
12061
|
AND sources.source_id = rows.source_id
|
|
11817
12062
|
AND sources.materialization_id = rows.materialization_id
|
|
11818
|
-
|
|
12063
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
12064
|
+
WHERE sources.active = 1`).all(requestedJson, this.projectId);
|
|
11819
12065
|
const slotOverrideRows = this.db.query(`WITH requested AS (
|
|
11820
12066
|
SELECT DISTINCT
|
|
11821
12067
|
json_extract(value, '$.sourceTypeId') AS source_type_id,
|
|
@@ -11871,13 +12117,13 @@ class SqliteMaterializationStateReader {
|
|
|
11871
12117
|
ORDER BY requested.scope_sort_key, ${linkSortExpression("links")}
|
|
11872
12118
|
`).iterate(requestedJson, this.projectId);
|
|
11873
12119
|
for (const row of rows) {
|
|
11874
|
-
const
|
|
11875
|
-
if (!
|
|
11876
|
-
throw new
|
|
12120
|
+
const accumulator = accumulators.get(row.scope_sort_key);
|
|
12121
|
+
if (!accumulator) {
|
|
12122
|
+
throw new MaterializationConflictError3("effective-state", `Unexpected link scope '${row.scope_sort_key}' returned by storage.`);
|
|
11877
12123
|
}
|
|
11878
12124
|
const snapshot = effectiveLinkSnapshot(row);
|
|
11879
|
-
appendScopeSnapshot(
|
|
11880
|
-
effective.set(row.scope_sort_key,
|
|
12125
|
+
appendScopeSnapshot(accumulator, snapshot);
|
|
12126
|
+
effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null);
|
|
11881
12127
|
}
|
|
11882
12128
|
return {
|
|
11883
12129
|
revisions: scopes.map(({ source, linkId }) => finishScopeAccumulator(accumulators.get(linkScopeSortKey(source, linkId)))),
|
|
@@ -11904,11 +12150,12 @@ class SqliteMaterializationStateReader {
|
|
|
11904
12150
|
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
11905
12151
|
rows.target_type_id, rows.target_primary_id
|
|
11906
12152
|
FROM ontology_source_rows AS rows
|
|
11907
|
-
JOIN
|
|
12153
|
+
JOIN ontology_source_roots AS sources
|
|
11908
12154
|
ON sources.project_id = rows.project_id
|
|
11909
12155
|
AND sources.source_id = rows.source_id
|
|
11910
12156
|
AND sources.materialization_id = rows.materialization_id
|
|
11911
|
-
|
|
12157
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
12158
|
+
WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.active = 1
|
|
11912
12159
|
), selected AS (
|
|
11913
12160
|
SELECT DISTINCT links.*,
|
|
11914
12161
|
${linkSortExpression("links")} AS sort_key
|
|
@@ -11951,19 +12198,19 @@ class SqliteMaterializationStateReader {
|
|
|
11951
12198
|
const candidates = this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
|
|
11952
12199
|
return base.map((state) => ({
|
|
11953
12200
|
ref: state.ref,
|
|
11954
|
-
candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(
|
|
12201
|
+
candidateSource: sourceObject(candidates.byMaterialization.get(candidateMaterializationId)?.get(projectionEntityKey2({ kind: "object", ref: state.ref }))),
|
|
11955
12202
|
override: state.override,
|
|
11956
12203
|
effective: state.effective,
|
|
11957
12204
|
latestTelemetry: state.latestTelemetry
|
|
11958
12205
|
}));
|
|
11959
12206
|
}
|
|
11960
|
-
replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities) {
|
|
12207
|
+
replacementLinkStates(sourceId, candidateMaterializationId, materializationIds, identities, incremental = false) {
|
|
11961
12208
|
const refs = identities.map((identity) => identity.ref);
|
|
11962
12209
|
const base = this.linkStates(refs);
|
|
11963
|
-
const replacements = this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })));
|
|
12210
|
+
const replacements = this.replacementSources(sourceId, materializationIds, refs.map((ref) => ({ kind: "link", ref })), incremental);
|
|
11964
12211
|
const candidate = replacements.byMaterialization.get(candidateMaterializationId);
|
|
11965
12212
|
return base.map((state, index) => {
|
|
11966
|
-
const key =
|
|
12213
|
+
const key = projectionEntityKey2({ kind: "link", ref: state.ref });
|
|
11967
12214
|
return {
|
|
11968
12215
|
ref: state.ref,
|
|
11969
12216
|
candidateSource: replacements.owned.has(key) ? sourceLink(candidate?.get(key)) : state.source,
|
|
@@ -11987,32 +12234,23 @@ class SqliteMaterializationStateReader {
|
|
|
11987
12234
|
`).get(this.projectId, ref.source.objectTypeId, ref.source.primaryId, ref.linkId, ref.target.objectTypeId, ref.target.primaryId);
|
|
11988
12235
|
return row ? row.last_commit_id : undefined;
|
|
11989
12236
|
}
|
|
11990
|
-
replacementSources(sourceId, materializationIds, refs) {
|
|
12237
|
+
replacementSources(sourceId, materializationIds, refs, incremental = false) {
|
|
11991
12238
|
if (materializationIds.length === 0 || refs.length === 0) {
|
|
11992
12239
|
return { owned: new Set, byMaterialization: new Map };
|
|
11993
12240
|
}
|
|
11994
|
-
const
|
|
11995
|
-
|
|
11996
|
-
|
|
11997
|
-
|
|
11998
|
-
|
|
11999
|
-
|
|
12000
|
-
|
|
12001
|
-
|
|
12002
|
-
)
|
|
12003
|
-
SELECT rows.* FROM requested_materializations
|
|
12004
|
-
CROSS JOIN requested_entities
|
|
12005
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
12006
|
-
ON rows.project_id = ?
|
|
12007
|
-
AND rows.source_id = ?
|
|
12008
|
-
AND rows.materialization_id = requested_materializations.materialization_id
|
|
12009
|
-
AND rows.entity_kind = requested_entities.entity_kind
|
|
12010
|
-
AND rows.entity_key = requested_entities.entity_key`).all(requestedEntities, requestedMaterializations, this.projectId, sourceId);
|
|
12241
|
+
const rows = replacementAssertionRows(this.db, {
|
|
12242
|
+
projectId: this.projectId,
|
|
12243
|
+
sourceId,
|
|
12244
|
+
materializationId: materializationIds[0],
|
|
12245
|
+
incremental,
|
|
12246
|
+
includePrevious: materializationIds.length > 1,
|
|
12247
|
+
refs
|
|
12248
|
+
});
|
|
12011
12249
|
const owned = new Set;
|
|
12012
12250
|
const byMaterialization = new Map;
|
|
12013
12251
|
for (const row of rows) {
|
|
12014
12252
|
const source = storedSource(row);
|
|
12015
|
-
const key =
|
|
12253
|
+
const key = projectionEntityKey2(source.assertion);
|
|
12016
12254
|
owned.add(key);
|
|
12017
12255
|
const materialization = byMaterialization.get(row.materialization_id) ?? new Map;
|
|
12018
12256
|
materialization.set(key, source);
|
|
@@ -12028,19 +12266,30 @@ class SqliteMaterializationStateReader {
|
|
|
12028
12266
|
this.prepareReplacementLinks(input);
|
|
12029
12267
|
}
|
|
12030
12268
|
prepareReplacementObjects(input) {
|
|
12269
|
+
const rows = replacementSourceRows({
|
|
12270
|
+
projectId: this.projectId,
|
|
12271
|
+
sourceId: input.sourceId,
|
|
12272
|
+
materializationId: input.candidateMaterializationId,
|
|
12273
|
+
incremental: input.incremental ?? false
|
|
12274
|
+
});
|
|
12031
12275
|
this.db.query(`
|
|
12032
12276
|
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
12033
12277
|
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
12034
12278
|
)
|
|
12035
12279
|
SELECT ?, 'object', json_array(object_type_id, primary_id),
|
|
12036
12280
|
MIN(entity_sort_key), 1
|
|
12037
|
-
FROM
|
|
12038
|
-
WHERE
|
|
12039
|
-
AND materialization_id IN (?, COALESCE(?, ''))
|
|
12281
|
+
FROM (${rows.sql}) AS rows
|
|
12282
|
+
WHERE entity_kind = 'object'
|
|
12040
12283
|
GROUP BY object_type_id, primary_id
|
|
12041
|
-
`).run(input.sessionId,
|
|
12284
|
+
`).run(input.sessionId, ...rows.values);
|
|
12042
12285
|
}
|
|
12043
12286
|
prepareReplacementLinks(input) {
|
|
12287
|
+
const rows = replacementSourceRows({
|
|
12288
|
+
projectId: this.projectId,
|
|
12289
|
+
sourceId: input.sourceId,
|
|
12290
|
+
materializationId: input.candidateMaterializationId,
|
|
12291
|
+
incremental: input.incremental ?? false
|
|
12292
|
+
});
|
|
12044
12293
|
this.db.query(`
|
|
12045
12294
|
WITH incident_objects AS (
|
|
12046
12295
|
SELECT DISTINCT
|
|
@@ -12051,9 +12300,8 @@ class SqliteMaterializationStateReader {
|
|
|
12051
12300
|
), replacement_links AS (
|
|
12052
12301
|
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
12053
12302
|
target_type_id, target_primary_id AS target_id
|
|
12054
|
-
FROM
|
|
12055
|
-
WHERE
|
|
12056
|
-
AND materialization_id IN (?, COALESCE(?, ''))
|
|
12303
|
+
FROM (${rows.sql}) AS rows
|
|
12304
|
+
WHERE entity_kind = 'link'
|
|
12057
12305
|
), incident_links AS (
|
|
12058
12306
|
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
12059
12307
|
links.target_type_id, links.target_id
|
|
@@ -12094,28 +12342,30 @@ class SqliteMaterializationStateReader {
|
|
|
12094
12342
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
12095
12343
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
12096
12344
|
FROM ontology_source_rows AS rows
|
|
12097
|
-
JOIN
|
|
12345
|
+
JOIN ontology_source_roots AS sources
|
|
12098
12346
|
ON sources.project_id = rows.project_id
|
|
12099
12347
|
AND sources.source_id = rows.source_id
|
|
12100
12348
|
AND sources.materialization_id = rows.materialization_id
|
|
12349
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
12101
12350
|
JOIN incident_objects
|
|
12102
12351
|
ON incident_objects.object_type_id = rows.source_type_id
|
|
12103
12352
|
AND incident_objects.primary_id = rows.source_primary_id
|
|
12104
12353
|
WHERE rows.project_id = ? AND rows.entity_kind = 'link'
|
|
12105
|
-
AND sources.
|
|
12354
|
+
AND sources.active = 1
|
|
12106
12355
|
UNION
|
|
12107
12356
|
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
12108
12357
|
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
12109
12358
|
FROM ontology_source_rows AS rows
|
|
12110
|
-
JOIN
|
|
12359
|
+
JOIN ontology_source_roots AS sources
|
|
12111
12360
|
ON sources.project_id = rows.project_id
|
|
12112
12361
|
AND sources.source_id = rows.source_id
|
|
12113
12362
|
AND sources.materialization_id = rows.materialization_id
|
|
12363
|
+
AND sources.root_sort_key = rows.root_sort_key
|
|
12114
12364
|
JOIN incident_objects
|
|
12115
12365
|
ON incident_objects.object_type_id = rows.target_type_id
|
|
12116
12366
|
AND incident_objects.primary_id = rows.target_primary_id
|
|
12117
12367
|
WHERE rows.project_id = ? AND rows.entity_kind = 'link'
|
|
12118
|
-
AND sources.
|
|
12368
|
+
AND sources.active = 1
|
|
12119
12369
|
), diff_links AS (
|
|
12120
12370
|
SELECT * FROM replacement_links
|
|
12121
12371
|
UNION SELECT * FROM incident_links
|
|
@@ -12153,7 +12403,7 @@ class SqliteMaterializationStateReader {
|
|
|
12153
12403
|
json_array(source_type_id, source_id, link_id, target_type_id, target_id),
|
|
12154
12404
|
sort_key, diff_required
|
|
12155
12405
|
FROM selected
|
|
12156
|
-
`).run(input.sessionId,
|
|
12406
|
+
`).run(input.sessionId, ...rows.values, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, input.sessionId);
|
|
12157
12407
|
}
|
|
12158
12408
|
replacementRows(input, cursor) {
|
|
12159
12409
|
return this.db.query(`
|
|
@@ -12191,9 +12441,9 @@ function activeSourceMap(rows) {
|
|
|
12191
12441
|
const result = new Map;
|
|
12192
12442
|
for (const row of rows) {
|
|
12193
12443
|
const source = storedSource(row);
|
|
12194
|
-
const key =
|
|
12444
|
+
const key = projectionEntityKey2(source.assertion);
|
|
12195
12445
|
if (result.has(key)) {
|
|
12196
|
-
throw new
|
|
12446
|
+
throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert ${key}.`);
|
|
12197
12447
|
}
|
|
12198
12448
|
result.set(key, source);
|
|
12199
12449
|
}
|
|
@@ -12208,7 +12458,7 @@ function activeScopeSourceMap(rows) {
|
|
|
12208
12458
|
const ref = source.assertion.ref;
|
|
12209
12459
|
const key = linkScopeSortKey(ref.source, ref.linkId);
|
|
12210
12460
|
if (result.has(key)) {
|
|
12211
|
-
throw new
|
|
12461
|
+
throw new MaterializationConflictError3("source-materialization", `Multiple active sources assert cardinality-one link slot ${key}.`);
|
|
12212
12462
|
}
|
|
12213
12463
|
result.set(key, source);
|
|
12214
12464
|
}
|
|
@@ -12272,7 +12522,7 @@ function storedSource(row) {
|
|
|
12272
12522
|
}
|
|
12273
12523
|
function effectiveObjectSnapshot(row) {
|
|
12274
12524
|
if (!row.last_commit_id) {
|
|
12275
|
-
throw new
|
|
12525
|
+
throw new MaterializationConflictError3("effective-state", `Effective object ${row.object_type_id}:${row.primary_id} lacks materializer provenance.`);
|
|
12276
12526
|
}
|
|
12277
12527
|
return {
|
|
12278
12528
|
ref: objectRefFromColumns(row),
|
|
@@ -12285,7 +12535,7 @@ function effectiveObjectSnapshot(row) {
|
|
|
12285
12535
|
}
|
|
12286
12536
|
function effectiveLinkSnapshot(row) {
|
|
12287
12537
|
if (!row.last_commit_id) {
|
|
12288
|
-
throw new
|
|
12538
|
+
throw new MaterializationConflictError3("effective-state", `Effective link ${linkRefKey(linkRefFromColumns(row))} lacks materializer provenance.`);
|
|
12289
12539
|
}
|
|
12290
12540
|
return {
|
|
12291
12541
|
ref: linkRefFromColumns(row),
|
|
@@ -12299,7 +12549,7 @@ function effectiveLinkSnapshot(row) {
|
|
|
12299
12549
|
}
|
|
12300
12550
|
function storedPoint(row) {
|
|
12301
12551
|
if (!row.last_commit_id) {
|
|
12302
|
-
throw new
|
|
12552
|
+
throw new MaterializationConflictError3("timeseries-point", `Telemetry point ${telemetryPointSortKey({
|
|
12303
12553
|
object: { objectTypeId: row.object_type_id, primaryId: row.object_id },
|
|
12304
12554
|
propertyId: row.property_id
|
|
12305
12555
|
}, row.at)} lacks materializer provenance.`);
|
|
@@ -12338,7 +12588,7 @@ class SqliteMaterializationSessions {
|
|
|
12338
12588
|
}
|
|
12339
12589
|
create(header) {
|
|
12340
12590
|
if (!this.context?.active) {
|
|
12341
|
-
throw new
|
|
12591
|
+
throw new MaterializationConflictError4("effective-state", "Materialization sessions require an active storage transaction.");
|
|
12342
12592
|
}
|
|
12343
12593
|
this.ensureTables();
|
|
12344
12594
|
const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id);
|
|
@@ -12350,7 +12600,7 @@ class SqliteMaterializationSessions {
|
|
|
12350
12600
|
require(session) {
|
|
12351
12601
|
const value = this.sessions.get(session.providerToken);
|
|
12352
12602
|
if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
|
|
12353
|
-
throw new
|
|
12603
|
+
throw new MaterializationConflictError4("effective-state", "Materialization session is inactive.");
|
|
12354
12604
|
}
|
|
12355
12605
|
return value;
|
|
12356
12606
|
}
|
|
@@ -12410,7 +12660,7 @@ class SqliteMaterializationSessions {
|
|
|
12410
12660
|
const session = this.require(input.session);
|
|
12411
12661
|
const stream = session.workStreams[input.order];
|
|
12412
12662
|
if (stream.started) {
|
|
12413
|
-
throw new
|
|
12663
|
+
throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
|
|
12414
12664
|
}
|
|
12415
12665
|
stream.started = true;
|
|
12416
12666
|
session.workSealed = true;
|
|
@@ -12687,7 +12937,7 @@ function recordSummary(summary, record, columns) {
|
|
|
12687
12937
|
// src/ontology-storage/materialization-writer.ts
|
|
12688
12938
|
import {
|
|
12689
12939
|
linkRefKey as linkRefKey2,
|
|
12690
|
-
MaterializationConflictError as
|
|
12940
|
+
MaterializationConflictError as MaterializationConflictError5,
|
|
12691
12941
|
objectRefKey as objectRefKey2,
|
|
12692
12942
|
telemetryPointKey
|
|
12693
12943
|
} from "@sixb/core/internal/materialization";
|
|
@@ -12859,7 +13109,7 @@ class SqliteMaterializationWriter {
|
|
|
12859
13109
|
`).run(projectId, point.series.object.objectTypeId, point.series.object.primaryId, point.series.propertyId, canonicalJson(point.value), point.unit ?? null, point.at, point.lastCommitId);
|
|
12860
13110
|
} catch (error) {
|
|
12861
13111
|
if (isSqliteConstraintError(error)) {
|
|
12862
|
-
throw new
|
|
13112
|
+
throw new MaterializationConflictError5("timeseries-point", `Telemetry point ${telemetryPointKey(point.series, point.at)} changed.`);
|
|
12863
13113
|
}
|
|
12864
13114
|
throw error;
|
|
12865
13115
|
}
|
|
@@ -12921,13 +13171,13 @@ class SqliteOntologyMaterializationStorage {
|
|
|
12921
13171
|
const linkScopeRevisions = reader.linkScopeRevisions(input.expected.linkScopes);
|
|
12922
13172
|
for (const [index, expected] of input.expected.linkScopes.entries()) {
|
|
12923
13173
|
if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
|
|
12924
|
-
throw new
|
|
13174
|
+
throw new MaterializationConflictError6("effective-state", `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`);
|
|
12925
13175
|
}
|
|
12926
13176
|
}
|
|
12927
13177
|
for (const expected of input.expected.points) {
|
|
12928
13178
|
const point = reader.exactPoint(expected.series, expected.at);
|
|
12929
13179
|
if ((point?.lastCommitId ?? null) !== expected.lastCommitId) {
|
|
12930
|
-
throw new
|
|
13180
|
+
throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey2(expected.series, expected.at)} changed.`);
|
|
12931
13181
|
}
|
|
12932
13182
|
}
|
|
12933
13183
|
return session.publicSession();
|
|
@@ -12985,10 +13235,10 @@ class SqliteOntologyMaterializationStorage {
|
|
|
12985
13235
|
const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
|
|
12986
13236
|
if (input.entityKind === "object") {
|
|
12987
13237
|
if (replacement.projectionKind !== "object") {
|
|
12988
|
-
throw new
|
|
13238
|
+
throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
|
|
12989
13239
|
}
|
|
12990
13240
|
if (replacement.objectStreamStarted) {
|
|
12991
|
-
throw new
|
|
13241
|
+
throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
|
|
12992
13242
|
}
|
|
12993
13243
|
replacement.objectStreamStarted = true;
|
|
12994
13244
|
for (const identities of reader.replacementIdentities({
|
|
@@ -12996,6 +13246,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
12996
13246
|
sourceId: replacement.sourceId,
|
|
12997
13247
|
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
12998
13248
|
previousMaterializationId: replacement.previousMaterializationId,
|
|
13249
|
+
incremental: replacement.incremental,
|
|
12999
13250
|
kind: "object",
|
|
13000
13251
|
pageRows: input.pageRows
|
|
13001
13252
|
})) {
|
|
@@ -13014,10 +13265,10 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13014
13265
|
return;
|
|
13015
13266
|
}
|
|
13016
13267
|
if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
|
|
13017
|
-
throw new
|
|
13268
|
+
throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
|
|
13018
13269
|
}
|
|
13019
13270
|
if (replacement.linkStreamStarted) {
|
|
13020
|
-
throw new
|
|
13271
|
+
throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
|
|
13021
13272
|
}
|
|
13022
13273
|
replacement.linkStreamStarted = true;
|
|
13023
13274
|
const materializationIds = [
|
|
@@ -13029,6 +13280,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13029
13280
|
sourceId: replacement.sourceId,
|
|
13030
13281
|
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
13031
13282
|
previousMaterializationId: replacement.previousMaterializationId,
|
|
13283
|
+
incremental: replacement.incremental,
|
|
13032
13284
|
kind: "link",
|
|
13033
13285
|
pageRows: input.pageRows
|
|
13034
13286
|
})) {
|
|
@@ -13039,7 +13291,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13039
13291
|
}
|
|
13040
13292
|
return identity;
|
|
13041
13293
|
});
|
|
13042
|
-
const links = reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities);
|
|
13294
|
+
const links = reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
|
|
13043
13295
|
yield { objects: [], links };
|
|
13044
13296
|
}
|
|
13045
13297
|
this.sessions.require(input.session);
|
|
@@ -13091,14 +13343,14 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13091
13343
|
requireReplacement(session, input) {
|
|
13092
13344
|
if (session.replacement) {
|
|
13093
13345
|
if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
|
|
13094
|
-
throw new
|
|
13346
|
+
throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
|
|
13095
13347
|
}
|
|
13096
13348
|
return session.replacement;
|
|
13097
13349
|
}
|
|
13098
13350
|
const projectId = session.header.commit.projectId;
|
|
13099
13351
|
const candidate = this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId);
|
|
13100
13352
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
13101
|
-
throw new
|
|
13353
|
+
throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
|
|
13102
13354
|
}
|
|
13103
13355
|
assertProjectionExecution(this.db, {
|
|
13104
13356
|
projectId,
|
|
@@ -13108,12 +13360,13 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13108
13360
|
});
|
|
13109
13361
|
const previous = this.getActiveSource(projectId, input.source.projectionId);
|
|
13110
13362
|
if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
|
|
13111
|
-
throw new
|
|
13363
|
+
throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
|
|
13112
13364
|
}
|
|
13113
13365
|
session.replacement = {
|
|
13114
13366
|
sourceId: input.source.projectionId,
|
|
13115
13367
|
candidateMaterializationId: input.candidateMaterializationId,
|
|
13116
13368
|
previousMaterializationId: previous?.materialization_id ?? null,
|
|
13369
|
+
incremental: candidate.base_materialization_id !== null,
|
|
13117
13370
|
projectionKind: candidate.projection_kind,
|
|
13118
13371
|
objectStreamStarted: false,
|
|
13119
13372
|
objectStreamCompleted: false,
|
|
@@ -13142,38 +13395,38 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13142
13395
|
if (!duplicate)
|
|
13143
13396
|
return;
|
|
13144
13397
|
if (duplicate.duplicate === "origin") {
|
|
13145
|
-
throw new
|
|
13398
|
+
throw new MaterializationConflictError6("run-correlation", "Ontology commit origin already has an authoritative commit.");
|
|
13146
13399
|
}
|
|
13147
|
-
throw new
|
|
13400
|
+
throw new MaterializationConflictError6("idempotency", duplicate.duplicate === "id" ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
|
|
13148
13401
|
}
|
|
13149
13402
|
assertSource(expected, projectId) {
|
|
13150
13403
|
const active = this.getActiveSource(projectId, expected.source.projectionId);
|
|
13151
13404
|
if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
|
|
13152
|
-
throw new
|
|
13405
|
+
throw new MaterializationConflictError6("projection-fence", `Source '${expected.source.projectionId}' changed.`);
|
|
13153
13406
|
}
|
|
13154
13407
|
}
|
|
13155
13408
|
assertObject(reader, expected) {
|
|
13156
13409
|
const row = reader.effectiveObjectRevision(expected.ref);
|
|
13157
13410
|
if (!expected.exists) {
|
|
13158
13411
|
if (row) {
|
|
13159
|
-
throw new
|
|
13412
|
+
throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} to be absent.`);
|
|
13160
13413
|
}
|
|
13161
13414
|
return;
|
|
13162
13415
|
}
|
|
13163
13416
|
if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
|
|
13164
|
-
throw new
|
|
13417
|
+
throw new MaterializationConflictError6("effective-state", `Expected object ${objectRefKey3(expected.ref)} changed.`);
|
|
13165
13418
|
}
|
|
13166
13419
|
}
|
|
13167
13420
|
assertLink(reader, expected) {
|
|
13168
13421
|
const lastCommitId = reader.effectiveLinkLastCommit(expected.ref);
|
|
13169
13422
|
if (!expected.exists) {
|
|
13170
13423
|
if (lastCommitId !== undefined) {
|
|
13171
|
-
throw new
|
|
13424
|
+
throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} to be absent.`);
|
|
13172
13425
|
}
|
|
13173
13426
|
return;
|
|
13174
13427
|
}
|
|
13175
13428
|
if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
|
|
13176
|
-
throw new
|
|
13429
|
+
throw new MaterializationConflictError6("effective-state", `Expected link ${linkRefKey3(expected.ref)} changed.`);
|
|
13177
13430
|
}
|
|
13178
13431
|
}
|
|
13179
13432
|
async assertFinalization(session, input) {
|
|
@@ -13267,7 +13520,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13267
13520
|
assertSourceActivationCorrelation(session, activation);
|
|
13268
13521
|
const candidate = this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId);
|
|
13269
13522
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
13270
|
-
throw new
|
|
13523
|
+
throw new MaterializationConflictError6("source-materialization", "Source activation candidate is missing or is not ready.");
|
|
13271
13524
|
}
|
|
13272
13525
|
assertProjectionExecution(this.db, {
|
|
13273
13526
|
projectId: commit.projectId,
|
|
@@ -13279,6 +13532,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13279
13532
|
invalidCorrelation2("Source activation does not match its ready candidate identity.");
|
|
13280
13533
|
}
|
|
13281
13534
|
this.assertSource(activation.expected, commit.projectId);
|
|
13535
|
+
activateSourceRoots(this.db, commit.projectId, candidate, activation);
|
|
13282
13536
|
const previous = this.getActiveSource(commit.projectId, activation.source.projectionId);
|
|
13283
13537
|
if (previous) {
|
|
13284
13538
|
assertPinnedDatasetWatermark({
|
|
@@ -13318,7 +13572,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13318
13572
|
`).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.actor === undefined ? null : canonicalJson(commit.actor), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
|
|
13319
13573
|
} catch (error) {
|
|
13320
13574
|
if (isSqliteConstraintError(error)) {
|
|
13321
|
-
throw new
|
|
13575
|
+
throw new MaterializationConflictError6("idempotency", "Ontology commit identity already exists.");
|
|
13322
13576
|
}
|
|
13323
13577
|
throw error;
|
|
13324
13578
|
}
|
|
@@ -13338,8 +13592,8 @@ class SqliteOntologyMaterializationStorage {
|
|
|
13338
13592
|
// src/ontology-storage/outbox.ts
|
|
13339
13593
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
13340
13594
|
import {
|
|
13341
|
-
MaterializationConflictError as
|
|
13342
|
-
MaterializationValidationError
|
|
13595
|
+
MaterializationConflictError as MaterializationConflictError7,
|
|
13596
|
+
MaterializationValidationError as MaterializationValidationError2
|
|
13343
13597
|
} from "@sixb/core/internal/materialization";
|
|
13344
13598
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
|
|
13345
13599
|
class SqliteOntologyOutboxStorage {
|
|
@@ -13357,7 +13611,7 @@ class SqliteOntologyOutboxStorage {
|
|
|
13357
13611
|
const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
|
|
13358
13612
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
13359
13613
|
if (leaseExpiry <= now) {
|
|
13360
|
-
throw new
|
|
13614
|
+
throw new MaterializationValidationError2("Ontology outbox lease expiry must be later than the claim time.");
|
|
13361
13615
|
}
|
|
13362
13616
|
const rows = this.db.query(`
|
|
13363
13617
|
UPDATE ontology_outbox
|
|
@@ -13375,7 +13629,7 @@ class SqliteOntologyOutboxStorage {
|
|
|
13375
13629
|
`).all(input.leaseId, input.leaseExpiresAt, input.projectId, input.now, input.now, input.limit);
|
|
13376
13630
|
return rows.map((row) => outboxRecord(row)).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.envelope.id.localeCompare(right.envelope.id)).map((record) => {
|
|
13377
13631
|
if (record.leaseId === null || record.leaseExpiresAt === null) {
|
|
13378
|
-
throw new
|
|
13632
|
+
throw new MaterializationConflictError7("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
|
|
13379
13633
|
}
|
|
13380
13634
|
return { ...record, leaseId: record.leaseId, leaseExpiresAt: record.leaseExpiresAt };
|
|
13381
13635
|
});
|
|
@@ -13474,7 +13728,7 @@ function validateLeaseIds(ids) {
|
|
|
13474
13728
|
for (const id of ids) {
|
|
13475
13729
|
assertNonblank(id, "Ontology outbox event id");
|
|
13476
13730
|
if (seen.has(id)) {
|
|
13477
|
-
throw new
|
|
13731
|
+
throw new MaterializationValidationError2(`Ontology outbox lease batch repeats event '${id}'.`);
|
|
13478
13732
|
}
|
|
13479
13733
|
seen.add(id);
|
|
13480
13734
|
}
|
|
@@ -13485,11 +13739,11 @@ function assertLeaseInput(input) {
|
|
|
13485
13739
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
13486
13740
|
}
|
|
13487
13741
|
function leaseConflict() {
|
|
13488
|
-
return new
|
|
13742
|
+
return new MaterializationConflictError7("outbox-lease", "Ontology outbox lease does not match.");
|
|
13489
13743
|
}
|
|
13490
13744
|
|
|
13491
13745
|
// src/ontology-storage/sources.ts
|
|
13492
|
-
import { MaterializationValidationError as
|
|
13746
|
+
import { MaterializationValidationError as MaterializationValidationError3 } from "@sixb/core/internal/materialization";
|
|
13493
13747
|
import {
|
|
13494
13748
|
assertSourceBeginInput as assertBeginInput,
|
|
13495
13749
|
assertSourceExecutionIdentity as assertExecutionIdentity,
|
|
@@ -13503,8 +13757,53 @@ import {
|
|
|
13503
13757
|
sourceMaterializationIdentity,
|
|
13504
13758
|
sourceStageRow,
|
|
13505
13759
|
sourceStageRows,
|
|
13506
|
-
utf8SortKey
|
|
13760
|
+
utf8SortKey as utf8SortKey2
|
|
13507
13761
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
13762
|
+
|
|
13763
|
+
// src/ontology-storage/source-cleanup.ts
|
|
13764
|
+
function cleanupSourceVersions(db, input) {
|
|
13765
|
+
let remaining = input.limit;
|
|
13766
|
+
let rowsDeleted = db.query(`DELETE FROM ontology_source_rows WHERE rowid IN (
|
|
13767
|
+
SELECT rows.rowid FROM ontology_source_roots AS roots
|
|
13768
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
|
|
13769
|
+
CROSS JOIN ontology_source_rows AS rows
|
|
13770
|
+
ON rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
13771
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key
|
|
13772
|
+
WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
|
|
13773
|
+
AND sources.status IN ('superseded', 'abandoned')
|
|
13774
|
+
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key, rows.entity_sort_key
|
|
13775
|
+
LIMIT ?
|
|
13776
|
+
)`).run(input.projectId, input.terminalBefore, remaining).changes;
|
|
13777
|
+
remaining -= rowsDeleted;
|
|
13778
|
+
if (remaining > 0) {
|
|
13779
|
+
const removed = db.query(`DELETE FROM ontology_source_roots WHERE rowid IN (
|
|
13780
|
+
SELECT roots.rowid FROM ontology_source_roots AS roots
|
|
13781
|
+
JOIN ontology_sources AS sources USING (project_id, source_id, materialization_id)
|
|
13782
|
+
WHERE roots.project_id = ? AND roots.active = 0 AND roots.retired_at < ?
|
|
13783
|
+
AND sources.status IN ('superseded', 'abandoned')
|
|
13784
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
13785
|
+
WHERE rows.project_id = roots.project_id AND rows.source_id = roots.source_id
|
|
13786
|
+
AND rows.materialization_id = roots.materialization_id AND rows.root_sort_key = roots.root_sort_key)
|
|
13787
|
+
ORDER BY roots.retired_at, roots.source_id, roots.materialization_id, roots.root_sort_key LIMIT ?
|
|
13788
|
+
)`).run(input.projectId, input.terminalBefore, remaining).changes;
|
|
13789
|
+
rowsDeleted += removed;
|
|
13790
|
+
remaining -= removed;
|
|
13791
|
+
}
|
|
13792
|
+
const materializationsDeleted = remaining === 0 ? 0 : db.query(`DELETE FROM ontology_sources WHERE rowid IN (
|
|
13793
|
+
SELECT sources.rowid FROM ontology_sources AS sources
|
|
13794
|
+
WHERE project_id = ? AND status IN ('superseded', 'abandoned') AND terminal_at < ?
|
|
13795
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_roots AS roots
|
|
13796
|
+
WHERE roots.project_id = sources.project_id AND roots.source_id = sources.source_id
|
|
13797
|
+
AND roots.materialization_id = sources.materialization_id)
|
|
13798
|
+
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows
|
|
13799
|
+
WHERE rows.project_id = sources.project_id AND rows.source_id = sources.source_id
|
|
13800
|
+
AND rows.materialization_id = sources.materialization_id)
|
|
13801
|
+
ORDER BY terminal_at, source_id, materialization_id LIMIT ?
|
|
13802
|
+
)`).run(input.projectId, input.terminalBefore, remaining).changes;
|
|
13803
|
+
return { rowsDeleted, materializationsDeleted };
|
|
13804
|
+
}
|
|
13805
|
+
|
|
13806
|
+
// src/ontology-storage/sources.ts
|
|
13508
13807
|
class SqliteOntologySourceStorage {
|
|
13509
13808
|
db;
|
|
13510
13809
|
runRootOperation;
|
|
@@ -13539,10 +13838,10 @@ class SqliteOntologySourceStorage {
|
|
|
13539
13838
|
dataset_id, dataset_version_id, dataset_version_created_at,
|
|
13540
13839
|
projection_revision, ownership_hash, ontology_revision,
|
|
13541
13840
|
root_count, assertion_count, created_at, ready_at, activated_at,
|
|
13542
|
-
terminal_at, last_commit_id, updated_at
|
|
13841
|
+
terminal_at, last_commit_id, updated_at, base_materialization_id, base_commit_id
|
|
13543
13842
|
) VALUES (?, ?, ?, ?, ?, 'replacement', 'staging', ?, ?, ?, ?, ?, ?, ?,
|
|
13544
|
-
NULL, NULL, ?, NULL, NULL, NULL, NULL, ?)
|
|
13545
|
-
`).run(input.projectId, input.source.projectionId, input.materializationId, input.execution.projectionRunId, input.projectionKind, input.execution.executionToken, input.datasetVersion.datasetId, input.datasetVersion.versionId, input.datasetVersion.createdAt, input.projectionRevision, input.ownershipHash, input.ontologyRevision, input.createdAt, input.createdAt);
|
|
13843
|
+
NULL, NULL, ?, NULL, NULL, NULL, NULL, ?, ?, ?)
|
|
13844
|
+
`).run(input.projectId, input.source.projectionId, input.materializationId, input.execution.projectionRunId, input.projectionKind, input.execution.executionToken, input.datasetVersion.datasetId, input.datasetVersion.versionId, input.datasetVersion.createdAt, input.projectionRevision, input.ownershipHash, input.ontologyRevision, input.createdAt, input.createdAt, input.base?.materializationId ?? null, input.base?.lastCommitId ?? null);
|
|
13546
13845
|
return sourceRecord(this.requireManifest(input.projectId, input.source.projectionId, input.materializationId));
|
|
13547
13846
|
});
|
|
13548
13847
|
}
|
|
@@ -13558,6 +13857,7 @@ class SqliteOntologySourceStorage {
|
|
|
13558
13857
|
const rows = sourceStageRows(manifest.projection_kind, input.rows);
|
|
13559
13858
|
const existing = this.findStageRows(input, rows);
|
|
13560
13859
|
const { pending, unchanged } = reconcileSourceStageRows(rows, existing);
|
|
13860
|
+
stageSourceRoots(this.db, manifest, input);
|
|
13561
13861
|
this.insertStageRows(input, pending);
|
|
13562
13862
|
return { inserted: pending.length, unchanged };
|
|
13563
13863
|
});
|
|
@@ -13583,7 +13883,7 @@ class SqliteOntologySourceStorage {
|
|
|
13583
13883
|
throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
|
|
13584
13884
|
}
|
|
13585
13885
|
if (input.readyAt < manifest.created_at) {
|
|
13586
|
-
throw new
|
|
13886
|
+
throw new MaterializationValidationError3("Source readyAt cannot precede source createdAt.");
|
|
13587
13887
|
}
|
|
13588
13888
|
this.assertReadyState(manifest, input.rootCount, input.assertionCount);
|
|
13589
13889
|
const result = this.db.query(`
|
|
@@ -13622,55 +13922,7 @@ class SqliteOntologySourceStorage {
|
|
|
13622
13922
|
assertNonblank(input.projectId, "Terminal source cleanup project id");
|
|
13623
13923
|
assertTimestamp(input.terminalBefore, "Terminal source cleanup cutoff", true);
|
|
13624
13924
|
assertPositiveInteger(input.limit, "Terminal source cleanup limit");
|
|
13625
|
-
|
|
13626
|
-
SELECT * FROM ontology_sources
|
|
13627
|
-
WHERE project_id = ? AND status IN ('superseded', 'abandoned')
|
|
13628
|
-
AND terminal_at < ?
|
|
13629
|
-
ORDER BY terminal_at, source_id, materialization_id
|
|
13630
|
-
LIMIT ?
|
|
13631
|
-
`).all(input.projectId, input.terminalBefore, input.limit);
|
|
13632
|
-
let remaining = input.limit;
|
|
13633
|
-
let rowsDeleted = 0;
|
|
13634
|
-
let materializationsDeleted = 0;
|
|
13635
|
-
for (const manifest of manifests) {
|
|
13636
|
-
if (remaining === 0)
|
|
13637
|
-
break;
|
|
13638
|
-
const deleted = this.db.query(`
|
|
13639
|
-
DELETE FROM ontology_source_rows
|
|
13640
|
-
WHERE rowid IN (
|
|
13641
|
-
SELECT rows.rowid
|
|
13642
|
-
FROM ontology_source_rows AS rows
|
|
13643
|
-
JOIN ontology_sources AS sources
|
|
13644
|
-
ON sources.project_id = rows.project_id
|
|
13645
|
-
AND sources.source_id = rows.source_id
|
|
13646
|
-
AND sources.materialization_id = rows.materialization_id
|
|
13647
|
-
WHERE rows.project_id = ? AND rows.source_id = ? AND rows.materialization_id = ?
|
|
13648
|
-
AND sources.status IN ('superseded', 'abandoned')
|
|
13649
|
-
AND sources.terminal_at = ? AND sources.terminal_at < ?
|
|
13650
|
-
ORDER BY rows.entity_sort_key
|
|
13651
|
-
LIMIT ?
|
|
13652
|
-
)
|
|
13653
|
-
`).run(manifest.project_id, manifest.source_id, manifest.materialization_id, manifest.terminal_at, input.terminalBefore, remaining).changes;
|
|
13654
|
-
rowsDeleted += deleted;
|
|
13655
|
-
remaining -= deleted;
|
|
13656
|
-
if (remaining === 0)
|
|
13657
|
-
break;
|
|
13658
|
-
const removed = this.db.query(`
|
|
13659
|
-
DELETE FROM ontology_sources
|
|
13660
|
-
WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
13661
|
-
AND status IN ('superseded', 'abandoned')
|
|
13662
|
-
AND terminal_at = ? AND terminal_at < ?
|
|
13663
|
-
AND NOT EXISTS (
|
|
13664
|
-
SELECT 1 FROM ontology_source_rows AS rows
|
|
13665
|
-
WHERE rows.project_id = ontology_sources.project_id
|
|
13666
|
-
AND rows.source_id = ontology_sources.source_id
|
|
13667
|
-
AND rows.materialization_id = ontology_sources.materialization_id
|
|
13668
|
-
)
|
|
13669
|
-
`).run(manifest.project_id, manifest.source_id, manifest.materialization_id, manifest.terminal_at, input.terminalBefore).changes;
|
|
13670
|
-
materializationsDeleted += removed;
|
|
13671
|
-
remaining -= removed;
|
|
13672
|
-
}
|
|
13673
|
-
return { rowsDeleted, materializationsDeleted };
|
|
13925
|
+
return cleanupSourceVersions(this.db, input);
|
|
13674
13926
|
});
|
|
13675
13927
|
}
|
|
13676
13928
|
async summarizeTerminal(input) {
|
|
@@ -13714,7 +13966,7 @@ class SqliteOntologySourceStorage {
|
|
|
13714
13966
|
}
|
|
13715
13967
|
transitionToAbandoned(manifest, abandonedAt) {
|
|
13716
13968
|
if (abandonedAt < manifest.created_at || manifest.ready_at !== null && abandonedAt < manifest.ready_at) {
|
|
13717
|
-
throw new
|
|
13969
|
+
throw new MaterializationValidationError3("Source abandonedAt cannot precede source creation or readiness.");
|
|
13718
13970
|
}
|
|
13719
13971
|
const changed = this.db.query(`
|
|
13720
13972
|
UPDATE ontology_sources
|
|
@@ -13725,19 +13977,23 @@ class SqliteOntologySourceStorage {
|
|
|
13725
13977
|
if (changed !== 1) {
|
|
13726
13978
|
throw sourceConflict(`Source materialization '${manifest.materialization_id}' cannot transition to 'abandoned'.`);
|
|
13727
13979
|
}
|
|
13980
|
+
this.db.query(`UPDATE ontology_source_roots SET retired_at = ?
|
|
13981
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ? AND active = 0`).run(abandonedAt, manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
13728
13982
|
return sourceRecord(this.requireManifest(manifest.project_id, manifest.source_id, manifest.materialization_id));
|
|
13729
13983
|
}
|
|
13730
13984
|
assertReadyState(manifest, rootCount, assertionCount) {
|
|
13731
13985
|
const counts = this.db.query(`
|
|
13732
|
-
SELECT
|
|
13986
|
+
SELECT (SELECT COUNT(*) FROM ontology_source_rows
|
|
13987
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?) AS assertions, COUNT(*) AS roots,
|
|
13733
13988
|
COUNT(DISTINCT staging_ordinal) AS ordinals,
|
|
13734
13989
|
MIN(staging_ordinal) AS min_ordinal, MAX(staging_ordinal) AS max_ordinal
|
|
13735
|
-
FROM
|
|
13990
|
+
FROM ontology_source_roots
|
|
13736
13991
|
WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
13737
|
-
`).get(manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
13992
|
+
`).get(manifest.project_id, manifest.source_id, manifest.materialization_id, manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
13738
13993
|
if (counts.assertions !== assertionCount || counts.roots !== rootCount || counts.ordinals !== rootCount || rootCount > 0 && (counts.min_ordinal !== 0 || counts.max_ordinal !== rootCount - 1)) {
|
|
13739
|
-
throw new
|
|
13994
|
+
throw new MaterializationValidationError3("Source ready counts do not match the staged roots and assertions.");
|
|
13740
13995
|
}
|
|
13996
|
+
assertSourceRootCoverage(this.db, manifest);
|
|
13741
13997
|
const invalid = this.db.query(manifest.projection_kind === "link" ? `
|
|
13742
13998
|
SELECT root_key
|
|
13743
13999
|
FROM ontology_source_rows
|
|
@@ -13763,15 +14019,15 @@ class SqliteOntologySourceStorage {
|
|
|
13763
14019
|
LIMIT 1
|
|
13764
14020
|
`).get(manifest.project_id, manifest.source_id, manifest.materialization_id);
|
|
13765
14021
|
if (invalid) {
|
|
13766
|
-
throw new
|
|
14022
|
+
throw new MaterializationValidationError3(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
13767
14023
|
}
|
|
13768
14024
|
}
|
|
13769
14025
|
findStageRows(input, rows) {
|
|
13770
14026
|
if (rows.length === 0)
|
|
13771
14027
|
return [];
|
|
13772
|
-
const rootSortKeys = [...new Set(rows.map((row) =>
|
|
14028
|
+
const rootSortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.rootKey)))];
|
|
13773
14029
|
const ordinals = [...new Set(rows.map((row) => row.row.stagingOrdinal))];
|
|
13774
|
-
const entitySortKeys = [...new Set(rows.map((row) =>
|
|
14030
|
+
const entitySortKeys = [...new Set(rows.map((row) => utf8SortKey2(row.entityKey)))];
|
|
13775
14031
|
const existing = this.db.query(`
|
|
13776
14032
|
SELECT * FROM ontology_source_rows
|
|
13777
14033
|
WHERE project_id = ? AND source_id = ? AND materialization_id = ?
|
|
@@ -13792,10 +14048,10 @@ class SqliteOntologySourceStorage {
|
|
|
13792
14048
|
return {
|
|
13793
14049
|
entityKind: row.assertion.kind,
|
|
13794
14050
|
entityKey,
|
|
13795
|
-
entitySortKey:
|
|
14051
|
+
entitySortKey: utf8SortKey2(entityKey),
|
|
13796
14052
|
rootKind: row.root.kind,
|
|
13797
14053
|
rootKey,
|
|
13798
|
-
rootSortKey:
|
|
14054
|
+
rootSortKey: utf8SortKey2(rootKey),
|
|
13799
14055
|
stagingOrdinal: row.stagingOrdinal,
|
|
13800
14056
|
root: row.root,
|
|
13801
14057
|
assertion: row.assertion,
|
|
@@ -13889,11 +14145,8 @@ class SqliteOntologyStorage {
|
|
|
13889
14145
|
// src/pipeline-run-storage.ts
|
|
13890
14146
|
import { parseSixbFailure as parseSixbFailure3, serializeSixbFailure as serializeSixbFailure3 } from "@sixb/core/internal/errors";
|
|
13891
14147
|
import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-storage-provider";
|
|
13892
|
-
import {
|
|
13893
|
-
|
|
13894
|
-
PIPELINE_RUN_FAILURE_CODES,
|
|
13895
|
-
PipelineRunError
|
|
13896
|
-
} from "@sixb/core/storage";
|
|
14148
|
+
import { canRequeuePipelineRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
|
|
14149
|
+
import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage";
|
|
13897
14150
|
|
|
13898
14151
|
// src/latest-run-query.ts
|
|
13899
14152
|
function queryLatestRunsByOwnerId(input) {
|
|
@@ -14310,7 +14563,7 @@ class SqliteProjectionRunStorage {
|
|
|
14310
14563
|
input_exhausted = ?, missing_target_object_type_id = NULL,
|
|
14311
14564
|
missing_target_object_id = NULL, missing_target_batch_ordinal = NULL,
|
|
14312
14565
|
missing_target_first_seen_at = NULL, source_rows_read = 0,
|
|
14313
|
-
source_rows_skipped = 0, error = NULL
|
|
14566
|
+
source_rows_skipped = 0, source_changes_read = NULL, error = NULL
|
|
14314
14567
|
WHERE project_id = ? AND id = ? AND status = 'failed'
|
|
14315
14568
|
AND json_extract(error, '$.code') = 'queue.enqueue_failed'
|
|
14316
14569
|
`).run(queuedAt.toISOString(), input.fixedBatchSize === undefined ? null : 0, input.fixedBatchSize === undefined ? null : 0, input.fixedBatchSize === undefined ? null : 0, input.projectId, input.id);
|
|
@@ -14405,10 +14658,11 @@ class SqliteProjectionRunStorage {
|
|
|
14405
14658
|
execution_token = NULL,
|
|
14406
14659
|
source_rows_read = ?,
|
|
14407
14660
|
source_rows_skipped = ?,
|
|
14661
|
+
source_changes_read = ?,
|
|
14408
14662
|
input_exhausted = ?,
|
|
14409
14663
|
error = ?
|
|
14410
14664
|
WHERE project_id = ? AND id = ? AND status = 'running' AND execution_token = ?
|
|
14411
|
-
`).run(input.status, plan.finishedAt.toISOString(), plan.progress.sourceRowsRead, plan.progress.sourceRowsSkipped, plan.inputExhausted ? 1 : existingRow.input_exhausted, plan.error === undefined ? null : serializeSixbFailure4(plan.error, PROJECTION_RUN_FAILURE_CODES), input.projectId, input.id, input.executionToken);
|
|
14665
|
+
`).run(input.status, plan.finishedAt.toISOString(), plan.progress.sourceRowsRead, plan.progress.sourceRowsSkipped, plan.progress.sourceChangesRead ?? null, plan.inputExhausted ? 1 : existingRow.input_exhausted, plan.error === undefined ? null : serializeSixbFailure4(plan.error, PROJECTION_RUN_FAILURE_CODES), input.projectId, input.id, input.executionToken);
|
|
14412
14666
|
if (result.changes !== 1)
|
|
14413
14667
|
throw staleProjectionRunExecution(input.id);
|
|
14414
14668
|
return rowToProjectionRunRecord(this.requireRow(input.projectId, input.id));
|
|
@@ -14574,6 +14828,7 @@ class SqliteProjectionRunStorage {
|
|
|
14574
14828
|
const args = [
|
|
14575
14829
|
progress.sourceRowsRead,
|
|
14576
14830
|
progress.sourceRowsSkipped,
|
|
14831
|
+
progress.sourceChangesRead ?? null,
|
|
14577
14832
|
existing.project_id,
|
|
14578
14833
|
existing.id,
|
|
14579
14834
|
executionToken
|
|
@@ -14582,7 +14837,8 @@ class SqliteProjectionRunStorage {
|
|
|
14582
14837
|
UPDATE projection_runs
|
|
14583
14838
|
SET
|
|
14584
14839
|
source_rows_read = ?,
|
|
14585
|
-
source_rows_skipped =
|
|
14840
|
+
source_rows_skipped = ?,
|
|
14841
|
+
source_changes_read = ?
|
|
14586
14842
|
WHERE project_id = ? AND id = ? AND status = 'running'
|
|
14587
14843
|
AND execution_token = ?
|
|
14588
14844
|
`).run(...args);
|
|
@@ -14652,7 +14908,8 @@ function restoreProjectionRunRow(row) {
|
|
|
14652
14908
|
missingTargetFirstSeenAt: row.missing_target_first_seen_at ? new Date(row.missing_target_first_seen_at) : undefined,
|
|
14653
14909
|
progress: {
|
|
14654
14910
|
sourceRowsRead: databaseSafeInteger(row.source_rows_read, "sourceRowsRead"),
|
|
14655
|
-
sourceRowsSkipped: databaseSafeInteger(row.source_rows_skipped, "sourceRowsSkipped")
|
|
14911
|
+
sourceRowsSkipped: databaseSafeInteger(row.source_rows_skipped, "sourceRowsSkipped"),
|
|
14912
|
+
...row.source_changes_read == null ? {} : { sourceChangesRead: databaseSafeInteger(row.source_changes_read, "sourceChangesRead") }
|
|
14656
14913
|
},
|
|
14657
14914
|
error: row.error ?? undefined
|
|
14658
14915
|
};
|
|
@@ -15175,12 +15432,9 @@ function invalidRecord2(message) {
|
|
|
15175
15432
|
|
|
15176
15433
|
// src/sync-run-storage.ts
|
|
15177
15434
|
import { parseSixbFailure as parseSixbFailure4, serializeSixbFailure as serializeSixbFailure5 } from "@sixb/core/internal/errors";
|
|
15435
|
+
import { canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage";
|
|
15178
15436
|
import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider";
|
|
15179
|
-
import {
|
|
15180
|
-
canRequeueSyncRunAfterEnqueueFailure,
|
|
15181
|
-
SYNC_RUN_FAILURE_CODES,
|
|
15182
|
-
SyncRunError
|
|
15183
|
-
} from "@sixb/core/storage";
|
|
15437
|
+
import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage";
|
|
15184
15438
|
class SqliteSyncRunStorage {
|
|
15185
15439
|
connection;
|
|
15186
15440
|
db;
|
|
@@ -15896,7 +16150,6 @@ function assertNonNegativeInteger(value, fieldName) {
|
|
|
15896
16150
|
}
|
|
15897
16151
|
|
|
15898
16152
|
// src/workflow-run-storage.ts
|
|
15899
|
-
import { normalizeRequesterGroupIds as normalizeRequesterGroupIds2 } from "@sixb/core/internal/auth";
|
|
15900
16153
|
import { parseSixbFailure as parseSixbFailure6, serializeSixbFailure as serializeSixbFailure7 } from "@sixb/core/internal/errors";
|
|
15901
16154
|
import {
|
|
15902
16155
|
assertWorkflowAgentNodeRunExecution,
|
|
@@ -15943,10 +16196,9 @@ class SqliteWorkflowRunStorage {
|
|
|
15943
16196
|
input,
|
|
15944
16197
|
queued_at,
|
|
15945
16198
|
started_at,
|
|
15946
|
-
requester_group_ids,
|
|
15947
16199
|
attempt
|
|
15948
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?,
|
|
15949
|
-
`).run(input.projectId, input.id, input.executionId, input.workflowId, "queued", serializeRecord2(input.input), queuedAt.toISOString(), queuedAt.toISOString(),
|
|
16200
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
16201
|
+
`).run(input.projectId, input.id, input.executionId, input.workflowId, "queued", serializeRecord2(input.input), queuedAt.toISOString(), queuedAt.toISOString(), 0);
|
|
15950
16202
|
} catch (error) {
|
|
15951
16203
|
if (isUniqueConstraintError2(error)) {
|
|
15952
16204
|
throw new WorkflowRunError(`[SixbSqlite] Workflow run '${input.id}' or execution '${input.executionId}' is already linked for project '${input.projectId}'.`);
|
|
@@ -16457,7 +16709,6 @@ function rowToWorkflowRunRecord(row) {
|
|
|
16457
16709
|
startedAt: new Date(row.started_at),
|
|
16458
16710
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
16459
16711
|
error: row.error === null ? undefined : parseSixbFailure6(row.error, WORKFLOW_RUN_FAILURE_CODES),
|
|
16460
|
-
requesterGroupIds: JSON.parse(row.requester_group_ids),
|
|
16461
16712
|
attempt: row.attempt,
|
|
16462
16713
|
...row.execution_token && row.execution_queue_lease_expires_at ? {
|
|
16463
16714
|
execution: {
|
|
@@ -16729,8 +16980,8 @@ function createSqliteStores(connection, options) {
|
|
|
16729
16980
|
};
|
|
16730
16981
|
}
|
|
16731
16982
|
export {
|
|
16732
|
-
|
|
16733
|
-
|
|
16983
|
+
SqliteStorage,
|
|
16984
|
+
migrateSqliteStorage
|
|
16734
16985
|
};
|
|
16735
16986
|
|
|
16736
|
-
//# debugId=
|
|
16987
|
+
//# debugId=B98ED3E1F74F05CB64756E2164756E21
|