@sixb/sqlite 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-limit-storage.d.ts.map +1 -1
- package/dist/ai-usage-storage.d.ts.map +1 -1
- package/dist/execution-storage.d.ts +28 -0
- package/dist/execution-storage.d.ts.map +1 -1
- package/dist/file-download-grant-storage.d.ts +11 -0
- package/dist/file-download-grant-storage.d.ts.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1362 -579
- package/dist/index.js.map +27 -25
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/index.d.ts +2 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +17 -37
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +3 -26
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +3 -6
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/replacement-plans.d.ts +88 -0
- package/dist/ontology-storage/replacement-plans.d.ts.map +1 -0
- package/dist/ontology-storage/shared.d.ts +8 -5
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
- package/dist/ontology-storage/source-roots.d.ts +15 -2
- package/dist/ontology-storage/source-roots.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +1 -2
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/projection-run-storage.d.ts.map +1 -1
- package/dist/rules-storage.d.ts.map +1 -1
- package/dist/testing.d.ts +4 -1
- package/dist/testing.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/ai-cost-storage.ts +10 -0
- package/src/ai-limit-storage.ts +13 -2
- package/src/ai-usage-storage.ts +9 -1
- package/src/execution-storage.ts +7 -2
- package/src/file-download-grant-storage.ts +76 -0
- package/src/index.ts +10 -3
- package/src/migrations/053-replacement-plans.sql +73 -0
- package/src/migrations/054-slim-ontology-outbox.sql +21 -0
- package/src/migrations/055-commit-touches.sql +31 -0
- package/src/migrations/056-audio-transcription.sql +10 -0
- package/src/migrations/057-reranking-model-kind.sql +16 -0
- package/src/migrations/058-file-download-grants.sql +21 -0
- package/src/migrations/059-drained-source-versions.sql +11 -0
- package/src/migrations.ts +115 -0
- package/src/objects/query-compiler.ts +5 -0
- package/src/objects/reader.ts +1 -0
- package/src/objects/storage.ts +5 -11
- package/src/ontology-storage/index.ts +6 -1
- package/src/ontology-storage/materialization-session.ts +120 -211
- package/src/ontology-storage/materialization-state.ts +5 -261
- package/src/ontology-storage/materialization-writer.ts +11 -13
- package/src/ontology-storage/materializations.ts +133 -210
- package/src/ontology-storage/outbox.ts +50 -16
- package/src/ontology-storage/replacement-plans.ts +1064 -0
- package/src/ontology-storage/shared.ts +17 -12
- package/src/ontology-storage/source-cleanup.ts +24 -9
- package/src/ontology-storage/source-roots.ts +38 -14
- package/src/ontology-storage/sources.ts +0 -26
- package/src/projection-run-storage.ts +7 -1
- package/src/rules-storage.ts +9 -14
- package/src/testing.ts +6 -4
package/dist/index.js
CHANGED
|
@@ -4670,6 +4670,172 @@ DROP TABLE legacy_ontology_sources;
|
|
|
4670
4670
|
-- describe empty tables forever and steer the planner to partial indexes over primary keys.
|
|
4671
4671
|
`;
|
|
4672
4672
|
|
|
4673
|
+
// src/migrations/053-replacement-plans.sql
|
|
4674
|
+
var _053_replacement_plans_default = `-- The plan of a ready replacement candidate, built outside its commit transaction. Its rows are a
|
|
4675
|
+
-- cache the commit verifies again, keyed by the candidate's source version, which outlives it:
|
|
4676
|
+
-- maintenance deletes a plan once its candidate is no longer ready, and only then the version.
|
|
4677
|
+
CREATE TABLE ontology_replacement_plans (
|
|
4678
|
+
version_id INTEGER PRIMARY KEY REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
|
|
4679
|
+
project_id TEXT NOT NULL,
|
|
4680
|
+
-- The commit the plan's work carries, at the time fixed when the plan opened.
|
|
4681
|
+
commit_id TEXT NOT NULL,
|
|
4682
|
+
committed_at TEXT NOT NULL,
|
|
4683
|
+
-- The active source the plan replaces, NULL when none was: a plan resumed once that moved
|
|
4684
|
+
-- starts over.
|
|
4685
|
+
replaced_materialization_id TEXT,
|
|
4686
|
+
replaced_last_commit_id TEXT,
|
|
4687
|
+
-- The last commit row every planned identity is known fresh against.
|
|
4688
|
+
watermark INTEGER NOT NULL
|
|
4689
|
+
);
|
|
4690
|
+
|
|
4691
|
+
-- One row per entity the plan decides. \`read_revision\` is the revision of the inputs its state was
|
|
4692
|
+
-- last read at; \`planned_revision\` the one its work was planned from, NULL while still to plan.
|
|
4693
|
+
-- \`classified\` and \`change\` summarize that work for the commit's change counts.
|
|
4694
|
+
CREATE TABLE ontology_replacement_plan_identities (
|
|
4695
|
+
version_id INTEGER NOT NULL,
|
|
4696
|
+
entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
|
|
4697
|
+
identity_key TEXT NOT NULL,
|
|
4698
|
+
sort_key TEXT NOT NULL,
|
|
4699
|
+
diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
|
|
4700
|
+
read_revision TEXT,
|
|
4701
|
+
planned_revision TEXT,
|
|
4702
|
+
classified INTEGER NOT NULL DEFAULT 0 CHECK (classified IN (0, 1)),
|
|
4703
|
+
change TEXT CHECK (change IN ('created', 'updated', 'deleted')),
|
|
4704
|
+
PRIMARY KEY (version_id, entity_kind, identity_key)
|
|
4705
|
+
);
|
|
4706
|
+
-- The identities still to plan, in planning order: a round that plans a few again finds them
|
|
4707
|
+
-- without walking the rest.
|
|
4708
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_unplanned
|
|
4709
|
+
ON ontology_replacement_plan_identities (version_id, entity_kind, sort_key)
|
|
4710
|
+
WHERE planned_revision IS NULL;
|
|
4711
|
+
|
|
4712
|
+
-- Planned work, with the same columns as a session's work table plus the identity it belongs to.
|
|
4713
|
+
CREATE TABLE ontology_replacement_plan_work (
|
|
4714
|
+
work_id INTEGER NOT NULL,
|
|
4715
|
+
entity_kind TEXT NOT NULL,
|
|
4716
|
+
identity_key TEXT NOT NULL,
|
|
4717
|
+
record_key TEXT NOT NULL,
|
|
4718
|
+
unique_key TEXT NOT NULL,
|
|
4719
|
+
kind TEXT NOT NULL,
|
|
4720
|
+
lane TEXT NOT NULL,
|
|
4721
|
+
major_order INTEGER NOT NULL,
|
|
4722
|
+
minor_order INTEGER NOT NULL,
|
|
4723
|
+
sort_one TEXT NOT NULL,
|
|
4724
|
+
sort_two TEXT NOT NULL,
|
|
4725
|
+
classification_entity_kind TEXT,
|
|
4726
|
+
classification_identity_key TEXT,
|
|
4727
|
+
cardinality_view TEXT,
|
|
4728
|
+
cardinality_occupied INTEGER,
|
|
4729
|
+
cardinality_source_type_id TEXT,
|
|
4730
|
+
cardinality_source_primary_id TEXT,
|
|
4731
|
+
cardinality_link_id TEXT,
|
|
4732
|
+
cardinality_target_type_id TEXT,
|
|
4733
|
+
cardinality_target_primary_id TEXT,
|
|
4734
|
+
payload TEXT NOT NULL CHECK (json_valid(payload)),
|
|
4735
|
+
PRIMARY KEY (work_id, record_key),
|
|
4736
|
+
UNIQUE (work_id, unique_key)
|
|
4737
|
+
);
|
|
4738
|
+
CREATE INDEX idx_ontology_replacement_plan_work_identity
|
|
4739
|
+
ON ontology_replacement_plan_work (work_id, entity_kind, identity_key);
|
|
4740
|
+
-- The few objects whose existence the plan flips, which link expansion starts from.
|
|
4741
|
+
CREATE INDEX idx_ontology_replacement_plan_work_incident
|
|
4742
|
+
ON ontology_replacement_plan_work (work_id) WHERE kind = 'incident-object';
|
|
4743
|
+
CREATE INDEX idx_ontology_replacement_plan_work_lane
|
|
4744
|
+
ON ontology_replacement_plan_work (
|
|
4745
|
+
work_id, lane, major_order, minor_order, sort_one, sort_two, record_key
|
|
4746
|
+
);
|
|
4747
|
+
`;
|
|
4748
|
+
|
|
4749
|
+
// src/migrations/054-slim-ontology-outbox.sql
|
|
4750
|
+
var _054_slim_ontology_outbox_default = `-- An outbox event stores what changed and nothing it can be rebuilt from: delivery rebuilds the
|
|
4751
|
+
-- event from its commit and the execution that commit ran under. The commit also stops copying
|
|
4752
|
+
-- \`requested_by\` and \`executor\` from that execution (047): the execution is canonical, immutable,
|
|
4753
|
+
-- and already referenced by the commit. The migration step checks beforehand that every stored
|
|
4754
|
+
-- event rebuilds exactly as it was stored.
|
|
4755
|
+
--
|
|
4756
|
+
-- SQLite cannot add a NOT NULL column without a default; every insert supplies the event, and the
|
|
4757
|
+
-- UPDATE below replaces the placeholder on every row.
|
|
4758
|
+
ALTER TABLE ontology_outbox ADD COLUMN event TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(event));
|
|
4759
|
+
UPDATE ontology_outbox SET event = json_object(
|
|
4760
|
+
'payload', CASE
|
|
4761
|
+
WHEN envelope ->> '$.type' IN ('object.created', 'link.created')
|
|
4762
|
+
THEN json_remove(envelope -> '$.payload', '$.propertyChanges')
|
|
4763
|
+
ELSE envelope -> '$.payload'
|
|
4764
|
+
END,
|
|
4765
|
+
'type', envelope ->> '$.type'
|
|
4766
|
+
);
|
|
4767
|
+
ALTER TABLE ontology_outbox DROP COLUMN envelope;
|
|
4768
|
+
|
|
4769
|
+
ALTER TABLE ontology_commits DROP COLUMN requested_by;
|
|
4770
|
+
ALTER TABLE ontology_commits DROP COLUMN executor;
|
|
4771
|
+
`;
|
|
4772
|
+
|
|
4773
|
+
// src/migrations/055-commit-touches.sql
|
|
4774
|
+
var _055_commit_touches_default = `-- Every commit records the entities whose plan inputs it changed, so a replacement plan's refresh
|
|
4775
|
+
-- rechecks only what the commits since it was last fresh touched, instead of the whole plan.
|
|
4776
|
+
CREATE TABLE ontology_commit_touches (
|
|
4777
|
+
project_id TEXT NOT NULL,
|
|
4778
|
+
commit_id TEXT NOT NULL,
|
|
4779
|
+
entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link', 'scope')),
|
|
4780
|
+
-- \`objectRefKey\`, \`linkRefKey\` or \`linkScopeKey\`.
|
|
4781
|
+
identity_key TEXT NOT NULL
|
|
4782
|
+
);
|
|
4783
|
+
CREATE INDEX idx_ontology_commit_touches_commit ON ontology_commit_touches (project_id, commit_id);
|
|
4784
|
+
|
|
4785
|
+
-- Existing plans predate the touches of the commits since their watermark: they plan again.
|
|
4786
|
+
DELETE FROM ontology_replacement_plan_work;
|
|
4787
|
+
DELETE FROM ontology_replacement_plan_identities;
|
|
4788
|
+
DELETE FROM ontology_replacement_plans;
|
|
4789
|
+
|
|
4790
|
+
-- Link expansion starts only from what changed since it last ran: \`expansions\` counts the runs,
|
|
4791
|
+
-- and an identity's \`expand_at\` is the run that has yet to start from it (an object once planned,
|
|
4792
|
+
-- a link once it needs a diff). \`source_key\` and \`target_key\` find a link from either endpoint.
|
|
4793
|
+
ALTER TABLE ontology_replacement_plans ADD COLUMN expansions INTEGER NOT NULL DEFAULT 0;
|
|
4794
|
+
ALTER TABLE ontology_replacement_plan_identities ADD COLUMN expand_at INTEGER;
|
|
4795
|
+
ALTER TABLE ontology_replacement_plan_identities ADD COLUMN source_key TEXT;
|
|
4796
|
+
ALTER TABLE ontology_replacement_plan_identities ADD COLUMN target_key TEXT;
|
|
4797
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_expand
|
|
4798
|
+
ON ontology_replacement_plan_identities (version_id, entity_kind, expand_at);
|
|
4799
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_source
|
|
4800
|
+
ON ontology_replacement_plan_identities (version_id, source_key)
|
|
4801
|
+
WHERE source_key IS NOT NULL;
|
|
4802
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_target
|
|
4803
|
+
ON ontology_replacement_plan_identities (version_id, target_key)
|
|
4804
|
+
WHERE target_key IS NOT NULL;
|
|
4805
|
+
`;
|
|
4806
|
+
|
|
4807
|
+
// src/migrations/056-audio-transcription.sql
|
|
4808
|
+
var _056_audio_transcription_default = `ALTER TABLE ai_model_call_usage
|
|
4809
|
+
ADD COLUMN audio_duration_ms INTEGER
|
|
4810
|
+
CHECK (audio_duration_ms IS NULL OR audio_duration_ms >= 0);
|
|
4811
|
+
|
|
4812
|
+
ALTER TABLE ai_model_call_usage
|
|
4813
|
+
ADD COLUMN model_kind TEXT
|
|
4814
|
+
CHECK (
|
|
4815
|
+
model_kind IS NULL
|
|
4816
|
+
OR model_kind IN ('language', 'image', 'video', 'embedding', 'decision', 'transcription')
|
|
4817
|
+
);
|
|
4818
|
+
`;
|
|
4819
|
+
|
|
4820
|
+
// src/migrations/057-reranking-model-kind.sql
|
|
4821
|
+
var _057_reranking_model_kind_default = `-- SQLite cannot alter a CHECK constraint. Swapping the column avoids rebuilding a table that
|
|
4822
|
+
-- usage groups, costs and limit reservations reference.
|
|
4823
|
+
ALTER TABLE ai_model_call_usage
|
|
4824
|
+
ADD COLUMN model_kind_next TEXT
|
|
4825
|
+
CHECK (
|
|
4826
|
+
model_kind_next IS NULL
|
|
4827
|
+
OR model_kind_next IN (
|
|
4828
|
+
'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'reranking'
|
|
4829
|
+
)
|
|
4830
|
+
);
|
|
4831
|
+
|
|
4832
|
+
UPDATE ai_model_call_usage SET model_kind_next = model_kind;
|
|
4833
|
+
|
|
4834
|
+
ALTER TABLE ai_model_call_usage DROP COLUMN model_kind;
|
|
4835
|
+
|
|
4836
|
+
ALTER TABLE ai_model_call_usage RENAME COLUMN model_kind_next TO model_kind;
|
|
4837
|
+
`;
|
|
4838
|
+
|
|
4673
4839
|
// src/migrations/037-execution-requester-groups.sql
|
|
4674
4840
|
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids TEXT NOT NULL DEFAULT '[]';
|
|
4675
4841
|
|
|
@@ -4790,6 +4956,44 @@ CREATE INDEX idx_file_upload_sessions_abandoned
|
|
|
4790
4956
|
WHERE status = 'pending' AND provider_upload IS NOT NULL;
|
|
4791
4957
|
`;
|
|
4792
4958
|
|
|
4959
|
+
// src/migrations/058-file-download-grants.sql
|
|
4960
|
+
var _058_file_download_grants_default = `-- URLs issued by blobs.createDownloadUrl(). Only the SHA-256 of each URL's token is stored.
|
|
4961
|
+
-- Timestamps are ISO-8601 UTC text and compare lexicographically.
|
|
4962
|
+
CREATE TABLE file_download_grants (
|
|
4963
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
4964
|
+
id TEXT NOT NULL CHECK (length(trim(id)) > 0),
|
|
4965
|
+
token_hash TEXT NOT NULL CHECK (
|
|
4966
|
+
length(token_hash) = 64
|
|
4967
|
+
AND token_hash = lower(token_hash)
|
|
4968
|
+
AND token_hash NOT GLOB '*[^0-9a-f]*'
|
|
4969
|
+
),
|
|
4970
|
+
file TEXT NOT NULL CHECK (json_type(file) = 'object'),
|
|
4971
|
+
execution_id TEXT NOT NULL CHECK (length(trim(execution_id)) > 0),
|
|
4972
|
+
created_at TEXT NOT NULL,
|
|
4973
|
+
expires_at TEXT NOT NULL,
|
|
4974
|
+
revoked_at TEXT,
|
|
4975
|
+
PRIMARY KEY (project_id, id),
|
|
4976
|
+
UNIQUE (project_id, token_hash),
|
|
4977
|
+
CHECK (expires_at > created_at)
|
|
4978
|
+
);
|
|
4979
|
+
|
|
4980
|
+
CREATE INDEX idx_file_download_grants_expiry ON file_download_grants (project_id, expires_at);
|
|
4981
|
+
`;
|
|
4982
|
+
|
|
4983
|
+
// src/migrations/059-drained-source-versions.sql
|
|
4984
|
+
var _059_drained_source_versions_default = `-- A superseded version's \`root_count\` counts the roots it still holds: cleanup lowers it as it
|
|
4985
|
+
-- deletes them. A version that reaches zero is found through this index, instead of every
|
|
4986
|
+
-- superseded version being walked and probed for roots on every maintenance pass.
|
|
4987
|
+
UPDATE ontology_sources
|
|
4988
|
+
SET root_count = (SELECT count(*) FROM ontology_source_roots AS roots
|
|
4989
|
+
WHERE roots.version_id = ontology_sources.version_id)
|
|
4990
|
+
WHERE status = 'superseded' AND root_count <> (SELECT count(*) FROM ontology_source_roots AS roots
|
|
4991
|
+
WHERE roots.version_id = ontology_sources.version_id);
|
|
4992
|
+
CREATE INDEX idx_ontology_sources_drained
|
|
4993
|
+
ON ontology_sources (project_id, terminal_at, source_id, materialization_id)
|
|
4994
|
+
WHERE status = 'superseded' AND root_count = 0;
|
|
4995
|
+
`;
|
|
4996
|
+
|
|
4793
4997
|
// src/migrations.ts
|
|
4794
4998
|
var MIGRATIONS_TABLE_SQL = `
|
|
4795
4999
|
CREATE TABLE IF NOT EXISTS sixb_migrations (
|
|
@@ -4883,7 +5087,20 @@ var sqliteStorageMigrations = defineMigrations({
|
|
|
4883
5087
|
if (after.versions !== before.versions || after.roots !== before.roots || after.rows !== before.rows) {
|
|
4884
5088
|
throw new Error("[SixbSqliteStorage] Source versions, roots or rows were lost while compacting source storage.");
|
|
4885
5089
|
}
|
|
4886
|
-
}, { checksum: checksum(_052_compact_source_storage_default) })
|
|
5090
|
+
}, { checksum: checksum(_052_compact_source_storage_default) }),
|
|
5091
|
+
sqliteSql("053-replacement-plans", _053_replacement_plans_default),
|
|
5092
|
+
sqliteStep("054-slim-ontology-outbox", (db) => {
|
|
5093
|
+
const mismatched = unrebuildableOutboxEvents(db);
|
|
5094
|
+
if (mismatched > 0) {
|
|
5095
|
+
throw new Error(`[SixbSqliteStorage] ${mismatched} outbox events would not rebuild from their commit.`);
|
|
5096
|
+
}
|
|
5097
|
+
db.run(_054_slim_ontology_outbox_default);
|
|
5098
|
+
}, { checksum: checksum(_054_slim_ontology_outbox_default) }),
|
|
5099
|
+
sqliteSql("055-commit-touches", _055_commit_touches_default),
|
|
5100
|
+
sqliteSql("056-audio-transcription", _056_audio_transcription_default),
|
|
5101
|
+
sqliteSql("057-reranking-model-kind", _057_reranking_model_kind_default),
|
|
5102
|
+
sqliteSql("058-file-download-grants", _058_file_download_grants_default),
|
|
5103
|
+
sqliteSql("059-drained-source-versions", _059_drained_source_versions_default)
|
|
4887
5104
|
]
|
|
4888
5105
|
});
|
|
4889
5106
|
function sqliteStoragePath(basePath) {
|
|
@@ -5003,6 +5220,82 @@ function sourceStorageCounts(db) {
|
|
|
5003
5220
|
(SELECT count(*) FROM ontology_source_roots) AS roots,
|
|
5004
5221
|
(SELECT count(*) FROM ontology_source_rows) AS rows`).get();
|
|
5005
5222
|
}
|
|
5223
|
+
function unrebuildableOutboxEvents(db) {
|
|
5224
|
+
const row = db.query(`SELECT COUNT(*) AS mismatched
|
|
5225
|
+
FROM ontology_outbox AS outbox
|
|
5226
|
+
LEFT JOIN ontology_commits AS commits
|
|
5227
|
+
ON commits.project_id = outbox.project_id AND commits.id = outbox.commit_id
|
|
5228
|
+
LEFT JOIN executions
|
|
5229
|
+
ON executions.project_id = commits.project_id AND executions.id = commits.execution_id
|
|
5230
|
+
WHERE executions.id IS NULL
|
|
5231
|
+
OR outbox.envelope ->> '$.id' IS NOT outbox.id
|
|
5232
|
+
OR outbox.envelope ->> '$.commitOrdinal' IS NOT outbox.commit_ordinal
|
|
5233
|
+
OR outbox.envelope ->> '$.commitId' IS NOT commits.id
|
|
5234
|
+
OR outbox.envelope ->> '$.projectId' IS NOT commits.project_id
|
|
5235
|
+
OR outbox.envelope ->> '$.schemaVersion' IS NOT 1
|
|
5236
|
+
OR outbox.envelope ->> '$.occurredAt' IS NOT commits.committed_at
|
|
5237
|
+
OR outbox.envelope -> '$.origin' IS NOT json(commits.origin)
|
|
5238
|
+
OR outbox.envelope ->> '$.correlationId' IS NOT executions.correlation_id
|
|
5239
|
+
OR outbox.envelope -> '$.requestedBy' IS NOT CASE
|
|
5240
|
+
WHEN executions.requested_by_user_id IS NOT NULL
|
|
5241
|
+
THEN json_object('id', executions.requested_by_user_id, 'type', 'user')
|
|
5242
|
+
WHEN executions.requested_by_service_account_id IS NOT NULL
|
|
5243
|
+
THEN json_object('id', executions.requested_by_service_account_id, 'type', 'serviceAccount')
|
|
5244
|
+
END
|
|
5245
|
+
OR outbox.envelope -> '$.executor' IS NOT CASE executions.executor_kind
|
|
5246
|
+
WHEN 'request' THEN json_object('requestId', executions.executor_id, 'type', 'request')
|
|
5247
|
+
WHEN 'agent' THEN json_object('runId', executions.executor_id, 'type', 'agent')
|
|
5248
|
+
WHEN 'kernel' THEN json_object(
|
|
5249
|
+
'operation',
|
|
5250
|
+
CASE executions.authority_kernel_operation
|
|
5251
|
+
WHEN 'ontology.recover'
|
|
5252
|
+
THEN json_object('recoveryId', executions.executor_id, 'type', 'ontology.recover')
|
|
5253
|
+
ELSE json_object('indexingId', executions.executor_id, 'type', 'ontology.indexVectors')
|
|
5254
|
+
END,
|
|
5255
|
+
'type', 'kernel'
|
|
5256
|
+
)
|
|
5257
|
+
ELSE json_object(
|
|
5258
|
+
'id', executions.authority_primitive_id,
|
|
5259
|
+
'kind', executions.executor_kind,
|
|
5260
|
+
'runId', executions.executor_id,
|
|
5261
|
+
'type', 'primitive'
|
|
5262
|
+
)
|
|
5263
|
+
END
|
|
5264
|
+
OR outbox.envelope ->> '$.topic' IS NOT CASE substr(outbox.envelope ->> '$.type', 1, 4)
|
|
5265
|
+
WHEN 'obje' THEN 'objects'
|
|
5266
|
+
WHEN 'link' THEN 'links'
|
|
5267
|
+
WHEN 'tele' THEN 'telemetry'
|
|
5268
|
+
END
|
|
5269
|
+
OR outbox.envelope ->> '$.partitionKey' IS NOT CASE outbox.envelope ->> '$.topic'
|
|
5270
|
+
WHEN 'objects' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
|
|
5271
|
+
(outbox.envelope ->> '$.payload.primaryId')
|
|
5272
|
+
WHEN 'links' THEN (outbox.envelope ->> '$.payload.sourceTypeId') || ':' ||
|
|
5273
|
+
(outbox.envelope ->> '$.payload.sourceId') || ':' ||
|
|
5274
|
+
(outbox.envelope ->> '$.payload.linkId')
|
|
5275
|
+
WHEN 'telemetry' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
|
|
5276
|
+
(outbox.envelope ->> '$.payload.objectId') || ':' ||
|
|
5277
|
+
(outbox.envelope ->> '$.payload.propertyId')
|
|
5278
|
+
END
|
|
5279
|
+
OR (
|
|
5280
|
+
outbox.envelope ->> '$.type' IN ('object.created', 'link.created')
|
|
5281
|
+
AND (
|
|
5282
|
+
(SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.propertyChanges'))
|
|
5283
|
+
IS NOT (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.properties'))
|
|
5284
|
+
OR EXISTS (
|
|
5285
|
+
SELECT 1
|
|
5286
|
+
FROM json_each(outbox.envelope, '$.payload.propertyChanges') AS change
|
|
5287
|
+
LEFT JOIN json_each(outbox.envelope, '$.payload.properties') AS property
|
|
5288
|
+
ON property.key = change.key
|
|
5289
|
+
WHERE property.key IS NULL
|
|
5290
|
+
OR change.value ->> '$.operation' IS NOT 'created'
|
|
5291
|
+
OR json_type(change.value, '$.after') IS NOT property.type
|
|
5292
|
+
OR change.value ->> '$.after' IS NOT property.value
|
|
5293
|
+
OR (SELECT COUNT(*) FROM json_each(change.value)) IS NOT 2
|
|
5294
|
+
)
|
|
5295
|
+
)
|
|
5296
|
+
)`).get();
|
|
5297
|
+
return row.mismatched;
|
|
5298
|
+
}
|
|
5006
5299
|
function checksum(value) {
|
|
5007
5300
|
return createHash("sha256").update(value).digest("hex");
|
|
5008
5301
|
}
|
|
@@ -6592,6 +6885,7 @@ class SqliteAiUsageStorage {
|
|
|
6592
6885
|
provider_id,
|
|
6593
6886
|
provider_ids,
|
|
6594
6887
|
requested_model_id,
|
|
6888
|
+
model_kind,
|
|
6595
6889
|
requested_reasoning,
|
|
6596
6890
|
response_model_id,
|
|
6597
6891
|
response_id,
|
|
@@ -6603,14 +6897,15 @@ class SqliteAiUsageStorage {
|
|
|
6603
6897
|
cache_write_input_tokens,
|
|
6604
6898
|
text_output_tokens,
|
|
6605
6899
|
reasoning_output_tokens,
|
|
6900
|
+
audio_duration_ms,
|
|
6606
6901
|
reporting_status,
|
|
6607
6902
|
raw_usage,
|
|
6608
6903
|
occurred_at,
|
|
6609
6904
|
recorded_at
|
|
6610
6905
|
) VALUES (
|
|
6611
|
-
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
6906
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
6612
6907
|
)
|
|
6613
|
-
`).run(record.projectId, record.id, record.executionId, record.attempt, record.callId, record.providerId, record.providerIds === undefined ? null : JSON.stringify(record.providerIds), record.requestedModelId, record.requestedReasoning === undefined ? null : JSON.stringify(record.requestedReasoning), record.responseModelId ?? null, record.responseId, record.usage.inputTokens ?? null, record.usage.outputTokens ?? null, record.usage.totalTokens ?? null, record.usage.uncachedInputTokens ?? null, record.usage.cacheReadInputTokens ?? null, record.usage.cacheWriteInputTokens ?? null, record.usage.textOutputTokens ?? null, record.usage.reasoningOutputTokens ?? null, record.usage.reportingStatus, record.rawUsage === undefined ? null : JSON.stringify(record.rawUsage), record.occurredAt.toISOString(), record.recordedAt.toISOString());
|
|
6908
|
+
`).run(record.projectId, record.id, record.executionId, record.attempt, record.callId, record.providerId, record.providerIds === undefined ? null : JSON.stringify(record.providerIds), record.requestedModelId, record.modelKind ?? null, record.requestedReasoning === undefined ? null : JSON.stringify(record.requestedReasoning), record.responseModelId ?? null, record.responseId, record.usage.inputTokens ?? null, record.usage.outputTokens ?? null, record.usage.totalTokens ?? null, record.usage.uncachedInputTokens ?? null, record.usage.cacheReadInputTokens ?? null, record.usage.cacheWriteInputTokens ?? null, record.usage.textOutputTokens ?? null, record.usage.reasoningOutputTokens ?? null, record.usage.audioDurationMs ?? null, record.usage.reportingStatus, record.rawUsage === undefined ? null : JSON.stringify(record.rawUsage), record.occurredAt.toISOString(), record.recordedAt.toISOString());
|
|
6614
6909
|
}
|
|
6615
6910
|
findByIdempotencyKey(record) {
|
|
6616
6911
|
const row = this.connection.db.query(`
|
|
@@ -6645,6 +6940,7 @@ class SqliteAiUsageStorage {
|
|
|
6645
6940
|
providerId: row.provider_id,
|
|
6646
6941
|
...row.provider_ids === null ? {} : { providerIds: normalizeModelProviderIds(JSON.parse(row.provider_ids)) },
|
|
6647
6942
|
requestedModelId: row.requested_model_id,
|
|
6943
|
+
...row.model_kind === null ? {} : { modelKind: row.model_kind },
|
|
6648
6944
|
...row.requested_reasoning === null ? {} : { requestedReasoning: requestedReasoningFromRow(row.requested_reasoning) },
|
|
6649
6945
|
...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
|
|
6650
6946
|
responseId: row.response_id,
|
|
@@ -6701,7 +6997,8 @@ function usageFromRow(row) {
|
|
|
6701
6997
|
...row.cache_read_input_tokens === null ? {} : { cacheReadInputTokens: row.cache_read_input_tokens },
|
|
6702
6998
|
...row.cache_write_input_tokens === null ? {} : { cacheWriteInputTokens: row.cache_write_input_tokens },
|
|
6703
6999
|
...row.text_output_tokens === null ? {} : { textOutputTokens: row.text_output_tokens },
|
|
6704
|
-
...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens }
|
|
7000
|
+
...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens },
|
|
7001
|
+
...row.audio_duration_ms === null ? {} : { audioDurationMs: row.audio_duration_ms }
|
|
6705
7002
|
};
|
|
6706
7003
|
}
|
|
6707
7004
|
function requestedReasoningFromRow(value) {
|
|
@@ -6882,6 +7179,7 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
6882
7179
|
usage.cache_write_input_tokens,
|
|
6883
7180
|
usage.text_output_tokens,
|
|
6884
7181
|
usage.reasoning_output_tokens,
|
|
7182
|
+
usage.audio_duration_ms,
|
|
6885
7183
|
usage.reporting_status,
|
|
6886
7184
|
CASE
|
|
6887
7185
|
WHEN direct_agent.kind = 'conversation' THEN 'agent'
|
|
@@ -6966,6 +7264,8 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
6966
7264
|
CAST(SUM(text_output_tokens) AS TEXT) AS text_output_tokens_sum,
|
|
6967
7265
|
CAST(COUNT(reasoning_output_tokens) AS TEXT) AS reasoning_output_tokens_count,
|
|
6968
7266
|
CAST(SUM(reasoning_output_tokens) AS TEXT) AS reasoning_output_tokens_sum,
|
|
7267
|
+
CAST(COUNT(audio_duration_ms) AS TEXT) AS audio_duration_ms_count,
|
|
7268
|
+
CAST(SUM(audio_duration_ms) AS TEXT) AS audio_duration_ms_sum,
|
|
6969
7269
|
CAST(SUM(CASE WHEN valuation_status = 'rated' THEN 1 ELSE 0 END) AS TEXT)
|
|
6970
7270
|
AS rated_call_count,
|
|
6971
7271
|
CAST(SUM(CASE WHEN valuation_status = 'unpriceable' THEN 1 ELSE 0 END) AS TEXT)
|
|
@@ -7011,7 +7311,8 @@ function aggregateFragmentFromRow(row) {
|
|
|
7011
7311
|
cacheReadInputTokens: meterFragment(row.cache_read_input_tokens_count, row.cache_read_input_tokens_sum),
|
|
7012
7312
|
cacheWriteInputTokens: meterFragment(row.cache_write_input_tokens_count, row.cache_write_input_tokens_sum),
|
|
7013
7313
|
textOutputTokens: meterFragment(row.text_output_tokens_count, row.text_output_tokens_sum),
|
|
7014
|
-
reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum)
|
|
7314
|
+
reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum),
|
|
7315
|
+
audioDurationMs: meterFragment(row.audio_duration_ms_count, row.audio_duration_ms_sum)
|
|
7015
7316
|
},
|
|
7016
7317
|
costs: {
|
|
7017
7318
|
...row.amount_currency === null || row.amount_high === null || row.amount_middle === null || row.amount_low === null ? {} : {
|
|
@@ -7067,6 +7368,7 @@ function usageFromRow2(row, requesterGroupIds) {
|
|
|
7067
7368
|
providerId: row.provider_id,
|
|
7068
7369
|
...row.provider_ids === null ? {} : { providerIds: normalizeModelProviderIds2(JSON.parse(row.provider_ids)) },
|
|
7069
7370
|
requestedModelId: row.requested_model_id,
|
|
7371
|
+
...row.model_kind === null ? {} : { modelKind: row.model_kind },
|
|
7070
7372
|
...row.requested_reasoning === null ? {} : { requestedReasoning: requestedReasoningFromRow(row.requested_reasoning) },
|
|
7071
7373
|
...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
|
|
7072
7374
|
responseId: row.response_id,
|
|
@@ -7079,6 +7381,7 @@ function usageFromRow2(row, requesterGroupIds) {
|
|
|
7079
7381
|
...row.cache_write_input_tokens === null ? {} : { cacheWriteInputTokens: row.cache_write_input_tokens },
|
|
7080
7382
|
...row.text_output_tokens === null ? {} : { textOutputTokens: row.text_output_tokens },
|
|
7081
7383
|
...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens },
|
|
7384
|
+
...row.audio_duration_ms === null ? {} : { audioDurationMs: row.audio_duration_ms },
|
|
7082
7385
|
reportingStatus: row.reporting_status
|
|
7083
7386
|
},
|
|
7084
7387
|
...row.raw_usage === null ? {} : { rawUsage: JSON.parse(row.raw_usage) },
|
|
@@ -7219,6 +7522,7 @@ function required(value) {
|
|
|
7219
7522
|
// src/ai-limit-storage.ts
|
|
7220
7523
|
import {
|
|
7221
7524
|
aiLimitAmountKey,
|
|
7525
|
+
aiLimitMeterApplies,
|
|
7222
7526
|
aiLimitQuantityFromAmount,
|
|
7223
7527
|
aiLimitReservationBuckets,
|
|
7224
7528
|
aiLimitReservationRequestKey,
|
|
@@ -7327,8 +7631,9 @@ class SqliteAiLimitStorage {
|
|
|
7327
7631
|
const unavailablePolicies = [];
|
|
7328
7632
|
const unavailableReasons = new Set;
|
|
7329
7633
|
for (const policy of enabledPolicies) {
|
|
7330
|
-
if (!subjectKeys.has(aiLimitSubjectKey(policy.subject)))
|
|
7634
|
+
if (!subjectKeys.has(aiLimitSubjectKey(policy.subject)) || !aiLimitMeterApplies(policy.limit.meter, request.modelKind)) {
|
|
7331
7635
|
continue;
|
|
7636
|
+
}
|
|
7332
7637
|
const limit = normalizeAiLimitAmount(policy.limit);
|
|
7333
7638
|
const estimate = estimates.get(aiLimitAmountKey(limit));
|
|
7334
7639
|
const status = this.statusForPolicy(policy, request.period);
|
|
@@ -7362,7 +7667,7 @@ class SqliteAiLimitStorage {
|
|
|
7362
7667
|
resetAt: new Date(Math.min(...exhaustedPolicies.map((status) => status.period.resetAt.getTime())))
|
|
7363
7668
|
};
|
|
7364
7669
|
}
|
|
7365
|
-
const buckets = aiLimitReservationBuckets(enabledPolicies, request.subjects, request.estimates);
|
|
7670
|
+
const buckets = aiLimitReservationBuckets(enabledPolicies, request.subjects, request.estimates, request.modelKind);
|
|
7366
7671
|
if (buckets.length === 0)
|
|
7367
7672
|
return { status: "notRequired" };
|
|
7368
7673
|
for (const bucket of buckets) {
|
|
@@ -7576,6 +7881,7 @@ class SqliteAiLimitStorage {
|
|
|
7576
7881
|
SELECT usage.project_id, usage.id AS usage_record_id, usage.execution_id,
|
|
7577
7882
|
usage.attempt, usage.call_id, usage.occurred_at,
|
|
7578
7883
|
CAST(usage.total_tokens AS TEXT) AS total_tokens,
|
|
7884
|
+
usage.model_kind,
|
|
7579
7885
|
executions.requested_by_user_id,
|
|
7580
7886
|
executions.requested_by_service_account_id,
|
|
7581
7887
|
valuations.status AS valuation_status,
|
|
@@ -7614,6 +7920,7 @@ class SqliteAiLimitStorage {
|
|
|
7614
7920
|
SELECT usage.project_id, usage.id AS usage_record_id, usage.execution_id,
|
|
7615
7921
|
usage.attempt, usage.call_id, usage.occurred_at,
|
|
7616
7922
|
CAST(usage.total_tokens AS TEXT) AS total_tokens,
|
|
7923
|
+
usage.model_kind,
|
|
7617
7924
|
executions.requested_by_user_id,
|
|
7618
7925
|
executions.requested_by_service_account_id,
|
|
7619
7926
|
valuations.status AS valuation_status,
|
|
@@ -7691,6 +7998,7 @@ function accountingEntryFromRow(row, subject, requesterGroupIds = subject?.type
|
|
|
7691
7998
|
attempt: row.attempt,
|
|
7692
7999
|
callId: row.call_id,
|
|
7693
8000
|
occurredAt: new Date(row.occurred_at),
|
|
8001
|
+
...row.model_kind === null ? {} : { modelKind: row.model_kind },
|
|
7694
8002
|
...totalTokens !== undefined && Number.isSafeInteger(totalTokens) ? { totalTokens } : {},
|
|
7695
8003
|
requesterGroupIds,
|
|
7696
8004
|
...requester ? { requester } : {},
|
|
@@ -10410,9 +10718,12 @@ class SqliteExecutionStorage {
|
|
|
10410
10718
|
FROM executions
|
|
10411
10719
|
WHERE project_id = ? AND id = ?
|
|
10412
10720
|
`).get(params.projectId, params.id);
|
|
10413
|
-
return row ?
|
|
10721
|
+
return row ? sqliteExecutionRecord(row) : null;
|
|
10414
10722
|
}
|
|
10415
10723
|
}
|
|
10724
|
+
function sqliteExecutionRecord(row) {
|
|
10725
|
+
return executionRecordFromStorageRow(toStorageRow(row));
|
|
10726
|
+
}
|
|
10416
10727
|
function toStorageRow(row) {
|
|
10417
10728
|
return {
|
|
10418
10729
|
projectId: row.project_id,
|
|
@@ -10444,6 +10755,43 @@ function isDuplicateExecutionError(error) {
|
|
|
10444
10755
|
return error instanceof Error && error.message.includes("UNIQUE constraint failed: executions.project_id, executions.id");
|
|
10445
10756
|
}
|
|
10446
10757
|
|
|
10758
|
+
// src/file-download-grant-storage.ts
|
|
10759
|
+
import { parseFileDownloadGrantRow } from "@sixb/core/internal/storage";
|
|
10760
|
+
var GRANT_COLUMNS = `
|
|
10761
|
+
project_id AS projectId, id, token_hash AS tokenHash, file, execution_id AS executionId,
|
|
10762
|
+
created_at AS createdAt, expires_at AS expiresAt, revoked_at AS revokedAt
|
|
10763
|
+
`;
|
|
10764
|
+
|
|
10765
|
+
class SqliteFileDownloadGrantStorage {
|
|
10766
|
+
db;
|
|
10767
|
+
constructor(connection) {
|
|
10768
|
+
this.db = connection.db;
|
|
10769
|
+
}
|
|
10770
|
+
async create(record) {
|
|
10771
|
+
this.db.query(`INSERT INTO file_download_grants (
|
|
10772
|
+
project_id, id, token_hash, file, execution_id, created_at, expires_at
|
|
10773
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`).run(record.projectId, record.id, record.tokenHash, JSON.stringify(record.file), record.executionId, record.createdAt.toISOString(), record.expiresAt.toISOString());
|
|
10774
|
+
}
|
|
10775
|
+
async findByTokenHash(input) {
|
|
10776
|
+
const row = this.db.query(`SELECT ${GRANT_COLUMNS} FROM file_download_grants WHERE project_id = ? AND token_hash = ?`).get(input.projectId, input.tokenHash);
|
|
10777
|
+
return row ? parseFileDownloadGrantRow(row) : null;
|
|
10778
|
+
}
|
|
10779
|
+
async revoke(input) {
|
|
10780
|
+
const row = this.db.query(`UPDATE file_download_grants SET revoked_at = COALESCE(revoked_at, ?)
|
|
10781
|
+
WHERE project_id = ? AND id = ?
|
|
10782
|
+
RETURNING ${GRANT_COLUMNS}`).get(input.revokedAt.toISOString(), input.projectId, input.id);
|
|
10783
|
+
return row ? parseFileDownloadGrantRow(row) : null;
|
|
10784
|
+
}
|
|
10785
|
+
async deleteExpired(input) {
|
|
10786
|
+
return this.db.query(`DELETE FROM file_download_grants WHERE rowid IN (
|
|
10787
|
+
SELECT rowid FROM file_download_grants
|
|
10788
|
+
WHERE project_id = ? AND expires_at < ?
|
|
10789
|
+
ORDER BY expires_at
|
|
10790
|
+
LIMIT ?
|
|
10791
|
+
)`).run(input.projectId, input.expiredBefore.toISOString(), input.limit).changes;
|
|
10792
|
+
}
|
|
10793
|
+
}
|
|
10794
|
+
|
|
10447
10795
|
// src/file-upload-session-storage.ts
|
|
10448
10796
|
import {
|
|
10449
10797
|
DurableFileUploadSessions,
|
|
@@ -10584,6 +10932,8 @@ function exactContext(ctx) {
|
|
|
10584
10932
|
}
|
|
10585
10933
|
function compileObjectQueryInternal(projectId, query, ctx) {
|
|
10586
10934
|
switch (query.kind) {
|
|
10935
|
+
case "rerank":
|
|
10936
|
+
throw new ObjectQueryExecutionError("reranking_requires_runtime", "Reranking must execute in the model runtime, not storage.");
|
|
10587
10937
|
case "start":
|
|
10588
10938
|
return compileStart(projectId, query, ctx);
|
|
10589
10939
|
case "refs":
|
|
@@ -11716,6 +12066,7 @@ class SqliteObjectReader {
|
|
|
11716
12066
|
source
|
|
11717
12067
|
});
|
|
11718
12068
|
return {
|
|
12069
|
+
total: readTotal(this.db, compiled),
|
|
11719
12070
|
facets: params.facets.map((facet) => ({
|
|
11720
12071
|
propertyId: facet.propertyId,
|
|
11721
12072
|
buckets: readFacetBuckets(this.db, compiled, facet)
|
|
@@ -12496,9 +12847,7 @@ class SqliteObjectStorage {
|
|
|
12496
12847
|
return this.reader.existsObjects(params);
|
|
12497
12848
|
}
|
|
12498
12849
|
async facetObjects(params) {
|
|
12499
|
-
|
|
12500
|
-
return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
|
|
12501
|
-
return this.reader.facetObjects(params);
|
|
12850
|
+
return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
|
|
12502
12851
|
}
|
|
12503
12852
|
async getByPrimaryId(params) {
|
|
12504
12853
|
return this.reader.getByPrimaryId(params);
|
|
@@ -12547,13 +12896,14 @@ class SqliteObjectStorage {
|
|
|
12547
12896
|
}
|
|
12548
12897
|
async listByPrimaryIdPage(params) {
|
|
12549
12898
|
assertReconciliationPageLimit(params.limit);
|
|
12899
|
+
const cursor = params.afterPrimaryId === undefined ? [] : [params.afterPrimaryId];
|
|
12550
12900
|
const rows = this.db.query(`
|
|
12551
12901
|
SELECT * FROM objects
|
|
12552
12902
|
WHERE project_id = ? AND object_type_id = ?
|
|
12553
|
-
|
|
12903
|
+
${cursor.length > 0 ? "AND primary_id > ?" : ""}
|
|
12554
12904
|
ORDER BY primary_id ASC
|
|
12555
12905
|
LIMIT ?
|
|
12556
|
-
`).all(params.projectId, params.objectTypeId,
|
|
12906
|
+
`).all(params.projectId, params.objectTypeId, ...cursor, params.limit + 1);
|
|
12557
12907
|
const hasMore = rows.length > params.limit;
|
|
12558
12908
|
const objects = rows.slice(0, params.limit).map((row) => rowToObject(row));
|
|
12559
12909
|
const last = objects.at(-1);
|
|
@@ -12646,10 +12996,6 @@ function commitRecord(row) {
|
|
|
12646
12996
|
requestHash: row.request_hash,
|
|
12647
12997
|
executionId: row.execution_id,
|
|
12648
12998
|
origin: parseJson2(row.origin),
|
|
12649
|
-
...row.requested_by === null ? {} : {
|
|
12650
|
-
requestedBy: parseJson2(row.requested_by)
|
|
12651
|
-
},
|
|
12652
|
-
executor: parseJson2(row.executor),
|
|
12653
12999
|
ontologyRevision: row.ontology_revision,
|
|
12654
13000
|
...row.projection_revision === null ? {} : { projectionRevision: row.projection_revision },
|
|
12655
13001
|
...row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash },
|
|
@@ -12659,9 +13005,15 @@ function commitRecord(row) {
|
|
|
12659
13005
|
const result = parseJson2(row.result);
|
|
12660
13006
|
return { ...base, intent, result };
|
|
12661
13007
|
}
|
|
12662
|
-
function
|
|
13008
|
+
function claimedOutboxDraft(row) {
|
|
13009
|
+
if (row.lease_id === null || row.lease_expires_at === null) {
|
|
13010
|
+
throw new MaterializationConflictError("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
|
|
13011
|
+
}
|
|
12663
13012
|
return {
|
|
12664
|
-
|
|
13013
|
+
id: row.id,
|
|
13014
|
+
commitId: row.commit_id,
|
|
13015
|
+
commitOrdinal: row.commit_ordinal,
|
|
13016
|
+
event: parseJson2(row.event),
|
|
12665
13017
|
availableAt: row.available_at,
|
|
12666
13018
|
attempts: row.attempts,
|
|
12667
13019
|
leaseId: row.lease_id,
|
|
@@ -12795,9 +13147,9 @@ import {
|
|
|
12795
13147
|
linkRefKey as linkRefKey3,
|
|
12796
13148
|
linkRefSortKey,
|
|
12797
13149
|
linkScopeSortKey as linkScopeSortKey2,
|
|
12798
|
-
MaterializationConflictError as
|
|
12799
|
-
MaterializationValidationError as
|
|
12800
|
-
objectRefKey as
|
|
13150
|
+
MaterializationConflictError as MaterializationConflictError7,
|
|
13151
|
+
MaterializationValidationError as MaterializationValidationError3,
|
|
13152
|
+
objectRefKey as objectRefKey4,
|
|
12801
13153
|
objectRefSortKey,
|
|
12802
13154
|
telemetryPointKey as telemetryPointKey2,
|
|
12803
13155
|
telemetryPointSortKey as telemetryPointSortKey2
|
|
@@ -12808,24 +13160,24 @@ import {
|
|
|
12808
13160
|
assertExpectedObjectRevision,
|
|
12809
13161
|
assertMaterializationFinalizationCorrelation,
|
|
12810
13162
|
assertMaterializationHeader,
|
|
12811
|
-
|
|
12812
|
-
assertPageRows,
|
|
12813
|
-
assertPlanChunkCorrelations,
|
|
13163
|
+
assertPageRows as assertPageRows2,
|
|
12814
13164
|
assertSourceActivationCorrelation,
|
|
13165
|
+
beginMaterializationApply,
|
|
13166
|
+
beginMaterializationVectorChanges,
|
|
13167
|
+
createCardinalityValidator,
|
|
12815
13168
|
invalidCorrelation as invalidCorrelation2,
|
|
13169
|
+
materializationOutboxWrite,
|
|
13170
|
+
materializationPlanChunk,
|
|
12816
13171
|
sameNonnegativeCounts as sameCounts,
|
|
12817
13172
|
uniqueSorted
|
|
12818
13173
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
12819
13174
|
|
|
12820
13175
|
// src/ontology-storage/materialization-session.ts
|
|
12821
13176
|
import {
|
|
12822
|
-
MaterializationConflictError as
|
|
13177
|
+
MaterializationConflictError as MaterializationConflictError5
|
|
12823
13178
|
} from "@sixb/core/internal/materialization";
|
|
12824
13179
|
import {
|
|
12825
|
-
correlateMaterializationChunk,
|
|
12826
13180
|
duplicateMaterializationWork as duplicateWork,
|
|
12827
|
-
invalidCorrelation,
|
|
12828
|
-
materializationPlanItems,
|
|
12829
13181
|
ProviderMaterializationSessionState,
|
|
12830
13182
|
prepareMaterializationWork
|
|
12831
13183
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
@@ -12936,18 +13288,35 @@ function stageSourceRoots(db, manifest, input) {
|
|
|
12936
13288
|
}
|
|
12937
13289
|
}
|
|
12938
13290
|
}
|
|
12939
|
-
function activateSourceRoots(db, projectId, candidate, activation) {
|
|
13291
|
+
function activateSourceRoots(db, projectId, commitId, candidate, activation) {
|
|
12940
13292
|
if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
|
|
12941
13293
|
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
12942
13294
|
if (candidate.base_materialization_id !== null && candidate.root_count === 0)
|
|
12943
13295
|
return;
|
|
12944
13296
|
const delta = candidate.base_materialization_id !== null;
|
|
12945
|
-
|
|
12946
|
-
|
|
12947
|
-
AND
|
|
12948
|
-
|
|
12949
|
-
|
|
12950
|
-
|
|
13297
|
+
const replaced = `retired_at IS NULL AND deleted = 0
|
|
13298
|
+
AND version_id IN (SELECT version_id FROM ontology_sources
|
|
13299
|
+
WHERE project_id = ? AND source_id = ? AND status IN ${PUBLISHED})
|
|
13300
|
+
${delta ? `AND project_id = ? AND root_key IN (
|
|
13301
|
+
SELECT root_key FROM ontology_source_roots WHERE version_id = ?)` : ""}`;
|
|
13302
|
+
const replacedParams = [
|
|
13303
|
+
projectId,
|
|
13304
|
+
activation.source.projectionId,
|
|
13305
|
+
...delta ? [projectId, candidate.version_id] : []
|
|
13306
|
+
];
|
|
13307
|
+
db.query(`INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
|
|
13308
|
+
SELECT DISTINCT ?, ?, 'link', json_array(
|
|
13309
|
+
rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
|
|
13310
|
+
rows.target_primary_id
|
|
13311
|
+
)
|
|
13312
|
+
FROM (
|
|
13313
|
+
SELECT id FROM ontology_source_roots WHERE ${replaced}
|
|
13314
|
+
UNION ALL
|
|
13315
|
+
SELECT id FROM ontology_source_roots WHERE version_id = ?
|
|
13316
|
+
) AS changed
|
|
13317
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.root_id = changed.id
|
|
13318
|
+
WHERE rows.entity_kind = 'link'`).run(projectId, commitId, ...replacedParams, candidate.version_id);
|
|
13319
|
+
db.query(`UPDATE ontology_source_roots SET retired_at = ? WHERE ${replaced}`).run(activation.updatedAt, ...replacedParams);
|
|
12951
13320
|
db.query(`UPDATE ontology_source_roots SET retired_at = ?
|
|
12952
13321
|
WHERE version_id = ? AND deleted = 1`).run(activation.updatedAt, candidate.version_id);
|
|
12953
13322
|
}
|
|
@@ -13000,7 +13369,6 @@ function replacementAssertionRows(db, input) {
|
|
|
13000
13369
|
|
|
13001
13370
|
// src/ontology-storage/materialization-state.ts
|
|
13002
13371
|
var SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
|
|
13003
|
-
var SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
|
|
13004
13372
|
function linkScopeRequestJson(scopes) {
|
|
13005
13373
|
return canonicalJson(scopes.map(({ source, linkId }) => ({
|
|
13006
13374
|
scopeSortKey: linkScopeSortKey(source, linkId),
|
|
@@ -13328,20 +13696,6 @@ class SqliteMaterializationStateReader {
|
|
|
13328
13696
|
cursor = rows[rows.length - 1]?.sort_key ?? null;
|
|
13329
13697
|
}
|
|
13330
13698
|
}
|
|
13331
|
-
*replacementIdentities(input) {
|
|
13332
|
-
this.prepareReplacementIdentities(input);
|
|
13333
|
-
let cursor = null;
|
|
13334
|
-
while (true) {
|
|
13335
|
-
const rows = this.replacementRows(input, cursor);
|
|
13336
|
-
if (rows.length === 0)
|
|
13337
|
-
break;
|
|
13338
|
-
yield rows.map(replacementIdentity);
|
|
13339
|
-
if (rows.length < input.pageRows)
|
|
13340
|
-
break;
|
|
13341
|
-
const last = rows[rows.length - 1];
|
|
13342
|
-
cursor = { sortKey: last.sort_key, identityKey: last.identity_key };
|
|
13343
|
-
}
|
|
13344
|
-
}
|
|
13345
13699
|
replacementObjectStates(sourceId, candidateMaterializationId, refs) {
|
|
13346
13700
|
const base = this.objectStates(refs);
|
|
13347
13701
|
const candidates = this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
|
|
@@ -13407,174 +13761,6 @@ class SqliteMaterializationStateReader {
|
|
|
13407
13761
|
}
|
|
13408
13762
|
return { owned, byMaterialization };
|
|
13409
13763
|
}
|
|
13410
|
-
prepareReplacementIdentities(input) {
|
|
13411
|
-
if (input.kind === "object") {
|
|
13412
|
-
this.prepareReplacementObjects(input);
|
|
13413
|
-
return;
|
|
13414
|
-
}
|
|
13415
|
-
this.prepareReplacementLinks(input);
|
|
13416
|
-
}
|
|
13417
|
-
prepareReplacementObjects(input) {
|
|
13418
|
-
const rows = replacementSourceRows(this.db, {
|
|
13419
|
-
projectId: this.projectId,
|
|
13420
|
-
sourceId: input.sourceId,
|
|
13421
|
-
materializationId: input.candidateMaterializationId,
|
|
13422
|
-
incremental: input.incremental ?? false
|
|
13423
|
-
});
|
|
13424
|
-
this.db.query(`
|
|
13425
|
-
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
13426
|
-
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
13427
|
-
)
|
|
13428
|
-
SELECT ?, 'object', json_array(object_type_id, primary_id),
|
|
13429
|
-
${objectSortExpression()}, 1
|
|
13430
|
-
FROM (${rows.sql}) AS rows
|
|
13431
|
-
WHERE entity_kind = 'object'
|
|
13432
|
-
GROUP BY object_type_id, primary_id
|
|
13433
|
-
`).run(input.sessionId, ...rows.values);
|
|
13434
|
-
}
|
|
13435
|
-
prepareReplacementLinks(input) {
|
|
13436
|
-
const rows = replacementSourceRows(this.db, {
|
|
13437
|
-
projectId: this.projectId,
|
|
13438
|
-
sourceId: input.sourceId,
|
|
13439
|
-
materializationId: input.candidateMaterializationId,
|
|
13440
|
-
incremental: input.incremental ?? false
|
|
13441
|
-
});
|
|
13442
|
-
this.db.query(`
|
|
13443
|
-
WITH incident_objects AS (
|
|
13444
|
-
SELECT DISTINCT
|
|
13445
|
-
json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
|
|
13446
|
-
json_extract(payload, '$.ref.primaryId') AS primary_id
|
|
13447
|
-
FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
|
|
13448
|
-
WHERE session_id = ? AND kind = 'incident-object'
|
|
13449
|
-
), replacement_links AS (
|
|
13450
|
-
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
13451
|
-
target_type_id, target_primary_id AS target_id
|
|
13452
|
-
FROM (${rows.sql}) AS rows
|
|
13453
|
-
WHERE entity_kind = 'link'
|
|
13454
|
-
), incident_links AS (
|
|
13455
|
-
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
13456
|
-
links.target_type_id, links.target_id
|
|
13457
|
-
FROM links
|
|
13458
|
-
JOIN incident_objects
|
|
13459
|
-
ON incident_objects.object_type_id = links.source_type_id
|
|
13460
|
-
AND incident_objects.primary_id = links.source_id
|
|
13461
|
-
WHERE links.project_id = ?
|
|
13462
|
-
UNION
|
|
13463
|
-
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
13464
|
-
links.target_type_id, links.target_id
|
|
13465
|
-
FROM links
|
|
13466
|
-
JOIN incident_objects
|
|
13467
|
-
ON incident_objects.object_type_id = links.target_type_id
|
|
13468
|
-
AND incident_objects.primary_id = links.target_id
|
|
13469
|
-
WHERE links.project_id = ?
|
|
13470
|
-
UNION
|
|
13471
|
-
SELECT overrides.source_type_id,
|
|
13472
|
-
overrides.source_primary_id AS source_id,
|
|
13473
|
-
overrides.link_id, overrides.target_type_id,
|
|
13474
|
-
overrides.target_primary_id AS target_id
|
|
13475
|
-
FROM ontology_link_overrides AS overrides
|
|
13476
|
-
JOIN incident_objects
|
|
13477
|
-
ON incident_objects.object_type_id = overrides.source_type_id
|
|
13478
|
-
AND incident_objects.primary_id = overrides.source_primary_id
|
|
13479
|
-
WHERE overrides.project_id = ?
|
|
13480
|
-
UNION
|
|
13481
|
-
SELECT overrides.source_type_id,
|
|
13482
|
-
overrides.source_primary_id AS source_id,
|
|
13483
|
-
overrides.link_id, overrides.target_type_id,
|
|
13484
|
-
overrides.target_primary_id AS target_id
|
|
13485
|
-
FROM ontology_link_overrides AS overrides
|
|
13486
|
-
JOIN incident_objects
|
|
13487
|
-
ON incident_objects.object_type_id = overrides.target_type_id
|
|
13488
|
-
AND incident_objects.primary_id = overrides.target_primary_id
|
|
13489
|
-
WHERE overrides.project_id = ?
|
|
13490
|
-
UNION
|
|
13491
|
-
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
13492
|
-
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
13493
|
-
FROM incident_objects
|
|
13494
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
13495
|
-
ON rows.entity_kind = 'link'
|
|
13496
|
-
AND rows.source_type_id = incident_objects.object_type_id
|
|
13497
|
-
AND rows.source_primary_id = incident_objects.primary_id
|
|
13498
|
-
${LIVE_SOURCE_ROWS_JOIN}
|
|
13499
|
-
UNION
|
|
13500
|
-
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
13501
|
-
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
13502
|
-
FROM incident_objects
|
|
13503
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
13504
|
-
ON rows.entity_kind = 'link'
|
|
13505
|
-
AND rows.target_type_id = incident_objects.object_type_id
|
|
13506
|
-
AND rows.target_primary_id = incident_objects.primary_id
|
|
13507
|
-
${LIVE_SOURCE_ROWS_JOIN}
|
|
13508
|
-
), diff_links AS (
|
|
13509
|
-
SELECT * FROM replacement_links
|
|
13510
|
-
UNION SELECT * FROM incident_links
|
|
13511
|
-
), affected_scopes AS (
|
|
13512
|
-
SELECT DISTINCT source_type_id, source_id, link_id FROM diff_links
|
|
13513
|
-
), all_links AS (
|
|
13514
|
-
SELECT *, 1 AS diff_required FROM diff_links
|
|
13515
|
-
UNION ALL
|
|
13516
|
-
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
13517
|
-
links.target_type_id, links.target_id, 0 AS diff_required
|
|
13518
|
-
FROM links
|
|
13519
|
-
JOIN affected_scopes USING (source_type_id, source_id, link_id)
|
|
13520
|
-
WHERE links.project_id = ?
|
|
13521
|
-
UNION ALL
|
|
13522
|
-
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
13523
|
-
overrides.link_id, overrides.target_type_id,
|
|
13524
|
-
overrides.target_primary_id AS target_id, 0 AS diff_required
|
|
13525
|
-
FROM ontology_link_overrides AS overrides
|
|
13526
|
-
JOIN affected_scopes
|
|
13527
|
-
ON affected_scopes.source_type_id = overrides.source_type_id
|
|
13528
|
-
AND affected_scopes.source_id = overrides.source_primary_id
|
|
13529
|
-
AND affected_scopes.link_id = overrides.link_id
|
|
13530
|
-
WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
|
|
13531
|
-
), selected AS (
|
|
13532
|
-
SELECT source_type_id, source_id, link_id, target_type_id, target_id,
|
|
13533
|
-
${linkSortExpression("all_links")} AS sort_key,
|
|
13534
|
-
MAX(diff_required) AS diff_required
|
|
13535
|
-
FROM all_links
|
|
13536
|
-
GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
|
|
13537
|
-
)
|
|
13538
|
-
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
13539
|
-
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
13540
|
-
)
|
|
13541
|
-
SELECT ?, 'link',
|
|
13542
|
-
json_array(source_type_id, source_id, link_id, target_type_id, target_id),
|
|
13543
|
-
sort_key, diff_required
|
|
13544
|
-
FROM selected
|
|
13545
|
-
`).run(input.sessionId, ...rows.values, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, input.sessionId);
|
|
13546
|
-
}
|
|
13547
|
-
replacementRows(input, cursor) {
|
|
13548
|
-
return this.db.query(`
|
|
13549
|
-
SELECT entity_kind, identity_key, sort_key, diff_required
|
|
13550
|
-
FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
|
|
13551
|
-
WHERE session_id = ? AND entity_kind = ?
|
|
13552
|
-
AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
|
|
13553
|
-
ORDER BY sort_key, identity_key
|
|
13554
|
-
LIMIT ?
|
|
13555
|
-
`).all(input.sessionId, input.kind, cursor?.sortKey ?? null, cursor?.sortKey ?? null, cursor?.identityKey ?? null, input.pageRows);
|
|
13556
|
-
}
|
|
13557
|
-
}
|
|
13558
|
-
function replacementIdentity(row) {
|
|
13559
|
-
const parts = parseJson2(row.identity_key);
|
|
13560
|
-
if (row.entity_kind === "object") {
|
|
13561
|
-
return {
|
|
13562
|
-
kind: "object",
|
|
13563
|
-
ref: { objectTypeId: parts[0], primaryId: parts[1] },
|
|
13564
|
-
sortKey: row.sort_key,
|
|
13565
|
-
diffRequired: true
|
|
13566
|
-
};
|
|
13567
|
-
}
|
|
13568
|
-
return {
|
|
13569
|
-
kind: "link",
|
|
13570
|
-
ref: {
|
|
13571
|
-
source: { objectTypeId: parts[0], primaryId: parts[1] },
|
|
13572
|
-
linkId: parts[2],
|
|
13573
|
-
target: { objectTypeId: parts[3], primaryId: parts[4] }
|
|
13574
|
-
},
|
|
13575
|
-
sortKey: row.sort_key,
|
|
13576
|
-
diffRequired: row.diff_required === 1
|
|
13577
|
-
};
|
|
13578
13764
|
}
|
|
13579
13765
|
function activeSourceMap(rows) {
|
|
13580
13766
|
const result = new Map;
|
|
@@ -13702,20 +13888,709 @@ function storedPoint(row) {
|
|
|
13702
13888
|
lastCommitId: row.last_commit_id
|
|
13703
13889
|
};
|
|
13704
13890
|
}
|
|
13705
|
-
function objectSortExpression(alias) {
|
|
13706
|
-
const prefix = alias ? `${alias}.` : "";
|
|
13707
|
-
return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`;
|
|
13708
|
-
}
|
|
13709
13891
|
function linkSortExpression(alias) {
|
|
13710
13892
|
const prefix = alias ? `${alias}.` : "";
|
|
13711
13893
|
return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`;
|
|
13712
13894
|
}
|
|
13713
13895
|
|
|
13896
|
+
// src/ontology-storage/replacement-plans.ts
|
|
13897
|
+
import {
|
|
13898
|
+
MaterializationConflictError as MaterializationConflictError4,
|
|
13899
|
+
MaterializationValidationError as MaterializationValidationError2,
|
|
13900
|
+
objectRefKey as objectRefKey2
|
|
13901
|
+
} from "@sixb/core/internal/materialization";
|
|
13902
|
+
import {
|
|
13903
|
+
assertNonblank as assertNonblank2,
|
|
13904
|
+
assertPageRows,
|
|
13905
|
+
assertPositiveInteger as assertPositiveInteger2,
|
|
13906
|
+
assertReplacementPlanCommit,
|
|
13907
|
+
canonicalJson as canonicalJson2,
|
|
13908
|
+
duplicateMaterializationWork,
|
|
13909
|
+
invalidCorrelation,
|
|
13910
|
+
materializationWorkColumns,
|
|
13911
|
+
prepareReplacementIdentities,
|
|
13912
|
+
workUniquenessKey
|
|
13913
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
13914
|
+
var SQLITE_PLAN_WORK_TABLE = "ontology_replacement_plan_work";
|
|
13915
|
+
var IDENTITIES = "ontology_replacement_plan_identities";
|
|
13916
|
+
var SORT_KEY = "LOWER(HEX(CAST(identity_key AS BLOB)))";
|
|
13917
|
+
function objectRevision(type, id) {
|
|
13918
|
+
const series = (alias) => `${alias}.project_id = $projectId AND ${alias}.object_type_id = ${type} AND ${alias}.object_id = ${id}`;
|
|
13919
|
+
return `json_array(
|
|
13920
|
+
(SELECT json_array(version, last_commit_id) FROM objects
|
|
13921
|
+
WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}),
|
|
13922
|
+
(SELECT last_commit_id FROM ontology_object_overrides
|
|
13923
|
+
WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}),
|
|
13924
|
+
(WITH RECURSIVE properties(property_id) AS (
|
|
13925
|
+
SELECT MIN(first.property_id) FROM timeseries AS first WHERE ${series("first")}
|
|
13926
|
+
UNION ALL
|
|
13927
|
+
SELECT (SELECT MIN(next.property_id) FROM timeseries AS next
|
|
13928
|
+
WHERE ${series("next")} AND next.property_id > properties.property_id)
|
|
13929
|
+
FROM properties WHERE properties.property_id IS NOT NULL
|
|
13930
|
+
)
|
|
13931
|
+
SELECT json_group_array(json((
|
|
13932
|
+
SELECT json_array(points.property_id, points.at, points.last_commit_id)
|
|
13933
|
+
FROM timeseries AS points
|
|
13934
|
+
WHERE ${series("points")} AND points.property_id = properties.property_id
|
|
13935
|
+
ORDER BY points.at DESC LIMIT 1
|
|
13936
|
+
))) FROM properties WHERE properties.property_id IS NOT NULL)
|
|
13937
|
+
)`;
|
|
13938
|
+
}
|
|
13939
|
+
function endpointRevision(type, id) {
|
|
13940
|
+
return `CASE WHEN EXISTS (
|
|
13941
|
+
SELECT 1 FROM ${IDENTITIES} AS endpoint
|
|
13942
|
+
WHERE endpoint.version_id = $versionId AND endpoint.entity_kind = 'object'
|
|
13943
|
+
AND endpoint.identity_key = json_array(${type}, ${id})
|
|
13944
|
+
) THEN ${objectRevision(type, id)}
|
|
13945
|
+
ELSE json(CASE WHEN EXISTS (
|
|
13946
|
+
SELECT 1 FROM objects
|
|
13947
|
+
WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}
|
|
13948
|
+
) THEN 'true' ELSE 'false' END)
|
|
13949
|
+
END`;
|
|
13950
|
+
}
|
|
13951
|
+
function linkRevision(key) {
|
|
13952
|
+
const part = (index) => `json_extract(${key}, '$[${index}]')`;
|
|
13953
|
+
return `json_array(
|
|
13954
|
+
(SELECT last_commit_id FROM links
|
|
13955
|
+
WHERE project_id = $projectId AND source_type_id = ${part(0)} AND source_id = ${part(1)}
|
|
13956
|
+
AND link_id = ${part(2)} AND target_type_id = ${part(3)} AND target_id = ${part(4)}),
|
|
13957
|
+
(SELECT overrides.last_commit_id FROM ontology_link_overrides AS overrides
|
|
13958
|
+
WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'edge'
|
|
13959
|
+
AND overrides.identity_key = ${key}),
|
|
13960
|
+
(SELECT overrides.last_commit_id FROM ontology_link_overrides AS overrides
|
|
13961
|
+
WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
|
|
13962
|
+
AND overrides.identity_key = json_array(${part(0)}, ${part(1)}, ${part(2)})),
|
|
13963
|
+
(SELECT MAX(roots.id) FROM ontology_source_roots AS roots
|
|
13964
|
+
CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
13965
|
+
AND versions.status IN ${PUBLISHED}
|
|
13966
|
+
WHERE roots.project_id = $projectId AND roots.retired_at IS NULL AND roots.deleted = 0
|
|
13967
|
+
AND roots.root_key IN (
|
|
13968
|
+
'["link",' || substr(${key}, 2),
|
|
13969
|
+
'["object",' || substr(json_array(${part(0)}, ${part(1)}), 2)
|
|
13970
|
+
)
|
|
13971
|
+
AND EXISTS (
|
|
13972
|
+
-- \`+\` keeps the lookup on the root's few rows, not on every row of a hub source.
|
|
13973
|
+
SELECT 1 FROM ontology_source_rows AS rows
|
|
13974
|
+
WHERE rows.root_id = roots.id AND +rows.entity_kind = 'link'
|
|
13975
|
+
AND +rows.source_type_id = ${part(0)} AND +rows.source_primary_id = ${part(1)}
|
|
13976
|
+
AND +rows.link_id = ${part(2)} AND +rows.target_type_id = ${part(3)}
|
|
13977
|
+
AND +rows.target_primary_id = ${part(4)}
|
|
13978
|
+
)),
|
|
13979
|
+
${endpointRevision(part(0), part(1))},
|
|
13980
|
+
${endpointRevision(part(3), part(4))}
|
|
13981
|
+
)`;
|
|
13982
|
+
}
|
|
13983
|
+
function identityRevision(kind) {
|
|
13984
|
+
const key = "identities.identity_key";
|
|
13985
|
+
return kind === "object" ? objectRevision(`json_extract(${key}, '$[0]')`, `json_extract(${key}, '$[1]')`) : linkRevision(key);
|
|
13986
|
+
}
|
|
13987
|
+
function linkKeys(relation) {
|
|
13988
|
+
return `SELECT json_array(source_type_id, source_id, link_id, target_type_id, target_id)
|
|
13989
|
+
AS identity_key, json_array(source_type_id, source_id) AS source_key,
|
|
13990
|
+
json_array(target_type_id, target_id) AS target_key
|
|
13991
|
+
FROM ${relation}`;
|
|
13992
|
+
}
|
|
13993
|
+
var NEXT_EXPANSION = `(
|
|
13994
|
+
SELECT expansions FROM ontology_replacement_plans WHERE version_id = $versionId
|
|
13995
|
+
)`;
|
|
13996
|
+
var INSERT_INCIDENT_LINKS = `
|
|
13997
|
+
INSERT INTO ${IDENTITIES} (
|
|
13998
|
+
version_id, entity_kind, identity_key, sort_key, diff_required, expand_at, source_key,
|
|
13999
|
+
target_key
|
|
14000
|
+
)
|
|
14001
|
+
SELECT $versionId, 'link', identity_key, ${SORT_KEY}, 1, ${NEXT_EXPANSION}, source_key,
|
|
14002
|
+
target_key
|
|
14003
|
+
FROM (${linkKeys("incident_links")}) WHERE true
|
|
14004
|
+
ON CONFLICT (version_id, entity_kind, identity_key) DO UPDATE
|
|
14005
|
+
SET diff_required = 1, read_revision = NULL, planned_revision = NULL, classified = 0,
|
|
14006
|
+
change = NULL, expand_at = excluded.expand_at
|
|
14007
|
+
WHERE ${IDENTITIES}.diff_required = 0`;
|
|
14008
|
+
var INSERT_MEMBER_LINKS = `
|
|
14009
|
+
INSERT INTO ${IDENTITIES} (
|
|
14010
|
+
version_id, entity_kind, identity_key, sort_key, diff_required, source_key, target_key
|
|
14011
|
+
)
|
|
14012
|
+
SELECT $versionId, 'link', identity_key, ${SORT_KEY}, 0, source_key, target_key
|
|
14013
|
+
FROM (${linkKeys("members")}) WHERE true
|
|
14014
|
+
ON CONFLICT (version_id, entity_kind, identity_key) DO NOTHING`;
|
|
14015
|
+
var LIVE_ROWS = `CROSS JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
|
|
14016
|
+
AND roots.project_id = $projectId AND roots.retired_at IS NULL AND roots.deleted = 0
|
|
14017
|
+
CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
14018
|
+
AND versions.status IN ${PUBLISHED}`;
|
|
14019
|
+
var EXPAND_LINK_STATEMENTS = {
|
|
14020
|
+
incident: `WITH incident_objects AS MATERIALIZED (
|
|
14021
|
+
SELECT json_extract(work.payload, '$.ref.objectTypeId') AS object_type_id,
|
|
14022
|
+
json_extract(work.payload, '$.ref.primaryId') AS primary_id
|
|
14023
|
+
FROM ${IDENTITIES} AS objects
|
|
14024
|
+
CROSS JOIN ${SQLITE_PLAN_WORK_TABLE} AS work ON work.work_id = $versionId
|
|
14025
|
+
AND work.unique_key = 'incident-object:' || objects.identity_key
|
|
14026
|
+
WHERE objects.version_id = $versionId AND objects.entity_kind = 'object'
|
|
14027
|
+
AND objects.expand_at = ${NEXT_EXPANSION}
|
|
14028
|
+
), incident_links AS (
|
|
14029
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
14030
|
+
links.target_type_id, links.target_id
|
|
14031
|
+
FROM incident_objects CROSS JOIN links
|
|
14032
|
+
ON links.project_id = $projectId
|
|
14033
|
+
AND links.source_type_id = incident_objects.object_type_id
|
|
14034
|
+
AND links.source_id = incident_objects.primary_id
|
|
14035
|
+
UNION
|
|
14036
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
14037
|
+
links.target_type_id, links.target_id
|
|
14038
|
+
FROM incident_objects CROSS JOIN links INDEXED BY idx_links_target
|
|
14039
|
+
ON links.project_id = $projectId
|
|
14040
|
+
AND links.target_type_id = incident_objects.object_type_id
|
|
14041
|
+
AND links.target_id = incident_objects.primary_id
|
|
14042
|
+
UNION
|
|
14043
|
+
SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
|
|
14044
|
+
overrides.target_type_id, overrides.target_primary_id
|
|
14045
|
+
FROM incident_objects CROSS JOIN ontology_link_overrides AS overrides
|
|
14046
|
+
ON overrides.project_id = $projectId AND overrides.identity_kind IN ('edge', 'slot')
|
|
14047
|
+
AND overrides.source_type_id = incident_objects.object_type_id
|
|
14048
|
+
AND overrides.source_primary_id = incident_objects.primary_id
|
|
14049
|
+
UNION
|
|
14050
|
+
SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
|
|
14051
|
+
overrides.target_type_id, overrides.target_primary_id
|
|
14052
|
+
FROM incident_objects
|
|
14053
|
+
CROSS JOIN ontology_link_overrides AS overrides INDEXED BY idx_ontology_link_overrides_target
|
|
14054
|
+
ON overrides.project_id = $projectId
|
|
14055
|
+
AND overrides.target_type_id = incident_objects.object_type_id
|
|
14056
|
+
AND overrides.target_primary_id = incident_objects.primary_id
|
|
14057
|
+
UNION
|
|
14058
|
+
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
14059
|
+
rows.target_type_id, rows.target_primary_id
|
|
14060
|
+
FROM incident_objects CROSS JOIN ontology_source_rows AS rows
|
|
14061
|
+
ON rows.entity_kind = 'link'
|
|
14062
|
+
AND rows.source_type_id = incident_objects.object_type_id
|
|
14063
|
+
AND rows.source_primary_id = incident_objects.primary_id
|
|
14064
|
+
${LIVE_ROWS}
|
|
14065
|
+
UNION
|
|
14066
|
+
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
14067
|
+
rows.target_type_id, rows.target_primary_id
|
|
14068
|
+
FROM incident_objects CROSS JOIN ontology_source_rows AS rows
|
|
14069
|
+
ON rows.entity_kind = 'link'
|
|
14070
|
+
AND rows.target_type_id = incident_objects.object_type_id
|
|
14071
|
+
AND rows.target_primary_id = incident_objects.primary_id
|
|
14072
|
+
${LIVE_ROWS}
|
|
14073
|
+
)
|
|
14074
|
+
${INSERT_INCIDENT_LINKS}`,
|
|
14075
|
+
members: `WITH scopes AS (
|
|
14076
|
+
SELECT DISTINCT json_extract(identity_key, '$[0]') AS source_type_id,
|
|
14077
|
+
json_extract(identity_key, '$[1]') AS source_id,
|
|
14078
|
+
json_extract(identity_key, '$[2]') AS link_id
|
|
14079
|
+
FROM ${IDENTITIES}
|
|
14080
|
+
WHERE version_id = $versionId AND entity_kind = 'link' AND diff_required = 1
|
|
14081
|
+
AND expand_at = ${NEXT_EXPANSION}
|
|
14082
|
+
), members AS (
|
|
14083
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
14084
|
+
links.target_type_id, links.target_id
|
|
14085
|
+
FROM scopes CROSS JOIN links
|
|
14086
|
+
ON links.project_id = $projectId
|
|
14087
|
+
AND links.source_type_id = scopes.source_type_id
|
|
14088
|
+
AND links.source_id = scopes.source_id AND links.link_id = scopes.link_id
|
|
14089
|
+
UNION
|
|
14090
|
+
SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
|
|
14091
|
+
overrides.target_type_id, overrides.target_primary_id
|
|
14092
|
+
FROM scopes CROSS JOIN ontology_link_overrides AS overrides
|
|
14093
|
+
ON overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
|
|
14094
|
+
AND overrides.source_type_id = scopes.source_type_id
|
|
14095
|
+
AND overrides.source_primary_id = scopes.source_id
|
|
14096
|
+
AND overrides.link_id = scopes.link_id
|
|
14097
|
+
)
|
|
14098
|
+
${INSERT_MEMBER_LINKS}`
|
|
14099
|
+
};
|
|
14100
|
+
var TOUCHED = `touched AS MATERIALIZED (
|
|
14101
|
+
SELECT DISTINCT touches.entity_kind, touches.identity_key
|
|
14102
|
+
FROM ontology_commits AS commits
|
|
14103
|
+
CROSS JOIN ontology_commit_touches AS touches
|
|
14104
|
+
ON touches.project_id = commits.project_id AND touches.commit_id = commits.id
|
|
14105
|
+
WHERE commits.rowid > (
|
|
14106
|
+
SELECT watermark FROM ontology_replacement_plans WHERE version_id = $versionId
|
|
14107
|
+
) AND +commits.project_id = $projectId
|
|
14108
|
+
)`;
|
|
14109
|
+
var TOUCHED_REFRESH_STATEMENTS = {
|
|
14110
|
+
object: `WITH ${TOUCHED}
|
|
14111
|
+
UPDATE ${IDENTITIES} AS identities
|
|
14112
|
+
SET read_revision = NULL, planned_revision = NULL, classified = 0, change = NULL
|
|
14113
|
+
WHERE identities.version_id = $versionId AND identities.entity_kind = 'object'
|
|
14114
|
+
AND identities.identity_key IN (SELECT identity_key FROM touched WHERE entity_kind = 'object')
|
|
14115
|
+
AND identities.planned_revision IS NOT ${identityRevision("object")}`,
|
|
14116
|
+
link: `WITH ${TOUCHED}, candidates AS (
|
|
14117
|
+
SELECT identity_key FROM touched WHERE entity_kind = 'link'
|
|
14118
|
+
UNION
|
|
14119
|
+
SELECT links.identity_key FROM touched CROSS JOIN ${IDENTITIES} AS links
|
|
14120
|
+
ON links.version_id = $versionId
|
|
14121
|
+
AND links.source_key = json_array(
|
|
14122
|
+
json_extract(touched.identity_key, '$[0]'), json_extract(touched.identity_key, '$[1]')
|
|
14123
|
+
)
|
|
14124
|
+
WHERE touched.entity_kind IN ('object', 'scope')
|
|
14125
|
+
UNION
|
|
14126
|
+
SELECT links.identity_key FROM touched CROSS JOIN ${IDENTITIES} AS links
|
|
14127
|
+
ON links.version_id = $versionId AND links.target_key = touched.identity_key
|
|
14128
|
+
WHERE touched.entity_kind = 'object'
|
|
14129
|
+
)
|
|
14130
|
+
UPDATE ${IDENTITIES} AS identities
|
|
14131
|
+
SET read_revision = NULL, planned_revision = NULL, classified = 0, change = NULL
|
|
14132
|
+
WHERE identities.version_id = $versionId AND identities.entity_kind = 'link'
|
|
14133
|
+
AND identities.identity_key IN (SELECT identity_key FROM candidates)
|
|
14134
|
+
AND identities.planned_revision IS NOT ${identityRevision("link")}`
|
|
14135
|
+
};
|
|
14136
|
+
var TOUCHED_LINKS = `touched_links AS (
|
|
14137
|
+
SELECT identity_key, json_extract(identity_key, '$[0]') AS source_type_id,
|
|
14138
|
+
json_extract(identity_key, '$[1]') AS source_id, json_extract(identity_key, '$[2]') AS link_id,
|
|
14139
|
+
json_extract(identity_key, '$[3]') AS target_type_id,
|
|
14140
|
+
json_extract(identity_key, '$[4]') AS target_id
|
|
14141
|
+
FROM touched WHERE entity_kind = 'link'
|
|
14142
|
+
), slot_targets AS (
|
|
14143
|
+
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id, overrides.link_id,
|
|
14144
|
+
overrides.target_type_id, overrides.target_primary_id AS target_id
|
|
14145
|
+
FROM touched CROSS JOIN ontology_link_overrides AS overrides
|
|
14146
|
+
ON overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
|
|
14147
|
+
AND overrides.identity_key = touched.identity_key
|
|
14148
|
+
WHERE touched.entity_kind = 'scope'
|
|
14149
|
+
)`;
|
|
14150
|
+
var LINK_COLUMNS = "source_type_id, source_id, link_id, target_type_id, target_id";
|
|
14151
|
+
var EFFECTIVE_TOUCHED_LINK = `EXISTS (
|
|
14152
|
+
SELECT 1 FROM links AS stored
|
|
14153
|
+
WHERE stored.project_id = $projectId AND stored.source_type_id = touched_links.source_type_id
|
|
14154
|
+
AND stored.source_id = touched_links.source_id AND stored.link_id = touched_links.link_id
|
|
14155
|
+
AND stored.target_type_id = touched_links.target_type_id
|
|
14156
|
+
AND stored.target_id = touched_links.target_id
|
|
14157
|
+
)`;
|
|
14158
|
+
var EXPAND_TOUCHED_STATEMENTS = {
|
|
14159
|
+
incident: `WITH ${TOUCHED}, ${TOUCHED_LINKS}, incident_links AS (
|
|
14160
|
+
SELECT candidates.* FROM (
|
|
14161
|
+
SELECT ${LINK_COLUMNS} FROM touched_links
|
|
14162
|
+
WHERE ${EFFECTIVE_TOUCHED_LINK} OR EXISTS (
|
|
14163
|
+
SELECT 1 FROM ontology_link_overrides AS overrides
|
|
14164
|
+
WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'edge'
|
|
14165
|
+
AND overrides.identity_key = touched_links.identity_key
|
|
14166
|
+
) OR EXISTS (
|
|
14167
|
+
SELECT 1 FROM ontology_source_rows AS rows ${LIVE_ROWS}
|
|
14168
|
+
WHERE rows.entity_kind = 'link' AND rows.source_type_id = touched_links.source_type_id
|
|
14169
|
+
AND rows.source_primary_id = touched_links.source_id
|
|
14170
|
+
AND rows.link_id = touched_links.link_id
|
|
14171
|
+
AND rows.target_type_id = touched_links.target_type_id
|
|
14172
|
+
AND rows.target_primary_id = touched_links.target_id
|
|
14173
|
+
)
|
|
14174
|
+
UNION
|
|
14175
|
+
SELECT ${LINK_COLUMNS} FROM slot_targets
|
|
14176
|
+
) AS candidates
|
|
14177
|
+
WHERE EXISTS (
|
|
14178
|
+
SELECT 1 FROM ${SQLITE_PLAN_WORK_TABLE} AS work
|
|
14179
|
+
WHERE work.work_id = $versionId AND work.unique_key IN (
|
|
14180
|
+
'incident-object:' || json_array(candidates.source_type_id, candidates.source_id),
|
|
14181
|
+
'incident-object:' || json_array(candidates.target_type_id, candidates.target_id)
|
|
14182
|
+
)
|
|
14183
|
+
)
|
|
14184
|
+
)
|
|
14185
|
+
${INSERT_INCIDENT_LINKS}`,
|
|
14186
|
+
members: `WITH ${TOUCHED}, ${TOUCHED_LINKS}, members AS (
|
|
14187
|
+
SELECT candidates.* FROM (
|
|
14188
|
+
SELECT ${LINK_COLUMNS} FROM touched_links WHERE ${EFFECTIVE_TOUCHED_LINK}
|
|
14189
|
+
UNION
|
|
14190
|
+
SELECT ${LINK_COLUMNS} FROM slot_targets
|
|
14191
|
+
) AS candidates
|
|
14192
|
+
WHERE EXISTS (
|
|
14193
|
+
SELECT 1 FROM ${IDENTITIES} AS changed
|
|
14194
|
+
WHERE changed.version_id = $versionId
|
|
14195
|
+
AND changed.source_key = json_array(candidates.source_type_id, candidates.source_id)
|
|
14196
|
+
AND changed.diff_required = 1
|
|
14197
|
+
AND json_extract(changed.identity_key, '$[2]') = candidates.link_id
|
|
14198
|
+
)
|
|
14199
|
+
)
|
|
14200
|
+
${INSERT_MEMBER_LINKS}`
|
|
14201
|
+
};
|
|
14202
|
+
var PURGE_TOUCHES_STATEMENT = `DELETE FROM ontology_commit_touches WHERE rowid IN (
|
|
14203
|
+
SELECT touches.rowid FROM ontology_commit_touches AS touches
|
|
14204
|
+
CROSS JOIN ontology_commits AS commits
|
|
14205
|
+
ON commits.project_id = touches.project_id AND commits.id = touches.commit_id
|
|
14206
|
+
WHERE touches.project_id = $projectId AND commits.rowid <= COALESCE(
|
|
14207
|
+
(SELECT MIN(plans.watermark) FROM ontology_replacement_plans AS plans
|
|
14208
|
+
JOIN ontology_sources AS versions USING (version_id)
|
|
14209
|
+
WHERE plans.project_id = $projectId AND versions.status = 'ready'),
|
|
14210
|
+
(SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits)
|
|
14211
|
+
)
|
|
14212
|
+
LIMIT $limit
|
|
14213
|
+
)`;
|
|
14214
|
+
|
|
14215
|
+
class SqliteOntologyReplacementPlanStorage {
|
|
14216
|
+
db;
|
|
14217
|
+
runRootOperation;
|
|
14218
|
+
constructor(db, runRootOperation) {
|
|
14219
|
+
this.db = db;
|
|
14220
|
+
this.runRootOperation = runRootOperation;
|
|
14221
|
+
}
|
|
14222
|
+
async open(input) {
|
|
14223
|
+
return this.runRootOperation(() => {
|
|
14224
|
+
const { commit } = input;
|
|
14225
|
+
assertReplacementPlanCommit(commit);
|
|
14226
|
+
const candidate = this.requireCandidate(input);
|
|
14227
|
+
const active = this.db.query(`SELECT materialization_id, last_commit_id FROM ontology_sources
|
|
14228
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'`).get(input.projectId, input.source.projectionId);
|
|
14229
|
+
const replacedMaterializationId = active?.materialization_id ?? null;
|
|
14230
|
+
const replacedLastCommitId = active?.last_commit_id ?? null;
|
|
14231
|
+
const existing = planRow(this.db, candidate.version_id);
|
|
14232
|
+
if (existing?.replaced_materialization_id === replacedMaterializationId && existing.replaced_last_commit_id === replacedLastCommitId) {
|
|
14233
|
+
return { committedAt: existing.committed_at };
|
|
14234
|
+
}
|
|
14235
|
+
if (existing)
|
|
14236
|
+
deletePlan(this.db, candidate.version_id);
|
|
14237
|
+
this.db.query(`INSERT INTO ontology_replacement_plans (
|
|
14238
|
+
version_id, project_id, commit_id, committed_at, replaced_materialization_id,
|
|
14239
|
+
replaced_last_commit_id, watermark
|
|
14240
|
+
) VALUES (?, ?, ?, ?, ?, ?, (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits))`).run(candidate.version_id, input.projectId, commit.id, commit.committedAt, replacedMaterializationId, replacedLastCommitId);
|
|
14241
|
+
const rows = replacementSourceRows(this.db, {
|
|
14242
|
+
projectId: input.projectId,
|
|
14243
|
+
sourceId: input.source.projectionId,
|
|
14244
|
+
materializationId: input.materializationId,
|
|
14245
|
+
incremental: candidate.base_materialization_id !== null
|
|
14246
|
+
});
|
|
14247
|
+
this.db.query(`INSERT INTO ${IDENTITIES} (
|
|
14248
|
+
version_id, entity_kind, identity_key, sort_key, diff_required, expand_at,
|
|
14249
|
+
source_key, target_key
|
|
14250
|
+
)
|
|
14251
|
+
SELECT ?, entity_kind, identity_key, ${SORT_KEY}, 1,
|
|
14252
|
+
CASE entity_kind WHEN 'link' THEN 0 END, source_key, target_key
|
|
14253
|
+
FROM (
|
|
14254
|
+
SELECT entity_kind, CASE entity_kind
|
|
14255
|
+
WHEN 'object' THEN json_array(object_type_id, primary_id)
|
|
14256
|
+
ELSE json_array(
|
|
14257
|
+
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
14258
|
+
)
|
|
14259
|
+
END AS identity_key,
|
|
14260
|
+
CASE entity_kind
|
|
14261
|
+
WHEN 'link' THEN json_array(source_type_id, source_primary_id)
|
|
14262
|
+
END AS source_key,
|
|
14263
|
+
CASE entity_kind
|
|
14264
|
+
WHEN 'link' THEN json_array(target_type_id, target_primary_id)
|
|
14265
|
+
END AS target_key
|
|
14266
|
+
FROM (${rows.sql}) AS rows
|
|
14267
|
+
)
|
|
14268
|
+
GROUP BY entity_kind, identity_key`).run(candidate.version_id, ...rows.values);
|
|
14269
|
+
return { committedAt: commit.committedAt };
|
|
14270
|
+
});
|
|
14271
|
+
}
|
|
14272
|
+
async* streamState(input) {
|
|
14273
|
+
assertPageRows(input.pageRows);
|
|
14274
|
+
await this.runRootOperation(() => {
|
|
14275
|
+
const { candidate } = this.requirePlan(input);
|
|
14276
|
+
if (input.entityKind === "object" && candidate.projection_kind !== "object") {
|
|
14277
|
+
throw new MaterializationConflictError4("source-materialization", "Link projection replacement cannot stream object state.");
|
|
14278
|
+
}
|
|
14279
|
+
if (input.entityKind === "link") {
|
|
14280
|
+
if (unplannedCount(this.db, candidate.version_id, "object") > 0) {
|
|
14281
|
+
throw new MaterializationConflictError4("effective-state", "Object projection replacement must plan every object before its links.");
|
|
14282
|
+
}
|
|
14283
|
+
this.expandLinks(candidate.version_id, input.projectId);
|
|
14284
|
+
}
|
|
14285
|
+
});
|
|
14286
|
+
let after = "";
|
|
14287
|
+
while (true) {
|
|
14288
|
+
const page = await this.runRootOperation(() => {
|
|
14289
|
+
const { candidate } = this.requirePlan(input);
|
|
14290
|
+
const rows = this.db.query(`SELECT identity_key, sort_key, diff_required FROM ${IDENTITIES}
|
|
14291
|
+
WHERE version_id = ? AND entity_kind = ? AND planned_revision IS NULL AND sort_key > ?
|
|
14292
|
+
ORDER BY sort_key
|
|
14293
|
+
LIMIT ?`).all(candidate.version_id, input.entityKind, after, input.pageRows);
|
|
14294
|
+
if (rows.length === 0)
|
|
14295
|
+
return null;
|
|
14296
|
+
after = rows[rows.length - 1].sort_key;
|
|
14297
|
+
const state = this.readPage(input, candidate, rows);
|
|
14298
|
+
this.db.query(`UPDATE ${IDENTITIES} AS identities SET read_revision = ${identityRevision(input.entityKind)}
|
|
14299
|
+
WHERE identities.version_id = $versionId AND identities.entity_kind = $entityKind
|
|
14300
|
+
AND identities.identity_key IN (SELECT value FROM json_each($keys))`).run({
|
|
14301
|
+
$projectId: input.projectId,
|
|
14302
|
+
$versionId: candidate.version_id,
|
|
14303
|
+
$entityKind: input.entityKind,
|
|
14304
|
+
$keys: JSON.stringify(rows.map((row) => row.identity_key))
|
|
14305
|
+
});
|
|
14306
|
+
return state;
|
|
14307
|
+
});
|
|
14308
|
+
if (!page)
|
|
14309
|
+
return;
|
|
14310
|
+
yield page;
|
|
14311
|
+
}
|
|
14312
|
+
}
|
|
14313
|
+
async stage(input) {
|
|
14314
|
+
await this.runRootOperation(() => {
|
|
14315
|
+
const { plan, candidate } = this.requirePlan(input);
|
|
14316
|
+
const identities = prepareReplacementIdentities(input, {
|
|
14317
|
+
commitId: plan.commit_id,
|
|
14318
|
+
committedAt: plan.committed_at
|
|
14319
|
+
});
|
|
14320
|
+
const versionId = candidate.version_id;
|
|
14321
|
+
this.db.run("SAVEPOINT sixb_replacement_plan_stage");
|
|
14322
|
+
let recordKey;
|
|
14323
|
+
try {
|
|
14324
|
+
const known = this.db.query(`SELECT read_revision, planned_revision FROM ${IDENTITIES}
|
|
14325
|
+
WHERE version_id = ? AND entity_kind = ? AND identity_key = ?`);
|
|
14326
|
+
const insert = this.db.query(`INSERT INTO ${SQLITE_PLAN_WORK_TABLE} (
|
|
14327
|
+
work_id, entity_kind, identity_key, record_key, unique_key, kind, lane,
|
|
14328
|
+
major_order, minor_order, sort_one, sort_two,
|
|
14329
|
+
classification_entity_kind, classification_identity_key,
|
|
14330
|
+
cardinality_view, cardinality_occupied, cardinality_source_type_id,
|
|
14331
|
+
cardinality_source_primary_id, cardinality_link_id,
|
|
14332
|
+
cardinality_target_type_id, cardinality_target_primary_id, payload
|
|
14333
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, json(?))`);
|
|
14334
|
+
const planned = this.db.query(`UPDATE ${IDENTITIES}
|
|
14335
|
+
SET planned_revision = read_revision, classified = ?, change = ?,
|
|
14336
|
+
expand_at = CASE entity_kind
|
|
14337
|
+
WHEN 'object' THEN (
|
|
14338
|
+
SELECT expansions FROM ontology_replacement_plans WHERE version_id = ?
|
|
14339
|
+
)
|
|
14340
|
+
ELSE expand_at
|
|
14341
|
+
END
|
|
14342
|
+
WHERE version_id = ? AND entity_kind = ? AND identity_key = ?`);
|
|
14343
|
+
for (const identity of identities) {
|
|
14344
|
+
const row = known.get(versionId, identity.kind, identity.key);
|
|
14345
|
+
if (row?.read_revision == null || row.planned_revision !== null) {
|
|
14346
|
+
throw new MaterializationValidationError2(`Replacement identity ${identity.entityKey} is not streamed and still to plan.`);
|
|
14347
|
+
}
|
|
14348
|
+
for (const record of identity.records) {
|
|
14349
|
+
recordKey = record.recordKey;
|
|
14350
|
+
const columns = materializationWorkColumns(record);
|
|
14351
|
+
const classification = record.kind === "classification" ? record : null;
|
|
14352
|
+
const cardinality = record.kind === "cardinality" ? record : null;
|
|
14353
|
+
insert.run(versionId, identity.kind, identity.key, record.recordKey, workUniquenessKey(record), record.kind, columns.lane, columns.majorOrder, columns.minorOrder, columns.sortOne, columns.sortTwo, classification?.entityKind ?? null, classification?.identityKey ?? null, cardinality?.view ?? null, cardinality ? Number(cardinality.occupied) : null, cardinality?.ref.source.objectTypeId ?? null, cardinality?.ref.source.primaryId ?? null, cardinality?.ref.linkId ?? null, cardinality?.ref.target.objectTypeId ?? null, cardinality?.ref.target.primaryId ?? null, canonicalJson2(record));
|
|
14354
|
+
}
|
|
14355
|
+
planned.run(Number(identity.classified), identity.change, versionId, versionId, identity.kind, identity.key);
|
|
14356
|
+
}
|
|
14357
|
+
this.db.run("RELEASE SAVEPOINT sixb_replacement_plan_stage");
|
|
14358
|
+
} catch (error) {
|
|
14359
|
+
this.db.run("ROLLBACK TO SAVEPOINT sixb_replacement_plan_stage");
|
|
14360
|
+
this.db.run("RELEASE SAVEPOINT sixb_replacement_plan_stage");
|
|
14361
|
+
if (isSqliteConstraintError(error))
|
|
14362
|
+
throw duplicateMaterializationWork(recordKey);
|
|
14363
|
+
throw error;
|
|
14364
|
+
}
|
|
14365
|
+
});
|
|
14366
|
+
}
|
|
14367
|
+
async refresh(input) {
|
|
14368
|
+
return this.runRootOperation(() => {
|
|
14369
|
+
const { plan, candidate } = this.requirePlan(input);
|
|
14370
|
+
const versionId = candidate.version_id;
|
|
14371
|
+
const pending = unplannedCount(this.db, versionId);
|
|
14372
|
+
if (pending > 0)
|
|
14373
|
+
return { fresh: false, unplanned: pending };
|
|
14374
|
+
if (!committedSince(this.db, plan, input.projectId)) {
|
|
14375
|
+
return { fresh: true, counts: replacementPlanCounts(this.db, versionId) };
|
|
14376
|
+
}
|
|
14377
|
+
const params = { $projectId: input.projectId, $versionId: versionId };
|
|
14378
|
+
let unplanned = 0;
|
|
14379
|
+
for (const kind of ["object", "link"]) {
|
|
14380
|
+
const stale = this.db.query(TOUCHED_REFRESH_STATEMENTS[kind]).run(params).changes;
|
|
14381
|
+
if (stale === 0)
|
|
14382
|
+
continue;
|
|
14383
|
+
dropUnplannedWork(this.db, versionId, kind);
|
|
14384
|
+
unplanned += stale;
|
|
14385
|
+
}
|
|
14386
|
+
unplanned += this.expandTouched(versionId, input.projectId);
|
|
14387
|
+
this.db.query(`UPDATE ontology_replacement_plans
|
|
14388
|
+
SET watermark = (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits)
|
|
14389
|
+
WHERE version_id = ?`).run(versionId);
|
|
14390
|
+
if (unplanned > 0)
|
|
14391
|
+
return { fresh: false, unplanned };
|
|
14392
|
+
return { fresh: true, counts: replacementPlanCounts(this.db, versionId) };
|
|
14393
|
+
});
|
|
14394
|
+
}
|
|
14395
|
+
async purge(input) {
|
|
14396
|
+
return this.runRootOperation(() => {
|
|
14397
|
+
assertNonblank2(input.projectId, "Replacement plan purge project id");
|
|
14398
|
+
assertPositiveInteger2(input.limit, "Replacement plan purge limit");
|
|
14399
|
+
const spentPlan = this.db.query(`SELECT plans.version_id FROM ontology_replacement_plans AS plans
|
|
14400
|
+
JOIN ontology_sources AS versions USING (version_id)
|
|
14401
|
+
WHERE plans.project_id = ? AND versions.status <> 'ready'
|
|
14402
|
+
ORDER BY plans.version_id
|
|
14403
|
+
LIMIT 1`);
|
|
14404
|
+
let deleted = 0;
|
|
14405
|
+
while (deleted < input.limit) {
|
|
14406
|
+
const spent = spentPlan.get(input.projectId);
|
|
14407
|
+
if (!spent)
|
|
14408
|
+
break;
|
|
14409
|
+
for (const [table, column] of [
|
|
14410
|
+
[SQLITE_PLAN_WORK_TABLE, "work_id"],
|
|
14411
|
+
[IDENTITIES, "version_id"]
|
|
14412
|
+
]) {
|
|
14413
|
+
deleted += this.db.query(`DELETE FROM ${table} WHERE rowid IN (
|
|
14414
|
+
SELECT rowid FROM ${table} WHERE ${column} = ? LIMIT ?
|
|
14415
|
+
)`).run(spent.version_id, input.limit - deleted).changes;
|
|
14416
|
+
if (deleted >= input.limit)
|
|
14417
|
+
return deleted;
|
|
14418
|
+
}
|
|
14419
|
+
this.db.query(`DELETE FROM ontology_replacement_plans WHERE version_id = ?`).run(spent.version_id);
|
|
14420
|
+
deleted += 1;
|
|
14421
|
+
}
|
|
14422
|
+
if (deleted >= input.limit)
|
|
14423
|
+
return deleted;
|
|
14424
|
+
return deleted + this.db.query(PURGE_TOUCHES_STATEMENT).run({ $projectId: input.projectId, $limit: input.limit - deleted }).changes;
|
|
14425
|
+
});
|
|
14426
|
+
}
|
|
14427
|
+
readPage(input, candidate, rows) {
|
|
14428
|
+
const reader = new SqliteMaterializationStateReader(this.db, input.projectId);
|
|
14429
|
+
if (input.entityKind === "object") {
|
|
14430
|
+
const refs = rows.map((row) => {
|
|
14431
|
+
const [objectTypeId, primaryId] = parseJson2(row.identity_key);
|
|
14432
|
+
return { objectTypeId, primaryId };
|
|
14433
|
+
});
|
|
14434
|
+
return {
|
|
14435
|
+
objects: reader.replacementObjectStates(input.source.projectionId, input.materializationId, refs),
|
|
14436
|
+
links: [],
|
|
14437
|
+
endpoints: []
|
|
14438
|
+
};
|
|
14439
|
+
}
|
|
14440
|
+
const identities = rows.map((row) => {
|
|
14441
|
+
const parts = parseJson2(row.identity_key);
|
|
14442
|
+
const ref = {
|
|
14443
|
+
source: { objectTypeId: parts[0], primaryId: parts[1] },
|
|
14444
|
+
linkId: parts[2],
|
|
14445
|
+
target: { objectTypeId: parts[3], primaryId: parts[4] }
|
|
14446
|
+
};
|
|
14447
|
+
return {
|
|
14448
|
+
ref,
|
|
14449
|
+
sortKey: row.sort_key,
|
|
14450
|
+
diffRequired: row.diff_required === 1
|
|
14451
|
+
};
|
|
14452
|
+
});
|
|
14453
|
+
const previous = this.db.query(`SELECT materialization_id FROM ontology_sources
|
|
14454
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'`).get(input.projectId, input.source.projectionId);
|
|
14455
|
+
const links = reader.replacementLinkStates(input.source.projectionId, input.materializationId, [input.materializationId, ...previous ? [previous.materialization_id] : []], identities, candidate.base_materialization_id !== null);
|
|
14456
|
+
return {
|
|
14457
|
+
objects: [],
|
|
14458
|
+
links,
|
|
14459
|
+
endpoints: this.endpoints(candidate.version_id, input.projectId, identities.flatMap(({ ref }) => [ref.source, ref.target]))
|
|
14460
|
+
};
|
|
14461
|
+
}
|
|
14462
|
+
endpoints(versionId, projectId, refs) {
|
|
14463
|
+
const unique = [...new Map(refs.map((ref) => [objectRefKey2(ref), ref])).values()];
|
|
14464
|
+
const rows = this.db.query(`SELECT requested.value AS identity_key, (
|
|
14465
|
+
SELECT json_extract(work.payload, '$.exists') FROM ${SQLITE_PLAN_WORK_TABLE} AS work
|
|
14466
|
+
WHERE work.work_id = ? AND work.unique_key = 'object-existence:' || requested.value
|
|
14467
|
+
) AS planned, EXISTS (
|
|
14468
|
+
SELECT 1 FROM objects
|
|
14469
|
+
WHERE project_id = ? AND object_type_id = json_extract(requested.value, '$[0]')
|
|
14470
|
+
AND primary_id = json_extract(requested.value, '$[1]')
|
|
14471
|
+
) AS effective
|
|
14472
|
+
FROM json_each(?) AS requested`).all(versionId, projectId, JSON.stringify(unique.map((ref) => objectRefKey2(ref))));
|
|
14473
|
+
const existence = new Map(rows.map((row) => [row.identity_key, (row.planned ?? row.effective) === 1]));
|
|
14474
|
+
return unique.map((ref) => ({ ref, exists: existence.get(objectRefKey2(ref)) ?? false }));
|
|
14475
|
+
}
|
|
14476
|
+
expandLinks(versionId, projectId) {
|
|
14477
|
+
const added = this.expand(versionId, projectId, EXPAND_LINK_STATEMENTS);
|
|
14478
|
+
this.db.query(`UPDATE ontology_replacement_plans SET expansions = expansions + 1 WHERE version_id = ?`).run(versionId);
|
|
14479
|
+
return added;
|
|
14480
|
+
}
|
|
14481
|
+
expandTouched(versionId, projectId) {
|
|
14482
|
+
return this.expand(versionId, projectId, EXPAND_TOUCHED_STATEMENTS);
|
|
14483
|
+
}
|
|
14484
|
+
expand(versionId, projectId, statements) {
|
|
14485
|
+
const run = (statement) => this.db.query(statement).run({ $projectId: projectId, $versionId: versionId }).changes;
|
|
14486
|
+
const diffLinks = run(statements.incident);
|
|
14487
|
+
if (diffLinks > 0)
|
|
14488
|
+
dropUnplannedWork(this.db, versionId, "link");
|
|
14489
|
+
return diffLinks + run(statements.members);
|
|
14490
|
+
}
|
|
14491
|
+
requireCandidate(input) {
|
|
14492
|
+
const candidate = sourceRow(this.db, input.projectId, input.source.projectionId, input.materializationId);
|
|
14493
|
+
if (!candidate || candidate.status !== "ready" || candidate.projection_run_id !== input.execution.projectionRunId || candidate.execution_token !== input.execution.executionToken) {
|
|
14494
|
+
throw new MaterializationConflictError4("source-materialization", `Candidate source materialization '${input.materializationId}' is missing, not ready, or owned by another execution.`);
|
|
14495
|
+
}
|
|
14496
|
+
assertProjectionExecution(this.db, {
|
|
14497
|
+
projectId: input.projectId,
|
|
14498
|
+
sourceId: input.source.projectionId,
|
|
14499
|
+
projectionRunId: input.execution.projectionRunId,
|
|
14500
|
+
executionToken: input.execution.executionToken
|
|
14501
|
+
});
|
|
14502
|
+
return candidate;
|
|
14503
|
+
}
|
|
14504
|
+
requirePlan(input) {
|
|
14505
|
+
const candidate = this.requireCandidate(input);
|
|
14506
|
+
const plan = planRow(this.db, candidate.version_id);
|
|
14507
|
+
if (!plan) {
|
|
14508
|
+
throw new MaterializationConflictError4("source-materialization", `Candidate source materialization '${input.materializationId}' has no open plan.`);
|
|
14509
|
+
}
|
|
14510
|
+
return { plan, candidate };
|
|
14511
|
+
}
|
|
14512
|
+
}
|
|
14513
|
+
function assertReplacementPlanCoverage(db, plan) {
|
|
14514
|
+
const mismatch = db.query(`SELECT 1 FROM ${IDENTITIES}
|
|
14515
|
+
WHERE version_id = ?
|
|
14516
|
+
AND classified <> (diff_required AND (entity_kind = 'link' OR ? = 1))
|
|
14517
|
+
LIMIT 1`).get(plan.versionId, plan.projectionKind === "object" ? 1 : 0);
|
|
14518
|
+
if (mismatch) {
|
|
14519
|
+
invalidCorrelation("Projection replacement classification coverage does not match its plan.");
|
|
14520
|
+
}
|
|
14521
|
+
}
|
|
14522
|
+
function replacementPlanCounts(db, versionId) {
|
|
14523
|
+
const rows = db.query(`SELECT entity_kind, COALESCE(change, 'unchanged') AS change, COUNT(*) AS count
|
|
14524
|
+
FROM ${IDENTITIES}
|
|
14525
|
+
WHERE version_id = ? AND classified = 1
|
|
14526
|
+
GROUP BY entity_kind, COALESCE(change, 'unchanged')`).all(versionId);
|
|
14527
|
+
const count = (kind, change) => rows.find((row) => row.entity_kind === kind && row.change === change)?.count ?? 0;
|
|
14528
|
+
return {
|
|
14529
|
+
objectsCreated: count("object", "created"),
|
|
14530
|
+
objectsUpdated: count("object", "updated"),
|
|
14531
|
+
objectsDeleted: count("object", "deleted"),
|
|
14532
|
+
objectsUnchanged: count("object", "unchanged"),
|
|
14533
|
+
linksCreated: count("link", "created"),
|
|
14534
|
+
linksUpdated: count("link", "updated"),
|
|
14535
|
+
linksDeleted: count("link", "deleted"),
|
|
14536
|
+
linksUnchanged: count("link", "unchanged")
|
|
14537
|
+
};
|
|
14538
|
+
}
|
|
14539
|
+
function unplannedCount(db, versionId, kind) {
|
|
14540
|
+
const row = db.query(`SELECT COUNT(*) AS count FROM ${IDENTITIES}
|
|
14541
|
+
WHERE version_id = ? AND planned_revision IS NULL AND (? IS NULL OR entity_kind = ?)`).get(versionId, kind ?? null, kind ?? null);
|
|
14542
|
+
return row.count;
|
|
14543
|
+
}
|
|
14544
|
+
function committedSince(db, plan, projectId) {
|
|
14545
|
+
return db.query(`SELECT 1 FROM ontology_commits WHERE rowid > ? AND +project_id = ? LIMIT 1`).get(plan.watermark, projectId) !== null;
|
|
14546
|
+
}
|
|
14547
|
+
function dropUnplannedWork(db, versionId, kind) {
|
|
14548
|
+
db.query(`DELETE FROM ${SQLITE_PLAN_WORK_TABLE}
|
|
14549
|
+
WHERE work_id = ? AND entity_kind = ? AND identity_key IN (
|
|
14550
|
+
SELECT identity_key FROM ${IDENTITIES}
|
|
14551
|
+
WHERE version_id = ? AND entity_kind = ? AND planned_revision IS NULL
|
|
14552
|
+
)`).run(versionId, kind, versionId, kind);
|
|
14553
|
+
}
|
|
14554
|
+
function deletePlan(db, versionId) {
|
|
14555
|
+
db.query(`DELETE FROM ${SQLITE_PLAN_WORK_TABLE} WHERE work_id = ?`).run(versionId);
|
|
14556
|
+
db.query(`DELETE FROM ${IDENTITIES} WHERE version_id = ?`).run(versionId);
|
|
14557
|
+
db.query(`DELETE FROM ontology_replacement_plans WHERE version_id = ?`).run(versionId);
|
|
14558
|
+
}
|
|
14559
|
+
function planRow(db, versionId) {
|
|
14560
|
+
return db.query(`SELECT * FROM ontology_replacement_plans WHERE version_id = ?`).get(versionId);
|
|
14561
|
+
}
|
|
14562
|
+
function sourceRow(db, projectId, sourceId, materializationId) {
|
|
14563
|
+
return db.query(`SELECT * FROM ontology_sources
|
|
14564
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).get(projectId, sourceId, materializationId);
|
|
14565
|
+
}
|
|
14566
|
+
function boundReplacementPlan(db, header) {
|
|
14567
|
+
const ref = header.plan;
|
|
14568
|
+
const candidate = ref ? sourceRow(db, header.commit.projectId, ref.source.projectionId, ref.materializationId) : null;
|
|
14569
|
+
const plan = candidate ? planRow(db, candidate.version_id) : null;
|
|
14570
|
+
if (!ref || !candidate || !plan || header.commit.intent.kind !== "projection") {
|
|
14571
|
+
throw new MaterializationConflictError4("source-materialization", "A plan-bound session needs the open plan of a projection candidate.");
|
|
14572
|
+
}
|
|
14573
|
+
if (plan.commit_id !== header.commit.id || plan.committed_at !== header.commit.committedAt) {
|
|
14574
|
+
throw new MaterializationValidationError2("A plan-bound session must begin with the commit its plan carries.");
|
|
14575
|
+
}
|
|
14576
|
+
if (committedSince(db, plan, header.commit.projectId) || unplannedCount(db, candidate.version_id) > 0) {
|
|
14577
|
+
throw new MaterializationValidationError2("A replacement plan applies only fully planned and just refreshed.");
|
|
14578
|
+
}
|
|
14579
|
+
return { versionId: candidate.version_id, projectionKind: candidate.projection_kind };
|
|
14580
|
+
}
|
|
14581
|
+
|
|
13714
14582
|
// src/ontology-storage/materialization-session.ts
|
|
13715
14583
|
class SqliteMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
13716
|
-
|
|
13717
|
-
appliedEventCursor = null;
|
|
14584
|
+
plan;
|
|
13718
14585
|
summary = emptyWorkSummary();
|
|
14586
|
+
workTable;
|
|
14587
|
+
workId;
|
|
14588
|
+
constructor(header, transactionId, plan) {
|
|
14589
|
+
super(header, transactionId);
|
|
14590
|
+
this.plan = plan;
|
|
14591
|
+
this.workTable = plan ? SQLITE_PLAN_WORK_TABLE : SQLITE_MATERIALIZATION_WORK_TABLE;
|
|
14592
|
+
this.workId = plan ? plan.versionId : this.id;
|
|
14593
|
+
}
|
|
13719
14594
|
}
|
|
13720
14595
|
|
|
13721
14596
|
class SqliteMaterializationSessions {
|
|
@@ -13727,12 +14602,12 @@ class SqliteMaterializationSessions {
|
|
|
13727
14602
|
this.db = db;
|
|
13728
14603
|
this.context = context;
|
|
13729
14604
|
}
|
|
13730
|
-
create(header) {
|
|
14605
|
+
create(header, plan) {
|
|
13731
14606
|
if (!this.context?.active) {
|
|
13732
|
-
throw new
|
|
14607
|
+
throw new MaterializationConflictError5("effective-state", "Materialization sessions require an active storage transaction.");
|
|
13733
14608
|
}
|
|
13734
14609
|
this.ensureTables();
|
|
13735
|
-
const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id);
|
|
14610
|
+
const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id, plan);
|
|
13736
14611
|
this.sessions.set(session.providerToken, session);
|
|
13737
14612
|
this.live.add(session);
|
|
13738
14613
|
this.context.materializations.register(session.providerToken);
|
|
@@ -13741,7 +14616,7 @@ class SqliteMaterializationSessions {
|
|
|
13741
14616
|
require(session) {
|
|
13742
14617
|
const value = this.sessions.get(session.providerToken);
|
|
13743
14618
|
if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
|
|
13744
|
-
throw new
|
|
14619
|
+
throw new MaterializationConflictError5("effective-state", "Materialization session is inactive.");
|
|
13745
14620
|
}
|
|
13746
14621
|
return value;
|
|
13747
14622
|
}
|
|
@@ -13749,11 +14624,10 @@ class SqliteMaterializationSessions {
|
|
|
13749
14624
|
if (!session.active)
|
|
13750
14625
|
return;
|
|
13751
14626
|
session.active = false;
|
|
13752
|
-
session.replacement = null;
|
|
13753
14627
|
this.live.delete(session);
|
|
13754
14628
|
if (this.live.size === 0)
|
|
13755
14629
|
this.dropWorkTables();
|
|
13756
|
-
else
|
|
14630
|
+
else if (!session.plan)
|
|
13757
14631
|
this.deleteWork(session.id);
|
|
13758
14632
|
this.context?.materializations.complete(session.providerToken);
|
|
13759
14633
|
}
|
|
@@ -13768,7 +14642,7 @@ class SqliteMaterializationSessions {
|
|
|
13768
14642
|
return;
|
|
13769
14643
|
const insert = this.db.query(`
|
|
13770
14644
|
INSERT INTO ${SQLITE_MATERIALIZATION_WORK_TABLE} (
|
|
13771
|
-
|
|
14645
|
+
work_id, record_key, unique_key, kind, lane,
|
|
13772
14646
|
major_order, minor_order, sort_one, sort_two,
|
|
13773
14647
|
classification_entity_kind, classification_identity_key,
|
|
13774
14648
|
cardinality_view, cardinality_occupied, cardinality_source_type_id,
|
|
@@ -13795,74 +14669,62 @@ class SqliteMaterializationSessions {
|
|
|
13795
14669
|
throw error;
|
|
13796
14670
|
}
|
|
13797
14671
|
for (const prepared of records)
|
|
13798
|
-
recordSummary(session.summary, prepared.record
|
|
14672
|
+
recordSummary(session.summary, prepared.record);
|
|
13799
14673
|
}
|
|
13800
|
-
|
|
13801
|
-
const session = this.require(input.session);
|
|
13802
|
-
const stream = session.workStreams[input.order];
|
|
13803
|
-
if (stream.started) {
|
|
13804
|
-
throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
|
|
13805
|
-
}
|
|
13806
|
-
stream.started = true;
|
|
13807
|
-
session.workSealed = true;
|
|
14674
|
+
*lanePages(session, lane, pageRows) {
|
|
13808
14675
|
let cursor = null;
|
|
13809
14676
|
while (true) {
|
|
13810
|
-
this.require(
|
|
13811
|
-
const rows = this.readLane(
|
|
14677
|
+
this.require(session.publicSession());
|
|
14678
|
+
const rows = this.readLane(session, lane, cursor, pageRows);
|
|
13812
14679
|
if (rows.length === 0)
|
|
13813
|
-
|
|
13814
|
-
const records = rows.map((row) => parseJson2(row.payload));
|
|
14680
|
+
return;
|
|
13815
14681
|
cursor = workCursor(rows[rows.length - 1]);
|
|
13816
|
-
|
|
13817
|
-
yield { records };
|
|
14682
|
+
yield rows.map((row) => parseJson2(row.payload));
|
|
13818
14683
|
}
|
|
13819
|
-
this.require(input.session);
|
|
13820
|
-
stream.completed = true;
|
|
13821
14684
|
}
|
|
13822
|
-
|
|
13823
|
-
|
|
13824
|
-
|
|
13825
|
-
|
|
14685
|
+
*vectorChangePages(session, objectTypeIds, pageRows) {
|
|
14686
|
+
if (objectTypeIds.length === 0 && !this.db.query(`SELECT 1 FROM object_vectors WHERE project_id = ? LIMIT 1`).get(session.header.commit.projectId)) {
|
|
14687
|
+
return;
|
|
14688
|
+
}
|
|
14689
|
+
const query = this.db.query(`
|
|
14690
|
+
WITH changes AS (
|
|
14691
|
+
SELECT record_key, payload,
|
|
14692
|
+
coalesce(
|
|
14693
|
+
json_extract(payload, '$.item.value.row.ref.objectTypeId'),
|
|
14694
|
+
json_extract(payload, '$.item.value.ref.objectTypeId')
|
|
14695
|
+
) AS object_type_id,
|
|
14696
|
+
coalesce(
|
|
14697
|
+
json_extract(payload, '$.item.value.row.ref.primaryId'),
|
|
14698
|
+
json_extract(payload, '$.item.value.ref.primaryId')
|
|
14699
|
+
) AS primary_id
|
|
14700
|
+
FROM ${session.workTable}
|
|
14701
|
+
WHERE work_id = ? AND lane = 'apply' AND kind = 'plan'
|
|
14702
|
+
AND json_extract(payload, '$.item.kind') IN ('object-upsert', 'object-delete')
|
|
14703
|
+
AND record_key > ?
|
|
14704
|
+
)
|
|
14705
|
+
SELECT record_key, payload FROM changes
|
|
14706
|
+
WHERE object_type_id IN (SELECT value FROM json_each(?))
|
|
14707
|
+
OR EXISTS (
|
|
14708
|
+
SELECT 1 FROM object_vectors
|
|
14709
|
+
WHERE project_id = ? AND object_type_id = changes.object_type_id
|
|
14710
|
+
AND primary_id = changes.primary_id
|
|
14711
|
+
)
|
|
14712
|
+
ORDER BY record_key
|
|
14713
|
+
LIMIT ?
|
|
13826
14714
|
`);
|
|
13827
|
-
|
|
13828
|
-
|
|
13829
|
-
|
|
13830
|
-
|
|
13831
|
-
|
|
13832
|
-
|
|
13833
|
-
|
|
13834
|
-
|
|
13835
|
-
|
|
13836
|
-
prepareChunkSequence(session, chunk) {
|
|
13837
|
-
const items = materializationPlanItems(chunk);
|
|
13838
|
-
const planRows = this.readLane("apply", session.id, session.appliedPlanCursor, items.length);
|
|
13839
|
-
const expectedItems = planRows.map((row) => {
|
|
13840
|
-
const record = parseJson2(row.payload);
|
|
13841
|
-
return record.item;
|
|
13842
|
-
});
|
|
13843
|
-
const eventRows = this.readLane("event", session.id, session.appliedEventCursor, chunk.outbox.length);
|
|
13844
|
-
const expectedEvents = eventRows.map((row) => parseJson2(row.payload));
|
|
13845
|
-
const progress = correlateMaterializationChunk(session, chunk, items, expectedItems, expectedEvents);
|
|
13846
|
-
return {
|
|
13847
|
-
...progress,
|
|
13848
|
-
appliedPlanCursor: planRows.length === 0 ? session.appliedPlanCursor : workCursor(planRows[planRows.length - 1]),
|
|
13849
|
-
appliedEventCursor: eventRows.length === 0 ? session.appliedEventCursor : workCursor(eventRows[eventRows.length - 1])
|
|
13850
|
-
};
|
|
13851
|
-
}
|
|
13852
|
-
commitChunkSequence(session, progress) {
|
|
13853
|
-
session.appliedPlanCount = progress.appliedPlanCount;
|
|
13854
|
-
session.appliedOutboxCount = progress.appliedOutboxCount;
|
|
13855
|
-
session.appliedPlanCursor = progress.appliedPlanCursor;
|
|
13856
|
-
session.appliedEventCursor = progress.appliedEventCursor;
|
|
13857
|
-
}
|
|
13858
|
-
laneCounts(session) {
|
|
13859
|
-
return {
|
|
13860
|
-
apply: session.summary.apply,
|
|
13861
|
-
cardinality: session.summary.cardinality,
|
|
13862
|
-
event: session.summary.event
|
|
13863
|
-
};
|
|
14715
|
+
let after = "";
|
|
14716
|
+
while (true) {
|
|
14717
|
+
this.require(session.publicSession());
|
|
14718
|
+
const rows = query.all(session.workId, after, JSON.stringify(objectTypeIds), session.header.commit.projectId, pageRows);
|
|
14719
|
+
if (rows.length === 0)
|
|
14720
|
+
return;
|
|
14721
|
+
after = rows[rows.length - 1].record_key;
|
|
14722
|
+
yield rows.map((row) => parseJson2(row.payload).item);
|
|
14723
|
+
}
|
|
13864
14724
|
}
|
|
13865
14725
|
projectionCounts(session) {
|
|
14726
|
+
if (session.plan)
|
|
14727
|
+
return replacementPlanCounts(this.db, session.plan.versionId);
|
|
13866
14728
|
const summary = session.summary;
|
|
13867
14729
|
return {
|
|
13868
14730
|
objectsCreated: summary.objectsCreated,
|
|
@@ -13888,67 +14750,29 @@ class SqliteMaterializationSessions {
|
|
|
13888
14750
|
};
|
|
13889
14751
|
}
|
|
13890
14752
|
assertClassificationCoverage(session) {
|
|
13891
|
-
if (
|
|
13892
|
-
|
|
13893
|
-
const expectedObject = session.replacement.projectionKind === "object" ? 1 : 0;
|
|
13894
|
-
const mismatch = this.db.query(`
|
|
13895
|
-
WITH invalid AS (
|
|
13896
|
-
SELECT 1
|
|
13897
|
-
FROM ${SQLITE_REPLACEMENT_WORK_TABLE} AS expected
|
|
13898
|
-
WHERE expected.session_id = ? AND expected.diff_required = 1
|
|
13899
|
-
AND (expected.entity_kind = 'link' OR ? = 1)
|
|
13900
|
-
AND NOT EXISTS (
|
|
13901
|
-
SELECT 1 FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} AS actual
|
|
13902
|
-
WHERE actual.session_id = expected.session_id
|
|
13903
|
-
AND actual.unique_key = 'classification:' || expected.entity_kind || ':'
|
|
13904
|
-
|| expected.identity_key
|
|
13905
|
-
)
|
|
13906
|
-
UNION ALL
|
|
13907
|
-
SELECT 1
|
|
13908
|
-
FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} AS actual
|
|
13909
|
-
WHERE actual.session_id = ?
|
|
13910
|
-
-- A bounded range uses the existing (session_id, unique_key) unique index and scans
|
|
13911
|
-
-- classification entries only, instead of sorting the complete JSON work spool.
|
|
13912
|
-
AND actual.unique_key >= 'classification:'
|
|
13913
|
-
AND actual.unique_key < 'classification;'
|
|
13914
|
-
AND (
|
|
13915
|
-
actual.classification_entity_kind = 'point'
|
|
13916
|
-
OR NOT EXISTS (
|
|
13917
|
-
SELECT 1 FROM ${SQLITE_REPLACEMENT_WORK_TABLE} AS expected
|
|
13918
|
-
WHERE expected.session_id = actual.session_id
|
|
13919
|
-
AND expected.entity_kind = actual.classification_entity_kind
|
|
13920
|
-
AND expected.identity_key = actual.classification_identity_key
|
|
13921
|
-
AND expected.diff_required = 1
|
|
13922
|
-
AND (expected.entity_kind = 'link' OR ? = 1)
|
|
13923
|
-
)
|
|
13924
|
-
)
|
|
13925
|
-
)
|
|
13926
|
-
SELECT 1 FROM invalid LIMIT 1
|
|
13927
|
-
`).get(session.id, expectedObject, session.id, expectedObject);
|
|
13928
|
-
if (mismatch) {
|
|
13929
|
-
invalidCorrelation("Projection replacement classification coverage does not match its streamed state.");
|
|
13930
|
-
}
|
|
14753
|
+
if (session.plan)
|
|
14754
|
+
assertReplacementPlanCoverage(this.db, session.plan);
|
|
13931
14755
|
}
|
|
13932
|
-
readLane(
|
|
14756
|
+
readLane(session, lane, cursor, limit) {
|
|
13933
14757
|
if (limit === 0)
|
|
13934
14758
|
return [];
|
|
13935
14759
|
const selected = `record_key, major_order, minor_order, sort_one, sort_two, payload`;
|
|
13936
14760
|
if (!cursor) {
|
|
13937
|
-
return this.db.query(`SELECT ${selected} FROM ${
|
|
13938
|
-
WHERE
|
|
14761
|
+
return this.db.query(`SELECT ${selected} FROM ${session.workTable}
|
|
14762
|
+
WHERE work_id = ? AND lane = ?
|
|
13939
14763
|
ORDER BY major_order, minor_order, sort_one, sort_two, record_key
|
|
13940
|
-
LIMIT ?`).all(
|
|
14764
|
+
LIMIT ?`).all(session.workId, lane, limit);
|
|
13941
14765
|
}
|
|
13942
|
-
return this.db.query(`SELECT ${selected} FROM ${
|
|
13943
|
-
WHERE
|
|
14766
|
+
return this.db.query(`SELECT ${selected} FROM ${session.workTable}
|
|
14767
|
+
WHERE work_id = ? AND lane = ?
|
|
13944
14768
|
AND (major_order, minor_order, sort_one, sort_two, record_key) > (?, ?, ?, ?, ?)
|
|
13945
14769
|
ORDER BY major_order, minor_order, sort_one, sort_two, record_key
|
|
13946
|
-
LIMIT ?`).all(
|
|
14770
|
+
LIMIT ?`).all(session.workId, lane, cursor.majorOrder, cursor.minorOrder, cursor.sortOne, cursor.sortTwo, cursor.recordKey, limit);
|
|
13947
14771
|
}
|
|
13948
14772
|
ensureTables() {
|
|
13949
14773
|
this.db.run(`
|
|
13950
14774
|
CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE} (
|
|
13951
|
-
|
|
14775
|
+
work_id TEXT NOT NULL,
|
|
13952
14776
|
record_key TEXT NOT NULL,
|
|
13953
14777
|
unique_key TEXT NOT NULL,
|
|
13954
14778
|
kind TEXT NOT NULL,
|
|
@@ -13967,35 +14791,21 @@ class SqliteMaterializationSessions {
|
|
|
13967
14791
|
cardinality_target_type_id TEXT,
|
|
13968
14792
|
cardinality_target_primary_id TEXT,
|
|
13969
14793
|
payload TEXT NOT NULL CHECK (json_valid(payload)),
|
|
13970
|
-
PRIMARY KEY (
|
|
13971
|
-
UNIQUE (
|
|
14794
|
+
PRIMARY KEY (work_id, record_key),
|
|
14795
|
+
UNIQUE (work_id, unique_key)
|
|
13972
14796
|
);
|
|
13973
14797
|
CREATE INDEX IF NOT EXISTS idx_ontology_materialization_work_lane
|
|
13974
14798
|
ON ${SQLITE_MATERIALIZATION_WORK_TABLE}(
|
|
13975
|
-
|
|
13976
|
-
);
|
|
13977
|
-
CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
13978
|
-
session_id TEXT NOT NULL,
|
|
13979
|
-
entity_kind TEXT NOT NULL,
|
|
13980
|
-
identity_key TEXT NOT NULL,
|
|
13981
|
-
sort_key TEXT NOT NULL,
|
|
13982
|
-
diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
|
|
13983
|
-
PRIMARY KEY (session_id, entity_kind, identity_key)
|
|
13984
|
-
);
|
|
13985
|
-
CREATE INDEX IF NOT EXISTS idx_ontology_replacement_work_order
|
|
13986
|
-
ON ${SQLITE_REPLACEMENT_WORK_TABLE}(
|
|
13987
|
-
session_id, entity_kind, sort_key, identity_key
|
|
14799
|
+
work_id, lane, major_order, minor_order, sort_one, sort_two, record_key
|
|
13988
14800
|
);
|
|
13989
14801
|
`);
|
|
13990
14802
|
}
|
|
13991
14803
|
deleteWork(sessionId) {
|
|
13992
|
-
this.db.query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE
|
|
13993
|
-
this.db.query(`DELETE FROM ${SQLITE_REPLACEMENT_WORK_TABLE} WHERE session_id = ?`).run(sessionId);
|
|
14804
|
+
this.db.query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE work_id = ?`).run(sessionId);
|
|
13994
14805
|
}
|
|
13995
14806
|
dropWorkTables() {
|
|
13996
14807
|
this.db.run(`
|
|
13997
14808
|
DROP TABLE IF EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE};
|
|
13998
|
-
DROP TABLE IF EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE};
|
|
13999
14809
|
`);
|
|
14000
14810
|
}
|
|
14001
14811
|
}
|
|
@@ -14010,9 +14820,6 @@ function workCursor(row) {
|
|
|
14010
14820
|
}
|
|
14011
14821
|
function emptyWorkSummary() {
|
|
14012
14822
|
return {
|
|
14013
|
-
apply: 0,
|
|
14014
|
-
cardinality: 0,
|
|
14015
|
-
event: 0,
|
|
14016
14823
|
objectClassifications: 0,
|
|
14017
14824
|
linkClassifications: 0,
|
|
14018
14825
|
pointClassifications: 0,
|
|
@@ -14027,9 +14834,7 @@ function emptyWorkSummary() {
|
|
|
14027
14834
|
latestObjectsChanged: 0
|
|
14028
14835
|
};
|
|
14029
14836
|
}
|
|
14030
|
-
function recordSummary(summary, record
|
|
14031
|
-
if (columns.lane !== "none")
|
|
14032
|
-
summary[columns.lane] += 1;
|
|
14837
|
+
function recordSummary(summary, record) {
|
|
14033
14838
|
if (record.kind === "classification") {
|
|
14034
14839
|
if (record.entityKind === "object")
|
|
14035
14840
|
summary.objectClassifications += 1;
|
|
@@ -14078,11 +14883,13 @@ function recordSummary(summary, record, columns) {
|
|
|
14078
14883
|
// src/ontology-storage/materialization-writer.ts
|
|
14079
14884
|
import {
|
|
14080
14885
|
linkRefKey as linkRefKey2,
|
|
14081
|
-
MaterializationConflictError as
|
|
14082
|
-
objectRefKey as
|
|
14886
|
+
MaterializationConflictError as MaterializationConflictError6,
|
|
14887
|
+
objectRefKey as objectRefKey3,
|
|
14083
14888
|
telemetryPointKey
|
|
14084
14889
|
} from "@sixb/core/internal/materialization";
|
|
14085
|
-
import {
|
|
14890
|
+
import {
|
|
14891
|
+
effectiveConflict
|
|
14892
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
14086
14893
|
function linkOverrideConflictMessage(identityKind) {
|
|
14087
14894
|
return identityKind === "edge" ? "Expected link edge override changed." : "Expected link slot override changed.";
|
|
14088
14895
|
}
|
|
@@ -14178,7 +14985,7 @@ class SqliteMaterializationWriter {
|
|
|
14178
14985
|
for (const item of chunk.effective.objectDeletes) {
|
|
14179
14986
|
requireChanges(this.db.query(`DELETE FROM objects
|
|
14180
14987
|
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14181
|
-
AND version = ? AND last_commit_id = ?`).run(projectId, item.ref.objectTypeId, item.ref.primaryId, item.expected.version, item.expected.lastCommitId).changes, "effective-state", `Expected object ${
|
|
14988
|
+
AND version = ? AND last_commit_id = ?`).run(projectId, item.ref.objectTypeId, item.ref.primaryId, item.expected.version, item.expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey3(item.ref)} changed.`);
|
|
14182
14989
|
}
|
|
14183
14990
|
for (const item of chunk.effective.objectUpserts)
|
|
14184
14991
|
this.applyObject(projectId, item);
|
|
@@ -14197,7 +15004,7 @@ class SqliteMaterializationWriter {
|
|
|
14197
15004
|
`).run(projectId, row.ref.objectTypeId, row.ref.primaryId, canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId);
|
|
14198
15005
|
} catch (error) {
|
|
14199
15006
|
if (isSqliteConstraintError(error)) {
|
|
14200
|
-
throw effectiveConflict(`Expected object ${
|
|
15007
|
+
throw effectiveConflict(`Expected object ${objectRefKey3(row.ref)} to be absent.`);
|
|
14201
15008
|
}
|
|
14202
15009
|
throw error;
|
|
14203
15010
|
}
|
|
@@ -14209,7 +15016,7 @@ class SqliteMaterializationWriter {
|
|
|
14209
15016
|
last_commit_id = ?
|
|
14210
15017
|
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14211
15018
|
AND version = ? AND last_commit_id = ?
|
|
14212
|
-
`).run(canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId, projectId, row.ref.objectTypeId, row.ref.primaryId, expected.version, expected.lastCommitId).changes, "effective-state", `Expected object ${
|
|
15019
|
+
`).run(canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId, projectId, row.ref.objectTypeId, row.ref.primaryId, expected.version, expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey3(row.ref)} changed.`);
|
|
14213
15020
|
}
|
|
14214
15021
|
applyLink(projectId, item) {
|
|
14215
15022
|
const { row, expected } = item;
|
|
@@ -14250,7 +15057,7 @@ class SqliteMaterializationWriter {
|
|
|
14250
15057
|
`).run(projectId, point.series.object.objectTypeId, point.series.object.primaryId, point.series.propertyId, canonicalJson(point.value), point.unit ?? null, point.at, point.lastCommitId);
|
|
14251
15058
|
} catch (error) {
|
|
14252
15059
|
if (isSqliteConstraintError(error)) {
|
|
14253
|
-
throw new
|
|
15060
|
+
throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey(point.series, point.at)} changed.`);
|
|
14254
15061
|
}
|
|
14255
15062
|
throw error;
|
|
14256
15063
|
}
|
|
@@ -14267,19 +15074,18 @@ class SqliteMaterializationWriter {
|
|
|
14267
15074
|
applyOutbox(projectId, commitId, chunk) {
|
|
14268
15075
|
const insert = this.db.query(`
|
|
14269
15076
|
INSERT INTO ontology_outbox (
|
|
14270
|
-
project_id, id, commit_id, commit_ordinal,
|
|
15077
|
+
project_id, id, commit_id, commit_ordinal, event,
|
|
14271
15078
|
available_at, attempts, lease_id, lease_expires_at,
|
|
14272
15079
|
published_at, last_failure, created_at
|
|
14273
15080
|
) VALUES (?, ?, ?, ?, json(?), ?, 0, NULL, NULL, NULL, NULL, ?)
|
|
14274
15081
|
`);
|
|
14275
15082
|
for (const item of chunk.outbox) {
|
|
14276
|
-
assertTimestamp(item.availableAt, "Outbox availableAt");
|
|
14277
15083
|
assertTimestamp(item.createdAt, "Outbox createdAt");
|
|
14278
15084
|
try {
|
|
14279
|
-
insert.run(projectId, item.
|
|
15085
|
+
insert.run(projectId, item.id, commitId, item.commitOrdinal, canonicalJson(item.event), item.createdAt, item.createdAt);
|
|
14280
15086
|
} catch (error) {
|
|
14281
15087
|
if (isSqliteConstraintError(error)) {
|
|
14282
|
-
throw effectiveConflict(`Duplicate outbox event '${item.
|
|
15088
|
+
throw effectiveConflict(`Duplicate outbox event '${item.id}'.`);
|
|
14283
15089
|
}
|
|
14284
15090
|
throw error;
|
|
14285
15091
|
}
|
|
@@ -14288,6 +15094,48 @@ class SqliteMaterializationWriter {
|
|
|
14288
15094
|
}
|
|
14289
15095
|
|
|
14290
15096
|
// src/ontology-storage/materializations.ts
|
|
15097
|
+
var APPLY_PAGE_ROWS = 1000;
|
|
15098
|
+
function touchAppliedWorkStatement(workTable) {
|
|
15099
|
+
return `
|
|
15100
|
+
INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
|
|
15101
|
+
SELECT DISTINCT $projectId, $commitId, kind, CASE kind
|
|
15102
|
+
WHEN 'object' THEN
|
|
15103
|
+
json_array(json_extract(ref, '$.objectTypeId'), json_extract(ref, '$.primaryId'))
|
|
15104
|
+
WHEN 'scope' THEN json_array(
|
|
15105
|
+
json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
|
|
15106
|
+
json_extract(ref, '$.linkId')
|
|
15107
|
+
)
|
|
15108
|
+
ELSE json_array(
|
|
15109
|
+
json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
|
|
15110
|
+
json_extract(ref, '$.linkId'), json_extract(ref, '$.target.objectTypeId'),
|
|
15111
|
+
json_extract(ref, '$.target.primaryId')
|
|
15112
|
+
)
|
|
15113
|
+
END
|
|
15114
|
+
FROM (
|
|
15115
|
+
SELECT CASE json_extract(payload, '$.item.kind')
|
|
15116
|
+
WHEN 'object-upsert' THEN 'object'
|
|
15117
|
+
WHEN 'object-delete' THEN 'object'
|
|
15118
|
+
WHEN 'object-override-upsert' THEN 'object'
|
|
15119
|
+
WHEN 'object-override-delete' THEN 'object'
|
|
15120
|
+
WHEN 'point-upsert' THEN 'object'
|
|
15121
|
+
WHEN 'link-upsert' THEN 'link'
|
|
15122
|
+
WHEN 'link-delete' THEN 'link'
|
|
15123
|
+
WHEN 'link-override-upsert' THEN 'link'
|
|
15124
|
+
WHEN 'link-override-delete' THEN 'link'
|
|
15125
|
+
WHEN 'link-slot-override-upsert' THEN 'scope'
|
|
15126
|
+
WHEN 'link-slot-override-delete' THEN 'scope'
|
|
15127
|
+
END AS kind,
|
|
15128
|
+
CASE json_extract(payload, '$.item.kind')
|
|
15129
|
+
WHEN 'object-upsert' THEN json_extract(payload, '$.item.value.row.ref')
|
|
15130
|
+
WHEN 'link-upsert' THEN json_extract(payload, '$.item.value.row.ref')
|
|
15131
|
+
WHEN 'point-upsert' THEN json_extract(payload, '$.item.value.point.series.object')
|
|
15132
|
+
ELSE json_extract(payload, '$.item.value.ref')
|
|
15133
|
+
END AS ref
|
|
15134
|
+
FROM ${workTable}
|
|
15135
|
+
WHERE work_id = $workId AND lane = 'apply'
|
|
15136
|
+
)`;
|
|
15137
|
+
}
|
|
15138
|
+
|
|
14291
15139
|
class SqliteOntologyMaterializationStorage {
|
|
14292
15140
|
db;
|
|
14293
15141
|
sessions;
|
|
@@ -14300,12 +15148,12 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14300
15148
|
assertVectorSession(session, projectId, commitId) {
|
|
14301
15149
|
const active = this.sessions.require(session);
|
|
14302
15150
|
if (active.header.commit.projectId !== projectId || commitId !== undefined && active.header.commit.id !== commitId) {
|
|
14303
|
-
throw new
|
|
15151
|
+
throw new MaterializationValidationError3("Vector mutation does not belong to this materialization session.");
|
|
14304
15152
|
}
|
|
14305
15153
|
}
|
|
14306
15154
|
async begin(input) {
|
|
14307
15155
|
assertMaterializationHeader(input);
|
|
14308
|
-
const session = this.sessions.create(input);
|
|
15156
|
+
const session = this.sessions.create(input, input.plan ? boundReplacementPlan(this.db, input) : null);
|
|
14309
15157
|
try {
|
|
14310
15158
|
this.assertCommitAbsent(input);
|
|
14311
15159
|
const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId);
|
|
@@ -14324,7 +15172,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14324
15172
|
for (const expected of input.expected.points) {
|
|
14325
15173
|
const point = reader.exactPoint(expected.series, expected.at);
|
|
14326
15174
|
if ((point?.lastCommitId ?? null) !== expected.lastCommitId) {
|
|
14327
|
-
throw new
|
|
15175
|
+
throw new MaterializationConflictError7("timeseries-point", `Telemetry point ${telemetryPointKey2(expected.series, expected.at)} changed.`);
|
|
14328
15176
|
}
|
|
14329
15177
|
}
|
|
14330
15178
|
return session.publicSession();
|
|
@@ -14335,11 +15183,11 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14335
15183
|
}
|
|
14336
15184
|
async* streamState(input) {
|
|
14337
15185
|
const session = this.sessions.require(input.session);
|
|
14338
|
-
|
|
15186
|
+
assertPageRows2(input.pageRows);
|
|
14339
15187
|
const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
|
|
14340
15188
|
for await (const request of input.requests) {
|
|
14341
15189
|
this.sessions.require(input.session);
|
|
14342
|
-
const objects = uniqueSorted(request.objects,
|
|
15190
|
+
const objects = uniqueSorted(request.objects, objectRefKey4, objectRefSortKey);
|
|
14343
15191
|
for (let offset = 0;offset < objects.length; offset += input.pageRows) {
|
|
14344
15192
|
const page = reader.objectStates(objects.slice(offset, offset + input.pageRows));
|
|
14345
15193
|
yield { objects: page, links: [], linkScopes: [], points: [] };
|
|
@@ -14375,103 +15223,50 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14375
15223
|
}
|
|
14376
15224
|
}
|
|
14377
15225
|
}
|
|
14378
|
-
async* streamSourceReplacementState(input) {
|
|
14379
|
-
const session = this.sessions.require(input.session);
|
|
14380
|
-
assertPageRows(input.pageRows);
|
|
14381
|
-
const replacement = this.requireReplacement(session, input);
|
|
14382
|
-
const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
|
|
14383
|
-
if (input.entityKind === "object") {
|
|
14384
|
-
if (replacement.projectionKind !== "object") {
|
|
14385
|
-
throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
|
|
14386
|
-
}
|
|
14387
|
-
if (replacement.objectStreamStarted) {
|
|
14388
|
-
throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
|
|
14389
|
-
}
|
|
14390
|
-
replacement.objectStreamStarted = true;
|
|
14391
|
-
for (const identities of reader.replacementIdentities({
|
|
14392
|
-
sessionId: session.id,
|
|
14393
|
-
sourceId: replacement.sourceId,
|
|
14394
|
-
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
14395
|
-
previousMaterializationId: replacement.previousMaterializationId,
|
|
14396
|
-
incremental: replacement.incremental,
|
|
14397
|
-
kind: "object",
|
|
14398
|
-
pageRows: input.pageRows
|
|
14399
|
-
})) {
|
|
14400
|
-
this.sessions.require(input.session);
|
|
14401
|
-
const refs = identities.map((identity) => {
|
|
14402
|
-
if (identity.kind !== "object") {
|
|
14403
|
-
invalidCorrelation2("Object replacement returned a link identity.");
|
|
14404
|
-
}
|
|
14405
|
-
return identity.ref;
|
|
14406
|
-
});
|
|
14407
|
-
const objects = reader.replacementObjectStates(replacement.sourceId, replacement.candidateMaterializationId, refs);
|
|
14408
|
-
yield { objects, links: [] };
|
|
14409
|
-
}
|
|
14410
|
-
this.sessions.require(input.session);
|
|
14411
|
-
replacement.objectStreamCompleted = true;
|
|
14412
|
-
return;
|
|
14413
|
-
}
|
|
14414
|
-
if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
|
|
14415
|
-
throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
|
|
14416
|
-
}
|
|
14417
|
-
if (replacement.linkStreamStarted) {
|
|
14418
|
-
throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
|
|
14419
|
-
}
|
|
14420
|
-
replacement.linkStreamStarted = true;
|
|
14421
|
-
const materializationIds = [
|
|
14422
|
-
replacement.candidateMaterializationId,
|
|
14423
|
-
...replacement.previousMaterializationId ? [replacement.previousMaterializationId] : []
|
|
14424
|
-
];
|
|
14425
|
-
for (const identities of reader.replacementIdentities({
|
|
14426
|
-
sessionId: session.id,
|
|
14427
|
-
sourceId: replacement.sourceId,
|
|
14428
|
-
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
14429
|
-
previousMaterializationId: replacement.previousMaterializationId,
|
|
14430
|
-
incremental: replacement.incremental,
|
|
14431
|
-
kind: "link",
|
|
14432
|
-
pageRows: input.pageRows
|
|
14433
|
-
})) {
|
|
14434
|
-
this.sessions.require(input.session);
|
|
14435
|
-
const linkIdentities = identities.map((identity) => {
|
|
14436
|
-
if (identity.kind !== "link") {
|
|
14437
|
-
invalidCorrelation2("Link replacement returned an object identity.");
|
|
14438
|
-
}
|
|
14439
|
-
return identity;
|
|
14440
|
-
});
|
|
14441
|
-
const links = reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
|
|
14442
|
-
yield { objects: [], links };
|
|
14443
|
-
}
|
|
14444
|
-
this.sessions.require(input.session);
|
|
14445
|
-
replacement.linkStreamCompleted = true;
|
|
14446
|
-
}
|
|
14447
15226
|
async stageWork(input) {
|
|
14448
15227
|
this.sessions.stage(input);
|
|
14449
15228
|
await yieldSqliteEventLoop();
|
|
14450
15229
|
}
|
|
14451
|
-
|
|
14452
|
-
assertPageRows(input.pageRows);
|
|
14453
|
-
return this.sessions.stream(input);
|
|
14454
|
-
}
|
|
14455
|
-
async readObjectExistence(input) {
|
|
15230
|
+
async* streamVectorChanges(input) {
|
|
14456
15231
|
const session = this.sessions.require(input.session);
|
|
14457
|
-
|
|
15232
|
+
assertPageRows2(input.pageRows);
|
|
15233
|
+
beginMaterializationVectorChanges(session);
|
|
15234
|
+
for (const items of this.sessions.vectorChangePages(session, input.objectTypeIds, input.pageRows)) {
|
|
15235
|
+
yield { items };
|
|
15236
|
+
}
|
|
14458
15237
|
}
|
|
14459
|
-
async
|
|
15238
|
+
async apply(input) {
|
|
14460
15239
|
const session = this.sessions.require(input.session);
|
|
15240
|
+
beginMaterializationApply(session);
|
|
14461
15241
|
const { commit } = session.header;
|
|
14462
|
-
|
|
14463
|
-
const
|
|
14464
|
-
|
|
15242
|
+
const cardinality = createCardinalityValidator();
|
|
15243
|
+
for (const records of this.sessions.lanePages(session, "cardinality", APPLY_PAGE_ROWS)) {
|
|
15244
|
+
for (const record of records)
|
|
15245
|
+
cardinality.accept(record);
|
|
15246
|
+
}
|
|
15247
|
+
let eventCount = 0;
|
|
15248
|
+
this.db.run("SAVEPOINT sixb_ontology_apply");
|
|
14465
15249
|
try {
|
|
14466
|
-
this.
|
|
14467
|
-
|
|
14468
|
-
|
|
15250
|
+
for (const records of this.sessions.lanePages(session, "apply", APPLY_PAGE_ROWS)) {
|
|
15251
|
+
for (const phase of splitByPhase(records)) {
|
|
15252
|
+
this.writer.apply(commit.projectId, commit.id, materializationPlanChunk(phase.map((record) => record.item)));
|
|
15253
|
+
}
|
|
15254
|
+
await yieldSqliteEventLoop();
|
|
15255
|
+
}
|
|
15256
|
+
for (const records of this.sessions.lanePages(session, "event", APPLY_PAGE_ROWS)) {
|
|
15257
|
+
const outbox = records.map((record) => materializationOutboxWrite(commit, record.draft, eventCount++));
|
|
15258
|
+
this.writer.apply(commit.projectId, commit.id, materializationPlanChunk([], outbox));
|
|
15259
|
+
await yieldSqliteEventLoop();
|
|
15260
|
+
}
|
|
15261
|
+
this.db.query(touchAppliedWorkStatement(session.workTable)).run({ $projectId: commit.projectId, $commitId: commit.id, $workId: session.workId });
|
|
15262
|
+
this.db.run("RELEASE SAVEPOINT sixb_ontology_apply");
|
|
14469
15263
|
} catch (error) {
|
|
14470
|
-
this.db.run("ROLLBACK TO SAVEPOINT
|
|
14471
|
-
this.db.run("RELEASE SAVEPOINT
|
|
15264
|
+
this.db.run("ROLLBACK TO SAVEPOINT sixb_ontology_apply");
|
|
15265
|
+
this.db.run("RELEASE SAVEPOINT sixb_ontology_apply");
|
|
14472
15266
|
throw error;
|
|
14473
15267
|
}
|
|
14474
|
-
|
|
15268
|
+
session.appliedEventCount = eventCount;
|
|
15269
|
+
return { eventCount };
|
|
14475
15270
|
}
|
|
14476
15271
|
async finalize(input) {
|
|
14477
15272
|
const session = this.sessions.require(input.session);
|
|
@@ -14487,41 +15282,6 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14487
15282
|
deactivateSessions() {
|
|
14488
15283
|
this.sessions.deactivateAll();
|
|
14489
15284
|
}
|
|
14490
|
-
requireReplacement(session, input) {
|
|
14491
|
-
if (session.replacement) {
|
|
14492
|
-
if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
|
|
14493
|
-
throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
|
|
14494
|
-
}
|
|
14495
|
-
return session.replacement;
|
|
14496
|
-
}
|
|
14497
|
-
const projectId = session.header.commit.projectId;
|
|
14498
|
-
const candidate = this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId);
|
|
14499
|
-
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
14500
|
-
throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
|
|
14501
|
-
}
|
|
14502
|
-
assertProjectionExecution(this.db, {
|
|
14503
|
-
projectId,
|
|
14504
|
-
sourceId: input.source.projectionId,
|
|
14505
|
-
projectionRunId: candidate.projection_run_id,
|
|
14506
|
-
executionToken: candidate.execution_token
|
|
14507
|
-
});
|
|
14508
|
-
const previous = this.getActiveSource(projectId, input.source.projectionId);
|
|
14509
|
-
if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
|
|
14510
|
-
throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
|
|
14511
|
-
}
|
|
14512
|
-
session.replacement = {
|
|
14513
|
-
sourceId: input.source.projectionId,
|
|
14514
|
-
candidateMaterializationId: input.candidateMaterializationId,
|
|
14515
|
-
previousMaterializationId: previous?.materialization_id ?? null,
|
|
14516
|
-
incremental: candidate.base_materialization_id !== null,
|
|
14517
|
-
projectionKind: candidate.projection_kind,
|
|
14518
|
-
objectStreamStarted: false,
|
|
14519
|
-
objectStreamCompleted: false,
|
|
14520
|
-
linkStreamStarted: false,
|
|
14521
|
-
linkStreamCompleted: false
|
|
14522
|
-
};
|
|
14523
|
-
return session.replacement;
|
|
14524
|
-
}
|
|
14525
15285
|
assertCommitAbsent(header) {
|
|
14526
15286
|
const origin = originColumns(header.commit.origin);
|
|
14527
15287
|
const duplicate = this.db.query(`
|
|
@@ -14542,25 +15302,23 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14542
15302
|
if (!duplicate)
|
|
14543
15303
|
return;
|
|
14544
15304
|
if (duplicate.duplicate === "origin") {
|
|
14545
|
-
throw new
|
|
15305
|
+
throw new MaterializationConflictError7("run-correlation", "Ontology commit origin already has an authoritative commit.");
|
|
14546
15306
|
}
|
|
14547
|
-
throw new
|
|
15307
|
+
throw new MaterializationConflictError7("idempotency", duplicate.duplicate === "id" ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
|
|
14548
15308
|
}
|
|
14549
15309
|
assertSource(expected, projectId) {
|
|
14550
15310
|
const active = this.getActiveSource(projectId, expected.source.projectionId);
|
|
14551
15311
|
if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
|
|
14552
|
-
throw new
|
|
15312
|
+
throw new MaterializationConflictError7("projection-fence", `Source '${expected.source.projectionId}' changed.`);
|
|
14553
15313
|
}
|
|
14554
15314
|
}
|
|
14555
15315
|
async assertFinalization(session, input) {
|
|
14556
15316
|
const { commit } = session.header;
|
|
14557
15317
|
const { result } = input.finalization;
|
|
14558
15318
|
assertMaterializationFinalizationCorrelation(session, input);
|
|
14559
|
-
const laneCounts = this.sessions.laneCounts(session);
|
|
14560
|
-
assertMaterializationLaneCompletion(session, laneCounts);
|
|
14561
15319
|
this.assertFinalCardinality(session);
|
|
14562
15320
|
await yieldSqliteEventLoop();
|
|
14563
|
-
const eventCount =
|
|
15321
|
+
const eventCount = session.appliedEventCount;
|
|
14564
15322
|
const outbox = this.db.query(`
|
|
14565
15323
|
SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum, MAX(commit_ordinal) AS maximum
|
|
14566
15324
|
FROM ontology_outbox WHERE project_id = ? AND commit_id = ?
|
|
@@ -14603,8 +15361,8 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14603
15361
|
THEN cardinality_target_type_id END) AS expected_target_type_id,
|
|
14604
15362
|
MAX(CASE WHEN cardinality_occupied = 1
|
|
14605
15363
|
THEN cardinality_target_primary_id END) AS expected_target_id
|
|
14606
|
-
FROM ${
|
|
14607
|
-
WHERE
|
|
15364
|
+
FROM ${session.workTable}
|
|
15365
|
+
WHERE work_id = ? AND kind = 'cardinality'
|
|
14608
15366
|
AND cardinality_view = 'effective'
|
|
14609
15367
|
GROUP BY sort_one, cardinality_source_type_id,
|
|
14610
15368
|
cardinality_source_primary_id, cardinality_link_id
|
|
@@ -14624,16 +15382,11 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14624
15382
|
GROUP BY scopes.scope_sort_key, scopes.expected_count,
|
|
14625
15383
|
scopes.expected_target_type_id, scopes.expected_target_id
|
|
14626
15384
|
)
|
|
14627
|
-
SELECT
|
|
15385
|
+
SELECT 1 AS mismatch
|
|
14628
15386
|
FROM validated
|
|
14629
|
-
WHERE expected_count
|
|
14630
|
-
OR actual_count <> expected_count
|
|
14631
|
-
OR matching_count <> expected_count
|
|
15387
|
+
WHERE actual_count <> expected_count OR matching_count <> expected_count
|
|
14632
15388
|
LIMIT 1
|
|
14633
|
-
`).get(session.
|
|
14634
|
-
if (violation?.reason === "duplicate") {
|
|
14635
|
-
invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
|
|
14636
|
-
}
|
|
15389
|
+
`).get(session.workId, session.header.commit.projectId);
|
|
14637
15390
|
if (violation) {
|
|
14638
15391
|
invalidCorrelation2("Materialization cardinality work does not match the final effective link scope.");
|
|
14639
15392
|
}
|
|
@@ -14643,7 +15396,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14643
15396
|
assertSourceActivationCorrelation(session, activation);
|
|
14644
15397
|
const candidate = this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId);
|
|
14645
15398
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
14646
|
-
throw new
|
|
15399
|
+
throw new MaterializationConflictError7("source-materialization", "Source activation candidate is missing or is not ready.");
|
|
14647
15400
|
}
|
|
14648
15401
|
assertProjectionExecution(this.db, {
|
|
14649
15402
|
projectId: commit.projectId,
|
|
@@ -14655,7 +15408,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14655
15408
|
invalidCorrelation2("Source activation does not match its ready candidate identity.");
|
|
14656
15409
|
}
|
|
14657
15410
|
this.assertSource(activation.expected, commit.projectId);
|
|
14658
|
-
activateSourceRoots(this.db, commit.projectId, candidate, activation);
|
|
15411
|
+
activateSourceRoots(this.db, commit.projectId, commit.id, candidate, activation);
|
|
14659
15412
|
const previous = this.getActiveSource(commit.projectId, activation.source.projectionId);
|
|
14660
15413
|
if (previous) {
|
|
14661
15414
|
assertPinnedDatasetWatermark({
|
|
@@ -14688,16 +15441,14 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14688
15441
|
this.db.query(`
|
|
14689
15442
|
INSERT INTO ontology_commits (
|
|
14690
15443
|
project_id, id, idempotency_key, request_hash, execution_id,
|
|
14691
|
-
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
15444
|
+
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
14692
15445
|
ontology_revision, projection_revision, ownership_hash,
|
|
14693
15446
|
intent, result, committed_at
|
|
14694
|
-
) VALUES (
|
|
14695
|
-
|
|
14696
|
-
)
|
|
14697
|
-
`).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy), canonicalJson(commit.executor), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
|
|
15447
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, json(?), ?, ?, ?, json(?), json(?), ?)
|
|
15448
|
+
`).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
|
|
14698
15449
|
} catch (error) {
|
|
14699
15450
|
if (isSqliteConstraintError(error)) {
|
|
14700
|
-
throw new
|
|
15451
|
+
throw new MaterializationConflictError7("idempotency", "Ontology commit identity already exists.");
|
|
14701
15452
|
}
|
|
14702
15453
|
throw error;
|
|
14703
15454
|
}
|
|
@@ -14713,13 +15464,27 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14713
15464
|
WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).get(projectId, sourceId, materializationId);
|
|
14714
15465
|
}
|
|
14715
15466
|
}
|
|
15467
|
+
function splitByPhase(records) {
|
|
15468
|
+
const phases = [];
|
|
15469
|
+
for (const record of records) {
|
|
15470
|
+
const current = phases[phases.length - 1];
|
|
15471
|
+
if (current && current[0].applyPhase === record.applyPhase)
|
|
15472
|
+
current.push(record);
|
|
15473
|
+
else
|
|
15474
|
+
phases.push([record]);
|
|
15475
|
+
}
|
|
15476
|
+
return phases;
|
|
15477
|
+
}
|
|
14716
15478
|
|
|
14717
15479
|
// src/ontology-storage/outbox.ts
|
|
14718
15480
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
14719
15481
|
import {
|
|
14720
|
-
MaterializationConflictError as
|
|
14721
|
-
MaterializationValidationError as
|
|
15482
|
+
MaterializationConflictError as MaterializationConflictError8,
|
|
15483
|
+
MaterializationValidationError as MaterializationValidationError4
|
|
14722
15484
|
} from "@sixb/core/internal/materialization";
|
|
15485
|
+
import {
|
|
15486
|
+
claimedOutboxRows
|
|
15487
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
14723
15488
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
|
|
14724
15489
|
class SqliteOntologyOutboxStorage {
|
|
14725
15490
|
db;
|
|
@@ -14736,7 +15501,7 @@ class SqliteOntologyOutboxStorage {
|
|
|
14736
15501
|
const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
|
|
14737
15502
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
14738
15503
|
if (leaseExpiry <= now) {
|
|
14739
|
-
throw new
|
|
15504
|
+
throw new MaterializationValidationError4("Ontology outbox lease expiry must be later than the claim time.");
|
|
14740
15505
|
}
|
|
14741
15506
|
const eligible = this.db.query(`SELECT CAST(rowid AS TEXT) AS rowId
|
|
14742
15507
|
FROM ontology_outbox INDEXED BY idx_ontology_outbox_claim
|
|
@@ -14755,15 +15520,19 @@ class SqliteOntologyOutboxStorage {
|
|
|
14755
15520
|
UPDATE ontology_outbox
|
|
14756
15521
|
SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
|
|
14757
15522
|
WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
|
|
14758
|
-
RETURNING
|
|
14759
|
-
published_at, last_failure, created_at
|
|
15523
|
+
RETURNING id, commit_id, commit_ordinal, event, available_at, attempts, lease_id,
|
|
15524
|
+
lease_expires_at, published_at, last_failure, created_at
|
|
14760
15525
|
`).all(input.leaseId, input.leaseExpiresAt, JSON.stringify(selected.map((row) => row.rowId)));
|
|
14761
|
-
|
|
14762
|
-
|
|
14763
|
-
|
|
14764
|
-
|
|
14765
|
-
|
|
14766
|
-
|
|
15526
|
+
const contexts = this.db.query(`SELECT commits.id AS commit_id, commits.committed_at AS commit_committed_at,
|
|
15527
|
+
commits.origin AS commit_origin, executions.*
|
|
15528
|
+
FROM json_each(?) AS requested
|
|
15529
|
+
CROSS JOIN ontology_commits AS commits
|
|
15530
|
+
ON commits.project_id = ? AND commits.id = requested.value
|
|
15531
|
+
CROSS JOIN executions
|
|
15532
|
+
ON executions.project_id = commits.project_id AND executions.id = commits.execution_id`).all(JSON.stringify([...new Set(rows.map((row) => row.commit_id))]), input.projectId);
|
|
15533
|
+
const contextOf = new Map(contexts.map((row) => [row.commit_id, outboxEventContext(input.projectId, row)]));
|
|
15534
|
+
const drafts = rows.map(claimedOutboxDraft).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
15535
|
+
return claimedOutboxRows(drafts, (commitId) => contextOf.get(commitId));
|
|
14767
15536
|
});
|
|
14768
15537
|
}
|
|
14769
15538
|
async markPublished(input) {
|
|
@@ -14856,12 +15625,23 @@ class SqliteOntologyOutboxStorage {
|
|
|
14856
15625
|
});
|
|
14857
15626
|
}
|
|
14858
15627
|
}
|
|
15628
|
+
function outboxEventContext(projectId, row) {
|
|
15629
|
+
return {
|
|
15630
|
+
commit: {
|
|
15631
|
+
projectId,
|
|
15632
|
+
id: row.commit_id,
|
|
15633
|
+
committedAt: row.commit_committed_at,
|
|
15634
|
+
origin: parseJson2(row.commit_origin)
|
|
15635
|
+
},
|
|
15636
|
+
execution: sqliteExecutionRecord(row)
|
|
15637
|
+
};
|
|
15638
|
+
}
|
|
14859
15639
|
function validateLeaseIds(ids) {
|
|
14860
15640
|
const seen = new Set;
|
|
14861
15641
|
for (const id of ids) {
|
|
14862
15642
|
assertNonblank(id, "Ontology outbox event id");
|
|
14863
15643
|
if (seen.has(id)) {
|
|
14864
|
-
throw new
|
|
15644
|
+
throw new MaterializationValidationError4(`Ontology outbox lease batch repeats event '${id}'.`);
|
|
14865
15645
|
}
|
|
14866
15646
|
seen.add(id);
|
|
14867
15647
|
}
|
|
@@ -14872,11 +15652,11 @@ function assertLeaseInput(input) {
|
|
|
14872
15652
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
14873
15653
|
}
|
|
14874
15654
|
function leaseConflict() {
|
|
14875
|
-
return new
|
|
15655
|
+
return new MaterializationConflictError8("outbox-lease", "Ontology outbox lease does not match.");
|
|
14876
15656
|
}
|
|
14877
15657
|
|
|
14878
15658
|
// src/ontology-storage/sources.ts
|
|
14879
|
-
import { MaterializationValidationError as
|
|
15659
|
+
import { MaterializationValidationError as MaterializationValidationError5 } from "@sixb/core/internal/materialization";
|
|
14880
15660
|
import {
|
|
14881
15661
|
assertSourceBeginInput as assertBeginInput,
|
|
14882
15662
|
assertSourceExecutionIdentity as assertExecutionIdentity,
|
|
@@ -14894,13 +15674,16 @@ import {
|
|
|
14894
15674
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
14895
15675
|
|
|
14896
15676
|
// src/ontology-storage/source-cleanup.ts
|
|
15677
|
+
var PLAN_PURGED = `NOT EXISTS (
|
|
15678
|
+
SELECT 1 FROM ontology_replacement_plans AS plans WHERE plans.version_id = versions.version_id
|
|
15679
|
+
)`;
|
|
14897
15680
|
function purgeAbandonedSourceVersions(db, input) {
|
|
14898
15681
|
let remaining = input.limit;
|
|
14899
15682
|
let rowsDeleted = 0;
|
|
14900
15683
|
let materializationsDeleted = 0;
|
|
14901
15684
|
const nextCandidate = db.query(`
|
|
14902
|
-
SELECT version_id FROM ontology_sources
|
|
14903
|
-
WHERE project_id = ? AND status = 'abandoned'
|
|
15685
|
+
SELECT version_id FROM ontology_sources AS versions
|
|
15686
|
+
WHERE project_id = ? AND status = 'abandoned' AND ${PLAN_PURGED}
|
|
14904
15687
|
ORDER BY terminal_at, source_id, materialization_id LIMIT 1
|
|
14905
15688
|
`);
|
|
14906
15689
|
const deleteRows = db.query(`DELETE FROM ontology_source_rows WHERE rowid IN (
|
|
@@ -14951,15 +15734,20 @@ function cleanupSourceVersions(db, input) {
|
|
|
14951
15734
|
WHERE ${retired}
|
|
14952
15735
|
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
|
|
14953
15736
|
ORDER BY roots.retired_at, roots.id LIMIT ?
|
|
14954
|
-
)`).
|
|
14955
|
-
|
|
14956
|
-
|
|
15737
|
+
) RETURNING version_id`).all(input.projectId, input.terminalBefore, remaining);
|
|
15738
|
+
const counts = new Map;
|
|
15739
|
+
for (const root of removed)
|
|
15740
|
+
counts.set(root.version_id, (counts.get(root.version_id) ?? 0) + 1);
|
|
15741
|
+
const drain = db.query("UPDATE ontology_sources SET root_count = root_count - ? WHERE version_id = ?");
|
|
15742
|
+
for (const [versionId, count] of counts)
|
|
15743
|
+
drain.run(count, versionId);
|
|
15744
|
+
rowsDeleted += removed.length;
|
|
15745
|
+
remaining -= removed.length;
|
|
14957
15746
|
}
|
|
14958
15747
|
const materializationsDeleted = remaining === 0 ? 0 : db.query(`DELETE FROM ontology_sources WHERE version_id IN (
|
|
14959
15748
|
SELECT versions.version_id FROM ontology_sources AS versions
|
|
14960
|
-
WHERE project_id = ? AND status = 'superseded' AND terminal_at < ?
|
|
14961
|
-
AND
|
|
14962
|
-
WHERE roots.version_id = versions.version_id)
|
|
15749
|
+
WHERE project_id = ? AND status = 'superseded' AND root_count = 0 AND terminal_at < ?
|
|
15750
|
+
AND ${PLAN_PURGED}
|
|
14963
15751
|
ORDER BY terminal_at, source_id, materialization_id LIMIT ?
|
|
14964
15752
|
)`).run(input.projectId, input.terminalBefore, remaining).changes;
|
|
14965
15753
|
return { rowsDeleted, materializationsDeleted };
|
|
@@ -15045,7 +15833,7 @@ class SqliteOntologySourceStorage {
|
|
|
15045
15833
|
throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
|
|
15046
15834
|
}
|
|
15047
15835
|
if (input.readyAt < manifest.created_at) {
|
|
15048
|
-
throw new
|
|
15836
|
+
throw new MaterializationValidationError5("Source readyAt cannot precede source createdAt.");
|
|
15049
15837
|
}
|
|
15050
15838
|
this.assertReadyState(manifest, input.rootCount, input.assertionCount);
|
|
15051
15839
|
const result = this.db.query(`
|
|
@@ -15121,20 +15909,6 @@ class SqliteOntologySourceStorage {
|
|
|
15121
15909
|
return purgeAbandonedSourceVersions(this.db, input);
|
|
15122
15910
|
});
|
|
15123
15911
|
}
|
|
15124
|
-
async summarizeTerminal(input) {
|
|
15125
|
-
return this.runRootOperation(() => {
|
|
15126
|
-
assertNonblank(input.projectId, "Terminal source summary project id");
|
|
15127
|
-
const summary = this.db.query(`
|
|
15128
|
-
SELECT COUNT(*) AS terminal_count, MIN(terminal_at) AS oldest_terminal_at
|
|
15129
|
-
FROM ontology_sources
|
|
15130
|
-
WHERE project_id = ? AND status IN ('superseded', 'abandoned')
|
|
15131
|
-
`).get(input.projectId);
|
|
15132
|
-
return {
|
|
15133
|
-
count: summary.terminal_count,
|
|
15134
|
-
oldestTerminalAt: summary.oldest_terminal_at
|
|
15135
|
-
};
|
|
15136
|
-
});
|
|
15137
|
-
}
|
|
15138
15912
|
abandonCandidate(input) {
|
|
15139
15913
|
const manifest = this.requireManifest(input.projectId, input.source.projectionId, input.materializationId);
|
|
15140
15914
|
if (manifest.status === "abandoned") {
|
|
@@ -15163,7 +15937,7 @@ class SqliteOntologySourceStorage {
|
|
|
15163
15937
|
}
|
|
15164
15938
|
transitionToAbandoned(manifest, abandonedAt) {
|
|
15165
15939
|
if (abandonedAt < manifest.created_at || manifest.ready_at !== null && abandonedAt < manifest.ready_at) {
|
|
15166
|
-
throw new
|
|
15940
|
+
throw new MaterializationValidationError5("Source abandonedAt cannot precede source creation or readiness.");
|
|
15167
15941
|
}
|
|
15168
15942
|
const changed = this.db.query(`
|
|
15169
15943
|
UPDATE ontology_sources
|
|
@@ -15187,7 +15961,7 @@ class SqliteOntologySourceStorage {
|
|
|
15187
15961
|
WHERE version_id = ?1
|
|
15188
15962
|
`).get(manifest.version_id);
|
|
15189
15963
|
if (counts.assertions !== assertionCount || counts.roots !== rootCount || counts.ordinals !== rootCount || rootCount > 0 && (counts.min_ordinal !== 0 || counts.max_ordinal !== rootCount - 1)) {
|
|
15190
|
-
throw new
|
|
15964
|
+
throw new MaterializationValidationError5("Source ready counts do not match the staged roots and assertions.");
|
|
15191
15965
|
}
|
|
15192
15966
|
assertSourceRootCoverage(this.db, manifest);
|
|
15193
15967
|
const invalid = this.db.query(manifest.projection_kind === "link" ? `
|
|
@@ -15230,7 +16004,7 @@ class SqliteOntologySourceStorage {
|
|
|
15230
16004
|
LIMIT 1
|
|
15231
16005
|
`).get(manifest.version_id);
|
|
15232
16006
|
if (invalid) {
|
|
15233
|
-
throw new
|
|
16007
|
+
throw new MaterializationValidationError5(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
15234
16008
|
}
|
|
15235
16009
|
}
|
|
15236
16010
|
findStageRows(manifest, rows) {
|
|
@@ -15414,7 +16188,7 @@ function decode(row) {
|
|
|
15414
16188
|
}
|
|
15415
16189
|
|
|
15416
16190
|
// src/ontology-storage/vectors.ts
|
|
15417
|
-
import { MaterializationConflictError as
|
|
16191
|
+
import { MaterializationConflictError as MaterializationConflictError9 } from "@sixb/core/internal/materialization";
|
|
15418
16192
|
class SqliteOntologyVectorStorage {
|
|
15419
16193
|
db;
|
|
15420
16194
|
runRootOperation;
|
|
@@ -15520,7 +16294,7 @@ class SqliteOntologyVectorStorage {
|
|
|
15520
16294
|
}
|
|
15521
16295
|
}
|
|
15522
16296
|
function vectorConflict() {
|
|
15523
|
-
return new
|
|
16297
|
+
return new MaterializationConflictError9("effective-state", "Vector changed since it was prepared.");
|
|
15524
16298
|
}
|
|
15525
16299
|
|
|
15526
16300
|
// src/ontology-storage/index.ts
|
|
@@ -15528,6 +16302,7 @@ class SqliteOntologyStorage {
|
|
|
15528
16302
|
commits;
|
|
15529
16303
|
sources;
|
|
15530
16304
|
materializations;
|
|
16305
|
+
replacementPlans;
|
|
15531
16306
|
vectorIndexing;
|
|
15532
16307
|
vectors;
|
|
15533
16308
|
outbox;
|
|
@@ -15535,6 +16310,7 @@ class SqliteOntologyStorage {
|
|
|
15535
16310
|
this.commits = new SqliteOntologyCommitStorage(input.db, input.runRootOperation);
|
|
15536
16311
|
this.sources = new SqliteOntologySourceStorage(input.db, input.runRootOperation);
|
|
15537
16312
|
this.materializations = new SqliteOntologyMaterializationStorage(input.db, input.transactionContext);
|
|
16313
|
+
this.replacementPlans = new SqliteOntologyReplacementPlanStorage(input.db, input.runRootOperation);
|
|
15538
16314
|
this.vectorIndexing = new SqliteVectorIndexingStorage(input.db, input.runRootOperation, (session, projectId) => this.materializations.assertVectorSession(session, projectId));
|
|
15539
16315
|
this.vectors = new SqliteOntologyVectorStorage(input.db, input.runRootOperation, (session, projectId, commitId) => this.materializations.assertVectorSession(session, projectId, commitId));
|
|
15540
16316
|
this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation);
|
|
@@ -16137,7 +16913,7 @@ class SqliteProjectionRunStorage {
|
|
|
16137
16913
|
}
|
|
16138
16914
|
async list(input) {
|
|
16139
16915
|
assertProjectionRunNonEmpty(input.projectId, "projectId");
|
|
16140
|
-
if (input.statuses
|
|
16916
|
+
if (input.statuses?.length === 0 || input.errorCodes?.length === 0) {
|
|
16141
16917
|
return { runs: [], hasMore: false, total: 0 };
|
|
16142
16918
|
}
|
|
16143
16919
|
const whereClauses = ["project_id = ?"];
|
|
@@ -16170,6 +16946,10 @@ class SqliteProjectionRunStorage {
|
|
|
16170
16946
|
whereClauses.push(`status IN (${input.statuses.map(() => "?").join(", ")})`);
|
|
16171
16947
|
args.push(...input.statuses);
|
|
16172
16948
|
}
|
|
16949
|
+
if (input.errorCodes) {
|
|
16950
|
+
whereClauses.push(`json_extract(error, '$.code') IN (${input.errorCodes.map(() => "?").join(", ")})`);
|
|
16951
|
+
args.push(...input.errorCodes);
|
|
16952
|
+
}
|
|
16173
16953
|
if (input.startedAfter) {
|
|
16174
16954
|
whereClauses.push("COALESCE(started_at, queued_at) >= ?");
|
|
16175
16955
|
args.push(input.startedAfter.toISOString());
|
|
@@ -16479,17 +17259,16 @@ class SqliteRulesStorage {
|
|
|
16479
17259
|
}
|
|
16480
17260
|
async listReconciliationPage(input) {
|
|
16481
17261
|
assertPositiveLimit(input.limit);
|
|
17262
|
+
const { after } = input;
|
|
17263
|
+
const cursor = after ? [after.ruleId, after.objectTypeId, after.primaryId] : [];
|
|
16482
17264
|
const rows = this.db.query(`
|
|
16483
17265
|
SELECT project_id, rule_id, subject_kind, object_type_id, primary_id, triggered_at
|
|
16484
17266
|
FROM rule_states
|
|
16485
|
-
WHERE project_id = ?
|
|
16486
|
-
AND (
|
|
16487
|
-
|
|
16488
|
-
(rule_id, object_type_id, primary_id) > (?, ?, ?)
|
|
16489
|
-
)
|
|
16490
|
-
ORDER BY rule_id ASC, object_type_id ASC, primary_id ASC
|
|
17267
|
+
WHERE project_id = ?
|
|
17268
|
+
${cursor.length > 0 ? "AND (rule_id, subject_kind, object_type_id, primary_id) > (?, 'object', ?, ?)" : ""}
|
|
17269
|
+
ORDER BY rule_id ASC, subject_kind ASC, object_type_id ASC, primary_id ASC
|
|
16491
17270
|
LIMIT ?
|
|
16492
|
-
`).all(input.projectId,
|
|
17271
|
+
`).all(input.projectId, ...cursor, input.limit + 1);
|
|
16493
17272
|
return reconciliationPage(rows, input.limit);
|
|
16494
17273
|
}
|
|
16495
17274
|
async applyTriggered(event) {
|
|
@@ -17080,10 +17859,8 @@ function isUniqueConstraintError4(error) {
|
|
|
17080
17859
|
|
|
17081
17860
|
// src/testing.ts
|
|
17082
17861
|
var adapters = new WeakMap;
|
|
17083
|
-
function registerSqliteStorageTestingAdapter(storage,
|
|
17084
|
-
adapters.set(storage,
|
|
17085
|
-
advanceConnectorConnectionTime
|
|
17086
|
-
});
|
|
17862
|
+
function registerSqliteStorageTestingAdapter(storage, adapter) {
|
|
17863
|
+
adapters.set(storage, adapter);
|
|
17087
17864
|
}
|
|
17088
17865
|
|
|
17089
17866
|
// src/timeseries-storage.ts
|
|
@@ -18239,6 +19016,7 @@ class SqliteStorage {
|
|
|
18239
19016
|
shareSessions;
|
|
18240
19017
|
connectorConnections;
|
|
18241
19018
|
fileUploadSessions;
|
|
19019
|
+
fileDownloadGrants;
|
|
18242
19020
|
migrators;
|
|
18243
19021
|
connection;
|
|
18244
19022
|
readConnection;
|
|
@@ -18290,8 +19068,12 @@ class SqliteStorage {
|
|
|
18290
19068
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope);
|
|
18291
19069
|
this.connectorConnections = createOperationScopedFacade(stores.connectorConnections, scope);
|
|
18292
19070
|
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope);
|
|
19071
|
+
this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope);
|
|
18293
19072
|
this.migrators = options.path ? createSqliteStorageMigrators(options.path) : [];
|
|
18294
|
-
registerSqliteStorageTestingAdapter(this,
|
|
19073
|
+
registerSqliteStorageTestingAdapter(this, {
|
|
19074
|
+
advanceConnectorConnectionTime: (durationMs) => stores.connectorConnections.advanceTimeForTesting(durationMs),
|
|
19075
|
+
db: this.connection.db
|
|
19076
|
+
});
|
|
18295
19077
|
}
|
|
18296
19078
|
async ping() {
|
|
18297
19079
|
this.assertRootOperationAvailable();
|
|
@@ -18415,7 +19197,8 @@ function createSqliteStores(connection, options) {
|
|
|
18415
19197
|
shareGrants,
|
|
18416
19198
|
shareSessions,
|
|
18417
19199
|
connectorConnections: new SqliteConnectorConnectionStorage(connection),
|
|
18418
|
-
fileUploadSessions: new SqliteFileUploadSessionStorage(connection)
|
|
19200
|
+
fileUploadSessions: new SqliteFileUploadSessionStorage(connection),
|
|
19201
|
+
fileDownloadGrants: new SqliteFileDownloadGrantStorage(connection)
|
|
18419
19202
|
};
|
|
18420
19203
|
}
|
|
18421
19204
|
export {
|
|
@@ -18423,4 +19206,4 @@ export {
|
|
|
18423
19206
|
migrateSqliteStorage
|
|
18424
19207
|
};
|
|
18425
19208
|
|
|
18426
|
-
//# debugId=
|
|
19209
|
+
//# debugId=0E043C316BC5C57264756E2164756E21
|