@sixb/sqlite 0.1.15 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-limit-storage.d.ts.map +1 -1
- package/dist/ai-usage-storage.d.ts.map +1 -1
- package/dist/execution-storage.d.ts +28 -0
- package/dist/execution-storage.d.ts.map +1 -1
- package/dist/file-download-grant-storage.d.ts +11 -0
- package/dist/file-download-grant-storage.d.ts.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1428 -599
- package/dist/index.js.map +29 -27
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +3 -0
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts +2 -0
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +1 -0
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/rows.d.ts +1 -0
- package/dist/objects/rows.d.ts.map +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/index.d.ts +2 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +17 -37
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +3 -26
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +3 -6
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/replacement-plans.d.ts +88 -0
- package/dist/ontology-storage/replacement-plans.d.ts.map +1 -0
- package/dist/ontology-storage/shared.d.ts +8 -5
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
- package/dist/ontology-storage/source-roots.d.ts +15 -2
- package/dist/ontology-storage/source-roots.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +1 -2
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/projection-run-storage.d.ts.map +1 -1
- package/dist/rules-storage.d.ts.map +1 -1
- package/dist/testing.d.ts +4 -1
- package/dist/testing.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/ai-cost-storage.ts +10 -0
- package/src/ai-limit-storage.ts +13 -2
- package/src/ai-usage-storage.ts +9 -1
- package/src/execution-storage.ts +7 -2
- package/src/file-download-grant-storage.ts +76 -0
- package/src/index.ts +10 -3
- package/src/migrations/053-replacement-plans.sql +73 -0
- package/src/migrations/054-slim-ontology-outbox.sql +21 -0
- package/src/migrations/055-commit-touches.sql +31 -0
- package/src/migrations/056-audio-transcription.sql +10 -0
- package/src/migrations/057-reranking-model-kind.sql +16 -0
- package/src/migrations/058-file-download-grants.sql +21 -0
- package/src/migrations/059-drained-source-versions.sql +11 -0
- package/src/migrations.ts +115 -0
- package/src/objects/README.md +19 -0
- package/src/objects/query-compiler.ts +49 -7
- package/src/objects/read-scope.ts +10 -8
- package/src/objects/reader.ts +44 -8
- package/src/objects/rows.ts +1 -0
- package/src/objects/storage.ts +5 -11
- package/src/ontology-storage/index.ts +6 -1
- package/src/ontology-storage/materialization-session.ts +120 -211
- package/src/ontology-storage/materialization-state.ts +5 -261
- package/src/ontology-storage/materialization-writer.ts +11 -13
- package/src/ontology-storage/materializations.ts +133 -210
- package/src/ontology-storage/outbox.ts +50 -16
- package/src/ontology-storage/replacement-plans.ts +1064 -0
- package/src/ontology-storage/shared.ts +17 -12
- package/src/ontology-storage/source-cleanup.ts +24 -9
- package/src/ontology-storage/source-roots.ts +38 -14
- package/src/ontology-storage/sources.ts +0 -26
- package/src/projection-run-storage.ts +7 -1
- package/src/rules-storage.ts +9 -14
- package/src/testing.ts +6 -4
package/dist/index.js
CHANGED
|
@@ -4670,6 +4670,172 @@ DROP TABLE legacy_ontology_sources;
|
|
|
4670
4670
|
-- describe empty tables forever and steer the planner to partial indexes over primary keys.
|
|
4671
4671
|
`;
|
|
4672
4672
|
|
|
4673
|
+
// src/migrations/053-replacement-plans.sql
|
|
4674
|
+
var _053_replacement_plans_default = `-- The plan of a ready replacement candidate, built outside its commit transaction. Its rows are a
|
|
4675
|
+
-- cache the commit verifies again, keyed by the candidate's source version, which outlives it:
|
|
4676
|
+
-- maintenance deletes a plan once its candidate is no longer ready, and only then the version.
|
|
4677
|
+
CREATE TABLE ontology_replacement_plans (
|
|
4678
|
+
version_id INTEGER PRIMARY KEY REFERENCES ontology_sources (version_id) ON DELETE RESTRICT,
|
|
4679
|
+
project_id TEXT NOT NULL,
|
|
4680
|
+
-- The commit the plan's work carries, at the time fixed when the plan opened.
|
|
4681
|
+
commit_id TEXT NOT NULL,
|
|
4682
|
+
committed_at TEXT NOT NULL,
|
|
4683
|
+
-- The active source the plan replaces, NULL when none was: a plan resumed once that moved
|
|
4684
|
+
-- starts over.
|
|
4685
|
+
replaced_materialization_id TEXT,
|
|
4686
|
+
replaced_last_commit_id TEXT,
|
|
4687
|
+
-- The last commit row every planned identity is known fresh against.
|
|
4688
|
+
watermark INTEGER NOT NULL
|
|
4689
|
+
);
|
|
4690
|
+
|
|
4691
|
+
-- One row per entity the plan decides. \`read_revision\` is the revision of the inputs its state was
|
|
4692
|
+
-- last read at; \`planned_revision\` the one its work was planned from, NULL while still to plan.
|
|
4693
|
+
-- \`classified\` and \`change\` summarize that work for the commit's change counts.
|
|
4694
|
+
CREATE TABLE ontology_replacement_plan_identities (
|
|
4695
|
+
version_id INTEGER NOT NULL,
|
|
4696
|
+
entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link')),
|
|
4697
|
+
identity_key TEXT NOT NULL,
|
|
4698
|
+
sort_key TEXT NOT NULL,
|
|
4699
|
+
diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
|
|
4700
|
+
read_revision TEXT,
|
|
4701
|
+
planned_revision TEXT,
|
|
4702
|
+
classified INTEGER NOT NULL DEFAULT 0 CHECK (classified IN (0, 1)),
|
|
4703
|
+
change TEXT CHECK (change IN ('created', 'updated', 'deleted')),
|
|
4704
|
+
PRIMARY KEY (version_id, entity_kind, identity_key)
|
|
4705
|
+
);
|
|
4706
|
+
-- The identities still to plan, in planning order: a round that plans a few again finds them
|
|
4707
|
+
-- without walking the rest.
|
|
4708
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_unplanned
|
|
4709
|
+
ON ontology_replacement_plan_identities (version_id, entity_kind, sort_key)
|
|
4710
|
+
WHERE planned_revision IS NULL;
|
|
4711
|
+
|
|
4712
|
+
-- Planned work, with the same columns as a session's work table plus the identity it belongs to.
|
|
4713
|
+
CREATE TABLE ontology_replacement_plan_work (
|
|
4714
|
+
work_id INTEGER NOT NULL,
|
|
4715
|
+
entity_kind TEXT NOT NULL,
|
|
4716
|
+
identity_key TEXT NOT NULL,
|
|
4717
|
+
record_key TEXT NOT NULL,
|
|
4718
|
+
unique_key TEXT NOT NULL,
|
|
4719
|
+
kind TEXT NOT NULL,
|
|
4720
|
+
lane TEXT NOT NULL,
|
|
4721
|
+
major_order INTEGER NOT NULL,
|
|
4722
|
+
minor_order INTEGER NOT NULL,
|
|
4723
|
+
sort_one TEXT NOT NULL,
|
|
4724
|
+
sort_two TEXT NOT NULL,
|
|
4725
|
+
classification_entity_kind TEXT,
|
|
4726
|
+
classification_identity_key TEXT,
|
|
4727
|
+
cardinality_view TEXT,
|
|
4728
|
+
cardinality_occupied INTEGER,
|
|
4729
|
+
cardinality_source_type_id TEXT,
|
|
4730
|
+
cardinality_source_primary_id TEXT,
|
|
4731
|
+
cardinality_link_id TEXT,
|
|
4732
|
+
cardinality_target_type_id TEXT,
|
|
4733
|
+
cardinality_target_primary_id TEXT,
|
|
4734
|
+
payload TEXT NOT NULL CHECK (json_valid(payload)),
|
|
4735
|
+
PRIMARY KEY (work_id, record_key),
|
|
4736
|
+
UNIQUE (work_id, unique_key)
|
|
4737
|
+
);
|
|
4738
|
+
CREATE INDEX idx_ontology_replacement_plan_work_identity
|
|
4739
|
+
ON ontology_replacement_plan_work (work_id, entity_kind, identity_key);
|
|
4740
|
+
-- The few objects whose existence the plan flips, which link expansion starts from.
|
|
4741
|
+
CREATE INDEX idx_ontology_replacement_plan_work_incident
|
|
4742
|
+
ON ontology_replacement_plan_work (work_id) WHERE kind = 'incident-object';
|
|
4743
|
+
CREATE INDEX idx_ontology_replacement_plan_work_lane
|
|
4744
|
+
ON ontology_replacement_plan_work (
|
|
4745
|
+
work_id, lane, major_order, minor_order, sort_one, sort_two, record_key
|
|
4746
|
+
);
|
|
4747
|
+
`;
|
|
4748
|
+
|
|
4749
|
+
// src/migrations/054-slim-ontology-outbox.sql
|
|
4750
|
+
var _054_slim_ontology_outbox_default = `-- An outbox event stores what changed and nothing it can be rebuilt from: delivery rebuilds the
|
|
4751
|
+
-- event from its commit and the execution that commit ran under. The commit also stops copying
|
|
4752
|
+
-- \`requested_by\` and \`executor\` from that execution (047): the execution is canonical, immutable,
|
|
4753
|
+
-- and already referenced by the commit. The migration step checks beforehand that every stored
|
|
4754
|
+
-- event rebuilds exactly as it was stored.
|
|
4755
|
+
--
|
|
4756
|
+
-- SQLite cannot add a NOT NULL column without a default; every insert supplies the event, and the
|
|
4757
|
+
-- UPDATE below replaces the placeholder on every row.
|
|
4758
|
+
ALTER TABLE ontology_outbox ADD COLUMN event TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(event));
|
|
4759
|
+
UPDATE ontology_outbox SET event = json_object(
|
|
4760
|
+
'payload', CASE
|
|
4761
|
+
WHEN envelope ->> '$.type' IN ('object.created', 'link.created')
|
|
4762
|
+
THEN json_remove(envelope -> '$.payload', '$.propertyChanges')
|
|
4763
|
+
ELSE envelope -> '$.payload'
|
|
4764
|
+
END,
|
|
4765
|
+
'type', envelope ->> '$.type'
|
|
4766
|
+
);
|
|
4767
|
+
ALTER TABLE ontology_outbox DROP COLUMN envelope;
|
|
4768
|
+
|
|
4769
|
+
ALTER TABLE ontology_commits DROP COLUMN requested_by;
|
|
4770
|
+
ALTER TABLE ontology_commits DROP COLUMN executor;
|
|
4771
|
+
`;
|
|
4772
|
+
|
|
4773
|
+
// src/migrations/055-commit-touches.sql
|
|
4774
|
+
var _055_commit_touches_default = `-- Every commit records the entities whose plan inputs it changed, so a replacement plan's refresh
|
|
4775
|
+
-- rechecks only what the commits since it was last fresh touched, instead of the whole plan.
|
|
4776
|
+
CREATE TABLE ontology_commit_touches (
|
|
4777
|
+
project_id TEXT NOT NULL,
|
|
4778
|
+
commit_id TEXT NOT NULL,
|
|
4779
|
+
entity_kind TEXT NOT NULL CHECK (entity_kind IN ('object', 'link', 'scope')),
|
|
4780
|
+
-- \`objectRefKey\`, \`linkRefKey\` or \`linkScopeKey\`.
|
|
4781
|
+
identity_key TEXT NOT NULL
|
|
4782
|
+
);
|
|
4783
|
+
CREATE INDEX idx_ontology_commit_touches_commit ON ontology_commit_touches (project_id, commit_id);
|
|
4784
|
+
|
|
4785
|
+
-- Existing plans predate the touches of the commits since their watermark: they plan again.
|
|
4786
|
+
DELETE FROM ontology_replacement_plan_work;
|
|
4787
|
+
DELETE FROM ontology_replacement_plan_identities;
|
|
4788
|
+
DELETE FROM ontology_replacement_plans;
|
|
4789
|
+
|
|
4790
|
+
-- Link expansion starts only from what changed since it last ran: \`expansions\` counts the runs,
|
|
4791
|
+
-- and an identity's \`expand_at\` is the run that has yet to start from it (an object once planned,
|
|
4792
|
+
-- a link once it needs a diff). \`source_key\` and \`target_key\` find a link from either endpoint.
|
|
4793
|
+
ALTER TABLE ontology_replacement_plans ADD COLUMN expansions INTEGER NOT NULL DEFAULT 0;
|
|
4794
|
+
ALTER TABLE ontology_replacement_plan_identities ADD COLUMN expand_at INTEGER;
|
|
4795
|
+
ALTER TABLE ontology_replacement_plan_identities ADD COLUMN source_key TEXT;
|
|
4796
|
+
ALTER TABLE ontology_replacement_plan_identities ADD COLUMN target_key TEXT;
|
|
4797
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_expand
|
|
4798
|
+
ON ontology_replacement_plan_identities (version_id, entity_kind, expand_at);
|
|
4799
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_source
|
|
4800
|
+
ON ontology_replacement_plan_identities (version_id, source_key)
|
|
4801
|
+
WHERE source_key IS NOT NULL;
|
|
4802
|
+
CREATE INDEX idx_ontology_replacement_plan_identities_target
|
|
4803
|
+
ON ontology_replacement_plan_identities (version_id, target_key)
|
|
4804
|
+
WHERE target_key IS NOT NULL;
|
|
4805
|
+
`;
|
|
4806
|
+
|
|
4807
|
+
// src/migrations/056-audio-transcription.sql
|
|
4808
|
+
var _056_audio_transcription_default = `ALTER TABLE ai_model_call_usage
|
|
4809
|
+
ADD COLUMN audio_duration_ms INTEGER
|
|
4810
|
+
CHECK (audio_duration_ms IS NULL OR audio_duration_ms >= 0);
|
|
4811
|
+
|
|
4812
|
+
ALTER TABLE ai_model_call_usage
|
|
4813
|
+
ADD COLUMN model_kind TEXT
|
|
4814
|
+
CHECK (
|
|
4815
|
+
model_kind IS NULL
|
|
4816
|
+
OR model_kind IN ('language', 'image', 'video', 'embedding', 'decision', 'transcription')
|
|
4817
|
+
);
|
|
4818
|
+
`;
|
|
4819
|
+
|
|
4820
|
+
// src/migrations/057-reranking-model-kind.sql
|
|
4821
|
+
var _057_reranking_model_kind_default = `-- SQLite cannot alter a CHECK constraint. Swapping the column avoids rebuilding a table that
|
|
4822
|
+
-- usage groups, costs and limit reservations reference.
|
|
4823
|
+
ALTER TABLE ai_model_call_usage
|
|
4824
|
+
ADD COLUMN model_kind_next TEXT
|
|
4825
|
+
CHECK (
|
|
4826
|
+
model_kind_next IS NULL
|
|
4827
|
+
OR model_kind_next IN (
|
|
4828
|
+
'language', 'image', 'video', 'embedding', 'decision', 'transcription', 'reranking'
|
|
4829
|
+
)
|
|
4830
|
+
);
|
|
4831
|
+
|
|
4832
|
+
UPDATE ai_model_call_usage SET model_kind_next = model_kind;
|
|
4833
|
+
|
|
4834
|
+
ALTER TABLE ai_model_call_usage DROP COLUMN model_kind;
|
|
4835
|
+
|
|
4836
|
+
ALTER TABLE ai_model_call_usage RENAME COLUMN model_kind_next TO model_kind;
|
|
4837
|
+
`;
|
|
4838
|
+
|
|
4673
4839
|
// src/migrations/037-execution-requester-groups.sql
|
|
4674
4840
|
var _037_execution_requester_groups_default = `ALTER TABLE executions ADD COLUMN requester_group_ids TEXT NOT NULL DEFAULT '[]';
|
|
4675
4841
|
|
|
@@ -4790,6 +4956,44 @@ CREATE INDEX idx_file_upload_sessions_abandoned
|
|
|
4790
4956
|
WHERE status = 'pending' AND provider_upload IS NOT NULL;
|
|
4791
4957
|
`;
|
|
4792
4958
|
|
|
4959
|
+
// src/migrations/058-file-download-grants.sql
|
|
4960
|
+
var _058_file_download_grants_default = `-- URLs issued by blobs.createDownloadUrl(). Only the SHA-256 of each URL's token is stored.
|
|
4961
|
+
-- Timestamps are ISO-8601 UTC text and compare lexicographically.
|
|
4962
|
+
CREATE TABLE file_download_grants (
|
|
4963
|
+
project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
|
|
4964
|
+
id TEXT NOT NULL CHECK (length(trim(id)) > 0),
|
|
4965
|
+
token_hash TEXT NOT NULL CHECK (
|
|
4966
|
+
length(token_hash) = 64
|
|
4967
|
+
AND token_hash = lower(token_hash)
|
|
4968
|
+
AND token_hash NOT GLOB '*[^0-9a-f]*'
|
|
4969
|
+
),
|
|
4970
|
+
file TEXT NOT NULL CHECK (json_type(file) = 'object'),
|
|
4971
|
+
execution_id TEXT NOT NULL CHECK (length(trim(execution_id)) > 0),
|
|
4972
|
+
created_at TEXT NOT NULL,
|
|
4973
|
+
expires_at TEXT NOT NULL,
|
|
4974
|
+
revoked_at TEXT,
|
|
4975
|
+
PRIMARY KEY (project_id, id),
|
|
4976
|
+
UNIQUE (project_id, token_hash),
|
|
4977
|
+
CHECK (expires_at > created_at)
|
|
4978
|
+
);
|
|
4979
|
+
|
|
4980
|
+
CREATE INDEX idx_file_download_grants_expiry ON file_download_grants (project_id, expires_at);
|
|
4981
|
+
`;
|
|
4982
|
+
|
|
4983
|
+
// src/migrations/059-drained-source-versions.sql
|
|
4984
|
+
var _059_drained_source_versions_default = `-- A superseded version's \`root_count\` counts the roots it still holds: cleanup lowers it as it
|
|
4985
|
+
-- deletes them. A version that reaches zero is found through this index, instead of every
|
|
4986
|
+
-- superseded version being walked and probed for roots on every maintenance pass.
|
|
4987
|
+
UPDATE ontology_sources
|
|
4988
|
+
SET root_count = (SELECT count(*) FROM ontology_source_roots AS roots
|
|
4989
|
+
WHERE roots.version_id = ontology_sources.version_id)
|
|
4990
|
+
WHERE status = 'superseded' AND root_count <> (SELECT count(*) FROM ontology_source_roots AS roots
|
|
4991
|
+
WHERE roots.version_id = ontology_sources.version_id);
|
|
4992
|
+
CREATE INDEX idx_ontology_sources_drained
|
|
4993
|
+
ON ontology_sources (project_id, terminal_at, source_id, materialization_id)
|
|
4994
|
+
WHERE status = 'superseded' AND root_count = 0;
|
|
4995
|
+
`;
|
|
4996
|
+
|
|
4793
4997
|
// src/migrations.ts
|
|
4794
4998
|
var MIGRATIONS_TABLE_SQL = `
|
|
4795
4999
|
CREATE TABLE IF NOT EXISTS sixb_migrations (
|
|
@@ -4883,7 +5087,20 @@ var sqliteStorageMigrations = defineMigrations({
|
|
|
4883
5087
|
if (after.versions !== before.versions || after.roots !== before.roots || after.rows !== before.rows) {
|
|
4884
5088
|
throw new Error("[SixbSqliteStorage] Source versions, roots or rows were lost while compacting source storage.");
|
|
4885
5089
|
}
|
|
4886
|
-
}, { checksum: checksum(_052_compact_source_storage_default) })
|
|
5090
|
+
}, { checksum: checksum(_052_compact_source_storage_default) }),
|
|
5091
|
+
sqliteSql("053-replacement-plans", _053_replacement_plans_default),
|
|
5092
|
+
sqliteStep("054-slim-ontology-outbox", (db) => {
|
|
5093
|
+
const mismatched = unrebuildableOutboxEvents(db);
|
|
5094
|
+
if (mismatched > 0) {
|
|
5095
|
+
throw new Error(`[SixbSqliteStorage] ${mismatched} outbox events would not rebuild from their commit.`);
|
|
5096
|
+
}
|
|
5097
|
+
db.run(_054_slim_ontology_outbox_default);
|
|
5098
|
+
}, { checksum: checksum(_054_slim_ontology_outbox_default) }),
|
|
5099
|
+
sqliteSql("055-commit-touches", _055_commit_touches_default),
|
|
5100
|
+
sqliteSql("056-audio-transcription", _056_audio_transcription_default),
|
|
5101
|
+
sqliteSql("057-reranking-model-kind", _057_reranking_model_kind_default),
|
|
5102
|
+
sqliteSql("058-file-download-grants", _058_file_download_grants_default),
|
|
5103
|
+
sqliteSql("059-drained-source-versions", _059_drained_source_versions_default)
|
|
4887
5104
|
]
|
|
4888
5105
|
});
|
|
4889
5106
|
function sqliteStoragePath(basePath) {
|
|
@@ -5003,6 +5220,82 @@ function sourceStorageCounts(db) {
|
|
|
5003
5220
|
(SELECT count(*) FROM ontology_source_roots) AS roots,
|
|
5004
5221
|
(SELECT count(*) FROM ontology_source_rows) AS rows`).get();
|
|
5005
5222
|
}
|
|
5223
|
+
function unrebuildableOutboxEvents(db) {
|
|
5224
|
+
const row = db.query(`SELECT COUNT(*) AS mismatched
|
|
5225
|
+
FROM ontology_outbox AS outbox
|
|
5226
|
+
LEFT JOIN ontology_commits AS commits
|
|
5227
|
+
ON commits.project_id = outbox.project_id AND commits.id = outbox.commit_id
|
|
5228
|
+
LEFT JOIN executions
|
|
5229
|
+
ON executions.project_id = commits.project_id AND executions.id = commits.execution_id
|
|
5230
|
+
WHERE executions.id IS NULL
|
|
5231
|
+
OR outbox.envelope ->> '$.id' IS NOT outbox.id
|
|
5232
|
+
OR outbox.envelope ->> '$.commitOrdinal' IS NOT outbox.commit_ordinal
|
|
5233
|
+
OR outbox.envelope ->> '$.commitId' IS NOT commits.id
|
|
5234
|
+
OR outbox.envelope ->> '$.projectId' IS NOT commits.project_id
|
|
5235
|
+
OR outbox.envelope ->> '$.schemaVersion' IS NOT 1
|
|
5236
|
+
OR outbox.envelope ->> '$.occurredAt' IS NOT commits.committed_at
|
|
5237
|
+
OR outbox.envelope -> '$.origin' IS NOT json(commits.origin)
|
|
5238
|
+
OR outbox.envelope ->> '$.correlationId' IS NOT executions.correlation_id
|
|
5239
|
+
OR outbox.envelope -> '$.requestedBy' IS NOT CASE
|
|
5240
|
+
WHEN executions.requested_by_user_id IS NOT NULL
|
|
5241
|
+
THEN json_object('id', executions.requested_by_user_id, 'type', 'user')
|
|
5242
|
+
WHEN executions.requested_by_service_account_id IS NOT NULL
|
|
5243
|
+
THEN json_object('id', executions.requested_by_service_account_id, 'type', 'serviceAccount')
|
|
5244
|
+
END
|
|
5245
|
+
OR outbox.envelope -> '$.executor' IS NOT CASE executions.executor_kind
|
|
5246
|
+
WHEN 'request' THEN json_object('requestId', executions.executor_id, 'type', 'request')
|
|
5247
|
+
WHEN 'agent' THEN json_object('runId', executions.executor_id, 'type', 'agent')
|
|
5248
|
+
WHEN 'kernel' THEN json_object(
|
|
5249
|
+
'operation',
|
|
5250
|
+
CASE executions.authority_kernel_operation
|
|
5251
|
+
WHEN 'ontology.recover'
|
|
5252
|
+
THEN json_object('recoveryId', executions.executor_id, 'type', 'ontology.recover')
|
|
5253
|
+
ELSE json_object('indexingId', executions.executor_id, 'type', 'ontology.indexVectors')
|
|
5254
|
+
END,
|
|
5255
|
+
'type', 'kernel'
|
|
5256
|
+
)
|
|
5257
|
+
ELSE json_object(
|
|
5258
|
+
'id', executions.authority_primitive_id,
|
|
5259
|
+
'kind', executions.executor_kind,
|
|
5260
|
+
'runId', executions.executor_id,
|
|
5261
|
+
'type', 'primitive'
|
|
5262
|
+
)
|
|
5263
|
+
END
|
|
5264
|
+
OR outbox.envelope ->> '$.topic' IS NOT CASE substr(outbox.envelope ->> '$.type', 1, 4)
|
|
5265
|
+
WHEN 'obje' THEN 'objects'
|
|
5266
|
+
WHEN 'link' THEN 'links'
|
|
5267
|
+
WHEN 'tele' THEN 'telemetry'
|
|
5268
|
+
END
|
|
5269
|
+
OR outbox.envelope ->> '$.partitionKey' IS NOT CASE outbox.envelope ->> '$.topic'
|
|
5270
|
+
WHEN 'objects' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
|
|
5271
|
+
(outbox.envelope ->> '$.payload.primaryId')
|
|
5272
|
+
WHEN 'links' THEN (outbox.envelope ->> '$.payload.sourceTypeId') || ':' ||
|
|
5273
|
+
(outbox.envelope ->> '$.payload.sourceId') || ':' ||
|
|
5274
|
+
(outbox.envelope ->> '$.payload.linkId')
|
|
5275
|
+
WHEN 'telemetry' THEN (outbox.envelope ->> '$.payload.objectTypeId') || ':' ||
|
|
5276
|
+
(outbox.envelope ->> '$.payload.objectId') || ':' ||
|
|
5277
|
+
(outbox.envelope ->> '$.payload.propertyId')
|
|
5278
|
+
END
|
|
5279
|
+
OR (
|
|
5280
|
+
outbox.envelope ->> '$.type' IN ('object.created', 'link.created')
|
|
5281
|
+
AND (
|
|
5282
|
+
(SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.propertyChanges'))
|
|
5283
|
+
IS NOT (SELECT COUNT(*) FROM json_each(outbox.envelope, '$.payload.properties'))
|
|
5284
|
+
OR EXISTS (
|
|
5285
|
+
SELECT 1
|
|
5286
|
+
FROM json_each(outbox.envelope, '$.payload.propertyChanges') AS change
|
|
5287
|
+
LEFT JOIN json_each(outbox.envelope, '$.payload.properties') AS property
|
|
5288
|
+
ON property.key = change.key
|
|
5289
|
+
WHERE property.key IS NULL
|
|
5290
|
+
OR change.value ->> '$.operation' IS NOT 'created'
|
|
5291
|
+
OR json_type(change.value, '$.after') IS NOT property.type
|
|
5292
|
+
OR change.value ->> '$.after' IS NOT property.value
|
|
5293
|
+
OR (SELECT COUNT(*) FROM json_each(change.value)) IS NOT 2
|
|
5294
|
+
)
|
|
5295
|
+
)
|
|
5296
|
+
)`).get();
|
|
5297
|
+
return row.mismatched;
|
|
5298
|
+
}
|
|
5006
5299
|
function checksum(value) {
|
|
5007
5300
|
return createHash("sha256").update(value).digest("hex");
|
|
5008
5301
|
}
|
|
@@ -6592,6 +6885,7 @@ class SqliteAiUsageStorage {
|
|
|
6592
6885
|
provider_id,
|
|
6593
6886
|
provider_ids,
|
|
6594
6887
|
requested_model_id,
|
|
6888
|
+
model_kind,
|
|
6595
6889
|
requested_reasoning,
|
|
6596
6890
|
response_model_id,
|
|
6597
6891
|
response_id,
|
|
@@ -6603,14 +6897,15 @@ class SqliteAiUsageStorage {
|
|
|
6603
6897
|
cache_write_input_tokens,
|
|
6604
6898
|
text_output_tokens,
|
|
6605
6899
|
reasoning_output_tokens,
|
|
6900
|
+
audio_duration_ms,
|
|
6606
6901
|
reporting_status,
|
|
6607
6902
|
raw_usage,
|
|
6608
6903
|
occurred_at,
|
|
6609
6904
|
recorded_at
|
|
6610
6905
|
) VALUES (
|
|
6611
|
-
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
6906
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
6612
6907
|
)
|
|
6613
|
-
`).run(record.projectId, record.id, record.executionId, record.attempt, record.callId, record.providerId, record.providerIds === undefined ? null : JSON.stringify(record.providerIds), record.requestedModelId, record.requestedReasoning === undefined ? null : JSON.stringify(record.requestedReasoning), record.responseModelId ?? null, record.responseId, record.usage.inputTokens ?? null, record.usage.outputTokens ?? null, record.usage.totalTokens ?? null, record.usage.uncachedInputTokens ?? null, record.usage.cacheReadInputTokens ?? null, record.usage.cacheWriteInputTokens ?? null, record.usage.textOutputTokens ?? null, record.usage.reasoningOutputTokens ?? null, record.usage.reportingStatus, record.rawUsage === undefined ? null : JSON.stringify(record.rawUsage), record.occurredAt.toISOString(), record.recordedAt.toISOString());
|
|
6908
|
+
`).run(record.projectId, record.id, record.executionId, record.attempt, record.callId, record.providerId, record.providerIds === undefined ? null : JSON.stringify(record.providerIds), record.requestedModelId, record.modelKind ?? null, record.requestedReasoning === undefined ? null : JSON.stringify(record.requestedReasoning), record.responseModelId ?? null, record.responseId, record.usage.inputTokens ?? null, record.usage.outputTokens ?? null, record.usage.totalTokens ?? null, record.usage.uncachedInputTokens ?? null, record.usage.cacheReadInputTokens ?? null, record.usage.cacheWriteInputTokens ?? null, record.usage.textOutputTokens ?? null, record.usage.reasoningOutputTokens ?? null, record.usage.audioDurationMs ?? null, record.usage.reportingStatus, record.rawUsage === undefined ? null : JSON.stringify(record.rawUsage), record.occurredAt.toISOString(), record.recordedAt.toISOString());
|
|
6614
6909
|
}
|
|
6615
6910
|
findByIdempotencyKey(record) {
|
|
6616
6911
|
const row = this.connection.db.query(`
|
|
@@ -6645,6 +6940,7 @@ class SqliteAiUsageStorage {
|
|
|
6645
6940
|
providerId: row.provider_id,
|
|
6646
6941
|
...row.provider_ids === null ? {} : { providerIds: normalizeModelProviderIds(JSON.parse(row.provider_ids)) },
|
|
6647
6942
|
requestedModelId: row.requested_model_id,
|
|
6943
|
+
...row.model_kind === null ? {} : { modelKind: row.model_kind },
|
|
6648
6944
|
...row.requested_reasoning === null ? {} : { requestedReasoning: requestedReasoningFromRow(row.requested_reasoning) },
|
|
6649
6945
|
...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
|
|
6650
6946
|
responseId: row.response_id,
|
|
@@ -6701,7 +6997,8 @@ function usageFromRow(row) {
|
|
|
6701
6997
|
...row.cache_read_input_tokens === null ? {} : { cacheReadInputTokens: row.cache_read_input_tokens },
|
|
6702
6998
|
...row.cache_write_input_tokens === null ? {} : { cacheWriteInputTokens: row.cache_write_input_tokens },
|
|
6703
6999
|
...row.text_output_tokens === null ? {} : { textOutputTokens: row.text_output_tokens },
|
|
6704
|
-
...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens }
|
|
7000
|
+
...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens },
|
|
7001
|
+
...row.audio_duration_ms === null ? {} : { audioDurationMs: row.audio_duration_ms }
|
|
6705
7002
|
};
|
|
6706
7003
|
}
|
|
6707
7004
|
function requestedReasoningFromRow(value) {
|
|
@@ -6882,6 +7179,7 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
6882
7179
|
usage.cache_write_input_tokens,
|
|
6883
7180
|
usage.text_output_tokens,
|
|
6884
7181
|
usage.reasoning_output_tokens,
|
|
7182
|
+
usage.audio_duration_ms,
|
|
6885
7183
|
usage.reporting_status,
|
|
6886
7184
|
CASE
|
|
6887
7185
|
WHEN direct_agent.kind = 'conversation' THEN 'agent'
|
|
@@ -6966,6 +7264,8 @@ var SQLITE_ACCOUNTING_OVERVIEW_SQL = `
|
|
|
6966
7264
|
CAST(SUM(text_output_tokens) AS TEXT) AS text_output_tokens_sum,
|
|
6967
7265
|
CAST(COUNT(reasoning_output_tokens) AS TEXT) AS reasoning_output_tokens_count,
|
|
6968
7266
|
CAST(SUM(reasoning_output_tokens) AS TEXT) AS reasoning_output_tokens_sum,
|
|
7267
|
+
CAST(COUNT(audio_duration_ms) AS TEXT) AS audio_duration_ms_count,
|
|
7268
|
+
CAST(SUM(audio_duration_ms) AS TEXT) AS audio_duration_ms_sum,
|
|
6969
7269
|
CAST(SUM(CASE WHEN valuation_status = 'rated' THEN 1 ELSE 0 END) AS TEXT)
|
|
6970
7270
|
AS rated_call_count,
|
|
6971
7271
|
CAST(SUM(CASE WHEN valuation_status = 'unpriceable' THEN 1 ELSE 0 END) AS TEXT)
|
|
@@ -7011,7 +7311,8 @@ function aggregateFragmentFromRow(row) {
|
|
|
7011
7311
|
cacheReadInputTokens: meterFragment(row.cache_read_input_tokens_count, row.cache_read_input_tokens_sum),
|
|
7012
7312
|
cacheWriteInputTokens: meterFragment(row.cache_write_input_tokens_count, row.cache_write_input_tokens_sum),
|
|
7013
7313
|
textOutputTokens: meterFragment(row.text_output_tokens_count, row.text_output_tokens_sum),
|
|
7014
|
-
reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum)
|
|
7314
|
+
reasoningOutputTokens: meterFragment(row.reasoning_output_tokens_count, row.reasoning_output_tokens_sum),
|
|
7315
|
+
audioDurationMs: meterFragment(row.audio_duration_ms_count, row.audio_duration_ms_sum)
|
|
7015
7316
|
},
|
|
7016
7317
|
costs: {
|
|
7017
7318
|
...row.amount_currency === null || row.amount_high === null || row.amount_middle === null || row.amount_low === null ? {} : {
|
|
@@ -7067,6 +7368,7 @@ function usageFromRow2(row, requesterGroupIds) {
|
|
|
7067
7368
|
providerId: row.provider_id,
|
|
7068
7369
|
...row.provider_ids === null ? {} : { providerIds: normalizeModelProviderIds2(JSON.parse(row.provider_ids)) },
|
|
7069
7370
|
requestedModelId: row.requested_model_id,
|
|
7371
|
+
...row.model_kind === null ? {} : { modelKind: row.model_kind },
|
|
7070
7372
|
...row.requested_reasoning === null ? {} : { requestedReasoning: requestedReasoningFromRow(row.requested_reasoning) },
|
|
7071
7373
|
...row.response_model_id === null ? {} : { responseModelId: row.response_model_id },
|
|
7072
7374
|
responseId: row.response_id,
|
|
@@ -7079,6 +7381,7 @@ function usageFromRow2(row, requesterGroupIds) {
|
|
|
7079
7381
|
...row.cache_write_input_tokens === null ? {} : { cacheWriteInputTokens: row.cache_write_input_tokens },
|
|
7080
7382
|
...row.text_output_tokens === null ? {} : { textOutputTokens: row.text_output_tokens },
|
|
7081
7383
|
...row.reasoning_output_tokens === null ? {} : { reasoningOutputTokens: row.reasoning_output_tokens },
|
|
7384
|
+
...row.audio_duration_ms === null ? {} : { audioDurationMs: row.audio_duration_ms },
|
|
7082
7385
|
reportingStatus: row.reporting_status
|
|
7083
7386
|
},
|
|
7084
7387
|
...row.raw_usage === null ? {} : { rawUsage: JSON.parse(row.raw_usage) },
|
|
@@ -7219,6 +7522,7 @@ function required(value) {
|
|
|
7219
7522
|
// src/ai-limit-storage.ts
|
|
7220
7523
|
import {
|
|
7221
7524
|
aiLimitAmountKey,
|
|
7525
|
+
aiLimitMeterApplies,
|
|
7222
7526
|
aiLimitQuantityFromAmount,
|
|
7223
7527
|
aiLimitReservationBuckets,
|
|
7224
7528
|
aiLimitReservationRequestKey,
|
|
@@ -7327,8 +7631,9 @@ class SqliteAiLimitStorage {
|
|
|
7327
7631
|
const unavailablePolicies = [];
|
|
7328
7632
|
const unavailableReasons = new Set;
|
|
7329
7633
|
for (const policy of enabledPolicies) {
|
|
7330
|
-
if (!subjectKeys.has(aiLimitSubjectKey(policy.subject)))
|
|
7634
|
+
if (!subjectKeys.has(aiLimitSubjectKey(policy.subject)) || !aiLimitMeterApplies(policy.limit.meter, request.modelKind)) {
|
|
7331
7635
|
continue;
|
|
7636
|
+
}
|
|
7332
7637
|
const limit = normalizeAiLimitAmount(policy.limit);
|
|
7333
7638
|
const estimate = estimates.get(aiLimitAmountKey(limit));
|
|
7334
7639
|
const status = this.statusForPolicy(policy, request.period);
|
|
@@ -7362,7 +7667,7 @@ class SqliteAiLimitStorage {
|
|
|
7362
7667
|
resetAt: new Date(Math.min(...exhaustedPolicies.map((status) => status.period.resetAt.getTime())))
|
|
7363
7668
|
};
|
|
7364
7669
|
}
|
|
7365
|
-
const buckets = aiLimitReservationBuckets(enabledPolicies, request.subjects, request.estimates);
|
|
7670
|
+
const buckets = aiLimitReservationBuckets(enabledPolicies, request.subjects, request.estimates, request.modelKind);
|
|
7366
7671
|
if (buckets.length === 0)
|
|
7367
7672
|
return { status: "notRequired" };
|
|
7368
7673
|
for (const bucket of buckets) {
|
|
@@ -7576,6 +7881,7 @@ class SqliteAiLimitStorage {
|
|
|
7576
7881
|
SELECT usage.project_id, usage.id AS usage_record_id, usage.execution_id,
|
|
7577
7882
|
usage.attempt, usage.call_id, usage.occurred_at,
|
|
7578
7883
|
CAST(usage.total_tokens AS TEXT) AS total_tokens,
|
|
7884
|
+
usage.model_kind,
|
|
7579
7885
|
executions.requested_by_user_id,
|
|
7580
7886
|
executions.requested_by_service_account_id,
|
|
7581
7887
|
valuations.status AS valuation_status,
|
|
@@ -7614,6 +7920,7 @@ class SqliteAiLimitStorage {
|
|
|
7614
7920
|
SELECT usage.project_id, usage.id AS usage_record_id, usage.execution_id,
|
|
7615
7921
|
usage.attempt, usage.call_id, usage.occurred_at,
|
|
7616
7922
|
CAST(usage.total_tokens AS TEXT) AS total_tokens,
|
|
7923
|
+
usage.model_kind,
|
|
7617
7924
|
executions.requested_by_user_id,
|
|
7618
7925
|
executions.requested_by_service_account_id,
|
|
7619
7926
|
valuations.status AS valuation_status,
|
|
@@ -7691,6 +7998,7 @@ function accountingEntryFromRow(row, subject, requesterGroupIds = subject?.type
|
|
|
7691
7998
|
attempt: row.attempt,
|
|
7692
7999
|
callId: row.call_id,
|
|
7693
8000
|
occurredAt: new Date(row.occurred_at),
|
|
8001
|
+
...row.model_kind === null ? {} : { modelKind: row.model_kind },
|
|
7694
8002
|
...totalTokens !== undefined && Number.isSafeInteger(totalTokens) ? { totalTokens } : {},
|
|
7695
8003
|
requesterGroupIds,
|
|
7696
8004
|
...requester ? { requester } : {},
|
|
@@ -10410,9 +10718,12 @@ class SqliteExecutionStorage {
|
|
|
10410
10718
|
FROM executions
|
|
10411
10719
|
WHERE project_id = ? AND id = ?
|
|
10412
10720
|
`).get(params.projectId, params.id);
|
|
10413
|
-
return row ?
|
|
10721
|
+
return row ? sqliteExecutionRecord(row) : null;
|
|
10414
10722
|
}
|
|
10415
10723
|
}
|
|
10724
|
+
function sqliteExecutionRecord(row) {
|
|
10725
|
+
return executionRecordFromStorageRow(toStorageRow(row));
|
|
10726
|
+
}
|
|
10416
10727
|
function toStorageRow(row) {
|
|
10417
10728
|
return {
|
|
10418
10729
|
projectId: row.project_id,
|
|
@@ -10444,6 +10755,43 @@ function isDuplicateExecutionError(error) {
|
|
|
10444
10755
|
return error instanceof Error && error.message.includes("UNIQUE constraint failed: executions.project_id, executions.id");
|
|
10445
10756
|
}
|
|
10446
10757
|
|
|
10758
|
+
// src/file-download-grant-storage.ts
|
|
10759
|
+
import { parseFileDownloadGrantRow } from "@sixb/core/internal/storage";
|
|
10760
|
+
var GRANT_COLUMNS = `
|
|
10761
|
+
project_id AS projectId, id, token_hash AS tokenHash, file, execution_id AS executionId,
|
|
10762
|
+
created_at AS createdAt, expires_at AS expiresAt, revoked_at AS revokedAt
|
|
10763
|
+
`;
|
|
10764
|
+
|
|
10765
|
+
class SqliteFileDownloadGrantStorage {
|
|
10766
|
+
db;
|
|
10767
|
+
constructor(connection) {
|
|
10768
|
+
this.db = connection.db;
|
|
10769
|
+
}
|
|
10770
|
+
async create(record) {
|
|
10771
|
+
this.db.query(`INSERT INTO file_download_grants (
|
|
10772
|
+
project_id, id, token_hash, file, execution_id, created_at, expires_at
|
|
10773
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`).run(record.projectId, record.id, record.tokenHash, JSON.stringify(record.file), record.executionId, record.createdAt.toISOString(), record.expiresAt.toISOString());
|
|
10774
|
+
}
|
|
10775
|
+
async findByTokenHash(input) {
|
|
10776
|
+
const row = this.db.query(`SELECT ${GRANT_COLUMNS} FROM file_download_grants WHERE project_id = ? AND token_hash = ?`).get(input.projectId, input.tokenHash);
|
|
10777
|
+
return row ? parseFileDownloadGrantRow(row) : null;
|
|
10778
|
+
}
|
|
10779
|
+
async revoke(input) {
|
|
10780
|
+
const row = this.db.query(`UPDATE file_download_grants SET revoked_at = COALESCE(revoked_at, ?)
|
|
10781
|
+
WHERE project_id = ? AND id = ?
|
|
10782
|
+
RETURNING ${GRANT_COLUMNS}`).get(input.revokedAt.toISOString(), input.projectId, input.id);
|
|
10783
|
+
return row ? parseFileDownloadGrantRow(row) : null;
|
|
10784
|
+
}
|
|
10785
|
+
async deleteExpired(input) {
|
|
10786
|
+
return this.db.query(`DELETE FROM file_download_grants WHERE rowid IN (
|
|
10787
|
+
SELECT rowid FROM file_download_grants
|
|
10788
|
+
WHERE project_id = ? AND expires_at < ?
|
|
10789
|
+
ORDER BY expires_at
|
|
10790
|
+
LIMIT ?
|
|
10791
|
+
)`).run(input.projectId, input.expiredBefore.toISOString(), input.limit).changes;
|
|
10792
|
+
}
|
|
10793
|
+
}
|
|
10794
|
+
|
|
10447
10795
|
// src/file-upload-session-storage.ts
|
|
10448
10796
|
import {
|
|
10449
10797
|
DurableFileUploadSessions,
|
|
@@ -10567,7 +10915,7 @@ var PAGE_TOKEN_PREFIX = "keyset:";
|
|
|
10567
10915
|
var DEFAULT_EXPANSION_FANOUT = 1000;
|
|
10568
10916
|
function compileObjectQuery(projectId, query, options = {}) {
|
|
10569
10917
|
if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
|
|
10570
|
-
throw new ObjectQueryExecutionError("unsupported_vector_composition", "Vector search supports one profile with filters before ranking and limit/project after.");
|
|
10918
|
+
throw new ObjectQueryExecutionError("unsupported_vector_composition", "Vector search supports one profile with filters, traversals or sets before ranking and limit/project after.");
|
|
10571
10919
|
}
|
|
10572
10920
|
const source = options.source ?? DEFAULT_OBJECT_QUERY_SOURCE;
|
|
10573
10921
|
const ctx = { probeLimit: options.includeTotal === false, source };
|
|
@@ -10584,6 +10932,8 @@ function exactContext(ctx) {
|
|
|
10584
10932
|
}
|
|
10585
10933
|
function compileObjectQueryInternal(projectId, query, ctx) {
|
|
10586
10934
|
switch (query.kind) {
|
|
10935
|
+
case "rerank":
|
|
10936
|
+
throw new ObjectQueryExecutionError("reranking_requires_runtime", "Reranking must execute in the model runtime, not storage.");
|
|
10587
10937
|
case "start":
|
|
10588
10938
|
return compileStart(projectId, query, ctx);
|
|
10589
10939
|
case "refs":
|
|
@@ -10640,9 +10990,29 @@ function compileVector(projectId, query, ctx) {
|
|
|
10640
10990
|
{ kind: "column", column: "_vector_score", direction: "desc" },
|
|
10641
10991
|
...identityOrderFields().map((field) => ({ ...field, binary: true }))
|
|
10642
10992
|
]);
|
|
10643
|
-
const
|
|
10644
|
-
|
|
10645
|
-
|
|
10993
|
+
const rankedOrder = compileOrder(order.fields, "ranked");
|
|
10994
|
+
const sql = `SELECT stored.project_id, stored.object_type_id, stored.primary_id,
|
|
10995
|
+
stored.properties, stored.properties AS _cursor_properties,
|
|
10996
|
+
stored.created_at, stored.updated_at, stored.version, stored.last_commit_id,
|
|
10997
|
+
ranked._vector_candidate_count, ranked._vector_score
|
|
10998
|
+
FROM (
|
|
10999
|
+
WITH _sixb_vector_candidates AS MATERIALIZED (
|
|
11000
|
+
SELECT input.project_id, input.object_type_id, input.primary_id,
|
|
11001
|
+
vectors.embedding AS _vector_embedding ${candidates} LIMIT ?
|
|
11002
|
+
), _sixb_vector_budget AS (
|
|
11003
|
+
SELECT COUNT(*) AS _vector_candidate_count FROM _sixb_vector_candidates
|
|
11004
|
+
)
|
|
11005
|
+
SELECT input.project_id, input.object_type_id, input.primary_id, budget._vector_candidate_count,
|
|
11006
|
+
CASE WHEN budget._vector_candidate_count <= ?
|
|
11007
|
+
THEN max(-1, min(1, 1 - vec_distance_cosine(input._vector_embedding, ?)))
|
|
11008
|
+
END AS _vector_score
|
|
11009
|
+
FROM _sixb_vector_candidates AS input CROSS JOIN _sixb_vector_budget AS budget
|
|
11010
|
+
ORDER BY ${order.sql} LIMIT ?
|
|
11011
|
+
) AS ranked CROSS JOIN ${ctx.source.objectsTable} AS stored
|
|
11012
|
+
ON stored.project_id = ranked.project_id AND stored.object_type_id = ranked.object_type_id
|
|
11013
|
+
AND stored.primary_id = ranked.primary_id
|
|
11014
|
+
ORDER BY ${rankedOrder.sql}`;
|
|
11015
|
+
const args = [...candidateArgs, limit + 1, limit, encodeSqliteVector(values), query.k];
|
|
10646
11016
|
return {
|
|
10647
11017
|
sql,
|
|
10648
11018
|
args,
|
|
@@ -10652,7 +11022,9 @@ function compileVector(projectId, query, ctx) {
|
|
|
10652
11022
|
vectorProbe: {
|
|
10653
11023
|
sql: `SELECT COUNT(*) AS total FROM (SELECT 1 ${candidates} LIMIT ?) AS candidates`,
|
|
10654
11024
|
args: [...candidateArgs, limit + 1],
|
|
10655
|
-
limit
|
|
11025
|
+
limit,
|
|
11026
|
+
rowLimit: query.k,
|
|
11027
|
+
totalLimit: query.k
|
|
10656
11028
|
},
|
|
10657
11029
|
hasMore: () => false,
|
|
10658
11030
|
trimRows: identityRows,
|
|
@@ -10811,7 +11183,11 @@ function compileLimit(projectId, inputQuery, rawLimit, ctx) {
|
|
|
10811
11183
|
FROM (${input.sql}) AS input
|
|
10812
11184
|
`,
|
|
10813
11185
|
totalArgs: input.args,
|
|
10814
|
-
vectorProbe: input.vectorProbe
|
|
11186
|
+
vectorProbe: input.vectorProbe ? {
|
|
11187
|
+
...input.vectorProbe,
|
|
11188
|
+
rowLimit: Math.min(input.vectorProbe.rowLimit, rowLimit),
|
|
11189
|
+
totalLimit: input.vectorProbe.rowLimit
|
|
11190
|
+
} : undefined,
|
|
10815
11191
|
order: input.order,
|
|
10816
11192
|
hasMore: (rowCount, total) => total === undefined ? ctx.probeLimit && rowCount > limit : limit < total,
|
|
10817
11193
|
trimRows: ctx.probeLimit ? (rows) => rows.slice(0, limit) : identityRows,
|
|
@@ -11088,7 +11464,7 @@ function compileProject(projectId, inputQuery, properties, ctx) {
|
|
|
11088
11464
|
input.created_at,
|
|
11089
11465
|
input.updated_at,
|
|
11090
11466
|
input.version,
|
|
11091
|
-
input.last_commit_id${input.vectorProbe ? ", input._vector_score" : ""}
|
|
11467
|
+
input.last_commit_id${input.vectorProbe ? ", input._vector_score, input._vector_candidate_count" : ""}
|
|
11092
11468
|
FROM (${input.sql}) AS input
|
|
11093
11469
|
ORDER BY ${inputOrder.sql}
|
|
11094
11470
|
`,
|
|
@@ -11632,19 +12008,20 @@ class SqliteObjectReader {
|
|
|
11632
12008
|
ensureSqliteVectorSearch(this.db);
|
|
11633
12009
|
const probe = compiled.vectorProbe;
|
|
11634
12010
|
const row = this.db.query(probe.sql).get(...probe.args);
|
|
11635
|
-
|
|
11636
|
-
throw new ObjectQueryExecutionError3("vector_work_limit_exceeded", `Exact vector search supports at most ${probe.limit} eligible vectors at this dimension. Narrow the query filters.`);
|
|
11637
|
-
}
|
|
12011
|
+
assertVectorCandidateCount(row?.total, probe.limit);
|
|
11638
12012
|
}
|
|
11639
12013
|
queryObjects(params) {
|
|
11640
|
-
this.assertVectorSearch(params);
|
|
11641
12014
|
const source = this.source;
|
|
11642
12015
|
const compiled = compileObjectQuery(params.projectId, params.query, {
|
|
11643
12016
|
includeTotal: params.includeTotal,
|
|
11644
12017
|
source
|
|
11645
12018
|
});
|
|
11646
|
-
|
|
12019
|
+
if (compiled.vectorProbe)
|
|
12020
|
+
ensureSqliteVectorSearch(this.db);
|
|
12021
|
+
const ordinaryTotal = params.includeTotal === false || compiled.vectorProbe ? undefined : readTotal(this.db, compiled);
|
|
11647
12022
|
const rawRows = this.db.query(compiled.sql).all(...compiled.args);
|
|
12023
|
+
const vectorTotal = this.vectorResultTotal(compiled, rawRows);
|
|
12024
|
+
const total = params.includeTotal === false ? undefined : vectorTotal ?? ordinaryTotal;
|
|
11648
12025
|
const rows = compiled.trimRows(rawRows);
|
|
11649
12026
|
const hasMore = total === undefined && compiled.hasMoreProbe ? compiled.hasMoreProbe.hasMore(this.db.query(compiled.hasMoreProbe.sql).all(...compiled.hasMoreProbe.args).length) : compiled.hasMore(rawRows.length, total);
|
|
11650
12027
|
return {
|
|
@@ -11654,6 +12031,17 @@ class SqliteObjectReader {
|
|
|
11654
12031
|
...total === undefined ? {} : { total }
|
|
11655
12032
|
};
|
|
11656
12033
|
}
|
|
12034
|
+
vectorResultTotal(compiled, rows) {
|
|
12035
|
+
const probe = compiled.vectorProbe;
|
|
12036
|
+
if (!probe)
|
|
12037
|
+
return;
|
|
12038
|
+
let count = rows[0]?._vector_candidate_count;
|
|
12039
|
+
if (rows.length === 0) {
|
|
12040
|
+
count = probe.rowLimit === 0 ? this.db.query(probe.sql).get(...probe.args)?.total : 0;
|
|
12041
|
+
}
|
|
12042
|
+
assertVectorCandidateCount(count, probe.limit);
|
|
12043
|
+
return Math.min(count, probe.totalLimit);
|
|
12044
|
+
}
|
|
11657
12045
|
countObjects(params) {
|
|
11658
12046
|
this.assertVectorSearch(params);
|
|
11659
12047
|
const source = this.source;
|
|
@@ -11678,6 +12066,7 @@ class SqliteObjectReader {
|
|
|
11678
12066
|
source
|
|
11679
12067
|
});
|
|
11680
12068
|
return {
|
|
12069
|
+
total: readTotal(this.db, compiled),
|
|
11681
12070
|
facets: params.facets.map((facet) => ({
|
|
11682
12071
|
propertyId: facet.propertyId,
|
|
11683
12072
|
buckets: readFacetBuckets(this.db, compiled, facet)
|
|
@@ -11967,6 +12356,14 @@ function sqliteFacetValue(valueType, value) {
|
|
|
11967
12356
|
}
|
|
11968
12357
|
return value;
|
|
11969
12358
|
}
|
|
12359
|
+
function assertVectorCandidateCount(count, limit) {
|
|
12360
|
+
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0) {
|
|
12361
|
+
throw new ObjectQueryExecutionError3("invalid_vector_candidate_count", "Storage did not return a valid vector candidate count.");
|
|
12362
|
+
}
|
|
12363
|
+
if (count > limit) {
|
|
12364
|
+
throw new ObjectQueryExecutionError3("vector_work_limit_exceeded", `Exact vector search supports at most ${limit} eligible vectors at this dimension. Narrow the query filters.`);
|
|
12365
|
+
}
|
|
12366
|
+
}
|
|
11970
12367
|
function existsProbeQuery(query) {
|
|
11971
12368
|
return { kind: "limit", limit: 1, input: stripOuterRowShape(query) };
|
|
11972
12369
|
}
|
|
@@ -12058,7 +12455,7 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
12058
12455
|
_sixb_scope_root_facts(project_id, root_id, node_id, object_type_id, primary_id) AS (
|
|
12059
12456
|
SELECT anchor.project_id, root.root_id, root.node_id, root.object_type_id, root.primary_id
|
|
12060
12457
|
FROM _sixb_scope_roots AS root
|
|
12061
|
-
JOIN objects AS anchor
|
|
12458
|
+
CROSS JOIN objects AS anchor
|
|
12062
12459
|
ON anchor.project_id = ?
|
|
12063
12460
|
AND anchor.object_type_id = root.object_type_id
|
|
12064
12461
|
AND anchor.primary_id = root.primary_id
|
|
@@ -12140,7 +12537,7 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
12140
12537
|
GROUP BY reachable.project_id, reachable.object_type_id, reachable.primary_id
|
|
12141
12538
|
),
|
|
12142
12539
|
_sixb_scope_objects AS (
|
|
12143
|
-
SELECT
|
|
12540
|
+
SELECT grants.project_id, grants.object_type_id, grants.primary_id,
|
|
12144
12541
|
COALESCE((
|
|
12145
12542
|
SELECT json_group_object(property.key, ${sqliteJsonEachValue("property")})
|
|
12146
12543
|
FROM json_each(stored.properties) AS property
|
|
@@ -12230,12 +12627,12 @@ function compileSelectedReadScopeCte(scopeJson, projectId) {
|
|
|
12230
12627
|
),
|
|
12231
12628
|
_sixb_scope_links AS (
|
|
12232
12629
|
SELECT
|
|
12233
|
-
|
|
12234
|
-
|
|
12235
|
-
|
|
12236
|
-
|
|
12237
|
-
|
|
12238
|
-
|
|
12630
|
+
visible.project_id,
|
|
12631
|
+
visible.source_type_id,
|
|
12632
|
+
visible.source_id,
|
|
12633
|
+
visible.link_id,
|
|
12634
|
+
visible.target_type_id,
|
|
12635
|
+
visible.target_id,
|
|
12239
12636
|
CASE
|
|
12240
12637
|
WHEN stored.properties IS NULL THEN NULL
|
|
12241
12638
|
ELSE NULLIF(
|
|
@@ -12450,9 +12847,7 @@ class SqliteObjectStorage {
|
|
|
12450
12847
|
return this.reader.existsObjects(params);
|
|
12451
12848
|
}
|
|
12452
12849
|
async facetObjects(params) {
|
|
12453
|
-
|
|
12454
|
-
return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
|
|
12455
|
-
return this.reader.facetObjects(params);
|
|
12850
|
+
return runDeferredReadTransaction(this.db, () => this.reader.facetObjects(params));
|
|
12456
12851
|
}
|
|
12457
12852
|
async getByPrimaryId(params) {
|
|
12458
12853
|
return this.reader.getByPrimaryId(params);
|
|
@@ -12501,13 +12896,14 @@ class SqliteObjectStorage {
|
|
|
12501
12896
|
}
|
|
12502
12897
|
async listByPrimaryIdPage(params) {
|
|
12503
12898
|
assertReconciliationPageLimit(params.limit);
|
|
12899
|
+
const cursor = params.afterPrimaryId === undefined ? [] : [params.afterPrimaryId];
|
|
12504
12900
|
const rows = this.db.query(`
|
|
12505
12901
|
SELECT * FROM objects
|
|
12506
12902
|
WHERE project_id = ? AND object_type_id = ?
|
|
12507
|
-
|
|
12903
|
+
${cursor.length > 0 ? "AND primary_id > ?" : ""}
|
|
12508
12904
|
ORDER BY primary_id ASC
|
|
12509
12905
|
LIMIT ?
|
|
12510
|
-
`).all(params.projectId, params.objectTypeId,
|
|
12906
|
+
`).all(params.projectId, params.objectTypeId, ...cursor, params.limit + 1);
|
|
12511
12907
|
const hasMore = rows.length > params.limit;
|
|
12512
12908
|
const objects = rows.slice(0, params.limit).map((row) => rowToObject(row));
|
|
12513
12909
|
const last = objects.at(-1);
|
|
@@ -12600,10 +12996,6 @@ function commitRecord(row) {
|
|
|
12600
12996
|
requestHash: row.request_hash,
|
|
12601
12997
|
executionId: row.execution_id,
|
|
12602
12998
|
origin: parseJson2(row.origin),
|
|
12603
|
-
...row.requested_by === null ? {} : {
|
|
12604
|
-
requestedBy: parseJson2(row.requested_by)
|
|
12605
|
-
},
|
|
12606
|
-
executor: parseJson2(row.executor),
|
|
12607
12999
|
ontologyRevision: row.ontology_revision,
|
|
12608
13000
|
...row.projection_revision === null ? {} : { projectionRevision: row.projection_revision },
|
|
12609
13001
|
...row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash },
|
|
@@ -12613,9 +13005,15 @@ function commitRecord(row) {
|
|
|
12613
13005
|
const result = parseJson2(row.result);
|
|
12614
13006
|
return { ...base, intent, result };
|
|
12615
13007
|
}
|
|
12616
|
-
function
|
|
13008
|
+
function claimedOutboxDraft(row) {
|
|
13009
|
+
if (row.lease_id === null || row.lease_expires_at === null) {
|
|
13010
|
+
throw new MaterializationConflictError("outbox-lease", "Ontology outbox claim returned an unpaired lease.");
|
|
13011
|
+
}
|
|
12617
13012
|
return {
|
|
12618
|
-
|
|
13013
|
+
id: row.id,
|
|
13014
|
+
commitId: row.commit_id,
|
|
13015
|
+
commitOrdinal: row.commit_ordinal,
|
|
13016
|
+
event: parseJson2(row.event),
|
|
12619
13017
|
availableAt: row.available_at,
|
|
12620
13018
|
attempts: row.attempts,
|
|
12621
13019
|
leaseId: row.lease_id,
|
|
@@ -12749,9 +13147,9 @@ import {
|
|
|
12749
13147
|
linkRefKey as linkRefKey3,
|
|
12750
13148
|
linkRefSortKey,
|
|
12751
13149
|
linkScopeSortKey as linkScopeSortKey2,
|
|
12752
|
-
MaterializationConflictError as
|
|
12753
|
-
MaterializationValidationError as
|
|
12754
|
-
objectRefKey as
|
|
13150
|
+
MaterializationConflictError as MaterializationConflictError7,
|
|
13151
|
+
MaterializationValidationError as MaterializationValidationError3,
|
|
13152
|
+
objectRefKey as objectRefKey4,
|
|
12755
13153
|
objectRefSortKey,
|
|
12756
13154
|
telemetryPointKey as telemetryPointKey2,
|
|
12757
13155
|
telemetryPointSortKey as telemetryPointSortKey2
|
|
@@ -12762,24 +13160,24 @@ import {
|
|
|
12762
13160
|
assertExpectedObjectRevision,
|
|
12763
13161
|
assertMaterializationFinalizationCorrelation,
|
|
12764
13162
|
assertMaterializationHeader,
|
|
12765
|
-
|
|
12766
|
-
assertPageRows,
|
|
12767
|
-
assertPlanChunkCorrelations,
|
|
13163
|
+
assertPageRows as assertPageRows2,
|
|
12768
13164
|
assertSourceActivationCorrelation,
|
|
13165
|
+
beginMaterializationApply,
|
|
13166
|
+
beginMaterializationVectorChanges,
|
|
13167
|
+
createCardinalityValidator,
|
|
12769
13168
|
invalidCorrelation as invalidCorrelation2,
|
|
13169
|
+
materializationOutboxWrite,
|
|
13170
|
+
materializationPlanChunk,
|
|
12770
13171
|
sameNonnegativeCounts as sameCounts,
|
|
12771
13172
|
uniqueSorted
|
|
12772
13173
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
12773
13174
|
|
|
12774
13175
|
// src/ontology-storage/materialization-session.ts
|
|
12775
13176
|
import {
|
|
12776
|
-
MaterializationConflictError as
|
|
13177
|
+
MaterializationConflictError as MaterializationConflictError5
|
|
12777
13178
|
} from "@sixb/core/internal/materialization";
|
|
12778
13179
|
import {
|
|
12779
|
-
correlateMaterializationChunk,
|
|
12780
13180
|
duplicateMaterializationWork as duplicateWork,
|
|
12781
|
-
invalidCorrelation,
|
|
12782
|
-
materializationPlanItems,
|
|
12783
13181
|
ProviderMaterializationSessionState,
|
|
12784
13182
|
prepareMaterializationWork
|
|
12785
13183
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
@@ -12890,18 +13288,35 @@ function stageSourceRoots(db, manifest, input) {
|
|
|
12890
13288
|
}
|
|
12891
13289
|
}
|
|
12892
13290
|
}
|
|
12893
|
-
function activateSourceRoots(db, projectId, candidate, activation) {
|
|
13291
|
+
function activateSourceRoots(db, projectId, commitId, candidate, activation) {
|
|
12894
13292
|
if (candidate.base_materialization_id !== null && (candidate.base_materialization_id !== activation.expected.activeMaterializationId || candidate.base_commit_id !== activation.expected.lastCommitId))
|
|
12895
13293
|
throw new MaterializationConflictError2("projection-fence", "Source delta base changed before activation.");
|
|
12896
13294
|
if (candidate.base_materialization_id !== null && candidate.root_count === 0)
|
|
12897
13295
|
return;
|
|
12898
13296
|
const delta = candidate.base_materialization_id !== null;
|
|
12899
|
-
|
|
12900
|
-
|
|
12901
|
-
AND
|
|
12902
|
-
|
|
12903
|
-
|
|
12904
|
-
|
|
13297
|
+
const replaced = `retired_at IS NULL AND deleted = 0
|
|
13298
|
+
AND version_id IN (SELECT version_id FROM ontology_sources
|
|
13299
|
+
WHERE project_id = ? AND source_id = ? AND status IN ${PUBLISHED})
|
|
13300
|
+
${delta ? `AND project_id = ? AND root_key IN (
|
|
13301
|
+
SELECT root_key FROM ontology_source_roots WHERE version_id = ?)` : ""}`;
|
|
13302
|
+
const replacedParams = [
|
|
13303
|
+
projectId,
|
|
13304
|
+
activation.source.projectionId,
|
|
13305
|
+
...delta ? [projectId, candidate.version_id] : []
|
|
13306
|
+
];
|
|
13307
|
+
db.query(`INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
|
|
13308
|
+
SELECT DISTINCT ?, ?, 'link', json_array(
|
|
13309
|
+
rows.source_type_id, rows.source_primary_id, rows.link_id, rows.target_type_id,
|
|
13310
|
+
rows.target_primary_id
|
|
13311
|
+
)
|
|
13312
|
+
FROM (
|
|
13313
|
+
SELECT id FROM ontology_source_roots WHERE ${replaced}
|
|
13314
|
+
UNION ALL
|
|
13315
|
+
SELECT id FROM ontology_source_roots WHERE version_id = ?
|
|
13316
|
+
) AS changed
|
|
13317
|
+
CROSS JOIN ontology_source_rows AS rows ON rows.root_id = changed.id
|
|
13318
|
+
WHERE rows.entity_kind = 'link'`).run(projectId, commitId, ...replacedParams, candidate.version_id);
|
|
13319
|
+
db.query(`UPDATE ontology_source_roots SET retired_at = ? WHERE ${replaced}`).run(activation.updatedAt, ...replacedParams);
|
|
12905
13320
|
db.query(`UPDATE ontology_source_roots SET retired_at = ?
|
|
12906
13321
|
WHERE version_id = ? AND deleted = 1`).run(activation.updatedAt, candidate.version_id);
|
|
12907
13322
|
}
|
|
@@ -12954,7 +13369,6 @@ function replacementAssertionRows(db, input) {
|
|
|
12954
13369
|
|
|
12955
13370
|
// src/ontology-storage/materialization-state.ts
|
|
12956
13371
|
var SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work";
|
|
12957
|
-
var SQLITE_REPLACEMENT_WORK_TABLE = "ontology_replacement_work";
|
|
12958
13372
|
function linkScopeRequestJson(scopes) {
|
|
12959
13373
|
return canonicalJson(scopes.map(({ source, linkId }) => ({
|
|
12960
13374
|
scopeSortKey: linkScopeSortKey(source, linkId),
|
|
@@ -13282,20 +13696,6 @@ class SqliteMaterializationStateReader {
|
|
|
13282
13696
|
cursor = rows[rows.length - 1]?.sort_key ?? null;
|
|
13283
13697
|
}
|
|
13284
13698
|
}
|
|
13285
|
-
*replacementIdentities(input) {
|
|
13286
|
-
this.prepareReplacementIdentities(input);
|
|
13287
|
-
let cursor = null;
|
|
13288
|
-
while (true) {
|
|
13289
|
-
const rows = this.replacementRows(input, cursor);
|
|
13290
|
-
if (rows.length === 0)
|
|
13291
|
-
break;
|
|
13292
|
-
yield rows.map(replacementIdentity);
|
|
13293
|
-
if (rows.length < input.pageRows)
|
|
13294
|
-
break;
|
|
13295
|
-
const last = rows[rows.length - 1];
|
|
13296
|
-
cursor = { sortKey: last.sort_key, identityKey: last.identity_key };
|
|
13297
|
-
}
|
|
13298
|
-
}
|
|
13299
13699
|
replacementObjectStates(sourceId, candidateMaterializationId, refs) {
|
|
13300
13700
|
const base = this.objectStates(refs);
|
|
13301
13701
|
const candidates = this.replacementSources(sourceId, [candidateMaterializationId], refs.map((ref) => ({ kind: "object", ref })));
|
|
@@ -13361,174 +13761,6 @@ class SqliteMaterializationStateReader {
|
|
|
13361
13761
|
}
|
|
13362
13762
|
return { owned, byMaterialization };
|
|
13363
13763
|
}
|
|
13364
|
-
prepareReplacementIdentities(input) {
|
|
13365
|
-
if (input.kind === "object") {
|
|
13366
|
-
this.prepareReplacementObjects(input);
|
|
13367
|
-
return;
|
|
13368
|
-
}
|
|
13369
|
-
this.prepareReplacementLinks(input);
|
|
13370
|
-
}
|
|
13371
|
-
prepareReplacementObjects(input) {
|
|
13372
|
-
const rows = replacementSourceRows(this.db, {
|
|
13373
|
-
projectId: this.projectId,
|
|
13374
|
-
sourceId: input.sourceId,
|
|
13375
|
-
materializationId: input.candidateMaterializationId,
|
|
13376
|
-
incremental: input.incremental ?? false
|
|
13377
|
-
});
|
|
13378
|
-
this.db.query(`
|
|
13379
|
-
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
13380
|
-
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
13381
|
-
)
|
|
13382
|
-
SELECT ?, 'object', json_array(object_type_id, primary_id),
|
|
13383
|
-
${objectSortExpression()}, 1
|
|
13384
|
-
FROM (${rows.sql}) AS rows
|
|
13385
|
-
WHERE entity_kind = 'object'
|
|
13386
|
-
GROUP BY object_type_id, primary_id
|
|
13387
|
-
`).run(input.sessionId, ...rows.values);
|
|
13388
|
-
}
|
|
13389
|
-
prepareReplacementLinks(input) {
|
|
13390
|
-
const rows = replacementSourceRows(this.db, {
|
|
13391
|
-
projectId: this.projectId,
|
|
13392
|
-
sourceId: input.sourceId,
|
|
13393
|
-
materializationId: input.candidateMaterializationId,
|
|
13394
|
-
incremental: input.incremental ?? false
|
|
13395
|
-
});
|
|
13396
|
-
this.db.query(`
|
|
13397
|
-
WITH incident_objects AS (
|
|
13398
|
-
SELECT DISTINCT
|
|
13399
|
-
json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
|
|
13400
|
-
json_extract(payload, '$.ref.primaryId') AS primary_id
|
|
13401
|
-
FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
|
|
13402
|
-
WHERE session_id = ? AND kind = 'incident-object'
|
|
13403
|
-
), replacement_links AS (
|
|
13404
|
-
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
13405
|
-
target_type_id, target_primary_id AS target_id
|
|
13406
|
-
FROM (${rows.sql}) AS rows
|
|
13407
|
-
WHERE entity_kind = 'link'
|
|
13408
|
-
), incident_links AS (
|
|
13409
|
-
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
13410
|
-
links.target_type_id, links.target_id
|
|
13411
|
-
FROM links
|
|
13412
|
-
JOIN incident_objects
|
|
13413
|
-
ON incident_objects.object_type_id = links.source_type_id
|
|
13414
|
-
AND incident_objects.primary_id = links.source_id
|
|
13415
|
-
WHERE links.project_id = ?
|
|
13416
|
-
UNION
|
|
13417
|
-
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
13418
|
-
links.target_type_id, links.target_id
|
|
13419
|
-
FROM links
|
|
13420
|
-
JOIN incident_objects
|
|
13421
|
-
ON incident_objects.object_type_id = links.target_type_id
|
|
13422
|
-
AND incident_objects.primary_id = links.target_id
|
|
13423
|
-
WHERE links.project_id = ?
|
|
13424
|
-
UNION
|
|
13425
|
-
SELECT overrides.source_type_id,
|
|
13426
|
-
overrides.source_primary_id AS source_id,
|
|
13427
|
-
overrides.link_id, overrides.target_type_id,
|
|
13428
|
-
overrides.target_primary_id AS target_id
|
|
13429
|
-
FROM ontology_link_overrides AS overrides
|
|
13430
|
-
JOIN incident_objects
|
|
13431
|
-
ON incident_objects.object_type_id = overrides.source_type_id
|
|
13432
|
-
AND incident_objects.primary_id = overrides.source_primary_id
|
|
13433
|
-
WHERE overrides.project_id = ?
|
|
13434
|
-
UNION
|
|
13435
|
-
SELECT overrides.source_type_id,
|
|
13436
|
-
overrides.source_primary_id AS source_id,
|
|
13437
|
-
overrides.link_id, overrides.target_type_id,
|
|
13438
|
-
overrides.target_primary_id AS target_id
|
|
13439
|
-
FROM ontology_link_overrides AS overrides
|
|
13440
|
-
JOIN incident_objects
|
|
13441
|
-
ON incident_objects.object_type_id = overrides.target_type_id
|
|
13442
|
-
AND incident_objects.primary_id = overrides.target_primary_id
|
|
13443
|
-
WHERE overrides.project_id = ?
|
|
13444
|
-
UNION
|
|
13445
|
-
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
13446
|
-
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
13447
|
-
FROM incident_objects
|
|
13448
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
13449
|
-
ON rows.entity_kind = 'link'
|
|
13450
|
-
AND rows.source_type_id = incident_objects.object_type_id
|
|
13451
|
-
AND rows.source_primary_id = incident_objects.primary_id
|
|
13452
|
-
${LIVE_SOURCE_ROWS_JOIN}
|
|
13453
|
-
UNION
|
|
13454
|
-
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
13455
|
-
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
13456
|
-
FROM incident_objects
|
|
13457
|
-
CROSS JOIN ontology_source_rows AS rows
|
|
13458
|
-
ON rows.entity_kind = 'link'
|
|
13459
|
-
AND rows.target_type_id = incident_objects.object_type_id
|
|
13460
|
-
AND rows.target_primary_id = incident_objects.primary_id
|
|
13461
|
-
${LIVE_SOURCE_ROWS_JOIN}
|
|
13462
|
-
), diff_links AS (
|
|
13463
|
-
SELECT * FROM replacement_links
|
|
13464
|
-
UNION SELECT * FROM incident_links
|
|
13465
|
-
), affected_scopes AS (
|
|
13466
|
-
SELECT DISTINCT source_type_id, source_id, link_id FROM diff_links
|
|
13467
|
-
), all_links AS (
|
|
13468
|
-
SELECT *, 1 AS diff_required FROM diff_links
|
|
13469
|
-
UNION ALL
|
|
13470
|
-
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
13471
|
-
links.target_type_id, links.target_id, 0 AS diff_required
|
|
13472
|
-
FROM links
|
|
13473
|
-
JOIN affected_scopes USING (source_type_id, source_id, link_id)
|
|
13474
|
-
WHERE links.project_id = ?
|
|
13475
|
-
UNION ALL
|
|
13476
|
-
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
13477
|
-
overrides.link_id, overrides.target_type_id,
|
|
13478
|
-
overrides.target_primary_id AS target_id, 0 AS diff_required
|
|
13479
|
-
FROM ontology_link_overrides AS overrides
|
|
13480
|
-
JOIN affected_scopes
|
|
13481
|
-
ON affected_scopes.source_type_id = overrides.source_type_id
|
|
13482
|
-
AND affected_scopes.source_id = overrides.source_primary_id
|
|
13483
|
-
AND affected_scopes.link_id = overrides.link_id
|
|
13484
|
-
WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
|
|
13485
|
-
), selected AS (
|
|
13486
|
-
SELECT source_type_id, source_id, link_id, target_type_id, target_id,
|
|
13487
|
-
${linkSortExpression("all_links")} AS sort_key,
|
|
13488
|
-
MAX(diff_required) AS diff_required
|
|
13489
|
-
FROM all_links
|
|
13490
|
-
GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
|
|
13491
|
-
)
|
|
13492
|
-
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
13493
|
-
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
13494
|
-
)
|
|
13495
|
-
SELECT ?, 'link',
|
|
13496
|
-
json_array(source_type_id, source_id, link_id, target_type_id, target_id),
|
|
13497
|
-
sort_key, diff_required
|
|
13498
|
-
FROM selected
|
|
13499
|
-
`).run(input.sessionId, ...rows.values, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, this.projectId, input.sessionId);
|
|
13500
|
-
}
|
|
13501
|
-
replacementRows(input, cursor) {
|
|
13502
|
-
return this.db.query(`
|
|
13503
|
-
SELECT entity_kind, identity_key, sort_key, diff_required
|
|
13504
|
-
FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
|
|
13505
|
-
WHERE session_id = ? AND entity_kind = ?
|
|
13506
|
-
AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
|
|
13507
|
-
ORDER BY sort_key, identity_key
|
|
13508
|
-
LIMIT ?
|
|
13509
|
-
`).all(input.sessionId, input.kind, cursor?.sortKey ?? null, cursor?.sortKey ?? null, cursor?.identityKey ?? null, input.pageRows);
|
|
13510
|
-
}
|
|
13511
|
-
}
|
|
13512
|
-
function replacementIdentity(row) {
|
|
13513
|
-
const parts = parseJson2(row.identity_key);
|
|
13514
|
-
if (row.entity_kind === "object") {
|
|
13515
|
-
return {
|
|
13516
|
-
kind: "object",
|
|
13517
|
-
ref: { objectTypeId: parts[0], primaryId: parts[1] },
|
|
13518
|
-
sortKey: row.sort_key,
|
|
13519
|
-
diffRequired: true
|
|
13520
|
-
};
|
|
13521
|
-
}
|
|
13522
|
-
return {
|
|
13523
|
-
kind: "link",
|
|
13524
|
-
ref: {
|
|
13525
|
-
source: { objectTypeId: parts[0], primaryId: parts[1] },
|
|
13526
|
-
linkId: parts[2],
|
|
13527
|
-
target: { objectTypeId: parts[3], primaryId: parts[4] }
|
|
13528
|
-
},
|
|
13529
|
-
sortKey: row.sort_key,
|
|
13530
|
-
diffRequired: row.diff_required === 1
|
|
13531
|
-
};
|
|
13532
13764
|
}
|
|
13533
13765
|
function activeSourceMap(rows) {
|
|
13534
13766
|
const result = new Map;
|
|
@@ -13656,20 +13888,709 @@ function storedPoint(row) {
|
|
|
13656
13888
|
lastCommitId: row.last_commit_id
|
|
13657
13889
|
};
|
|
13658
13890
|
}
|
|
13659
|
-
function objectSortExpression(alias) {
|
|
13660
|
-
const prefix = alias ? `${alias}.` : "";
|
|
13661
|
-
return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`;
|
|
13662
|
-
}
|
|
13663
13891
|
function linkSortExpression(alias) {
|
|
13664
13892
|
const prefix = alias ? `${alias}.` : "";
|
|
13665
13893
|
return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`;
|
|
13666
13894
|
}
|
|
13667
13895
|
|
|
13896
|
+
// src/ontology-storage/replacement-plans.ts
|
|
13897
|
+
import {
|
|
13898
|
+
MaterializationConflictError as MaterializationConflictError4,
|
|
13899
|
+
MaterializationValidationError as MaterializationValidationError2,
|
|
13900
|
+
objectRefKey as objectRefKey2
|
|
13901
|
+
} from "@sixb/core/internal/materialization";
|
|
13902
|
+
import {
|
|
13903
|
+
assertNonblank as assertNonblank2,
|
|
13904
|
+
assertPageRows,
|
|
13905
|
+
assertPositiveInteger as assertPositiveInteger2,
|
|
13906
|
+
assertReplacementPlanCommit,
|
|
13907
|
+
canonicalJson as canonicalJson2,
|
|
13908
|
+
duplicateMaterializationWork,
|
|
13909
|
+
invalidCorrelation,
|
|
13910
|
+
materializationWorkColumns,
|
|
13911
|
+
prepareReplacementIdentities,
|
|
13912
|
+
workUniquenessKey
|
|
13913
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
13914
|
+
var SQLITE_PLAN_WORK_TABLE = "ontology_replacement_plan_work";
|
|
13915
|
+
var IDENTITIES = "ontology_replacement_plan_identities";
|
|
13916
|
+
var SORT_KEY = "LOWER(HEX(CAST(identity_key AS BLOB)))";
|
|
13917
|
+
function objectRevision(type, id) {
|
|
13918
|
+
const series = (alias) => `${alias}.project_id = $projectId AND ${alias}.object_type_id = ${type} AND ${alias}.object_id = ${id}`;
|
|
13919
|
+
return `json_array(
|
|
13920
|
+
(SELECT json_array(version, last_commit_id) FROM objects
|
|
13921
|
+
WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}),
|
|
13922
|
+
(SELECT last_commit_id FROM ontology_object_overrides
|
|
13923
|
+
WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}),
|
|
13924
|
+
(WITH RECURSIVE properties(property_id) AS (
|
|
13925
|
+
SELECT MIN(first.property_id) FROM timeseries AS first WHERE ${series("first")}
|
|
13926
|
+
UNION ALL
|
|
13927
|
+
SELECT (SELECT MIN(next.property_id) FROM timeseries AS next
|
|
13928
|
+
WHERE ${series("next")} AND next.property_id > properties.property_id)
|
|
13929
|
+
FROM properties WHERE properties.property_id IS NOT NULL
|
|
13930
|
+
)
|
|
13931
|
+
SELECT json_group_array(json((
|
|
13932
|
+
SELECT json_array(points.property_id, points.at, points.last_commit_id)
|
|
13933
|
+
FROM timeseries AS points
|
|
13934
|
+
WHERE ${series("points")} AND points.property_id = properties.property_id
|
|
13935
|
+
ORDER BY points.at DESC LIMIT 1
|
|
13936
|
+
))) FROM properties WHERE properties.property_id IS NOT NULL)
|
|
13937
|
+
)`;
|
|
13938
|
+
}
|
|
13939
|
+
function endpointRevision(type, id) {
|
|
13940
|
+
return `CASE WHEN EXISTS (
|
|
13941
|
+
SELECT 1 FROM ${IDENTITIES} AS endpoint
|
|
13942
|
+
WHERE endpoint.version_id = $versionId AND endpoint.entity_kind = 'object'
|
|
13943
|
+
AND endpoint.identity_key = json_array(${type}, ${id})
|
|
13944
|
+
) THEN ${objectRevision(type, id)}
|
|
13945
|
+
ELSE json(CASE WHEN EXISTS (
|
|
13946
|
+
SELECT 1 FROM objects
|
|
13947
|
+
WHERE project_id = $projectId AND object_type_id = ${type} AND primary_id = ${id}
|
|
13948
|
+
) THEN 'true' ELSE 'false' END)
|
|
13949
|
+
END`;
|
|
13950
|
+
}
|
|
13951
|
+
function linkRevision(key) {
|
|
13952
|
+
const part = (index) => `json_extract(${key}, '$[${index}]')`;
|
|
13953
|
+
return `json_array(
|
|
13954
|
+
(SELECT last_commit_id FROM links
|
|
13955
|
+
WHERE project_id = $projectId AND source_type_id = ${part(0)} AND source_id = ${part(1)}
|
|
13956
|
+
AND link_id = ${part(2)} AND target_type_id = ${part(3)} AND target_id = ${part(4)}),
|
|
13957
|
+
(SELECT overrides.last_commit_id FROM ontology_link_overrides AS overrides
|
|
13958
|
+
WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'edge'
|
|
13959
|
+
AND overrides.identity_key = ${key}),
|
|
13960
|
+
(SELECT overrides.last_commit_id FROM ontology_link_overrides AS overrides
|
|
13961
|
+
WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
|
|
13962
|
+
AND overrides.identity_key = json_array(${part(0)}, ${part(1)}, ${part(2)})),
|
|
13963
|
+
(SELECT MAX(roots.id) FROM ontology_source_roots AS roots
|
|
13964
|
+
CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
13965
|
+
AND versions.status IN ${PUBLISHED}
|
|
13966
|
+
WHERE roots.project_id = $projectId AND roots.retired_at IS NULL AND roots.deleted = 0
|
|
13967
|
+
AND roots.root_key IN (
|
|
13968
|
+
'["link",' || substr(${key}, 2),
|
|
13969
|
+
'["object",' || substr(json_array(${part(0)}, ${part(1)}), 2)
|
|
13970
|
+
)
|
|
13971
|
+
AND EXISTS (
|
|
13972
|
+
-- \`+\` keeps the lookup on the root's few rows, not on every row of a hub source.
|
|
13973
|
+
SELECT 1 FROM ontology_source_rows AS rows
|
|
13974
|
+
WHERE rows.root_id = roots.id AND +rows.entity_kind = 'link'
|
|
13975
|
+
AND +rows.source_type_id = ${part(0)} AND +rows.source_primary_id = ${part(1)}
|
|
13976
|
+
AND +rows.link_id = ${part(2)} AND +rows.target_type_id = ${part(3)}
|
|
13977
|
+
AND +rows.target_primary_id = ${part(4)}
|
|
13978
|
+
)),
|
|
13979
|
+
${endpointRevision(part(0), part(1))},
|
|
13980
|
+
${endpointRevision(part(3), part(4))}
|
|
13981
|
+
)`;
|
|
13982
|
+
}
|
|
13983
|
+
function identityRevision(kind) {
|
|
13984
|
+
const key = "identities.identity_key";
|
|
13985
|
+
return kind === "object" ? objectRevision(`json_extract(${key}, '$[0]')`, `json_extract(${key}, '$[1]')`) : linkRevision(key);
|
|
13986
|
+
}
|
|
13987
|
+
function linkKeys(relation) {
|
|
13988
|
+
return `SELECT json_array(source_type_id, source_id, link_id, target_type_id, target_id)
|
|
13989
|
+
AS identity_key, json_array(source_type_id, source_id) AS source_key,
|
|
13990
|
+
json_array(target_type_id, target_id) AS target_key
|
|
13991
|
+
FROM ${relation}`;
|
|
13992
|
+
}
|
|
13993
|
+
var NEXT_EXPANSION = `(
|
|
13994
|
+
SELECT expansions FROM ontology_replacement_plans WHERE version_id = $versionId
|
|
13995
|
+
)`;
|
|
13996
|
+
var INSERT_INCIDENT_LINKS = `
|
|
13997
|
+
INSERT INTO ${IDENTITIES} (
|
|
13998
|
+
version_id, entity_kind, identity_key, sort_key, diff_required, expand_at, source_key,
|
|
13999
|
+
target_key
|
|
14000
|
+
)
|
|
14001
|
+
SELECT $versionId, 'link', identity_key, ${SORT_KEY}, 1, ${NEXT_EXPANSION}, source_key,
|
|
14002
|
+
target_key
|
|
14003
|
+
FROM (${linkKeys("incident_links")}) WHERE true
|
|
14004
|
+
ON CONFLICT (version_id, entity_kind, identity_key) DO UPDATE
|
|
14005
|
+
SET diff_required = 1, read_revision = NULL, planned_revision = NULL, classified = 0,
|
|
14006
|
+
change = NULL, expand_at = excluded.expand_at
|
|
14007
|
+
WHERE ${IDENTITIES}.diff_required = 0`;
|
|
14008
|
+
var INSERT_MEMBER_LINKS = `
|
|
14009
|
+
INSERT INTO ${IDENTITIES} (
|
|
14010
|
+
version_id, entity_kind, identity_key, sort_key, diff_required, source_key, target_key
|
|
14011
|
+
)
|
|
14012
|
+
SELECT $versionId, 'link', identity_key, ${SORT_KEY}, 0, source_key, target_key
|
|
14013
|
+
FROM (${linkKeys("members")}) WHERE true
|
|
14014
|
+
ON CONFLICT (version_id, entity_kind, identity_key) DO NOTHING`;
|
|
14015
|
+
var LIVE_ROWS = `CROSS JOIN ontology_source_roots AS roots ON roots.id = rows.root_id
|
|
14016
|
+
AND roots.project_id = $projectId AND roots.retired_at IS NULL AND roots.deleted = 0
|
|
14017
|
+
CROSS JOIN ontology_sources AS versions ON versions.version_id = roots.version_id
|
|
14018
|
+
AND versions.status IN ${PUBLISHED}`;
|
|
14019
|
+
var EXPAND_LINK_STATEMENTS = {
|
|
14020
|
+
incident: `WITH incident_objects AS MATERIALIZED (
|
|
14021
|
+
SELECT json_extract(work.payload, '$.ref.objectTypeId') AS object_type_id,
|
|
14022
|
+
json_extract(work.payload, '$.ref.primaryId') AS primary_id
|
|
14023
|
+
FROM ${IDENTITIES} AS objects
|
|
14024
|
+
CROSS JOIN ${SQLITE_PLAN_WORK_TABLE} AS work ON work.work_id = $versionId
|
|
14025
|
+
AND work.unique_key = 'incident-object:' || objects.identity_key
|
|
14026
|
+
WHERE objects.version_id = $versionId AND objects.entity_kind = 'object'
|
|
14027
|
+
AND objects.expand_at = ${NEXT_EXPANSION}
|
|
14028
|
+
), incident_links AS (
|
|
14029
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
14030
|
+
links.target_type_id, links.target_id
|
|
14031
|
+
FROM incident_objects CROSS JOIN links
|
|
14032
|
+
ON links.project_id = $projectId
|
|
14033
|
+
AND links.source_type_id = incident_objects.object_type_id
|
|
14034
|
+
AND links.source_id = incident_objects.primary_id
|
|
14035
|
+
UNION
|
|
14036
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
14037
|
+
links.target_type_id, links.target_id
|
|
14038
|
+
FROM incident_objects CROSS JOIN links INDEXED BY idx_links_target
|
|
14039
|
+
ON links.project_id = $projectId
|
|
14040
|
+
AND links.target_type_id = incident_objects.object_type_id
|
|
14041
|
+
AND links.target_id = incident_objects.primary_id
|
|
14042
|
+
UNION
|
|
14043
|
+
SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
|
|
14044
|
+
overrides.target_type_id, overrides.target_primary_id
|
|
14045
|
+
FROM incident_objects CROSS JOIN ontology_link_overrides AS overrides
|
|
14046
|
+
ON overrides.project_id = $projectId AND overrides.identity_kind IN ('edge', 'slot')
|
|
14047
|
+
AND overrides.source_type_id = incident_objects.object_type_id
|
|
14048
|
+
AND overrides.source_primary_id = incident_objects.primary_id
|
|
14049
|
+
UNION
|
|
14050
|
+
SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
|
|
14051
|
+
overrides.target_type_id, overrides.target_primary_id
|
|
14052
|
+
FROM incident_objects
|
|
14053
|
+
CROSS JOIN ontology_link_overrides AS overrides INDEXED BY idx_ontology_link_overrides_target
|
|
14054
|
+
ON overrides.project_id = $projectId
|
|
14055
|
+
AND overrides.target_type_id = incident_objects.object_type_id
|
|
14056
|
+
AND overrides.target_primary_id = incident_objects.primary_id
|
|
14057
|
+
UNION
|
|
14058
|
+
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
14059
|
+
rows.target_type_id, rows.target_primary_id
|
|
14060
|
+
FROM incident_objects CROSS JOIN ontology_source_rows AS rows
|
|
14061
|
+
ON rows.entity_kind = 'link'
|
|
14062
|
+
AND rows.source_type_id = incident_objects.object_type_id
|
|
14063
|
+
AND rows.source_primary_id = incident_objects.primary_id
|
|
14064
|
+
${LIVE_ROWS}
|
|
14065
|
+
UNION
|
|
14066
|
+
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
14067
|
+
rows.target_type_id, rows.target_primary_id
|
|
14068
|
+
FROM incident_objects CROSS JOIN ontology_source_rows AS rows
|
|
14069
|
+
ON rows.entity_kind = 'link'
|
|
14070
|
+
AND rows.target_type_id = incident_objects.object_type_id
|
|
14071
|
+
AND rows.target_primary_id = incident_objects.primary_id
|
|
14072
|
+
${LIVE_ROWS}
|
|
14073
|
+
)
|
|
14074
|
+
${INSERT_INCIDENT_LINKS}`,
|
|
14075
|
+
members: `WITH scopes AS (
|
|
14076
|
+
SELECT DISTINCT json_extract(identity_key, '$[0]') AS source_type_id,
|
|
14077
|
+
json_extract(identity_key, '$[1]') AS source_id,
|
|
14078
|
+
json_extract(identity_key, '$[2]') AS link_id
|
|
14079
|
+
FROM ${IDENTITIES}
|
|
14080
|
+
WHERE version_id = $versionId AND entity_kind = 'link' AND diff_required = 1
|
|
14081
|
+
AND expand_at = ${NEXT_EXPANSION}
|
|
14082
|
+
), members AS (
|
|
14083
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
14084
|
+
links.target_type_id, links.target_id
|
|
14085
|
+
FROM scopes CROSS JOIN links
|
|
14086
|
+
ON links.project_id = $projectId
|
|
14087
|
+
AND links.source_type_id = scopes.source_type_id
|
|
14088
|
+
AND links.source_id = scopes.source_id AND links.link_id = scopes.link_id
|
|
14089
|
+
UNION
|
|
14090
|
+
SELECT overrides.source_type_id, overrides.source_primary_id, overrides.link_id,
|
|
14091
|
+
overrides.target_type_id, overrides.target_primary_id
|
|
14092
|
+
FROM scopes CROSS JOIN ontology_link_overrides AS overrides
|
|
14093
|
+
ON overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
|
|
14094
|
+
AND overrides.source_type_id = scopes.source_type_id
|
|
14095
|
+
AND overrides.source_primary_id = scopes.source_id
|
|
14096
|
+
AND overrides.link_id = scopes.link_id
|
|
14097
|
+
)
|
|
14098
|
+
${INSERT_MEMBER_LINKS}`
|
|
14099
|
+
};
|
|
14100
|
+
var TOUCHED = `touched AS MATERIALIZED (
|
|
14101
|
+
SELECT DISTINCT touches.entity_kind, touches.identity_key
|
|
14102
|
+
FROM ontology_commits AS commits
|
|
14103
|
+
CROSS JOIN ontology_commit_touches AS touches
|
|
14104
|
+
ON touches.project_id = commits.project_id AND touches.commit_id = commits.id
|
|
14105
|
+
WHERE commits.rowid > (
|
|
14106
|
+
SELECT watermark FROM ontology_replacement_plans WHERE version_id = $versionId
|
|
14107
|
+
) AND +commits.project_id = $projectId
|
|
14108
|
+
)`;
|
|
14109
|
+
var TOUCHED_REFRESH_STATEMENTS = {
|
|
14110
|
+
object: `WITH ${TOUCHED}
|
|
14111
|
+
UPDATE ${IDENTITIES} AS identities
|
|
14112
|
+
SET read_revision = NULL, planned_revision = NULL, classified = 0, change = NULL
|
|
14113
|
+
WHERE identities.version_id = $versionId AND identities.entity_kind = 'object'
|
|
14114
|
+
AND identities.identity_key IN (SELECT identity_key FROM touched WHERE entity_kind = 'object')
|
|
14115
|
+
AND identities.planned_revision IS NOT ${identityRevision("object")}`,
|
|
14116
|
+
link: `WITH ${TOUCHED}, candidates AS (
|
|
14117
|
+
SELECT identity_key FROM touched WHERE entity_kind = 'link'
|
|
14118
|
+
UNION
|
|
14119
|
+
SELECT links.identity_key FROM touched CROSS JOIN ${IDENTITIES} AS links
|
|
14120
|
+
ON links.version_id = $versionId
|
|
14121
|
+
AND links.source_key = json_array(
|
|
14122
|
+
json_extract(touched.identity_key, '$[0]'), json_extract(touched.identity_key, '$[1]')
|
|
14123
|
+
)
|
|
14124
|
+
WHERE touched.entity_kind IN ('object', 'scope')
|
|
14125
|
+
UNION
|
|
14126
|
+
SELECT links.identity_key FROM touched CROSS JOIN ${IDENTITIES} AS links
|
|
14127
|
+
ON links.version_id = $versionId AND links.target_key = touched.identity_key
|
|
14128
|
+
WHERE touched.entity_kind = 'object'
|
|
14129
|
+
)
|
|
14130
|
+
UPDATE ${IDENTITIES} AS identities
|
|
14131
|
+
SET read_revision = NULL, planned_revision = NULL, classified = 0, change = NULL
|
|
14132
|
+
WHERE identities.version_id = $versionId AND identities.entity_kind = 'link'
|
|
14133
|
+
AND identities.identity_key IN (SELECT identity_key FROM candidates)
|
|
14134
|
+
AND identities.planned_revision IS NOT ${identityRevision("link")}`
|
|
14135
|
+
};
|
|
14136
|
+
var TOUCHED_LINKS = `touched_links AS (
|
|
14137
|
+
SELECT identity_key, json_extract(identity_key, '$[0]') AS source_type_id,
|
|
14138
|
+
json_extract(identity_key, '$[1]') AS source_id, json_extract(identity_key, '$[2]') AS link_id,
|
|
14139
|
+
json_extract(identity_key, '$[3]') AS target_type_id,
|
|
14140
|
+
json_extract(identity_key, '$[4]') AS target_id
|
|
14141
|
+
FROM touched WHERE entity_kind = 'link'
|
|
14142
|
+
), slot_targets AS (
|
|
14143
|
+
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id, overrides.link_id,
|
|
14144
|
+
overrides.target_type_id, overrides.target_primary_id AS target_id
|
|
14145
|
+
FROM touched CROSS JOIN ontology_link_overrides AS overrides
|
|
14146
|
+
ON overrides.project_id = $projectId AND overrides.identity_kind = 'slot'
|
|
14147
|
+
AND overrides.identity_key = touched.identity_key
|
|
14148
|
+
WHERE touched.entity_kind = 'scope'
|
|
14149
|
+
)`;
|
|
14150
|
+
var LINK_COLUMNS = "source_type_id, source_id, link_id, target_type_id, target_id";
|
|
14151
|
+
var EFFECTIVE_TOUCHED_LINK = `EXISTS (
|
|
14152
|
+
SELECT 1 FROM links AS stored
|
|
14153
|
+
WHERE stored.project_id = $projectId AND stored.source_type_id = touched_links.source_type_id
|
|
14154
|
+
AND stored.source_id = touched_links.source_id AND stored.link_id = touched_links.link_id
|
|
14155
|
+
AND stored.target_type_id = touched_links.target_type_id
|
|
14156
|
+
AND stored.target_id = touched_links.target_id
|
|
14157
|
+
)`;
|
|
14158
|
+
var EXPAND_TOUCHED_STATEMENTS = {
|
|
14159
|
+
incident: `WITH ${TOUCHED}, ${TOUCHED_LINKS}, incident_links AS (
|
|
14160
|
+
SELECT candidates.* FROM (
|
|
14161
|
+
SELECT ${LINK_COLUMNS} FROM touched_links
|
|
14162
|
+
WHERE ${EFFECTIVE_TOUCHED_LINK} OR EXISTS (
|
|
14163
|
+
SELECT 1 FROM ontology_link_overrides AS overrides
|
|
14164
|
+
WHERE overrides.project_id = $projectId AND overrides.identity_kind = 'edge'
|
|
14165
|
+
AND overrides.identity_key = touched_links.identity_key
|
|
14166
|
+
) OR EXISTS (
|
|
14167
|
+
SELECT 1 FROM ontology_source_rows AS rows ${LIVE_ROWS}
|
|
14168
|
+
WHERE rows.entity_kind = 'link' AND rows.source_type_id = touched_links.source_type_id
|
|
14169
|
+
AND rows.source_primary_id = touched_links.source_id
|
|
14170
|
+
AND rows.link_id = touched_links.link_id
|
|
14171
|
+
AND rows.target_type_id = touched_links.target_type_id
|
|
14172
|
+
AND rows.target_primary_id = touched_links.target_id
|
|
14173
|
+
)
|
|
14174
|
+
UNION
|
|
14175
|
+
SELECT ${LINK_COLUMNS} FROM slot_targets
|
|
14176
|
+
) AS candidates
|
|
14177
|
+
WHERE EXISTS (
|
|
14178
|
+
SELECT 1 FROM ${SQLITE_PLAN_WORK_TABLE} AS work
|
|
14179
|
+
WHERE work.work_id = $versionId AND work.unique_key IN (
|
|
14180
|
+
'incident-object:' || json_array(candidates.source_type_id, candidates.source_id),
|
|
14181
|
+
'incident-object:' || json_array(candidates.target_type_id, candidates.target_id)
|
|
14182
|
+
)
|
|
14183
|
+
)
|
|
14184
|
+
)
|
|
14185
|
+
${INSERT_INCIDENT_LINKS}`,
|
|
14186
|
+
members: `WITH ${TOUCHED}, ${TOUCHED_LINKS}, members AS (
|
|
14187
|
+
SELECT candidates.* FROM (
|
|
14188
|
+
SELECT ${LINK_COLUMNS} FROM touched_links WHERE ${EFFECTIVE_TOUCHED_LINK}
|
|
14189
|
+
UNION
|
|
14190
|
+
SELECT ${LINK_COLUMNS} FROM slot_targets
|
|
14191
|
+
) AS candidates
|
|
14192
|
+
WHERE EXISTS (
|
|
14193
|
+
SELECT 1 FROM ${IDENTITIES} AS changed
|
|
14194
|
+
WHERE changed.version_id = $versionId
|
|
14195
|
+
AND changed.source_key = json_array(candidates.source_type_id, candidates.source_id)
|
|
14196
|
+
AND changed.diff_required = 1
|
|
14197
|
+
AND json_extract(changed.identity_key, '$[2]') = candidates.link_id
|
|
14198
|
+
)
|
|
14199
|
+
)
|
|
14200
|
+
${INSERT_MEMBER_LINKS}`
|
|
14201
|
+
};
|
|
14202
|
+
var PURGE_TOUCHES_STATEMENT = `DELETE FROM ontology_commit_touches WHERE rowid IN (
|
|
14203
|
+
SELECT touches.rowid FROM ontology_commit_touches AS touches
|
|
14204
|
+
CROSS JOIN ontology_commits AS commits
|
|
14205
|
+
ON commits.project_id = touches.project_id AND commits.id = touches.commit_id
|
|
14206
|
+
WHERE touches.project_id = $projectId AND commits.rowid <= COALESCE(
|
|
14207
|
+
(SELECT MIN(plans.watermark) FROM ontology_replacement_plans AS plans
|
|
14208
|
+
JOIN ontology_sources AS versions USING (version_id)
|
|
14209
|
+
WHERE plans.project_id = $projectId AND versions.status = 'ready'),
|
|
14210
|
+
(SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits)
|
|
14211
|
+
)
|
|
14212
|
+
LIMIT $limit
|
|
14213
|
+
)`;
|
|
14214
|
+
|
|
14215
|
+
class SqliteOntologyReplacementPlanStorage {
|
|
14216
|
+
db;
|
|
14217
|
+
runRootOperation;
|
|
14218
|
+
constructor(db, runRootOperation) {
|
|
14219
|
+
this.db = db;
|
|
14220
|
+
this.runRootOperation = runRootOperation;
|
|
14221
|
+
}
|
|
14222
|
+
async open(input) {
|
|
14223
|
+
return this.runRootOperation(() => {
|
|
14224
|
+
const { commit } = input;
|
|
14225
|
+
assertReplacementPlanCommit(commit);
|
|
14226
|
+
const candidate = this.requireCandidate(input);
|
|
14227
|
+
const active = this.db.query(`SELECT materialization_id, last_commit_id FROM ontology_sources
|
|
14228
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'`).get(input.projectId, input.source.projectionId);
|
|
14229
|
+
const replacedMaterializationId = active?.materialization_id ?? null;
|
|
14230
|
+
const replacedLastCommitId = active?.last_commit_id ?? null;
|
|
14231
|
+
const existing = planRow(this.db, candidate.version_id);
|
|
14232
|
+
if (existing?.replaced_materialization_id === replacedMaterializationId && existing.replaced_last_commit_id === replacedLastCommitId) {
|
|
14233
|
+
return { committedAt: existing.committed_at };
|
|
14234
|
+
}
|
|
14235
|
+
if (existing)
|
|
14236
|
+
deletePlan(this.db, candidate.version_id);
|
|
14237
|
+
this.db.query(`INSERT INTO ontology_replacement_plans (
|
|
14238
|
+
version_id, project_id, commit_id, committed_at, replaced_materialization_id,
|
|
14239
|
+
replaced_last_commit_id, watermark
|
|
14240
|
+
) VALUES (?, ?, ?, ?, ?, ?, (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits))`).run(candidate.version_id, input.projectId, commit.id, commit.committedAt, replacedMaterializationId, replacedLastCommitId);
|
|
14241
|
+
const rows = replacementSourceRows(this.db, {
|
|
14242
|
+
projectId: input.projectId,
|
|
14243
|
+
sourceId: input.source.projectionId,
|
|
14244
|
+
materializationId: input.materializationId,
|
|
14245
|
+
incremental: candidate.base_materialization_id !== null
|
|
14246
|
+
});
|
|
14247
|
+
this.db.query(`INSERT INTO ${IDENTITIES} (
|
|
14248
|
+
version_id, entity_kind, identity_key, sort_key, diff_required, expand_at,
|
|
14249
|
+
source_key, target_key
|
|
14250
|
+
)
|
|
14251
|
+
SELECT ?, entity_kind, identity_key, ${SORT_KEY}, 1,
|
|
14252
|
+
CASE entity_kind WHEN 'link' THEN 0 END, source_key, target_key
|
|
14253
|
+
FROM (
|
|
14254
|
+
SELECT entity_kind, CASE entity_kind
|
|
14255
|
+
WHEN 'object' THEN json_array(object_type_id, primary_id)
|
|
14256
|
+
ELSE json_array(
|
|
14257
|
+
source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
14258
|
+
)
|
|
14259
|
+
END AS identity_key,
|
|
14260
|
+
CASE entity_kind
|
|
14261
|
+
WHEN 'link' THEN json_array(source_type_id, source_primary_id)
|
|
14262
|
+
END AS source_key,
|
|
14263
|
+
CASE entity_kind
|
|
14264
|
+
WHEN 'link' THEN json_array(target_type_id, target_primary_id)
|
|
14265
|
+
END AS target_key
|
|
14266
|
+
FROM (${rows.sql}) AS rows
|
|
14267
|
+
)
|
|
14268
|
+
GROUP BY entity_kind, identity_key`).run(candidate.version_id, ...rows.values);
|
|
14269
|
+
return { committedAt: commit.committedAt };
|
|
14270
|
+
});
|
|
14271
|
+
}
|
|
14272
|
+
async* streamState(input) {
|
|
14273
|
+
assertPageRows(input.pageRows);
|
|
14274
|
+
await this.runRootOperation(() => {
|
|
14275
|
+
const { candidate } = this.requirePlan(input);
|
|
14276
|
+
if (input.entityKind === "object" && candidate.projection_kind !== "object") {
|
|
14277
|
+
throw new MaterializationConflictError4("source-materialization", "Link projection replacement cannot stream object state.");
|
|
14278
|
+
}
|
|
14279
|
+
if (input.entityKind === "link") {
|
|
14280
|
+
if (unplannedCount(this.db, candidate.version_id, "object") > 0) {
|
|
14281
|
+
throw new MaterializationConflictError4("effective-state", "Object projection replacement must plan every object before its links.");
|
|
14282
|
+
}
|
|
14283
|
+
this.expandLinks(candidate.version_id, input.projectId);
|
|
14284
|
+
}
|
|
14285
|
+
});
|
|
14286
|
+
let after = "";
|
|
14287
|
+
while (true) {
|
|
14288
|
+
const page = await this.runRootOperation(() => {
|
|
14289
|
+
const { candidate } = this.requirePlan(input);
|
|
14290
|
+
const rows = this.db.query(`SELECT identity_key, sort_key, diff_required FROM ${IDENTITIES}
|
|
14291
|
+
WHERE version_id = ? AND entity_kind = ? AND planned_revision IS NULL AND sort_key > ?
|
|
14292
|
+
ORDER BY sort_key
|
|
14293
|
+
LIMIT ?`).all(candidate.version_id, input.entityKind, after, input.pageRows);
|
|
14294
|
+
if (rows.length === 0)
|
|
14295
|
+
return null;
|
|
14296
|
+
after = rows[rows.length - 1].sort_key;
|
|
14297
|
+
const state = this.readPage(input, candidate, rows);
|
|
14298
|
+
this.db.query(`UPDATE ${IDENTITIES} AS identities SET read_revision = ${identityRevision(input.entityKind)}
|
|
14299
|
+
WHERE identities.version_id = $versionId AND identities.entity_kind = $entityKind
|
|
14300
|
+
AND identities.identity_key IN (SELECT value FROM json_each($keys))`).run({
|
|
14301
|
+
$projectId: input.projectId,
|
|
14302
|
+
$versionId: candidate.version_id,
|
|
14303
|
+
$entityKind: input.entityKind,
|
|
14304
|
+
$keys: JSON.stringify(rows.map((row) => row.identity_key))
|
|
14305
|
+
});
|
|
14306
|
+
return state;
|
|
14307
|
+
});
|
|
14308
|
+
if (!page)
|
|
14309
|
+
return;
|
|
14310
|
+
yield page;
|
|
14311
|
+
}
|
|
14312
|
+
}
|
|
14313
|
+
async stage(input) {
|
|
14314
|
+
await this.runRootOperation(() => {
|
|
14315
|
+
const { plan, candidate } = this.requirePlan(input);
|
|
14316
|
+
const identities = prepareReplacementIdentities(input, {
|
|
14317
|
+
commitId: plan.commit_id,
|
|
14318
|
+
committedAt: plan.committed_at
|
|
14319
|
+
});
|
|
14320
|
+
const versionId = candidate.version_id;
|
|
14321
|
+
this.db.run("SAVEPOINT sixb_replacement_plan_stage");
|
|
14322
|
+
let recordKey;
|
|
14323
|
+
try {
|
|
14324
|
+
const known = this.db.query(`SELECT read_revision, planned_revision FROM ${IDENTITIES}
|
|
14325
|
+
WHERE version_id = ? AND entity_kind = ? AND identity_key = ?`);
|
|
14326
|
+
const insert = this.db.query(`INSERT INTO ${SQLITE_PLAN_WORK_TABLE} (
|
|
14327
|
+
work_id, entity_kind, identity_key, record_key, unique_key, kind, lane,
|
|
14328
|
+
major_order, minor_order, sort_one, sort_two,
|
|
14329
|
+
classification_entity_kind, classification_identity_key,
|
|
14330
|
+
cardinality_view, cardinality_occupied, cardinality_source_type_id,
|
|
14331
|
+
cardinality_source_primary_id, cardinality_link_id,
|
|
14332
|
+
cardinality_target_type_id, cardinality_target_primary_id, payload
|
|
14333
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, json(?))`);
|
|
14334
|
+
const planned = this.db.query(`UPDATE ${IDENTITIES}
|
|
14335
|
+
SET planned_revision = read_revision, classified = ?, change = ?,
|
|
14336
|
+
expand_at = CASE entity_kind
|
|
14337
|
+
WHEN 'object' THEN (
|
|
14338
|
+
SELECT expansions FROM ontology_replacement_plans WHERE version_id = ?
|
|
14339
|
+
)
|
|
14340
|
+
ELSE expand_at
|
|
14341
|
+
END
|
|
14342
|
+
WHERE version_id = ? AND entity_kind = ? AND identity_key = ?`);
|
|
14343
|
+
for (const identity of identities) {
|
|
14344
|
+
const row = known.get(versionId, identity.kind, identity.key);
|
|
14345
|
+
if (row?.read_revision == null || row.planned_revision !== null) {
|
|
14346
|
+
throw new MaterializationValidationError2(`Replacement identity ${identity.entityKey} is not streamed and still to plan.`);
|
|
14347
|
+
}
|
|
14348
|
+
for (const record of identity.records) {
|
|
14349
|
+
recordKey = record.recordKey;
|
|
14350
|
+
const columns = materializationWorkColumns(record);
|
|
14351
|
+
const classification = record.kind === "classification" ? record : null;
|
|
14352
|
+
const cardinality = record.kind === "cardinality" ? record : null;
|
|
14353
|
+
insert.run(versionId, identity.kind, identity.key, record.recordKey, workUniquenessKey(record), record.kind, columns.lane, columns.majorOrder, columns.minorOrder, columns.sortOne, columns.sortTwo, classification?.entityKind ?? null, classification?.identityKey ?? null, cardinality?.view ?? null, cardinality ? Number(cardinality.occupied) : null, cardinality?.ref.source.objectTypeId ?? null, cardinality?.ref.source.primaryId ?? null, cardinality?.ref.linkId ?? null, cardinality?.ref.target.objectTypeId ?? null, cardinality?.ref.target.primaryId ?? null, canonicalJson2(record));
|
|
14354
|
+
}
|
|
14355
|
+
planned.run(Number(identity.classified), identity.change, versionId, versionId, identity.kind, identity.key);
|
|
14356
|
+
}
|
|
14357
|
+
this.db.run("RELEASE SAVEPOINT sixb_replacement_plan_stage");
|
|
14358
|
+
} catch (error) {
|
|
14359
|
+
this.db.run("ROLLBACK TO SAVEPOINT sixb_replacement_plan_stage");
|
|
14360
|
+
this.db.run("RELEASE SAVEPOINT sixb_replacement_plan_stage");
|
|
14361
|
+
if (isSqliteConstraintError(error))
|
|
14362
|
+
throw duplicateMaterializationWork(recordKey);
|
|
14363
|
+
throw error;
|
|
14364
|
+
}
|
|
14365
|
+
});
|
|
14366
|
+
}
|
|
14367
|
+
async refresh(input) {
|
|
14368
|
+
return this.runRootOperation(() => {
|
|
14369
|
+
const { plan, candidate } = this.requirePlan(input);
|
|
14370
|
+
const versionId = candidate.version_id;
|
|
14371
|
+
const pending = unplannedCount(this.db, versionId);
|
|
14372
|
+
if (pending > 0)
|
|
14373
|
+
return { fresh: false, unplanned: pending };
|
|
14374
|
+
if (!committedSince(this.db, plan, input.projectId)) {
|
|
14375
|
+
return { fresh: true, counts: replacementPlanCounts(this.db, versionId) };
|
|
14376
|
+
}
|
|
14377
|
+
const params = { $projectId: input.projectId, $versionId: versionId };
|
|
14378
|
+
let unplanned = 0;
|
|
14379
|
+
for (const kind of ["object", "link"]) {
|
|
14380
|
+
const stale = this.db.query(TOUCHED_REFRESH_STATEMENTS[kind]).run(params).changes;
|
|
14381
|
+
if (stale === 0)
|
|
14382
|
+
continue;
|
|
14383
|
+
dropUnplannedWork(this.db, versionId, kind);
|
|
14384
|
+
unplanned += stale;
|
|
14385
|
+
}
|
|
14386
|
+
unplanned += this.expandTouched(versionId, input.projectId);
|
|
14387
|
+
this.db.query(`UPDATE ontology_replacement_plans
|
|
14388
|
+
SET watermark = (SELECT COALESCE(MAX(rowid), 0) FROM ontology_commits)
|
|
14389
|
+
WHERE version_id = ?`).run(versionId);
|
|
14390
|
+
if (unplanned > 0)
|
|
14391
|
+
return { fresh: false, unplanned };
|
|
14392
|
+
return { fresh: true, counts: replacementPlanCounts(this.db, versionId) };
|
|
14393
|
+
});
|
|
14394
|
+
}
|
|
14395
|
+
async purge(input) {
|
|
14396
|
+
return this.runRootOperation(() => {
|
|
14397
|
+
assertNonblank2(input.projectId, "Replacement plan purge project id");
|
|
14398
|
+
assertPositiveInteger2(input.limit, "Replacement plan purge limit");
|
|
14399
|
+
const spentPlan = this.db.query(`SELECT plans.version_id FROM ontology_replacement_plans AS plans
|
|
14400
|
+
JOIN ontology_sources AS versions USING (version_id)
|
|
14401
|
+
WHERE plans.project_id = ? AND versions.status <> 'ready'
|
|
14402
|
+
ORDER BY plans.version_id
|
|
14403
|
+
LIMIT 1`);
|
|
14404
|
+
let deleted = 0;
|
|
14405
|
+
while (deleted < input.limit) {
|
|
14406
|
+
const spent = spentPlan.get(input.projectId);
|
|
14407
|
+
if (!spent)
|
|
14408
|
+
break;
|
|
14409
|
+
for (const [table, column] of [
|
|
14410
|
+
[SQLITE_PLAN_WORK_TABLE, "work_id"],
|
|
14411
|
+
[IDENTITIES, "version_id"]
|
|
14412
|
+
]) {
|
|
14413
|
+
deleted += this.db.query(`DELETE FROM ${table} WHERE rowid IN (
|
|
14414
|
+
SELECT rowid FROM ${table} WHERE ${column} = ? LIMIT ?
|
|
14415
|
+
)`).run(spent.version_id, input.limit - deleted).changes;
|
|
14416
|
+
if (deleted >= input.limit)
|
|
14417
|
+
return deleted;
|
|
14418
|
+
}
|
|
14419
|
+
this.db.query(`DELETE FROM ontology_replacement_plans WHERE version_id = ?`).run(spent.version_id);
|
|
14420
|
+
deleted += 1;
|
|
14421
|
+
}
|
|
14422
|
+
if (deleted >= input.limit)
|
|
14423
|
+
return deleted;
|
|
14424
|
+
return deleted + this.db.query(PURGE_TOUCHES_STATEMENT).run({ $projectId: input.projectId, $limit: input.limit - deleted }).changes;
|
|
14425
|
+
});
|
|
14426
|
+
}
|
|
14427
|
+
readPage(input, candidate, rows) {
|
|
14428
|
+
const reader = new SqliteMaterializationStateReader(this.db, input.projectId);
|
|
14429
|
+
if (input.entityKind === "object") {
|
|
14430
|
+
const refs = rows.map((row) => {
|
|
14431
|
+
const [objectTypeId, primaryId] = parseJson2(row.identity_key);
|
|
14432
|
+
return { objectTypeId, primaryId };
|
|
14433
|
+
});
|
|
14434
|
+
return {
|
|
14435
|
+
objects: reader.replacementObjectStates(input.source.projectionId, input.materializationId, refs),
|
|
14436
|
+
links: [],
|
|
14437
|
+
endpoints: []
|
|
14438
|
+
};
|
|
14439
|
+
}
|
|
14440
|
+
const identities = rows.map((row) => {
|
|
14441
|
+
const parts = parseJson2(row.identity_key);
|
|
14442
|
+
const ref = {
|
|
14443
|
+
source: { objectTypeId: parts[0], primaryId: parts[1] },
|
|
14444
|
+
linkId: parts[2],
|
|
14445
|
+
target: { objectTypeId: parts[3], primaryId: parts[4] }
|
|
14446
|
+
};
|
|
14447
|
+
return {
|
|
14448
|
+
ref,
|
|
14449
|
+
sortKey: row.sort_key,
|
|
14450
|
+
diffRequired: row.diff_required === 1
|
|
14451
|
+
};
|
|
14452
|
+
});
|
|
14453
|
+
const previous = this.db.query(`SELECT materialization_id FROM ontology_sources
|
|
14454
|
+
WHERE project_id = ? AND source_id = ? AND status = 'active'`).get(input.projectId, input.source.projectionId);
|
|
14455
|
+
const links = reader.replacementLinkStates(input.source.projectionId, input.materializationId, [input.materializationId, ...previous ? [previous.materialization_id] : []], identities, candidate.base_materialization_id !== null);
|
|
14456
|
+
return {
|
|
14457
|
+
objects: [],
|
|
14458
|
+
links,
|
|
14459
|
+
endpoints: this.endpoints(candidate.version_id, input.projectId, identities.flatMap(({ ref }) => [ref.source, ref.target]))
|
|
14460
|
+
};
|
|
14461
|
+
}
|
|
14462
|
+
endpoints(versionId, projectId, refs) {
|
|
14463
|
+
const unique = [...new Map(refs.map((ref) => [objectRefKey2(ref), ref])).values()];
|
|
14464
|
+
const rows = this.db.query(`SELECT requested.value AS identity_key, (
|
|
14465
|
+
SELECT json_extract(work.payload, '$.exists') FROM ${SQLITE_PLAN_WORK_TABLE} AS work
|
|
14466
|
+
WHERE work.work_id = ? AND work.unique_key = 'object-existence:' || requested.value
|
|
14467
|
+
) AS planned, EXISTS (
|
|
14468
|
+
SELECT 1 FROM objects
|
|
14469
|
+
WHERE project_id = ? AND object_type_id = json_extract(requested.value, '$[0]')
|
|
14470
|
+
AND primary_id = json_extract(requested.value, '$[1]')
|
|
14471
|
+
) AS effective
|
|
14472
|
+
FROM json_each(?) AS requested`).all(versionId, projectId, JSON.stringify(unique.map((ref) => objectRefKey2(ref))));
|
|
14473
|
+
const existence = new Map(rows.map((row) => [row.identity_key, (row.planned ?? row.effective) === 1]));
|
|
14474
|
+
return unique.map((ref) => ({ ref, exists: existence.get(objectRefKey2(ref)) ?? false }));
|
|
14475
|
+
}
|
|
14476
|
+
expandLinks(versionId, projectId) {
|
|
14477
|
+
const added = this.expand(versionId, projectId, EXPAND_LINK_STATEMENTS);
|
|
14478
|
+
this.db.query(`UPDATE ontology_replacement_plans SET expansions = expansions + 1 WHERE version_id = ?`).run(versionId);
|
|
14479
|
+
return added;
|
|
14480
|
+
}
|
|
14481
|
+
expandTouched(versionId, projectId) {
|
|
14482
|
+
return this.expand(versionId, projectId, EXPAND_TOUCHED_STATEMENTS);
|
|
14483
|
+
}
|
|
14484
|
+
expand(versionId, projectId, statements) {
|
|
14485
|
+
const run = (statement) => this.db.query(statement).run({ $projectId: projectId, $versionId: versionId }).changes;
|
|
14486
|
+
const diffLinks = run(statements.incident);
|
|
14487
|
+
if (diffLinks > 0)
|
|
14488
|
+
dropUnplannedWork(this.db, versionId, "link");
|
|
14489
|
+
return diffLinks + run(statements.members);
|
|
14490
|
+
}
|
|
14491
|
+
requireCandidate(input) {
|
|
14492
|
+
const candidate = sourceRow(this.db, input.projectId, input.source.projectionId, input.materializationId);
|
|
14493
|
+
if (!candidate || candidate.status !== "ready" || candidate.projection_run_id !== input.execution.projectionRunId || candidate.execution_token !== input.execution.executionToken) {
|
|
14494
|
+
throw new MaterializationConflictError4("source-materialization", `Candidate source materialization '${input.materializationId}' is missing, not ready, or owned by another execution.`);
|
|
14495
|
+
}
|
|
14496
|
+
assertProjectionExecution(this.db, {
|
|
14497
|
+
projectId: input.projectId,
|
|
14498
|
+
sourceId: input.source.projectionId,
|
|
14499
|
+
projectionRunId: input.execution.projectionRunId,
|
|
14500
|
+
executionToken: input.execution.executionToken
|
|
14501
|
+
});
|
|
14502
|
+
return candidate;
|
|
14503
|
+
}
|
|
14504
|
+
requirePlan(input) {
|
|
14505
|
+
const candidate = this.requireCandidate(input);
|
|
14506
|
+
const plan = planRow(this.db, candidate.version_id);
|
|
14507
|
+
if (!plan) {
|
|
14508
|
+
throw new MaterializationConflictError4("source-materialization", `Candidate source materialization '${input.materializationId}' has no open plan.`);
|
|
14509
|
+
}
|
|
14510
|
+
return { plan, candidate };
|
|
14511
|
+
}
|
|
14512
|
+
}
|
|
14513
|
+
function assertReplacementPlanCoverage(db, plan) {
|
|
14514
|
+
const mismatch = db.query(`SELECT 1 FROM ${IDENTITIES}
|
|
14515
|
+
WHERE version_id = ?
|
|
14516
|
+
AND classified <> (diff_required AND (entity_kind = 'link' OR ? = 1))
|
|
14517
|
+
LIMIT 1`).get(plan.versionId, plan.projectionKind === "object" ? 1 : 0);
|
|
14518
|
+
if (mismatch) {
|
|
14519
|
+
invalidCorrelation("Projection replacement classification coverage does not match its plan.");
|
|
14520
|
+
}
|
|
14521
|
+
}
|
|
14522
|
+
function replacementPlanCounts(db, versionId) {
|
|
14523
|
+
const rows = db.query(`SELECT entity_kind, COALESCE(change, 'unchanged') AS change, COUNT(*) AS count
|
|
14524
|
+
FROM ${IDENTITIES}
|
|
14525
|
+
WHERE version_id = ? AND classified = 1
|
|
14526
|
+
GROUP BY entity_kind, COALESCE(change, 'unchanged')`).all(versionId);
|
|
14527
|
+
const count = (kind, change) => rows.find((row) => row.entity_kind === kind && row.change === change)?.count ?? 0;
|
|
14528
|
+
return {
|
|
14529
|
+
objectsCreated: count("object", "created"),
|
|
14530
|
+
objectsUpdated: count("object", "updated"),
|
|
14531
|
+
objectsDeleted: count("object", "deleted"),
|
|
14532
|
+
objectsUnchanged: count("object", "unchanged"),
|
|
14533
|
+
linksCreated: count("link", "created"),
|
|
14534
|
+
linksUpdated: count("link", "updated"),
|
|
14535
|
+
linksDeleted: count("link", "deleted"),
|
|
14536
|
+
linksUnchanged: count("link", "unchanged")
|
|
14537
|
+
};
|
|
14538
|
+
}
|
|
14539
|
+
function unplannedCount(db, versionId, kind) {
|
|
14540
|
+
const row = db.query(`SELECT COUNT(*) AS count FROM ${IDENTITIES}
|
|
14541
|
+
WHERE version_id = ? AND planned_revision IS NULL AND (? IS NULL OR entity_kind = ?)`).get(versionId, kind ?? null, kind ?? null);
|
|
14542
|
+
return row.count;
|
|
14543
|
+
}
|
|
14544
|
+
function committedSince(db, plan, projectId) {
|
|
14545
|
+
return db.query(`SELECT 1 FROM ontology_commits WHERE rowid > ? AND +project_id = ? LIMIT 1`).get(plan.watermark, projectId) !== null;
|
|
14546
|
+
}
|
|
14547
|
+
function dropUnplannedWork(db, versionId, kind) {
|
|
14548
|
+
db.query(`DELETE FROM ${SQLITE_PLAN_WORK_TABLE}
|
|
14549
|
+
WHERE work_id = ? AND entity_kind = ? AND identity_key IN (
|
|
14550
|
+
SELECT identity_key FROM ${IDENTITIES}
|
|
14551
|
+
WHERE version_id = ? AND entity_kind = ? AND planned_revision IS NULL
|
|
14552
|
+
)`).run(versionId, kind, versionId, kind);
|
|
14553
|
+
}
|
|
14554
|
+
function deletePlan(db, versionId) {
|
|
14555
|
+
db.query(`DELETE FROM ${SQLITE_PLAN_WORK_TABLE} WHERE work_id = ?`).run(versionId);
|
|
14556
|
+
db.query(`DELETE FROM ${IDENTITIES} WHERE version_id = ?`).run(versionId);
|
|
14557
|
+
db.query(`DELETE FROM ontology_replacement_plans WHERE version_id = ?`).run(versionId);
|
|
14558
|
+
}
|
|
14559
|
+
function planRow(db, versionId) {
|
|
14560
|
+
return db.query(`SELECT * FROM ontology_replacement_plans WHERE version_id = ?`).get(versionId);
|
|
14561
|
+
}
|
|
14562
|
+
function sourceRow(db, projectId, sourceId, materializationId) {
|
|
14563
|
+
return db.query(`SELECT * FROM ontology_sources
|
|
14564
|
+
WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).get(projectId, sourceId, materializationId);
|
|
14565
|
+
}
|
|
14566
|
+
function boundReplacementPlan(db, header) {
|
|
14567
|
+
const ref = header.plan;
|
|
14568
|
+
const candidate = ref ? sourceRow(db, header.commit.projectId, ref.source.projectionId, ref.materializationId) : null;
|
|
14569
|
+
const plan = candidate ? planRow(db, candidate.version_id) : null;
|
|
14570
|
+
if (!ref || !candidate || !plan || header.commit.intent.kind !== "projection") {
|
|
14571
|
+
throw new MaterializationConflictError4("source-materialization", "A plan-bound session needs the open plan of a projection candidate.");
|
|
14572
|
+
}
|
|
14573
|
+
if (plan.commit_id !== header.commit.id || plan.committed_at !== header.commit.committedAt) {
|
|
14574
|
+
throw new MaterializationValidationError2("A plan-bound session must begin with the commit its plan carries.");
|
|
14575
|
+
}
|
|
14576
|
+
if (committedSince(db, plan, header.commit.projectId) || unplannedCount(db, candidate.version_id) > 0) {
|
|
14577
|
+
throw new MaterializationValidationError2("A replacement plan applies only fully planned and just refreshed.");
|
|
14578
|
+
}
|
|
14579
|
+
return { versionId: candidate.version_id, projectionKind: candidate.projection_kind };
|
|
14580
|
+
}
|
|
14581
|
+
|
|
13668
14582
|
// src/ontology-storage/materialization-session.ts
|
|
13669
14583
|
class SqliteMaterializationSessionState extends ProviderMaterializationSessionState {
|
|
13670
|
-
|
|
13671
|
-
appliedEventCursor = null;
|
|
14584
|
+
plan;
|
|
13672
14585
|
summary = emptyWorkSummary();
|
|
14586
|
+
workTable;
|
|
14587
|
+
workId;
|
|
14588
|
+
constructor(header, transactionId, plan) {
|
|
14589
|
+
super(header, transactionId);
|
|
14590
|
+
this.plan = plan;
|
|
14591
|
+
this.workTable = plan ? SQLITE_PLAN_WORK_TABLE : SQLITE_MATERIALIZATION_WORK_TABLE;
|
|
14592
|
+
this.workId = plan ? plan.versionId : this.id;
|
|
14593
|
+
}
|
|
13673
14594
|
}
|
|
13674
14595
|
|
|
13675
14596
|
class SqliteMaterializationSessions {
|
|
@@ -13681,12 +14602,12 @@ class SqliteMaterializationSessions {
|
|
|
13681
14602
|
this.db = db;
|
|
13682
14603
|
this.context = context;
|
|
13683
14604
|
}
|
|
13684
|
-
create(header) {
|
|
14605
|
+
create(header, plan) {
|
|
13685
14606
|
if (!this.context?.active) {
|
|
13686
|
-
throw new
|
|
14607
|
+
throw new MaterializationConflictError5("effective-state", "Materialization sessions require an active storage transaction.");
|
|
13687
14608
|
}
|
|
13688
14609
|
this.ensureTables();
|
|
13689
|
-
const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id);
|
|
14610
|
+
const session = new SqliteMaterializationSessionState(structuredClone(header), this.context.id, plan);
|
|
13690
14611
|
this.sessions.set(session.providerToken, session);
|
|
13691
14612
|
this.live.add(session);
|
|
13692
14613
|
this.context.materializations.register(session.providerToken);
|
|
@@ -13695,7 +14616,7 @@ class SqliteMaterializationSessions {
|
|
|
13695
14616
|
require(session) {
|
|
13696
14617
|
const value = this.sessions.get(session.providerToken);
|
|
13697
14618
|
if (!value || !value.active || !this.context?.active || value.transactionId !== this.context.id) {
|
|
13698
|
-
throw new
|
|
14619
|
+
throw new MaterializationConflictError5("effective-state", "Materialization session is inactive.");
|
|
13699
14620
|
}
|
|
13700
14621
|
return value;
|
|
13701
14622
|
}
|
|
@@ -13703,11 +14624,10 @@ class SqliteMaterializationSessions {
|
|
|
13703
14624
|
if (!session.active)
|
|
13704
14625
|
return;
|
|
13705
14626
|
session.active = false;
|
|
13706
|
-
session.replacement = null;
|
|
13707
14627
|
this.live.delete(session);
|
|
13708
14628
|
if (this.live.size === 0)
|
|
13709
14629
|
this.dropWorkTables();
|
|
13710
|
-
else
|
|
14630
|
+
else if (!session.plan)
|
|
13711
14631
|
this.deleteWork(session.id);
|
|
13712
14632
|
this.context?.materializations.complete(session.providerToken);
|
|
13713
14633
|
}
|
|
@@ -13722,7 +14642,7 @@ class SqliteMaterializationSessions {
|
|
|
13722
14642
|
return;
|
|
13723
14643
|
const insert = this.db.query(`
|
|
13724
14644
|
INSERT INTO ${SQLITE_MATERIALIZATION_WORK_TABLE} (
|
|
13725
|
-
|
|
14645
|
+
work_id, record_key, unique_key, kind, lane,
|
|
13726
14646
|
major_order, minor_order, sort_one, sort_two,
|
|
13727
14647
|
classification_entity_kind, classification_identity_key,
|
|
13728
14648
|
cardinality_view, cardinality_occupied, cardinality_source_type_id,
|
|
@@ -13749,74 +14669,62 @@ class SqliteMaterializationSessions {
|
|
|
13749
14669
|
throw error;
|
|
13750
14670
|
}
|
|
13751
14671
|
for (const prepared of records)
|
|
13752
|
-
recordSummary(session.summary, prepared.record
|
|
14672
|
+
recordSummary(session.summary, prepared.record);
|
|
13753
14673
|
}
|
|
13754
|
-
|
|
13755
|
-
const session = this.require(input.session);
|
|
13756
|
-
const stream = session.workStreams[input.order];
|
|
13757
|
-
if (stream.started) {
|
|
13758
|
-
throw new MaterializationConflictError4("effective-state", `Materialization ${input.order} work may only be streamed once per session.`);
|
|
13759
|
-
}
|
|
13760
|
-
stream.started = true;
|
|
13761
|
-
session.workSealed = true;
|
|
14674
|
+
*lanePages(session, lane, pageRows) {
|
|
13762
14675
|
let cursor = null;
|
|
13763
14676
|
while (true) {
|
|
13764
|
-
this.require(
|
|
13765
|
-
const rows = this.readLane(
|
|
14677
|
+
this.require(session.publicSession());
|
|
14678
|
+
const rows = this.readLane(session, lane, cursor, pageRows);
|
|
13766
14679
|
if (rows.length === 0)
|
|
13767
|
-
|
|
13768
|
-
const records = rows.map((row) => parseJson2(row.payload));
|
|
14680
|
+
return;
|
|
13769
14681
|
cursor = workCursor(rows[rows.length - 1]);
|
|
13770
|
-
|
|
13771
|
-
yield { records };
|
|
14682
|
+
yield rows.map((row) => parseJson2(row.payload));
|
|
13772
14683
|
}
|
|
13773
|
-
this.require(input.session);
|
|
13774
|
-
stream.completed = true;
|
|
13775
14684
|
}
|
|
13776
|
-
|
|
13777
|
-
|
|
13778
|
-
|
|
13779
|
-
|
|
14685
|
+
*vectorChangePages(session, objectTypeIds, pageRows) {
|
|
14686
|
+
if (objectTypeIds.length === 0 && !this.db.query(`SELECT 1 FROM object_vectors WHERE project_id = ? LIMIT 1`).get(session.header.commit.projectId)) {
|
|
14687
|
+
return;
|
|
14688
|
+
}
|
|
14689
|
+
const query = this.db.query(`
|
|
14690
|
+
WITH changes AS (
|
|
14691
|
+
SELECT record_key, payload,
|
|
14692
|
+
coalesce(
|
|
14693
|
+
json_extract(payload, '$.item.value.row.ref.objectTypeId'),
|
|
14694
|
+
json_extract(payload, '$.item.value.ref.objectTypeId')
|
|
14695
|
+
) AS object_type_id,
|
|
14696
|
+
coalesce(
|
|
14697
|
+
json_extract(payload, '$.item.value.row.ref.primaryId'),
|
|
14698
|
+
json_extract(payload, '$.item.value.ref.primaryId')
|
|
14699
|
+
) AS primary_id
|
|
14700
|
+
FROM ${session.workTable}
|
|
14701
|
+
WHERE work_id = ? AND lane = 'apply' AND kind = 'plan'
|
|
14702
|
+
AND json_extract(payload, '$.item.kind') IN ('object-upsert', 'object-delete')
|
|
14703
|
+
AND record_key > ?
|
|
14704
|
+
)
|
|
14705
|
+
SELECT record_key, payload FROM changes
|
|
14706
|
+
WHERE object_type_id IN (SELECT value FROM json_each(?))
|
|
14707
|
+
OR EXISTS (
|
|
14708
|
+
SELECT 1 FROM object_vectors
|
|
14709
|
+
WHERE project_id = ? AND object_type_id = changes.object_type_id
|
|
14710
|
+
AND primary_id = changes.primary_id
|
|
14711
|
+
)
|
|
14712
|
+
ORDER BY record_key
|
|
14713
|
+
LIMIT ?
|
|
13780
14714
|
`);
|
|
13781
|
-
|
|
13782
|
-
|
|
13783
|
-
|
|
13784
|
-
|
|
13785
|
-
|
|
13786
|
-
|
|
13787
|
-
|
|
13788
|
-
|
|
13789
|
-
|
|
13790
|
-
prepareChunkSequence(session, chunk) {
|
|
13791
|
-
const items = materializationPlanItems(chunk);
|
|
13792
|
-
const planRows = this.readLane("apply", session.id, session.appliedPlanCursor, items.length);
|
|
13793
|
-
const expectedItems = planRows.map((row) => {
|
|
13794
|
-
const record = parseJson2(row.payload);
|
|
13795
|
-
return record.item;
|
|
13796
|
-
});
|
|
13797
|
-
const eventRows = this.readLane("event", session.id, session.appliedEventCursor, chunk.outbox.length);
|
|
13798
|
-
const expectedEvents = eventRows.map((row) => parseJson2(row.payload));
|
|
13799
|
-
const progress = correlateMaterializationChunk(session, chunk, items, expectedItems, expectedEvents);
|
|
13800
|
-
return {
|
|
13801
|
-
...progress,
|
|
13802
|
-
appliedPlanCursor: planRows.length === 0 ? session.appliedPlanCursor : workCursor(planRows[planRows.length - 1]),
|
|
13803
|
-
appliedEventCursor: eventRows.length === 0 ? session.appliedEventCursor : workCursor(eventRows[eventRows.length - 1])
|
|
13804
|
-
};
|
|
13805
|
-
}
|
|
13806
|
-
commitChunkSequence(session, progress) {
|
|
13807
|
-
session.appliedPlanCount = progress.appliedPlanCount;
|
|
13808
|
-
session.appliedOutboxCount = progress.appliedOutboxCount;
|
|
13809
|
-
session.appliedPlanCursor = progress.appliedPlanCursor;
|
|
13810
|
-
session.appliedEventCursor = progress.appliedEventCursor;
|
|
13811
|
-
}
|
|
13812
|
-
laneCounts(session) {
|
|
13813
|
-
return {
|
|
13814
|
-
apply: session.summary.apply,
|
|
13815
|
-
cardinality: session.summary.cardinality,
|
|
13816
|
-
event: session.summary.event
|
|
13817
|
-
};
|
|
14715
|
+
let after = "";
|
|
14716
|
+
while (true) {
|
|
14717
|
+
this.require(session.publicSession());
|
|
14718
|
+
const rows = query.all(session.workId, after, JSON.stringify(objectTypeIds), session.header.commit.projectId, pageRows);
|
|
14719
|
+
if (rows.length === 0)
|
|
14720
|
+
return;
|
|
14721
|
+
after = rows[rows.length - 1].record_key;
|
|
14722
|
+
yield rows.map((row) => parseJson2(row.payload).item);
|
|
14723
|
+
}
|
|
13818
14724
|
}
|
|
13819
14725
|
projectionCounts(session) {
|
|
14726
|
+
if (session.plan)
|
|
14727
|
+
return replacementPlanCounts(this.db, session.plan.versionId);
|
|
13820
14728
|
const summary = session.summary;
|
|
13821
14729
|
return {
|
|
13822
14730
|
objectsCreated: summary.objectsCreated,
|
|
@@ -13842,67 +14750,29 @@ class SqliteMaterializationSessions {
|
|
|
13842
14750
|
};
|
|
13843
14751
|
}
|
|
13844
14752
|
assertClassificationCoverage(session) {
|
|
13845
|
-
if (
|
|
13846
|
-
|
|
13847
|
-
const expectedObject = session.replacement.projectionKind === "object" ? 1 : 0;
|
|
13848
|
-
const mismatch = this.db.query(`
|
|
13849
|
-
WITH invalid AS (
|
|
13850
|
-
SELECT 1
|
|
13851
|
-
FROM ${SQLITE_REPLACEMENT_WORK_TABLE} AS expected
|
|
13852
|
-
WHERE expected.session_id = ? AND expected.diff_required = 1
|
|
13853
|
-
AND (expected.entity_kind = 'link' OR ? = 1)
|
|
13854
|
-
AND NOT EXISTS (
|
|
13855
|
-
SELECT 1 FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} AS actual
|
|
13856
|
-
WHERE actual.session_id = expected.session_id
|
|
13857
|
-
AND actual.unique_key = 'classification:' || expected.entity_kind || ':'
|
|
13858
|
-
|| expected.identity_key
|
|
13859
|
-
)
|
|
13860
|
-
UNION ALL
|
|
13861
|
-
SELECT 1
|
|
13862
|
-
FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} AS actual
|
|
13863
|
-
WHERE actual.session_id = ?
|
|
13864
|
-
-- A bounded range uses the existing (session_id, unique_key) unique index and scans
|
|
13865
|
-
-- classification entries only, instead of sorting the complete JSON work spool.
|
|
13866
|
-
AND actual.unique_key >= 'classification:'
|
|
13867
|
-
AND actual.unique_key < 'classification;'
|
|
13868
|
-
AND (
|
|
13869
|
-
actual.classification_entity_kind = 'point'
|
|
13870
|
-
OR NOT EXISTS (
|
|
13871
|
-
SELECT 1 FROM ${SQLITE_REPLACEMENT_WORK_TABLE} AS expected
|
|
13872
|
-
WHERE expected.session_id = actual.session_id
|
|
13873
|
-
AND expected.entity_kind = actual.classification_entity_kind
|
|
13874
|
-
AND expected.identity_key = actual.classification_identity_key
|
|
13875
|
-
AND expected.diff_required = 1
|
|
13876
|
-
AND (expected.entity_kind = 'link' OR ? = 1)
|
|
13877
|
-
)
|
|
13878
|
-
)
|
|
13879
|
-
)
|
|
13880
|
-
SELECT 1 FROM invalid LIMIT 1
|
|
13881
|
-
`).get(session.id, expectedObject, session.id, expectedObject);
|
|
13882
|
-
if (mismatch) {
|
|
13883
|
-
invalidCorrelation("Projection replacement classification coverage does not match its streamed state.");
|
|
13884
|
-
}
|
|
14753
|
+
if (session.plan)
|
|
14754
|
+
assertReplacementPlanCoverage(this.db, session.plan);
|
|
13885
14755
|
}
|
|
13886
|
-
readLane(
|
|
14756
|
+
readLane(session, lane, cursor, limit) {
|
|
13887
14757
|
if (limit === 0)
|
|
13888
14758
|
return [];
|
|
13889
14759
|
const selected = `record_key, major_order, minor_order, sort_one, sort_two, payload`;
|
|
13890
14760
|
if (!cursor) {
|
|
13891
|
-
return this.db.query(`SELECT ${selected} FROM ${
|
|
13892
|
-
WHERE
|
|
14761
|
+
return this.db.query(`SELECT ${selected} FROM ${session.workTable}
|
|
14762
|
+
WHERE work_id = ? AND lane = ?
|
|
13893
14763
|
ORDER BY major_order, minor_order, sort_one, sort_two, record_key
|
|
13894
|
-
LIMIT ?`).all(
|
|
14764
|
+
LIMIT ?`).all(session.workId, lane, limit);
|
|
13895
14765
|
}
|
|
13896
|
-
return this.db.query(`SELECT ${selected} FROM ${
|
|
13897
|
-
WHERE
|
|
14766
|
+
return this.db.query(`SELECT ${selected} FROM ${session.workTable}
|
|
14767
|
+
WHERE work_id = ? AND lane = ?
|
|
13898
14768
|
AND (major_order, minor_order, sort_one, sort_two, record_key) > (?, ?, ?, ?, ?)
|
|
13899
14769
|
ORDER BY major_order, minor_order, sort_one, sort_two, record_key
|
|
13900
|
-
LIMIT ?`).all(
|
|
14770
|
+
LIMIT ?`).all(session.workId, lane, cursor.majorOrder, cursor.minorOrder, cursor.sortOne, cursor.sortTwo, cursor.recordKey, limit);
|
|
13901
14771
|
}
|
|
13902
14772
|
ensureTables() {
|
|
13903
14773
|
this.db.run(`
|
|
13904
14774
|
CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE} (
|
|
13905
|
-
|
|
14775
|
+
work_id TEXT NOT NULL,
|
|
13906
14776
|
record_key TEXT NOT NULL,
|
|
13907
14777
|
unique_key TEXT NOT NULL,
|
|
13908
14778
|
kind TEXT NOT NULL,
|
|
@@ -13921,35 +14791,21 @@ class SqliteMaterializationSessions {
|
|
|
13921
14791
|
cardinality_target_type_id TEXT,
|
|
13922
14792
|
cardinality_target_primary_id TEXT,
|
|
13923
14793
|
payload TEXT NOT NULL CHECK (json_valid(payload)),
|
|
13924
|
-
PRIMARY KEY (
|
|
13925
|
-
UNIQUE (
|
|
14794
|
+
PRIMARY KEY (work_id, record_key),
|
|
14795
|
+
UNIQUE (work_id, unique_key)
|
|
13926
14796
|
);
|
|
13927
14797
|
CREATE INDEX IF NOT EXISTS idx_ontology_materialization_work_lane
|
|
13928
14798
|
ON ${SQLITE_MATERIALIZATION_WORK_TABLE}(
|
|
13929
|
-
|
|
13930
|
-
);
|
|
13931
|
-
CREATE TEMP TABLE IF NOT EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
13932
|
-
session_id TEXT NOT NULL,
|
|
13933
|
-
entity_kind TEXT NOT NULL,
|
|
13934
|
-
identity_key TEXT NOT NULL,
|
|
13935
|
-
sort_key TEXT NOT NULL,
|
|
13936
|
-
diff_required INTEGER NOT NULL CHECK (diff_required IN (0, 1)),
|
|
13937
|
-
PRIMARY KEY (session_id, entity_kind, identity_key)
|
|
13938
|
-
);
|
|
13939
|
-
CREATE INDEX IF NOT EXISTS idx_ontology_replacement_work_order
|
|
13940
|
-
ON ${SQLITE_REPLACEMENT_WORK_TABLE}(
|
|
13941
|
-
session_id, entity_kind, sort_key, identity_key
|
|
14799
|
+
work_id, lane, major_order, minor_order, sort_one, sort_two, record_key
|
|
13942
14800
|
);
|
|
13943
14801
|
`);
|
|
13944
14802
|
}
|
|
13945
14803
|
deleteWork(sessionId) {
|
|
13946
|
-
this.db.query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE
|
|
13947
|
-
this.db.query(`DELETE FROM ${SQLITE_REPLACEMENT_WORK_TABLE} WHERE session_id = ?`).run(sessionId);
|
|
14804
|
+
this.db.query(`DELETE FROM ${SQLITE_MATERIALIZATION_WORK_TABLE} WHERE work_id = ?`).run(sessionId);
|
|
13948
14805
|
}
|
|
13949
14806
|
dropWorkTables() {
|
|
13950
14807
|
this.db.run(`
|
|
13951
14808
|
DROP TABLE IF EXISTS ${SQLITE_MATERIALIZATION_WORK_TABLE};
|
|
13952
|
-
DROP TABLE IF EXISTS ${SQLITE_REPLACEMENT_WORK_TABLE};
|
|
13953
14809
|
`);
|
|
13954
14810
|
}
|
|
13955
14811
|
}
|
|
@@ -13964,9 +14820,6 @@ function workCursor(row) {
|
|
|
13964
14820
|
}
|
|
13965
14821
|
function emptyWorkSummary() {
|
|
13966
14822
|
return {
|
|
13967
|
-
apply: 0,
|
|
13968
|
-
cardinality: 0,
|
|
13969
|
-
event: 0,
|
|
13970
14823
|
objectClassifications: 0,
|
|
13971
14824
|
linkClassifications: 0,
|
|
13972
14825
|
pointClassifications: 0,
|
|
@@ -13981,9 +14834,7 @@ function emptyWorkSummary() {
|
|
|
13981
14834
|
latestObjectsChanged: 0
|
|
13982
14835
|
};
|
|
13983
14836
|
}
|
|
13984
|
-
function recordSummary(summary, record
|
|
13985
|
-
if (columns.lane !== "none")
|
|
13986
|
-
summary[columns.lane] += 1;
|
|
14837
|
+
function recordSummary(summary, record) {
|
|
13987
14838
|
if (record.kind === "classification") {
|
|
13988
14839
|
if (record.entityKind === "object")
|
|
13989
14840
|
summary.objectClassifications += 1;
|
|
@@ -14032,11 +14883,13 @@ function recordSummary(summary, record, columns) {
|
|
|
14032
14883
|
// src/ontology-storage/materialization-writer.ts
|
|
14033
14884
|
import {
|
|
14034
14885
|
linkRefKey as linkRefKey2,
|
|
14035
|
-
MaterializationConflictError as
|
|
14036
|
-
objectRefKey as
|
|
14886
|
+
MaterializationConflictError as MaterializationConflictError6,
|
|
14887
|
+
objectRefKey as objectRefKey3,
|
|
14037
14888
|
telemetryPointKey
|
|
14038
14889
|
} from "@sixb/core/internal/materialization";
|
|
14039
|
-
import {
|
|
14890
|
+
import {
|
|
14891
|
+
effectiveConflict
|
|
14892
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
14040
14893
|
function linkOverrideConflictMessage(identityKind) {
|
|
14041
14894
|
return identityKind === "edge" ? "Expected link edge override changed." : "Expected link slot override changed.";
|
|
14042
14895
|
}
|
|
@@ -14132,7 +14985,7 @@ class SqliteMaterializationWriter {
|
|
|
14132
14985
|
for (const item of chunk.effective.objectDeletes) {
|
|
14133
14986
|
requireChanges(this.db.query(`DELETE FROM objects
|
|
14134
14987
|
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14135
|
-
AND version = ? AND last_commit_id = ?`).run(projectId, item.ref.objectTypeId, item.ref.primaryId, item.expected.version, item.expected.lastCommitId).changes, "effective-state", `Expected object ${
|
|
14988
|
+
AND version = ? AND last_commit_id = ?`).run(projectId, item.ref.objectTypeId, item.ref.primaryId, item.expected.version, item.expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey3(item.ref)} changed.`);
|
|
14136
14989
|
}
|
|
14137
14990
|
for (const item of chunk.effective.objectUpserts)
|
|
14138
14991
|
this.applyObject(projectId, item);
|
|
@@ -14151,7 +15004,7 @@ class SqliteMaterializationWriter {
|
|
|
14151
15004
|
`).run(projectId, row.ref.objectTypeId, row.ref.primaryId, canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId);
|
|
14152
15005
|
} catch (error) {
|
|
14153
15006
|
if (isSqliteConstraintError(error)) {
|
|
14154
|
-
throw effectiveConflict(`Expected object ${
|
|
15007
|
+
throw effectiveConflict(`Expected object ${objectRefKey3(row.ref)} to be absent.`);
|
|
14155
15008
|
}
|
|
14156
15009
|
throw error;
|
|
14157
15010
|
}
|
|
@@ -14163,7 +15016,7 @@ class SqliteMaterializationWriter {
|
|
|
14163
15016
|
last_commit_id = ?
|
|
14164
15017
|
WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
|
|
14165
15018
|
AND version = ? AND last_commit_id = ?
|
|
14166
|
-
`).run(canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId, projectId, row.ref.objectTypeId, row.ref.primaryId, expected.version, expected.lastCommitId).changes, "effective-state", `Expected object ${
|
|
15019
|
+
`).run(canonicalJson(row.properties), row.createdAt, row.updatedAt, row.version, row.lastCommitId, projectId, row.ref.objectTypeId, row.ref.primaryId, expected.version, expected.lastCommitId).changes, "effective-state", `Expected object ${objectRefKey3(row.ref)} changed.`);
|
|
14167
15020
|
}
|
|
14168
15021
|
applyLink(projectId, item) {
|
|
14169
15022
|
const { row, expected } = item;
|
|
@@ -14204,7 +15057,7 @@ class SqliteMaterializationWriter {
|
|
|
14204
15057
|
`).run(projectId, point.series.object.objectTypeId, point.series.object.primaryId, point.series.propertyId, canonicalJson(point.value), point.unit ?? null, point.at, point.lastCommitId);
|
|
14205
15058
|
} catch (error) {
|
|
14206
15059
|
if (isSqliteConstraintError(error)) {
|
|
14207
|
-
throw new
|
|
15060
|
+
throw new MaterializationConflictError6("timeseries-point", `Telemetry point ${telemetryPointKey(point.series, point.at)} changed.`);
|
|
14208
15061
|
}
|
|
14209
15062
|
throw error;
|
|
14210
15063
|
}
|
|
@@ -14221,19 +15074,18 @@ class SqliteMaterializationWriter {
|
|
|
14221
15074
|
applyOutbox(projectId, commitId, chunk) {
|
|
14222
15075
|
const insert = this.db.query(`
|
|
14223
15076
|
INSERT INTO ontology_outbox (
|
|
14224
|
-
project_id, id, commit_id, commit_ordinal,
|
|
15077
|
+
project_id, id, commit_id, commit_ordinal, event,
|
|
14225
15078
|
available_at, attempts, lease_id, lease_expires_at,
|
|
14226
15079
|
published_at, last_failure, created_at
|
|
14227
15080
|
) VALUES (?, ?, ?, ?, json(?), ?, 0, NULL, NULL, NULL, NULL, ?)
|
|
14228
15081
|
`);
|
|
14229
15082
|
for (const item of chunk.outbox) {
|
|
14230
|
-
assertTimestamp(item.availableAt, "Outbox availableAt");
|
|
14231
15083
|
assertTimestamp(item.createdAt, "Outbox createdAt");
|
|
14232
15084
|
try {
|
|
14233
|
-
insert.run(projectId, item.
|
|
15085
|
+
insert.run(projectId, item.id, commitId, item.commitOrdinal, canonicalJson(item.event), item.createdAt, item.createdAt);
|
|
14234
15086
|
} catch (error) {
|
|
14235
15087
|
if (isSqliteConstraintError(error)) {
|
|
14236
|
-
throw effectiveConflict(`Duplicate outbox event '${item.
|
|
15088
|
+
throw effectiveConflict(`Duplicate outbox event '${item.id}'.`);
|
|
14237
15089
|
}
|
|
14238
15090
|
throw error;
|
|
14239
15091
|
}
|
|
@@ -14242,6 +15094,48 @@ class SqliteMaterializationWriter {
|
|
|
14242
15094
|
}
|
|
14243
15095
|
|
|
14244
15096
|
// src/ontology-storage/materializations.ts
|
|
15097
|
+
var APPLY_PAGE_ROWS = 1000;
|
|
15098
|
+
function touchAppliedWorkStatement(workTable) {
|
|
15099
|
+
return `
|
|
15100
|
+
INSERT INTO ontology_commit_touches (project_id, commit_id, entity_kind, identity_key)
|
|
15101
|
+
SELECT DISTINCT $projectId, $commitId, kind, CASE kind
|
|
15102
|
+
WHEN 'object' THEN
|
|
15103
|
+
json_array(json_extract(ref, '$.objectTypeId'), json_extract(ref, '$.primaryId'))
|
|
15104
|
+
WHEN 'scope' THEN json_array(
|
|
15105
|
+
json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
|
|
15106
|
+
json_extract(ref, '$.linkId')
|
|
15107
|
+
)
|
|
15108
|
+
ELSE json_array(
|
|
15109
|
+
json_extract(ref, '$.source.objectTypeId'), json_extract(ref, '$.source.primaryId'),
|
|
15110
|
+
json_extract(ref, '$.linkId'), json_extract(ref, '$.target.objectTypeId'),
|
|
15111
|
+
json_extract(ref, '$.target.primaryId')
|
|
15112
|
+
)
|
|
15113
|
+
END
|
|
15114
|
+
FROM (
|
|
15115
|
+
SELECT CASE json_extract(payload, '$.item.kind')
|
|
15116
|
+
WHEN 'object-upsert' THEN 'object'
|
|
15117
|
+
WHEN 'object-delete' THEN 'object'
|
|
15118
|
+
WHEN 'object-override-upsert' THEN 'object'
|
|
15119
|
+
WHEN 'object-override-delete' THEN 'object'
|
|
15120
|
+
WHEN 'point-upsert' THEN 'object'
|
|
15121
|
+
WHEN 'link-upsert' THEN 'link'
|
|
15122
|
+
WHEN 'link-delete' THEN 'link'
|
|
15123
|
+
WHEN 'link-override-upsert' THEN 'link'
|
|
15124
|
+
WHEN 'link-override-delete' THEN 'link'
|
|
15125
|
+
WHEN 'link-slot-override-upsert' THEN 'scope'
|
|
15126
|
+
WHEN 'link-slot-override-delete' THEN 'scope'
|
|
15127
|
+
END AS kind,
|
|
15128
|
+
CASE json_extract(payload, '$.item.kind')
|
|
15129
|
+
WHEN 'object-upsert' THEN json_extract(payload, '$.item.value.row.ref')
|
|
15130
|
+
WHEN 'link-upsert' THEN json_extract(payload, '$.item.value.row.ref')
|
|
15131
|
+
WHEN 'point-upsert' THEN json_extract(payload, '$.item.value.point.series.object')
|
|
15132
|
+
ELSE json_extract(payload, '$.item.value.ref')
|
|
15133
|
+
END AS ref
|
|
15134
|
+
FROM ${workTable}
|
|
15135
|
+
WHERE work_id = $workId AND lane = 'apply'
|
|
15136
|
+
)`;
|
|
15137
|
+
}
|
|
15138
|
+
|
|
14245
15139
|
class SqliteOntologyMaterializationStorage {
|
|
14246
15140
|
db;
|
|
14247
15141
|
sessions;
|
|
@@ -14254,12 +15148,12 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14254
15148
|
assertVectorSession(session, projectId, commitId) {
|
|
14255
15149
|
const active = this.sessions.require(session);
|
|
14256
15150
|
if (active.header.commit.projectId !== projectId || commitId !== undefined && active.header.commit.id !== commitId) {
|
|
14257
|
-
throw new
|
|
15151
|
+
throw new MaterializationValidationError3("Vector mutation does not belong to this materialization session.");
|
|
14258
15152
|
}
|
|
14259
15153
|
}
|
|
14260
15154
|
async begin(input) {
|
|
14261
15155
|
assertMaterializationHeader(input);
|
|
14262
|
-
const session = this.sessions.create(input);
|
|
15156
|
+
const session = this.sessions.create(input, input.plan ? boundReplacementPlan(this.db, input) : null);
|
|
14263
15157
|
try {
|
|
14264
15158
|
this.assertCommitAbsent(input);
|
|
14265
15159
|
const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId);
|
|
@@ -14278,7 +15172,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14278
15172
|
for (const expected of input.expected.points) {
|
|
14279
15173
|
const point = reader.exactPoint(expected.series, expected.at);
|
|
14280
15174
|
if ((point?.lastCommitId ?? null) !== expected.lastCommitId) {
|
|
14281
|
-
throw new
|
|
15175
|
+
throw new MaterializationConflictError7("timeseries-point", `Telemetry point ${telemetryPointKey2(expected.series, expected.at)} changed.`);
|
|
14282
15176
|
}
|
|
14283
15177
|
}
|
|
14284
15178
|
return session.publicSession();
|
|
@@ -14289,11 +15183,11 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14289
15183
|
}
|
|
14290
15184
|
async* streamState(input) {
|
|
14291
15185
|
const session = this.sessions.require(input.session);
|
|
14292
|
-
|
|
15186
|
+
assertPageRows2(input.pageRows);
|
|
14293
15187
|
const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
|
|
14294
15188
|
for await (const request of input.requests) {
|
|
14295
15189
|
this.sessions.require(input.session);
|
|
14296
|
-
const objects = uniqueSorted(request.objects,
|
|
15190
|
+
const objects = uniqueSorted(request.objects, objectRefKey4, objectRefSortKey);
|
|
14297
15191
|
for (let offset = 0;offset < objects.length; offset += input.pageRows) {
|
|
14298
15192
|
const page = reader.objectStates(objects.slice(offset, offset + input.pageRows));
|
|
14299
15193
|
yield { objects: page, links: [], linkScopes: [], points: [] };
|
|
@@ -14329,103 +15223,50 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14329
15223
|
}
|
|
14330
15224
|
}
|
|
14331
15225
|
}
|
|
14332
|
-
async* streamSourceReplacementState(input) {
|
|
14333
|
-
const session = this.sessions.require(input.session);
|
|
14334
|
-
assertPageRows(input.pageRows);
|
|
14335
|
-
const replacement = this.requireReplacement(session, input);
|
|
14336
|
-
const reader = new SqliteMaterializationStateReader(this.db, session.header.commit.projectId);
|
|
14337
|
-
if (input.entityKind === "object") {
|
|
14338
|
-
if (replacement.projectionKind !== "object") {
|
|
14339
|
-
throw new MaterializationConflictError6("source-materialization", "Link projection replacement cannot stream object state.");
|
|
14340
|
-
}
|
|
14341
|
-
if (replacement.objectStreamStarted) {
|
|
14342
|
-
throw new MaterializationConflictError6("effective-state", "Replacement object state may only be streamed once per session.");
|
|
14343
|
-
}
|
|
14344
|
-
replacement.objectStreamStarted = true;
|
|
14345
|
-
for (const identities of reader.replacementIdentities({
|
|
14346
|
-
sessionId: session.id,
|
|
14347
|
-
sourceId: replacement.sourceId,
|
|
14348
|
-
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
14349
|
-
previousMaterializationId: replacement.previousMaterializationId,
|
|
14350
|
-
incremental: replacement.incremental,
|
|
14351
|
-
kind: "object",
|
|
14352
|
-
pageRows: input.pageRows
|
|
14353
|
-
})) {
|
|
14354
|
-
this.sessions.require(input.session);
|
|
14355
|
-
const refs = identities.map((identity) => {
|
|
14356
|
-
if (identity.kind !== "object") {
|
|
14357
|
-
invalidCorrelation2("Object replacement returned a link identity.");
|
|
14358
|
-
}
|
|
14359
|
-
return identity.ref;
|
|
14360
|
-
});
|
|
14361
|
-
const objects = reader.replacementObjectStates(replacement.sourceId, replacement.candidateMaterializationId, refs);
|
|
14362
|
-
yield { objects, links: [] };
|
|
14363
|
-
}
|
|
14364
|
-
this.sessions.require(input.session);
|
|
14365
|
-
replacement.objectStreamCompleted = true;
|
|
14366
|
-
return;
|
|
14367
|
-
}
|
|
14368
|
-
if (replacement.projectionKind === "object" && !replacement.objectStreamCompleted) {
|
|
14369
|
-
throw new MaterializationConflictError6("effective-state", "Object projection replacement must fully stream object state before link state.");
|
|
14370
|
-
}
|
|
14371
|
-
if (replacement.linkStreamStarted) {
|
|
14372
|
-
throw new MaterializationConflictError6("effective-state", "Replacement link state may only be streamed once per session.");
|
|
14373
|
-
}
|
|
14374
|
-
replacement.linkStreamStarted = true;
|
|
14375
|
-
const materializationIds = [
|
|
14376
|
-
replacement.candidateMaterializationId,
|
|
14377
|
-
...replacement.previousMaterializationId ? [replacement.previousMaterializationId] : []
|
|
14378
|
-
];
|
|
14379
|
-
for (const identities of reader.replacementIdentities({
|
|
14380
|
-
sessionId: session.id,
|
|
14381
|
-
sourceId: replacement.sourceId,
|
|
14382
|
-
candidateMaterializationId: replacement.candidateMaterializationId,
|
|
14383
|
-
previousMaterializationId: replacement.previousMaterializationId,
|
|
14384
|
-
incremental: replacement.incremental,
|
|
14385
|
-
kind: "link",
|
|
14386
|
-
pageRows: input.pageRows
|
|
14387
|
-
})) {
|
|
14388
|
-
this.sessions.require(input.session);
|
|
14389
|
-
const linkIdentities = identities.map((identity) => {
|
|
14390
|
-
if (identity.kind !== "link") {
|
|
14391
|
-
invalidCorrelation2("Link replacement returned an object identity.");
|
|
14392
|
-
}
|
|
14393
|
-
return identity;
|
|
14394
|
-
});
|
|
14395
|
-
const links = reader.replacementLinkStates(replacement.sourceId, replacement.candidateMaterializationId, materializationIds, linkIdentities, replacement.incremental);
|
|
14396
|
-
yield { objects: [], links };
|
|
14397
|
-
}
|
|
14398
|
-
this.sessions.require(input.session);
|
|
14399
|
-
replacement.linkStreamCompleted = true;
|
|
14400
|
-
}
|
|
14401
15226
|
async stageWork(input) {
|
|
14402
15227
|
this.sessions.stage(input);
|
|
14403
15228
|
await yieldSqliteEventLoop();
|
|
14404
15229
|
}
|
|
14405
|
-
|
|
14406
|
-
assertPageRows(input.pageRows);
|
|
14407
|
-
return this.sessions.stream(input);
|
|
14408
|
-
}
|
|
14409
|
-
async readObjectExistence(input) {
|
|
15230
|
+
async* streamVectorChanges(input) {
|
|
14410
15231
|
const session = this.sessions.require(input.session);
|
|
14411
|
-
|
|
15232
|
+
assertPageRows2(input.pageRows);
|
|
15233
|
+
beginMaterializationVectorChanges(session);
|
|
15234
|
+
for (const items of this.sessions.vectorChangePages(session, input.objectTypeIds, input.pageRows)) {
|
|
15235
|
+
yield { items };
|
|
15236
|
+
}
|
|
14412
15237
|
}
|
|
14413
|
-
async
|
|
15238
|
+
async apply(input) {
|
|
14414
15239
|
const session = this.sessions.require(input.session);
|
|
15240
|
+
beginMaterializationApply(session);
|
|
14415
15241
|
const { commit } = session.header;
|
|
14416
|
-
|
|
14417
|
-
const
|
|
14418
|
-
|
|
15242
|
+
const cardinality = createCardinalityValidator();
|
|
15243
|
+
for (const records of this.sessions.lanePages(session, "cardinality", APPLY_PAGE_ROWS)) {
|
|
15244
|
+
for (const record of records)
|
|
15245
|
+
cardinality.accept(record);
|
|
15246
|
+
}
|
|
15247
|
+
let eventCount = 0;
|
|
15248
|
+
this.db.run("SAVEPOINT sixb_ontology_apply");
|
|
14419
15249
|
try {
|
|
14420
|
-
this.
|
|
14421
|
-
|
|
14422
|
-
|
|
15250
|
+
for (const records of this.sessions.lanePages(session, "apply", APPLY_PAGE_ROWS)) {
|
|
15251
|
+
for (const phase of splitByPhase(records)) {
|
|
15252
|
+
this.writer.apply(commit.projectId, commit.id, materializationPlanChunk(phase.map((record) => record.item)));
|
|
15253
|
+
}
|
|
15254
|
+
await yieldSqliteEventLoop();
|
|
15255
|
+
}
|
|
15256
|
+
for (const records of this.sessions.lanePages(session, "event", APPLY_PAGE_ROWS)) {
|
|
15257
|
+
const outbox = records.map((record) => materializationOutboxWrite(commit, record.draft, eventCount++));
|
|
15258
|
+
this.writer.apply(commit.projectId, commit.id, materializationPlanChunk([], outbox));
|
|
15259
|
+
await yieldSqliteEventLoop();
|
|
15260
|
+
}
|
|
15261
|
+
this.db.query(touchAppliedWorkStatement(session.workTable)).run({ $projectId: commit.projectId, $commitId: commit.id, $workId: session.workId });
|
|
15262
|
+
this.db.run("RELEASE SAVEPOINT sixb_ontology_apply");
|
|
14423
15263
|
} catch (error) {
|
|
14424
|
-
this.db.run("ROLLBACK TO SAVEPOINT
|
|
14425
|
-
this.db.run("RELEASE SAVEPOINT
|
|
15264
|
+
this.db.run("ROLLBACK TO SAVEPOINT sixb_ontology_apply");
|
|
15265
|
+
this.db.run("RELEASE SAVEPOINT sixb_ontology_apply");
|
|
14426
15266
|
throw error;
|
|
14427
15267
|
}
|
|
14428
|
-
|
|
15268
|
+
session.appliedEventCount = eventCount;
|
|
15269
|
+
return { eventCount };
|
|
14429
15270
|
}
|
|
14430
15271
|
async finalize(input) {
|
|
14431
15272
|
const session = this.sessions.require(input.session);
|
|
@@ -14441,41 +15282,6 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14441
15282
|
deactivateSessions() {
|
|
14442
15283
|
this.sessions.deactivateAll();
|
|
14443
15284
|
}
|
|
14444
|
-
requireReplacement(session, input) {
|
|
14445
|
-
if (session.replacement) {
|
|
14446
|
-
if (session.replacement.sourceId !== input.source.projectionId || session.replacement.candidateMaterializationId !== input.candidateMaterializationId) {
|
|
14447
|
-
throw new MaterializationConflictError6("source-materialization", "Materialization session already owns another replacement union.");
|
|
14448
|
-
}
|
|
14449
|
-
return session.replacement;
|
|
14450
|
-
}
|
|
14451
|
-
const projectId = session.header.commit.projectId;
|
|
14452
|
-
const candidate = this.getSource(projectId, input.source.projectionId, input.candidateMaterializationId);
|
|
14453
|
-
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
14454
|
-
throw new MaterializationConflictError6("source-materialization", `Candidate source materialization '${input.candidateMaterializationId}' is missing or is not ready.`);
|
|
14455
|
-
}
|
|
14456
|
-
assertProjectionExecution(this.db, {
|
|
14457
|
-
projectId,
|
|
14458
|
-
sourceId: input.source.projectionId,
|
|
14459
|
-
projectionRunId: candidate.projection_run_id,
|
|
14460
|
-
executionToken: candidate.execution_token
|
|
14461
|
-
});
|
|
14462
|
-
const previous = this.getActiveSource(projectId, input.source.projectionId);
|
|
14463
|
-
if (candidate.protocol !== "replacement" || previous && (previous.protocol !== candidate.protocol || previous.projection_kind !== candidate.projection_kind)) {
|
|
14464
|
-
throw new MaterializationConflictError6("source-materialization", "Source replacement kind or protocol does not match its active materialization.");
|
|
14465
|
-
}
|
|
14466
|
-
session.replacement = {
|
|
14467
|
-
sourceId: input.source.projectionId,
|
|
14468
|
-
candidateMaterializationId: input.candidateMaterializationId,
|
|
14469
|
-
previousMaterializationId: previous?.materialization_id ?? null,
|
|
14470
|
-
incremental: candidate.base_materialization_id !== null,
|
|
14471
|
-
projectionKind: candidate.projection_kind,
|
|
14472
|
-
objectStreamStarted: false,
|
|
14473
|
-
objectStreamCompleted: false,
|
|
14474
|
-
linkStreamStarted: false,
|
|
14475
|
-
linkStreamCompleted: false
|
|
14476
|
-
};
|
|
14477
|
-
return session.replacement;
|
|
14478
|
-
}
|
|
14479
15285
|
assertCommitAbsent(header) {
|
|
14480
15286
|
const origin = originColumns(header.commit.origin);
|
|
14481
15287
|
const duplicate = this.db.query(`
|
|
@@ -14496,25 +15302,23 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14496
15302
|
if (!duplicate)
|
|
14497
15303
|
return;
|
|
14498
15304
|
if (duplicate.duplicate === "origin") {
|
|
14499
|
-
throw new
|
|
15305
|
+
throw new MaterializationConflictError7("run-correlation", "Ontology commit origin already has an authoritative commit.");
|
|
14500
15306
|
}
|
|
14501
|
-
throw new
|
|
15307
|
+
throw new MaterializationConflictError7("idempotency", duplicate.duplicate === "id" ? `Ontology commit '${header.commit.id}' already exists.` : "Ontology idempotency key already exists.");
|
|
14502
15308
|
}
|
|
14503
15309
|
assertSource(expected, projectId) {
|
|
14504
15310
|
const active = this.getActiveSource(projectId, expected.source.projectionId);
|
|
14505
15311
|
if ((active?.materialization_id ?? null) !== expected.activeMaterializationId || (active?.last_commit_id ?? null) !== expected.lastCommitId) {
|
|
14506
|
-
throw new
|
|
15312
|
+
throw new MaterializationConflictError7("projection-fence", `Source '${expected.source.projectionId}' changed.`);
|
|
14507
15313
|
}
|
|
14508
15314
|
}
|
|
14509
15315
|
async assertFinalization(session, input) {
|
|
14510
15316
|
const { commit } = session.header;
|
|
14511
15317
|
const { result } = input.finalization;
|
|
14512
15318
|
assertMaterializationFinalizationCorrelation(session, input);
|
|
14513
|
-
const laneCounts = this.sessions.laneCounts(session);
|
|
14514
|
-
assertMaterializationLaneCompletion(session, laneCounts);
|
|
14515
15319
|
this.assertFinalCardinality(session);
|
|
14516
15320
|
await yieldSqliteEventLoop();
|
|
14517
|
-
const eventCount =
|
|
15321
|
+
const eventCount = session.appliedEventCount;
|
|
14518
15322
|
const outbox = this.db.query(`
|
|
14519
15323
|
SELECT COUNT(*) AS count, MIN(commit_ordinal) AS minimum, MAX(commit_ordinal) AS maximum
|
|
14520
15324
|
FROM ontology_outbox WHERE project_id = ? AND commit_id = ?
|
|
@@ -14557,8 +15361,8 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14557
15361
|
THEN cardinality_target_type_id END) AS expected_target_type_id,
|
|
14558
15362
|
MAX(CASE WHEN cardinality_occupied = 1
|
|
14559
15363
|
THEN cardinality_target_primary_id END) AS expected_target_id
|
|
14560
|
-
FROM ${
|
|
14561
|
-
WHERE
|
|
15364
|
+
FROM ${session.workTable}
|
|
15365
|
+
WHERE work_id = ? AND kind = 'cardinality'
|
|
14562
15366
|
AND cardinality_view = 'effective'
|
|
14563
15367
|
GROUP BY sort_one, cardinality_source_type_id,
|
|
14564
15368
|
cardinality_source_primary_id, cardinality_link_id
|
|
@@ -14578,16 +15382,11 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14578
15382
|
GROUP BY scopes.scope_sort_key, scopes.expected_count,
|
|
14579
15383
|
scopes.expected_target_type_id, scopes.expected_target_id
|
|
14580
15384
|
)
|
|
14581
|
-
SELECT
|
|
15385
|
+
SELECT 1 AS mismatch
|
|
14582
15386
|
FROM validated
|
|
14583
|
-
WHERE expected_count
|
|
14584
|
-
OR actual_count <> expected_count
|
|
14585
|
-
OR matching_count <> expected_count
|
|
15387
|
+
WHERE actual_count <> expected_count OR matching_count <> expected_count
|
|
14586
15388
|
LIMIT 1
|
|
14587
|
-
`).get(session.
|
|
14588
|
-
if (violation?.reason === "duplicate") {
|
|
14589
|
-
invalidCorrelation2("Materialization cardinality work violates cardinality-one.");
|
|
14590
|
-
}
|
|
15389
|
+
`).get(session.workId, session.header.commit.projectId);
|
|
14591
15390
|
if (violation) {
|
|
14592
15391
|
invalidCorrelation2("Materialization cardinality work does not match the final effective link scope.");
|
|
14593
15392
|
}
|
|
@@ -14597,7 +15396,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14597
15396
|
assertSourceActivationCorrelation(session, activation);
|
|
14598
15397
|
const candidate = this.getSource(commit.projectId, activation.source.projectionId, activation.materializationId);
|
|
14599
15398
|
if (!candidate || candidate.status !== "ready" || candidate.execution_token === null) {
|
|
14600
|
-
throw new
|
|
15399
|
+
throw new MaterializationConflictError7("source-materialization", "Source activation candidate is missing or is not ready.");
|
|
14601
15400
|
}
|
|
14602
15401
|
assertProjectionExecution(this.db, {
|
|
14603
15402
|
projectId: commit.projectId,
|
|
@@ -14609,7 +15408,7 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14609
15408
|
invalidCorrelation2("Source activation does not match its ready candidate identity.");
|
|
14610
15409
|
}
|
|
14611
15410
|
this.assertSource(activation.expected, commit.projectId);
|
|
14612
|
-
activateSourceRoots(this.db, commit.projectId, candidate, activation);
|
|
15411
|
+
activateSourceRoots(this.db, commit.projectId, commit.id, candidate, activation);
|
|
14613
15412
|
const previous = this.getActiveSource(commit.projectId, activation.source.projectionId);
|
|
14614
15413
|
if (previous) {
|
|
14615
15414
|
assertPinnedDatasetWatermark({
|
|
@@ -14642,16 +15441,14 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14642
15441
|
this.db.query(`
|
|
14643
15442
|
INSERT INTO ontology_commits (
|
|
14644
15443
|
project_id, id, idempotency_key, request_hash, execution_id,
|
|
14645
|
-
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
15444
|
+
origin_kind, origin_run_id, origin_batch_ordinal, origin,
|
|
14646
15445
|
ontology_revision, projection_revision, ownership_hash,
|
|
14647
15446
|
intent, result, committed_at
|
|
14648
|
-
) VALUES (
|
|
14649
|
-
|
|
14650
|
-
)
|
|
14651
|
-
`).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy), canonicalJson(commit.executor), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
|
|
15447
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, json(?), ?, ?, ?, json(?), json(?), ?)
|
|
15448
|
+
`).run(commit.projectId, commit.id, commit.idempotencyKey, commit.requestHash, commit.executionId, origin.kind, origin.runId, origin.batchOrdinal, canonicalJson(commit.origin), commit.ontologyRevision, commit.projectionRevision ?? null, commit.ownershipHash ?? null, canonicalJson(commit.intent), canonicalJson(input.finalization.result), commit.committedAt);
|
|
14652
15449
|
} catch (error) {
|
|
14653
15450
|
if (isSqliteConstraintError(error)) {
|
|
14654
|
-
throw new
|
|
15451
|
+
throw new MaterializationConflictError7("idempotency", "Ontology commit identity already exists.");
|
|
14655
15452
|
}
|
|
14656
15453
|
throw error;
|
|
14657
15454
|
}
|
|
@@ -14667,13 +15464,27 @@ class SqliteOntologyMaterializationStorage {
|
|
|
14667
15464
|
WHERE project_id = ? AND source_id = ? AND materialization_id = ?`).get(projectId, sourceId, materializationId);
|
|
14668
15465
|
}
|
|
14669
15466
|
}
|
|
15467
|
+
function splitByPhase(records) {
|
|
15468
|
+
const phases = [];
|
|
15469
|
+
for (const record of records) {
|
|
15470
|
+
const current = phases[phases.length - 1];
|
|
15471
|
+
if (current && current[0].applyPhase === record.applyPhase)
|
|
15472
|
+
current.push(record);
|
|
15473
|
+
else
|
|
15474
|
+
phases.push([record]);
|
|
15475
|
+
}
|
|
15476
|
+
return phases;
|
|
15477
|
+
}
|
|
14670
15478
|
|
|
14671
15479
|
// src/ontology-storage/outbox.ts
|
|
14672
15480
|
import { serializeSixbFailure as serializeSixbFailure2 } from "@sixb/core/internal/errors";
|
|
14673
15481
|
import {
|
|
14674
|
-
MaterializationConflictError as
|
|
14675
|
-
MaterializationValidationError as
|
|
15482
|
+
MaterializationConflictError as MaterializationConflictError8,
|
|
15483
|
+
MaterializationValidationError as MaterializationValidationError4
|
|
14676
15484
|
} from "@sixb/core/internal/materialization";
|
|
15485
|
+
import {
|
|
15486
|
+
claimedOutboxRows
|
|
15487
|
+
} from "@sixb/core/internal/ontology-storage-provider";
|
|
14677
15488
|
import { ONTOLOGY_OUTBOX_FAILURE_CODES as ONTOLOGY_OUTBOX_FAILURE_CODES2 } from "@sixb/core/storage";
|
|
14678
15489
|
class SqliteOntologyOutboxStorage {
|
|
14679
15490
|
db;
|
|
@@ -14690,7 +15501,7 @@ class SqliteOntologyOutboxStorage {
|
|
|
14690
15501
|
const leaseExpiry = assertTimestamp(input.leaseExpiresAt, "Ontology outbox lease expiry");
|
|
14691
15502
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
14692
15503
|
if (leaseExpiry <= now) {
|
|
14693
|
-
throw new
|
|
15504
|
+
throw new MaterializationValidationError4("Ontology outbox lease expiry must be later than the claim time.");
|
|
14694
15505
|
}
|
|
14695
15506
|
const eligible = this.db.query(`SELECT CAST(rowid AS TEXT) AS rowId
|
|
14696
15507
|
FROM ontology_outbox INDEXED BY idx_ontology_outbox_claim
|
|
@@ -14709,15 +15520,19 @@ class SqliteOntologyOutboxStorage {
|
|
|
14709
15520
|
UPDATE ontology_outbox
|
|
14710
15521
|
SET attempts = attempts + 1, lease_id = ?, lease_expires_at = ?
|
|
14711
15522
|
WHERE rowid IN (SELECT CAST(value AS INTEGER) FROM json_each(?))
|
|
14712
|
-
RETURNING
|
|
14713
|
-
published_at, last_failure, created_at
|
|
15523
|
+
RETURNING id, commit_id, commit_ordinal, event, available_at, attempts, lease_id,
|
|
15524
|
+
lease_expires_at, published_at, last_failure, created_at
|
|
14714
15525
|
`).all(input.leaseId, input.leaseExpiresAt, JSON.stringify(selected.map((row) => row.rowId)));
|
|
14715
|
-
|
|
14716
|
-
|
|
14717
|
-
|
|
14718
|
-
|
|
14719
|
-
|
|
14720
|
-
|
|
15526
|
+
const contexts = this.db.query(`SELECT commits.id AS commit_id, commits.committed_at AS commit_committed_at,
|
|
15527
|
+
commits.origin AS commit_origin, executions.*
|
|
15528
|
+
FROM json_each(?) AS requested
|
|
15529
|
+
CROSS JOIN ontology_commits AS commits
|
|
15530
|
+
ON commits.project_id = ? AND commits.id = requested.value
|
|
15531
|
+
CROSS JOIN executions
|
|
15532
|
+
ON executions.project_id = commits.project_id AND executions.id = commits.execution_id`).all(JSON.stringify([...new Set(rows.map((row) => row.commit_id))]), input.projectId);
|
|
15533
|
+
const contextOf = new Map(contexts.map((row) => [row.commit_id, outboxEventContext(input.projectId, row)]));
|
|
15534
|
+
const drafts = rows.map(claimedOutboxDraft).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
15535
|
+
return claimedOutboxRows(drafts, (commitId) => contextOf.get(commitId));
|
|
14721
15536
|
});
|
|
14722
15537
|
}
|
|
14723
15538
|
async markPublished(input) {
|
|
@@ -14810,12 +15625,23 @@ class SqliteOntologyOutboxStorage {
|
|
|
14810
15625
|
});
|
|
14811
15626
|
}
|
|
14812
15627
|
}
|
|
15628
|
+
function outboxEventContext(projectId, row) {
|
|
15629
|
+
return {
|
|
15630
|
+
commit: {
|
|
15631
|
+
projectId,
|
|
15632
|
+
id: row.commit_id,
|
|
15633
|
+
committedAt: row.commit_committed_at,
|
|
15634
|
+
origin: parseJson2(row.commit_origin)
|
|
15635
|
+
},
|
|
15636
|
+
execution: sqliteExecutionRecord(row)
|
|
15637
|
+
};
|
|
15638
|
+
}
|
|
14813
15639
|
function validateLeaseIds(ids) {
|
|
14814
15640
|
const seen = new Set;
|
|
14815
15641
|
for (const id of ids) {
|
|
14816
15642
|
assertNonblank(id, "Ontology outbox event id");
|
|
14817
15643
|
if (seen.has(id)) {
|
|
14818
|
-
throw new
|
|
15644
|
+
throw new MaterializationValidationError4(`Ontology outbox lease batch repeats event '${id}'.`);
|
|
14819
15645
|
}
|
|
14820
15646
|
seen.add(id);
|
|
14821
15647
|
}
|
|
@@ -14826,11 +15652,11 @@ function assertLeaseInput(input) {
|
|
|
14826
15652
|
assertNonblank(input.leaseId, "Ontology outbox lease id");
|
|
14827
15653
|
}
|
|
14828
15654
|
function leaseConflict() {
|
|
14829
|
-
return new
|
|
15655
|
+
return new MaterializationConflictError8("outbox-lease", "Ontology outbox lease does not match.");
|
|
14830
15656
|
}
|
|
14831
15657
|
|
|
14832
15658
|
// src/ontology-storage/sources.ts
|
|
14833
|
-
import { MaterializationValidationError as
|
|
15659
|
+
import { MaterializationValidationError as MaterializationValidationError5 } from "@sixb/core/internal/materialization";
|
|
14834
15660
|
import {
|
|
14835
15661
|
assertSourceBeginInput as assertBeginInput,
|
|
14836
15662
|
assertSourceExecutionIdentity as assertExecutionIdentity,
|
|
@@ -14848,13 +15674,16 @@ import {
|
|
|
14848
15674
|
} from "@sixb/core/internal/ontology-storage-provider";
|
|
14849
15675
|
|
|
14850
15676
|
// src/ontology-storage/source-cleanup.ts
|
|
15677
|
+
var PLAN_PURGED = `NOT EXISTS (
|
|
15678
|
+
SELECT 1 FROM ontology_replacement_plans AS plans WHERE plans.version_id = versions.version_id
|
|
15679
|
+
)`;
|
|
14851
15680
|
function purgeAbandonedSourceVersions(db, input) {
|
|
14852
15681
|
let remaining = input.limit;
|
|
14853
15682
|
let rowsDeleted = 0;
|
|
14854
15683
|
let materializationsDeleted = 0;
|
|
14855
15684
|
const nextCandidate = db.query(`
|
|
14856
|
-
SELECT version_id FROM ontology_sources
|
|
14857
|
-
WHERE project_id = ? AND status = 'abandoned'
|
|
15685
|
+
SELECT version_id FROM ontology_sources AS versions
|
|
15686
|
+
WHERE project_id = ? AND status = 'abandoned' AND ${PLAN_PURGED}
|
|
14858
15687
|
ORDER BY terminal_at, source_id, materialization_id LIMIT 1
|
|
14859
15688
|
`);
|
|
14860
15689
|
const deleteRows = db.query(`DELETE FROM ontology_source_rows WHERE rowid IN (
|
|
@@ -14905,15 +15734,20 @@ function cleanupSourceVersions(db, input) {
|
|
|
14905
15734
|
WHERE ${retired}
|
|
14906
15735
|
AND NOT EXISTS (SELECT 1 FROM ontology_source_rows AS rows WHERE rows.root_id = roots.id)
|
|
14907
15736
|
ORDER BY roots.retired_at, roots.id LIMIT ?
|
|
14908
|
-
)`).
|
|
14909
|
-
|
|
14910
|
-
|
|
15737
|
+
) RETURNING version_id`).all(input.projectId, input.terminalBefore, remaining);
|
|
15738
|
+
const counts = new Map;
|
|
15739
|
+
for (const root of removed)
|
|
15740
|
+
counts.set(root.version_id, (counts.get(root.version_id) ?? 0) + 1);
|
|
15741
|
+
const drain = db.query("UPDATE ontology_sources SET root_count = root_count - ? WHERE version_id = ?");
|
|
15742
|
+
for (const [versionId, count] of counts)
|
|
15743
|
+
drain.run(count, versionId);
|
|
15744
|
+
rowsDeleted += removed.length;
|
|
15745
|
+
remaining -= removed.length;
|
|
14911
15746
|
}
|
|
14912
15747
|
const materializationsDeleted = remaining === 0 ? 0 : db.query(`DELETE FROM ontology_sources WHERE version_id IN (
|
|
14913
15748
|
SELECT versions.version_id FROM ontology_sources AS versions
|
|
14914
|
-
WHERE project_id = ? AND status = 'superseded' AND terminal_at < ?
|
|
14915
|
-
AND
|
|
14916
|
-
WHERE roots.version_id = versions.version_id)
|
|
15749
|
+
WHERE project_id = ? AND status = 'superseded' AND root_count = 0 AND terminal_at < ?
|
|
15750
|
+
AND ${PLAN_PURGED}
|
|
14917
15751
|
ORDER BY terminal_at, source_id, materialization_id LIMIT ?
|
|
14918
15752
|
)`).run(input.projectId, input.terminalBefore, remaining).changes;
|
|
14919
15753
|
return { rowsDeleted, materializationsDeleted };
|
|
@@ -14999,7 +15833,7 @@ class SqliteOntologySourceStorage {
|
|
|
14999
15833
|
throw sourceConflict(`Source materialization '${input.materializationId}' cannot transition from '${manifest.status}' to 'ready'.`);
|
|
15000
15834
|
}
|
|
15001
15835
|
if (input.readyAt < manifest.created_at) {
|
|
15002
|
-
throw new
|
|
15836
|
+
throw new MaterializationValidationError5("Source readyAt cannot precede source createdAt.");
|
|
15003
15837
|
}
|
|
15004
15838
|
this.assertReadyState(manifest, input.rootCount, input.assertionCount);
|
|
15005
15839
|
const result = this.db.query(`
|
|
@@ -15075,20 +15909,6 @@ class SqliteOntologySourceStorage {
|
|
|
15075
15909
|
return purgeAbandonedSourceVersions(this.db, input);
|
|
15076
15910
|
});
|
|
15077
15911
|
}
|
|
15078
|
-
async summarizeTerminal(input) {
|
|
15079
|
-
return this.runRootOperation(() => {
|
|
15080
|
-
assertNonblank(input.projectId, "Terminal source summary project id");
|
|
15081
|
-
const summary = this.db.query(`
|
|
15082
|
-
SELECT COUNT(*) AS terminal_count, MIN(terminal_at) AS oldest_terminal_at
|
|
15083
|
-
FROM ontology_sources
|
|
15084
|
-
WHERE project_id = ? AND status IN ('superseded', 'abandoned')
|
|
15085
|
-
`).get(input.projectId);
|
|
15086
|
-
return {
|
|
15087
|
-
count: summary.terminal_count,
|
|
15088
|
-
oldestTerminalAt: summary.oldest_terminal_at
|
|
15089
|
-
};
|
|
15090
|
-
});
|
|
15091
|
-
}
|
|
15092
15912
|
abandonCandidate(input) {
|
|
15093
15913
|
const manifest = this.requireManifest(input.projectId, input.source.projectionId, input.materializationId);
|
|
15094
15914
|
if (manifest.status === "abandoned") {
|
|
@@ -15117,7 +15937,7 @@ class SqliteOntologySourceStorage {
|
|
|
15117
15937
|
}
|
|
15118
15938
|
transitionToAbandoned(manifest, abandonedAt) {
|
|
15119
15939
|
if (abandonedAt < manifest.created_at || manifest.ready_at !== null && abandonedAt < manifest.ready_at) {
|
|
15120
|
-
throw new
|
|
15940
|
+
throw new MaterializationValidationError5("Source abandonedAt cannot precede source creation or readiness.");
|
|
15121
15941
|
}
|
|
15122
15942
|
const changed = this.db.query(`
|
|
15123
15943
|
UPDATE ontology_sources
|
|
@@ -15141,7 +15961,7 @@ class SqliteOntologySourceStorage {
|
|
|
15141
15961
|
WHERE version_id = ?1
|
|
15142
15962
|
`).get(manifest.version_id);
|
|
15143
15963
|
if (counts.assertions !== assertionCount || counts.roots !== rootCount || counts.ordinals !== rootCount || rootCount > 0 && (counts.min_ordinal !== 0 || counts.max_ordinal !== rootCount - 1)) {
|
|
15144
|
-
throw new
|
|
15964
|
+
throw new MaterializationValidationError5("Source ready counts do not match the staged roots and assertions.");
|
|
15145
15965
|
}
|
|
15146
15966
|
assertSourceRootCoverage(this.db, manifest);
|
|
15147
15967
|
const invalid = this.db.query(manifest.projection_kind === "link" ? `
|
|
@@ -15184,7 +16004,7 @@ class SqliteOntologySourceStorage {
|
|
|
15184
16004
|
LIMIT 1
|
|
15185
16005
|
`).get(manifest.version_id);
|
|
15186
16006
|
if (invalid) {
|
|
15187
|
-
throw new
|
|
16007
|
+
throw new MaterializationValidationError5(manifest.projection_kind === "link" ? "Link projection roots must contain exactly their matching link assertion." : "Object projection roots must contain exactly their matching object assertion plus links sourced from that root.");
|
|
15188
16008
|
}
|
|
15189
16009
|
}
|
|
15190
16010
|
findStageRows(manifest, rows) {
|
|
@@ -15368,7 +16188,7 @@ function decode(row) {
|
|
|
15368
16188
|
}
|
|
15369
16189
|
|
|
15370
16190
|
// src/ontology-storage/vectors.ts
|
|
15371
|
-
import { MaterializationConflictError as
|
|
16191
|
+
import { MaterializationConflictError as MaterializationConflictError9 } from "@sixb/core/internal/materialization";
|
|
15372
16192
|
class SqliteOntologyVectorStorage {
|
|
15373
16193
|
db;
|
|
15374
16194
|
runRootOperation;
|
|
@@ -15474,7 +16294,7 @@ class SqliteOntologyVectorStorage {
|
|
|
15474
16294
|
}
|
|
15475
16295
|
}
|
|
15476
16296
|
function vectorConflict() {
|
|
15477
|
-
return new
|
|
16297
|
+
return new MaterializationConflictError9("effective-state", "Vector changed since it was prepared.");
|
|
15478
16298
|
}
|
|
15479
16299
|
|
|
15480
16300
|
// src/ontology-storage/index.ts
|
|
@@ -15482,6 +16302,7 @@ class SqliteOntologyStorage {
|
|
|
15482
16302
|
commits;
|
|
15483
16303
|
sources;
|
|
15484
16304
|
materializations;
|
|
16305
|
+
replacementPlans;
|
|
15485
16306
|
vectorIndexing;
|
|
15486
16307
|
vectors;
|
|
15487
16308
|
outbox;
|
|
@@ -15489,6 +16310,7 @@ class SqliteOntologyStorage {
|
|
|
15489
16310
|
this.commits = new SqliteOntologyCommitStorage(input.db, input.runRootOperation);
|
|
15490
16311
|
this.sources = new SqliteOntologySourceStorage(input.db, input.runRootOperation);
|
|
15491
16312
|
this.materializations = new SqliteOntologyMaterializationStorage(input.db, input.transactionContext);
|
|
16313
|
+
this.replacementPlans = new SqliteOntologyReplacementPlanStorage(input.db, input.runRootOperation);
|
|
15492
16314
|
this.vectorIndexing = new SqliteVectorIndexingStorage(input.db, input.runRootOperation, (session, projectId) => this.materializations.assertVectorSession(session, projectId));
|
|
15493
16315
|
this.vectors = new SqliteOntologyVectorStorage(input.db, input.runRootOperation, (session, projectId, commitId) => this.materializations.assertVectorSession(session, projectId, commitId));
|
|
15494
16316
|
this.outbox = new SqliteOntologyOutboxStorage(input.db, input.runRootOperation);
|
|
@@ -16091,7 +16913,7 @@ class SqliteProjectionRunStorage {
|
|
|
16091
16913
|
}
|
|
16092
16914
|
async list(input) {
|
|
16093
16915
|
assertProjectionRunNonEmpty(input.projectId, "projectId");
|
|
16094
|
-
if (input.statuses
|
|
16916
|
+
if (input.statuses?.length === 0 || input.errorCodes?.length === 0) {
|
|
16095
16917
|
return { runs: [], hasMore: false, total: 0 };
|
|
16096
16918
|
}
|
|
16097
16919
|
const whereClauses = ["project_id = ?"];
|
|
@@ -16124,6 +16946,10 @@ class SqliteProjectionRunStorage {
|
|
|
16124
16946
|
whereClauses.push(`status IN (${input.statuses.map(() => "?").join(", ")})`);
|
|
16125
16947
|
args.push(...input.statuses);
|
|
16126
16948
|
}
|
|
16949
|
+
if (input.errorCodes) {
|
|
16950
|
+
whereClauses.push(`json_extract(error, '$.code') IN (${input.errorCodes.map(() => "?").join(", ")})`);
|
|
16951
|
+
args.push(...input.errorCodes);
|
|
16952
|
+
}
|
|
16127
16953
|
if (input.startedAfter) {
|
|
16128
16954
|
whereClauses.push("COALESCE(started_at, queued_at) >= ?");
|
|
16129
16955
|
args.push(input.startedAfter.toISOString());
|
|
@@ -16433,17 +17259,16 @@ class SqliteRulesStorage {
|
|
|
16433
17259
|
}
|
|
16434
17260
|
async listReconciliationPage(input) {
|
|
16435
17261
|
assertPositiveLimit(input.limit);
|
|
17262
|
+
const { after } = input;
|
|
17263
|
+
const cursor = after ? [after.ruleId, after.objectTypeId, after.primaryId] : [];
|
|
16436
17264
|
const rows = this.db.query(`
|
|
16437
17265
|
SELECT project_id, rule_id, subject_kind, object_type_id, primary_id, triggered_at
|
|
16438
17266
|
FROM rule_states
|
|
16439
|
-
WHERE project_id = ?
|
|
16440
|
-
AND (
|
|
16441
|
-
|
|
16442
|
-
(rule_id, object_type_id, primary_id) > (?, ?, ?)
|
|
16443
|
-
)
|
|
16444
|
-
ORDER BY rule_id ASC, object_type_id ASC, primary_id ASC
|
|
17267
|
+
WHERE project_id = ?
|
|
17268
|
+
${cursor.length > 0 ? "AND (rule_id, subject_kind, object_type_id, primary_id) > (?, 'object', ?, ?)" : ""}
|
|
17269
|
+
ORDER BY rule_id ASC, subject_kind ASC, object_type_id ASC, primary_id ASC
|
|
16445
17270
|
LIMIT ?
|
|
16446
|
-
`).all(input.projectId,
|
|
17271
|
+
`).all(input.projectId, ...cursor, input.limit + 1);
|
|
16447
17272
|
return reconciliationPage(rows, input.limit);
|
|
16448
17273
|
}
|
|
16449
17274
|
async applyTriggered(event) {
|
|
@@ -17034,10 +17859,8 @@ function isUniqueConstraintError4(error) {
|
|
|
17034
17859
|
|
|
17035
17860
|
// src/testing.ts
|
|
17036
17861
|
var adapters = new WeakMap;
|
|
17037
|
-
function registerSqliteStorageTestingAdapter(storage,
|
|
17038
|
-
adapters.set(storage,
|
|
17039
|
-
advanceConnectorConnectionTime
|
|
17040
|
-
});
|
|
17862
|
+
function registerSqliteStorageTestingAdapter(storage, adapter) {
|
|
17863
|
+
adapters.set(storage, adapter);
|
|
17041
17864
|
}
|
|
17042
17865
|
|
|
17043
17866
|
// src/timeseries-storage.ts
|
|
@@ -18193,6 +19016,7 @@ class SqliteStorage {
|
|
|
18193
19016
|
shareSessions;
|
|
18194
19017
|
connectorConnections;
|
|
18195
19018
|
fileUploadSessions;
|
|
19019
|
+
fileDownloadGrants;
|
|
18196
19020
|
migrators;
|
|
18197
19021
|
connection;
|
|
18198
19022
|
readConnection;
|
|
@@ -18244,8 +19068,12 @@ class SqliteStorage {
|
|
|
18244
19068
|
this.shareSessions = createOperationScopedFacade(stores.shareSessions, scope);
|
|
18245
19069
|
this.connectorConnections = createOperationScopedFacade(stores.connectorConnections, scope);
|
|
18246
19070
|
this.fileUploadSessions = createOperationScopedFacade(stores.fileUploadSessions, scope);
|
|
19071
|
+
this.fileDownloadGrants = createOperationScopedFacade(stores.fileDownloadGrants, scope);
|
|
18247
19072
|
this.migrators = options.path ? createSqliteStorageMigrators(options.path) : [];
|
|
18248
|
-
registerSqliteStorageTestingAdapter(this,
|
|
19073
|
+
registerSqliteStorageTestingAdapter(this, {
|
|
19074
|
+
advanceConnectorConnectionTime: (durationMs) => stores.connectorConnections.advanceTimeForTesting(durationMs),
|
|
19075
|
+
db: this.connection.db
|
|
19076
|
+
});
|
|
18249
19077
|
}
|
|
18250
19078
|
async ping() {
|
|
18251
19079
|
this.assertRootOperationAvailable();
|
|
@@ -18369,7 +19197,8 @@ function createSqliteStores(connection, options) {
|
|
|
18369
19197
|
shareGrants,
|
|
18370
19198
|
shareSessions,
|
|
18371
19199
|
connectorConnections: new SqliteConnectorConnectionStorage(connection),
|
|
18372
|
-
fileUploadSessions: new SqliteFileUploadSessionStorage(connection)
|
|
19200
|
+
fileUploadSessions: new SqliteFileUploadSessionStorage(connection),
|
|
19201
|
+
fileDownloadGrants: new SqliteFileDownloadGrantStorage(connection)
|
|
18373
19202
|
};
|
|
18374
19203
|
}
|
|
18375
19204
|
export {
|
|
@@ -18377,4 +19206,4 @@ export {
|
|
|
18377
19206
|
migrateSqliteStorage
|
|
18378
19207
|
};
|
|
18379
19208
|
|
|
18380
|
-
//# debugId=
|
|
19209
|
+
//# debugId=0E043C316BC5C57264756E2164756E21
|