@sixb/sqlite 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -13
- package/dist/action-run-storage.d.ts +5 -2
- package/dist/action-run-storage.d.ts.map +1 -1
- package/dist/agents/agent-storage.d.ts +9 -5
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/checkpoints.d.ts +14 -0
- package/dist/agents/checkpoints.d.ts.map +1 -0
- package/dist/agents/index.d.ts +1 -0
- package/dist/agents/index.d.ts.map +1 -1
- package/dist/agents/messages.d.ts +2 -1
- package/dist/agents/messages.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +21 -11
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -2
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/ai-cost-storage.d.ts +15 -0
- package/dist/ai-cost-storage.d.ts.map +1 -0
- package/dist/ai-usage-storage.d.ts +22 -0
- package/dist/ai-usage-storage.d.ts.map +1 -0
- package/dist/auth-storage/service-accounts.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +1 -10
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts +9 -0
- package/dist/connector-connection-storage.d.ts.map +1 -0
- package/dist/execution-storage.d.ts +14 -0
- package/dist/execution-storage.d.ts.map +1 -0
- package/dist/index.d.ts +14 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4633 -1066
- package/dist/index.js.map +33 -28
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts +0 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +50 -8
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +4 -5
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +0 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +3 -3
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pipeline-run-storage.d.ts +7 -2
- package/dist/pipeline-run-storage.d.ts.map +1 -1
- package/dist/projection-run-storage.d.ts +9 -2
- package/dist/projection-run-storage.d.ts.map +1 -1
- package/dist/run-list-query.d.ts +1 -1
- package/dist/run-list-query.d.ts.map +1 -1
- package/dist/sync-run-storage.d.ts +7 -2
- package/dist/sync-run-storage.d.ts.map +1 -1
- package/dist/testing.d.ts +6 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/transactions.d.ts +3 -0
- package/dist/transactions.d.ts.map +1 -1
- package/dist/webhook-run-storage.d.ts +9 -2
- package/dist/webhook-run-storage.d.ts.map +1 -1
- package/dist/workflow-run-storage.d.ts +7 -3
- package/dist/workflow-run-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/action-run-storage.ts +51 -74
- package/src/agents/agent-storage.ts +11 -6
- package/src/agents/checkpoints.ts +175 -0
- package/src/agents/index.ts +1 -0
- package/src/agents/messages.ts +59 -0
- package/src/agents/rows.ts +52 -40
- package/src/agents/runs.ts +27 -21
- package/src/ai-cost-storage.ts +744 -0
- package/src/ai-usage-storage.ts +324 -0
- package/src/auth-storage/service-accounts.ts +14 -21
- package/src/auth-storage/shared.ts +0 -22
- package/src/connector-connection-storage.ts +784 -0
- package/src/execution-storage.ts +158 -0
- package/src/index.ts +69 -22
- package/src/latest-run-query.ts +7 -1
- package/src/migrations/004-executions.sql +142 -0
- package/src/migrations/005-workflow-executions.sql +7 -0
- package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
- package/src/migrations/007-split-overrides.sql +154 -0
- package/src/migrations/008-action-executions.sql +4 -0
- package/src/migrations/009-agent-executions.sql +115 -0
- package/src/migrations/010-ai-usage-accounting-foundation.sql +74 -0
- package/src/migrations/011-sync-failure-record.sql +22 -0
- package/src/migrations/012-pipeline-failure-record.sql +45 -0
- package/src/migrations/013-workflow-failure-record.sql +45 -0
- package/src/migrations/014-agent-failure-record.sql +59 -0
- package/src/migrations/015-projection-failure-record.sql +20 -0
- package/src/migrations/016-webhook-run-failure-record.sql +19 -0
- package/src/migrations/017-action-failure-record.sql +65 -0
- package/src/migrations/018-ontology-outbox-failure-record.sql +80 -0
- package/src/migrations/019-webhook-delivery-failure-record.sql +62 -0
- package/src/migrations/020-drop-run-usage-projections.sql +7 -0
- package/src/migrations/021-sync-pipeline-executions.sql +101 -0
- package/src/migrations/022-projection-executions.sql +244 -0
- package/src/migrations/023-webhook-executions.sql +51 -0
- package/src/migrations/024-ontology-commit-executions.sql +58 -0
- package/src/migrations/025-connector-connections.sql +257 -0
- package/src/migrations/026-ai-cost-accounting.sql +57 -0
- package/src/migrations/027-agent-context-checkpoints.sql +28 -0
- package/src/migrations/028-object-override-edit-times.sql +3 -0
- package/src/migrations.ts +129 -4
- package/src/ontology-storage/materialization-session.ts +317 -143
- package/src/ontology-storage/materialization-state.ts +323 -112
- package/src/ontology-storage/materialization-writer.ts +145 -67
- package/src/ontology-storage/materializations.ts +67 -141
- package/src/ontology-storage/outbox.ts +10 -3
- package/src/ontology-storage/shared.ts +10 -4
- package/src/ontology-storage/sources.ts +4 -1
- package/src/pipeline-run-storage.ts +128 -43
- package/src/projection-run-storage.ts +209 -97
- package/src/run-list-query.ts +9 -4
- package/src/sync-run-storage.ts +118 -32
- package/src/testing.ts +20 -0
- package/src/transactions.ts +10 -3
- package/src/webhook-run-storage.ts +118 -74
- package/src/workflow-run-storage.ts +117 -131
- package/dist/validation.d.ts +0 -5
- package/dist/validation.d.ts.map +0 -1
- package/dist/webhook-delivery-storage.d.ts +0 -27
- package/dist/webhook-delivery-storage.d.ts.map +0 -1
- package/src/validation.ts +0 -30
- package/src/webhook-delivery-storage.ts +0 -272
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
import {
|
|
19
19
|
appendScopeSnapshot,
|
|
20
20
|
finishScopeAccumulator,
|
|
21
|
+
linkSlotOverrideValue,
|
|
21
22
|
startScopeAccumulator,
|
|
22
23
|
} from "@sixb/core/internal/ontology-storage-provider"
|
|
23
24
|
import type {
|
|
@@ -27,6 +28,7 @@ import type {
|
|
|
27
28
|
SourceReplacementLinkState,
|
|
28
29
|
SourceReplacementObjectState,
|
|
29
30
|
StoredLinkOverride,
|
|
31
|
+
StoredLinkSlotOverride,
|
|
30
32
|
StoredObjectOverride,
|
|
31
33
|
StoredSourceAssertion,
|
|
32
34
|
StoredSourceLinkAssertion,
|
|
@@ -38,8 +40,8 @@ import {
|
|
|
38
40
|
linkRefFromColumns,
|
|
39
41
|
objectRefFromColumns,
|
|
40
42
|
parseJson,
|
|
41
|
-
type SqliteOntologyOverrideRow,
|
|
42
43
|
type SqliteOntologySourceAssertionRow,
|
|
44
|
+
type SqliteStoredOverrideRow,
|
|
43
45
|
} from "./shared"
|
|
44
46
|
|
|
45
47
|
export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
|
|
@@ -81,12 +83,13 @@ interface LinkScopeRow extends EffectiveLinkRow {
|
|
|
81
83
|
readonly scope_sort_key: string
|
|
82
84
|
}
|
|
83
85
|
|
|
84
|
-
interface ObjectOverrideRow extends
|
|
86
|
+
interface ObjectOverrideRow extends SqliteStoredOverrideRow {
|
|
85
87
|
readonly object_type_id: string
|
|
86
88
|
readonly primary_id: string
|
|
89
|
+
readonly edited_at: string
|
|
87
90
|
}
|
|
88
91
|
|
|
89
|
-
interface LinkOverrideRow extends
|
|
92
|
+
interface LinkOverrideRow extends SqliteStoredOverrideRow {
|
|
90
93
|
readonly source_type_id: string
|
|
91
94
|
readonly source_primary_id: string
|
|
92
95
|
readonly link_id: string
|
|
@@ -94,6 +97,17 @@ interface LinkOverrideRow extends SqliteOntologyOverrideRow {
|
|
|
94
97
|
readonly target_primary_id: string
|
|
95
98
|
}
|
|
96
99
|
|
|
100
|
+
interface LinkSlotOverrideRow {
|
|
101
|
+
readonly source_type_id: string
|
|
102
|
+
readonly source_primary_id: string
|
|
103
|
+
readonly link_id: string
|
|
104
|
+
readonly target_type_id: string
|
|
105
|
+
readonly target_primary_id: string
|
|
106
|
+
readonly value: string
|
|
107
|
+
readonly last_commit_id: string
|
|
108
|
+
readonly updated_at: string
|
|
109
|
+
}
|
|
110
|
+
|
|
97
111
|
interface ReplacementSources {
|
|
98
112
|
readonly owned: ReadonlySet<string>
|
|
99
113
|
readonly byMaterialization: ReadonlyMap<string, ReadonlyMap<string, StoredSourceAssertion>>
|
|
@@ -122,6 +136,18 @@ interface ReplacementIdentityInput {
|
|
|
122
136
|
readonly pageRows: number
|
|
123
137
|
}
|
|
124
138
|
|
|
139
|
+
interface ReplacementWorkRow {
|
|
140
|
+
readonly entity_kind: "object" | "link"
|
|
141
|
+
readonly identity_key: string
|
|
142
|
+
readonly sort_key: string
|
|
143
|
+
readonly diff_required: number
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
interface ReplacementCursor {
|
|
147
|
+
readonly sortKey: string
|
|
148
|
+
readonly identityKey: string
|
|
149
|
+
}
|
|
150
|
+
|
|
125
151
|
export class SqliteMaterializationStateReader {
|
|
126
152
|
constructor(
|
|
127
153
|
private readonly db: Database,
|
|
@@ -175,9 +201,8 @@ export class SqliteMaterializationStateReader {
|
|
|
175
201
|
.query(
|
|
176
202
|
`WITH requested AS (${requestedObjects})
|
|
177
203
|
SELECT overrides.* FROM requested
|
|
178
|
-
CROSS JOIN
|
|
204
|
+
CROSS JOIN ontology_object_overrides AS overrides
|
|
179
205
|
ON overrides.project_id = ?
|
|
180
|
-
AND overrides.entity_kind = 'object'
|
|
181
206
|
AND overrides.object_type_id = requested.object_type_id
|
|
182
207
|
AND overrides.primary_id = requested.primary_id`
|
|
183
208
|
)
|
|
@@ -305,9 +330,9 @@ export class SqliteMaterializationStateReader {
|
|
|
305
330
|
.query(
|
|
306
331
|
`WITH requested AS (${sourceRequest})
|
|
307
332
|
SELECT overrides.* FROM requested
|
|
308
|
-
CROSS JOIN
|
|
333
|
+
CROSS JOIN ontology_link_overrides AS overrides
|
|
309
334
|
ON overrides.project_id = ?
|
|
310
|
-
AND overrides.
|
|
335
|
+
AND overrides.identity_kind = 'edge'
|
|
311
336
|
AND overrides.source_type_id = requested.source_type_id
|
|
312
337
|
AND overrides.source_primary_id = requested.source_primary_id
|
|
313
338
|
AND overrides.link_id = requested.link_id
|
|
@@ -315,6 +340,18 @@ export class SqliteMaterializationStateReader {
|
|
|
315
340
|
AND overrides.target_primary_id = requested.target_primary_id`
|
|
316
341
|
)
|
|
317
342
|
.all(requested, this.projectId) as LinkOverrideRow[]
|
|
343
|
+
const slotOverrideRows = this.db
|
|
344
|
+
.query(
|
|
345
|
+
`WITH requested AS (${sourceRequest})
|
|
346
|
+
SELECT overrides.* FROM requested
|
|
347
|
+
CROSS JOIN ontology_link_overrides AS overrides
|
|
348
|
+
ON overrides.project_id = ?
|
|
349
|
+
AND overrides.identity_kind = 'slot'
|
|
350
|
+
AND overrides.source_type_id = requested.source_type_id
|
|
351
|
+
AND overrides.source_primary_id = requested.source_primary_id
|
|
352
|
+
AND overrides.link_id = requested.link_id`
|
|
353
|
+
)
|
|
354
|
+
.all(requested, this.projectId) as LinkSlotOverrideRow[]
|
|
318
355
|
const effective = new Map(
|
|
319
356
|
effectiveRows.map((row) => [linkRefKey(linkRefFromColumns(row)), row] as const)
|
|
320
357
|
)
|
|
@@ -324,6 +361,12 @@ export class SqliteMaterializationStateReader {
|
|
|
324
361
|
(row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
|
|
325
362
|
)
|
|
326
363
|
)
|
|
364
|
+
const slotOverrides = new Map(
|
|
365
|
+
slotOverrideRows.map((row) => {
|
|
366
|
+
const stored = storedLinkSlotOverride(row)
|
|
367
|
+
return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
|
|
368
|
+
})
|
|
369
|
+
)
|
|
327
370
|
return refs.map((ref) => {
|
|
328
371
|
const key = linkRefKey(ref)
|
|
329
372
|
const effectiveRow = effective.get(key)
|
|
@@ -331,6 +374,7 @@ export class SqliteMaterializationStateReader {
|
|
|
331
374
|
ref: structuredClone(ref),
|
|
332
375
|
source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
|
|
333
376
|
override: overrides.get(key) ?? null,
|
|
377
|
+
slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
|
|
334
378
|
effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
|
|
335
379
|
}
|
|
336
380
|
})
|
|
@@ -395,9 +439,59 @@ export class SqliteMaterializationStateReader {
|
|
|
395
439
|
const accumulators = new Map(
|
|
396
440
|
scopes.map(({ source, linkId }) => {
|
|
397
441
|
const key = linkScopeSortKey(source, linkId)
|
|
398
|
-
return [key, startScopeAccumulator(source, linkId
|
|
442
|
+
return [key, startScopeAccumulator(source, linkId)] as const
|
|
399
443
|
})
|
|
400
444
|
)
|
|
445
|
+
const requestedJson = canonicalJson(requested)
|
|
446
|
+
const sourceRows = this.db
|
|
447
|
+
.query(
|
|
448
|
+
`WITH requested AS (
|
|
449
|
+
SELECT DISTINCT
|
|
450
|
+
json_extract(value, '$.sourceTypeId') AS source_type_id,
|
|
451
|
+
json_extract(value, '$.sourceId') AS source_primary_id,
|
|
452
|
+
json_extract(value, '$.linkId') AS link_id
|
|
453
|
+
FROM json_each(?)
|
|
454
|
+
)
|
|
455
|
+
SELECT rows.* FROM requested
|
|
456
|
+
CROSS JOIN ontology_source_rows AS rows
|
|
457
|
+
ON rows.project_id = ?
|
|
458
|
+
AND rows.entity_kind = 'link'
|
|
459
|
+
AND rows.source_type_id = requested.source_type_id
|
|
460
|
+
AND rows.source_primary_id = requested.source_primary_id
|
|
461
|
+
AND rows.link_id = requested.link_id
|
|
462
|
+
JOIN ontology_sources AS sources
|
|
463
|
+
ON sources.project_id = rows.project_id
|
|
464
|
+
AND sources.source_id = rows.source_id
|
|
465
|
+
AND sources.materialization_id = rows.materialization_id
|
|
466
|
+
WHERE sources.status = 'active'`
|
|
467
|
+
)
|
|
468
|
+
.all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
|
|
469
|
+
const slotOverrideRows = this.db
|
|
470
|
+
.query(
|
|
471
|
+
`WITH requested AS (
|
|
472
|
+
SELECT DISTINCT
|
|
473
|
+
json_extract(value, '$.sourceTypeId') AS source_type_id,
|
|
474
|
+
json_extract(value, '$.sourceId') AS source_primary_id,
|
|
475
|
+
json_extract(value, '$.linkId') AS link_id
|
|
476
|
+
FROM json_each(?)
|
|
477
|
+
)
|
|
478
|
+
SELECT overrides.* FROM requested
|
|
479
|
+
CROSS JOIN ontology_link_overrides AS overrides
|
|
480
|
+
ON overrides.project_id = ?
|
|
481
|
+
AND overrides.identity_kind = 'slot'
|
|
482
|
+
AND overrides.source_type_id = requested.source_type_id
|
|
483
|
+
AND overrides.source_primary_id = requested.source_primary_id
|
|
484
|
+
AND overrides.link_id = requested.link_id`
|
|
485
|
+
)
|
|
486
|
+
.all(requestedJson, this.projectId) as LinkSlotOverrideRow[]
|
|
487
|
+
const sources = activeScopeSourceMap(sourceRows)
|
|
488
|
+
const overrides = new Map(
|
|
489
|
+
slotOverrideRows.map((row) => {
|
|
490
|
+
const stored = storedLinkSlotOverride(row)
|
|
491
|
+
return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
|
|
492
|
+
})
|
|
493
|
+
)
|
|
494
|
+
const effective = new Map<string, EffectiveLinkSnapshot | null>()
|
|
401
495
|
const rows = this.db
|
|
402
496
|
.query(
|
|
403
497
|
`
|
|
@@ -419,7 +513,7 @@ export class SqliteMaterializationStateReader {
|
|
|
419
513
|
ORDER BY requested.scope_sort_key, ${linkSortExpression("links")}
|
|
420
514
|
`
|
|
421
515
|
)
|
|
422
|
-
.iterate(
|
|
516
|
+
.iterate(requestedJson, this.projectId) as Iterable<LinkScopeRow>
|
|
423
517
|
for (const row of rows) {
|
|
424
518
|
const accumulator = accumulators.get(row.scope_sort_key)
|
|
425
519
|
if (!accumulator) {
|
|
@@ -428,11 +522,19 @@ export class SqliteMaterializationStateReader {
|
|
|
428
522
|
`Unexpected link scope '${row.scope_sort_key}' returned by storage.`
|
|
429
523
|
)
|
|
430
524
|
}
|
|
431
|
-
|
|
525
|
+
const snapshot = effectiveLinkSnapshot(row)
|
|
526
|
+
appendScopeSnapshot(accumulator, snapshot)
|
|
527
|
+
effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
|
|
432
528
|
}
|
|
433
|
-
return scopes.map(({ source, linkId }) =>
|
|
434
|
-
|
|
435
|
-
|
|
529
|
+
return scopes.map(({ source, linkId }) => {
|
|
530
|
+
const key = linkScopeSortKey(source, linkId)
|
|
531
|
+
return {
|
|
532
|
+
...finishScopeAccumulator(accumulators.get(key)!),
|
|
533
|
+
sourceAssertion: sources.get(key) ?? null,
|
|
534
|
+
override: overrides.get(key) ?? null,
|
|
535
|
+
effective: effective.get(key) ?? null,
|
|
536
|
+
}
|
|
537
|
+
})
|
|
436
538
|
}
|
|
437
539
|
|
|
438
540
|
linkScope(source: OntologyObjectRef, linkId: string): MaterializationLinkScopeState {
|
|
@@ -464,8 +566,8 @@ export class SqliteMaterializationStateReader {
|
|
|
464
566
|
FROM links WHERE project_id = ?
|
|
465
567
|
UNION
|
|
466
568
|
SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
467
|
-
FROM
|
|
468
|
-
WHERE project_id = ?
|
|
569
|
+
FROM ontology_link_overrides
|
|
570
|
+
WHERE project_id = ?
|
|
469
571
|
UNION
|
|
470
572
|
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
471
573
|
rows.target_type_id, rows.target_primary_id
|
|
@@ -509,29 +611,15 @@ export class SqliteMaterializationStateReader {
|
|
|
509
611
|
}
|
|
510
612
|
|
|
511
613
|
*replacementIdentities(input: ReplacementIdentityInput): Iterable<ReplacementIdentity[]> {
|
|
512
|
-
|
|
614
|
+
this.prepareReplacementIdentities(input)
|
|
615
|
+
let cursor: ReplacementCursor | null = null
|
|
513
616
|
while (true) {
|
|
514
|
-
|
|
515
|
-
const rows = this.replacementObjectRows(input, cursor)
|
|
516
|
-
if (rows.length === 0) break
|
|
517
|
-
yield rows.map((row) => ({
|
|
518
|
-
kind: "object",
|
|
519
|
-
ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
|
|
520
|
-
sortKey: row.sort_key,
|
|
521
|
-
diffRequired: true,
|
|
522
|
-
}))
|
|
523
|
-
cursor = rows[rows.length - 1]?.sort_key ?? null
|
|
524
|
-
continue
|
|
525
|
-
}
|
|
526
|
-
const rows = this.replacementLinkRows(input, cursor)
|
|
617
|
+
const rows = this.replacementRows(input, cursor)
|
|
527
618
|
if (rows.length === 0) break
|
|
528
|
-
yield rows.map(
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
diffRequired: row.diff_required === 1,
|
|
533
|
-
}))
|
|
534
|
-
cursor = rows[rows.length - 1]?.sort_key ?? null
|
|
619
|
+
yield rows.map(replacementIdentity)
|
|
620
|
+
if (rows.length < input.pageRows) break
|
|
621
|
+
const last = rows[rows.length - 1]!
|
|
622
|
+
cursor = { sortKey: last.sort_key, identityKey: last.identity_key }
|
|
535
623
|
}
|
|
536
624
|
}
|
|
537
625
|
|
|
@@ -581,6 +669,7 @@ export class SqliteMaterializationStateReader {
|
|
|
581
669
|
? sourceLink(candidate?.get(key))
|
|
582
670
|
: state.source,
|
|
583
671
|
override: state.override,
|
|
672
|
+
slotOverride: state.slotOverride,
|
|
584
673
|
effective: state.effective,
|
|
585
674
|
diffRequired: identities[index]!.diffRequired,
|
|
586
675
|
}
|
|
@@ -623,16 +712,6 @@ export class SqliteMaterializationStateReader {
|
|
|
623
712
|
return row ? row.last_commit_id : undefined
|
|
624
713
|
}
|
|
625
714
|
|
|
626
|
-
overrideLastCommit(kind: "object" | "link", entityKey: string): string | null {
|
|
627
|
-
const row = this.db
|
|
628
|
-
.query(
|
|
629
|
-
`SELECT last_commit_id FROM ontology_overrides
|
|
630
|
-
WHERE project_id = ? AND entity_kind = ? AND entity_key = ?`
|
|
631
|
-
)
|
|
632
|
-
.get(this.projectId, kind, entityKey) as { readonly last_commit_id: string } | null
|
|
633
|
-
return row?.last_commit_id ?? null
|
|
634
|
-
}
|
|
635
|
-
|
|
636
715
|
private replacementSources(
|
|
637
716
|
sourceId: string,
|
|
638
717
|
materializationIds: readonly string[],
|
|
@@ -682,62 +761,48 @@ export class SqliteMaterializationStateReader {
|
|
|
682
761
|
return { owned, byMaterialization }
|
|
683
762
|
}
|
|
684
763
|
|
|
685
|
-
private
|
|
686
|
-
input
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
764
|
+
private prepareReplacementIdentities(input: ReplacementIdentityInput): void {
|
|
765
|
+
if (input.kind === "object") {
|
|
766
|
+
this.prepareReplacementObjects(input)
|
|
767
|
+
return
|
|
768
|
+
}
|
|
769
|
+
this.prepareReplacementLinks(input)
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
private prepareReplacementObjects(input: ReplacementIdentityInput): void {
|
|
773
|
+
this.db
|
|
690
774
|
.query(
|
|
691
775
|
`
|
|
692
|
-
|
|
776
|
+
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
777
|
+
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
778
|
+
)
|
|
779
|
+
SELECT ?, 'object', json_array(object_type_id, primary_id),
|
|
780
|
+
MIN(entity_sort_key), 1
|
|
693
781
|
FROM ontology_source_rows
|
|
694
782
|
WHERE project_id = ? AND source_id = ? AND entity_kind = 'object'
|
|
695
783
|
AND materialization_id IN (?, COALESCE(?, ''))
|
|
696
|
-
|
|
697
|
-
ORDER BY sort_key
|
|
698
|
-
LIMIT ?
|
|
784
|
+
GROUP BY object_type_id, primary_id
|
|
699
785
|
`
|
|
700
786
|
)
|
|
701
|
-
.
|
|
787
|
+
.run(
|
|
788
|
+
input.sessionId,
|
|
702
789
|
this.projectId,
|
|
703
790
|
input.sourceId,
|
|
704
791
|
input.candidateMaterializationId,
|
|
705
|
-
input.previousMaterializationId
|
|
706
|
-
|
|
707
|
-
cursor,
|
|
708
|
-
input.pageRows
|
|
709
|
-
) as (EffectiveObjectRow & { readonly sort_key: string })[]
|
|
792
|
+
input.previousMaterializationId
|
|
793
|
+
)
|
|
710
794
|
}
|
|
711
795
|
|
|
712
|
-
private
|
|
713
|
-
|
|
714
|
-
cursor: string | null
|
|
715
|
-
): (EffectiveLinkRow & { readonly sort_key: string; readonly diff_required: number })[] {
|
|
716
|
-
return this.db
|
|
796
|
+
private prepareReplacementLinks(input: ReplacementIdentityInput): void {
|
|
797
|
+
this.db
|
|
717
798
|
.query(
|
|
718
799
|
`
|
|
719
800
|
WITH incident_objects AS (
|
|
720
|
-
SELECT
|
|
801
|
+
SELECT DISTINCT
|
|
721
802
|
json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
|
|
722
803
|
json_extract(payload, '$.ref.primaryId') AS primary_id
|
|
723
804
|
FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
|
|
724
805
|
WHERE session_id = ? AND kind = 'incident-object'
|
|
725
|
-
), authority_links AS (
|
|
726
|
-
SELECT source_type_id, source_id, link_id, target_type_id, target_id
|
|
727
|
-
FROM links WHERE project_id = ?
|
|
728
|
-
UNION
|
|
729
|
-
SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
730
|
-
FROM ontology_overrides
|
|
731
|
-
WHERE project_id = ? AND entity_kind = 'link'
|
|
732
|
-
UNION
|
|
733
|
-
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
734
|
-
rows.target_type_id, rows.target_primary_id
|
|
735
|
-
FROM ontology_source_rows AS rows
|
|
736
|
-
JOIN ontology_sources AS sources
|
|
737
|
-
ON sources.project_id = rows.project_id
|
|
738
|
-
AND sources.source_id = rows.source_id
|
|
739
|
-
AND sources.materialization_id = rows.materialization_id
|
|
740
|
-
WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.status = 'active'
|
|
741
806
|
), replacement_links AS (
|
|
742
807
|
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
743
808
|
target_type_id, target_primary_id AS target_id
|
|
@@ -745,14 +810,67 @@ export class SqliteMaterializationStateReader {
|
|
|
745
810
|
WHERE project_id = ? AND source_id = ? AND entity_kind = 'link'
|
|
746
811
|
AND materialization_id IN (?, COALESCE(?, ''))
|
|
747
812
|
), incident_links AS (
|
|
748
|
-
SELECT links
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
813
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
814
|
+
links.target_type_id, links.target_id
|
|
815
|
+
FROM links
|
|
816
|
+
JOIN incident_objects
|
|
817
|
+
ON incident_objects.object_type_id = links.source_type_id
|
|
818
|
+
AND incident_objects.primary_id = links.source_id
|
|
819
|
+
WHERE links.project_id = ?
|
|
820
|
+
UNION
|
|
821
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
822
|
+
links.target_type_id, links.target_id
|
|
823
|
+
FROM links
|
|
824
|
+
JOIN incident_objects
|
|
825
|
+
ON incident_objects.object_type_id = links.target_type_id
|
|
826
|
+
AND incident_objects.primary_id = links.target_id
|
|
827
|
+
WHERE links.project_id = ?
|
|
828
|
+
UNION
|
|
829
|
+
SELECT overrides.source_type_id,
|
|
830
|
+
overrides.source_primary_id AS source_id,
|
|
831
|
+
overrides.link_id, overrides.target_type_id,
|
|
832
|
+
overrides.target_primary_id AS target_id
|
|
833
|
+
FROM ontology_link_overrides AS overrides
|
|
834
|
+
JOIN incident_objects
|
|
835
|
+
ON incident_objects.object_type_id = overrides.source_type_id
|
|
836
|
+
AND incident_objects.primary_id = overrides.source_primary_id
|
|
837
|
+
WHERE overrides.project_id = ?
|
|
838
|
+
UNION
|
|
839
|
+
SELECT overrides.source_type_id,
|
|
840
|
+
overrides.source_primary_id AS source_id,
|
|
841
|
+
overrides.link_id, overrides.target_type_id,
|
|
842
|
+
overrides.target_primary_id AS target_id
|
|
843
|
+
FROM ontology_link_overrides AS overrides
|
|
844
|
+
JOIN incident_objects
|
|
845
|
+
ON incident_objects.object_type_id = overrides.target_type_id
|
|
846
|
+
AND incident_objects.primary_id = overrides.target_primary_id
|
|
847
|
+
WHERE overrides.project_id = ?
|
|
848
|
+
UNION
|
|
849
|
+
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
850
|
+
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
851
|
+
FROM ontology_source_rows AS rows
|
|
852
|
+
JOIN ontology_sources AS sources
|
|
853
|
+
ON sources.project_id = rows.project_id
|
|
854
|
+
AND sources.source_id = rows.source_id
|
|
855
|
+
AND sources.materialization_id = rows.materialization_id
|
|
856
|
+
JOIN incident_objects
|
|
857
|
+
ON incident_objects.object_type_id = rows.source_type_id
|
|
858
|
+
AND incident_objects.primary_id = rows.source_primary_id
|
|
859
|
+
WHERE rows.project_id = ? AND rows.entity_kind = 'link'
|
|
860
|
+
AND sources.status = 'active'
|
|
861
|
+
UNION
|
|
862
|
+
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
863
|
+
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
864
|
+
FROM ontology_source_rows AS rows
|
|
865
|
+
JOIN ontology_sources AS sources
|
|
866
|
+
ON sources.project_id = rows.project_id
|
|
867
|
+
AND sources.source_id = rows.source_id
|
|
868
|
+
AND sources.materialization_id = rows.materialization_id
|
|
869
|
+
JOIN incident_objects
|
|
870
|
+
ON incident_objects.object_type_id = rows.target_type_id
|
|
871
|
+
AND incident_objects.primary_id = rows.target_primary_id
|
|
872
|
+
WHERE rows.project_id = ? AND rows.entity_kind = 'link'
|
|
873
|
+
AND sources.status = 'active'
|
|
756
874
|
), diff_links AS (
|
|
757
875
|
SELECT * FROM replacement_links
|
|
758
876
|
UNION SELECT * FROM incident_links
|
|
@@ -766,6 +884,16 @@ export class SqliteMaterializationStateReader {
|
|
|
766
884
|
FROM links
|
|
767
885
|
JOIN affected_scopes USING (source_type_id, source_id, link_id)
|
|
768
886
|
WHERE links.project_id = ?
|
|
887
|
+
UNION ALL
|
|
888
|
+
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
889
|
+
overrides.link_id, overrides.target_type_id,
|
|
890
|
+
overrides.target_primary_id AS target_id, 0 AS diff_required
|
|
891
|
+
FROM ontology_link_overrides AS overrides
|
|
892
|
+
JOIN affected_scopes
|
|
893
|
+
ON affected_scopes.source_type_id = overrides.source_type_id
|
|
894
|
+
AND affected_scopes.source_id = overrides.source_primary_id
|
|
895
|
+
AND affected_scopes.link_id = overrides.link_id
|
|
896
|
+
WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
|
|
769
897
|
), selected AS (
|
|
770
898
|
SELECT source_type_id, source_id, link_id, target_type_id, target_id,
|
|
771
899
|
${linkSortExpression("all_links")} AS sort_key,
|
|
@@ -773,26 +901,78 @@ export class SqliteMaterializationStateReader {
|
|
|
773
901
|
FROM all_links
|
|
774
902
|
GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
|
|
775
903
|
)
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
904
|
+
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
905
|
+
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
906
|
+
)
|
|
907
|
+
SELECT ?, 'link',
|
|
908
|
+
json_array(source_type_id, source_id, link_id, target_type_id, target_id),
|
|
909
|
+
sort_key, diff_required
|
|
910
|
+
FROM selected
|
|
780
911
|
`
|
|
781
912
|
)
|
|
782
|
-
.
|
|
913
|
+
.run(
|
|
783
914
|
input.sessionId,
|
|
784
915
|
this.projectId,
|
|
785
|
-
this.projectId,
|
|
786
|
-
this.projectId,
|
|
787
|
-
this.projectId,
|
|
788
916
|
input.sourceId,
|
|
789
917
|
input.candidateMaterializationId,
|
|
790
918
|
input.previousMaterializationId,
|
|
791
919
|
this.projectId,
|
|
792
|
-
|
|
793
|
-
|
|
920
|
+
this.projectId,
|
|
921
|
+
this.projectId,
|
|
922
|
+
this.projectId,
|
|
923
|
+
this.projectId,
|
|
924
|
+
this.projectId,
|
|
925
|
+
this.projectId,
|
|
926
|
+
this.projectId,
|
|
927
|
+
input.sessionId
|
|
928
|
+
)
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
private replacementRows(
|
|
932
|
+
input: ReplacementIdentityInput,
|
|
933
|
+
cursor: ReplacementCursor | null
|
|
934
|
+
): ReplacementWorkRow[] {
|
|
935
|
+
return this.db
|
|
936
|
+
.query(
|
|
937
|
+
`
|
|
938
|
+
SELECT entity_kind, identity_key, sort_key, diff_required
|
|
939
|
+
FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
|
|
940
|
+
WHERE session_id = ? AND entity_kind = ?
|
|
941
|
+
AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
|
|
942
|
+
ORDER BY sort_key, identity_key
|
|
943
|
+
LIMIT ?
|
|
944
|
+
`
|
|
945
|
+
)
|
|
946
|
+
.all(
|
|
947
|
+
input.sessionId,
|
|
948
|
+
input.kind,
|
|
949
|
+
cursor?.sortKey ?? null,
|
|
950
|
+
cursor?.sortKey ?? null,
|
|
951
|
+
cursor?.identityKey ?? null,
|
|
794
952
|
input.pageRows
|
|
795
|
-
) as
|
|
953
|
+
) as ReplacementWorkRow[]
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
function replacementIdentity(row: ReplacementWorkRow): ReplacementIdentity {
|
|
958
|
+
const parts = parseJson<string[]>(row.identity_key)
|
|
959
|
+
if (row.entity_kind === "object") {
|
|
960
|
+
return {
|
|
961
|
+
kind: "object",
|
|
962
|
+
ref: { objectTypeId: parts[0]!, primaryId: parts[1]! },
|
|
963
|
+
sortKey: row.sort_key,
|
|
964
|
+
diffRequired: true,
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
return {
|
|
968
|
+
kind: "link",
|
|
969
|
+
ref: {
|
|
970
|
+
source: { objectTypeId: parts[0]!, primaryId: parts[1]! },
|
|
971
|
+
linkId: parts[2]!,
|
|
972
|
+
target: { objectTypeId: parts[3]!, primaryId: parts[4]! },
|
|
973
|
+
},
|
|
974
|
+
sortKey: row.sort_key,
|
|
975
|
+
diffRequired: row.diff_required === 1,
|
|
796
976
|
}
|
|
797
977
|
}
|
|
798
978
|
|
|
@@ -814,6 +994,26 @@ function activeSourceMap(
|
|
|
814
994
|
return result
|
|
815
995
|
}
|
|
816
996
|
|
|
997
|
+
function activeScopeSourceMap(
|
|
998
|
+
rows: readonly SqliteOntologySourceAssertionRow[]
|
|
999
|
+
): ReadonlyMap<string, StoredSourceLinkAssertion> {
|
|
1000
|
+
const result = new Map<string, StoredSourceLinkAssertion>()
|
|
1001
|
+
for (const row of rows) {
|
|
1002
|
+
const source = sourceLink(storedSource(row))
|
|
1003
|
+
if (!source) continue
|
|
1004
|
+
const ref = source.assertion.ref
|
|
1005
|
+
const key = linkScopeSortKey(ref.source, ref.linkId)
|
|
1006
|
+
if (result.has(key)) {
|
|
1007
|
+
throw new MaterializationConflictError(
|
|
1008
|
+
"source-materialization",
|
|
1009
|
+
`Multiple active sources assert cardinality-one link slot ${key}.`
|
|
1010
|
+
)
|
|
1011
|
+
}
|
|
1012
|
+
result.set(key, source)
|
|
1013
|
+
}
|
|
1014
|
+
return result
|
|
1015
|
+
}
|
|
1016
|
+
|
|
817
1017
|
function sourceObject(
|
|
818
1018
|
source: StoredSourceAssertion | undefined
|
|
819
1019
|
): StoredSourceObjectAssertion | null {
|
|
@@ -829,6 +1029,7 @@ function storedObjectOverride(row: ObjectOverrideRow): StoredObjectOverride {
|
|
|
829
1029
|
return {
|
|
830
1030
|
ref,
|
|
831
1031
|
value: parseJson<StoredObjectOverride["value"]>(row.value),
|
|
1032
|
+
editedAt: parseJson<Readonly<Record<string, string>>>(row.edited_at),
|
|
832
1033
|
lastCommitId: row.last_commit_id,
|
|
833
1034
|
updatedAt: row.updated_at,
|
|
834
1035
|
}
|
|
@@ -844,6 +1045,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
|
|
|
844
1045
|
}
|
|
845
1046
|
}
|
|
846
1047
|
|
|
1048
|
+
function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
|
|
1049
|
+
const ref = {
|
|
1050
|
+
source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
|
|
1051
|
+
linkId: row.link_id,
|
|
1052
|
+
}
|
|
1053
|
+
return {
|
|
1054
|
+
ref,
|
|
1055
|
+
value: linkSlotOverrideValue(parseJson<unknown>(row.value), linkScopeLabel(ref)),
|
|
1056
|
+
lastCommitId: row.last_commit_id,
|
|
1057
|
+
updatedAt: row.updated_at,
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
847
1061
|
function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
|
|
848
1062
|
return {
|
|
849
1063
|
source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
|
|
@@ -852,6 +1066,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
|
|
|
852
1066
|
}
|
|
853
1067
|
}
|
|
854
1068
|
|
|
1069
|
+
function linkScopeLabel(scope: {
|
|
1070
|
+
readonly source: OntologyObjectRef
|
|
1071
|
+
readonly linkId: string
|
|
1072
|
+
}): string {
|
|
1073
|
+
return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
|
|
1074
|
+
}
|
|
1075
|
+
|
|
855
1076
|
function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssertion {
|
|
856
1077
|
return {
|
|
857
1078
|
source: { projectionId: row.source_id },
|
|
@@ -924,17 +1145,7 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
|
|
|
924
1145
|
}
|
|
925
1146
|
}
|
|
926
1147
|
|
|
927
|
-
export function objectSortExpression(alias?: string): string {
|
|
928
|
-
const prefix = alias ? `${alias}.` : ""
|
|
929
|
-
return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
|
|
930
|
-
}
|
|
931
|
-
|
|
932
1148
|
export function linkSortExpression(alias?: string): string {
|
|
933
1149
|
const prefix = alias ? `${alias}.` : ""
|
|
934
1150
|
return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`
|
|
935
1151
|
}
|
|
936
|
-
|
|
937
|
-
export function pointSortExpression(alias?: string): string {
|
|
938
|
-
const prefix = alias ? `${alias}.` : ""
|
|
939
|
-
return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}object_id, ${prefix}property_id, ${prefix}at) AS BLOB)))`
|
|
940
|
-
}
|