@sixb/sqlite 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -13
- package/dist/action-run-storage.d.ts +5 -2
- package/dist/action-run-storage.d.ts.map +1 -1
- package/dist/agents/agent-storage.d.ts +4 -2
- package/dist/agents/agent-storage.d.ts.map +1 -1
- package/dist/agents/rows.d.ts +3 -10
- package/dist/agents/rows.d.ts.map +1 -1
- package/dist/agents/runs.d.ts +3 -2
- package/dist/agents/runs.d.ts.map +1 -1
- package/dist/ai-usage-storage.d.ts +22 -0
- package/dist/ai-usage-storage.d.ts.map +1 -0
- package/dist/auth-storage/service-accounts.d.ts.map +1 -1
- package/dist/auth-storage/shared.d.ts +1 -10
- package/dist/auth-storage/shared.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts +9 -0
- package/dist/connector-connection-storage.d.ts.map +1 -0
- package/dist/execution-storage.d.ts +14 -0
- package/dist/execution-storage.d.ts.map +1 -0
- package/dist/index.d.ts +13 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3862 -1042
- package/dist/index.js.map +30 -27
- package/dist/latest-run-query.d.ts.map +1 -1
- package/dist/migrations.d.ts +0 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +50 -8
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +4 -5
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +0 -2
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/shared.d.ts +3 -3
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pipeline-run-storage.d.ts +7 -2
- package/dist/pipeline-run-storage.d.ts.map +1 -1
- package/dist/projection-run-storage.d.ts +9 -2
- package/dist/projection-run-storage.d.ts.map +1 -1
- package/dist/run-list-query.d.ts +1 -1
- package/dist/run-list-query.d.ts.map +1 -1
- package/dist/sync-run-storage.d.ts +7 -2
- package/dist/sync-run-storage.d.ts.map +1 -1
- package/dist/testing.d.ts +6 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/transactions.d.ts +3 -0
- package/dist/transactions.d.ts.map +1 -1
- package/dist/webhook-run-storage.d.ts +9 -2
- package/dist/webhook-run-storage.d.ts.map +1 -1
- package/dist/workflow-run-storage.d.ts +7 -3
- package/dist/workflow-run-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/action-run-storage.ts +51 -74
- package/src/agents/agent-storage.ts +5 -3
- package/src/agents/rows.ts +8 -40
- package/src/agents/runs.ts +27 -21
- package/src/ai-usage-storage.ts +324 -0
- package/src/auth-storage/service-accounts.ts +14 -21
- package/src/auth-storage/shared.ts +0 -22
- package/src/connector-connection-storage.ts +784 -0
- package/src/execution-storage.ts +158 -0
- package/src/index.ts +63 -22
- package/src/latest-run-query.ts +7 -1
- package/src/migrations/004-executions.sql +142 -0
- package/src/migrations/005-workflow-executions.sql +7 -0
- package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
- package/src/migrations/007-split-overrides.sql +154 -0
- package/src/migrations/008-action-executions.sql +4 -0
- package/src/migrations/009-agent-executions.sql +115 -0
- package/src/migrations/010-ai-usage-accounting-foundation.sql +74 -0
- package/src/migrations/011-sync-failure-record.sql +22 -0
- package/src/migrations/012-pipeline-failure-record.sql +45 -0
- package/src/migrations/013-workflow-failure-record.sql +45 -0
- package/src/migrations/014-agent-failure-record.sql +59 -0
- package/src/migrations/015-projection-failure-record.sql +20 -0
- package/src/migrations/016-webhook-run-failure-record.sql +19 -0
- package/src/migrations/017-action-failure-record.sql +65 -0
- package/src/migrations/018-ontology-outbox-failure-record.sql +80 -0
- package/src/migrations/019-webhook-delivery-failure-record.sql +62 -0
- package/src/migrations/020-drop-run-usage-projections.sql +7 -0
- package/src/migrations/021-sync-pipeline-executions.sql +101 -0
- package/src/migrations/022-projection-executions.sql +244 -0
- package/src/migrations/023-webhook-executions.sql +51 -0
- package/src/migrations/024-ontology-commit-executions.sql +58 -0
- package/src/migrations/025-connector-connections.sql +257 -0
- package/src/migrations.ts +119 -4
- package/src/ontology-storage/materialization-session.ts +317 -143
- package/src/ontology-storage/materialization-state.ts +321 -112
- package/src/ontology-storage/materialization-writer.ts +142 -67
- package/src/ontology-storage/materializations.ts +67 -141
- package/src/ontology-storage/outbox.ts +10 -3
- package/src/ontology-storage/shared.ts +10 -4
- package/src/ontology-storage/sources.ts +4 -1
- package/src/pipeline-run-storage.ts +128 -43
- package/src/projection-run-storage.ts +209 -97
- package/src/run-list-query.ts +9 -4
- package/src/sync-run-storage.ts +118 -32
- package/src/testing.ts +20 -0
- package/src/transactions.ts +10 -3
- package/src/webhook-run-storage.ts +118 -74
- package/src/workflow-run-storage.ts +117 -131
- package/dist/validation.d.ts +0 -5
- package/dist/validation.d.ts.map +0 -1
- package/dist/webhook-delivery-storage.d.ts +0 -27
- package/dist/webhook-delivery-storage.d.ts.map +0 -1
- package/src/validation.ts +0 -30
- package/src/webhook-delivery-storage.ts +0 -272
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
import {
|
|
19
19
|
appendScopeSnapshot,
|
|
20
20
|
finishScopeAccumulator,
|
|
21
|
+
linkSlotOverrideValue,
|
|
21
22
|
startScopeAccumulator,
|
|
22
23
|
} from "@sixb/core/internal/ontology-storage-provider"
|
|
23
24
|
import type {
|
|
@@ -27,6 +28,7 @@ import type {
|
|
|
27
28
|
SourceReplacementLinkState,
|
|
28
29
|
SourceReplacementObjectState,
|
|
29
30
|
StoredLinkOverride,
|
|
31
|
+
StoredLinkSlotOverride,
|
|
30
32
|
StoredObjectOverride,
|
|
31
33
|
StoredSourceAssertion,
|
|
32
34
|
StoredSourceLinkAssertion,
|
|
@@ -38,8 +40,8 @@ import {
|
|
|
38
40
|
linkRefFromColumns,
|
|
39
41
|
objectRefFromColumns,
|
|
40
42
|
parseJson,
|
|
41
|
-
type SqliteOntologyOverrideRow,
|
|
42
43
|
type SqliteOntologySourceAssertionRow,
|
|
44
|
+
type SqliteStoredOverrideRow,
|
|
43
45
|
} from "./shared"
|
|
44
46
|
|
|
45
47
|
export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
|
|
@@ -81,12 +83,12 @@ interface LinkScopeRow extends EffectiveLinkRow {
|
|
|
81
83
|
readonly scope_sort_key: string
|
|
82
84
|
}
|
|
83
85
|
|
|
84
|
-
interface ObjectOverrideRow extends
|
|
86
|
+
interface ObjectOverrideRow extends SqliteStoredOverrideRow {
|
|
85
87
|
readonly object_type_id: string
|
|
86
88
|
readonly primary_id: string
|
|
87
89
|
}
|
|
88
90
|
|
|
89
|
-
interface LinkOverrideRow extends
|
|
91
|
+
interface LinkOverrideRow extends SqliteStoredOverrideRow {
|
|
90
92
|
readonly source_type_id: string
|
|
91
93
|
readonly source_primary_id: string
|
|
92
94
|
readonly link_id: string
|
|
@@ -94,6 +96,17 @@ interface LinkOverrideRow extends SqliteOntologyOverrideRow {
|
|
|
94
96
|
readonly target_primary_id: string
|
|
95
97
|
}
|
|
96
98
|
|
|
99
|
+
interface LinkSlotOverrideRow {
|
|
100
|
+
readonly source_type_id: string
|
|
101
|
+
readonly source_primary_id: string
|
|
102
|
+
readonly link_id: string
|
|
103
|
+
readonly target_type_id: string
|
|
104
|
+
readonly target_primary_id: string
|
|
105
|
+
readonly value: string
|
|
106
|
+
readonly last_commit_id: string
|
|
107
|
+
readonly updated_at: string
|
|
108
|
+
}
|
|
109
|
+
|
|
97
110
|
interface ReplacementSources {
|
|
98
111
|
readonly owned: ReadonlySet<string>
|
|
99
112
|
readonly byMaterialization: ReadonlyMap<string, ReadonlyMap<string, StoredSourceAssertion>>
|
|
@@ -122,6 +135,18 @@ interface ReplacementIdentityInput {
|
|
|
122
135
|
readonly pageRows: number
|
|
123
136
|
}
|
|
124
137
|
|
|
138
|
+
interface ReplacementWorkRow {
|
|
139
|
+
readonly entity_kind: "object" | "link"
|
|
140
|
+
readonly identity_key: string
|
|
141
|
+
readonly sort_key: string
|
|
142
|
+
readonly diff_required: number
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
interface ReplacementCursor {
|
|
146
|
+
readonly sortKey: string
|
|
147
|
+
readonly identityKey: string
|
|
148
|
+
}
|
|
149
|
+
|
|
125
150
|
export class SqliteMaterializationStateReader {
|
|
126
151
|
constructor(
|
|
127
152
|
private readonly db: Database,
|
|
@@ -175,9 +200,8 @@ export class SqliteMaterializationStateReader {
|
|
|
175
200
|
.query(
|
|
176
201
|
`WITH requested AS (${requestedObjects})
|
|
177
202
|
SELECT overrides.* FROM requested
|
|
178
|
-
CROSS JOIN
|
|
203
|
+
CROSS JOIN ontology_object_overrides AS overrides
|
|
179
204
|
ON overrides.project_id = ?
|
|
180
|
-
AND overrides.entity_kind = 'object'
|
|
181
205
|
AND overrides.object_type_id = requested.object_type_id
|
|
182
206
|
AND overrides.primary_id = requested.primary_id`
|
|
183
207
|
)
|
|
@@ -305,9 +329,9 @@ export class SqliteMaterializationStateReader {
|
|
|
305
329
|
.query(
|
|
306
330
|
`WITH requested AS (${sourceRequest})
|
|
307
331
|
SELECT overrides.* FROM requested
|
|
308
|
-
CROSS JOIN
|
|
332
|
+
CROSS JOIN ontology_link_overrides AS overrides
|
|
309
333
|
ON overrides.project_id = ?
|
|
310
|
-
AND overrides.
|
|
334
|
+
AND overrides.identity_kind = 'edge'
|
|
311
335
|
AND overrides.source_type_id = requested.source_type_id
|
|
312
336
|
AND overrides.source_primary_id = requested.source_primary_id
|
|
313
337
|
AND overrides.link_id = requested.link_id
|
|
@@ -315,6 +339,18 @@ export class SqliteMaterializationStateReader {
|
|
|
315
339
|
AND overrides.target_primary_id = requested.target_primary_id`
|
|
316
340
|
)
|
|
317
341
|
.all(requested, this.projectId) as LinkOverrideRow[]
|
|
342
|
+
const slotOverrideRows = this.db
|
|
343
|
+
.query(
|
|
344
|
+
`WITH requested AS (${sourceRequest})
|
|
345
|
+
SELECT overrides.* FROM requested
|
|
346
|
+
CROSS JOIN ontology_link_overrides AS overrides
|
|
347
|
+
ON overrides.project_id = ?
|
|
348
|
+
AND overrides.identity_kind = 'slot'
|
|
349
|
+
AND overrides.source_type_id = requested.source_type_id
|
|
350
|
+
AND overrides.source_primary_id = requested.source_primary_id
|
|
351
|
+
AND overrides.link_id = requested.link_id`
|
|
352
|
+
)
|
|
353
|
+
.all(requested, this.projectId) as LinkSlotOverrideRow[]
|
|
318
354
|
const effective = new Map(
|
|
319
355
|
effectiveRows.map((row) => [linkRefKey(linkRefFromColumns(row)), row] as const)
|
|
320
356
|
)
|
|
@@ -324,6 +360,12 @@ export class SqliteMaterializationStateReader {
|
|
|
324
360
|
(row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
|
|
325
361
|
)
|
|
326
362
|
)
|
|
363
|
+
const slotOverrides = new Map(
|
|
364
|
+
slotOverrideRows.map((row) => {
|
|
365
|
+
const stored = storedLinkSlotOverride(row)
|
|
366
|
+
return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
|
|
367
|
+
})
|
|
368
|
+
)
|
|
327
369
|
return refs.map((ref) => {
|
|
328
370
|
const key = linkRefKey(ref)
|
|
329
371
|
const effectiveRow = effective.get(key)
|
|
@@ -331,6 +373,7 @@ export class SqliteMaterializationStateReader {
|
|
|
331
373
|
ref: structuredClone(ref),
|
|
332
374
|
source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
|
|
333
375
|
override: overrides.get(key) ?? null,
|
|
376
|
+
slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
|
|
334
377
|
effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
|
|
335
378
|
}
|
|
336
379
|
})
|
|
@@ -395,9 +438,59 @@ export class SqliteMaterializationStateReader {
|
|
|
395
438
|
const accumulators = new Map(
|
|
396
439
|
scopes.map(({ source, linkId }) => {
|
|
397
440
|
const key = linkScopeSortKey(source, linkId)
|
|
398
|
-
return [key, startScopeAccumulator(source, linkId
|
|
441
|
+
return [key, startScopeAccumulator(source, linkId)] as const
|
|
399
442
|
})
|
|
400
443
|
)
|
|
444
|
+
const requestedJson = canonicalJson(requested)
|
|
445
|
+
const sourceRows = this.db
|
|
446
|
+
.query(
|
|
447
|
+
`WITH requested AS (
|
|
448
|
+
SELECT DISTINCT
|
|
449
|
+
json_extract(value, '$.sourceTypeId') AS source_type_id,
|
|
450
|
+
json_extract(value, '$.sourceId') AS source_primary_id,
|
|
451
|
+
json_extract(value, '$.linkId') AS link_id
|
|
452
|
+
FROM json_each(?)
|
|
453
|
+
)
|
|
454
|
+
SELECT rows.* FROM requested
|
|
455
|
+
CROSS JOIN ontology_source_rows AS rows
|
|
456
|
+
ON rows.project_id = ?
|
|
457
|
+
AND rows.entity_kind = 'link'
|
|
458
|
+
AND rows.source_type_id = requested.source_type_id
|
|
459
|
+
AND rows.source_primary_id = requested.source_primary_id
|
|
460
|
+
AND rows.link_id = requested.link_id
|
|
461
|
+
JOIN ontology_sources AS sources
|
|
462
|
+
ON sources.project_id = rows.project_id
|
|
463
|
+
AND sources.source_id = rows.source_id
|
|
464
|
+
AND sources.materialization_id = rows.materialization_id
|
|
465
|
+
WHERE sources.status = 'active'`
|
|
466
|
+
)
|
|
467
|
+
.all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
|
|
468
|
+
const slotOverrideRows = this.db
|
|
469
|
+
.query(
|
|
470
|
+
`WITH requested AS (
|
|
471
|
+
SELECT DISTINCT
|
|
472
|
+
json_extract(value, '$.sourceTypeId') AS source_type_id,
|
|
473
|
+
json_extract(value, '$.sourceId') AS source_primary_id,
|
|
474
|
+
json_extract(value, '$.linkId') AS link_id
|
|
475
|
+
FROM json_each(?)
|
|
476
|
+
)
|
|
477
|
+
SELECT overrides.* FROM requested
|
|
478
|
+
CROSS JOIN ontology_link_overrides AS overrides
|
|
479
|
+
ON overrides.project_id = ?
|
|
480
|
+
AND overrides.identity_kind = 'slot'
|
|
481
|
+
AND overrides.source_type_id = requested.source_type_id
|
|
482
|
+
AND overrides.source_primary_id = requested.source_primary_id
|
|
483
|
+
AND overrides.link_id = requested.link_id`
|
|
484
|
+
)
|
|
485
|
+
.all(requestedJson, this.projectId) as LinkSlotOverrideRow[]
|
|
486
|
+
const sources = activeScopeSourceMap(sourceRows)
|
|
487
|
+
const overrides = new Map(
|
|
488
|
+
slotOverrideRows.map((row) => {
|
|
489
|
+
const stored = storedLinkSlotOverride(row)
|
|
490
|
+
return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
|
|
491
|
+
})
|
|
492
|
+
)
|
|
493
|
+
const effective = new Map<string, EffectiveLinkSnapshot | null>()
|
|
401
494
|
const rows = this.db
|
|
402
495
|
.query(
|
|
403
496
|
`
|
|
@@ -419,7 +512,7 @@ export class SqliteMaterializationStateReader {
|
|
|
419
512
|
ORDER BY requested.scope_sort_key, ${linkSortExpression("links")}
|
|
420
513
|
`
|
|
421
514
|
)
|
|
422
|
-
.iterate(
|
|
515
|
+
.iterate(requestedJson, this.projectId) as Iterable<LinkScopeRow>
|
|
423
516
|
for (const row of rows) {
|
|
424
517
|
const accumulator = accumulators.get(row.scope_sort_key)
|
|
425
518
|
if (!accumulator) {
|
|
@@ -428,11 +521,19 @@ export class SqliteMaterializationStateReader {
|
|
|
428
521
|
`Unexpected link scope '${row.scope_sort_key}' returned by storage.`
|
|
429
522
|
)
|
|
430
523
|
}
|
|
431
|
-
|
|
524
|
+
const snapshot = effectiveLinkSnapshot(row)
|
|
525
|
+
appendScopeSnapshot(accumulator, snapshot)
|
|
526
|
+
effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
|
|
432
527
|
}
|
|
433
|
-
return scopes.map(({ source, linkId }) =>
|
|
434
|
-
|
|
435
|
-
|
|
528
|
+
return scopes.map(({ source, linkId }) => {
|
|
529
|
+
const key = linkScopeSortKey(source, linkId)
|
|
530
|
+
return {
|
|
531
|
+
...finishScopeAccumulator(accumulators.get(key)!),
|
|
532
|
+
sourceAssertion: sources.get(key) ?? null,
|
|
533
|
+
override: overrides.get(key) ?? null,
|
|
534
|
+
effective: effective.get(key) ?? null,
|
|
535
|
+
}
|
|
536
|
+
})
|
|
436
537
|
}
|
|
437
538
|
|
|
438
539
|
linkScope(source: OntologyObjectRef, linkId: string): MaterializationLinkScopeState {
|
|
@@ -464,8 +565,8 @@ export class SqliteMaterializationStateReader {
|
|
|
464
565
|
FROM links WHERE project_id = ?
|
|
465
566
|
UNION
|
|
466
567
|
SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
467
|
-
FROM
|
|
468
|
-
WHERE project_id = ?
|
|
568
|
+
FROM ontology_link_overrides
|
|
569
|
+
WHERE project_id = ?
|
|
469
570
|
UNION
|
|
470
571
|
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
471
572
|
rows.target_type_id, rows.target_primary_id
|
|
@@ -509,29 +610,15 @@ export class SqliteMaterializationStateReader {
|
|
|
509
610
|
}
|
|
510
611
|
|
|
511
612
|
*replacementIdentities(input: ReplacementIdentityInput): Iterable<ReplacementIdentity[]> {
|
|
512
|
-
|
|
613
|
+
this.prepareReplacementIdentities(input)
|
|
614
|
+
let cursor: ReplacementCursor | null = null
|
|
513
615
|
while (true) {
|
|
514
|
-
|
|
515
|
-
const rows = this.replacementObjectRows(input, cursor)
|
|
516
|
-
if (rows.length === 0) break
|
|
517
|
-
yield rows.map((row) => ({
|
|
518
|
-
kind: "object",
|
|
519
|
-
ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
|
|
520
|
-
sortKey: row.sort_key,
|
|
521
|
-
diffRequired: true,
|
|
522
|
-
}))
|
|
523
|
-
cursor = rows[rows.length - 1]?.sort_key ?? null
|
|
524
|
-
continue
|
|
525
|
-
}
|
|
526
|
-
const rows = this.replacementLinkRows(input, cursor)
|
|
616
|
+
const rows = this.replacementRows(input, cursor)
|
|
527
617
|
if (rows.length === 0) break
|
|
528
|
-
yield rows.map(
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
diffRequired: row.diff_required === 1,
|
|
533
|
-
}))
|
|
534
|
-
cursor = rows[rows.length - 1]?.sort_key ?? null
|
|
618
|
+
yield rows.map(replacementIdentity)
|
|
619
|
+
if (rows.length < input.pageRows) break
|
|
620
|
+
const last = rows[rows.length - 1]!
|
|
621
|
+
cursor = { sortKey: last.sort_key, identityKey: last.identity_key }
|
|
535
622
|
}
|
|
536
623
|
}
|
|
537
624
|
|
|
@@ -581,6 +668,7 @@ export class SqliteMaterializationStateReader {
|
|
|
581
668
|
? sourceLink(candidate?.get(key))
|
|
582
669
|
: state.source,
|
|
583
670
|
override: state.override,
|
|
671
|
+
slotOverride: state.slotOverride,
|
|
584
672
|
effective: state.effective,
|
|
585
673
|
diffRequired: identities[index]!.diffRequired,
|
|
586
674
|
}
|
|
@@ -623,16 +711,6 @@ export class SqliteMaterializationStateReader {
|
|
|
623
711
|
return row ? row.last_commit_id : undefined
|
|
624
712
|
}
|
|
625
713
|
|
|
626
|
-
overrideLastCommit(kind: "object" | "link", entityKey: string): string | null {
|
|
627
|
-
const row = this.db
|
|
628
|
-
.query(
|
|
629
|
-
`SELECT last_commit_id FROM ontology_overrides
|
|
630
|
-
WHERE project_id = ? AND entity_kind = ? AND entity_key = ?`
|
|
631
|
-
)
|
|
632
|
-
.get(this.projectId, kind, entityKey) as { readonly last_commit_id: string } | null
|
|
633
|
-
return row?.last_commit_id ?? null
|
|
634
|
-
}
|
|
635
|
-
|
|
636
714
|
private replacementSources(
|
|
637
715
|
sourceId: string,
|
|
638
716
|
materializationIds: readonly string[],
|
|
@@ -682,62 +760,48 @@ export class SqliteMaterializationStateReader {
|
|
|
682
760
|
return { owned, byMaterialization }
|
|
683
761
|
}
|
|
684
762
|
|
|
685
|
-
private
|
|
686
|
-
input
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
763
|
+
private prepareReplacementIdentities(input: ReplacementIdentityInput): void {
|
|
764
|
+
if (input.kind === "object") {
|
|
765
|
+
this.prepareReplacementObjects(input)
|
|
766
|
+
return
|
|
767
|
+
}
|
|
768
|
+
this.prepareReplacementLinks(input)
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
private prepareReplacementObjects(input: ReplacementIdentityInput): void {
|
|
772
|
+
this.db
|
|
690
773
|
.query(
|
|
691
774
|
`
|
|
692
|
-
|
|
775
|
+
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
776
|
+
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
777
|
+
)
|
|
778
|
+
SELECT ?, 'object', json_array(object_type_id, primary_id),
|
|
779
|
+
MIN(entity_sort_key), 1
|
|
693
780
|
FROM ontology_source_rows
|
|
694
781
|
WHERE project_id = ? AND source_id = ? AND entity_kind = 'object'
|
|
695
782
|
AND materialization_id IN (?, COALESCE(?, ''))
|
|
696
|
-
|
|
697
|
-
ORDER BY sort_key
|
|
698
|
-
LIMIT ?
|
|
783
|
+
GROUP BY object_type_id, primary_id
|
|
699
784
|
`
|
|
700
785
|
)
|
|
701
|
-
.
|
|
786
|
+
.run(
|
|
787
|
+
input.sessionId,
|
|
702
788
|
this.projectId,
|
|
703
789
|
input.sourceId,
|
|
704
790
|
input.candidateMaterializationId,
|
|
705
|
-
input.previousMaterializationId
|
|
706
|
-
|
|
707
|
-
cursor,
|
|
708
|
-
input.pageRows
|
|
709
|
-
) as (EffectiveObjectRow & { readonly sort_key: string })[]
|
|
791
|
+
input.previousMaterializationId
|
|
792
|
+
)
|
|
710
793
|
}
|
|
711
794
|
|
|
712
|
-
private
|
|
713
|
-
|
|
714
|
-
cursor: string | null
|
|
715
|
-
): (EffectiveLinkRow & { readonly sort_key: string; readonly diff_required: number })[] {
|
|
716
|
-
return this.db
|
|
795
|
+
private prepareReplacementLinks(input: ReplacementIdentityInput): void {
|
|
796
|
+
this.db
|
|
717
797
|
.query(
|
|
718
798
|
`
|
|
719
799
|
WITH incident_objects AS (
|
|
720
|
-
SELECT
|
|
800
|
+
SELECT DISTINCT
|
|
721
801
|
json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
|
|
722
802
|
json_extract(payload, '$.ref.primaryId') AS primary_id
|
|
723
803
|
FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
|
|
724
804
|
WHERE session_id = ? AND kind = 'incident-object'
|
|
725
|
-
), authority_links AS (
|
|
726
|
-
SELECT source_type_id, source_id, link_id, target_type_id, target_id
|
|
727
|
-
FROM links WHERE project_id = ?
|
|
728
|
-
UNION
|
|
729
|
-
SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
|
|
730
|
-
FROM ontology_overrides
|
|
731
|
-
WHERE project_id = ? AND entity_kind = 'link'
|
|
732
|
-
UNION
|
|
733
|
-
SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
|
|
734
|
-
rows.target_type_id, rows.target_primary_id
|
|
735
|
-
FROM ontology_source_rows AS rows
|
|
736
|
-
JOIN ontology_sources AS sources
|
|
737
|
-
ON sources.project_id = rows.project_id
|
|
738
|
-
AND sources.source_id = rows.source_id
|
|
739
|
-
AND sources.materialization_id = rows.materialization_id
|
|
740
|
-
WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.status = 'active'
|
|
741
805
|
), replacement_links AS (
|
|
742
806
|
SELECT source_type_id, source_primary_id AS source_id, link_id,
|
|
743
807
|
target_type_id, target_primary_id AS target_id
|
|
@@ -745,14 +809,67 @@ export class SqliteMaterializationStateReader {
|
|
|
745
809
|
WHERE project_id = ? AND source_id = ? AND entity_kind = 'link'
|
|
746
810
|
AND materialization_id IN (?, COALESCE(?, ''))
|
|
747
811
|
), incident_links AS (
|
|
748
|
-
SELECT links
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
812
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
813
|
+
links.target_type_id, links.target_id
|
|
814
|
+
FROM links
|
|
815
|
+
JOIN incident_objects
|
|
816
|
+
ON incident_objects.object_type_id = links.source_type_id
|
|
817
|
+
AND incident_objects.primary_id = links.source_id
|
|
818
|
+
WHERE links.project_id = ?
|
|
819
|
+
UNION
|
|
820
|
+
SELECT links.source_type_id, links.source_id, links.link_id,
|
|
821
|
+
links.target_type_id, links.target_id
|
|
822
|
+
FROM links
|
|
823
|
+
JOIN incident_objects
|
|
824
|
+
ON incident_objects.object_type_id = links.target_type_id
|
|
825
|
+
AND incident_objects.primary_id = links.target_id
|
|
826
|
+
WHERE links.project_id = ?
|
|
827
|
+
UNION
|
|
828
|
+
SELECT overrides.source_type_id,
|
|
829
|
+
overrides.source_primary_id AS source_id,
|
|
830
|
+
overrides.link_id, overrides.target_type_id,
|
|
831
|
+
overrides.target_primary_id AS target_id
|
|
832
|
+
FROM ontology_link_overrides AS overrides
|
|
833
|
+
JOIN incident_objects
|
|
834
|
+
ON incident_objects.object_type_id = overrides.source_type_id
|
|
835
|
+
AND incident_objects.primary_id = overrides.source_primary_id
|
|
836
|
+
WHERE overrides.project_id = ?
|
|
837
|
+
UNION
|
|
838
|
+
SELECT overrides.source_type_id,
|
|
839
|
+
overrides.source_primary_id AS source_id,
|
|
840
|
+
overrides.link_id, overrides.target_type_id,
|
|
841
|
+
overrides.target_primary_id AS target_id
|
|
842
|
+
FROM ontology_link_overrides AS overrides
|
|
843
|
+
JOIN incident_objects
|
|
844
|
+
ON incident_objects.object_type_id = overrides.target_type_id
|
|
845
|
+
AND incident_objects.primary_id = overrides.target_primary_id
|
|
846
|
+
WHERE overrides.project_id = ?
|
|
847
|
+
UNION
|
|
848
|
+
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
849
|
+
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
850
|
+
FROM ontology_source_rows AS rows
|
|
851
|
+
JOIN ontology_sources AS sources
|
|
852
|
+
ON sources.project_id = rows.project_id
|
|
853
|
+
AND sources.source_id = rows.source_id
|
|
854
|
+
AND sources.materialization_id = rows.materialization_id
|
|
855
|
+
JOIN incident_objects
|
|
856
|
+
ON incident_objects.object_type_id = rows.source_type_id
|
|
857
|
+
AND incident_objects.primary_id = rows.source_primary_id
|
|
858
|
+
WHERE rows.project_id = ? AND rows.entity_kind = 'link'
|
|
859
|
+
AND sources.status = 'active'
|
|
860
|
+
UNION
|
|
861
|
+
SELECT rows.source_type_id, rows.source_primary_id AS source_id,
|
|
862
|
+
rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
|
|
863
|
+
FROM ontology_source_rows AS rows
|
|
864
|
+
JOIN ontology_sources AS sources
|
|
865
|
+
ON sources.project_id = rows.project_id
|
|
866
|
+
AND sources.source_id = rows.source_id
|
|
867
|
+
AND sources.materialization_id = rows.materialization_id
|
|
868
|
+
JOIN incident_objects
|
|
869
|
+
ON incident_objects.object_type_id = rows.target_type_id
|
|
870
|
+
AND incident_objects.primary_id = rows.target_primary_id
|
|
871
|
+
WHERE rows.project_id = ? AND rows.entity_kind = 'link'
|
|
872
|
+
AND sources.status = 'active'
|
|
756
873
|
), diff_links AS (
|
|
757
874
|
SELECT * FROM replacement_links
|
|
758
875
|
UNION SELECT * FROM incident_links
|
|
@@ -766,6 +883,16 @@ export class SqliteMaterializationStateReader {
|
|
|
766
883
|
FROM links
|
|
767
884
|
JOIN affected_scopes USING (source_type_id, source_id, link_id)
|
|
768
885
|
WHERE links.project_id = ?
|
|
886
|
+
UNION ALL
|
|
887
|
+
SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
|
|
888
|
+
overrides.link_id, overrides.target_type_id,
|
|
889
|
+
overrides.target_primary_id AS target_id, 0 AS diff_required
|
|
890
|
+
FROM ontology_link_overrides AS overrides
|
|
891
|
+
JOIN affected_scopes
|
|
892
|
+
ON affected_scopes.source_type_id = overrides.source_type_id
|
|
893
|
+
AND affected_scopes.source_id = overrides.source_primary_id
|
|
894
|
+
AND affected_scopes.link_id = overrides.link_id
|
|
895
|
+
WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
|
|
769
896
|
), selected AS (
|
|
770
897
|
SELECT source_type_id, source_id, link_id, target_type_id, target_id,
|
|
771
898
|
${linkSortExpression("all_links")} AS sort_key,
|
|
@@ -773,26 +900,78 @@ export class SqliteMaterializationStateReader {
|
|
|
773
900
|
FROM all_links
|
|
774
901
|
GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
|
|
775
902
|
)
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
903
|
+
INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
|
|
904
|
+
session_id, entity_kind, identity_key, sort_key, diff_required
|
|
905
|
+
)
|
|
906
|
+
SELECT ?, 'link',
|
|
907
|
+
json_array(source_type_id, source_id, link_id, target_type_id, target_id),
|
|
908
|
+
sort_key, diff_required
|
|
909
|
+
FROM selected
|
|
780
910
|
`
|
|
781
911
|
)
|
|
782
|
-
.
|
|
912
|
+
.run(
|
|
783
913
|
input.sessionId,
|
|
784
914
|
this.projectId,
|
|
785
|
-
this.projectId,
|
|
786
|
-
this.projectId,
|
|
787
|
-
this.projectId,
|
|
788
915
|
input.sourceId,
|
|
789
916
|
input.candidateMaterializationId,
|
|
790
917
|
input.previousMaterializationId,
|
|
791
918
|
this.projectId,
|
|
792
|
-
|
|
793
|
-
|
|
919
|
+
this.projectId,
|
|
920
|
+
this.projectId,
|
|
921
|
+
this.projectId,
|
|
922
|
+
this.projectId,
|
|
923
|
+
this.projectId,
|
|
924
|
+
this.projectId,
|
|
925
|
+
this.projectId,
|
|
926
|
+
input.sessionId
|
|
927
|
+
)
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
private replacementRows(
|
|
931
|
+
input: ReplacementIdentityInput,
|
|
932
|
+
cursor: ReplacementCursor | null
|
|
933
|
+
): ReplacementWorkRow[] {
|
|
934
|
+
return this.db
|
|
935
|
+
.query(
|
|
936
|
+
`
|
|
937
|
+
SELECT entity_kind, identity_key, sort_key, diff_required
|
|
938
|
+
FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
|
|
939
|
+
WHERE session_id = ? AND entity_kind = ?
|
|
940
|
+
AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
|
|
941
|
+
ORDER BY sort_key, identity_key
|
|
942
|
+
LIMIT ?
|
|
943
|
+
`
|
|
944
|
+
)
|
|
945
|
+
.all(
|
|
946
|
+
input.sessionId,
|
|
947
|
+
input.kind,
|
|
948
|
+
cursor?.sortKey ?? null,
|
|
949
|
+
cursor?.sortKey ?? null,
|
|
950
|
+
cursor?.identityKey ?? null,
|
|
794
951
|
input.pageRows
|
|
795
|
-
) as
|
|
952
|
+
) as ReplacementWorkRow[]
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
function replacementIdentity(row: ReplacementWorkRow): ReplacementIdentity {
|
|
957
|
+
const parts = parseJson<string[]>(row.identity_key)
|
|
958
|
+
if (row.entity_kind === "object") {
|
|
959
|
+
return {
|
|
960
|
+
kind: "object",
|
|
961
|
+
ref: { objectTypeId: parts[0]!, primaryId: parts[1]! },
|
|
962
|
+
sortKey: row.sort_key,
|
|
963
|
+
diffRequired: true,
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
return {
|
|
967
|
+
kind: "link",
|
|
968
|
+
ref: {
|
|
969
|
+
source: { objectTypeId: parts[0]!, primaryId: parts[1]! },
|
|
970
|
+
linkId: parts[2]!,
|
|
971
|
+
target: { objectTypeId: parts[3]!, primaryId: parts[4]! },
|
|
972
|
+
},
|
|
973
|
+
sortKey: row.sort_key,
|
|
974
|
+
diffRequired: row.diff_required === 1,
|
|
796
975
|
}
|
|
797
976
|
}
|
|
798
977
|
|
|
@@ -814,6 +993,26 @@ function activeSourceMap(
|
|
|
814
993
|
return result
|
|
815
994
|
}
|
|
816
995
|
|
|
996
|
+
function activeScopeSourceMap(
|
|
997
|
+
rows: readonly SqliteOntologySourceAssertionRow[]
|
|
998
|
+
): ReadonlyMap<string, StoredSourceLinkAssertion> {
|
|
999
|
+
const result = new Map<string, StoredSourceLinkAssertion>()
|
|
1000
|
+
for (const row of rows) {
|
|
1001
|
+
const source = sourceLink(storedSource(row))
|
|
1002
|
+
if (!source) continue
|
|
1003
|
+
const ref = source.assertion.ref
|
|
1004
|
+
const key = linkScopeSortKey(ref.source, ref.linkId)
|
|
1005
|
+
if (result.has(key)) {
|
|
1006
|
+
throw new MaterializationConflictError(
|
|
1007
|
+
"source-materialization",
|
|
1008
|
+
`Multiple active sources assert cardinality-one link slot ${key}.`
|
|
1009
|
+
)
|
|
1010
|
+
}
|
|
1011
|
+
result.set(key, source)
|
|
1012
|
+
}
|
|
1013
|
+
return result
|
|
1014
|
+
}
|
|
1015
|
+
|
|
817
1016
|
function sourceObject(
|
|
818
1017
|
source: StoredSourceAssertion | undefined
|
|
819
1018
|
): StoredSourceObjectAssertion | null {
|
|
@@ -844,6 +1043,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
|
|
|
844
1043
|
}
|
|
845
1044
|
}
|
|
846
1045
|
|
|
1046
|
+
function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
|
|
1047
|
+
const ref = {
|
|
1048
|
+
source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
|
|
1049
|
+
linkId: row.link_id,
|
|
1050
|
+
}
|
|
1051
|
+
return {
|
|
1052
|
+
ref,
|
|
1053
|
+
value: linkSlotOverrideValue(parseJson<unknown>(row.value), linkScopeLabel(ref)),
|
|
1054
|
+
lastCommitId: row.last_commit_id,
|
|
1055
|
+
updatedAt: row.updated_at,
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
|
|
847
1059
|
function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
|
|
848
1060
|
return {
|
|
849
1061
|
source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
|
|
@@ -852,6 +1064,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
|
|
|
852
1064
|
}
|
|
853
1065
|
}
|
|
854
1066
|
|
|
1067
|
+
function linkScopeLabel(scope: {
|
|
1068
|
+
readonly source: OntologyObjectRef
|
|
1069
|
+
readonly linkId: string
|
|
1070
|
+
}): string {
|
|
1071
|
+
return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
|
|
1072
|
+
}
|
|
1073
|
+
|
|
855
1074
|
function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssertion {
|
|
856
1075
|
return {
|
|
857
1076
|
source: { projectionId: row.source_id },
|
|
@@ -924,17 +1143,7 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
|
|
|
924
1143
|
}
|
|
925
1144
|
}
|
|
926
1145
|
|
|
927
|
-
export function objectSortExpression(alias?: string): string {
|
|
928
|
-
const prefix = alias ? `${alias}.` : ""
|
|
929
|
-
return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
|
|
930
|
-
}
|
|
931
|
-
|
|
932
1146
|
export function linkSortExpression(alias?: string): string {
|
|
933
1147
|
const prefix = alias ? `${alias}.` : ""
|
|
934
1148
|
return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`
|
|
935
1149
|
}
|
|
936
|
-
|
|
937
|
-
export function pointSortExpression(alias?: string): string {
|
|
938
|
-
const prefix = alias ? `${alias}.` : ""
|
|
939
|
-
return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}object_id, ${prefix}property_id, ${prefix}at) AS BLOB)))`
|
|
940
|
-
}
|