@sixb/sqlite 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +18 -13
  2. package/dist/action-run-storage.d.ts +5 -2
  3. package/dist/action-run-storage.d.ts.map +1 -1
  4. package/dist/agents/agent-storage.d.ts +4 -2
  5. package/dist/agents/agent-storage.d.ts.map +1 -1
  6. package/dist/agents/rows.d.ts +3 -10
  7. package/dist/agents/rows.d.ts.map +1 -1
  8. package/dist/agents/runs.d.ts +3 -2
  9. package/dist/agents/runs.d.ts.map +1 -1
  10. package/dist/ai-usage-storage.d.ts +22 -0
  11. package/dist/ai-usage-storage.d.ts.map +1 -0
  12. package/dist/auth-storage/service-accounts.d.ts.map +1 -1
  13. package/dist/auth-storage/shared.d.ts +1 -10
  14. package/dist/auth-storage/shared.d.ts.map +1 -1
  15. package/dist/connector-connection-storage.d.ts +9 -0
  16. package/dist/connector-connection-storage.d.ts.map +1 -0
  17. package/dist/execution-storage.d.ts +14 -0
  18. package/dist/execution-storage.d.ts.map +1 -0
  19. package/dist/index.d.ts +13 -9
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +3862 -1042
  22. package/dist/index.js.map +30 -27
  23. package/dist/latest-run-query.d.ts.map +1 -1
  24. package/dist/migrations.d.ts +0 -1
  25. package/dist/migrations.d.ts.map +1 -1
  26. package/dist/ontology-storage/materialization-session.d.ts +50 -8
  27. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  28. package/dist/ontology-storage/materialization-state.d.ts +4 -5
  29. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  30. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  31. package/dist/ontology-storage/materializations.d.ts +0 -2
  32. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  33. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  34. package/dist/ontology-storage/shared.d.ts +3 -3
  35. package/dist/ontology-storage/shared.d.ts.map +1 -1
  36. package/dist/ontology-storage/sources.d.ts.map +1 -1
  37. package/dist/pipeline-run-storage.d.ts +7 -2
  38. package/dist/pipeline-run-storage.d.ts.map +1 -1
  39. package/dist/projection-run-storage.d.ts +9 -2
  40. package/dist/projection-run-storage.d.ts.map +1 -1
  41. package/dist/run-list-query.d.ts +1 -1
  42. package/dist/run-list-query.d.ts.map +1 -1
  43. package/dist/sync-run-storage.d.ts +7 -2
  44. package/dist/sync-run-storage.d.ts.map +1 -1
  45. package/dist/testing.d.ts +6 -0
  46. package/dist/testing.d.ts.map +1 -0
  47. package/dist/transactions.d.ts +3 -0
  48. package/dist/transactions.d.ts.map +1 -1
  49. package/dist/webhook-run-storage.d.ts +9 -2
  50. package/dist/webhook-run-storage.d.ts.map +1 -1
  51. package/dist/workflow-run-storage.d.ts +7 -3
  52. package/dist/workflow-run-storage.d.ts.map +1 -1
  53. package/package.json +3 -3
  54. package/src/action-run-storage.ts +51 -74
  55. package/src/agents/agent-storage.ts +5 -3
  56. package/src/agents/rows.ts +8 -40
  57. package/src/agents/runs.ts +27 -21
  58. package/src/ai-usage-storage.ts +324 -0
  59. package/src/auth-storage/service-accounts.ts +14 -21
  60. package/src/auth-storage/shared.ts +0 -22
  61. package/src/connector-connection-storage.ts +784 -0
  62. package/src/execution-storage.ts +158 -0
  63. package/src/index.ts +63 -22
  64. package/src/latest-run-query.ts +7 -1
  65. package/src/migrations/004-executions.sql +142 -0
  66. package/src/migrations/005-workflow-executions.sql +7 -0
  67. package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
  68. package/src/migrations/007-split-overrides.sql +154 -0
  69. package/src/migrations/008-action-executions.sql +4 -0
  70. package/src/migrations/009-agent-executions.sql +115 -0
  71. package/src/migrations/010-ai-usage-accounting-foundation.sql +74 -0
  72. package/src/migrations/011-sync-failure-record.sql +22 -0
  73. package/src/migrations/012-pipeline-failure-record.sql +45 -0
  74. package/src/migrations/013-workflow-failure-record.sql +45 -0
  75. package/src/migrations/014-agent-failure-record.sql +59 -0
  76. package/src/migrations/015-projection-failure-record.sql +20 -0
  77. package/src/migrations/016-webhook-run-failure-record.sql +19 -0
  78. package/src/migrations/017-action-failure-record.sql +65 -0
  79. package/src/migrations/018-ontology-outbox-failure-record.sql +80 -0
  80. package/src/migrations/019-webhook-delivery-failure-record.sql +62 -0
  81. package/src/migrations/020-drop-run-usage-projections.sql +7 -0
  82. package/src/migrations/021-sync-pipeline-executions.sql +101 -0
  83. package/src/migrations/022-projection-executions.sql +244 -0
  84. package/src/migrations/023-webhook-executions.sql +51 -0
  85. package/src/migrations/024-ontology-commit-executions.sql +58 -0
  86. package/src/migrations/025-connector-connections.sql +257 -0
  87. package/src/migrations.ts +119 -4
  88. package/src/ontology-storage/materialization-session.ts +317 -143
  89. package/src/ontology-storage/materialization-state.ts +321 -112
  90. package/src/ontology-storage/materialization-writer.ts +142 -67
  91. package/src/ontology-storage/materializations.ts +67 -141
  92. package/src/ontology-storage/outbox.ts +10 -3
  93. package/src/ontology-storage/shared.ts +10 -4
  94. package/src/ontology-storage/sources.ts +4 -1
  95. package/src/pipeline-run-storage.ts +128 -43
  96. package/src/projection-run-storage.ts +209 -97
  97. package/src/run-list-query.ts +9 -4
  98. package/src/sync-run-storage.ts +118 -32
  99. package/src/testing.ts +20 -0
  100. package/src/transactions.ts +10 -3
  101. package/src/webhook-run-storage.ts +118 -74
  102. package/src/workflow-run-storage.ts +117 -131
  103. package/dist/validation.d.ts +0 -5
  104. package/dist/validation.d.ts.map +0 -1
  105. package/dist/webhook-delivery-storage.d.ts +0 -27
  106. package/dist/webhook-delivery-storage.d.ts.map +0 -1
  107. package/src/validation.ts +0 -30
  108. package/src/webhook-delivery-storage.ts +0 -272
@@ -18,6 +18,7 @@ import {
18
18
  import {
19
19
  appendScopeSnapshot,
20
20
  finishScopeAccumulator,
21
+ linkSlotOverrideValue,
21
22
  startScopeAccumulator,
22
23
  } from "@sixb/core/internal/ontology-storage-provider"
23
24
  import type {
@@ -27,6 +28,7 @@ import type {
27
28
  SourceReplacementLinkState,
28
29
  SourceReplacementObjectState,
29
30
  StoredLinkOverride,
31
+ StoredLinkSlotOverride,
30
32
  StoredObjectOverride,
31
33
  StoredSourceAssertion,
32
34
  StoredSourceLinkAssertion,
@@ -38,8 +40,8 @@ import {
38
40
  linkRefFromColumns,
39
41
  objectRefFromColumns,
40
42
  parseJson,
41
- type SqliteOntologyOverrideRow,
42
43
  type SqliteOntologySourceAssertionRow,
44
+ type SqliteStoredOverrideRow,
43
45
  } from "./shared"
44
46
 
45
47
  export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
@@ -81,12 +83,12 @@ interface LinkScopeRow extends EffectiveLinkRow {
81
83
  readonly scope_sort_key: string
82
84
  }
83
85
 
84
- interface ObjectOverrideRow extends SqliteOntologyOverrideRow {
86
+ interface ObjectOverrideRow extends SqliteStoredOverrideRow {
85
87
  readonly object_type_id: string
86
88
  readonly primary_id: string
87
89
  }
88
90
 
89
- interface LinkOverrideRow extends SqliteOntologyOverrideRow {
91
+ interface LinkOverrideRow extends SqliteStoredOverrideRow {
90
92
  readonly source_type_id: string
91
93
  readonly source_primary_id: string
92
94
  readonly link_id: string
@@ -94,6 +96,17 @@ interface LinkOverrideRow extends SqliteOntologyOverrideRow {
94
96
  readonly target_primary_id: string
95
97
  }
96
98
 
99
+ interface LinkSlotOverrideRow {
100
+ readonly source_type_id: string
101
+ readonly source_primary_id: string
102
+ readonly link_id: string
103
+ readonly target_type_id: string
104
+ readonly target_primary_id: string
105
+ readonly value: string
106
+ readonly last_commit_id: string
107
+ readonly updated_at: string
108
+ }
109
+
97
110
  interface ReplacementSources {
98
111
  readonly owned: ReadonlySet<string>
99
112
  readonly byMaterialization: ReadonlyMap<string, ReadonlyMap<string, StoredSourceAssertion>>
@@ -122,6 +135,18 @@ interface ReplacementIdentityInput {
122
135
  readonly pageRows: number
123
136
  }
124
137
 
138
+ interface ReplacementWorkRow {
139
+ readonly entity_kind: "object" | "link"
140
+ readonly identity_key: string
141
+ readonly sort_key: string
142
+ readonly diff_required: number
143
+ }
144
+
145
+ interface ReplacementCursor {
146
+ readonly sortKey: string
147
+ readonly identityKey: string
148
+ }
149
+
125
150
  export class SqliteMaterializationStateReader {
126
151
  constructor(
127
152
  private readonly db: Database,
@@ -175,9 +200,8 @@ export class SqliteMaterializationStateReader {
175
200
  .query(
176
201
  `WITH requested AS (${requestedObjects})
177
202
  SELECT overrides.* FROM requested
178
- CROSS JOIN ontology_overrides AS overrides
203
+ CROSS JOIN ontology_object_overrides AS overrides
179
204
  ON overrides.project_id = ?
180
- AND overrides.entity_kind = 'object'
181
205
  AND overrides.object_type_id = requested.object_type_id
182
206
  AND overrides.primary_id = requested.primary_id`
183
207
  )
@@ -305,9 +329,9 @@ export class SqliteMaterializationStateReader {
305
329
  .query(
306
330
  `WITH requested AS (${sourceRequest})
307
331
  SELECT overrides.* FROM requested
308
- CROSS JOIN ontology_overrides AS overrides
332
+ CROSS JOIN ontology_link_overrides AS overrides
309
333
  ON overrides.project_id = ?
310
- AND overrides.entity_kind = 'link'
334
+ AND overrides.identity_kind = 'edge'
311
335
  AND overrides.source_type_id = requested.source_type_id
312
336
  AND overrides.source_primary_id = requested.source_primary_id
313
337
  AND overrides.link_id = requested.link_id
@@ -315,6 +339,18 @@ export class SqliteMaterializationStateReader {
315
339
  AND overrides.target_primary_id = requested.target_primary_id`
316
340
  )
317
341
  .all(requested, this.projectId) as LinkOverrideRow[]
342
+ const slotOverrideRows = this.db
343
+ .query(
344
+ `WITH requested AS (${sourceRequest})
345
+ SELECT overrides.* FROM requested
346
+ CROSS JOIN ontology_link_overrides AS overrides
347
+ ON overrides.project_id = ?
348
+ AND overrides.identity_kind = 'slot'
349
+ AND overrides.source_type_id = requested.source_type_id
350
+ AND overrides.source_primary_id = requested.source_primary_id
351
+ AND overrides.link_id = requested.link_id`
352
+ )
353
+ .all(requested, this.projectId) as LinkSlotOverrideRow[]
318
354
  const effective = new Map(
319
355
  effectiveRows.map((row) => [linkRefKey(linkRefFromColumns(row)), row] as const)
320
356
  )
@@ -324,6 +360,12 @@ export class SqliteMaterializationStateReader {
324
360
  (row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
325
361
  )
326
362
  )
363
+ const slotOverrides = new Map(
364
+ slotOverrideRows.map((row) => {
365
+ const stored = storedLinkSlotOverride(row)
366
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
367
+ })
368
+ )
327
369
  return refs.map((ref) => {
328
370
  const key = linkRefKey(ref)
329
371
  const effectiveRow = effective.get(key)
@@ -331,6 +373,7 @@ export class SqliteMaterializationStateReader {
331
373
  ref: structuredClone(ref),
332
374
  source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
333
375
  override: overrides.get(key) ?? null,
376
+ slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
334
377
  effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
335
378
  }
336
379
  })
@@ -395,9 +438,59 @@ export class SqliteMaterializationStateReader {
395
438
  const accumulators = new Map(
396
439
  scopes.map(({ source, linkId }) => {
397
440
  const key = linkScopeSortKey(source, linkId)
398
- return [key, startScopeAccumulator(source, linkId, key)] as const
441
+ return [key, startScopeAccumulator(source, linkId)] as const
399
442
  })
400
443
  )
444
+ const requestedJson = canonicalJson(requested)
445
+ const sourceRows = this.db
446
+ .query(
447
+ `WITH requested AS (
448
+ SELECT DISTINCT
449
+ json_extract(value, '$.sourceTypeId') AS source_type_id,
450
+ json_extract(value, '$.sourceId') AS source_primary_id,
451
+ json_extract(value, '$.linkId') AS link_id
452
+ FROM json_each(?)
453
+ )
454
+ SELECT rows.* FROM requested
455
+ CROSS JOIN ontology_source_rows AS rows
456
+ ON rows.project_id = ?
457
+ AND rows.entity_kind = 'link'
458
+ AND rows.source_type_id = requested.source_type_id
459
+ AND rows.source_primary_id = requested.source_primary_id
460
+ AND rows.link_id = requested.link_id
461
+ JOIN ontology_sources AS sources
462
+ ON sources.project_id = rows.project_id
463
+ AND sources.source_id = rows.source_id
464
+ AND sources.materialization_id = rows.materialization_id
465
+ WHERE sources.status = 'active'`
466
+ )
467
+ .all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
468
+ const slotOverrideRows = this.db
469
+ .query(
470
+ `WITH requested AS (
471
+ SELECT DISTINCT
472
+ json_extract(value, '$.sourceTypeId') AS source_type_id,
473
+ json_extract(value, '$.sourceId') AS source_primary_id,
474
+ json_extract(value, '$.linkId') AS link_id
475
+ FROM json_each(?)
476
+ )
477
+ SELECT overrides.* FROM requested
478
+ CROSS JOIN ontology_link_overrides AS overrides
479
+ ON overrides.project_id = ?
480
+ AND overrides.identity_kind = 'slot'
481
+ AND overrides.source_type_id = requested.source_type_id
482
+ AND overrides.source_primary_id = requested.source_primary_id
483
+ AND overrides.link_id = requested.link_id`
484
+ )
485
+ .all(requestedJson, this.projectId) as LinkSlotOverrideRow[]
486
+ const sources = activeScopeSourceMap(sourceRows)
487
+ const overrides = new Map(
488
+ slotOverrideRows.map((row) => {
489
+ const stored = storedLinkSlotOverride(row)
490
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
491
+ })
492
+ )
493
+ const effective = new Map<string, EffectiveLinkSnapshot | null>()
401
494
  const rows = this.db
402
495
  .query(
403
496
  `
@@ -419,7 +512,7 @@ export class SqliteMaterializationStateReader {
419
512
  ORDER BY requested.scope_sort_key, ${linkSortExpression("links")}
420
513
  `
421
514
  )
422
- .iterate(canonicalJson(requested), this.projectId) as Iterable<LinkScopeRow>
515
+ .iterate(requestedJson, this.projectId) as Iterable<LinkScopeRow>
423
516
  for (const row of rows) {
424
517
  const accumulator = accumulators.get(row.scope_sort_key)
425
518
  if (!accumulator) {
@@ -428,11 +521,19 @@ export class SqliteMaterializationStateReader {
428
521
  `Unexpected link scope '${row.scope_sort_key}' returned by storage.`
429
522
  )
430
523
  }
431
- appendScopeSnapshot(accumulator, effectiveLinkSnapshot(row))
524
+ const snapshot = effectiveLinkSnapshot(row)
525
+ appendScopeSnapshot(accumulator, snapshot)
526
+ effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
432
527
  }
433
- return scopes.map(({ source, linkId }) =>
434
- finishScopeAccumulator(accumulators.get(linkScopeSortKey(source, linkId))!)
435
- )
528
+ return scopes.map(({ source, linkId }) => {
529
+ const key = linkScopeSortKey(source, linkId)
530
+ return {
531
+ ...finishScopeAccumulator(accumulators.get(key)!),
532
+ sourceAssertion: sources.get(key) ?? null,
533
+ override: overrides.get(key) ?? null,
534
+ effective: effective.get(key) ?? null,
535
+ }
536
+ })
436
537
  }
437
538
 
438
539
  linkScope(source: OntologyObjectRef, linkId: string): MaterializationLinkScopeState {
@@ -464,8 +565,8 @@ export class SqliteMaterializationStateReader {
464
565
  FROM links WHERE project_id = ?
465
566
  UNION
466
567
  SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
467
- FROM ontology_overrides
468
- WHERE project_id = ? AND entity_kind = 'link'
568
+ FROM ontology_link_overrides
569
+ WHERE project_id = ?
469
570
  UNION
470
571
  SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
471
572
  rows.target_type_id, rows.target_primary_id
@@ -509,29 +610,15 @@ export class SqliteMaterializationStateReader {
509
610
  }
510
611
 
511
612
  *replacementIdentities(input: ReplacementIdentityInput): Iterable<ReplacementIdentity[]> {
512
- let cursor: string | null = null
613
+ this.prepareReplacementIdentities(input)
614
+ let cursor: ReplacementCursor | null = null
513
615
  while (true) {
514
- if (input.kind === "object") {
515
- const rows = this.replacementObjectRows(input, cursor)
516
- if (rows.length === 0) break
517
- yield rows.map((row) => ({
518
- kind: "object",
519
- ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
520
- sortKey: row.sort_key,
521
- diffRequired: true,
522
- }))
523
- cursor = rows[rows.length - 1]?.sort_key ?? null
524
- continue
525
- }
526
- const rows = this.replacementLinkRows(input, cursor)
616
+ const rows = this.replacementRows(input, cursor)
527
617
  if (rows.length === 0) break
528
- yield rows.map((row) => ({
529
- kind: "link",
530
- ref: linkRefFromColumns(row),
531
- sortKey: row.sort_key,
532
- diffRequired: row.diff_required === 1,
533
- }))
534
- cursor = rows[rows.length - 1]?.sort_key ?? null
618
+ yield rows.map(replacementIdentity)
619
+ if (rows.length < input.pageRows) break
620
+ const last = rows[rows.length - 1]!
621
+ cursor = { sortKey: last.sort_key, identityKey: last.identity_key }
535
622
  }
536
623
  }
537
624
 
@@ -581,6 +668,7 @@ export class SqliteMaterializationStateReader {
581
668
  ? sourceLink(candidate?.get(key))
582
669
  : state.source,
583
670
  override: state.override,
671
+ slotOverride: state.slotOverride,
584
672
  effective: state.effective,
585
673
  diffRequired: identities[index]!.diffRequired,
586
674
  }
@@ -623,16 +711,6 @@ export class SqliteMaterializationStateReader {
623
711
  return row ? row.last_commit_id : undefined
624
712
  }
625
713
 
626
- overrideLastCommit(kind: "object" | "link", entityKey: string): string | null {
627
- const row = this.db
628
- .query(
629
- `SELECT last_commit_id FROM ontology_overrides
630
- WHERE project_id = ? AND entity_kind = ? AND entity_key = ?`
631
- )
632
- .get(this.projectId, kind, entityKey) as { readonly last_commit_id: string } | null
633
- return row?.last_commit_id ?? null
634
- }
635
-
636
714
  private replacementSources(
637
715
  sourceId: string,
638
716
  materializationIds: readonly string[],
@@ -682,62 +760,48 @@ export class SqliteMaterializationStateReader {
682
760
  return { owned, byMaterialization }
683
761
  }
684
762
 
685
- private replacementObjectRows(
686
- input: ReplacementIdentityInput,
687
- cursor: string | null
688
- ): (EffectiveObjectRow & { readonly sort_key: string })[] {
689
- return this.db
763
+ private prepareReplacementIdentities(input: ReplacementIdentityInput): void {
764
+ if (input.kind === "object") {
765
+ this.prepareReplacementObjects(input)
766
+ return
767
+ }
768
+ this.prepareReplacementLinks(input)
769
+ }
770
+
771
+ private prepareReplacementObjects(input: ReplacementIdentityInput): void {
772
+ this.db
690
773
  .query(
691
774
  `
692
- SELECT DISTINCT object_type_id, primary_id, entity_sort_key AS sort_key
775
+ INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
776
+ session_id, entity_kind, identity_key, sort_key, diff_required
777
+ )
778
+ SELECT ?, 'object', json_array(object_type_id, primary_id),
779
+ MIN(entity_sort_key), 1
693
780
  FROM ontology_source_rows
694
781
  WHERE project_id = ? AND source_id = ? AND entity_kind = 'object'
695
782
  AND materialization_id IN (?, COALESCE(?, ''))
696
- AND (? IS NULL OR entity_sort_key > ?)
697
- ORDER BY sort_key
698
- LIMIT ?
783
+ GROUP BY object_type_id, primary_id
699
784
  `
700
785
  )
701
- .all(
786
+ .run(
787
+ input.sessionId,
702
788
  this.projectId,
703
789
  input.sourceId,
704
790
  input.candidateMaterializationId,
705
- input.previousMaterializationId,
706
- cursor,
707
- cursor,
708
- input.pageRows
709
- ) as (EffectiveObjectRow & { readonly sort_key: string })[]
791
+ input.previousMaterializationId
792
+ )
710
793
  }
711
794
 
712
- private replacementLinkRows(
713
- input: ReplacementIdentityInput,
714
- cursor: string | null
715
- ): (EffectiveLinkRow & { readonly sort_key: string; readonly diff_required: number })[] {
716
- return this.db
795
+ private prepareReplacementLinks(input: ReplacementIdentityInput): void {
796
+ this.db
717
797
  .query(
718
798
  `
719
799
  WITH incident_objects AS (
720
- SELECT
800
+ SELECT DISTINCT
721
801
  json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
722
802
  json_extract(payload, '$.ref.primaryId') AS primary_id
723
803
  FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
724
804
  WHERE session_id = ? AND kind = 'incident-object'
725
- ), authority_links AS (
726
- SELECT source_type_id, source_id, link_id, target_type_id, target_id
727
- FROM links WHERE project_id = ?
728
- UNION
729
- SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
730
- FROM ontology_overrides
731
- WHERE project_id = ? AND entity_kind = 'link'
732
- UNION
733
- SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
734
- rows.target_type_id, rows.target_primary_id
735
- FROM ontology_source_rows AS rows
736
- JOIN ontology_sources AS sources
737
- ON sources.project_id = rows.project_id
738
- AND sources.source_id = rows.source_id
739
- AND sources.materialization_id = rows.materialization_id
740
- WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.status = 'active'
741
805
  ), replacement_links AS (
742
806
  SELECT source_type_id, source_primary_id AS source_id, link_id,
743
807
  target_type_id, target_primary_id AS target_id
@@ -745,14 +809,67 @@ export class SqliteMaterializationStateReader {
745
809
  WHERE project_id = ? AND source_id = ? AND entity_kind = 'link'
746
810
  AND materialization_id IN (?, COALESCE(?, ''))
747
811
  ), incident_links AS (
748
- SELECT links.* FROM authority_links AS links
749
- WHERE EXISTS (
750
- SELECT 1 FROM incident_objects
751
- WHERE (incident_objects.object_type_id = links.source_type_id
752
- AND incident_objects.primary_id = links.source_id)
753
- OR (incident_objects.object_type_id = links.target_type_id
754
- AND incident_objects.primary_id = links.target_id)
755
- )
812
+ SELECT links.source_type_id, links.source_id, links.link_id,
813
+ links.target_type_id, links.target_id
814
+ FROM links
815
+ JOIN incident_objects
816
+ ON incident_objects.object_type_id = links.source_type_id
817
+ AND incident_objects.primary_id = links.source_id
818
+ WHERE links.project_id = ?
819
+ UNION
820
+ SELECT links.source_type_id, links.source_id, links.link_id,
821
+ links.target_type_id, links.target_id
822
+ FROM links
823
+ JOIN incident_objects
824
+ ON incident_objects.object_type_id = links.target_type_id
825
+ AND incident_objects.primary_id = links.target_id
826
+ WHERE links.project_id = ?
827
+ UNION
828
+ SELECT overrides.source_type_id,
829
+ overrides.source_primary_id AS source_id,
830
+ overrides.link_id, overrides.target_type_id,
831
+ overrides.target_primary_id AS target_id
832
+ FROM ontology_link_overrides AS overrides
833
+ JOIN incident_objects
834
+ ON incident_objects.object_type_id = overrides.source_type_id
835
+ AND incident_objects.primary_id = overrides.source_primary_id
836
+ WHERE overrides.project_id = ?
837
+ UNION
838
+ SELECT overrides.source_type_id,
839
+ overrides.source_primary_id AS source_id,
840
+ overrides.link_id, overrides.target_type_id,
841
+ overrides.target_primary_id AS target_id
842
+ FROM ontology_link_overrides AS overrides
843
+ JOIN incident_objects
844
+ ON incident_objects.object_type_id = overrides.target_type_id
845
+ AND incident_objects.primary_id = overrides.target_primary_id
846
+ WHERE overrides.project_id = ?
847
+ UNION
848
+ SELECT rows.source_type_id, rows.source_primary_id AS source_id,
849
+ rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
850
+ FROM ontology_source_rows AS rows
851
+ JOIN ontology_sources AS sources
852
+ ON sources.project_id = rows.project_id
853
+ AND sources.source_id = rows.source_id
854
+ AND sources.materialization_id = rows.materialization_id
855
+ JOIN incident_objects
856
+ ON incident_objects.object_type_id = rows.source_type_id
857
+ AND incident_objects.primary_id = rows.source_primary_id
858
+ WHERE rows.project_id = ? AND rows.entity_kind = 'link'
859
+ AND sources.status = 'active'
860
+ UNION
861
+ SELECT rows.source_type_id, rows.source_primary_id AS source_id,
862
+ rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
863
+ FROM ontology_source_rows AS rows
864
+ JOIN ontology_sources AS sources
865
+ ON sources.project_id = rows.project_id
866
+ AND sources.source_id = rows.source_id
867
+ AND sources.materialization_id = rows.materialization_id
868
+ JOIN incident_objects
869
+ ON incident_objects.object_type_id = rows.target_type_id
870
+ AND incident_objects.primary_id = rows.target_primary_id
871
+ WHERE rows.project_id = ? AND rows.entity_kind = 'link'
872
+ AND sources.status = 'active'
756
873
  ), diff_links AS (
757
874
  SELECT * FROM replacement_links
758
875
  UNION SELECT * FROM incident_links
@@ -766,6 +883,16 @@ export class SqliteMaterializationStateReader {
766
883
  FROM links
767
884
  JOIN affected_scopes USING (source_type_id, source_id, link_id)
768
885
  WHERE links.project_id = ?
886
+ UNION ALL
887
+ SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
888
+ overrides.link_id, overrides.target_type_id,
889
+ overrides.target_primary_id AS target_id, 0 AS diff_required
890
+ FROM ontology_link_overrides AS overrides
891
+ JOIN affected_scopes
892
+ ON affected_scopes.source_type_id = overrides.source_type_id
893
+ AND affected_scopes.source_id = overrides.source_primary_id
894
+ AND affected_scopes.link_id = overrides.link_id
895
+ WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
769
896
  ), selected AS (
770
897
  SELECT source_type_id, source_id, link_id, target_type_id, target_id,
771
898
  ${linkSortExpression("all_links")} AS sort_key,
@@ -773,26 +900,78 @@ export class SqliteMaterializationStateReader {
773
900
  FROM all_links
774
901
  GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
775
902
  )
776
- SELECT * FROM selected
777
- WHERE (? IS NULL OR sort_key > ?)
778
- ORDER BY sort_key
779
- LIMIT ?
903
+ INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
904
+ session_id, entity_kind, identity_key, sort_key, diff_required
905
+ )
906
+ SELECT ?, 'link',
907
+ json_array(source_type_id, source_id, link_id, target_type_id, target_id),
908
+ sort_key, diff_required
909
+ FROM selected
780
910
  `
781
911
  )
782
- .all(
912
+ .run(
783
913
  input.sessionId,
784
914
  this.projectId,
785
- this.projectId,
786
- this.projectId,
787
- this.projectId,
788
915
  input.sourceId,
789
916
  input.candidateMaterializationId,
790
917
  input.previousMaterializationId,
791
918
  this.projectId,
792
- cursor,
793
- cursor,
919
+ this.projectId,
920
+ this.projectId,
921
+ this.projectId,
922
+ this.projectId,
923
+ this.projectId,
924
+ this.projectId,
925
+ this.projectId,
926
+ input.sessionId
927
+ )
928
+ }
929
+
930
+ private replacementRows(
931
+ input: ReplacementIdentityInput,
932
+ cursor: ReplacementCursor | null
933
+ ): ReplacementWorkRow[] {
934
+ return this.db
935
+ .query(
936
+ `
937
+ SELECT entity_kind, identity_key, sort_key, diff_required
938
+ FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
939
+ WHERE session_id = ? AND entity_kind = ?
940
+ AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
941
+ ORDER BY sort_key, identity_key
942
+ LIMIT ?
943
+ `
944
+ )
945
+ .all(
946
+ input.sessionId,
947
+ input.kind,
948
+ cursor?.sortKey ?? null,
949
+ cursor?.sortKey ?? null,
950
+ cursor?.identityKey ?? null,
794
951
  input.pageRows
795
- ) as (EffectiveLinkRow & { readonly sort_key: string; readonly diff_required: number })[]
952
+ ) as ReplacementWorkRow[]
953
+ }
954
+ }
955
+
956
+ function replacementIdentity(row: ReplacementWorkRow): ReplacementIdentity {
957
+ const parts = parseJson<string[]>(row.identity_key)
958
+ if (row.entity_kind === "object") {
959
+ return {
960
+ kind: "object",
961
+ ref: { objectTypeId: parts[0]!, primaryId: parts[1]! },
962
+ sortKey: row.sort_key,
963
+ diffRequired: true,
964
+ }
965
+ }
966
+ return {
967
+ kind: "link",
968
+ ref: {
969
+ source: { objectTypeId: parts[0]!, primaryId: parts[1]! },
970
+ linkId: parts[2]!,
971
+ target: { objectTypeId: parts[3]!, primaryId: parts[4]! },
972
+ },
973
+ sortKey: row.sort_key,
974
+ diffRequired: row.diff_required === 1,
796
975
  }
797
976
  }
798
977
 
@@ -814,6 +993,26 @@ function activeSourceMap(
814
993
  return result
815
994
  }
816
995
 
996
+ function activeScopeSourceMap(
997
+ rows: readonly SqliteOntologySourceAssertionRow[]
998
+ ): ReadonlyMap<string, StoredSourceLinkAssertion> {
999
+ const result = new Map<string, StoredSourceLinkAssertion>()
1000
+ for (const row of rows) {
1001
+ const source = sourceLink(storedSource(row))
1002
+ if (!source) continue
1003
+ const ref = source.assertion.ref
1004
+ const key = linkScopeSortKey(ref.source, ref.linkId)
1005
+ if (result.has(key)) {
1006
+ throw new MaterializationConflictError(
1007
+ "source-materialization",
1008
+ `Multiple active sources assert cardinality-one link slot ${key}.`
1009
+ )
1010
+ }
1011
+ result.set(key, source)
1012
+ }
1013
+ return result
1014
+ }
1015
+
817
1016
  function sourceObject(
818
1017
  source: StoredSourceAssertion | undefined
819
1018
  ): StoredSourceObjectAssertion | null {
@@ -844,6 +1043,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
844
1043
  }
845
1044
  }
846
1045
 
1046
+ function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
1047
+ const ref = {
1048
+ source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
1049
+ linkId: row.link_id,
1050
+ }
1051
+ return {
1052
+ ref,
1053
+ value: linkSlotOverrideValue(parseJson<unknown>(row.value), linkScopeLabel(ref)),
1054
+ lastCommitId: row.last_commit_id,
1055
+ updatedAt: row.updated_at,
1056
+ }
1057
+ }
1058
+
847
1059
  function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
848
1060
  return {
849
1061
  source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
@@ -852,6 +1064,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
852
1064
  }
853
1065
  }
854
1066
 
1067
+ function linkScopeLabel(scope: {
1068
+ readonly source: OntologyObjectRef
1069
+ readonly linkId: string
1070
+ }): string {
1071
+ return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
1072
+ }
1073
+
855
1074
  function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssertion {
856
1075
  return {
857
1076
  source: { projectionId: row.source_id },
@@ -924,17 +1143,7 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
924
1143
  }
925
1144
  }
926
1145
 
927
- export function objectSortExpression(alias?: string): string {
928
- const prefix = alias ? `${alias}.` : ""
929
- return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
930
- }
931
-
932
1146
  export function linkSortExpression(alias?: string): string {
933
1147
  const prefix = alias ? `${alias}.` : ""
934
1148
  return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`
935
1149
  }
936
-
937
- export function pointSortExpression(alias?: string): string {
938
- const prefix = alias ? `${alias}.` : ""
939
- return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}object_id, ${prefix}property_id, ${prefix}at) AS BLOB)))`
940
- }