@sixb/sqlite 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +18 -13
  2. package/dist/action-run-storage.d.ts +5 -2
  3. package/dist/action-run-storage.d.ts.map +1 -1
  4. package/dist/agents/agent-storage.d.ts +9 -5
  5. package/dist/agents/agent-storage.d.ts.map +1 -1
  6. package/dist/agents/checkpoints.d.ts +14 -0
  7. package/dist/agents/checkpoints.d.ts.map +1 -0
  8. package/dist/agents/index.d.ts +1 -0
  9. package/dist/agents/index.d.ts.map +1 -1
  10. package/dist/agents/messages.d.ts +2 -1
  11. package/dist/agents/messages.d.ts.map +1 -1
  12. package/dist/agents/rows.d.ts +21 -11
  13. package/dist/agents/rows.d.ts.map +1 -1
  14. package/dist/agents/runs.d.ts +3 -2
  15. package/dist/agents/runs.d.ts.map +1 -1
  16. package/dist/ai-cost-storage.d.ts +15 -0
  17. package/dist/ai-cost-storage.d.ts.map +1 -0
  18. package/dist/ai-usage-storage.d.ts +22 -0
  19. package/dist/ai-usage-storage.d.ts.map +1 -0
  20. package/dist/auth-storage/service-accounts.d.ts.map +1 -1
  21. package/dist/auth-storage/shared.d.ts +1 -10
  22. package/dist/auth-storage/shared.d.ts.map +1 -1
  23. package/dist/connector-connection-storage.d.ts +9 -0
  24. package/dist/connector-connection-storage.d.ts.map +1 -0
  25. package/dist/execution-storage.d.ts +14 -0
  26. package/dist/execution-storage.d.ts.map +1 -0
  27. package/dist/index.d.ts +14 -9
  28. package/dist/index.d.ts.map +1 -1
  29. package/dist/index.js +4633 -1066
  30. package/dist/index.js.map +33 -28
  31. package/dist/latest-run-query.d.ts.map +1 -1
  32. package/dist/migrations.d.ts +0 -1
  33. package/dist/migrations.d.ts.map +1 -1
  34. package/dist/ontology-storage/materialization-session.d.ts +50 -8
  35. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  36. package/dist/ontology-storage/materialization-state.d.ts +4 -5
  37. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  38. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  39. package/dist/ontology-storage/materializations.d.ts +0 -2
  40. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  41. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  42. package/dist/ontology-storage/shared.d.ts +3 -3
  43. package/dist/ontology-storage/shared.d.ts.map +1 -1
  44. package/dist/ontology-storage/sources.d.ts.map +1 -1
  45. package/dist/pipeline-run-storage.d.ts +7 -2
  46. package/dist/pipeline-run-storage.d.ts.map +1 -1
  47. package/dist/projection-run-storage.d.ts +9 -2
  48. package/dist/projection-run-storage.d.ts.map +1 -1
  49. package/dist/run-list-query.d.ts +1 -1
  50. package/dist/run-list-query.d.ts.map +1 -1
  51. package/dist/sync-run-storage.d.ts +7 -2
  52. package/dist/sync-run-storage.d.ts.map +1 -1
  53. package/dist/testing.d.ts +6 -0
  54. package/dist/testing.d.ts.map +1 -0
  55. package/dist/transactions.d.ts +3 -0
  56. package/dist/transactions.d.ts.map +1 -1
  57. package/dist/webhook-run-storage.d.ts +9 -2
  58. package/dist/webhook-run-storage.d.ts.map +1 -1
  59. package/dist/workflow-run-storage.d.ts +7 -3
  60. package/dist/workflow-run-storage.d.ts.map +1 -1
  61. package/package.json +3 -3
  62. package/src/action-run-storage.ts +51 -74
  63. package/src/agents/agent-storage.ts +11 -6
  64. package/src/agents/checkpoints.ts +175 -0
  65. package/src/agents/index.ts +1 -0
  66. package/src/agents/messages.ts +59 -0
  67. package/src/agents/rows.ts +52 -40
  68. package/src/agents/runs.ts +27 -21
  69. package/src/ai-cost-storage.ts +744 -0
  70. package/src/ai-usage-storage.ts +324 -0
  71. package/src/auth-storage/service-accounts.ts +14 -21
  72. package/src/auth-storage/shared.ts +0 -22
  73. package/src/connector-connection-storage.ts +784 -0
  74. package/src/execution-storage.ts +158 -0
  75. package/src/index.ts +69 -22
  76. package/src/latest-run-query.ts +7 -1
  77. package/src/migrations/004-executions.sql +142 -0
  78. package/src/migrations/005-workflow-executions.sql +7 -0
  79. package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
  80. package/src/migrations/007-split-overrides.sql +154 -0
  81. package/src/migrations/008-action-executions.sql +4 -0
  82. package/src/migrations/009-agent-executions.sql +115 -0
  83. package/src/migrations/010-ai-usage-accounting-foundation.sql +74 -0
  84. package/src/migrations/011-sync-failure-record.sql +22 -0
  85. package/src/migrations/012-pipeline-failure-record.sql +45 -0
  86. package/src/migrations/013-workflow-failure-record.sql +45 -0
  87. package/src/migrations/014-agent-failure-record.sql +59 -0
  88. package/src/migrations/015-projection-failure-record.sql +20 -0
  89. package/src/migrations/016-webhook-run-failure-record.sql +19 -0
  90. package/src/migrations/017-action-failure-record.sql +65 -0
  91. package/src/migrations/018-ontology-outbox-failure-record.sql +80 -0
  92. package/src/migrations/019-webhook-delivery-failure-record.sql +62 -0
  93. package/src/migrations/020-drop-run-usage-projections.sql +7 -0
  94. package/src/migrations/021-sync-pipeline-executions.sql +101 -0
  95. package/src/migrations/022-projection-executions.sql +244 -0
  96. package/src/migrations/023-webhook-executions.sql +51 -0
  97. package/src/migrations/024-ontology-commit-executions.sql +58 -0
  98. package/src/migrations/025-connector-connections.sql +257 -0
  99. package/src/migrations/026-ai-cost-accounting.sql +57 -0
  100. package/src/migrations/027-agent-context-checkpoints.sql +28 -0
  101. package/src/migrations/028-object-override-edit-times.sql +3 -0
  102. package/src/migrations.ts +129 -4
  103. package/src/ontology-storage/materialization-session.ts +317 -143
  104. package/src/ontology-storage/materialization-state.ts +323 -112
  105. package/src/ontology-storage/materialization-writer.ts +145 -67
  106. package/src/ontology-storage/materializations.ts +67 -141
  107. package/src/ontology-storage/outbox.ts +10 -3
  108. package/src/ontology-storage/shared.ts +10 -4
  109. package/src/ontology-storage/sources.ts +4 -1
  110. package/src/pipeline-run-storage.ts +128 -43
  111. package/src/projection-run-storage.ts +209 -97
  112. package/src/run-list-query.ts +9 -4
  113. package/src/sync-run-storage.ts +118 -32
  114. package/src/testing.ts +20 -0
  115. package/src/transactions.ts +10 -3
  116. package/src/webhook-run-storage.ts +118 -74
  117. package/src/workflow-run-storage.ts +117 -131
  118. package/dist/validation.d.ts +0 -5
  119. package/dist/validation.d.ts.map +0 -1
  120. package/dist/webhook-delivery-storage.d.ts +0 -27
  121. package/dist/webhook-delivery-storage.d.ts.map +0 -1
  122. package/src/validation.ts +0 -30
  123. package/src/webhook-delivery-storage.ts +0 -272
@@ -18,6 +18,7 @@ import {
18
18
  import {
19
19
  appendScopeSnapshot,
20
20
  finishScopeAccumulator,
21
+ linkSlotOverrideValue,
21
22
  startScopeAccumulator,
22
23
  } from "@sixb/core/internal/ontology-storage-provider"
23
24
  import type {
@@ -27,6 +28,7 @@ import type {
27
28
  SourceReplacementLinkState,
28
29
  SourceReplacementObjectState,
29
30
  StoredLinkOverride,
31
+ StoredLinkSlotOverride,
30
32
  StoredObjectOverride,
31
33
  StoredSourceAssertion,
32
34
  StoredSourceLinkAssertion,
@@ -38,8 +40,8 @@ import {
38
40
  linkRefFromColumns,
39
41
  objectRefFromColumns,
40
42
  parseJson,
41
- type SqliteOntologyOverrideRow,
42
43
  type SqliteOntologySourceAssertionRow,
44
+ type SqliteStoredOverrideRow,
43
45
  } from "./shared"
44
46
 
45
47
  export const SQLITE_MATERIALIZATION_WORK_TABLE = "ontology_materialization_work"
@@ -81,12 +83,13 @@ interface LinkScopeRow extends EffectiveLinkRow {
81
83
  readonly scope_sort_key: string
82
84
  }
83
85
 
84
- interface ObjectOverrideRow extends SqliteOntologyOverrideRow {
86
+ interface ObjectOverrideRow extends SqliteStoredOverrideRow {
85
87
  readonly object_type_id: string
86
88
  readonly primary_id: string
89
+ readonly edited_at: string
87
90
  }
88
91
 
89
- interface LinkOverrideRow extends SqliteOntologyOverrideRow {
92
+ interface LinkOverrideRow extends SqliteStoredOverrideRow {
90
93
  readonly source_type_id: string
91
94
  readonly source_primary_id: string
92
95
  readonly link_id: string
@@ -94,6 +97,17 @@ interface LinkOverrideRow extends SqliteOntologyOverrideRow {
94
97
  readonly target_primary_id: string
95
98
  }
96
99
 
100
+ interface LinkSlotOverrideRow {
101
+ readonly source_type_id: string
102
+ readonly source_primary_id: string
103
+ readonly link_id: string
104
+ readonly target_type_id: string
105
+ readonly target_primary_id: string
106
+ readonly value: string
107
+ readonly last_commit_id: string
108
+ readonly updated_at: string
109
+ }
110
+
97
111
  interface ReplacementSources {
98
112
  readonly owned: ReadonlySet<string>
99
113
  readonly byMaterialization: ReadonlyMap<string, ReadonlyMap<string, StoredSourceAssertion>>
@@ -122,6 +136,18 @@ interface ReplacementIdentityInput {
122
136
  readonly pageRows: number
123
137
  }
124
138
 
139
+ interface ReplacementWorkRow {
140
+ readonly entity_kind: "object" | "link"
141
+ readonly identity_key: string
142
+ readonly sort_key: string
143
+ readonly diff_required: number
144
+ }
145
+
146
+ interface ReplacementCursor {
147
+ readonly sortKey: string
148
+ readonly identityKey: string
149
+ }
150
+
125
151
  export class SqliteMaterializationStateReader {
126
152
  constructor(
127
153
  private readonly db: Database,
@@ -175,9 +201,8 @@ export class SqliteMaterializationStateReader {
175
201
  .query(
176
202
  `WITH requested AS (${requestedObjects})
177
203
  SELECT overrides.* FROM requested
178
- CROSS JOIN ontology_overrides AS overrides
204
+ CROSS JOIN ontology_object_overrides AS overrides
179
205
  ON overrides.project_id = ?
180
- AND overrides.entity_kind = 'object'
181
206
  AND overrides.object_type_id = requested.object_type_id
182
207
  AND overrides.primary_id = requested.primary_id`
183
208
  )
@@ -305,9 +330,9 @@ export class SqliteMaterializationStateReader {
305
330
  .query(
306
331
  `WITH requested AS (${sourceRequest})
307
332
  SELECT overrides.* FROM requested
308
- CROSS JOIN ontology_overrides AS overrides
333
+ CROSS JOIN ontology_link_overrides AS overrides
309
334
  ON overrides.project_id = ?
310
- AND overrides.entity_kind = 'link'
335
+ AND overrides.identity_kind = 'edge'
311
336
  AND overrides.source_type_id = requested.source_type_id
312
337
  AND overrides.source_primary_id = requested.source_primary_id
313
338
  AND overrides.link_id = requested.link_id
@@ -315,6 +340,18 @@ export class SqliteMaterializationStateReader {
315
340
  AND overrides.target_primary_id = requested.target_primary_id`
316
341
  )
317
342
  .all(requested, this.projectId) as LinkOverrideRow[]
343
+ const slotOverrideRows = this.db
344
+ .query(
345
+ `WITH requested AS (${sourceRequest})
346
+ SELECT overrides.* FROM requested
347
+ CROSS JOIN ontology_link_overrides AS overrides
348
+ ON overrides.project_id = ?
349
+ AND overrides.identity_kind = 'slot'
350
+ AND overrides.source_type_id = requested.source_type_id
351
+ AND overrides.source_primary_id = requested.source_primary_id
352
+ AND overrides.link_id = requested.link_id`
353
+ )
354
+ .all(requested, this.projectId) as LinkSlotOverrideRow[]
318
355
  const effective = new Map(
319
356
  effectiveRows.map((row) => [linkRefKey(linkRefFromColumns(row)), row] as const)
320
357
  )
@@ -324,6 +361,12 @@ export class SqliteMaterializationStateReader {
324
361
  (row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
325
362
  )
326
363
  )
364
+ const slotOverrides = new Map(
365
+ slotOverrideRows.map((row) => {
366
+ const stored = storedLinkSlotOverride(row)
367
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
368
+ })
369
+ )
327
370
  return refs.map((ref) => {
328
371
  const key = linkRefKey(ref)
329
372
  const effectiveRow = effective.get(key)
@@ -331,6 +374,7 @@ export class SqliteMaterializationStateReader {
331
374
  ref: structuredClone(ref),
332
375
  source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
333
376
  override: overrides.get(key) ?? null,
377
+ slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
334
378
  effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
335
379
  }
336
380
  })
@@ -395,9 +439,59 @@ export class SqliteMaterializationStateReader {
395
439
  const accumulators = new Map(
396
440
  scopes.map(({ source, linkId }) => {
397
441
  const key = linkScopeSortKey(source, linkId)
398
- return [key, startScopeAccumulator(source, linkId, key)] as const
442
+ return [key, startScopeAccumulator(source, linkId)] as const
399
443
  })
400
444
  )
445
+ const requestedJson = canonicalJson(requested)
446
+ const sourceRows = this.db
447
+ .query(
448
+ `WITH requested AS (
449
+ SELECT DISTINCT
450
+ json_extract(value, '$.sourceTypeId') AS source_type_id,
451
+ json_extract(value, '$.sourceId') AS source_primary_id,
452
+ json_extract(value, '$.linkId') AS link_id
453
+ FROM json_each(?)
454
+ )
455
+ SELECT rows.* FROM requested
456
+ CROSS JOIN ontology_source_rows AS rows
457
+ ON rows.project_id = ?
458
+ AND rows.entity_kind = 'link'
459
+ AND rows.source_type_id = requested.source_type_id
460
+ AND rows.source_primary_id = requested.source_primary_id
461
+ AND rows.link_id = requested.link_id
462
+ JOIN ontology_sources AS sources
463
+ ON sources.project_id = rows.project_id
464
+ AND sources.source_id = rows.source_id
465
+ AND sources.materialization_id = rows.materialization_id
466
+ WHERE sources.status = 'active'`
467
+ )
468
+ .all(requestedJson, this.projectId) as SqliteOntologySourceAssertionRow[]
469
+ const slotOverrideRows = this.db
470
+ .query(
471
+ `WITH requested AS (
472
+ SELECT DISTINCT
473
+ json_extract(value, '$.sourceTypeId') AS source_type_id,
474
+ json_extract(value, '$.sourceId') AS source_primary_id,
475
+ json_extract(value, '$.linkId') AS link_id
476
+ FROM json_each(?)
477
+ )
478
+ SELECT overrides.* FROM requested
479
+ CROSS JOIN ontology_link_overrides AS overrides
480
+ ON overrides.project_id = ?
481
+ AND overrides.identity_kind = 'slot'
482
+ AND overrides.source_type_id = requested.source_type_id
483
+ AND overrides.source_primary_id = requested.source_primary_id
484
+ AND overrides.link_id = requested.link_id`
485
+ )
486
+ .all(requestedJson, this.projectId) as LinkSlotOverrideRow[]
487
+ const sources = activeScopeSourceMap(sourceRows)
488
+ const overrides = new Map(
489
+ slotOverrideRows.map((row) => {
490
+ const stored = storedLinkSlotOverride(row)
491
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
492
+ })
493
+ )
494
+ const effective = new Map<string, EffectiveLinkSnapshot | null>()
401
495
  const rows = this.db
402
496
  .query(
403
497
  `
@@ -419,7 +513,7 @@ export class SqliteMaterializationStateReader {
419
513
  ORDER BY requested.scope_sort_key, ${linkSortExpression("links")}
420
514
  `
421
515
  )
422
- .iterate(canonicalJson(requested), this.projectId) as Iterable<LinkScopeRow>
516
+ .iterate(requestedJson, this.projectId) as Iterable<LinkScopeRow>
423
517
  for (const row of rows) {
424
518
  const accumulator = accumulators.get(row.scope_sort_key)
425
519
  if (!accumulator) {
@@ -428,11 +522,19 @@ export class SqliteMaterializationStateReader {
428
522
  `Unexpected link scope '${row.scope_sort_key}' returned by storage.`
429
523
  )
430
524
  }
431
- appendScopeSnapshot(accumulator, effectiveLinkSnapshot(row))
525
+ const snapshot = effectiveLinkSnapshot(row)
526
+ appendScopeSnapshot(accumulator, snapshot)
527
+ effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
432
528
  }
433
- return scopes.map(({ source, linkId }) =>
434
- finishScopeAccumulator(accumulators.get(linkScopeSortKey(source, linkId))!)
435
- )
529
+ return scopes.map(({ source, linkId }) => {
530
+ const key = linkScopeSortKey(source, linkId)
531
+ return {
532
+ ...finishScopeAccumulator(accumulators.get(key)!),
533
+ sourceAssertion: sources.get(key) ?? null,
534
+ override: overrides.get(key) ?? null,
535
+ effective: effective.get(key) ?? null,
536
+ }
537
+ })
436
538
  }
437
539
 
438
540
  linkScope(source: OntologyObjectRef, linkId: string): MaterializationLinkScopeState {
@@ -464,8 +566,8 @@ export class SqliteMaterializationStateReader {
464
566
  FROM links WHERE project_id = ?
465
567
  UNION
466
568
  SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
467
- FROM ontology_overrides
468
- WHERE project_id = ? AND entity_kind = 'link'
569
+ FROM ontology_link_overrides
570
+ WHERE project_id = ?
469
571
  UNION
470
572
  SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
471
573
  rows.target_type_id, rows.target_primary_id
@@ -509,29 +611,15 @@ export class SqliteMaterializationStateReader {
509
611
  }
510
612
 
511
613
  *replacementIdentities(input: ReplacementIdentityInput): Iterable<ReplacementIdentity[]> {
512
- let cursor: string | null = null
614
+ this.prepareReplacementIdentities(input)
615
+ let cursor: ReplacementCursor | null = null
513
616
  while (true) {
514
- if (input.kind === "object") {
515
- const rows = this.replacementObjectRows(input, cursor)
516
- if (rows.length === 0) break
517
- yield rows.map((row) => ({
518
- kind: "object",
519
- ref: { objectTypeId: row.object_type_id, primaryId: row.primary_id },
520
- sortKey: row.sort_key,
521
- diffRequired: true,
522
- }))
523
- cursor = rows[rows.length - 1]?.sort_key ?? null
524
- continue
525
- }
526
- const rows = this.replacementLinkRows(input, cursor)
617
+ const rows = this.replacementRows(input, cursor)
527
618
  if (rows.length === 0) break
528
- yield rows.map((row) => ({
529
- kind: "link",
530
- ref: linkRefFromColumns(row),
531
- sortKey: row.sort_key,
532
- diffRequired: row.diff_required === 1,
533
- }))
534
- cursor = rows[rows.length - 1]?.sort_key ?? null
619
+ yield rows.map(replacementIdentity)
620
+ if (rows.length < input.pageRows) break
621
+ const last = rows[rows.length - 1]!
622
+ cursor = { sortKey: last.sort_key, identityKey: last.identity_key }
535
623
  }
536
624
  }
537
625
 
@@ -581,6 +669,7 @@ export class SqliteMaterializationStateReader {
581
669
  ? sourceLink(candidate?.get(key))
582
670
  : state.source,
583
671
  override: state.override,
672
+ slotOverride: state.slotOverride,
584
673
  effective: state.effective,
585
674
  diffRequired: identities[index]!.diffRequired,
586
675
  }
@@ -623,16 +712,6 @@ export class SqliteMaterializationStateReader {
623
712
  return row ? row.last_commit_id : undefined
624
713
  }
625
714
 
626
- overrideLastCommit(kind: "object" | "link", entityKey: string): string | null {
627
- const row = this.db
628
- .query(
629
- `SELECT last_commit_id FROM ontology_overrides
630
- WHERE project_id = ? AND entity_kind = ? AND entity_key = ?`
631
- )
632
- .get(this.projectId, kind, entityKey) as { readonly last_commit_id: string } | null
633
- return row?.last_commit_id ?? null
634
- }
635
-
636
715
  private replacementSources(
637
716
  sourceId: string,
638
717
  materializationIds: readonly string[],
@@ -682,62 +761,48 @@ export class SqliteMaterializationStateReader {
682
761
  return { owned, byMaterialization }
683
762
  }
684
763
 
685
- private replacementObjectRows(
686
- input: ReplacementIdentityInput,
687
- cursor: string | null
688
- ): (EffectiveObjectRow & { readonly sort_key: string })[] {
689
- return this.db
764
+ private prepareReplacementIdentities(input: ReplacementIdentityInput): void {
765
+ if (input.kind === "object") {
766
+ this.prepareReplacementObjects(input)
767
+ return
768
+ }
769
+ this.prepareReplacementLinks(input)
770
+ }
771
+
772
+ private prepareReplacementObjects(input: ReplacementIdentityInput): void {
773
+ this.db
690
774
  .query(
691
775
  `
692
- SELECT DISTINCT object_type_id, primary_id, entity_sort_key AS sort_key
776
+ INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
777
+ session_id, entity_kind, identity_key, sort_key, diff_required
778
+ )
779
+ SELECT ?, 'object', json_array(object_type_id, primary_id),
780
+ MIN(entity_sort_key), 1
693
781
  FROM ontology_source_rows
694
782
  WHERE project_id = ? AND source_id = ? AND entity_kind = 'object'
695
783
  AND materialization_id IN (?, COALESCE(?, ''))
696
- AND (? IS NULL OR entity_sort_key > ?)
697
- ORDER BY sort_key
698
- LIMIT ?
784
+ GROUP BY object_type_id, primary_id
699
785
  `
700
786
  )
701
- .all(
787
+ .run(
788
+ input.sessionId,
702
789
  this.projectId,
703
790
  input.sourceId,
704
791
  input.candidateMaterializationId,
705
- input.previousMaterializationId,
706
- cursor,
707
- cursor,
708
- input.pageRows
709
- ) as (EffectiveObjectRow & { readonly sort_key: string })[]
792
+ input.previousMaterializationId
793
+ )
710
794
  }
711
795
 
712
- private replacementLinkRows(
713
- input: ReplacementIdentityInput,
714
- cursor: string | null
715
- ): (EffectiveLinkRow & { readonly sort_key: string; readonly diff_required: number })[] {
716
- return this.db
796
+ private prepareReplacementLinks(input: ReplacementIdentityInput): void {
797
+ this.db
717
798
  .query(
718
799
  `
719
800
  WITH incident_objects AS (
720
- SELECT
801
+ SELECT DISTINCT
721
802
  json_extract(payload, '$.ref.objectTypeId') AS object_type_id,
722
803
  json_extract(payload, '$.ref.primaryId') AS primary_id
723
804
  FROM ${SQLITE_MATERIALIZATION_WORK_TABLE}
724
805
  WHERE session_id = ? AND kind = 'incident-object'
725
- ), authority_links AS (
726
- SELECT source_type_id, source_id, link_id, target_type_id, target_id
727
- FROM links WHERE project_id = ?
728
- UNION
729
- SELECT source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
730
- FROM ontology_overrides
731
- WHERE project_id = ? AND entity_kind = 'link'
732
- UNION
733
- SELECT rows.source_type_id, rows.source_primary_id, rows.link_id,
734
- rows.target_type_id, rows.target_primary_id
735
- FROM ontology_source_rows AS rows
736
- JOIN ontology_sources AS sources
737
- ON sources.project_id = rows.project_id
738
- AND sources.source_id = rows.source_id
739
- AND sources.materialization_id = rows.materialization_id
740
- WHERE rows.project_id = ? AND rows.entity_kind = 'link' AND sources.status = 'active'
741
806
  ), replacement_links AS (
742
807
  SELECT source_type_id, source_primary_id AS source_id, link_id,
743
808
  target_type_id, target_primary_id AS target_id
@@ -745,14 +810,67 @@ export class SqliteMaterializationStateReader {
745
810
  WHERE project_id = ? AND source_id = ? AND entity_kind = 'link'
746
811
  AND materialization_id IN (?, COALESCE(?, ''))
747
812
  ), incident_links AS (
748
- SELECT links.* FROM authority_links AS links
749
- WHERE EXISTS (
750
- SELECT 1 FROM incident_objects
751
- WHERE (incident_objects.object_type_id = links.source_type_id
752
- AND incident_objects.primary_id = links.source_id)
753
- OR (incident_objects.object_type_id = links.target_type_id
754
- AND incident_objects.primary_id = links.target_id)
755
- )
813
+ SELECT links.source_type_id, links.source_id, links.link_id,
814
+ links.target_type_id, links.target_id
815
+ FROM links
816
+ JOIN incident_objects
817
+ ON incident_objects.object_type_id = links.source_type_id
818
+ AND incident_objects.primary_id = links.source_id
819
+ WHERE links.project_id = ?
820
+ UNION
821
+ SELECT links.source_type_id, links.source_id, links.link_id,
822
+ links.target_type_id, links.target_id
823
+ FROM links
824
+ JOIN incident_objects
825
+ ON incident_objects.object_type_id = links.target_type_id
826
+ AND incident_objects.primary_id = links.target_id
827
+ WHERE links.project_id = ?
828
+ UNION
829
+ SELECT overrides.source_type_id,
830
+ overrides.source_primary_id AS source_id,
831
+ overrides.link_id, overrides.target_type_id,
832
+ overrides.target_primary_id AS target_id
833
+ FROM ontology_link_overrides AS overrides
834
+ JOIN incident_objects
835
+ ON incident_objects.object_type_id = overrides.source_type_id
836
+ AND incident_objects.primary_id = overrides.source_primary_id
837
+ WHERE overrides.project_id = ?
838
+ UNION
839
+ SELECT overrides.source_type_id,
840
+ overrides.source_primary_id AS source_id,
841
+ overrides.link_id, overrides.target_type_id,
842
+ overrides.target_primary_id AS target_id
843
+ FROM ontology_link_overrides AS overrides
844
+ JOIN incident_objects
845
+ ON incident_objects.object_type_id = overrides.target_type_id
846
+ AND incident_objects.primary_id = overrides.target_primary_id
847
+ WHERE overrides.project_id = ?
848
+ UNION
849
+ SELECT rows.source_type_id, rows.source_primary_id AS source_id,
850
+ rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
851
+ FROM ontology_source_rows AS rows
852
+ JOIN ontology_sources AS sources
853
+ ON sources.project_id = rows.project_id
854
+ AND sources.source_id = rows.source_id
855
+ AND sources.materialization_id = rows.materialization_id
856
+ JOIN incident_objects
857
+ ON incident_objects.object_type_id = rows.source_type_id
858
+ AND incident_objects.primary_id = rows.source_primary_id
859
+ WHERE rows.project_id = ? AND rows.entity_kind = 'link'
860
+ AND sources.status = 'active'
861
+ UNION
862
+ SELECT rows.source_type_id, rows.source_primary_id AS source_id,
863
+ rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
864
+ FROM ontology_source_rows AS rows
865
+ JOIN ontology_sources AS sources
866
+ ON sources.project_id = rows.project_id
867
+ AND sources.source_id = rows.source_id
868
+ AND sources.materialization_id = rows.materialization_id
869
+ JOIN incident_objects
870
+ ON incident_objects.object_type_id = rows.target_type_id
871
+ AND incident_objects.primary_id = rows.target_primary_id
872
+ WHERE rows.project_id = ? AND rows.entity_kind = 'link'
873
+ AND sources.status = 'active'
756
874
  ), diff_links AS (
757
875
  SELECT * FROM replacement_links
758
876
  UNION SELECT * FROM incident_links
@@ -766,6 +884,16 @@ export class SqliteMaterializationStateReader {
766
884
  FROM links
767
885
  JOIN affected_scopes USING (source_type_id, source_id, link_id)
768
886
  WHERE links.project_id = ?
887
+ UNION ALL
888
+ SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
889
+ overrides.link_id, overrides.target_type_id,
890
+ overrides.target_primary_id AS target_id, 0 AS diff_required
891
+ FROM ontology_link_overrides AS overrides
892
+ JOIN affected_scopes
893
+ ON affected_scopes.source_type_id = overrides.source_type_id
894
+ AND affected_scopes.source_id = overrides.source_primary_id
895
+ AND affected_scopes.link_id = overrides.link_id
896
+ WHERE overrides.project_id = ? AND overrides.identity_kind = 'slot'
769
897
  ), selected AS (
770
898
  SELECT source_type_id, source_id, link_id, target_type_id, target_id,
771
899
  ${linkSortExpression("all_links")} AS sort_key,
@@ -773,26 +901,78 @@ export class SqliteMaterializationStateReader {
773
901
  FROM all_links
774
902
  GROUP BY source_type_id, source_id, link_id, target_type_id, target_id
775
903
  )
776
- SELECT * FROM selected
777
- WHERE (? IS NULL OR sort_key > ?)
778
- ORDER BY sort_key
779
- LIMIT ?
904
+ INSERT INTO ${SQLITE_REPLACEMENT_WORK_TABLE} (
905
+ session_id, entity_kind, identity_key, sort_key, diff_required
906
+ )
907
+ SELECT ?, 'link',
908
+ json_array(source_type_id, source_id, link_id, target_type_id, target_id),
909
+ sort_key, diff_required
910
+ FROM selected
780
911
  `
781
912
  )
782
- .all(
913
+ .run(
783
914
  input.sessionId,
784
915
  this.projectId,
785
- this.projectId,
786
- this.projectId,
787
- this.projectId,
788
916
  input.sourceId,
789
917
  input.candidateMaterializationId,
790
918
  input.previousMaterializationId,
791
919
  this.projectId,
792
- cursor,
793
- cursor,
920
+ this.projectId,
921
+ this.projectId,
922
+ this.projectId,
923
+ this.projectId,
924
+ this.projectId,
925
+ this.projectId,
926
+ this.projectId,
927
+ input.sessionId
928
+ )
929
+ }
930
+
931
+ private replacementRows(
932
+ input: ReplacementIdentityInput,
933
+ cursor: ReplacementCursor | null
934
+ ): ReplacementWorkRow[] {
935
+ return this.db
936
+ .query(
937
+ `
938
+ SELECT entity_kind, identity_key, sort_key, diff_required
939
+ FROM ${SQLITE_REPLACEMENT_WORK_TABLE}
940
+ WHERE session_id = ? AND entity_kind = ?
941
+ AND (? IS NULL OR (sort_key, identity_key) > (?, ?))
942
+ ORDER BY sort_key, identity_key
943
+ LIMIT ?
944
+ `
945
+ )
946
+ .all(
947
+ input.sessionId,
948
+ input.kind,
949
+ cursor?.sortKey ?? null,
950
+ cursor?.sortKey ?? null,
951
+ cursor?.identityKey ?? null,
794
952
  input.pageRows
795
- ) as (EffectiveLinkRow & { readonly sort_key: string; readonly diff_required: number })[]
953
+ ) as ReplacementWorkRow[]
954
+ }
955
+ }
956
+
957
+ function replacementIdentity(row: ReplacementWorkRow): ReplacementIdentity {
958
+ const parts = parseJson<string[]>(row.identity_key)
959
+ if (row.entity_kind === "object") {
960
+ return {
961
+ kind: "object",
962
+ ref: { objectTypeId: parts[0]!, primaryId: parts[1]! },
963
+ sortKey: row.sort_key,
964
+ diffRequired: true,
965
+ }
966
+ }
967
+ return {
968
+ kind: "link",
969
+ ref: {
970
+ source: { objectTypeId: parts[0]!, primaryId: parts[1]! },
971
+ linkId: parts[2]!,
972
+ target: { objectTypeId: parts[3]!, primaryId: parts[4]! },
973
+ },
974
+ sortKey: row.sort_key,
975
+ diffRequired: row.diff_required === 1,
796
976
  }
797
977
  }
798
978
 
@@ -814,6 +994,26 @@ function activeSourceMap(
814
994
  return result
815
995
  }
816
996
 
997
+ function activeScopeSourceMap(
998
+ rows: readonly SqliteOntologySourceAssertionRow[]
999
+ ): ReadonlyMap<string, StoredSourceLinkAssertion> {
1000
+ const result = new Map<string, StoredSourceLinkAssertion>()
1001
+ for (const row of rows) {
1002
+ const source = sourceLink(storedSource(row))
1003
+ if (!source) continue
1004
+ const ref = source.assertion.ref
1005
+ const key = linkScopeSortKey(ref.source, ref.linkId)
1006
+ if (result.has(key)) {
1007
+ throw new MaterializationConflictError(
1008
+ "source-materialization",
1009
+ `Multiple active sources assert cardinality-one link slot ${key}.`
1010
+ )
1011
+ }
1012
+ result.set(key, source)
1013
+ }
1014
+ return result
1015
+ }
1016
+
817
1017
  function sourceObject(
818
1018
  source: StoredSourceAssertion | undefined
819
1019
  ): StoredSourceObjectAssertion | null {
@@ -829,6 +1029,7 @@ function storedObjectOverride(row: ObjectOverrideRow): StoredObjectOverride {
829
1029
  return {
830
1030
  ref,
831
1031
  value: parseJson<StoredObjectOverride["value"]>(row.value),
1032
+ editedAt: parseJson<Readonly<Record<string, string>>>(row.edited_at),
832
1033
  lastCommitId: row.last_commit_id,
833
1034
  updatedAt: row.updated_at,
834
1035
  }
@@ -844,6 +1045,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
844
1045
  }
845
1046
  }
846
1047
 
1048
+ function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
1049
+ const ref = {
1050
+ source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
1051
+ linkId: row.link_id,
1052
+ }
1053
+ return {
1054
+ ref,
1055
+ value: linkSlotOverrideValue(parseJson<unknown>(row.value), linkScopeLabel(ref)),
1056
+ lastCommitId: row.last_commit_id,
1057
+ updatedAt: row.updated_at,
1058
+ }
1059
+ }
1060
+
847
1061
  function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
848
1062
  return {
849
1063
  source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
@@ -852,6 +1066,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
852
1066
  }
853
1067
  }
854
1068
 
1069
+ function linkScopeLabel(scope: {
1070
+ readonly source: OntologyObjectRef
1071
+ readonly linkId: string
1072
+ }): string {
1073
+ return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
1074
+ }
1075
+
855
1076
  function storedSource(row: SqliteOntologySourceAssertionRow): StoredSourceAssertion {
856
1077
  return {
857
1078
  source: { projectionId: row.source_id },
@@ -924,17 +1145,7 @@ function storedPoint(row: TelemetryRow): StoredTelemetryPoint {
924
1145
  }
925
1146
  }
926
1147
 
927
- export function objectSortExpression(alias?: string): string {
928
- const prefix = alias ? `${alias}.` : ""
929
- return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}primary_id) AS BLOB)))`
930
- }
931
-
932
1148
  export function linkSortExpression(alias?: string): string {
933
1149
  const prefix = alias ? `${alias}.` : ""
934
1150
  return `LOWER(HEX(CAST(json_array(${prefix}source_type_id, ${prefix}source_id, ${prefix}link_id, ${prefix}target_type_id, ${prefix}target_id) AS BLOB)))`
935
1151
  }
936
-
937
- export function pointSortExpression(alias?: string): string {
938
- const prefix = alias ? `${alias}.` : ""
939
- return `LOWER(HEX(CAST(json_array(${prefix}object_type_id, ${prefix}object_id, ${prefix}property_id, ${prefix}at) AS BLOB)))`
940
- }