@sixb/pg 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +7 -6
  2. package/dist/agents/agent-storage.d.ts +7 -4
  3. package/dist/agents/agent-storage.d.ts.map +1 -1
  4. package/dist/agents/checkpoints.d.ts +12 -0
  5. package/dist/agents/checkpoints.d.ts.map +1 -0
  6. package/dist/agents/index.d.ts +1 -0
  7. package/dist/agents/index.d.ts.map +1 -1
  8. package/dist/agents/messages.d.ts +2 -1
  9. package/dist/agents/messages.d.ts.map +1 -1
  10. package/dist/agents/rows.d.ts +22 -12
  11. package/dist/agents/rows.d.ts.map +1 -1
  12. package/dist/agents/runs.d.ts +3 -2
  13. package/dist/agents/runs.d.ts.map +1 -1
  14. package/dist/auth-storage/service-accounts.d.ts.map +1 -1
  15. package/dist/auth-storage/shared.d.ts +0 -4
  16. package/dist/auth-storage/shared.d.ts.map +1 -1
  17. package/dist/connector-connection-storage.d.ts +9 -0
  18. package/dist/connector-connection-storage.d.ts.map +1 -0
  19. package/dist/index.d.ts +9 -5
  20. package/dist/index.d.ts.map +1 -1
  21. package/dist/index.js +5010 -1837
  22. package/dist/index.js.map +31 -26
  23. package/dist/latest-run-query.d.ts.map +1 -1
  24. package/dist/migrations.d.ts.map +1 -1
  25. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  26. package/dist/ontology-storage/materialization-state.d.ts +0 -2
  27. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  28. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  29. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  30. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  31. package/dist/ontology-storage/shared.d.ts +3 -6
  32. package/dist/ontology-storage/shared.d.ts.map +1 -1
  33. package/dist/pg-action-run-storage.d.ts +3 -2
  34. package/dist/pg-action-run-storage.d.ts.map +1 -1
  35. package/dist/pg-ai-cost-storage.d.ts +12 -0
  36. package/dist/pg-ai-cost-storage.d.ts.map +1 -0
  37. package/dist/pg-ai-usage-storage.d.ts +16 -0
  38. package/dist/pg-ai-usage-storage.d.ts.map +1 -0
  39. package/dist/pg-execution-storage.d.ts +14 -0
  40. package/dist/pg-execution-storage.d.ts.map +1 -0
  41. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  42. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  43. package/dist/pg-projection-run-storage.d.ts +5 -2
  44. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  45. package/dist/pg-sync-run-storage.d.ts +5 -2
  46. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  47. package/dist/pg-webhook-run-storage.d.ts +5 -2
  48. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  49. package/dist/pg-workflow-run-storage.d.ts +5 -3
  50. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  51. package/dist/run-list-query.d.ts +1 -1
  52. package/dist/run-list-query.d.ts.map +1 -1
  53. package/dist/testing.d.ts +6 -0
  54. package/dist/testing.d.ts.map +1 -0
  55. package/package.json +3 -3
  56. package/src/agents/agent-storage.ts +9 -5
  57. package/src/agents/checkpoints.ts +175 -0
  58. package/src/agents/index.ts +1 -0
  59. package/src/agents/messages.ts +71 -13
  60. package/src/agents/rows.ts +57 -43
  61. package/src/agents/runs.ts +29 -15
  62. package/src/auth-storage/service-accounts.ts +14 -15
  63. package/src/auth-storage/shared.ts +0 -8
  64. package/src/connector-connection-storage.ts +760 -0
  65. package/src/index.ts +38 -15
  66. package/src/latest-run-query.ts +7 -1
  67. package/src/migrations/004-executions.sql +140 -0
  68. package/src/migrations/005-workflow-executions.sql +10 -0
  69. package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
  70. package/src/migrations/007-split-overrides.sql +153 -0
  71. package/src/migrations/008-action-executions.sql +7 -0
  72. package/src/migrations/009-agent-executions.sql +21 -0
  73. package/src/migrations/010-ai-usage-accounting-foundation.sql +76 -0
  74. package/src/migrations/011-sync-failure-record.sql +23 -0
  75. package/src/migrations/012-pipeline-failure-record.sql +47 -0
  76. package/src/migrations/013-workflow-failure-record.sql +45 -0
  77. package/src/migrations/014-agent-failure-record.sql +56 -0
  78. package/src/migrations/015-projection-failure-record.sql +20 -0
  79. package/src/migrations/016-webhook-run-failure-record.sql +19 -0
  80. package/src/migrations/017-action-failure-record.sql +65 -0
  81. package/src/migrations/018-ontology-outbox-failure-record.sql +29 -0
  82. package/src/migrations/019-webhook-delivery-failure-record.sql +27 -0
  83. package/src/migrations/020-drop-run-usage-projections.sql +8 -0
  84. package/src/migrations/021-sync-pipeline-executions.sql +50 -0
  85. package/src/migrations/022-projection-executions.sql +115 -0
  86. package/src/migrations/023-webhook-executions.sql +30 -0
  87. package/src/migrations/024-ontology-commit-executions.sql +12 -0
  88. package/src/migrations/025-connector-connections.sql +231 -0
  89. package/src/migrations/026-ai-cost-accounting.sql +53 -0
  90. package/src/migrations/027-agent-context-checkpoints.sql +34 -0
  91. package/src/migrations/028-object-override-edit-times.sql +3 -0
  92. package/src/migrations.ts +82 -0
  93. package/src/ontology-storage/materialization-session.ts +25 -28
  94. package/src/ontology-storage/materialization-state.ts +161 -43
  95. package/src/ontology-storage/materialization-writer.ts +149 -76
  96. package/src/ontology-storage/materializations.ts +3 -1
  97. package/src/ontology-storage/outbox.ts +9 -2
  98. package/src/ontology-storage/shared.ts +10 -20
  99. package/src/pg-action-run-storage.ts +43 -67
  100. package/src/pg-ai-cost-storage.ts +707 -0
  101. package/src/pg-ai-usage-storage.ts +290 -0
  102. package/src/pg-execution-storage.ts +147 -0
  103. package/src/pg-pipeline-run-storage.ts +107 -41
  104. package/src/pg-projection-run-storage.ts +159 -91
  105. package/src/pg-sync-run-storage.ts +100 -28
  106. package/src/pg-webhook-run-storage.ts +89 -57
  107. package/src/pg-workflow-run-storage.ts +97 -130
  108. package/src/run-list-query.ts +9 -4
  109. package/src/testing.ts +18 -0
  110. package/dist/pg-webhook-delivery-storage.d.ts +0 -17
  111. package/dist/pg-webhook-delivery-storage.d.ts.map +0 -1
  112. package/src/pg-webhook-delivery-storage.ts +0 -180
@@ -18,6 +18,7 @@ import {
18
18
  import {
19
19
  appendScopeSnapshot,
20
20
  finishScopeAccumulator,
21
+ linkSlotOverrideValue,
21
22
  startScopeAccumulator,
22
23
  } from "@sixb/core/internal/ontology-storage-provider"
23
24
  import type {
@@ -27,6 +28,7 @@ import type {
27
28
  SourceReplacementLinkState,
28
29
  SourceReplacementObjectState,
29
30
  StoredLinkOverride,
31
+ StoredLinkSlotOverride,
30
32
  StoredObjectOverride,
31
33
  StoredSourceAssertion,
32
34
  StoredSourceLinkAssertion,
@@ -36,12 +38,11 @@ import type {
36
38
  import type { SQLClient } from "../pg-client"
37
39
  import {
38
40
  databaseSafeInteger,
39
- jsonKeyParameter,
40
41
  jsonParameter,
41
42
  linkRefFromColumns,
42
43
  objectRefFromColumns,
43
- type PgOntologyOverrideRow,
44
44
  type PgOntologySourceAssertionRow,
45
+ type PgStoredOverrideRow,
45
46
  toIsoString,
46
47
  } from "./shared"
47
48
 
@@ -94,12 +95,13 @@ interface LinkScopeRow extends EffectiveLinkRow {
94
95
  readonly link_sort_key: string
95
96
  }
96
97
 
97
- interface ObjectOverrideRow extends PgOntologyOverrideRow {
98
+ interface ObjectOverrideRow extends PgStoredOverrideRow {
98
99
  readonly object_type_id: string
99
100
  readonly primary_id: string
101
+ readonly edited_at: unknown
100
102
  }
101
103
 
102
- interface LinkOverrideRow extends PgOntologyOverrideRow {
104
+ interface LinkOverrideRow extends PgStoredOverrideRow {
103
105
  readonly source_type_id: string
104
106
  readonly source_primary_id: string
105
107
  readonly link_id: string
@@ -107,6 +109,17 @@ interface LinkOverrideRow extends PgOntologyOverrideRow {
107
109
  readonly target_primary_id: string
108
110
  }
109
111
 
112
+ interface LinkSlotOverrideRow {
113
+ readonly source_type_id: string
114
+ readonly source_primary_id: string
115
+ readonly link_id: string
116
+ readonly target_type_id: string
117
+ readonly target_primary_id: string
118
+ readonly value: unknown
119
+ readonly last_commit_id: string
120
+ readonly updated_at: Date | string
121
+ }
122
+
110
123
  export type ReplacementIdentity =
111
124
  | {
112
125
  readonly kind: "object"
@@ -193,10 +206,9 @@ export class PgMaterializationStateReader {
193
206
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
194
207
  AS requested_values(object_type_id TEXT, primary_id TEXT)
195
208
  )
196
- SELECT overrides.* FROM ontology_overrides AS overrides
209
+ SELECT overrides.* FROM ontology_object_overrides AS overrides
197
210
  JOIN requested USING (object_type_id, primary_id)
198
211
  WHERE overrides.project_id = ${this.projectId}
199
- AND overrides.entity_kind = 'object'
200
212
  `,
201
213
  this.sql<TelemetryRow[]>`
202
214
  WITH requested AS (
@@ -266,7 +278,7 @@ export class PgMaterializationStateReader {
266
278
  target_primary_id: ref.target.primaryId,
267
279
  }))
268
280
  const requestedParameter = jsonParameter(this.sql, requested)
269
- const [effectiveRows, sourceRows, overrideRows] = await Promise.all([
281
+ const [effectiveRows, sourceRows, overrideRows, slotOverrideRows] = await Promise.all([
270
282
  this.sql<EffectiveLinkRow[]>`
271
283
  WITH requested AS (
272
284
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
@@ -305,12 +317,23 @@ export class PgMaterializationStateReader {
305
317
  target_type_id TEXT, target_primary_id TEXT
306
318
  )
307
319
  )
308
- SELECT overrides.* FROM ontology_overrides AS overrides
320
+ SELECT overrides.* FROM ontology_link_overrides AS overrides
309
321
  JOIN requested USING (
310
322
  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
311
323
  )
312
324
  WHERE overrides.project_id = ${this.projectId}
313
- AND overrides.entity_kind = 'link'
325
+ AND overrides.identity_kind = 'edge'
326
+ `,
327
+ this.sql<LinkSlotOverrideRow[]>`
328
+ WITH requested AS (
329
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
330
+ source_type_id TEXT, source_primary_id TEXT, link_id TEXT
331
+ )
332
+ )
333
+ SELECT overrides.* FROM ontology_link_overrides AS overrides
334
+ JOIN requested USING (source_type_id, source_primary_id, link_id)
335
+ WHERE overrides.project_id = ${this.projectId}
336
+ AND overrides.identity_kind = 'slot'
314
337
  `,
315
338
  ])
316
339
  const effective = new Map(
@@ -322,6 +345,12 @@ export class PgMaterializationStateReader {
322
345
  (row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
323
346
  )
324
347
  )
348
+ const slotOverrides = new Map(
349
+ slotOverrideRows.map((row) => {
350
+ const stored = storedLinkSlotOverride(row)
351
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
352
+ })
353
+ )
325
354
  return refs.map((ref) => {
326
355
  const key = linkRefKey(ref)
327
356
  const effectiveRow = effective.get(key)
@@ -329,6 +358,7 @@ export class PgMaterializationStateReader {
329
358
  ref: structuredClone(ref),
330
359
  source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
331
360
  override: overrides.get(key) ?? null,
361
+ slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
332
362
  effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
333
363
  }
334
364
  })
@@ -390,15 +420,63 @@ export class PgMaterializationStateReader {
390
420
  const accumulators = new Map(
391
421
  scopes.map(({ source, linkId }) => {
392
422
  const key = linkScopeSortKey(source, linkId)
393
- return [key, startScopeAccumulator(source, linkId, key)] as const
423
+ return [key, startScopeAccumulator(source, linkId)] as const
394
424
  })
395
425
  )
426
+ const requestedParameter = jsonParameter(this.sql, requested)
427
+ const [sourceRows, slotOverrideRows] = await Promise.all([
428
+ this.sql<PgOntologySourceAssertionRow[]>`
429
+ WITH requested AS (
430
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
431
+ AS requested_values(
432
+ scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
433
+ )
434
+ )
435
+ SELECT rows.*
436
+ FROM ontology_source_rows AS rows
437
+ JOIN requested
438
+ ON requested.source_type_id = rows.source_type_id
439
+ AND requested.source_id = rows.source_primary_id
440
+ AND requested.link_id = rows.link_id
441
+ JOIN ontology_sources AS sources
442
+ ON sources.project_id = rows.project_id
443
+ AND sources.source_id = rows.source_id
444
+ AND sources.materialization_id = rows.materialization_id
445
+ WHERE rows.project_id = ${this.projectId}
446
+ AND rows.entity_kind = 'link'
447
+ AND sources.status = 'active'
448
+ `,
449
+ this.sql<LinkSlotOverrideRow[]>`
450
+ WITH requested AS (
451
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
452
+ AS requested_values(
453
+ scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
454
+ )
455
+ )
456
+ SELECT overrides.*
457
+ FROM ontology_link_overrides AS overrides
458
+ JOIN requested
459
+ ON requested.source_type_id = overrides.source_type_id
460
+ AND requested.source_id = overrides.source_primary_id
461
+ AND requested.link_id = overrides.link_id
462
+ WHERE overrides.project_id = ${this.projectId}
463
+ AND overrides.identity_kind = 'slot'
464
+ `,
465
+ ])
466
+ const sources = activeScopeSourceMap(sourceRows)
467
+ const overrides = new Map(
468
+ slotOverrideRows.map((row) => {
469
+ const stored = storedLinkSlotOverride(row)
470
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
471
+ })
472
+ )
473
+ const effective = new Map<string, EffectiveLinkSnapshot | null>()
396
474
  let scopeCursor: string | null = null
397
475
  let linkCursor: string | null = null
398
476
  while (true) {
399
477
  const rows: LinkScopeRow[] = await this.sql`
400
478
  WITH requested AS (
401
- SELECT DISTINCT * FROM jsonb_to_recordset(${jsonParameter(this.sql, requested)})
479
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
402
480
  AS requested_values(
403
481
  scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
404
482
  )
@@ -423,16 +501,24 @@ export class PgMaterializationStateReader {
423
501
  `Unexpected link scope '${row.scope_sort_key}' returned by storage.`
424
502
  )
425
503
  }
426
- appendScopeSnapshot(accumulator, effectiveLinkSnapshot(row))
504
+ const snapshot = effectiveLinkSnapshot(row)
505
+ appendScopeSnapshot(accumulator, snapshot)
506
+ effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
427
507
  }
428
508
  if (rows.length < 500) break
429
509
  const last: LinkScopeRow = rows[rows.length - 1]!
430
510
  scopeCursor = last.scope_sort_key
431
511
  linkCursor = last.link_sort_key
432
512
  }
433
- return scopes.map(({ source, linkId }) =>
434
- finishScopeAccumulator(accumulators.get(linkScopeSortKey(source, linkId))!)
435
- )
513
+ return scopes.map(({ source, linkId }) => {
514
+ const key = linkScopeSortKey(source, linkId)
515
+ return {
516
+ ...finishScopeAccumulator(accumulators.get(key)!),
517
+ sourceAssertion: sources.get(key) ?? null,
518
+ override: overrides.get(key) ?? null,
519
+ effective: effective.get(key) ?? null,
520
+ }
521
+ })
436
522
  }
437
523
 
438
524
  async linkScope(
@@ -485,20 +571,20 @@ export class PgMaterializationStateReader {
485
571
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
486
572
  overrides.link_id, overrides.target_type_id,
487
573
  overrides.target_primary_id AS target_id
488
- FROM ontology_overrides AS overrides
574
+ FROM ontology_link_overrides AS overrides
489
575
  JOIN requested
490
576
  ON requested.object_type_id = overrides.source_type_id
491
577
  AND requested.primary_id = overrides.source_primary_id
492
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
578
+ WHERE overrides.project_id = ${this.projectId}
493
579
  UNION
494
580
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
495
581
  overrides.link_id, overrides.target_type_id,
496
582
  overrides.target_primary_id AS target_id
497
- FROM ontology_overrides AS overrides
583
+ FROM ontology_link_overrides AS overrides
498
584
  JOIN requested
499
585
  ON requested.object_type_id = overrides.target_type_id
500
586
  AND requested.primary_id = overrides.target_primary_id
501
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
587
+ WHERE overrides.project_id = ${this.projectId}
502
588
  UNION
503
589
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
504
590
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
@@ -609,6 +695,7 @@ export class PgMaterializationStateReader {
609
695
  ? sourceLink(candidate?.get(key))
610
696
  : state.source,
611
697
  override: state.override,
698
+ slotOverride: state.slotOverride,
612
699
  effective: state.effective,
613
700
  diffRequired: identities[index]!.diffRequired,
614
701
  }
@@ -701,16 +788,6 @@ export class PgMaterializationStateReader {
701
788
  )
702
789
  }
703
790
 
704
- async overrideLastCommit(kind: "object" | "link", entityKey: string): Promise<string | null> {
705
- const [row] = await this.sql<{ readonly last_commit_id: string }[]>`
706
- SELECT last_commit_id FROM ontology_overrides
707
- WHERE project_id = ${this.projectId}
708
- AND entity_kind = ${kind}
709
- AND entity_key = ${jsonKeyParameter(this.sql, entityKey)}
710
- `
711
- return row?.last_commit_id ?? null
712
- }
713
-
714
791
  private async replacementSources(
715
792
  sourceId: string,
716
793
  materializationIds: readonly string[],
@@ -824,20 +901,20 @@ export class PgMaterializationStateReader {
824
901
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
825
902
  overrides.link_id, overrides.target_type_id,
826
903
  overrides.target_primary_id AS target_id
827
- FROM ontology_overrides AS overrides
904
+ FROM ontology_link_overrides AS overrides
828
905
  JOIN incident_objects
829
906
  ON incident_objects.object_type_id = overrides.source_type_id
830
907
  AND incident_objects.primary_id = overrides.source_primary_id
831
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
908
+ WHERE overrides.project_id = ${this.projectId}
832
909
  UNION
833
910
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
834
911
  overrides.link_id, overrides.target_type_id,
835
912
  overrides.target_primary_id AS target_id
836
- FROM ontology_overrides AS overrides
913
+ FROM ontology_link_overrides AS overrides
837
914
  JOIN incident_objects
838
915
  ON incident_objects.object_type_id = overrides.target_type_id
839
916
  AND incident_objects.primary_id = overrides.target_primary_id
840
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
917
+ WHERE overrides.project_id = ${this.projectId}
841
918
  UNION
842
919
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
843
920
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
@@ -879,6 +956,16 @@ export class PgMaterializationStateReader {
879
956
  FROM links
880
957
  JOIN affected_scopes USING (source_type_id, source_id, link_id)
881
958
  WHERE links.project_id = ${this.projectId}
959
+ UNION ALL
960
+ SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
961
+ overrides.link_id, overrides.target_type_id,
962
+ overrides.target_primary_id AS target_id, FALSE AS diff_required
963
+ FROM ontology_link_overrides AS overrides
964
+ JOIN affected_scopes
965
+ ON affected_scopes.source_type_id = overrides.source_type_id
966
+ AND affected_scopes.source_id = overrides.source_primary_id
967
+ AND affected_scopes.link_id = overrides.link_id
968
+ WHERE overrides.project_id = ${this.projectId} AND overrides.identity_kind = 'slot'
882
969
  ), selected AS (
883
970
  SELECT source_type_id, source_id, link_id, target_type_id, target_id,
884
971
  ${this.sql.unsafe(linkKeyExpression("all_links"))} AS identity_key,
@@ -939,6 +1026,26 @@ function activeSourceMap(
939
1026
  return result
940
1027
  }
941
1028
 
1029
+ function activeScopeSourceMap(
1030
+ rows: readonly PgOntologySourceAssertionRow[]
1031
+ ): ReadonlyMap<string, StoredSourceLinkAssertion> {
1032
+ const result = new Map<string, StoredSourceLinkAssertion>()
1033
+ for (const row of rows) {
1034
+ const source = sourceLink(storedSource(row))
1035
+ if (!source) continue
1036
+ const ref = source.assertion.ref
1037
+ const key = linkScopeSortKey(ref.source, ref.linkId)
1038
+ if (result.has(key)) {
1039
+ throw new MaterializationConflictError(
1040
+ "source-materialization",
1041
+ `Multiple active sources assert cardinality-one link slot ${key}.`
1042
+ )
1043
+ }
1044
+ result.set(key, source)
1045
+ }
1046
+ return result
1047
+ }
1048
+
942
1049
  function sourceObject(
943
1050
  source: StoredSourceAssertion | undefined
944
1051
  ): StoredSourceObjectAssertion | null {
@@ -954,6 +1061,7 @@ function storedObjectOverride(row: ObjectOverrideRow): StoredObjectOverride {
954
1061
  return {
955
1062
  ref,
956
1063
  value: structuredClone(row.value) as StoredObjectOverride["value"],
1064
+ editedAt: structuredClone(row.edited_at) as Readonly<Record<string, string>>,
957
1065
  lastCommitId: row.last_commit_id,
958
1066
  updatedAt: toIsoString(row.updated_at),
959
1067
  }
@@ -969,6 +1077,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
969
1077
  }
970
1078
  }
971
1079
 
1080
+ function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
1081
+ const ref = {
1082
+ source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
1083
+ linkId: row.link_id,
1084
+ }
1085
+ return {
1086
+ ref,
1087
+ value: linkSlotOverrideValue(row.value, linkScopeLabel(ref)),
1088
+ lastCommitId: row.last_commit_id,
1089
+ updatedAt: toIsoString(row.updated_at),
1090
+ }
1091
+ }
1092
+
972
1093
  function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
973
1094
  return {
974
1095
  source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
@@ -977,6 +1098,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
977
1098
  }
978
1099
  }
979
1100
 
1101
+ function linkScopeLabel(scope: {
1102
+ readonly source: OntologyObjectRef
1103
+ readonly linkId: string
1104
+ }): string {
1105
+ return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
1106
+ }
1107
+
980
1108
  function storedSource(row: PgOntologySourceAssertionRow): StoredSourceAssertion {
981
1109
  return {
982
1110
  source: { projectionId: row.source_id },
@@ -1090,16 +1218,6 @@ function linkKeyExpression(alias?: string): string {
1090
1218
  ])
1091
1219
  }
1092
1220
 
1093
- export function pointSortExpression(alias?: string): string {
1094
- const prefix = alias ? `${alias}.` : ""
1095
- return utf8SortExpression([
1096
- `to_jsonb(${prefix}object_type_id)::text`,
1097
- `to_jsonb(${prefix}object_id)::text`,
1098
- `to_jsonb(${prefix}property_id)::text`,
1099
- `to_jsonb(to_char(${prefix}at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))::text`,
1100
- ])
1101
- }
1102
-
1103
1221
  function utf8SortExpression(parts: readonly string[]): string {
1104
1222
  return `encode(convert_to(${jsonTupleExpression(parts)}, 'UTF8'), 'hex')`
1105
1223
  }
@@ -4,11 +4,7 @@ import {
4
4
  objectRefKey,
5
5
  telemetryPointKey,
6
6
  } from "@sixb/core/internal/materialization"
7
- import {
8
- effectiveConflict,
9
- type OverrideEntity,
10
- overrideEntityColumns as overrideColumns,
11
- } from "@sixb/core/internal/ontology-storage-provider"
7
+ import { effectiveConflict } from "@sixb/core/internal/ontology-storage-provider"
12
8
  import type { MaterializationPlanChunk } from "@sixb/core/storage"
13
9
  import type { SQLClient } from "../pg-client"
14
10
  import { assertTimestamp, jsonParameter } from "./shared"
@@ -25,82 +21,173 @@ export class PgMaterializationWriter {
25
21
  }
26
22
 
27
23
  private async applyOverrides(projectId: string, chunk: MaterializationPlanChunk): Promise<void> {
28
- const upserts = [
29
- ...chunk.overrides.objectUpserts.map((item) =>
30
- overrideWrite({ kind: "object", ref: item.ref }, item)
31
- ),
32
- ...chunk.overrides.linkUpserts.map((item) =>
33
- overrideWrite({ kind: "link", ref: item.ref }, item)
34
- ),
35
- ]
36
- const inserts = upserts.filter((item) => item.expectedLastCommitId === null)
37
- if (inserts.length > 0) {
38
- const rows = await this.sql<{ readonly entity_kind: "object" | "link" }[]>`
24
+ const objectUpserts = chunk.overrides.objects.upserts.map((item) => ({
25
+ objectTypeId: item.ref.objectTypeId,
26
+ primaryId: item.ref.primaryId,
27
+ value: item.value,
28
+ editedAt: item.editedAt,
29
+ lastCommitId: item.lastCommitId,
30
+ updatedAt: item.updatedAt,
31
+ expectedLastCommitId: item.expectedLastCommitId,
32
+ }))
33
+ const objectInserts = objectUpserts.filter((item) => item.expectedLastCommitId === null)
34
+ if (objectInserts.length > 0) {
35
+ const rows = await this.sql<{ readonly object_type_id: string }[]>`
39
36
  WITH staged AS (
40
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
37
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectInserts)}::jsonb)
41
38
  )
42
- INSERT INTO ontology_overrides (
43
- project_id, entity_kind, entity_key, entity_sort_key,
44
- object_type_id, primary_id, source_type_id, source_primary_id,
45
- link_id, target_type_id, target_primary_id,
46
- value, last_commit_id, updated_at
39
+ INSERT INTO ontology_object_overrides (
40
+ project_id, object_type_id, primary_id, value, edited_at, last_commit_id, updated_at
47
41
  )
48
- SELECT ${projectId}, value->>'kind', value->'entityKey', value->>'sortKey',
49
- value->>'objectTypeId', value->>'primaryId', value->>'sourceTypeId',
50
- value->>'sourcePrimaryId', value->>'linkId', value->>'targetTypeId',
51
- value->>'targetPrimaryId', value->'value', value->>'lastCommitId',
42
+ SELECT ${projectId}, value->>'objectTypeId', value->>'primaryId', value->'value',
43
+ value->'editedAt', value->>'lastCommitId',
52
44
  (value->>'updatedAt')::timestamptz
53
45
  FROM staged
54
46
  ON CONFLICT DO NOTHING
55
- RETURNING entity_kind
47
+ RETURNING object_type_id
56
48
  `
57
- if (rows.length !== inserts.length) throw overrideConflict(inserts[0]!.kind)
49
+ if (rows.length !== objectInserts.length) throw objectOverrideConflict()
58
50
  }
59
51
 
60
- const updates = upserts.filter((item) => item.expectedLastCommitId !== null)
61
- if (updates.length > 0) {
62
- const rows = await this.sql<{ readonly entity_kind: "object" | "link" }[]>`
52
+ const objectUpdates = objectUpserts.filter((item) => item.expectedLastCommitId !== null)
53
+ if (objectUpdates.length > 0) {
54
+ const rows = await this.sql<{ readonly object_type_id: string }[]>`
63
55
  WITH staged AS (
64
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, updates)}::jsonb)
56
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectUpdates)}::jsonb)
57
+ )
58
+ UPDATE ontology_object_overrides AS overrides
59
+ SET value = staged.value->'value', edited_at = staged.value->'editedAt',
60
+ last_commit_id = staged.value->>'lastCommitId',
61
+ updated_at = (staged.value->>'updatedAt')::timestamptz
62
+ FROM staged
63
+ WHERE overrides.project_id = ${projectId}
64
+ AND overrides.object_type_id = staged.value->>'objectTypeId'
65
+ AND overrides.primary_id = staged.value->>'primaryId'
66
+ AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
67
+ RETURNING overrides.object_type_id
68
+ `
69
+ if (rows.length !== objectUpdates.length) throw objectOverrideConflict()
70
+ }
71
+
72
+ const objectDeletes = chunk.overrides.objects.deletes.map((item) => ({
73
+ objectTypeId: item.ref.objectTypeId,
74
+ primaryId: item.ref.primaryId,
75
+ expectedLastCommitId: item.expectedLastCommitId,
76
+ }))
77
+ if (objectDeletes.length > 0) {
78
+ const rows = await this.sql<{ readonly object_type_id: string }[]>`
79
+ WITH staged AS (
80
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectDeletes)}::jsonb)
81
+ )
82
+ DELETE FROM ontology_object_overrides AS overrides USING staged
83
+ WHERE overrides.project_id = ${projectId}
84
+ AND overrides.object_type_id = staged.value->>'objectTypeId'
85
+ AND overrides.primary_id = staged.value->>'primaryId'
86
+ AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
87
+ RETURNING overrides.object_type_id
88
+ `
89
+ if (rows.length !== objectDeletes.length) throw objectOverrideConflict()
90
+ }
91
+
92
+ const linkUpserts = [
93
+ ...chunk.overrides.links.edges.upserts.map((item) => ({
94
+ identityKind: "edge" as const,
95
+ identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
96
+ sourceTypeId: item.ref.source.objectTypeId,
97
+ sourcePrimaryId: item.ref.source.primaryId,
98
+ linkId: item.ref.linkId,
99
+ targetTypeId: item.ref.target.objectTypeId,
100
+ targetPrimaryId: item.ref.target.primaryId,
101
+ value: item.value,
102
+ lastCommitId: item.lastCommitId,
103
+ updatedAt: item.updatedAt,
104
+ expectedLastCommitId: item.expectedLastCommitId,
105
+ })),
106
+ ...chunk.overrides.links.slots.upserts.map((item) => ({
107
+ identityKind: "slot" as const,
108
+ identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
109
+ sourceTypeId: item.ref.source.objectTypeId,
110
+ sourcePrimaryId: item.ref.source.primaryId,
111
+ linkId: item.ref.linkId,
112
+ targetTypeId: item.value.target.objectTypeId,
113
+ targetPrimaryId: item.value.target.primaryId,
114
+ value: item.value,
115
+ lastCommitId: item.lastCommitId,
116
+ updatedAt: item.updatedAt,
117
+ expectedLastCommitId: item.expectedLastCommitId,
118
+ })),
119
+ ]
120
+ const linkInserts = linkUpserts.filter((item) => item.expectedLastCommitId === null)
121
+ if (linkInserts.length > 0) {
122
+ const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
123
+ WITH staged AS (
124
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkInserts)}::jsonb)
65
125
  )
66
- UPDATE ontology_overrides AS overrides
67
- SET value = staged.value->'value', last_commit_id = staged.value->>'lastCommitId',
126
+ INSERT INTO ontology_link_overrides (
127
+ project_id, identity_kind, identity_key, source_type_id, source_primary_id, link_id,
128
+ target_type_id, target_primary_id, value, last_commit_id, updated_at
129
+ )
130
+ SELECT ${projectId}, value->>'identityKind', value->'identityKey',
131
+ value->>'sourceTypeId', value->>'sourcePrimaryId', value->>'linkId',
132
+ value->>'targetTypeId', value->>'targetPrimaryId', value->'value',
133
+ value->>'lastCommitId', (value->>'updatedAt')::timestamptz
134
+ FROM staged
135
+ ON CONFLICT DO NOTHING
136
+ RETURNING identity_kind
137
+ `
138
+ if (rows.length !== linkInserts.length)
139
+ throw linkOverrideConflict(linkInserts[0]!.identityKind)
140
+ }
141
+
142
+ const linkUpdates = linkUpserts.filter((item) => item.expectedLastCommitId !== null)
143
+ if (linkUpdates.length > 0) {
144
+ const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
145
+ WITH staged AS (
146
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkUpdates)}::jsonb)
147
+ )
148
+ UPDATE ontology_link_overrides AS overrides
149
+ SET target_type_id = staged.value->>'targetTypeId',
150
+ target_primary_id = staged.value->>'targetPrimaryId',
151
+ value = staged.value->'value',
152
+ last_commit_id = staged.value->>'lastCommitId',
68
153
  updated_at = (staged.value->>'updatedAt')::timestamptz
69
154
  FROM staged
70
155
  WHERE overrides.project_id = ${projectId}
71
- AND overrides.entity_kind = staged.value->>'kind'
72
- AND overrides.entity_key = staged.value->'entityKey'
156
+ AND overrides.identity_kind = staged.value->>'identityKind'
157
+ AND overrides.identity_key = staged.value->'identityKey'
73
158
  AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
74
- RETURNING overrides.entity_kind
159
+ RETURNING overrides.identity_kind
75
160
  `
76
- if (rows.length !== updates.length) throw overrideConflict(updates[0]!.kind)
161
+ if (rows.length !== linkUpdates.length)
162
+ throw linkOverrideConflict(linkUpdates[0]!.identityKind)
77
163
  }
78
164
 
79
- const deletes = [
80
- ...chunk.overrides.objectDeletes.map((item) => ({
81
- kind: "object" as const,
82
- entityKey: JSON.parse(objectRefKey(item.ref)) as unknown,
165
+ const linkDeletes = [
166
+ ...chunk.overrides.links.edges.deletes.map((item) => ({
167
+ identityKind: "edge" as const,
168
+ identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
83
169
  expectedLastCommitId: item.expectedLastCommitId,
84
170
  })),
85
- ...chunk.overrides.linkDeletes.map((item) => ({
86
- kind: "link" as const,
87
- entityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
171
+ ...chunk.overrides.links.slots.deletes.map((item) => ({
172
+ identityKind: "slot" as const,
173
+ identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
88
174
  expectedLastCommitId: item.expectedLastCommitId,
89
175
  })),
90
176
  ]
91
- if (deletes.length > 0) {
92
- const rows = await this.sql<{ readonly entity_kind: "object" | "link" }[]>`
177
+ if (linkDeletes.length > 0) {
178
+ const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
93
179
  WITH staged AS (
94
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, deletes)}::jsonb)
180
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkDeletes)}::jsonb)
95
181
  )
96
- DELETE FROM ontology_overrides AS overrides USING staged
182
+ DELETE FROM ontology_link_overrides AS overrides USING staged
97
183
  WHERE overrides.project_id = ${projectId}
98
- AND overrides.entity_kind = staged.value->>'kind'
99
- AND overrides.entity_key = staged.value->'entityKey'
184
+ AND overrides.identity_kind = staged.value->>'identityKind'
185
+ AND overrides.identity_key = staged.value->'identityKey'
100
186
  AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
101
- RETURNING overrides.entity_kind
187
+ RETURNING overrides.identity_kind
102
188
  `
103
- if (rows.length !== deletes.length) throw overrideConflict(deletes[0]!.kind)
189
+ if (rows.length !== linkDeletes.length)
190
+ throw linkOverrideConflict(linkDeletes[0]!.identityKind)
104
191
  }
105
192
  }
106
193
 
@@ -408,7 +495,7 @@ export class PgMaterializationWriter {
408
495
  INSERT INTO ontology_outbox (
409
496
  project_id, id, commit_id, commit_ordinal, envelope,
410
497
  available_at, attempts, lease_id, lease_expires_at,
411
- published_at, last_error, created_at
498
+ published_at, last_failure, created_at
412
499
  )
413
500
  SELECT ${projectId}, value->>'id', ${commitId},
414
501
  (value->>'commitOrdinal')::bigint, value->'envelope',
@@ -426,30 +513,16 @@ export class PgMaterializationWriter {
426
513
  }
427
514
  }
428
515
 
429
- function overrideWrite(
430
- entity: OverrideEntity,
431
- item: {
432
- readonly value: unknown
433
- readonly lastCommitId: string
434
- readonly updatedAt: string
435
- readonly expectedLastCommitId: string | null
436
- }
437
- ) {
438
- const key = entity.kind === "object" ? objectRefKey(entity.ref) : linkRefKey(entity.ref)
439
- const columns = overrideColumns(entity)
440
- return {
441
- kind: entity.kind,
442
- entityKey: JSON.parse(key) as unknown,
443
- ...columns,
444
- value: item.value,
445
- lastCommitId: item.lastCommitId,
446
- updatedAt: item.updatedAt,
447
- expectedLastCommitId: item.expectedLastCommitId,
448
- }
516
+ function objectOverrideConflict(): MaterializationConflictError {
517
+ return effectiveConflict("Expected object override changed.")
449
518
  }
450
519
 
451
- function overrideConflict(kind: "object" | "link"): MaterializationConflictError {
452
- return effectiveConflict(`Expected ${kind} override changed.`)
520
+ function linkOverrideConflict(identityKind: "edge" | "slot"): MaterializationConflictError {
521
+ return effectiveConflict(
522
+ identityKind === "edge"
523
+ ? "Expected link edge override changed."
524
+ : "Expected link slot override changed."
525
+ )
453
526
  }
454
527
 
455
528
  function objectIdentityKey(item: {