@sixb/pg 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/README.md +7 -6
  2. package/dist/agents/agent-storage.d.ts +2 -1
  3. package/dist/agents/agent-storage.d.ts.map +1 -1
  4. package/dist/agents/rows.d.ts +4 -11
  5. package/dist/agents/rows.d.ts.map +1 -1
  6. package/dist/agents/runs.d.ts +3 -2
  7. package/dist/agents/runs.d.ts.map +1 -1
  8. package/dist/auth-storage/service-accounts.d.ts.map +1 -1
  9. package/dist/auth-storage/shared.d.ts +0 -4
  10. package/dist/auth-storage/shared.d.ts.map +1 -1
  11. package/dist/connector-connection-storage.d.ts +9 -0
  12. package/dist/connector-connection-storage.d.ts.map +1 -0
  13. package/dist/index.d.ts +8 -5
  14. package/dist/index.d.ts.map +1 -1
  15. package/dist/index.js +3086 -730
  16. package/dist/index.js.map +28 -25
  17. package/dist/latest-run-query.d.ts.map +1 -1
  18. package/dist/migrations.d.ts.map +1 -1
  19. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  20. package/dist/ontology-storage/materialization-state.d.ts +0 -2
  21. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  22. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  23. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  24. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  25. package/dist/ontology-storage/shared.d.ts +3 -6
  26. package/dist/ontology-storage/shared.d.ts.map +1 -1
  27. package/dist/pg-action-run-storage.d.ts +3 -2
  28. package/dist/pg-action-run-storage.d.ts.map +1 -1
  29. package/dist/pg-ai-usage-storage.d.ts +16 -0
  30. package/dist/pg-ai-usage-storage.d.ts.map +1 -0
  31. package/dist/pg-execution-storage.d.ts +14 -0
  32. package/dist/pg-execution-storage.d.ts.map +1 -0
  33. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  34. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  35. package/dist/pg-projection-run-storage.d.ts +5 -2
  36. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  37. package/dist/pg-sync-run-storage.d.ts +5 -2
  38. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  39. package/dist/pg-webhook-run-storage.d.ts +5 -2
  40. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  41. package/dist/pg-workflow-run-storage.d.ts +5 -3
  42. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  43. package/dist/run-list-query.d.ts +1 -1
  44. package/dist/run-list-query.d.ts.map +1 -1
  45. package/dist/testing.d.ts +6 -0
  46. package/dist/testing.d.ts.map +1 -0
  47. package/package.json +3 -3
  48. package/src/agents/agent-storage.ts +3 -2
  49. package/src/agents/rows.ts +12 -43
  50. package/src/agents/runs.ts +29 -15
  51. package/src/auth-storage/service-accounts.ts +14 -15
  52. package/src/auth-storage/shared.ts +0 -8
  53. package/src/connector-connection-storage.ts +760 -0
  54. package/src/index.ts +32 -15
  55. package/src/latest-run-query.ts +7 -1
  56. package/src/migrations/004-executions.sql +140 -0
  57. package/src/migrations/005-workflow-executions.sql +10 -0
  58. package/src/migrations/006-narrow-ontology-source-root-index.sql +4 -0
  59. package/src/migrations/007-split-overrides.sql +153 -0
  60. package/src/migrations/008-action-executions.sql +7 -0
  61. package/src/migrations/009-agent-executions.sql +21 -0
  62. package/src/migrations/010-ai-usage-accounting-foundation.sql +76 -0
  63. package/src/migrations/011-sync-failure-record.sql +23 -0
  64. package/src/migrations/012-pipeline-failure-record.sql +47 -0
  65. package/src/migrations/013-workflow-failure-record.sql +45 -0
  66. package/src/migrations/014-agent-failure-record.sql +56 -0
  67. package/src/migrations/015-projection-failure-record.sql +20 -0
  68. package/src/migrations/016-webhook-run-failure-record.sql +19 -0
  69. package/src/migrations/017-action-failure-record.sql +65 -0
  70. package/src/migrations/018-ontology-outbox-failure-record.sql +29 -0
  71. package/src/migrations/019-webhook-delivery-failure-record.sql +27 -0
  72. package/src/migrations/020-drop-run-usage-projections.sql +8 -0
  73. package/src/migrations/021-sync-pipeline-executions.sql +50 -0
  74. package/src/migrations/022-projection-executions.sql +115 -0
  75. package/src/migrations/023-webhook-executions.sql +30 -0
  76. package/src/migrations/024-ontology-commit-executions.sql +12 -0
  77. package/src/migrations/025-connector-connections.sql +231 -0
  78. package/src/migrations.ts +72 -0
  79. package/src/ontology-storage/materialization-session.ts +25 -28
  80. package/src/ontology-storage/materialization-state.ts +159 -43
  81. package/src/ontology-storage/materialization-writer.ts +146 -75
  82. package/src/ontology-storage/materializations.ts +3 -1
  83. package/src/ontology-storage/outbox.ts +9 -2
  84. package/src/ontology-storage/shared.ts +10 -20
  85. package/src/pg-action-run-storage.ts +43 -67
  86. package/src/pg-ai-usage-storage.ts +290 -0
  87. package/src/pg-execution-storage.ts +147 -0
  88. package/src/pg-pipeline-run-storage.ts +107 -41
  89. package/src/pg-projection-run-storage.ts +159 -91
  90. package/src/pg-sync-run-storage.ts +100 -28
  91. package/src/pg-webhook-run-storage.ts +89 -57
  92. package/src/pg-workflow-run-storage.ts +97 -130
  93. package/src/run-list-query.ts +9 -4
  94. package/src/testing.ts +18 -0
  95. package/dist/pg-webhook-delivery-storage.d.ts +0 -17
  96. package/dist/pg-webhook-delivery-storage.d.ts.map +0 -1
  97. package/src/pg-webhook-delivery-storage.ts +0 -180
@@ -18,6 +18,7 @@ import {
18
18
  import {
19
19
  appendScopeSnapshot,
20
20
  finishScopeAccumulator,
21
+ linkSlotOverrideValue,
21
22
  startScopeAccumulator,
22
23
  } from "@sixb/core/internal/ontology-storage-provider"
23
24
  import type {
@@ -27,6 +28,7 @@ import type {
27
28
  SourceReplacementLinkState,
28
29
  SourceReplacementObjectState,
29
30
  StoredLinkOverride,
31
+ StoredLinkSlotOverride,
30
32
  StoredObjectOverride,
31
33
  StoredSourceAssertion,
32
34
  StoredSourceLinkAssertion,
@@ -36,12 +38,11 @@ import type {
36
38
  import type { SQLClient } from "../pg-client"
37
39
  import {
38
40
  databaseSafeInteger,
39
- jsonKeyParameter,
40
41
  jsonParameter,
41
42
  linkRefFromColumns,
42
43
  objectRefFromColumns,
43
- type PgOntologyOverrideRow,
44
44
  type PgOntologySourceAssertionRow,
45
+ type PgStoredOverrideRow,
45
46
  toIsoString,
46
47
  } from "./shared"
47
48
 
@@ -94,12 +95,12 @@ interface LinkScopeRow extends EffectiveLinkRow {
94
95
  readonly link_sort_key: string
95
96
  }
96
97
 
97
- interface ObjectOverrideRow extends PgOntologyOverrideRow {
98
+ interface ObjectOverrideRow extends PgStoredOverrideRow {
98
99
  readonly object_type_id: string
99
100
  readonly primary_id: string
100
101
  }
101
102
 
102
- interface LinkOverrideRow extends PgOntologyOverrideRow {
103
+ interface LinkOverrideRow extends PgStoredOverrideRow {
103
104
  readonly source_type_id: string
104
105
  readonly source_primary_id: string
105
106
  readonly link_id: string
@@ -107,6 +108,17 @@ interface LinkOverrideRow extends PgOntologyOverrideRow {
107
108
  readonly target_primary_id: string
108
109
  }
109
110
 
111
+ interface LinkSlotOverrideRow {
112
+ readonly source_type_id: string
113
+ readonly source_primary_id: string
114
+ readonly link_id: string
115
+ readonly target_type_id: string
116
+ readonly target_primary_id: string
117
+ readonly value: unknown
118
+ readonly last_commit_id: string
119
+ readonly updated_at: Date | string
120
+ }
121
+
110
122
  export type ReplacementIdentity =
111
123
  | {
112
124
  readonly kind: "object"
@@ -193,10 +205,9 @@ export class PgMaterializationStateReader {
193
205
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
194
206
  AS requested_values(object_type_id TEXT, primary_id TEXT)
195
207
  )
196
- SELECT overrides.* FROM ontology_overrides AS overrides
208
+ SELECT overrides.* FROM ontology_object_overrides AS overrides
197
209
  JOIN requested USING (object_type_id, primary_id)
198
210
  WHERE overrides.project_id = ${this.projectId}
199
- AND overrides.entity_kind = 'object'
200
211
  `,
201
212
  this.sql<TelemetryRow[]>`
202
213
  WITH requested AS (
@@ -266,7 +277,7 @@ export class PgMaterializationStateReader {
266
277
  target_primary_id: ref.target.primaryId,
267
278
  }))
268
279
  const requestedParameter = jsonParameter(this.sql, requested)
269
- const [effectiveRows, sourceRows, overrideRows] = await Promise.all([
280
+ const [effectiveRows, sourceRows, overrideRows, slotOverrideRows] = await Promise.all([
270
281
  this.sql<EffectiveLinkRow[]>`
271
282
  WITH requested AS (
272
283
  SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
@@ -305,12 +316,23 @@ export class PgMaterializationStateReader {
305
316
  target_type_id TEXT, target_primary_id TEXT
306
317
  )
307
318
  )
308
- SELECT overrides.* FROM ontology_overrides AS overrides
319
+ SELECT overrides.* FROM ontology_link_overrides AS overrides
309
320
  JOIN requested USING (
310
321
  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
311
322
  )
312
323
  WHERE overrides.project_id = ${this.projectId}
313
- AND overrides.entity_kind = 'link'
324
+ AND overrides.identity_kind = 'edge'
325
+ `,
326
+ this.sql<LinkSlotOverrideRow[]>`
327
+ WITH requested AS (
328
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter}) AS requested_values(
329
+ source_type_id TEXT, source_primary_id TEXT, link_id TEXT
330
+ )
331
+ )
332
+ SELECT overrides.* FROM ontology_link_overrides AS overrides
333
+ JOIN requested USING (source_type_id, source_primary_id, link_id)
334
+ WHERE overrides.project_id = ${this.projectId}
335
+ AND overrides.identity_kind = 'slot'
314
336
  `,
315
337
  ])
316
338
  const effective = new Map(
@@ -322,6 +344,12 @@ export class PgMaterializationStateReader {
322
344
  (row) => [linkRefKey(linkRefFromOverrideColumns(row)), storedLinkOverride(row)] as const
323
345
  )
324
346
  )
347
+ const slotOverrides = new Map(
348
+ slotOverrideRows.map((row) => {
349
+ const stored = storedLinkSlotOverride(row)
350
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
351
+ })
352
+ )
325
353
  return refs.map((ref) => {
326
354
  const key = linkRefKey(ref)
327
355
  const effectiveRow = effective.get(key)
@@ -329,6 +357,7 @@ export class PgMaterializationStateReader {
329
357
  ref: structuredClone(ref),
330
358
  source: sourceLink(sources.get(projectionEntityKey({ kind: "link", ref }))),
331
359
  override: overrides.get(key) ?? null,
360
+ slotOverride: slotOverrides.get(linkScopeSortKey(ref.source, ref.linkId)) ?? null,
332
361
  effective: effectiveRow ? effectiveLinkSnapshot(effectiveRow) : null,
333
362
  }
334
363
  })
@@ -390,15 +419,63 @@ export class PgMaterializationStateReader {
390
419
  const accumulators = new Map(
391
420
  scopes.map(({ source, linkId }) => {
392
421
  const key = linkScopeSortKey(source, linkId)
393
- return [key, startScopeAccumulator(source, linkId, key)] as const
422
+ return [key, startScopeAccumulator(source, linkId)] as const
394
423
  })
395
424
  )
425
+ const requestedParameter = jsonParameter(this.sql, requested)
426
+ const [sourceRows, slotOverrideRows] = await Promise.all([
427
+ this.sql<PgOntologySourceAssertionRow[]>`
428
+ WITH requested AS (
429
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
430
+ AS requested_values(
431
+ scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
432
+ )
433
+ )
434
+ SELECT rows.*
435
+ FROM ontology_source_rows AS rows
436
+ JOIN requested
437
+ ON requested.source_type_id = rows.source_type_id
438
+ AND requested.source_id = rows.source_primary_id
439
+ AND requested.link_id = rows.link_id
440
+ JOIN ontology_sources AS sources
441
+ ON sources.project_id = rows.project_id
442
+ AND sources.source_id = rows.source_id
443
+ AND sources.materialization_id = rows.materialization_id
444
+ WHERE rows.project_id = ${this.projectId}
445
+ AND rows.entity_kind = 'link'
446
+ AND sources.status = 'active'
447
+ `,
448
+ this.sql<LinkSlotOverrideRow[]>`
449
+ WITH requested AS (
450
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
451
+ AS requested_values(
452
+ scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
453
+ )
454
+ )
455
+ SELECT overrides.*
456
+ FROM ontology_link_overrides AS overrides
457
+ JOIN requested
458
+ ON requested.source_type_id = overrides.source_type_id
459
+ AND requested.source_id = overrides.source_primary_id
460
+ AND requested.link_id = overrides.link_id
461
+ WHERE overrides.project_id = ${this.projectId}
462
+ AND overrides.identity_kind = 'slot'
463
+ `,
464
+ ])
465
+ const sources = activeScopeSourceMap(sourceRows)
466
+ const overrides = new Map(
467
+ slotOverrideRows.map((row) => {
468
+ const stored = storedLinkSlotOverride(row)
469
+ return [linkScopeSortKey(stored.ref.source, stored.ref.linkId), stored] as const
470
+ })
471
+ )
472
+ const effective = new Map<string, EffectiveLinkSnapshot | null>()
396
473
  let scopeCursor: string | null = null
397
474
  let linkCursor: string | null = null
398
475
  while (true) {
399
476
  const rows: LinkScopeRow[] = await this.sql`
400
477
  WITH requested AS (
401
- SELECT DISTINCT * FROM jsonb_to_recordset(${jsonParameter(this.sql, requested)})
478
+ SELECT DISTINCT * FROM jsonb_to_recordset(${requestedParameter})
402
479
  AS requested_values(
403
480
  scope_sort_key TEXT, source_type_id TEXT, source_id TEXT, link_id TEXT
404
481
  )
@@ -423,16 +500,24 @@ export class PgMaterializationStateReader {
423
500
  `Unexpected link scope '${row.scope_sort_key}' returned by storage.`
424
501
  )
425
502
  }
426
- appendScopeSnapshot(accumulator, effectiveLinkSnapshot(row))
503
+ const snapshot = effectiveLinkSnapshot(row)
504
+ appendScopeSnapshot(accumulator, snapshot)
505
+ effective.set(row.scope_sort_key, accumulator.effectiveCount === 1 ? snapshot : null)
427
506
  }
428
507
  if (rows.length < 500) break
429
508
  const last: LinkScopeRow = rows[rows.length - 1]!
430
509
  scopeCursor = last.scope_sort_key
431
510
  linkCursor = last.link_sort_key
432
511
  }
433
- return scopes.map(({ source, linkId }) =>
434
- finishScopeAccumulator(accumulators.get(linkScopeSortKey(source, linkId))!)
435
- )
512
+ return scopes.map(({ source, linkId }) => {
513
+ const key = linkScopeSortKey(source, linkId)
514
+ return {
515
+ ...finishScopeAccumulator(accumulators.get(key)!),
516
+ sourceAssertion: sources.get(key) ?? null,
517
+ override: overrides.get(key) ?? null,
518
+ effective: effective.get(key) ?? null,
519
+ }
520
+ })
436
521
  }
437
522
 
438
523
  async linkScope(
@@ -485,20 +570,20 @@ export class PgMaterializationStateReader {
485
570
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
486
571
  overrides.link_id, overrides.target_type_id,
487
572
  overrides.target_primary_id AS target_id
488
- FROM ontology_overrides AS overrides
573
+ FROM ontology_link_overrides AS overrides
489
574
  JOIN requested
490
575
  ON requested.object_type_id = overrides.source_type_id
491
576
  AND requested.primary_id = overrides.source_primary_id
492
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
577
+ WHERE overrides.project_id = ${this.projectId}
493
578
  UNION
494
579
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
495
580
  overrides.link_id, overrides.target_type_id,
496
581
  overrides.target_primary_id AS target_id
497
- FROM ontology_overrides AS overrides
582
+ FROM ontology_link_overrides AS overrides
498
583
  JOIN requested
499
584
  ON requested.object_type_id = overrides.target_type_id
500
585
  AND requested.primary_id = overrides.target_primary_id
501
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
586
+ WHERE overrides.project_id = ${this.projectId}
502
587
  UNION
503
588
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
504
589
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
@@ -609,6 +694,7 @@ export class PgMaterializationStateReader {
609
694
  ? sourceLink(candidate?.get(key))
610
695
  : state.source,
611
696
  override: state.override,
697
+ slotOverride: state.slotOverride,
612
698
  effective: state.effective,
613
699
  diffRequired: identities[index]!.diffRequired,
614
700
  }
@@ -701,16 +787,6 @@ export class PgMaterializationStateReader {
701
787
  )
702
788
  }
703
789
 
704
- async overrideLastCommit(kind: "object" | "link", entityKey: string): Promise<string | null> {
705
- const [row] = await this.sql<{ readonly last_commit_id: string }[]>`
706
- SELECT last_commit_id FROM ontology_overrides
707
- WHERE project_id = ${this.projectId}
708
- AND entity_kind = ${kind}
709
- AND entity_key = ${jsonKeyParameter(this.sql, entityKey)}
710
- `
711
- return row?.last_commit_id ?? null
712
- }
713
-
714
790
  private async replacementSources(
715
791
  sourceId: string,
716
792
  materializationIds: readonly string[],
@@ -824,20 +900,20 @@ export class PgMaterializationStateReader {
824
900
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
825
901
  overrides.link_id, overrides.target_type_id,
826
902
  overrides.target_primary_id AS target_id
827
- FROM ontology_overrides AS overrides
903
+ FROM ontology_link_overrides AS overrides
828
904
  JOIN incident_objects
829
905
  ON incident_objects.object_type_id = overrides.source_type_id
830
906
  AND incident_objects.primary_id = overrides.source_primary_id
831
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
907
+ WHERE overrides.project_id = ${this.projectId}
832
908
  UNION
833
909
  SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
834
910
  overrides.link_id, overrides.target_type_id,
835
911
  overrides.target_primary_id AS target_id
836
- FROM ontology_overrides AS overrides
912
+ FROM ontology_link_overrides AS overrides
837
913
  JOIN incident_objects
838
914
  ON incident_objects.object_type_id = overrides.target_type_id
839
915
  AND incident_objects.primary_id = overrides.target_primary_id
840
- WHERE overrides.project_id = ${this.projectId} AND overrides.entity_kind = 'link'
916
+ WHERE overrides.project_id = ${this.projectId}
841
917
  UNION
842
918
  SELECT rows.source_type_id, rows.source_primary_id AS source_id,
843
919
  rows.link_id, rows.target_type_id, rows.target_primary_id AS target_id
@@ -879,6 +955,16 @@ export class PgMaterializationStateReader {
879
955
  FROM links
880
956
  JOIN affected_scopes USING (source_type_id, source_id, link_id)
881
957
  WHERE links.project_id = ${this.projectId}
958
+ UNION ALL
959
+ SELECT overrides.source_type_id, overrides.source_primary_id AS source_id,
960
+ overrides.link_id, overrides.target_type_id,
961
+ overrides.target_primary_id AS target_id, FALSE AS diff_required
962
+ FROM ontology_link_overrides AS overrides
963
+ JOIN affected_scopes
964
+ ON affected_scopes.source_type_id = overrides.source_type_id
965
+ AND affected_scopes.source_id = overrides.source_primary_id
966
+ AND affected_scopes.link_id = overrides.link_id
967
+ WHERE overrides.project_id = ${this.projectId} AND overrides.identity_kind = 'slot'
882
968
  ), selected AS (
883
969
  SELECT source_type_id, source_id, link_id, target_type_id, target_id,
884
970
  ${this.sql.unsafe(linkKeyExpression("all_links"))} AS identity_key,
@@ -939,6 +1025,26 @@ function activeSourceMap(
939
1025
  return result
940
1026
  }
941
1027
 
1028
+ function activeScopeSourceMap(
1029
+ rows: readonly PgOntologySourceAssertionRow[]
1030
+ ): ReadonlyMap<string, StoredSourceLinkAssertion> {
1031
+ const result = new Map<string, StoredSourceLinkAssertion>()
1032
+ for (const row of rows) {
1033
+ const source = sourceLink(storedSource(row))
1034
+ if (!source) continue
1035
+ const ref = source.assertion.ref
1036
+ const key = linkScopeSortKey(ref.source, ref.linkId)
1037
+ if (result.has(key)) {
1038
+ throw new MaterializationConflictError(
1039
+ "source-materialization",
1040
+ `Multiple active sources assert cardinality-one link slot ${key}.`
1041
+ )
1042
+ }
1043
+ result.set(key, source)
1044
+ }
1045
+ return result
1046
+ }
1047
+
942
1048
  function sourceObject(
943
1049
  source: StoredSourceAssertion | undefined
944
1050
  ): StoredSourceObjectAssertion | null {
@@ -969,6 +1075,19 @@ function storedLinkOverride(row: LinkOverrideRow): StoredLinkOverride {
969
1075
  }
970
1076
  }
971
1077
 
1078
+ function storedLinkSlotOverride(row: LinkSlotOverrideRow): StoredLinkSlotOverride {
1079
+ const ref = {
1080
+ source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
1081
+ linkId: row.link_id,
1082
+ }
1083
+ return {
1084
+ ref,
1085
+ value: linkSlotOverrideValue(row.value, linkScopeLabel(ref)),
1086
+ lastCommitId: row.last_commit_id,
1087
+ updatedAt: toIsoString(row.updated_at),
1088
+ }
1089
+ }
1090
+
972
1091
  function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
973
1092
  return {
974
1093
  source: { objectTypeId: row.source_type_id, primaryId: row.source_primary_id },
@@ -977,6 +1096,13 @@ function linkRefFromOverrideColumns(row: LinkOverrideRow): OntologyLinkRef {
977
1096
  }
978
1097
  }
979
1098
 
1099
+ function linkScopeLabel(scope: {
1100
+ readonly source: OntologyObjectRef
1101
+ readonly linkId: string
1102
+ }): string {
1103
+ return JSON.stringify([scope.source.objectTypeId, scope.source.primaryId, scope.linkId])
1104
+ }
1105
+
980
1106
  function storedSource(row: PgOntologySourceAssertionRow): StoredSourceAssertion {
981
1107
  return {
982
1108
  source: { projectionId: row.source_id },
@@ -1090,16 +1216,6 @@ function linkKeyExpression(alias?: string): string {
1090
1216
  ])
1091
1217
  }
1092
1218
 
1093
- export function pointSortExpression(alias?: string): string {
1094
- const prefix = alias ? `${alias}.` : ""
1095
- return utf8SortExpression([
1096
- `to_jsonb(${prefix}object_type_id)::text`,
1097
- `to_jsonb(${prefix}object_id)::text`,
1098
- `to_jsonb(${prefix}property_id)::text`,
1099
- `to_jsonb(to_char(${prefix}at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))::text`,
1100
- ])
1101
- }
1102
-
1103
1219
  function utf8SortExpression(parts: readonly string[]): string {
1104
1220
  return `encode(convert_to(${jsonTupleExpression(parts)}, 'UTF8'), 'hex')`
1105
1221
  }
@@ -4,11 +4,7 @@ import {
4
4
  objectRefKey,
5
5
  telemetryPointKey,
6
6
  } from "@sixb/core/internal/materialization"
7
- import {
8
- effectiveConflict,
9
- type OverrideEntity,
10
- overrideEntityColumns as overrideColumns,
11
- } from "@sixb/core/internal/ontology-storage-provider"
7
+ import { effectiveConflict } from "@sixb/core/internal/ontology-storage-provider"
12
8
  import type { MaterializationPlanChunk } from "@sixb/core/storage"
13
9
  import type { SQLClient } from "../pg-client"
14
10
  import { assertTimestamp, jsonParameter } from "./shared"
@@ -25,82 +21,171 @@ export class PgMaterializationWriter {
25
21
  }
26
22
 
27
23
  private async applyOverrides(projectId: string, chunk: MaterializationPlanChunk): Promise<void> {
28
- const upserts = [
29
- ...chunk.overrides.objectUpserts.map((item) =>
30
- overrideWrite({ kind: "object", ref: item.ref }, item)
31
- ),
32
- ...chunk.overrides.linkUpserts.map((item) =>
33
- overrideWrite({ kind: "link", ref: item.ref }, item)
34
- ),
35
- ]
36
- const inserts = upserts.filter((item) => item.expectedLastCommitId === null)
37
- if (inserts.length > 0) {
38
- const rows = await this.sql<{ readonly entity_kind: "object" | "link" }[]>`
24
+ const objectUpserts = chunk.overrides.objects.upserts.map((item) => ({
25
+ objectTypeId: item.ref.objectTypeId,
26
+ primaryId: item.ref.primaryId,
27
+ value: item.value,
28
+ lastCommitId: item.lastCommitId,
29
+ updatedAt: item.updatedAt,
30
+ expectedLastCommitId: item.expectedLastCommitId,
31
+ }))
32
+ const objectInserts = objectUpserts.filter((item) => item.expectedLastCommitId === null)
33
+ if (objectInserts.length > 0) {
34
+ const rows = await this.sql<{ readonly object_type_id: string }[]>`
39
35
  WITH staged AS (
40
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
36
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectInserts)}::jsonb)
41
37
  )
42
- INSERT INTO ontology_overrides (
43
- project_id, entity_kind, entity_key, entity_sort_key,
44
- object_type_id, primary_id, source_type_id, source_primary_id,
45
- link_id, target_type_id, target_primary_id,
46
- value, last_commit_id, updated_at
38
+ INSERT INTO ontology_object_overrides (
39
+ project_id, object_type_id, primary_id, value, last_commit_id, updated_at
47
40
  )
48
- SELECT ${projectId}, value->>'kind', value->'entityKey', value->>'sortKey',
49
- value->>'objectTypeId', value->>'primaryId', value->>'sourceTypeId',
50
- value->>'sourcePrimaryId', value->>'linkId', value->>'targetTypeId',
51
- value->>'targetPrimaryId', value->'value', value->>'lastCommitId',
41
+ SELECT ${projectId}, value->>'objectTypeId', value->>'primaryId', value->'value',
42
+ value->>'lastCommitId',
52
43
  (value->>'updatedAt')::timestamptz
53
44
  FROM staged
54
45
  ON CONFLICT DO NOTHING
55
- RETURNING entity_kind
46
+ RETURNING object_type_id
56
47
  `
57
- if (rows.length !== inserts.length) throw overrideConflict(inserts[0]!.kind)
48
+ if (rows.length !== objectInserts.length) throw objectOverrideConflict()
58
49
  }
59
50
 
60
- const updates = upserts.filter((item) => item.expectedLastCommitId !== null)
61
- if (updates.length > 0) {
62
- const rows = await this.sql<{ readonly entity_kind: "object" | "link" }[]>`
51
+ const objectUpdates = objectUpserts.filter((item) => item.expectedLastCommitId !== null)
52
+ if (objectUpdates.length > 0) {
53
+ const rows = await this.sql<{ readonly object_type_id: string }[]>`
63
54
  WITH staged AS (
64
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, updates)}::jsonb)
55
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectUpdates)}::jsonb)
65
56
  )
66
- UPDATE ontology_overrides AS overrides
57
+ UPDATE ontology_object_overrides AS overrides
67
58
  SET value = staged.value->'value', last_commit_id = staged.value->>'lastCommitId',
68
59
  updated_at = (staged.value->>'updatedAt')::timestamptz
69
60
  FROM staged
70
61
  WHERE overrides.project_id = ${projectId}
71
- AND overrides.entity_kind = staged.value->>'kind'
72
- AND overrides.entity_key = staged.value->'entityKey'
62
+ AND overrides.object_type_id = staged.value->>'objectTypeId'
63
+ AND overrides.primary_id = staged.value->>'primaryId'
64
+ AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
65
+ RETURNING overrides.object_type_id
66
+ `
67
+ if (rows.length !== objectUpdates.length) throw objectOverrideConflict()
68
+ }
69
+
70
+ const objectDeletes = chunk.overrides.objects.deletes.map((item) => ({
71
+ objectTypeId: item.ref.objectTypeId,
72
+ primaryId: item.ref.primaryId,
73
+ expectedLastCommitId: item.expectedLastCommitId,
74
+ }))
75
+ if (objectDeletes.length > 0) {
76
+ const rows = await this.sql<{ readonly object_type_id: string }[]>`
77
+ WITH staged AS (
78
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectDeletes)}::jsonb)
79
+ )
80
+ DELETE FROM ontology_object_overrides AS overrides USING staged
81
+ WHERE overrides.project_id = ${projectId}
82
+ AND overrides.object_type_id = staged.value->>'objectTypeId'
83
+ AND overrides.primary_id = staged.value->>'primaryId'
84
+ AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
85
+ RETURNING overrides.object_type_id
86
+ `
87
+ if (rows.length !== objectDeletes.length) throw objectOverrideConflict()
88
+ }
89
+
90
+ const linkUpserts = [
91
+ ...chunk.overrides.links.edges.upserts.map((item) => ({
92
+ identityKind: "edge" as const,
93
+ identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
94
+ sourceTypeId: item.ref.source.objectTypeId,
95
+ sourcePrimaryId: item.ref.source.primaryId,
96
+ linkId: item.ref.linkId,
97
+ targetTypeId: item.ref.target.objectTypeId,
98
+ targetPrimaryId: item.ref.target.primaryId,
99
+ value: item.value,
100
+ lastCommitId: item.lastCommitId,
101
+ updatedAt: item.updatedAt,
102
+ expectedLastCommitId: item.expectedLastCommitId,
103
+ })),
104
+ ...chunk.overrides.links.slots.upserts.map((item) => ({
105
+ identityKind: "slot" as const,
106
+ identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
107
+ sourceTypeId: item.ref.source.objectTypeId,
108
+ sourcePrimaryId: item.ref.source.primaryId,
109
+ linkId: item.ref.linkId,
110
+ targetTypeId: item.value.target.objectTypeId,
111
+ targetPrimaryId: item.value.target.primaryId,
112
+ value: item.value,
113
+ lastCommitId: item.lastCommitId,
114
+ updatedAt: item.updatedAt,
115
+ expectedLastCommitId: item.expectedLastCommitId,
116
+ })),
117
+ ]
118
+ const linkInserts = linkUpserts.filter((item) => item.expectedLastCommitId === null)
119
+ if (linkInserts.length > 0) {
120
+ const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
121
+ WITH staged AS (
122
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkInserts)}::jsonb)
123
+ )
124
+ INSERT INTO ontology_link_overrides (
125
+ project_id, identity_kind, identity_key, source_type_id, source_primary_id, link_id,
126
+ target_type_id, target_primary_id, value, last_commit_id, updated_at
127
+ )
128
+ SELECT ${projectId}, value->>'identityKind', value->'identityKey',
129
+ value->>'sourceTypeId', value->>'sourcePrimaryId', value->>'linkId',
130
+ value->>'targetTypeId', value->>'targetPrimaryId', value->'value',
131
+ value->>'lastCommitId', (value->>'updatedAt')::timestamptz
132
+ FROM staged
133
+ ON CONFLICT DO NOTHING
134
+ RETURNING identity_kind
135
+ `
136
+ if (rows.length !== linkInserts.length)
137
+ throw linkOverrideConflict(linkInserts[0]!.identityKind)
138
+ }
139
+
140
+ const linkUpdates = linkUpserts.filter((item) => item.expectedLastCommitId !== null)
141
+ if (linkUpdates.length > 0) {
142
+ const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
143
+ WITH staged AS (
144
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkUpdates)}::jsonb)
145
+ )
146
+ UPDATE ontology_link_overrides AS overrides
147
+ SET target_type_id = staged.value->>'targetTypeId',
148
+ target_primary_id = staged.value->>'targetPrimaryId',
149
+ value = staged.value->'value',
150
+ last_commit_id = staged.value->>'lastCommitId',
151
+ updated_at = (staged.value->>'updatedAt')::timestamptz
152
+ FROM staged
153
+ WHERE overrides.project_id = ${projectId}
154
+ AND overrides.identity_kind = staged.value->>'identityKind'
155
+ AND overrides.identity_key = staged.value->'identityKey'
73
156
  AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
74
- RETURNING overrides.entity_kind
157
+ RETURNING overrides.identity_kind
75
158
  `
76
- if (rows.length !== updates.length) throw overrideConflict(updates[0]!.kind)
159
+ if (rows.length !== linkUpdates.length)
160
+ throw linkOverrideConflict(linkUpdates[0]!.identityKind)
77
161
  }
78
162
 
79
- const deletes = [
80
- ...chunk.overrides.objectDeletes.map((item) => ({
81
- kind: "object" as const,
82
- entityKey: JSON.parse(objectRefKey(item.ref)) as unknown,
163
+ const linkDeletes = [
164
+ ...chunk.overrides.links.edges.deletes.map((item) => ({
165
+ identityKind: "edge" as const,
166
+ identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
83
167
  expectedLastCommitId: item.expectedLastCommitId,
84
168
  })),
85
- ...chunk.overrides.linkDeletes.map((item) => ({
86
- kind: "link" as const,
87
- entityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
169
+ ...chunk.overrides.links.slots.deletes.map((item) => ({
170
+ identityKind: "slot" as const,
171
+ identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
88
172
  expectedLastCommitId: item.expectedLastCommitId,
89
173
  })),
90
174
  ]
91
- if (deletes.length > 0) {
92
- const rows = await this.sql<{ readonly entity_kind: "object" | "link" }[]>`
175
+ if (linkDeletes.length > 0) {
176
+ const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
93
177
  WITH staged AS (
94
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, deletes)}::jsonb)
178
+ SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkDeletes)}::jsonb)
95
179
  )
96
- DELETE FROM ontology_overrides AS overrides USING staged
180
+ DELETE FROM ontology_link_overrides AS overrides USING staged
97
181
  WHERE overrides.project_id = ${projectId}
98
- AND overrides.entity_kind = staged.value->>'kind'
99
- AND overrides.entity_key = staged.value->'entityKey'
182
+ AND overrides.identity_kind = staged.value->>'identityKind'
183
+ AND overrides.identity_key = staged.value->'identityKey'
100
184
  AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
101
- RETURNING overrides.entity_kind
185
+ RETURNING overrides.identity_kind
102
186
  `
103
- if (rows.length !== deletes.length) throw overrideConflict(deletes[0]!.kind)
187
+ if (rows.length !== linkDeletes.length)
188
+ throw linkOverrideConflict(linkDeletes[0]!.identityKind)
104
189
  }
105
190
  }
106
191
 
@@ -408,7 +493,7 @@ export class PgMaterializationWriter {
408
493
  INSERT INTO ontology_outbox (
409
494
  project_id, id, commit_id, commit_ordinal, envelope,
410
495
  available_at, attempts, lease_id, lease_expires_at,
411
- published_at, last_error, created_at
496
+ published_at, last_failure, created_at
412
497
  )
413
498
  SELECT ${projectId}, value->>'id', ${commitId},
414
499
  (value->>'commitOrdinal')::bigint, value->'envelope',
@@ -426,30 +511,16 @@ export class PgMaterializationWriter {
426
511
  }
427
512
  }
428
513
 
429
- function overrideWrite(
430
- entity: OverrideEntity,
431
- item: {
432
- readonly value: unknown
433
- readonly lastCommitId: string
434
- readonly updatedAt: string
435
- readonly expectedLastCommitId: string | null
436
- }
437
- ) {
438
- const key = entity.kind === "object" ? objectRefKey(entity.ref) : linkRefKey(entity.ref)
439
- const columns = overrideColumns(entity)
440
- return {
441
- kind: entity.kind,
442
- entityKey: JSON.parse(key) as unknown,
443
- ...columns,
444
- value: item.value,
445
- lastCommitId: item.lastCommitId,
446
- updatedAt: item.updatedAt,
447
- expectedLastCommitId: item.expectedLastCommitId,
448
- }
514
+ function objectOverrideConflict(): MaterializationConflictError {
515
+ return effectiveConflict("Expected object override changed.")
449
516
  }
450
517
 
451
- function overrideConflict(kind: "object" | "link"): MaterializationConflictError {
452
- return effectiveConflict(`Expected ${kind} override changed.`)
518
+ function linkOverrideConflict(identityKind: "edge" | "slot"): MaterializationConflictError {
519
+ return effectiveConflict(
520
+ identityKind === "edge"
521
+ ? "Expected link edge override changed."
522
+ : "Expected link slot override changed."
523
+ )
453
524
  }
454
525
 
455
526
  function objectIdentityKey(item: {