@sixb/pg 0.1.15 → 0.1.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +125 -0
  2. package/dist/index.d.ts +15 -2
  3. package/dist/index.d.ts.map +1 -1
  4. package/dist/index.js +3202 -1474
  5. package/dist/index.js.map +32 -25
  6. package/dist/migrations.d.ts.map +1 -1
  7. package/dist/objects/aggregate-admission.d.ts +10 -0
  8. package/dist/objects/aggregate-admission.d.ts.map +1 -0
  9. package/dist/objects/preparation-lock.d.ts +6 -0
  10. package/dist/objects/preparation-lock.d.ts.map +1 -0
  11. package/dist/objects/prepare-queries.d.ts +5 -0
  12. package/dist/objects/prepare-queries.d.ts.map +1 -0
  13. package/dist/objects/query-compiler.d.ts +19 -0
  14. package/dist/objects/query-compiler.d.ts.map +1 -1
  15. package/dist/objects/query-indexes.d.ts +43 -0
  16. package/dist/objects/query-indexes.d.ts.map +1 -0
  17. package/dist/objects/reader.d.ts +7 -1
  18. package/dist/objects/reader.d.ts.map +1 -1
  19. package/dist/objects/storage.d.ts +1 -1
  20. package/dist/objects/storage.d.ts.map +1 -1
  21. package/dist/objects/text-count-index.d.ts +24 -0
  22. package/dist/objects/text-count-index.d.ts.map +1 -0
  23. package/dist/ontology-storage/index.d.ts +2 -0
  24. package/dist/ontology-storage/index.d.ts.map +1 -1
  25. package/dist/ontology-storage/materialization-session.d.ts +18 -38
  26. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  27. package/dist/ontology-storage/materialization-state.d.ts +3 -26
  28. package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
  29. package/dist/ontology-storage/materialization-writer.d.ts +51 -9
  30. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  31. package/dist/ontology-storage/materializations.d.ts +5 -7
  32. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  33. package/dist/ontology-storage/outbox.d.ts.map +1 -1
  34. package/dist/ontology-storage/replacement-plans.d.ts +36 -0
  35. package/dist/ontology-storage/replacement-plans.d.ts.map +1 -0
  36. package/dist/ontology-storage/shared.d.ts +11 -5
  37. package/dist/ontology-storage/shared.d.ts.map +1 -1
  38. package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
  39. package/dist/ontology-storage/source-roots.d.ts +14 -1
  40. package/dist/ontology-storage/source-roots.d.ts.map +1 -1
  41. package/dist/ontology-storage/sources.d.ts +1 -2
  42. package/dist/ontology-storage/sources.d.ts.map +1 -1
  43. package/dist/pg-ai-cost-storage.d.ts.map +1 -1
  44. package/dist/pg-ai-limit-storage.d.ts.map +1 -1
  45. package/dist/pg-ai-usage-storage.d.ts +2 -0
  46. package/dist/pg-ai-usage-storage.d.ts.map +1 -1
  47. package/dist/pg-execution-storage.d.ts +28 -0
  48. package/dist/pg-execution-storage.d.ts.map +1 -1
  49. package/dist/pg-file-download-grant-storage.d.ts +11 -0
  50. package/dist/pg-file-download-grant-storage.d.ts.map +1 -0
  51. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  52. package/dist/pg-rules-storage.d.ts.map +1 -1
  53. package/dist/transactions.d.ts +6 -2
  54. package/dist/transactions.d.ts.map +1 -1
  55. package/package.json +3 -3
  56. package/src/index.ts +53 -3
  57. package/src/migrations/053-replacement-plans.sql +72 -0
  58. package/src/migrations/054-slim-ontology-outbox.sql +108 -0
  59. package/src/migrations/055-object-query-preparation.sql +11 -0
  60. package/src/migrations/056-commit-touches.sql +41 -0
  61. package/src/migrations/057-audio-transcription.sql +10 -0
  62. package/src/migrations/058-reranking-model-kind.sql +11 -0
  63. package/src/migrations/059-file-download-grants.sql +16 -0
  64. package/src/migrations/060-drained-source-versions.sql +11 -0
  65. package/src/migrations.ts +20 -0
  66. package/src/objects/aggregate-admission.ts +21 -0
  67. package/src/objects/preparation-lock.ts +15 -0
  68. package/src/objects/prepare-queries.ts +181 -0
  69. package/src/objects/query-compiler.ts +405 -115
  70. package/src/objects/query-indexes.ts +348 -0
  71. package/src/objects/reader.ts +140 -51
  72. package/src/objects/storage.ts +15 -3
  73. package/src/objects/text-count-index.ts +455 -0
  74. package/src/ontology-storage/index.ts +3 -0
  75. package/src/ontology-storage/materialization-session.ts +116 -275
  76. package/src/ontology-storage/materialization-state.ts +8 -259
  77. package/src/ontology-storage/materialization-writer.ts +618 -508
  78. package/src/ontology-storage/materializations.ts +106 -289
  79. package/src/ontology-storage/outbox.ts +46 -15
  80. package/src/ontology-storage/replacement-plans.ts +1110 -0
  81. package/src/ontology-storage/shared.ts +27 -15
  82. package/src/ontology-storage/source-cleanup.ts +22 -10
  83. package/src/ontology-storage/source-roots.ts +35 -8
  84. package/src/ontology-storage/sources.ts +0 -30
  85. package/src/pg-ai-cost-storage.ts +12 -0
  86. package/src/pg-ai-limit-storage.ts +13 -2
  87. package/src/pg-ai-usage-storage.ts +8 -0
  88. package/src/pg-execution-storage.ts +8 -3
  89. package/src/pg-file-download-grant-storage.ts +76 -0
  90. package/src/pg-projection-run-storage.ts +6 -1
  91. package/src/pg-rules-storage.ts +7 -3
  92. package/src/transactions.ts +10 -3
@@ -1,582 +1,692 @@
1
1
  import {
2
2
  linkRefKey,
3
3
  MaterializationConflictError,
4
+ type OntologyLinkRef,
5
+ type OntologyObjectRef,
4
6
  objectRefKey,
7
+ type TelemetrySeriesRef,
5
8
  telemetryPointKey,
6
9
  } from "@sixb/core/internal/materialization"
7
- import { effectiveConflict } from "@sixb/core/internal/ontology-storage-provider"
8
- import type { MaterializationPlanChunk } from "@sixb/core/storage"
9
- import type { SQLClient } from "../pg-client"
10
- import { assertTimestamp, jsonParameter } from "./shared"
10
+ import {
11
+ canonicalJson,
12
+ effectiveConflict,
13
+ materializationApplyPhase,
14
+ materializationPlanKindRank,
15
+ } from "@sixb/core/internal/ontology-storage-provider"
16
+ import type { MaterializationPlanWorkItem } from "@sixb/core/storage"
17
+ import type postgres from "postgres"
18
+ import type { SQL, SQLClient } from "../pg-client"
19
+
20
+ export interface PgStagedPlanTarget {
21
+ /** The work table and the id its rows carry: a session's temp table, or a durable plan. */
22
+ readonly workTable: string
23
+ readonly workId: string
24
+ readonly projectId: string
25
+ readonly commitId: string
26
+ readonly committedAt: string
27
+ }
28
+
29
+ export interface PgAppliedPlan {
30
+ readonly objectWrites: number
31
+ readonly linkWrites: number
32
+ readonly eventCount: number
33
+ }
34
+
35
+ interface WriteCounts {
36
+ readonly staged: number | string
37
+ readonly written: number | string
38
+ }
11
39
 
12
- /** Applies one already-validated exact plan chunk in bounded PostgreSQL batches. */
40
+ type Fragment = ReturnType<SQL["unsafe"]>
41
+ type Query = postgres.PendingQuery<postgres.Row[]>
42
+
43
+ /**
44
+ * Applies a session's staged plan where it is stored: each statement reads one item kind from the
45
+ * work table and writes it set-based, so planned rows never travel back through the client.
46
+ *
47
+ * Every write still checks the revision its plan expected. A statement that writes fewer rows than
48
+ * it staged is a conflict; only then does a second, indexed query find the entity to name, so the
49
+ * normal path never correlates the written rows back to the staged ones.
50
+ */
13
51
  export class PgMaterializationWriter {
14
52
  constructor(private readonly sql: SQLClient) {}
15
53
 
16
- async apply(projectId: string, commitId: string, chunk: MaterializationPlanChunk): Promise<void> {
17
- await this.applyOverrides(projectId, chunk)
18
- await this.applyEffective(projectId, chunk)
19
- await this.applyTimeseries(projectId, chunk)
20
- await this.applyOutbox(projectId, commitId, chunk)
54
+ async applyStaged(target: PgStagedPlanTarget): Promise<PgAppliedPlan> {
55
+ await this.applyObjectOverrides(target)
56
+ await this.applyLinkOverrides(target)
57
+ await this.applyPoints(target)
58
+ let linkWrites = await this.deleteLinks(target)
59
+ let objectWrites = await this.deleteObjects(target)
60
+ objectWrites += await this.upsertObjects(target)
61
+ linkWrites += await this.upsertLinks(target)
62
+ const eventCount = await this.writeOutbox(target)
63
+ await this.writeTouches(target)
64
+ return { objectWrites, linkWrites, eventCount }
65
+ }
66
+
67
+ /** The staged values of one plan item kind, as `value`. */
68
+ private staged(target: PgStagedPlanTarget, kind: MaterializationPlanWorkItem["kind"]) {
69
+ return this.sql`
70
+ SELECT payload->'item'->'value' AS value
71
+ FROM ${this.sql(target.workTable)}
72
+ WHERE work_id = ${target.workId} AND lane = 'apply'
73
+ AND major_order = ${materializationApplyPhase(kind)}
74
+ AND minor_order = ${materializationPlanKindRank(kind)}
75
+ `
21
76
  }
22
77
 
23
- private async applyOverrides(projectId: string, chunk: MaterializationPlanChunk): Promise<void> {
24
- const objectUpserts = chunk.overrides.objects.upserts.map((item) => ({
25
- objectTypeId: item.ref.objectTypeId,
26
- primaryId: item.ref.primaryId,
27
- value: item.value,
28
- editedAt: item.editedAt,
29
- lastCommitId: item.lastCommitId,
30
- updatedAt: item.updatedAt,
31
- expectedLastCommitId: item.expectedLastCommitId,
32
- }))
33
- const objectInserts = objectUpserts.filter((item) => item.expectedLastCommitId === null)
34
- if (objectInserts.length > 0) {
35
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
36
- WITH staged AS (
37
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectInserts)}::jsonb)
78
+ private async applyObjectOverrides(target: PgStagedPlanTarget): Promise<void> {
79
+ const { projectId } = target
80
+ const inserts = this.sql`
81
+ SELECT value FROM (${this.staged(target, "object-override-upsert")}) AS staged
82
+ WHERE value->>'expectedLastCommitId' IS NULL
83
+ `
84
+ await this.write(
85
+ this.sql<WriteCounts[]>`
86
+ WITH candidates AS (${inserts}), written AS (
87
+ INSERT INTO ontology_object_overrides (
88
+ project_id, object_type_id, primary_id, value, edited_at, last_commit_id, updated_at
89
+ )
90
+ SELECT ${projectId}, value->'ref'->>'objectTypeId', value->'ref'->>'primaryId',
91
+ value->'value', value->'editedAt', value->>'lastCommitId',
92
+ (value->>'updatedAt')::timestamptz
93
+ FROM candidates
94
+ ON CONFLICT DO NOTHING
95
+ RETURNING 1
38
96
  )
39
- INSERT INTO ontology_object_overrides (
40
- project_id, object_type_id, primary_id, value, edited_at, last_commit_id, updated_at
97
+ ${counts(this.sql)}
98
+ `,
99
+ async () => {
100
+ throw objectOverrideConflict()
101
+ }
102
+ )
103
+ const updates = this.sql`
104
+ SELECT value FROM (${this.staged(target, "object-override-upsert")}) AS staged
105
+ WHERE value->>'expectedLastCommitId' IS NOT NULL
106
+ `
107
+ await this.write(
108
+ this.sql<WriteCounts[]>`
109
+ WITH candidates AS (${updates}), written AS (
110
+ UPDATE ontology_object_overrides AS stored
111
+ SET value = candidates.value->'value', edited_at = candidates.value->'editedAt',
112
+ last_commit_id = candidates.value->>'lastCommitId',
113
+ updated_at = (candidates.value->>'updatedAt')::timestamptz
114
+ FROM candidates
115
+ WHERE stored.project_id = ${projectId} AND ${this.matches("object", "value->'ref'")}
116
+ AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
117
+ RETURNING 1
41
118
  )
42
- SELECT ${projectId}, value->>'objectTypeId', value->>'primaryId', value->'value',
43
- value->'editedAt', value->>'lastCommitId',
44
- (value->>'updatedAt')::timestamptz
45
- FROM staged
46
- ON CONFLICT DO NOTHING
47
- RETURNING object_type_id
48
- `
49
- if (rows.length !== objectInserts.length) throw objectOverrideConflict()
50
- }
51
-
52
- const objectUpdates = objectUpserts.filter((item) => item.expectedLastCommitId !== null)
53
- if (objectUpdates.length > 0) {
54
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
55
- WITH staged AS (
56
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectUpdates)}::jsonb)
119
+ ${counts(this.sql)}
120
+ `,
121
+ async () => {
122
+ throw objectOverrideConflict()
123
+ }
124
+ )
125
+ await this.write(
126
+ this.sql<WriteCounts[]>`
127
+ WITH candidates AS (${this.staged(target, "object-override-delete")}), written AS (
128
+ DELETE FROM ontology_object_overrides AS stored USING candidates
129
+ WHERE stored.project_id = ${projectId} AND ${this.matches("object", "value->'ref'")}
130
+ AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
131
+ RETURNING 1
57
132
  )
58
- UPDATE ontology_object_overrides AS overrides
59
- SET value = staged.value->'value', edited_at = staged.value->'editedAt',
60
- last_commit_id = staged.value->>'lastCommitId',
61
- updated_at = (staged.value->>'updatedAt')::timestamptz
62
- FROM staged
63
- WHERE overrides.project_id = ${projectId}
64
- AND overrides.object_type_id = staged.value->>'objectTypeId'
65
- AND overrides.primary_id = staged.value->>'primaryId'
66
- AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
67
- RETURNING overrides.object_type_id
68
- `
69
- if (rows.length !== objectUpdates.length) throw objectOverrideConflict()
70
- }
133
+ ${counts(this.sql)}
134
+ `,
135
+ async () => {
136
+ throw objectOverrideConflict()
137
+ }
138
+ )
139
+ }
71
140
 
72
- const objectDeletes = chunk.overrides.objects.deletes.map((item) => ({
73
- objectTypeId: item.ref.objectTypeId,
74
- primaryId: item.ref.primaryId,
75
- expectedLastCommitId: item.expectedLastCommitId,
76
- }))
77
- if (objectDeletes.length > 0) {
78
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
79
- WITH staged AS (
80
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectDeletes)}::jsonb)
81
- )
82
- DELETE FROM ontology_object_overrides AS overrides USING staged
83
- WHERE overrides.project_id = ${projectId}
84
- AND overrides.object_type_id = staged.value->>'objectTypeId'
85
- AND overrides.primary_id = staged.value->>'primaryId'
86
- AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
87
- RETURNING overrides.object_type_id
141
+ private async applyLinkOverrides(target: PgStagedPlanTarget): Promise<void> {
142
+ const { projectId, commitId } = target
143
+ // An edge is keyed by its whole link ref and targets it; a slot is keyed by its scope and
144
+ // targets the member its value names.
145
+ const identified = (
146
+ edges: MaterializationPlanWorkItem["kind"],
147
+ slots: MaterializationPlanWorkItem["kind"]
148
+ ) => this.sql`
149
+ SELECT 'edge' AS identity_kind,
150
+ jsonb_build_array(
151
+ value->'ref'->'source'->>'objectTypeId', value->'ref'->'source'->>'primaryId',
152
+ value->'ref'->>'linkId',
153
+ value->'ref'->'target'->>'objectTypeId', value->'ref'->'target'->>'primaryId'
154
+ ) AS identity_key,
155
+ value->'ref'->'target' AS target, value
156
+ FROM (${this.staged(target, edges)}) AS edges
157
+ UNION ALL
158
+ SELECT 'slot',
159
+ jsonb_build_array(
160
+ value->'ref'->'source'->>'objectTypeId', value->'ref'->'source'->>'primaryId',
161
+ value->'ref'->>'linkId'
162
+ ),
163
+ value->'value'->'target', value
164
+ FROM (${this.staged(target, slots)}) AS slots
165
+ `
166
+ const upserts = identified("link-override-upsert", "link-slot-override-upsert")
167
+ const writes = (inserts: boolean) => this.sql`
168
+ SELECT * FROM (${upserts}) AS upserts
169
+ WHERE (value->>'expectedLastCommitId' IS NULL) = ${inserts}
170
+ `
171
+ const deletes = identified("link-override-delete", "link-slot-override-delete")
172
+ const stored = this.sql`
173
+ SELECT 1 FROM ontology_link_overrides AS stored
174
+ WHERE stored.project_id = ${projectId}
175
+ AND stored.identity_kind = candidates.identity_kind
176
+ AND stored.identity_key = candidates.identity_key
177
+ `
178
+ const firstUnwritten = (source: typeof upserts) => async () => {
179
+ const [row] = await this.sql<{ readonly identity_kind: string }[]>`
180
+ SELECT identity_kind FROM (${source}) AS candidates
181
+ WHERE NOT EXISTS (${stored} AND stored.last_commit_id = ${commitId})
182
+ LIMIT 1
88
183
  `
89
- if (rows.length !== objectDeletes.length) throw objectOverrideConflict()
184
+ throw linkOverrideConflict(row?.identity_kind)
90
185
  }
91
-
92
- const linkUpserts = [
93
- ...chunk.overrides.links.edges.upserts.map((item) => ({
94
- identityKind: "edge" as const,
95
- identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
96
- sourceTypeId: item.ref.source.objectTypeId,
97
- sourcePrimaryId: item.ref.source.primaryId,
98
- linkId: item.ref.linkId,
99
- targetTypeId: item.ref.target.objectTypeId,
100
- targetPrimaryId: item.ref.target.primaryId,
101
- value: item.value,
102
- lastCommitId: item.lastCommitId,
103
- updatedAt: item.updatedAt,
104
- expectedLastCommitId: item.expectedLastCommitId,
105
- })),
106
- ...chunk.overrides.links.slots.upserts.map((item) => ({
107
- identityKind: "slot" as const,
108
- identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
109
- sourceTypeId: item.ref.source.objectTypeId,
110
- sourcePrimaryId: item.ref.source.primaryId,
111
- linkId: item.ref.linkId,
112
- targetTypeId: item.value.target.objectTypeId,
113
- targetPrimaryId: item.value.target.primaryId,
114
- value: item.value,
115
- lastCommitId: item.lastCommitId,
116
- updatedAt: item.updatedAt,
117
- expectedLastCommitId: item.expectedLastCommitId,
118
- })),
119
- ]
120
- const linkInserts = linkUpserts.filter((item) => item.expectedLastCommitId === null)
121
- if (linkInserts.length > 0) {
122
- const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
123
- WITH staged AS (
124
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkInserts)}::jsonb)
186
+ await this.write(
187
+ this.sql<WriteCounts[]>`
188
+ WITH candidates AS (${writes(true)}), written AS (
189
+ INSERT INTO ontology_link_overrides (
190
+ project_id, identity_kind, identity_key, source_type_id, source_primary_id, link_id,
191
+ target_type_id, target_primary_id, value, last_commit_id, updated_at
192
+ )
193
+ SELECT ${projectId}, identity_kind, identity_key,
194
+ value->'ref'->'source'->>'objectTypeId', value->'ref'->'source'->>'primaryId',
195
+ value->'ref'->>'linkId', target->>'objectTypeId', target->>'primaryId',
196
+ value->'value', value->>'lastCommitId', (value->>'updatedAt')::timestamptz
197
+ FROM candidates
198
+ ON CONFLICT DO NOTHING
199
+ RETURNING 1
125
200
  )
126
- INSERT INTO ontology_link_overrides (
127
- project_id, identity_kind, identity_key, source_type_id, source_primary_id, link_id,
128
- target_type_id, target_primary_id, value, last_commit_id, updated_at
201
+ ${counts(this.sql)}
202
+ `,
203
+ firstUnwritten(writes(true))
204
+ )
205
+ await this.write(
206
+ this.sql<WriteCounts[]>`
207
+ WITH candidates AS (${writes(false)}), written AS (
208
+ UPDATE ontology_link_overrides AS stored
209
+ SET target_type_id = candidates.target->>'objectTypeId',
210
+ target_primary_id = candidates.target->>'primaryId',
211
+ value = candidates.value->'value',
212
+ last_commit_id = candidates.value->>'lastCommitId',
213
+ updated_at = (candidates.value->>'updatedAt')::timestamptz
214
+ FROM candidates
215
+ WHERE stored.project_id = ${projectId}
216
+ AND stored.identity_kind = candidates.identity_kind
217
+ AND stored.identity_key = candidates.identity_key
218
+ AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
219
+ RETURNING 1
129
220
  )
130
- SELECT ${projectId}, value->>'identityKind', value->'identityKey',
131
- value->>'sourceTypeId', value->>'sourcePrimaryId', value->>'linkId',
132
- value->>'targetTypeId', value->>'targetPrimaryId', value->'value',
133
- value->>'lastCommitId', (value->>'updatedAt')::timestamptz
134
- FROM staged
135
- ON CONFLICT DO NOTHING
136
- RETURNING identity_kind
137
- `
138
- if (rows.length !== linkInserts.length)
139
- throw linkOverrideConflict(linkInserts[0]!.identityKind)
140
- }
141
-
142
- const linkUpdates = linkUpserts.filter((item) => item.expectedLastCommitId !== null)
143
- if (linkUpdates.length > 0) {
144
- const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
145
- WITH staged AS (
146
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkUpdates)}::jsonb)
147
- )
148
- UPDATE ontology_link_overrides AS overrides
149
- SET target_type_id = staged.value->>'targetTypeId',
150
- target_primary_id = staged.value->>'targetPrimaryId',
151
- value = staged.value->'value',
152
- last_commit_id = staged.value->>'lastCommitId',
153
- updated_at = (staged.value->>'updatedAt')::timestamptz
154
- FROM staged
155
- WHERE overrides.project_id = ${projectId}
156
- AND overrides.identity_kind = staged.value->>'identityKind'
157
- AND overrides.identity_key = staged.value->'identityKey'
158
- AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
159
- RETURNING overrides.identity_kind
160
- `
161
- if (rows.length !== linkUpdates.length)
162
- throw linkOverrideConflict(linkUpdates[0]!.identityKind)
163
- }
164
-
165
- const linkDeletes = [
166
- ...chunk.overrides.links.edges.deletes.map((item) => ({
167
- identityKind: "edge" as const,
168
- identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
169
- expectedLastCommitId: item.expectedLastCommitId,
170
- })),
171
- ...chunk.overrides.links.slots.deletes.map((item) => ({
172
- identityKind: "slot" as const,
173
- identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
174
- expectedLastCommitId: item.expectedLastCommitId,
175
- })),
176
- ]
177
- if (linkDeletes.length > 0) {
178
- const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
179
- WITH staged AS (
180
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkDeletes)}::jsonb)
221
+ ${counts(this.sql)}
222
+ `,
223
+ firstUnwritten(writes(false))
224
+ )
225
+ await this.write(
226
+ this.sql<WriteCounts[]>`
227
+ WITH candidates AS (${deletes}), written AS (
228
+ DELETE FROM ontology_link_overrides AS stored USING candidates
229
+ WHERE stored.project_id = ${projectId}
230
+ AND stored.identity_kind = candidates.identity_kind
231
+ AND stored.identity_key = candidates.identity_key
232
+ AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
233
+ RETURNING 1
181
234
  )
182
- DELETE FROM ontology_link_overrides AS overrides USING staged
183
- WHERE overrides.project_id = ${projectId}
184
- AND overrides.identity_kind = staged.value->>'identityKind'
185
- AND overrides.identity_key = staged.value->'identityKey'
186
- AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
187
- RETURNING overrides.identity_kind
188
- `
189
- if (rows.length !== linkDeletes.length)
190
- throw linkOverrideConflict(linkDeletes[0]!.identityKind)
191
- }
235
+ ${counts(this.sql)}
236
+ `,
237
+ async () => {
238
+ const [row] = await this.sql<{ readonly identity_kind: string }[]>`
239
+ SELECT identity_kind FROM (${deletes}) AS candidates WHERE EXISTS (${stored}) LIMIT 1
240
+ `
241
+ throw linkOverrideConflict(row?.identity_kind)
242
+ }
243
+ )
192
244
  }
193
245
 
194
- private async applyEffective(projectId: string, chunk: MaterializationPlanChunk): Promise<void> {
195
- const linkDeletes = chunk.effective.linkDeletes.map((item) => ({
196
- sourceTypeId: item.ref.source.objectTypeId,
197
- sourceId: item.ref.source.primaryId,
198
- linkId: item.ref.linkId,
199
- targetTypeId: item.ref.target.objectTypeId,
200
- targetId: item.ref.target.primaryId,
201
- expectedLastCommitId: item.expected.lastCommitId,
202
- }))
203
- if (linkDeletes.length > 0) {
204
- const rows = await this.sql<{ readonly source_type_id: string }[]>`
205
- WITH staged AS (
206
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkDeletes)}::jsonb)
246
+ private async applyPoints(target: PgStagedPlanTarget): Promise<void> {
247
+ const { projectId, commitId } = target
248
+ const points = (inserts: boolean) => this.sql`
249
+ SELECT value->'point'->'series'->'object'->>'objectTypeId' AS object_type_id,
250
+ value->'point'->'series'->'object'->>'primaryId' AS object_id,
251
+ value->'point'->'series'->>'propertyId' AS property_id,
252
+ value->'point'->'value' AS value, value->'point'->>'unit' AS unit,
253
+ (value->'point'->>'at')::timestamptz AS at,
254
+ value->'point'->>'lastCommitId' AS last_commit_id,
255
+ value->'expected'->>'lastCommitId' AS expected_last_commit_id,
256
+ value->'point' AS point
257
+ FROM (${this.staged(target, "point-upsert")}) AS staged
258
+ WHERE (value->'expected'->>'lastCommitId' IS NULL) = ${inserts}
259
+ `
260
+ // Both statements keep `timeseries_latest` on the newest point of each series.
261
+ const latest = this.sql`
262
+ latest_candidates AS (
263
+ SELECT DISTINCT ON (project_id, object_type_id, object_id, property_id)
264
+ project_id, object_type_id, object_id, property_id, value, unit, at, last_commit_id
265
+ FROM written
266
+ ORDER BY project_id, object_type_id, object_id, property_id, at DESC
267
+ ), latest AS (
268
+ INSERT INTO timeseries_latest (
269
+ project_id, object_type_id, object_id, property_id, value, unit, at, last_commit_id
207
270
  )
208
- DELETE FROM links AS effective USING staged
209
- WHERE effective.project_id = ${projectId}
210
- AND effective.source_type_id = staged.value->>'sourceTypeId'
211
- AND effective.source_id = staged.value->>'sourceId'
212
- AND effective.link_id = staged.value->>'linkId'
213
- AND effective.target_type_id = staged.value->>'targetTypeId'
214
- AND effective.target_id = staged.value->>'targetId'
215
- AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
216
- RETURNING effective.source_type_id
217
- `
218
- if (rows.length !== linkDeletes.length) throw effectiveLinkConflict(linkDeletes[0]!)
219
- }
220
-
221
- const objectDeletes = chunk.effective.objectDeletes.map((item) => ({
222
- objectTypeId: item.ref.objectTypeId,
223
- primaryId: item.ref.primaryId,
224
- expectedVersion: item.expected.version,
225
- expectedLastCommitId: item.expected.lastCommitId,
226
- }))
227
- if (objectDeletes.length > 0) {
228
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
229
- WITH staged AS (
230
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectDeletes)}::jsonb)
271
+ SELECT project_id, object_type_id, object_id, property_id, value, unit, at,
272
+ last_commit_id
273
+ FROM latest_candidates
274
+ ON CONFLICT (project_id, object_type_id, object_id, property_id)
275
+ DO UPDATE SET value = EXCLUDED.value, unit = EXCLUDED.unit, at = EXCLUDED.at,
276
+ last_commit_id = EXCLUDED.last_commit_id
277
+ WHERE EXCLUDED.at >= timeseries_latest.at
278
+ )
279
+ `
280
+ const firstUnwritten = (inserts: boolean) => async () => {
281
+ const [row] = await this.sql<{ readonly point: unknown }[]>`
282
+ SELECT point FROM (${points(inserts)}) AS candidates
283
+ WHERE NOT EXISTS (
284
+ SELECT 1 FROM timeseries AS stored
285
+ WHERE stored.project_id = ${projectId}
286
+ AND stored.object_type_id = candidates.object_type_id
287
+ AND stored.object_id = candidates.object_id
288
+ AND stored.property_id = candidates.property_id
289
+ AND stored.at = candidates.at
290
+ AND stored.last_commit_id = ${commitId}
231
291
  )
232
- DELETE FROM objects AS effective USING staged
233
- WHERE effective.project_id = ${projectId}
234
- AND effective.object_type_id = staged.value->>'objectTypeId'
235
- AND effective.primary_id = staged.value->>'primaryId'
236
- AND effective.version = (staged.value->>'expectedVersion')::integer
237
- AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
238
- RETURNING effective.object_type_id
292
+ LIMIT 1
239
293
  `
240
- if (rows.length !== objectDeletes.length) throw effectiveObjectConflict(objectDeletes[0]!)
294
+ throw pointConflict(row?.point)
241
295
  }
242
-
243
- await this.applyObjects(projectId, chunk.effective.objectUpserts)
244
- await this.applyLinks(projectId, chunk.effective.linkUpserts)
296
+ await this.write(
297
+ this.sql<WriteCounts[]>`
298
+ WITH candidates AS (${points(true)}), written AS (
299
+ INSERT INTO timeseries (
300
+ project_id, object_type_id, object_id, property_id, value, unit, at, last_commit_id
301
+ )
302
+ SELECT ${projectId}, object_type_id, object_id, property_id, value, unit, at,
303
+ last_commit_id
304
+ FROM candidates
305
+ ON CONFLICT DO NOTHING
306
+ RETURNING *
307
+ ), ${latest}
308
+ ${counts(this.sql)}
309
+ `,
310
+ firstUnwritten(true)
311
+ )
312
+ await this.write(
313
+ this.sql<WriteCounts[]>`
314
+ WITH candidates AS (${points(false)}), written AS (
315
+ UPDATE timeseries AS stored
316
+ SET value = candidates.value, unit = candidates.unit,
317
+ last_commit_id = candidates.last_commit_id
318
+ FROM candidates
319
+ WHERE stored.project_id = ${projectId}
320
+ AND stored.object_type_id = candidates.object_type_id
321
+ AND stored.object_id = candidates.object_id
322
+ AND stored.property_id = candidates.property_id
323
+ AND stored.at = candidates.at
324
+ AND stored.last_commit_id = candidates.expected_last_commit_id
325
+ RETURNING stored.*
326
+ ), ${latest}
327
+ ${counts(this.sql)}
328
+ `,
329
+ firstUnwritten(false)
330
+ )
245
331
  }
246
332
 
247
- private async applyObjects(
248
- projectId: string,
249
- items: MaterializationPlanChunk["effective"]["objectUpserts"]
250
- ): Promise<void> {
251
- const payload = items.map(({ row, expected }) => ({
252
- objectTypeId: row.ref.objectTypeId,
253
- primaryId: row.ref.primaryId,
254
- properties: row.properties,
255
- createdAt: row.createdAt,
256
- updatedAt: row.updatedAt,
257
- version: row.version,
258
- lastCommitId: row.lastCommitId,
259
- expectedExists: expected.exists,
260
- expectedVersion: expected.exists ? expected.version : null,
261
- expectedLastCommitId: expected.exists ? expected.lastCommitId : null,
262
- }))
263
- const inserts = payload.filter((item) => !item.expectedExists)
264
- if (inserts.length > 0) {
265
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
266
- WITH staged AS (
267
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
268
- )
269
- INSERT INTO objects (
270
- project_id, object_type_id, primary_id, properties, created_at,
271
- updated_at, version, last_commit_id
333
+ private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
334
+ const candidates = this.staged(target, "link-delete")
335
+ return this.write(
336
+ this.sql<WriteCounts[]>`
337
+ WITH candidates AS (${candidates}), written AS (
338
+ DELETE FROM links AS stored USING candidates
339
+ WHERE stored.project_id = ${target.projectId} AND ${this.matches("link", "value->'ref'")}
340
+ AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
341
+ RETURNING 1
272
342
  )
273
- SELECT ${projectId}, value->>'objectTypeId', value->>'primaryId',
274
- value->'properties', (value->>'createdAt')::timestamptz,
275
- (value->>'updatedAt')::timestamptz, (value->>'version')::integer,
276
- value->>'lastCommitId'
277
- FROM staged
278
- ON CONFLICT DO NOTHING
279
- RETURNING object_type_id
280
- `
281
- if (rows.length !== inserts.length) {
282
- throw effectiveConflict(`Expected object ${objectIdentityKey(inserts[0]!)} to be absent.`)
343
+ ${counts(this.sql)}
344
+ `,
345
+ async () => {
346
+ const ref = await this.firstConflict(target, candidates, "link", "value->'ref'", "delete")
347
+ throw effectiveLinkConflict(ref as OntologyLinkRef | undefined)
283
348
  }
284
- }
349
+ )
350
+ }
285
351
 
286
- const updates = payload.filter((item) => item.expectedExists)
287
- if (updates.length > 0) {
288
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
289
- WITH staged AS (
290
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, updates)}::jsonb)
352
+ private async deleteObjects(target: PgStagedPlanTarget): Promise<number> {
353
+ const candidates = this.staged(target, "object-delete")
354
+ return this.write(
355
+ this.sql<WriteCounts[]>`
356
+ WITH candidates AS (${candidates}), written AS (
357
+ DELETE FROM objects AS stored USING candidates
358
+ WHERE stored.project_id = ${target.projectId} AND ${this.matches("object", "value->'ref'")}
359
+ AND stored.version = (candidates.value->'expected'->>'version')::integer
360
+ AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
361
+ RETURNING 1
291
362
  )
292
- UPDATE objects AS effective
293
- SET properties = staged.value->'properties',
294
- created_at = (staged.value->>'createdAt')::timestamptz,
295
- updated_at = (staged.value->>'updatedAt')::timestamptz,
296
- version = (staged.value->>'version')::integer,
297
- last_commit_id = staged.value->>'lastCommitId'
298
- FROM staged
299
- WHERE effective.project_id = ${projectId}
300
- AND effective.object_type_id = staged.value->>'objectTypeId'
301
- AND effective.primary_id = staged.value->>'primaryId'
302
- AND effective.version = (staged.value->>'expectedVersion')::integer
303
- AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
304
- RETURNING effective.object_type_id
305
- `
306
- if (rows.length !== updates.length) throw effectiveObjectConflict(updates[0]!)
307
- }
363
+ ${counts(this.sql)}
364
+ `,
365
+ async () => {
366
+ const ref = await this.firstConflict(target, candidates, "object", "value->'ref'", "delete")
367
+ throw effectiveObjectConflict(ref as OntologyObjectRef | undefined)
368
+ }
369
+ )
308
370
  }
309
371
 
310
- private async applyLinks(
311
- projectId: string,
312
- items: MaterializationPlanChunk["effective"]["linkUpserts"]
313
- ): Promise<void> {
314
- const payload = items.map(({ row, expected }) => ({
315
- sourceTypeId: row.ref.source.objectTypeId,
316
- sourceId: row.ref.source.primaryId,
317
- linkId: row.ref.linkId,
318
- targetTypeId: row.ref.target.objectTypeId,
319
- targetId: row.ref.target.primaryId,
320
- ...(row.properties === undefined ? {} : { properties: row.properties }),
321
- createdAt: row.createdAt,
322
- updatedAt: row.updatedAt,
323
- lastCommitId: row.lastCommitId,
324
- expectedExists: expected.exists,
325
- expectedLastCommitId: expected.exists ? expected.lastCommitId : null,
326
- }))
327
- const inserts = payload.filter((item) => !item.expectedExists)
328
- if (inserts.length > 0) {
329
- const rows = await this.sql<{ readonly source_type_id: string }[]>`
330
- WITH staged AS (
331
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
372
+ private async upsertObjects(target: PgStagedPlanTarget): Promise<number> {
373
+ const { projectId } = target
374
+ const subset = (exists: boolean) => this.sql`
375
+ SELECT value FROM (${this.staged(target, "object-upsert")}) AS staged
376
+ WHERE ((value->'expected'->>'exists')::boolean IS TRUE) = ${exists}
377
+ `
378
+ const inserted = await this.write(
379
+ this.sql<WriteCounts[]>`
380
+ WITH candidates AS (${subset(false)}), written AS (
381
+ INSERT INTO objects (
382
+ project_id, object_type_id, primary_id, properties, created_at, updated_at,
383
+ version, last_commit_id
384
+ )
385
+ SELECT ${projectId}, value->'row'->'ref'->>'objectTypeId',
386
+ value->'row'->'ref'->>'primaryId', value->'row'->'properties',
387
+ (value->'row'->>'createdAt')::timestamptz, (value->'row'->>'updatedAt')::timestamptz,
388
+ (value->'row'->>'version')::integer, value->'row'->>'lastCommitId'
389
+ FROM candidates
390
+ ON CONFLICT DO NOTHING
391
+ RETURNING 1
332
392
  )
333
- INSERT INTO links (
334
- project_id, source_type_id, source_id, link_id, target_type_id, target_id,
335
- properties, created_at, updated_at, last_commit_id
393
+ ${counts(this.sql)}
394
+ `,
395
+ async () => {
396
+ const ref = await this.firstConflict(
397
+ target,
398
+ subset(false),
399
+ "object",
400
+ "value->'row'->'ref'",
401
+ "insert"
402
+ )
403
+ throw effectiveConflict(
404
+ `Expected ${objectLabel(ref as OntologyObjectRef | undefined)} to be absent.`
336
405
  )
337
- SELECT ${projectId}, value->>'sourceTypeId', value->>'sourceId', value->>'linkId',
338
- value->>'targetTypeId', value->>'targetId',
339
- CASE WHEN value ? 'properties' THEN value->'properties' ELSE NULL END,
340
- (value->>'createdAt')::timestamptz, (value->>'updatedAt')::timestamptz,
341
- value->>'lastCommitId'
342
- FROM staged
343
- ON CONFLICT DO NOTHING
344
- RETURNING source_type_id
345
- `
346
- if (rows.length !== inserts.length) {
347
- throw effectiveConflict(`Expected link ${linkIdentityKey(inserts[0]!)} to be absent.`)
348
406
  }
349
- }
350
-
351
- const updates = payload.filter((item) => item.expectedExists)
352
- if (updates.length > 0) {
353
- const rows = await this.sql<{ readonly source_type_id: string }[]>`
354
- WITH staged AS (
355
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, updates)}::jsonb)
407
+ )
408
+ const updated = await this.write(
409
+ this.sql<WriteCounts[]>`
410
+ WITH candidates AS (${subset(true)}), written AS (
411
+ UPDATE objects AS stored
412
+ SET properties = candidates.value->'row'->'properties',
413
+ created_at = (candidates.value->'row'->>'createdAt')::timestamptz,
414
+ updated_at = (candidates.value->'row'->>'updatedAt')::timestamptz,
415
+ version = (candidates.value->'row'->>'version')::integer,
416
+ last_commit_id = candidates.value->'row'->>'lastCommitId'
417
+ FROM candidates
418
+ WHERE stored.project_id = ${projectId}
419
+ AND ${this.matches("object", "value->'row'->'ref'")}
420
+ AND stored.version = (candidates.value->'expected'->>'version')::integer
421
+ AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
422
+ RETURNING 1
356
423
  )
357
- UPDATE links AS effective
358
- SET properties = CASE
359
- WHEN staged.value ? 'properties' THEN staged.value->'properties'
360
- ELSE NULL
361
- END,
362
- created_at = (staged.value->>'createdAt')::timestamptz,
363
- updated_at = (staged.value->>'updatedAt')::timestamptz,
364
- last_commit_id = staged.value->>'lastCommitId'
365
- FROM staged
366
- WHERE effective.project_id = ${projectId}
367
- AND effective.source_type_id = staged.value->>'sourceTypeId'
368
- AND effective.source_id = staged.value->>'sourceId'
369
- AND effective.link_id = staged.value->>'linkId'
370
- AND effective.target_type_id = staged.value->>'targetTypeId'
371
- AND effective.target_id = staged.value->>'targetId'
372
- AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
373
- RETURNING effective.source_type_id
374
- `
375
- if (rows.length !== updates.length) throw effectiveLinkConflict(updates[0]!)
376
- }
424
+ ${counts(this.sql)}
425
+ `,
426
+ async () => {
427
+ const ref = await this.firstConflict(
428
+ target,
429
+ subset(true),
430
+ "object",
431
+ "value->'row'->'ref'",
432
+ "update"
433
+ )
434
+ throw effectiveObjectConflict(ref as OntologyObjectRef | undefined)
435
+ }
436
+ )
437
+ return inserted + updated
377
438
  }
378
439
 
379
- private async applyTimeseries(projectId: string, chunk: MaterializationPlanChunk): Promise<void> {
380
- const payload = chunk.timeseries.pointUpserts.map(({ point, expected }) => ({
381
- objectTypeId: point.series.object.objectTypeId,
382
- objectId: point.series.object.primaryId,
383
- propertyId: point.series.propertyId,
384
- value: point.value,
385
- unit: point.unit ?? null,
386
- at: point.at,
387
- lastCommitId: point.lastCommitId,
388
- expectedLastCommitId: expected.lastCommitId,
389
- }))
390
- const inserts = payload.filter((item) => item.expectedLastCommitId === null)
391
- if (inserts.length > 0) {
392
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
393
- WITH staged AS (
394
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
395
- ), written AS (
396
- INSERT INTO timeseries (
397
- project_id, object_type_id, object_id, property_id,
398
- value, unit, at, last_commit_id
440
+ private async upsertLinks(target: PgStagedPlanTarget): Promise<number> {
441
+ const { projectId } = target
442
+ const subset = (exists: boolean) => this.sql`
443
+ SELECT value FROM (${this.staged(target, "link-upsert")}) AS staged
444
+ WHERE ((value->'expected'->>'exists')::boolean IS TRUE) = ${exists}
445
+ `
446
+ const inserted = await this.write(
447
+ this.sql<WriteCounts[]>`
448
+ WITH candidates AS (${subset(false)}), written AS (
449
+ INSERT INTO links (
450
+ project_id, source_type_id, source_id, link_id, target_type_id, target_id,
451
+ properties, created_at, updated_at, last_commit_id
399
452
  )
400
- SELECT ${projectId}, value->>'objectTypeId', value->>'objectId',
401
- value->>'propertyId', value->'value', value->>'unit',
402
- (value->>'at')::timestamptz, value->>'lastCommitId'
403
- FROM staged
453
+ SELECT ${projectId}, value->'row'->'ref'->'source'->>'objectTypeId',
454
+ value->'row'->'ref'->'source'->>'primaryId', value->'row'->'ref'->>'linkId',
455
+ value->'row'->'ref'->'target'->>'objectTypeId',
456
+ value->'row'->'ref'->'target'->>'primaryId', value->'row'->'properties',
457
+ (value->'row'->>'createdAt')::timestamptz, (value->'row'->>'updatedAt')::timestamptz,
458
+ value->'row'->>'lastCommitId'
459
+ FROM candidates
404
460
  ON CONFLICT DO NOTHING
405
- RETURNING *
406
- ), latest_candidates AS (
407
- SELECT DISTINCT ON (project_id, object_type_id, object_id, property_id)
408
- project_id, object_type_id, object_id, property_id,
409
- value, unit, at, last_commit_id
410
- FROM written
411
- ORDER BY project_id, object_type_id, object_id, property_id, at DESC
412
- ), latest AS (
413
- INSERT INTO timeseries_latest (
414
- project_id, object_type_id, object_id, property_id,
415
- value, unit, at, last_commit_id
416
- )
417
- SELECT project_id, object_type_id, object_id, property_id,
418
- value, unit, at, last_commit_id
419
- FROM latest_candidates
420
- ON CONFLICT (project_id, object_type_id, object_id, property_id)
421
- DO UPDATE SET value = EXCLUDED.value, unit = EXCLUDED.unit, at = EXCLUDED.at,
422
- last_commit_id = EXCLUDED.last_commit_id
423
- WHERE EXCLUDED.at >= timeseries_latest.at
424
- RETURNING object_type_id
461
+ RETURNING 1
425
462
  )
426
- SELECT object_type_id, (SELECT COUNT(*) FROM latest) AS latest_count FROM written
427
- `
428
- if (rows.length !== inserts.length) throw pointConflict(inserts[0]!)
429
- }
463
+ ${counts(this.sql)}
464
+ `,
465
+ async () => {
466
+ const ref = await this.firstConflict(
467
+ target,
468
+ subset(false),
469
+ "link",
470
+ "value->'row'->'ref'",
471
+ "insert"
472
+ )
473
+ throw effectiveConflict(
474
+ `Expected ${linkLabel(ref as OntologyLinkRef | undefined)} to be absent.`
475
+ )
476
+ }
477
+ )
478
+ const updated = await this.write(
479
+ this.sql<WriteCounts[]>`
480
+ WITH candidates AS (${subset(true)}), written AS (
481
+ UPDATE links AS stored
482
+ SET properties = candidates.value->'row'->'properties',
483
+ created_at = (candidates.value->'row'->>'createdAt')::timestamptz,
484
+ updated_at = (candidates.value->'row'->>'updatedAt')::timestamptz,
485
+ last_commit_id = candidates.value->'row'->>'lastCommitId'
486
+ FROM candidates
487
+ WHERE stored.project_id = ${projectId}
488
+ AND ${this.matches("link", "value->'row'->'ref'")}
489
+ AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
490
+ RETURNING 1
491
+ )
492
+ ${counts(this.sql)}
493
+ `,
494
+ async () => {
495
+ const ref = await this.firstConflict(
496
+ target,
497
+ subset(true),
498
+ "link",
499
+ "value->'row'->'ref'",
500
+ "update"
501
+ )
502
+ throw effectiveLinkConflict(ref as OntologyLinkRef | undefined)
503
+ }
504
+ )
505
+ return inserted + updated
506
+ }
430
507
 
431
- const updates = payload.filter((item) => item.expectedLastCommitId !== null)
432
- if (updates.length > 0) {
433
- const rows = await this.sql<{ readonly object_type_id: string }[]>`
434
- WITH staged AS (
435
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, updates)}::jsonb)
436
- ), written AS (
437
- UPDATE timeseries AS points
438
- SET value = staged.value->'value', unit = staged.value->>'unit',
439
- last_commit_id = staged.value->>'lastCommitId'
440
- FROM staged
441
- WHERE points.project_id = ${projectId}
442
- AND points.object_type_id = staged.value->>'objectTypeId'
443
- AND points.object_id = staged.value->>'objectId'
444
- AND points.property_id = staged.value->>'propertyId'
445
- AND points.at = (staged.value->>'at')::timestamptz
446
- AND points.last_commit_id = staged.value->>'expectedLastCommitId'
447
- RETURNING points.*
448
- ), latest_candidates AS (
449
- SELECT DISTINCT ON (project_id, object_type_id, object_id, property_id)
450
- project_id, object_type_id, object_id, property_id,
451
- value, unit, at, last_commit_id
452
- FROM written
453
- ORDER BY project_id, object_type_id, object_id, property_id, at DESC
454
- ), latest AS (
455
- INSERT INTO timeseries_latest (
456
- project_id, object_type_id, object_id, property_id,
457
- value, unit, at, last_commit_id
458
- )
459
- SELECT project_id, object_type_id, object_id, property_id,
460
- value, unit, at, last_commit_id
461
- FROM latest_candidates
462
- ON CONFLICT (project_id, object_type_id, object_id, property_id)
463
- DO UPDATE SET value = EXCLUDED.value, unit = EXCLUDED.unit, at = EXCLUDED.at,
464
- last_commit_id = EXCLUDED.last_commit_id
465
- WHERE EXCLUDED.at >= timeseries_latest.at
466
- RETURNING object_type_id
508
+ /**
509
+ * Writes the staged event drafts in canonical order. Event ids hash the canonical JSON of
510
+ * `[projectId, commitId, ordinal]`; the provider-fixed prefix is rendered here, the ordinal by
511
+ * PostgreSQL, so the outbox matches `createEventId` without one id crossing the wire.
512
+ */
513
+ private async writeOutbox(target: PgStagedPlanTarget): Promise<number> {
514
+ const prefix = `${canonicalJson([target.projectId, target.commitId]).slice(0, -1)},`
515
+ const [row] = await this.sql<WriteCounts[]>`
516
+ WITH candidates AS (
517
+ SELECT payload->'draft' AS draft,
518
+ row_number() OVER (
519
+ ORDER BY major_order, minor_order, sort_one, sort_two, record_key
520
+ ) - 1 AS ordinal
521
+ FROM ${this.sql(target.workTable)}
522
+ WHERE work_id = ${target.workId} AND lane = 'event'
523
+ ), identified AS (
524
+ SELECT draft, ordinal,
525
+ encode(sha256(convert_to(${prefix}::text || ordinal::text || ']', 'UTF8')), 'hex') AS id
526
+ FROM candidates
527
+ ), written AS (
528
+ INSERT INTO ontology_outbox (
529
+ project_id, id, commit_id, commit_ordinal, event,
530
+ available_at, attempts, lease_id, lease_expires_at,
531
+ published_at, last_failure, created_at
467
532
  )
468
- SELECT object_type_id, (SELECT COUNT(*) FROM latest) AS latest_count FROM written
469
- `
470
- if (rows.length !== updates.length) throw pointConflict(updates[0]!)
533
+ SELECT ${target.projectId}, id, ${target.commitId}, ordinal, draft,
534
+ ${target.committedAt}::timestamptz, 0, NULL, NULL, NULL, NULL,
535
+ ${target.committedAt}::timestamptz
536
+ FROM identified
537
+ ON CONFLICT DO NOTHING
538
+ RETURNING 1
539
+ )
540
+ ${counts(this.sql)}
541
+ `
542
+ const staged = Number(row?.staged ?? 0)
543
+ if (Number(row?.written ?? 0) !== staged) {
544
+ throw effectiveConflict(`Outbox events of commit '${target.commitId}' already exist.`)
471
545
  }
546
+ return staged
472
547
  }
473
548
 
474
- private async applyOutbox(
475
- projectId: string,
476
- commitId: string,
477
- chunk: MaterializationPlanChunk
478
- ): Promise<void> {
479
- if (chunk.outbox.length === 0) return
480
- const payload = chunk.outbox.map((item) => {
481
- assertTimestamp(item.availableAt, "Outbox availableAt")
482
- assertTimestamp(item.createdAt, "Outbox createdAt")
483
- return {
484
- id: item.envelope.id,
485
- commitOrdinal: item.envelope.commitOrdinal,
486
- envelope: item.envelope,
487
- availableAt: item.availableAt,
488
- createdAt: item.createdAt,
489
- }
490
- })
491
- const rows = await this.sql<{ readonly id: string }[]>`
492
- WITH staged AS (
493
- SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, payload)}::jsonb)
494
- )
495
- INSERT INTO ontology_outbox (
496
- project_id, id, commit_id, commit_ordinal, envelope,
497
- available_at, attempts, lease_id, lease_expires_at,
498
- published_at, last_failure, created_at
549
+ /**
550
+ * Records the entities whose plan inputs this commit changed, once each, as the in-memory
551
+ * `touchPlanItem` does: a replacement plan's refresh rechecks only those.
552
+ */
553
+ private async writeTouches(target: PgStagedPlanTarget): Promise<void> {
554
+ await this.sql`
555
+ INSERT INTO ontology_commit_touches (project_id, entity_kind, identity_key)
556
+ SELECT DISTINCT ${target.projectId}, entity.kind,
557
+ CASE entity.kind
558
+ WHEN 'object' THEN
559
+ concat('[', entity.ref->'objectTypeId', ',', entity.ref->'primaryId', ']')
560
+ WHEN 'scope' THEN concat('[', entity.ref->'source'->'objectTypeId', ',',
561
+ entity.ref->'source'->'primaryId', ',', entity.ref->'linkId', ']')
562
+ ELSE concat('[', entity.ref->'source'->'objectTypeId', ',',
563
+ entity.ref->'source'->'primaryId', ',', entity.ref->'linkId', ',',
564
+ entity.ref->'target'->'objectTypeId', ',', entity.ref->'target'->'primaryId', ']')
565
+ END
566
+ FROM ${this.sql(target.workTable)} AS work
567
+ CROSS JOIN LATERAL (
568
+ SELECT CASE work.payload->'item'->>'kind'
569
+ WHEN 'object-upsert' THEN 'object'
570
+ WHEN 'object-delete' THEN 'object'
571
+ WHEN 'object-override-upsert' THEN 'object'
572
+ WHEN 'object-override-delete' THEN 'object'
573
+ WHEN 'point-upsert' THEN 'object'
574
+ WHEN 'link-upsert' THEN 'link'
575
+ WHEN 'link-delete' THEN 'link'
576
+ WHEN 'link-override-upsert' THEN 'link'
577
+ WHEN 'link-override-delete' THEN 'link'
578
+ WHEN 'link-slot-override-upsert' THEN 'scope'
579
+ WHEN 'link-slot-override-delete' THEN 'scope'
580
+ END AS kind,
581
+ CASE work.payload->'item'->>'kind'
582
+ WHEN 'object-upsert' THEN work.payload->'item'->'value'->'row'->'ref'
583
+ WHEN 'link-upsert' THEN work.payload->'item'->'value'->'row'->'ref'
584
+ WHEN 'point-upsert' THEN work.payload->'item'->'value'->'point'->'series'->'object'
585
+ ELSE work.payload->'item'->'value'->'ref'
586
+ END AS ref
587
+ ) AS entity
588
+ WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
589
+ `
590
+ }
591
+
592
+ private async write(
593
+ statement: Promise<readonly WriteCounts[]>,
594
+ conflict: () => Promise<never>
595
+ ): Promise<number> {
596
+ const [row] = await statement
597
+ const written = Number(row?.written ?? 0)
598
+ if (written !== Number(row?.staged ?? 0)) await conflict()
599
+ return written
600
+ }
601
+
602
+ /** Matches a staged ref, at a static JSON path of `candidates.value`, to `stored`. */
603
+ private matches(kind: "object" | "link", path: string): Fragment {
604
+ const ref = `(candidates.${path})`
605
+ return this.sql.unsafe(
606
+ kind === "object"
607
+ ? `stored.object_type_id = ${ref}->>'objectTypeId'
608
+ AND stored.primary_id = ${ref}->>'primaryId'`
609
+ : `stored.source_type_id = ${ref}->'source'->>'objectTypeId'
610
+ AND stored.source_id = ${ref}->'source'->>'primaryId'
611
+ AND stored.link_id = ${ref}->>'linkId'
612
+ AND stored.target_type_id = ${ref}->'target'->>'objectTypeId'
613
+ AND stored.target_id = ${ref}->'target'->>'primaryId'`
614
+ )
615
+ }
616
+
617
+ /**
618
+ * The ref of the first staged entity whose stored row tells why a write missed it: it still
619
+ * exists after a delete, another commit wrote it before an insert, or an update left it without
620
+ * this commit. None when no stored row tells, as for a delete whose row is already gone.
621
+ */
622
+ private async firstConflict(
623
+ target: PgStagedPlanTarget,
624
+ candidates: Query,
625
+ kind: "object" | "link",
626
+ path: string,
627
+ missed: "delete" | "insert" | "update"
628
+ ): Promise<unknown> {
629
+ const table = kind === "object" ? "objects" : "links"
630
+ const revision =
631
+ missed === "insert"
632
+ ? this.sql`AND stored.last_commit_id <> ${target.commitId}`
633
+ : missed === "update"
634
+ ? this.sql`AND stored.last_commit_id = ${target.commitId}`
635
+ : this.sql``
636
+ const stored = this.sql`
637
+ EXISTS (
638
+ SELECT 1 FROM ${this.sql(table)} AS stored
639
+ WHERE stored.project_id = ${target.projectId} AND ${this.matches(kind, path)} ${revision}
499
640
  )
500
- SELECT ${projectId}, value->>'id', ${commitId},
501
- (value->>'commitOrdinal')::bigint, value->'envelope',
502
- (value->>'availableAt')::timestamptz, 0, NULL, NULL, NULL, NULL,
503
- (value->>'createdAt')::timestamptz
504
- FROM staged
505
- ON CONFLICT DO NOTHING
506
- RETURNING id
507
641
  `
508
- if (rows.length !== payload.length) {
509
- const inserted = new Set(rows.map((row) => row.id))
510
- const duplicate = payload.find((item) => !inserted.has(item.id)) ?? payload[0]!
511
- throw effectiveConflict(`Duplicate outbox event '${duplicate.id}'.`)
512
- }
642
+ const [row] = await this.sql<{ readonly ref: unknown }[]>`
643
+ SELECT ${this.sql.unsafe(`candidates.${path}`)} AS ref FROM (${candidates}) AS candidates
644
+ WHERE ${missed === "update" ? this.sql`NOT` : this.sql``} ${stored}
645
+ LIMIT 1
646
+ `
647
+ return row?.ref
513
648
  }
514
649
  }
515
650
 
651
+ function counts(sql: SQLClient) {
652
+ return sql`SELECT (SELECT COUNT(*) FROM candidates) AS staged,
653
+ (SELECT COUNT(*) FROM written) AS written`
654
+ }
655
+
516
656
  function objectOverrideConflict(): MaterializationConflictError {
517
657
  return effectiveConflict("Expected object override changed.")
518
658
  }
519
659
 
520
- function linkOverrideConflict(identityKind: "edge" | "slot"): MaterializationConflictError {
660
+ function linkOverrideConflict(identityKind: string | undefined): MaterializationConflictError {
521
661
  return effectiveConflict(
522
- identityKind === "edge"
523
- ? "Expected link edge override changed."
524
- : "Expected link slot override changed."
662
+ identityKind === "slot"
663
+ ? "Expected link slot override changed."
664
+ : "Expected link edge override changed."
525
665
  )
526
666
  }
527
667
 
528
- function objectIdentityKey(item: {
529
- readonly objectTypeId: string
530
- readonly primaryId: string
531
- }): string {
532
- return objectRefKey({ objectTypeId: item.objectTypeId, primaryId: item.primaryId })
668
+ function effectiveObjectConflict(ref: OntologyObjectRef | undefined): MaterializationConflictError {
669
+ return effectiveConflict(`Expected ${objectLabel(ref)} changed.`)
533
670
  }
534
671
 
535
- function linkIdentityKey(item: {
536
- readonly sourceTypeId: string
537
- readonly sourceId: string
538
- readonly linkId: string
539
- readonly targetTypeId: string
540
- readonly targetId: string
541
- }): string {
542
- return linkRefKey({
543
- source: { objectTypeId: item.sourceTypeId, primaryId: item.sourceId },
544
- linkId: item.linkId,
545
- target: { objectTypeId: item.targetTypeId, primaryId: item.targetId },
546
- })
672
+ function effectiveLinkConflict(ref: OntologyLinkRef | undefined): MaterializationConflictError {
673
+ return effectiveConflict(`Expected ${linkLabel(ref)} changed.`)
547
674
  }
548
675
 
549
- function effectiveObjectConflict(item: {
550
- readonly objectTypeId: string
551
- readonly primaryId: string
552
- }): MaterializationConflictError {
553
- return effectiveConflict(`Expected object ${objectIdentityKey(item)} changed.`)
676
+ function objectLabel(ref: OntologyObjectRef | undefined): string {
677
+ return ref ? `object ${objectRefKey(ref)}` : "object"
554
678
  }
555
679
 
556
- function effectiveLinkConflict(item: {
557
- readonly sourceTypeId: string
558
- readonly sourceId: string
559
- readonly linkId: string
560
- readonly targetTypeId: string
561
- readonly targetId: string
562
- }): MaterializationConflictError {
563
- return effectiveConflict(`Expected link ${linkIdentityKey(item)} changed.`)
680
+ function linkLabel(ref: OntologyLinkRef | undefined): string {
681
+ return ref ? `link ${linkRefKey(ref)}` : "link"
564
682
  }
565
683
 
566
- function pointConflict(item: {
567
- readonly objectTypeId: string
568
- readonly objectId: string
569
- readonly propertyId: string
570
- readonly at: string
571
- }): MaterializationConflictError {
684
+ function pointConflict(point: unknown): MaterializationConflictError {
685
+ const found = point as { readonly series: TelemetrySeriesRef; readonly at: string } | undefined
572
686
  return new MaterializationConflictError(
573
687
  "timeseries-point",
574
- `Telemetry point ${telemetryPointKey(
575
- {
576
- object: { objectTypeId: item.objectTypeId, primaryId: item.objectId },
577
- propertyId: item.propertyId,
578
- },
579
- item.at
580
- )} changed.`
688
+ found
689
+ ? `Telemetry point ${telemetryPointKey(found.series, found.at)} changed.`
690
+ : "Telemetry point changed."
581
691
  )
582
692
  }