@sixb/pg 0.1.15 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +125 -0
- package/dist/index.d.ts +15 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3202 -1474
- package/dist/index.js.map +32 -25
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/aggregate-admission.d.ts +10 -0
- package/dist/objects/aggregate-admission.d.ts.map +1 -0
- package/dist/objects/preparation-lock.d.ts +6 -0
- package/dist/objects/preparation-lock.d.ts.map +1 -0
- package/dist/objects/prepare-queries.d.ts +5 -0
- package/dist/objects/prepare-queries.d.ts.map +1 -0
- package/dist/objects/query-compiler.d.ts +19 -0
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +43 -0
- package/dist/objects/query-indexes.d.ts.map +1 -0
- package/dist/objects/reader.d.ts +7 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +1 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/text-count-index.d.ts +24 -0
- package/dist/objects/text-count-index.d.ts.map +1 -0
- package/dist/ontology-storage/index.d.ts +2 -0
- package/dist/ontology-storage/index.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +18 -38
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-state.d.ts +3 -26
- package/dist/ontology-storage/materialization-state.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +51 -9
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts +5 -7
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/ontology-storage/outbox.d.ts.map +1 -1
- package/dist/ontology-storage/replacement-plans.d.ts +36 -0
- package/dist/ontology-storage/replacement-plans.d.ts.map +1 -0
- package/dist/ontology-storage/shared.d.ts +11 -5
- package/dist/ontology-storage/shared.d.ts.map +1 -1
- package/dist/ontology-storage/source-cleanup.d.ts.map +1 -1
- package/dist/ontology-storage/source-roots.d.ts +14 -1
- package/dist/ontology-storage/source-roots.d.ts.map +1 -1
- package/dist/ontology-storage/sources.d.ts +1 -2
- package/dist/ontology-storage/sources.d.ts.map +1 -1
- package/dist/pg-ai-cost-storage.d.ts.map +1 -1
- package/dist/pg-ai-limit-storage.d.ts.map +1 -1
- package/dist/pg-ai-usage-storage.d.ts +2 -0
- package/dist/pg-ai-usage-storage.d.ts.map +1 -1
- package/dist/pg-execution-storage.d.ts +28 -0
- package/dist/pg-execution-storage.d.ts.map +1 -1
- package/dist/pg-file-download-grant-storage.d.ts +11 -0
- package/dist/pg-file-download-grant-storage.d.ts.map +1 -0
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-rules-storage.d.ts.map +1 -1
- package/dist/transactions.d.ts +6 -2
- package/dist/transactions.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/index.ts +53 -3
- package/src/migrations/053-replacement-plans.sql +72 -0
- package/src/migrations/054-slim-ontology-outbox.sql +108 -0
- package/src/migrations/055-object-query-preparation.sql +11 -0
- package/src/migrations/056-commit-touches.sql +41 -0
- package/src/migrations/057-audio-transcription.sql +10 -0
- package/src/migrations/058-reranking-model-kind.sql +11 -0
- package/src/migrations/059-file-download-grants.sql +16 -0
- package/src/migrations/060-drained-source-versions.sql +11 -0
- package/src/migrations.ts +20 -0
- package/src/objects/aggregate-admission.ts +21 -0
- package/src/objects/preparation-lock.ts +15 -0
- package/src/objects/prepare-queries.ts +181 -0
- package/src/objects/query-compiler.ts +405 -115
- package/src/objects/query-indexes.ts +348 -0
- package/src/objects/reader.ts +140 -51
- package/src/objects/storage.ts +15 -3
- package/src/objects/text-count-index.ts +455 -0
- package/src/ontology-storage/index.ts +3 -0
- package/src/ontology-storage/materialization-session.ts +116 -275
- package/src/ontology-storage/materialization-state.ts +8 -259
- package/src/ontology-storage/materialization-writer.ts +618 -508
- package/src/ontology-storage/materializations.ts +106 -289
- package/src/ontology-storage/outbox.ts +46 -15
- package/src/ontology-storage/replacement-plans.ts +1110 -0
- package/src/ontology-storage/shared.ts +27 -15
- package/src/ontology-storage/source-cleanup.ts +22 -10
- package/src/ontology-storage/source-roots.ts +35 -8
- package/src/ontology-storage/sources.ts +0 -30
- package/src/pg-ai-cost-storage.ts +12 -0
- package/src/pg-ai-limit-storage.ts +13 -2
- package/src/pg-ai-usage-storage.ts +8 -0
- package/src/pg-execution-storage.ts +8 -3
- package/src/pg-file-download-grant-storage.ts +76 -0
- package/src/pg-projection-run-storage.ts +6 -1
- package/src/pg-rules-storage.ts +7 -3
- package/src/transactions.ts +10 -3
|
@@ -1,582 +1,692 @@
|
|
|
1
1
|
import {
|
|
2
2
|
linkRefKey,
|
|
3
3
|
MaterializationConflictError,
|
|
4
|
+
type OntologyLinkRef,
|
|
5
|
+
type OntologyObjectRef,
|
|
4
6
|
objectRefKey,
|
|
7
|
+
type TelemetrySeriesRef,
|
|
5
8
|
telemetryPointKey,
|
|
6
9
|
} from "@sixb/core/internal/materialization"
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
10
|
+
import {
|
|
11
|
+
canonicalJson,
|
|
12
|
+
effectiveConflict,
|
|
13
|
+
materializationApplyPhase,
|
|
14
|
+
materializationPlanKindRank,
|
|
15
|
+
} from "@sixb/core/internal/ontology-storage-provider"
|
|
16
|
+
import type { MaterializationPlanWorkItem } from "@sixb/core/storage"
|
|
17
|
+
import type postgres from "postgres"
|
|
18
|
+
import type { SQL, SQLClient } from "../pg-client"
|
|
19
|
+
|
|
20
|
+
export interface PgStagedPlanTarget {
|
|
21
|
+
/** The work table and the id its rows carry: a session's temp table, or a durable plan. */
|
|
22
|
+
readonly workTable: string
|
|
23
|
+
readonly workId: string
|
|
24
|
+
readonly projectId: string
|
|
25
|
+
readonly commitId: string
|
|
26
|
+
readonly committedAt: string
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface PgAppliedPlan {
|
|
30
|
+
readonly objectWrites: number
|
|
31
|
+
readonly linkWrites: number
|
|
32
|
+
readonly eventCount: number
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface WriteCounts {
|
|
36
|
+
readonly staged: number | string
|
|
37
|
+
readonly written: number | string
|
|
38
|
+
}
|
|
11
39
|
|
|
12
|
-
|
|
40
|
+
type Fragment = ReturnType<SQL["unsafe"]>
|
|
41
|
+
type Query = postgres.PendingQuery<postgres.Row[]>
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Applies a session's staged plan where it is stored: each statement reads one item kind from the
|
|
45
|
+
* work table and writes it set-based, so planned rows never travel back through the client.
|
|
46
|
+
*
|
|
47
|
+
* Every write still checks the revision its plan expected. A statement that writes fewer rows than
|
|
48
|
+
* it staged is a conflict; only then does a second, indexed query find the entity to name, so the
|
|
49
|
+
* normal path never correlates the written rows back to the staged ones.
|
|
50
|
+
*/
|
|
13
51
|
export class PgMaterializationWriter {
|
|
14
52
|
constructor(private readonly sql: SQLClient) {}
|
|
15
53
|
|
|
16
|
-
async
|
|
17
|
-
await this.
|
|
18
|
-
await this.
|
|
19
|
-
await this.
|
|
20
|
-
await this.
|
|
54
|
+
async applyStaged(target: PgStagedPlanTarget): Promise<PgAppliedPlan> {
|
|
55
|
+
await this.applyObjectOverrides(target)
|
|
56
|
+
await this.applyLinkOverrides(target)
|
|
57
|
+
await this.applyPoints(target)
|
|
58
|
+
let linkWrites = await this.deleteLinks(target)
|
|
59
|
+
let objectWrites = await this.deleteObjects(target)
|
|
60
|
+
objectWrites += await this.upsertObjects(target)
|
|
61
|
+
linkWrites += await this.upsertLinks(target)
|
|
62
|
+
const eventCount = await this.writeOutbox(target)
|
|
63
|
+
await this.writeTouches(target)
|
|
64
|
+
return { objectWrites, linkWrites, eventCount }
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The staged values of one plan item kind, as `value`. */
|
|
68
|
+
private staged(target: PgStagedPlanTarget, kind: MaterializationPlanWorkItem["kind"]) {
|
|
69
|
+
return this.sql`
|
|
70
|
+
SELECT payload->'item'->'value' AS value
|
|
71
|
+
FROM ${this.sql(target.workTable)}
|
|
72
|
+
WHERE work_id = ${target.workId} AND lane = 'apply'
|
|
73
|
+
AND major_order = ${materializationApplyPhase(kind)}
|
|
74
|
+
AND minor_order = ${materializationPlanKindRank(kind)}
|
|
75
|
+
`
|
|
21
76
|
}
|
|
22
77
|
|
|
23
|
-
private async
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
value
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
78
|
+
private async applyObjectOverrides(target: PgStagedPlanTarget): Promise<void> {
|
|
79
|
+
const { projectId } = target
|
|
80
|
+
const inserts = this.sql`
|
|
81
|
+
SELECT value FROM (${this.staged(target, "object-override-upsert")}) AS staged
|
|
82
|
+
WHERE value->>'expectedLastCommitId' IS NULL
|
|
83
|
+
`
|
|
84
|
+
await this.write(
|
|
85
|
+
this.sql<WriteCounts[]>`
|
|
86
|
+
WITH candidates AS (${inserts}), written AS (
|
|
87
|
+
INSERT INTO ontology_object_overrides (
|
|
88
|
+
project_id, object_type_id, primary_id, value, edited_at, last_commit_id, updated_at
|
|
89
|
+
)
|
|
90
|
+
SELECT ${projectId}, value->'ref'->>'objectTypeId', value->'ref'->>'primaryId',
|
|
91
|
+
value->'value', value->'editedAt', value->>'lastCommitId',
|
|
92
|
+
(value->>'updatedAt')::timestamptz
|
|
93
|
+
FROM candidates
|
|
94
|
+
ON CONFLICT DO NOTHING
|
|
95
|
+
RETURNING 1
|
|
38
96
|
)
|
|
39
|
-
|
|
40
|
-
|
|
97
|
+
${counts(this.sql)}
|
|
98
|
+
`,
|
|
99
|
+
async () => {
|
|
100
|
+
throw objectOverrideConflict()
|
|
101
|
+
}
|
|
102
|
+
)
|
|
103
|
+
const updates = this.sql`
|
|
104
|
+
SELECT value FROM (${this.staged(target, "object-override-upsert")}) AS staged
|
|
105
|
+
WHERE value->>'expectedLastCommitId' IS NOT NULL
|
|
106
|
+
`
|
|
107
|
+
await this.write(
|
|
108
|
+
this.sql<WriteCounts[]>`
|
|
109
|
+
WITH candidates AS (${updates}), written AS (
|
|
110
|
+
UPDATE ontology_object_overrides AS stored
|
|
111
|
+
SET value = candidates.value->'value', edited_at = candidates.value->'editedAt',
|
|
112
|
+
last_commit_id = candidates.value->>'lastCommitId',
|
|
113
|
+
updated_at = (candidates.value->>'updatedAt')::timestamptz
|
|
114
|
+
FROM candidates
|
|
115
|
+
WHERE stored.project_id = ${projectId} AND ${this.matches("object", "value->'ref'")}
|
|
116
|
+
AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
|
|
117
|
+
RETURNING 1
|
|
41
118
|
)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
WITH staged AS (
|
|
56
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectUpdates)}::jsonb)
|
|
119
|
+
${counts(this.sql)}
|
|
120
|
+
`,
|
|
121
|
+
async () => {
|
|
122
|
+
throw objectOverrideConflict()
|
|
123
|
+
}
|
|
124
|
+
)
|
|
125
|
+
await this.write(
|
|
126
|
+
this.sql<WriteCounts[]>`
|
|
127
|
+
WITH candidates AS (${this.staged(target, "object-override-delete")}), written AS (
|
|
128
|
+
DELETE FROM ontology_object_overrides AS stored USING candidates
|
|
129
|
+
WHERE stored.project_id = ${projectId} AND ${this.matches("object", "value->'ref'")}
|
|
130
|
+
AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
|
|
131
|
+
RETURNING 1
|
|
57
132
|
)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
AND overrides.primary_id = staged.value->>'primaryId'
|
|
66
|
-
AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
67
|
-
RETURNING overrides.object_type_id
|
|
68
|
-
`
|
|
69
|
-
if (rows.length !== objectUpdates.length) throw objectOverrideConflict()
|
|
70
|
-
}
|
|
133
|
+
${counts(this.sql)}
|
|
134
|
+
`,
|
|
135
|
+
async () => {
|
|
136
|
+
throw objectOverrideConflict()
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
}
|
|
71
140
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
141
|
+
private async applyLinkOverrides(target: PgStagedPlanTarget): Promise<void> {
|
|
142
|
+
const { projectId, commitId } = target
|
|
143
|
+
// An edge is keyed by its whole link ref and targets it; a slot is keyed by its scope and
|
|
144
|
+
// targets the member its value names.
|
|
145
|
+
const identified = (
|
|
146
|
+
edges: MaterializationPlanWorkItem["kind"],
|
|
147
|
+
slots: MaterializationPlanWorkItem["kind"]
|
|
148
|
+
) => this.sql`
|
|
149
|
+
SELECT 'edge' AS identity_kind,
|
|
150
|
+
jsonb_build_array(
|
|
151
|
+
value->'ref'->'source'->>'objectTypeId', value->'ref'->'source'->>'primaryId',
|
|
152
|
+
value->'ref'->>'linkId',
|
|
153
|
+
value->'ref'->'target'->>'objectTypeId', value->'ref'->'target'->>'primaryId'
|
|
154
|
+
) AS identity_key,
|
|
155
|
+
value->'ref'->'target' AS target, value
|
|
156
|
+
FROM (${this.staged(target, edges)}) AS edges
|
|
157
|
+
UNION ALL
|
|
158
|
+
SELECT 'slot',
|
|
159
|
+
jsonb_build_array(
|
|
160
|
+
value->'ref'->'source'->>'objectTypeId', value->'ref'->'source'->>'primaryId',
|
|
161
|
+
value->'ref'->>'linkId'
|
|
162
|
+
),
|
|
163
|
+
value->'value'->'target', value
|
|
164
|
+
FROM (${this.staged(target, slots)}) AS slots
|
|
165
|
+
`
|
|
166
|
+
const upserts = identified("link-override-upsert", "link-slot-override-upsert")
|
|
167
|
+
const writes = (inserts: boolean) => this.sql`
|
|
168
|
+
SELECT * FROM (${upserts}) AS upserts
|
|
169
|
+
WHERE (value->>'expectedLastCommitId' IS NULL) = ${inserts}
|
|
170
|
+
`
|
|
171
|
+
const deletes = identified("link-override-delete", "link-slot-override-delete")
|
|
172
|
+
const stored = this.sql`
|
|
173
|
+
SELECT 1 FROM ontology_link_overrides AS stored
|
|
174
|
+
WHERE stored.project_id = ${projectId}
|
|
175
|
+
AND stored.identity_kind = candidates.identity_kind
|
|
176
|
+
AND stored.identity_key = candidates.identity_key
|
|
177
|
+
`
|
|
178
|
+
const firstUnwritten = (source: typeof upserts) => async () => {
|
|
179
|
+
const [row] = await this.sql<{ readonly identity_kind: string }[]>`
|
|
180
|
+
SELECT identity_kind FROM (${source}) AS candidates
|
|
181
|
+
WHERE NOT EXISTS (${stored} AND stored.last_commit_id = ${commitId})
|
|
182
|
+
LIMIT 1
|
|
88
183
|
`
|
|
89
|
-
|
|
184
|
+
throw linkOverrideConflict(row?.identity_kind)
|
|
90
185
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
})),
|
|
106
|
-
...chunk.overrides.links.slots.upserts.map((item) => ({
|
|
107
|
-
identityKind: "slot" as const,
|
|
108
|
-
identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
|
|
109
|
-
sourceTypeId: item.ref.source.objectTypeId,
|
|
110
|
-
sourcePrimaryId: item.ref.source.primaryId,
|
|
111
|
-
linkId: item.ref.linkId,
|
|
112
|
-
targetTypeId: item.value.target.objectTypeId,
|
|
113
|
-
targetPrimaryId: item.value.target.primaryId,
|
|
114
|
-
value: item.value,
|
|
115
|
-
lastCommitId: item.lastCommitId,
|
|
116
|
-
updatedAt: item.updatedAt,
|
|
117
|
-
expectedLastCommitId: item.expectedLastCommitId,
|
|
118
|
-
})),
|
|
119
|
-
]
|
|
120
|
-
const linkInserts = linkUpserts.filter((item) => item.expectedLastCommitId === null)
|
|
121
|
-
if (linkInserts.length > 0) {
|
|
122
|
-
const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
|
|
123
|
-
WITH staged AS (
|
|
124
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkInserts)}::jsonb)
|
|
186
|
+
await this.write(
|
|
187
|
+
this.sql<WriteCounts[]>`
|
|
188
|
+
WITH candidates AS (${writes(true)}), written AS (
|
|
189
|
+
INSERT INTO ontology_link_overrides (
|
|
190
|
+
project_id, identity_kind, identity_key, source_type_id, source_primary_id, link_id,
|
|
191
|
+
target_type_id, target_primary_id, value, last_commit_id, updated_at
|
|
192
|
+
)
|
|
193
|
+
SELECT ${projectId}, identity_kind, identity_key,
|
|
194
|
+
value->'ref'->'source'->>'objectTypeId', value->'ref'->'source'->>'primaryId',
|
|
195
|
+
value->'ref'->>'linkId', target->>'objectTypeId', target->>'primaryId',
|
|
196
|
+
value->'value', value->>'lastCommitId', (value->>'updatedAt')::timestamptz
|
|
197
|
+
FROM candidates
|
|
198
|
+
ON CONFLICT DO NOTHING
|
|
199
|
+
RETURNING 1
|
|
125
200
|
)
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
201
|
+
${counts(this.sql)}
|
|
202
|
+
`,
|
|
203
|
+
firstUnwritten(writes(true))
|
|
204
|
+
)
|
|
205
|
+
await this.write(
|
|
206
|
+
this.sql<WriteCounts[]>`
|
|
207
|
+
WITH candidates AS (${writes(false)}), written AS (
|
|
208
|
+
UPDATE ontology_link_overrides AS stored
|
|
209
|
+
SET target_type_id = candidates.target->>'objectTypeId',
|
|
210
|
+
target_primary_id = candidates.target->>'primaryId',
|
|
211
|
+
value = candidates.value->'value',
|
|
212
|
+
last_commit_id = candidates.value->>'lastCommitId',
|
|
213
|
+
updated_at = (candidates.value->>'updatedAt')::timestamptz
|
|
214
|
+
FROM candidates
|
|
215
|
+
WHERE stored.project_id = ${projectId}
|
|
216
|
+
AND stored.identity_kind = candidates.identity_kind
|
|
217
|
+
AND stored.identity_key = candidates.identity_key
|
|
218
|
+
AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
|
|
219
|
+
RETURNING 1
|
|
129
220
|
)
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
if (linkUpdates.length > 0) {
|
|
144
|
-
const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
|
|
145
|
-
WITH staged AS (
|
|
146
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkUpdates)}::jsonb)
|
|
147
|
-
)
|
|
148
|
-
UPDATE ontology_link_overrides AS overrides
|
|
149
|
-
SET target_type_id = staged.value->>'targetTypeId',
|
|
150
|
-
target_primary_id = staged.value->>'targetPrimaryId',
|
|
151
|
-
value = staged.value->'value',
|
|
152
|
-
last_commit_id = staged.value->>'lastCommitId',
|
|
153
|
-
updated_at = (staged.value->>'updatedAt')::timestamptz
|
|
154
|
-
FROM staged
|
|
155
|
-
WHERE overrides.project_id = ${projectId}
|
|
156
|
-
AND overrides.identity_kind = staged.value->>'identityKind'
|
|
157
|
-
AND overrides.identity_key = staged.value->'identityKey'
|
|
158
|
-
AND overrides.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
159
|
-
RETURNING overrides.identity_kind
|
|
160
|
-
`
|
|
161
|
-
if (rows.length !== linkUpdates.length)
|
|
162
|
-
throw linkOverrideConflict(linkUpdates[0]!.identityKind)
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
const linkDeletes = [
|
|
166
|
-
...chunk.overrides.links.edges.deletes.map((item) => ({
|
|
167
|
-
identityKind: "edge" as const,
|
|
168
|
-
identityKey: JSON.parse(linkRefKey(item.ref)) as unknown,
|
|
169
|
-
expectedLastCommitId: item.expectedLastCommitId,
|
|
170
|
-
})),
|
|
171
|
-
...chunk.overrides.links.slots.deletes.map((item) => ({
|
|
172
|
-
identityKind: "slot" as const,
|
|
173
|
-
identityKey: [item.ref.source.objectTypeId, item.ref.source.primaryId, item.ref.linkId],
|
|
174
|
-
expectedLastCommitId: item.expectedLastCommitId,
|
|
175
|
-
})),
|
|
176
|
-
]
|
|
177
|
-
if (linkDeletes.length > 0) {
|
|
178
|
-
const rows = await this.sql<{ readonly identity_kind: "edge" | "slot" }[]>`
|
|
179
|
-
WITH staged AS (
|
|
180
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, linkDeletes)}::jsonb)
|
|
221
|
+
${counts(this.sql)}
|
|
222
|
+
`,
|
|
223
|
+
firstUnwritten(writes(false))
|
|
224
|
+
)
|
|
225
|
+
await this.write(
|
|
226
|
+
this.sql<WriteCounts[]>`
|
|
227
|
+
WITH candidates AS (${deletes}), written AS (
|
|
228
|
+
DELETE FROM ontology_link_overrides AS stored USING candidates
|
|
229
|
+
WHERE stored.project_id = ${projectId}
|
|
230
|
+
AND stored.identity_kind = candidates.identity_kind
|
|
231
|
+
AND stored.identity_key = candidates.identity_key
|
|
232
|
+
AND stored.last_commit_id = candidates.value->>'expectedLastCommitId'
|
|
233
|
+
RETURNING 1
|
|
181
234
|
)
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
}
|
|
235
|
+
${counts(this.sql)}
|
|
236
|
+
`,
|
|
237
|
+
async () => {
|
|
238
|
+
const [row] = await this.sql<{ readonly identity_kind: string }[]>`
|
|
239
|
+
SELECT identity_kind FROM (${deletes}) AS candidates WHERE EXISTS (${stored}) LIMIT 1
|
|
240
|
+
`
|
|
241
|
+
throw linkOverrideConflict(row?.identity_kind)
|
|
242
|
+
}
|
|
243
|
+
)
|
|
192
244
|
}
|
|
193
245
|
|
|
194
|
-
private async
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
246
|
+
private async applyPoints(target: PgStagedPlanTarget): Promise<void> {
|
|
247
|
+
const { projectId, commitId } = target
|
|
248
|
+
const points = (inserts: boolean) => this.sql`
|
|
249
|
+
SELECT value->'point'->'series'->'object'->>'objectTypeId' AS object_type_id,
|
|
250
|
+
value->'point'->'series'->'object'->>'primaryId' AS object_id,
|
|
251
|
+
value->'point'->'series'->>'propertyId' AS property_id,
|
|
252
|
+
value->'point'->'value' AS value, value->'point'->>'unit' AS unit,
|
|
253
|
+
(value->'point'->>'at')::timestamptz AS at,
|
|
254
|
+
value->'point'->>'lastCommitId' AS last_commit_id,
|
|
255
|
+
value->'expected'->>'lastCommitId' AS expected_last_commit_id,
|
|
256
|
+
value->'point' AS point
|
|
257
|
+
FROM (${this.staged(target, "point-upsert")}) AS staged
|
|
258
|
+
WHERE (value->'expected'->>'lastCommitId' IS NULL) = ${inserts}
|
|
259
|
+
`
|
|
260
|
+
// Both statements keep `timeseries_latest` on the newest point of each series.
|
|
261
|
+
const latest = this.sql`
|
|
262
|
+
latest_candidates AS (
|
|
263
|
+
SELECT DISTINCT ON (project_id, object_type_id, object_id, property_id)
|
|
264
|
+
project_id, object_type_id, object_id, property_id, value, unit, at, last_commit_id
|
|
265
|
+
FROM written
|
|
266
|
+
ORDER BY project_id, object_type_id, object_id, property_id, at DESC
|
|
267
|
+
), latest AS (
|
|
268
|
+
INSERT INTO timeseries_latest (
|
|
269
|
+
project_id, object_type_id, object_id, property_id, value, unit, at, last_commit_id
|
|
207
270
|
)
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
const rows = await this.sql<{ readonly object_type_id: string }[]>`
|
|
229
|
-
WITH staged AS (
|
|
230
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, objectDeletes)}::jsonb)
|
|
271
|
+
SELECT project_id, object_type_id, object_id, property_id, value, unit, at,
|
|
272
|
+
last_commit_id
|
|
273
|
+
FROM latest_candidates
|
|
274
|
+
ON CONFLICT (project_id, object_type_id, object_id, property_id)
|
|
275
|
+
DO UPDATE SET value = EXCLUDED.value, unit = EXCLUDED.unit, at = EXCLUDED.at,
|
|
276
|
+
last_commit_id = EXCLUDED.last_commit_id
|
|
277
|
+
WHERE EXCLUDED.at >= timeseries_latest.at
|
|
278
|
+
)
|
|
279
|
+
`
|
|
280
|
+
const firstUnwritten = (inserts: boolean) => async () => {
|
|
281
|
+
const [row] = await this.sql<{ readonly point: unknown }[]>`
|
|
282
|
+
SELECT point FROM (${points(inserts)}) AS candidates
|
|
283
|
+
WHERE NOT EXISTS (
|
|
284
|
+
SELECT 1 FROM timeseries AS stored
|
|
285
|
+
WHERE stored.project_id = ${projectId}
|
|
286
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
287
|
+
AND stored.object_id = candidates.object_id
|
|
288
|
+
AND stored.property_id = candidates.property_id
|
|
289
|
+
AND stored.at = candidates.at
|
|
290
|
+
AND stored.last_commit_id = ${commitId}
|
|
231
291
|
)
|
|
232
|
-
|
|
233
|
-
WHERE effective.project_id = ${projectId}
|
|
234
|
-
AND effective.object_type_id = staged.value->>'objectTypeId'
|
|
235
|
-
AND effective.primary_id = staged.value->>'primaryId'
|
|
236
|
-
AND effective.version = (staged.value->>'expectedVersion')::integer
|
|
237
|
-
AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
238
|
-
RETURNING effective.object_type_id
|
|
292
|
+
LIMIT 1
|
|
239
293
|
`
|
|
240
|
-
|
|
294
|
+
throw pointConflict(row?.point)
|
|
241
295
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
296
|
+
await this.write(
|
|
297
|
+
this.sql<WriteCounts[]>`
|
|
298
|
+
WITH candidates AS (${points(true)}), written AS (
|
|
299
|
+
INSERT INTO timeseries (
|
|
300
|
+
project_id, object_type_id, object_id, property_id, value, unit, at, last_commit_id
|
|
301
|
+
)
|
|
302
|
+
SELECT ${projectId}, object_type_id, object_id, property_id, value, unit, at,
|
|
303
|
+
last_commit_id
|
|
304
|
+
FROM candidates
|
|
305
|
+
ON CONFLICT DO NOTHING
|
|
306
|
+
RETURNING *
|
|
307
|
+
), ${latest}
|
|
308
|
+
${counts(this.sql)}
|
|
309
|
+
`,
|
|
310
|
+
firstUnwritten(true)
|
|
311
|
+
)
|
|
312
|
+
await this.write(
|
|
313
|
+
this.sql<WriteCounts[]>`
|
|
314
|
+
WITH candidates AS (${points(false)}), written AS (
|
|
315
|
+
UPDATE timeseries AS stored
|
|
316
|
+
SET value = candidates.value, unit = candidates.unit,
|
|
317
|
+
last_commit_id = candidates.last_commit_id
|
|
318
|
+
FROM candidates
|
|
319
|
+
WHERE stored.project_id = ${projectId}
|
|
320
|
+
AND stored.object_type_id = candidates.object_type_id
|
|
321
|
+
AND stored.object_id = candidates.object_id
|
|
322
|
+
AND stored.property_id = candidates.property_id
|
|
323
|
+
AND stored.at = candidates.at
|
|
324
|
+
AND stored.last_commit_id = candidates.expected_last_commit_id
|
|
325
|
+
RETURNING stored.*
|
|
326
|
+
), ${latest}
|
|
327
|
+
${counts(this.sql)}
|
|
328
|
+
`,
|
|
329
|
+
firstUnwritten(false)
|
|
330
|
+
)
|
|
245
331
|
}
|
|
246
332
|
|
|
247
|
-
private async
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
updatedAt: row.updatedAt,
|
|
257
|
-
version: row.version,
|
|
258
|
-
lastCommitId: row.lastCommitId,
|
|
259
|
-
expectedExists: expected.exists,
|
|
260
|
-
expectedVersion: expected.exists ? expected.version : null,
|
|
261
|
-
expectedLastCommitId: expected.exists ? expected.lastCommitId : null,
|
|
262
|
-
}))
|
|
263
|
-
const inserts = payload.filter((item) => !item.expectedExists)
|
|
264
|
-
if (inserts.length > 0) {
|
|
265
|
-
const rows = await this.sql<{ readonly object_type_id: string }[]>`
|
|
266
|
-
WITH staged AS (
|
|
267
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
|
|
268
|
-
)
|
|
269
|
-
INSERT INTO objects (
|
|
270
|
-
project_id, object_type_id, primary_id, properties, created_at,
|
|
271
|
-
updated_at, version, last_commit_id
|
|
333
|
+
private async deleteLinks(target: PgStagedPlanTarget): Promise<number> {
|
|
334
|
+
const candidates = this.staged(target, "link-delete")
|
|
335
|
+
return this.write(
|
|
336
|
+
this.sql<WriteCounts[]>`
|
|
337
|
+
WITH candidates AS (${candidates}), written AS (
|
|
338
|
+
DELETE FROM links AS stored USING candidates
|
|
339
|
+
WHERE stored.project_id = ${target.projectId} AND ${this.matches("link", "value->'ref'")}
|
|
340
|
+
AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
|
|
341
|
+
RETURNING 1
|
|
272
342
|
)
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
ON CONFLICT DO NOTHING
|
|
279
|
-
RETURNING object_type_id
|
|
280
|
-
`
|
|
281
|
-
if (rows.length !== inserts.length) {
|
|
282
|
-
throw effectiveConflict(`Expected object ${objectIdentityKey(inserts[0]!)} to be absent.`)
|
|
343
|
+
${counts(this.sql)}
|
|
344
|
+
`,
|
|
345
|
+
async () => {
|
|
346
|
+
const ref = await this.firstConflict(target, candidates, "link", "value->'ref'", "delete")
|
|
347
|
+
throw effectiveLinkConflict(ref as OntologyLinkRef | undefined)
|
|
283
348
|
}
|
|
284
|
-
|
|
349
|
+
)
|
|
350
|
+
}
|
|
285
351
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
352
|
+
private async deleteObjects(target: PgStagedPlanTarget): Promise<number> {
|
|
353
|
+
const candidates = this.staged(target, "object-delete")
|
|
354
|
+
return this.write(
|
|
355
|
+
this.sql<WriteCounts[]>`
|
|
356
|
+
WITH candidates AS (${candidates}), written AS (
|
|
357
|
+
DELETE FROM objects AS stored USING candidates
|
|
358
|
+
WHERE stored.project_id = ${target.projectId} AND ${this.matches("object", "value->'ref'")}
|
|
359
|
+
AND stored.version = (candidates.value->'expected'->>'version')::integer
|
|
360
|
+
AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
|
|
361
|
+
RETURNING 1
|
|
291
362
|
)
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
WHERE effective.project_id = ${projectId}
|
|
300
|
-
AND effective.object_type_id = staged.value->>'objectTypeId'
|
|
301
|
-
AND effective.primary_id = staged.value->>'primaryId'
|
|
302
|
-
AND effective.version = (staged.value->>'expectedVersion')::integer
|
|
303
|
-
AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
304
|
-
RETURNING effective.object_type_id
|
|
305
|
-
`
|
|
306
|
-
if (rows.length !== updates.length) throw effectiveObjectConflict(updates[0]!)
|
|
307
|
-
}
|
|
363
|
+
${counts(this.sql)}
|
|
364
|
+
`,
|
|
365
|
+
async () => {
|
|
366
|
+
const ref = await this.firstConflict(target, candidates, "object", "value->'ref'", "delete")
|
|
367
|
+
throw effectiveObjectConflict(ref as OntologyObjectRef | undefined)
|
|
368
|
+
}
|
|
369
|
+
)
|
|
308
370
|
}
|
|
309
371
|
|
|
310
|
-
private async
|
|
311
|
-
projectId
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
WITH staged AS (
|
|
331
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
|
|
372
|
+
private async upsertObjects(target: PgStagedPlanTarget): Promise<number> {
|
|
373
|
+
const { projectId } = target
|
|
374
|
+
const subset = (exists: boolean) => this.sql`
|
|
375
|
+
SELECT value FROM (${this.staged(target, "object-upsert")}) AS staged
|
|
376
|
+
WHERE ((value->'expected'->>'exists')::boolean IS TRUE) = ${exists}
|
|
377
|
+
`
|
|
378
|
+
const inserted = await this.write(
|
|
379
|
+
this.sql<WriteCounts[]>`
|
|
380
|
+
WITH candidates AS (${subset(false)}), written AS (
|
|
381
|
+
INSERT INTO objects (
|
|
382
|
+
project_id, object_type_id, primary_id, properties, created_at, updated_at,
|
|
383
|
+
version, last_commit_id
|
|
384
|
+
)
|
|
385
|
+
SELECT ${projectId}, value->'row'->'ref'->>'objectTypeId',
|
|
386
|
+
value->'row'->'ref'->>'primaryId', value->'row'->'properties',
|
|
387
|
+
(value->'row'->>'createdAt')::timestamptz, (value->'row'->>'updatedAt')::timestamptz,
|
|
388
|
+
(value->'row'->>'version')::integer, value->'row'->>'lastCommitId'
|
|
389
|
+
FROM candidates
|
|
390
|
+
ON CONFLICT DO NOTHING
|
|
391
|
+
RETURNING 1
|
|
332
392
|
)
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
393
|
+
${counts(this.sql)}
|
|
394
|
+
`,
|
|
395
|
+
async () => {
|
|
396
|
+
const ref = await this.firstConflict(
|
|
397
|
+
target,
|
|
398
|
+
subset(false),
|
|
399
|
+
"object",
|
|
400
|
+
"value->'row'->'ref'",
|
|
401
|
+
"insert"
|
|
402
|
+
)
|
|
403
|
+
throw effectiveConflict(
|
|
404
|
+
`Expected ${objectLabel(ref as OntologyObjectRef | undefined)} to be absent.`
|
|
336
405
|
)
|
|
337
|
-
SELECT ${projectId}, value->>'sourceTypeId', value->>'sourceId', value->>'linkId',
|
|
338
|
-
value->>'targetTypeId', value->>'targetId',
|
|
339
|
-
CASE WHEN value ? 'properties' THEN value->'properties' ELSE NULL END,
|
|
340
|
-
(value->>'createdAt')::timestamptz, (value->>'updatedAt')::timestamptz,
|
|
341
|
-
value->>'lastCommitId'
|
|
342
|
-
FROM staged
|
|
343
|
-
ON CONFLICT DO NOTHING
|
|
344
|
-
RETURNING source_type_id
|
|
345
|
-
`
|
|
346
|
-
if (rows.length !== inserts.length) {
|
|
347
|
-
throw effectiveConflict(`Expected link ${linkIdentityKey(inserts[0]!)} to be absent.`)
|
|
348
406
|
}
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
407
|
+
)
|
|
408
|
+
const updated = await this.write(
|
|
409
|
+
this.sql<WriteCounts[]>`
|
|
410
|
+
WITH candidates AS (${subset(true)}), written AS (
|
|
411
|
+
UPDATE objects AS stored
|
|
412
|
+
SET properties = candidates.value->'row'->'properties',
|
|
413
|
+
created_at = (candidates.value->'row'->>'createdAt')::timestamptz,
|
|
414
|
+
updated_at = (candidates.value->'row'->>'updatedAt')::timestamptz,
|
|
415
|
+
version = (candidates.value->'row'->>'version')::integer,
|
|
416
|
+
last_commit_id = candidates.value->'row'->>'lastCommitId'
|
|
417
|
+
FROM candidates
|
|
418
|
+
WHERE stored.project_id = ${projectId}
|
|
419
|
+
AND ${this.matches("object", "value->'row'->'ref'")}
|
|
420
|
+
AND stored.version = (candidates.value->'expected'->>'version')::integer
|
|
421
|
+
AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
|
|
422
|
+
RETURNING 1
|
|
356
423
|
)
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
AND effective.target_id = staged.value->>'targetId'
|
|
372
|
-
AND effective.last_commit_id = staged.value->>'expectedLastCommitId'
|
|
373
|
-
RETURNING effective.source_type_id
|
|
374
|
-
`
|
|
375
|
-
if (rows.length !== updates.length) throw effectiveLinkConflict(updates[0]!)
|
|
376
|
-
}
|
|
424
|
+
${counts(this.sql)}
|
|
425
|
+
`,
|
|
426
|
+
async () => {
|
|
427
|
+
const ref = await this.firstConflict(
|
|
428
|
+
target,
|
|
429
|
+
subset(true),
|
|
430
|
+
"object",
|
|
431
|
+
"value->'row'->'ref'",
|
|
432
|
+
"update"
|
|
433
|
+
)
|
|
434
|
+
throw effectiveObjectConflict(ref as OntologyObjectRef | undefined)
|
|
435
|
+
}
|
|
436
|
+
)
|
|
437
|
+
return inserted + updated
|
|
377
438
|
}
|
|
378
439
|
|
|
379
|
-
private async
|
|
380
|
-
const
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
if (inserts.length > 0) {
|
|
392
|
-
const rows = await this.sql<{ readonly object_type_id: string }[]>`
|
|
393
|
-
WITH staged AS (
|
|
394
|
-
SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, inserts)}::jsonb)
|
|
395
|
-
), written AS (
|
|
396
|
-
INSERT INTO timeseries (
|
|
397
|
-
project_id, object_type_id, object_id, property_id,
|
|
398
|
-
value, unit, at, last_commit_id
|
|
440
|
+
private async upsertLinks(target: PgStagedPlanTarget): Promise<number> {
|
|
441
|
+
const { projectId } = target
|
|
442
|
+
const subset = (exists: boolean) => this.sql`
|
|
443
|
+
SELECT value FROM (${this.staged(target, "link-upsert")}) AS staged
|
|
444
|
+
WHERE ((value->'expected'->>'exists')::boolean IS TRUE) = ${exists}
|
|
445
|
+
`
|
|
446
|
+
const inserted = await this.write(
|
|
447
|
+
this.sql<WriteCounts[]>`
|
|
448
|
+
WITH candidates AS (${subset(false)}), written AS (
|
|
449
|
+
INSERT INTO links (
|
|
450
|
+
project_id, source_type_id, source_id, link_id, target_type_id, target_id,
|
|
451
|
+
properties, created_at, updated_at, last_commit_id
|
|
399
452
|
)
|
|
400
|
-
SELECT ${projectId}, value
|
|
401
|
-
value->>'
|
|
402
|
-
|
|
403
|
-
|
|
453
|
+
SELECT ${projectId}, value->'row'->'ref'->'source'->>'objectTypeId',
|
|
454
|
+
value->'row'->'ref'->'source'->>'primaryId', value->'row'->'ref'->>'linkId',
|
|
455
|
+
value->'row'->'ref'->'target'->>'objectTypeId',
|
|
456
|
+
value->'row'->'ref'->'target'->>'primaryId', value->'row'->'properties',
|
|
457
|
+
(value->'row'->>'createdAt')::timestamptz, (value->'row'->>'updatedAt')::timestamptz,
|
|
458
|
+
value->'row'->>'lastCommitId'
|
|
459
|
+
FROM candidates
|
|
404
460
|
ON CONFLICT DO NOTHING
|
|
405
|
-
RETURNING
|
|
406
|
-
), latest_candidates AS (
|
|
407
|
-
SELECT DISTINCT ON (project_id, object_type_id, object_id, property_id)
|
|
408
|
-
project_id, object_type_id, object_id, property_id,
|
|
409
|
-
value, unit, at, last_commit_id
|
|
410
|
-
FROM written
|
|
411
|
-
ORDER BY project_id, object_type_id, object_id, property_id, at DESC
|
|
412
|
-
), latest AS (
|
|
413
|
-
INSERT INTO timeseries_latest (
|
|
414
|
-
project_id, object_type_id, object_id, property_id,
|
|
415
|
-
value, unit, at, last_commit_id
|
|
416
|
-
)
|
|
417
|
-
SELECT project_id, object_type_id, object_id, property_id,
|
|
418
|
-
value, unit, at, last_commit_id
|
|
419
|
-
FROM latest_candidates
|
|
420
|
-
ON CONFLICT (project_id, object_type_id, object_id, property_id)
|
|
421
|
-
DO UPDATE SET value = EXCLUDED.value, unit = EXCLUDED.unit, at = EXCLUDED.at,
|
|
422
|
-
last_commit_id = EXCLUDED.last_commit_id
|
|
423
|
-
WHERE EXCLUDED.at >= timeseries_latest.at
|
|
424
|
-
RETURNING object_type_id
|
|
461
|
+
RETURNING 1
|
|
425
462
|
)
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
463
|
+
${counts(this.sql)}
|
|
464
|
+
`,
|
|
465
|
+
async () => {
|
|
466
|
+
const ref = await this.firstConflict(
|
|
467
|
+
target,
|
|
468
|
+
subset(false),
|
|
469
|
+
"link",
|
|
470
|
+
"value->'row'->'ref'",
|
|
471
|
+
"insert"
|
|
472
|
+
)
|
|
473
|
+
throw effectiveConflict(
|
|
474
|
+
`Expected ${linkLabel(ref as OntologyLinkRef | undefined)} to be absent.`
|
|
475
|
+
)
|
|
476
|
+
}
|
|
477
|
+
)
|
|
478
|
+
const updated = await this.write(
|
|
479
|
+
this.sql<WriteCounts[]>`
|
|
480
|
+
WITH candidates AS (${subset(true)}), written AS (
|
|
481
|
+
UPDATE links AS stored
|
|
482
|
+
SET properties = candidates.value->'row'->'properties',
|
|
483
|
+
created_at = (candidates.value->'row'->>'createdAt')::timestamptz,
|
|
484
|
+
updated_at = (candidates.value->'row'->>'updatedAt')::timestamptz,
|
|
485
|
+
last_commit_id = candidates.value->'row'->>'lastCommitId'
|
|
486
|
+
FROM candidates
|
|
487
|
+
WHERE stored.project_id = ${projectId}
|
|
488
|
+
AND ${this.matches("link", "value->'row'->'ref'")}
|
|
489
|
+
AND stored.last_commit_id = candidates.value->'expected'->>'lastCommitId'
|
|
490
|
+
RETURNING 1
|
|
491
|
+
)
|
|
492
|
+
${counts(this.sql)}
|
|
493
|
+
`,
|
|
494
|
+
async () => {
|
|
495
|
+
const ref = await this.firstConflict(
|
|
496
|
+
target,
|
|
497
|
+
subset(true),
|
|
498
|
+
"link",
|
|
499
|
+
"value->'row'->'ref'",
|
|
500
|
+
"update"
|
|
501
|
+
)
|
|
502
|
+
throw effectiveLinkConflict(ref as OntologyLinkRef | undefined)
|
|
503
|
+
}
|
|
504
|
+
)
|
|
505
|
+
return inserted + updated
|
|
506
|
+
}
|
|
430
507
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
INSERT INTO timeseries_latest (
|
|
456
|
-
project_id, object_type_id, object_id, property_id,
|
|
457
|
-
value, unit, at, last_commit_id
|
|
458
|
-
)
|
|
459
|
-
SELECT project_id, object_type_id, object_id, property_id,
|
|
460
|
-
value, unit, at, last_commit_id
|
|
461
|
-
FROM latest_candidates
|
|
462
|
-
ON CONFLICT (project_id, object_type_id, object_id, property_id)
|
|
463
|
-
DO UPDATE SET value = EXCLUDED.value, unit = EXCLUDED.unit, at = EXCLUDED.at,
|
|
464
|
-
last_commit_id = EXCLUDED.last_commit_id
|
|
465
|
-
WHERE EXCLUDED.at >= timeseries_latest.at
|
|
466
|
-
RETURNING object_type_id
|
|
508
|
+
/**
|
|
509
|
+
* Writes the staged event drafts in canonical order. Event ids hash the canonical JSON of
|
|
510
|
+
* `[projectId, commitId, ordinal]`; the provider-fixed prefix is rendered here, the ordinal by
|
|
511
|
+
* PostgreSQL, so the outbox matches `createEventId` without one id crossing the wire.
|
|
512
|
+
*/
|
|
513
|
+
private async writeOutbox(target: PgStagedPlanTarget): Promise<number> {
|
|
514
|
+
const prefix = `${canonicalJson([target.projectId, target.commitId]).slice(0, -1)},`
|
|
515
|
+
const [row] = await this.sql<WriteCounts[]>`
|
|
516
|
+
WITH candidates AS (
|
|
517
|
+
SELECT payload->'draft' AS draft,
|
|
518
|
+
row_number() OVER (
|
|
519
|
+
ORDER BY major_order, minor_order, sort_one, sort_two, record_key
|
|
520
|
+
) - 1 AS ordinal
|
|
521
|
+
FROM ${this.sql(target.workTable)}
|
|
522
|
+
WHERE work_id = ${target.workId} AND lane = 'event'
|
|
523
|
+
), identified AS (
|
|
524
|
+
SELECT draft, ordinal,
|
|
525
|
+
encode(sha256(convert_to(${prefix}::text || ordinal::text || ']', 'UTF8')), 'hex') AS id
|
|
526
|
+
FROM candidates
|
|
527
|
+
), written AS (
|
|
528
|
+
INSERT INTO ontology_outbox (
|
|
529
|
+
project_id, id, commit_id, commit_ordinal, event,
|
|
530
|
+
available_at, attempts, lease_id, lease_expires_at,
|
|
531
|
+
published_at, last_failure, created_at
|
|
467
532
|
)
|
|
468
|
-
SELECT
|
|
469
|
-
|
|
470
|
-
|
|
533
|
+
SELECT ${target.projectId}, id, ${target.commitId}, ordinal, draft,
|
|
534
|
+
${target.committedAt}::timestamptz, 0, NULL, NULL, NULL, NULL,
|
|
535
|
+
${target.committedAt}::timestamptz
|
|
536
|
+
FROM identified
|
|
537
|
+
ON CONFLICT DO NOTHING
|
|
538
|
+
RETURNING 1
|
|
539
|
+
)
|
|
540
|
+
${counts(this.sql)}
|
|
541
|
+
`
|
|
542
|
+
const staged = Number(row?.staged ?? 0)
|
|
543
|
+
if (Number(row?.written ?? 0) !== staged) {
|
|
544
|
+
throw effectiveConflict(`Outbox events of commit '${target.commitId}' already exist.`)
|
|
471
545
|
}
|
|
546
|
+
return staged
|
|
472
547
|
}
|
|
473
548
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
): Promise<void> {
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
SELECT
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
549
|
+
/**
|
|
550
|
+
* Records the entities whose plan inputs this commit changed, once each, as the in-memory
|
|
551
|
+
* `touchPlanItem` does: a replacement plan's refresh rechecks only those.
|
|
552
|
+
*/
|
|
553
|
+
private async writeTouches(target: PgStagedPlanTarget): Promise<void> {
|
|
554
|
+
await this.sql`
|
|
555
|
+
INSERT INTO ontology_commit_touches (project_id, entity_kind, identity_key)
|
|
556
|
+
SELECT DISTINCT ${target.projectId}, entity.kind,
|
|
557
|
+
CASE entity.kind
|
|
558
|
+
WHEN 'object' THEN
|
|
559
|
+
concat('[', entity.ref->'objectTypeId', ',', entity.ref->'primaryId', ']')
|
|
560
|
+
WHEN 'scope' THEN concat('[', entity.ref->'source'->'objectTypeId', ',',
|
|
561
|
+
entity.ref->'source'->'primaryId', ',', entity.ref->'linkId', ']')
|
|
562
|
+
ELSE concat('[', entity.ref->'source'->'objectTypeId', ',',
|
|
563
|
+
entity.ref->'source'->'primaryId', ',', entity.ref->'linkId', ',',
|
|
564
|
+
entity.ref->'target'->'objectTypeId', ',', entity.ref->'target'->'primaryId', ']')
|
|
565
|
+
END
|
|
566
|
+
FROM ${this.sql(target.workTable)} AS work
|
|
567
|
+
CROSS JOIN LATERAL (
|
|
568
|
+
SELECT CASE work.payload->'item'->>'kind'
|
|
569
|
+
WHEN 'object-upsert' THEN 'object'
|
|
570
|
+
WHEN 'object-delete' THEN 'object'
|
|
571
|
+
WHEN 'object-override-upsert' THEN 'object'
|
|
572
|
+
WHEN 'object-override-delete' THEN 'object'
|
|
573
|
+
WHEN 'point-upsert' THEN 'object'
|
|
574
|
+
WHEN 'link-upsert' THEN 'link'
|
|
575
|
+
WHEN 'link-delete' THEN 'link'
|
|
576
|
+
WHEN 'link-override-upsert' THEN 'link'
|
|
577
|
+
WHEN 'link-override-delete' THEN 'link'
|
|
578
|
+
WHEN 'link-slot-override-upsert' THEN 'scope'
|
|
579
|
+
WHEN 'link-slot-override-delete' THEN 'scope'
|
|
580
|
+
END AS kind,
|
|
581
|
+
CASE work.payload->'item'->>'kind'
|
|
582
|
+
WHEN 'object-upsert' THEN work.payload->'item'->'value'->'row'->'ref'
|
|
583
|
+
WHEN 'link-upsert' THEN work.payload->'item'->'value'->'row'->'ref'
|
|
584
|
+
WHEN 'point-upsert' THEN work.payload->'item'->'value'->'point'->'series'->'object'
|
|
585
|
+
ELSE work.payload->'item'->'value'->'ref'
|
|
586
|
+
END AS ref
|
|
587
|
+
) AS entity
|
|
588
|
+
WHERE work.work_id = ${target.workId} AND work.lane = 'apply'
|
|
589
|
+
`
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
private async write(
|
|
593
|
+
statement: Promise<readonly WriteCounts[]>,
|
|
594
|
+
conflict: () => Promise<never>
|
|
595
|
+
): Promise<number> {
|
|
596
|
+
const [row] = await statement
|
|
597
|
+
const written = Number(row?.written ?? 0)
|
|
598
|
+
if (written !== Number(row?.staged ?? 0)) await conflict()
|
|
599
|
+
return written
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** Matches a staged ref, at a static JSON path of `candidates.value`, to `stored`. */
|
|
603
|
+
private matches(kind: "object" | "link", path: string): Fragment {
|
|
604
|
+
const ref = `(candidates.${path})`
|
|
605
|
+
return this.sql.unsafe(
|
|
606
|
+
kind === "object"
|
|
607
|
+
? `stored.object_type_id = ${ref}->>'objectTypeId'
|
|
608
|
+
AND stored.primary_id = ${ref}->>'primaryId'`
|
|
609
|
+
: `stored.source_type_id = ${ref}->'source'->>'objectTypeId'
|
|
610
|
+
AND stored.source_id = ${ref}->'source'->>'primaryId'
|
|
611
|
+
AND stored.link_id = ${ref}->>'linkId'
|
|
612
|
+
AND stored.target_type_id = ${ref}->'target'->>'objectTypeId'
|
|
613
|
+
AND stored.target_id = ${ref}->'target'->>'primaryId'`
|
|
614
|
+
)
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* The ref of the first staged entity whose stored row tells why a write missed it: it still
|
|
619
|
+
* exists after a delete, another commit wrote it before an insert, or an update left it without
|
|
620
|
+
* this commit. None when no stored row tells, as for a delete whose row is already gone.
|
|
621
|
+
*/
|
|
622
|
+
private async firstConflict(
|
|
623
|
+
target: PgStagedPlanTarget,
|
|
624
|
+
candidates: Query,
|
|
625
|
+
kind: "object" | "link",
|
|
626
|
+
path: string,
|
|
627
|
+
missed: "delete" | "insert" | "update"
|
|
628
|
+
): Promise<unknown> {
|
|
629
|
+
const table = kind === "object" ? "objects" : "links"
|
|
630
|
+
const revision =
|
|
631
|
+
missed === "insert"
|
|
632
|
+
? this.sql`AND stored.last_commit_id <> ${target.commitId}`
|
|
633
|
+
: missed === "update"
|
|
634
|
+
? this.sql`AND stored.last_commit_id = ${target.commitId}`
|
|
635
|
+
: this.sql``
|
|
636
|
+
const stored = this.sql`
|
|
637
|
+
EXISTS (
|
|
638
|
+
SELECT 1 FROM ${this.sql(table)} AS stored
|
|
639
|
+
WHERE stored.project_id = ${target.projectId} AND ${this.matches(kind, path)} ${revision}
|
|
499
640
|
)
|
|
500
|
-
SELECT ${projectId}, value->>'id', ${commitId},
|
|
501
|
-
(value->>'commitOrdinal')::bigint, value->'envelope',
|
|
502
|
-
(value->>'availableAt')::timestamptz, 0, NULL, NULL, NULL, NULL,
|
|
503
|
-
(value->>'createdAt')::timestamptz
|
|
504
|
-
FROM staged
|
|
505
|
-
ON CONFLICT DO NOTHING
|
|
506
|
-
RETURNING id
|
|
507
641
|
`
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
642
|
+
const [row] = await this.sql<{ readonly ref: unknown }[]>`
|
|
643
|
+
SELECT ${this.sql.unsafe(`candidates.${path}`)} AS ref FROM (${candidates}) AS candidates
|
|
644
|
+
WHERE ${missed === "update" ? this.sql`NOT` : this.sql``} ${stored}
|
|
645
|
+
LIMIT 1
|
|
646
|
+
`
|
|
647
|
+
return row?.ref
|
|
513
648
|
}
|
|
514
649
|
}
|
|
515
650
|
|
|
651
|
+
function counts(sql: SQLClient) {
|
|
652
|
+
return sql`SELECT (SELECT COUNT(*) FROM candidates) AS staged,
|
|
653
|
+
(SELECT COUNT(*) FROM written) AS written`
|
|
654
|
+
}
|
|
655
|
+
|
|
516
656
|
function objectOverrideConflict(): MaterializationConflictError {
|
|
517
657
|
return effectiveConflict("Expected object override changed.")
|
|
518
658
|
}
|
|
519
659
|
|
|
520
|
-
function linkOverrideConflict(identityKind:
|
|
660
|
+
function linkOverrideConflict(identityKind: string | undefined): MaterializationConflictError {
|
|
521
661
|
return effectiveConflict(
|
|
522
|
-
identityKind === "
|
|
523
|
-
? "Expected link
|
|
524
|
-
: "Expected link
|
|
662
|
+
identityKind === "slot"
|
|
663
|
+
? "Expected link slot override changed."
|
|
664
|
+
: "Expected link edge override changed."
|
|
525
665
|
)
|
|
526
666
|
}
|
|
527
667
|
|
|
528
|
-
function
|
|
529
|
-
|
|
530
|
-
readonly primaryId: string
|
|
531
|
-
}): string {
|
|
532
|
-
return objectRefKey({ objectTypeId: item.objectTypeId, primaryId: item.primaryId })
|
|
668
|
+
function effectiveObjectConflict(ref: OntologyObjectRef | undefined): MaterializationConflictError {
|
|
669
|
+
return effectiveConflict(`Expected ${objectLabel(ref)} changed.`)
|
|
533
670
|
}
|
|
534
671
|
|
|
535
|
-
function
|
|
536
|
-
|
|
537
|
-
readonly sourceId: string
|
|
538
|
-
readonly linkId: string
|
|
539
|
-
readonly targetTypeId: string
|
|
540
|
-
readonly targetId: string
|
|
541
|
-
}): string {
|
|
542
|
-
return linkRefKey({
|
|
543
|
-
source: { objectTypeId: item.sourceTypeId, primaryId: item.sourceId },
|
|
544
|
-
linkId: item.linkId,
|
|
545
|
-
target: { objectTypeId: item.targetTypeId, primaryId: item.targetId },
|
|
546
|
-
})
|
|
672
|
+
function effectiveLinkConflict(ref: OntologyLinkRef | undefined): MaterializationConflictError {
|
|
673
|
+
return effectiveConflict(`Expected ${linkLabel(ref)} changed.`)
|
|
547
674
|
}
|
|
548
675
|
|
|
549
|
-
function
|
|
550
|
-
|
|
551
|
-
readonly primaryId: string
|
|
552
|
-
}): MaterializationConflictError {
|
|
553
|
-
return effectiveConflict(`Expected object ${objectIdentityKey(item)} changed.`)
|
|
676
|
+
function objectLabel(ref: OntologyObjectRef | undefined): string {
|
|
677
|
+
return ref ? `object ${objectRefKey(ref)}` : "object"
|
|
554
678
|
}
|
|
555
679
|
|
|
556
|
-
function
|
|
557
|
-
|
|
558
|
-
readonly sourceId: string
|
|
559
|
-
readonly linkId: string
|
|
560
|
-
readonly targetTypeId: string
|
|
561
|
-
readonly targetId: string
|
|
562
|
-
}): MaterializationConflictError {
|
|
563
|
-
return effectiveConflict(`Expected link ${linkIdentityKey(item)} changed.`)
|
|
680
|
+
function linkLabel(ref: OntologyLinkRef | undefined): string {
|
|
681
|
+
return ref ? `link ${linkRefKey(ref)}` : "link"
|
|
564
682
|
}
|
|
565
683
|
|
|
566
|
-
function pointConflict(
|
|
567
|
-
readonly
|
|
568
|
-
readonly objectId: string
|
|
569
|
-
readonly propertyId: string
|
|
570
|
-
readonly at: string
|
|
571
|
-
}): MaterializationConflictError {
|
|
684
|
+
function pointConflict(point: unknown): MaterializationConflictError {
|
|
685
|
+
const found = point as { readonly series: TelemetrySeriesRef; readonly at: string } | undefined
|
|
572
686
|
return new MaterializationConflictError(
|
|
573
687
|
"timeseries-point",
|
|
574
|
-
|
|
575
|
-
{
|
|
576
|
-
|
|
577
|
-
propertyId: item.propertyId,
|
|
578
|
-
},
|
|
579
|
-
item.at
|
|
580
|
-
)} changed.`
|
|
688
|
+
found
|
|
689
|
+
? `Telemetry point ${telemetryPointKey(found.series, found.at)} changed.`
|
|
690
|
+
: "Telemetry point changed."
|
|
581
691
|
)
|
|
582
692
|
}
|