@sixb/pg 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1807 -279
- package/dist/index.js.map +25 -21
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +10 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +40 -0
- package/src/objects/storage.ts +49 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +3 -1
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type DatasetIngestDeliveryRecord,
|
|
3
|
+
type DatasetIngestPendingCounts,
|
|
4
|
+
type DatasetIngestPersistence,
|
|
5
|
+
type DatasetIngestPersistenceBackend,
|
|
6
|
+
type DatasetIngestRejectionRecord,
|
|
7
|
+
type DatasetIngestState,
|
|
8
|
+
DurableDatasetIngestStorage,
|
|
9
|
+
initialDatasetIngestState,
|
|
10
|
+
} from "@sixb/core/internal/dataset-ingest-storage-provider"
|
|
11
|
+
import type { DatasetIngestDelivery } from "@sixb/core/storage"
|
|
12
|
+
import { type PgStoreClient, runPgTransaction } from "./transactions"
|
|
13
|
+
|
|
14
|
+
export class PgDatasetIngestStorage extends DurableDatasetIngestStorage {
|
|
15
|
+
constructor(sql: PgStoreClient) {
|
|
16
|
+
super(new PgDatasetIngestBackend(sql))
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
class PgDatasetIngestBackend implements DatasetIngestPersistenceBackend {
|
|
21
|
+
constructor(private readonly sql: PgStoreClient) {}
|
|
22
|
+
|
|
23
|
+
read<T>(run: (persistence: DatasetIngestPersistence) => Promise<T>): Promise<T> {
|
|
24
|
+
return run(new PgDatasetIngestPersistence(this.sql))
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
transaction<T>(run: (persistence: DatasetIngestPersistence) => Promise<T>): Promise<T> {
|
|
28
|
+
return runPgTransaction(this.sql, (tx) => run(new PgDatasetIngestPersistence(tx)))
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
interface PgStateRow {
|
|
33
|
+
readonly dataset_id: string
|
|
34
|
+
readonly request: string
|
|
35
|
+
readonly requested_at: Date | null
|
|
36
|
+
readonly lease_owner: string | null
|
|
37
|
+
readonly lease_expires_at: Date | null
|
|
38
|
+
readonly flush_id: string | null
|
|
39
|
+
readonly flush_started_at: Date | null
|
|
40
|
+
readonly failure_streak: number
|
|
41
|
+
readonly last_flushed_at: Date | null
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
interface PgDeliveryRow {
|
|
45
|
+
readonly id: string
|
|
46
|
+
readonly dataset_id: string
|
|
47
|
+
readonly execution_id: string
|
|
48
|
+
readonly correlation_id: string
|
|
49
|
+
readonly received_at: Date
|
|
50
|
+
readonly change_count: number
|
|
51
|
+
readonly definition_hash: string
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface PgRejectionRow extends PgDeliveryRow {
|
|
55
|
+
readonly changes: string
|
|
56
|
+
readonly rejected_at: Date
|
|
57
|
+
readonly rejection: string
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
class PgDatasetIngestPersistence implements DatasetIngestPersistence {
|
|
61
|
+
constructor(private readonly sql: PgStoreClient) {}
|
|
62
|
+
|
|
63
|
+
async now(): Promise<Date> {
|
|
64
|
+
const [row] = await this.sql<{ readonly now: Date }[]>`SELECT clock_timestamp() AS now`
|
|
65
|
+
if (!row) throw new Error("[SixbPg] Database clock returned no row.")
|
|
66
|
+
return new Date(row.now)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async lockState(projectId: string, datasetId: string): Promise<DatasetIngestState> {
|
|
70
|
+
await this.sql`
|
|
71
|
+
INSERT INTO dataset_ingest_state (project_id, dataset_id)
|
|
72
|
+
VALUES (${projectId}, ${datasetId})
|
|
73
|
+
ON CONFLICT (project_id, dataset_id) DO NOTHING
|
|
74
|
+
`
|
|
75
|
+
|
|
76
|
+
// Read committed: this statement runs once the lock is ours, so it sees every delivery and
|
|
77
|
+
// request committed before.
|
|
78
|
+
const [row] = await this.sql<PgStateRow[]>`
|
|
79
|
+
SELECT dataset_id, request::text AS request, requested_at, lease_owner, lease_expires_at,
|
|
80
|
+
flush_id, flush_started_at, failure_streak, last_flushed_at
|
|
81
|
+
FROM dataset_ingest_state
|
|
82
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId}
|
|
83
|
+
FOR UPDATE
|
|
84
|
+
`
|
|
85
|
+
return row ? stateFromRow(row) : initialDatasetIngestState(datasetId)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async writeState(projectId: string, state: DatasetIngestState): Promise<void> {
|
|
89
|
+
await this.sql`
|
|
90
|
+
UPDATE dataset_ingest_state
|
|
91
|
+
SET request = ${state.request},
|
|
92
|
+
requested_at = ${state.requestedAt},
|
|
93
|
+
lease_owner = ${state.leaseOwner},
|
|
94
|
+
lease_expires_at = ${state.leaseExpiresAt},
|
|
95
|
+
flush_id = ${state.flushId},
|
|
96
|
+
flush_started_at = ${state.flushStartedAt},
|
|
97
|
+
failure_streak = ${state.failureStreak},
|
|
98
|
+
last_flushed_at = ${state.lastFlushedAt}
|
|
99
|
+
WHERE project_id = ${projectId} AND dataset_id = ${state.datasetId}
|
|
100
|
+
`
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
async insertDelivery(projectId: string, record: DatasetIngestDeliveryRecord): Promise<string> {
|
|
104
|
+
const [row] = await this.sql<{ readonly id: string }[]>`
|
|
105
|
+
INSERT INTO dataset_ingest_deliveries (
|
|
106
|
+
project_id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
107
|
+
definition_hash, changes
|
|
108
|
+
) VALUES (
|
|
109
|
+
${projectId}, ${record.datasetId}, ${record.executionId}, ${record.correlationId},
|
|
110
|
+
${record.receivedAt}, ${record.changeCount}, ${record.definitionHash}, ${record.changes}
|
|
111
|
+
)
|
|
112
|
+
RETURNING id::text AS id
|
|
113
|
+
`
|
|
114
|
+
if (!row) throw new Error("[SixbPg] Dataset ingest delivery insert returned no id.")
|
|
115
|
+
return row.id
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async listUntaken(
|
|
119
|
+
projectId: string,
|
|
120
|
+
datasetId: string,
|
|
121
|
+
limit: number
|
|
122
|
+
): Promise<readonly DatasetIngestDelivery[]> {
|
|
123
|
+
const rows = await this.sql<PgDeliveryRow[]>`
|
|
124
|
+
-- The id is read as text: ORDER BY names the table's column, not that alias.
|
|
125
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
126
|
+
definition_hash
|
|
127
|
+
FROM dataset_ingest_deliveries
|
|
128
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id IS NULL
|
|
129
|
+
AND rejected_at IS NULL
|
|
130
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
131
|
+
LIMIT ${limit}
|
|
132
|
+
`
|
|
133
|
+
return rows.map(deliveryFromRow)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async take(
|
|
137
|
+
projectId: string,
|
|
138
|
+
datasetId: string,
|
|
139
|
+
deliveryIds: readonly string[],
|
|
140
|
+
flushId: string
|
|
141
|
+
): Promise<void> {
|
|
142
|
+
await this.sql`
|
|
143
|
+
UPDATE dataset_ingest_deliveries
|
|
144
|
+
SET flush_id = ${flushId}
|
|
145
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId}
|
|
146
|
+
AND id = ANY(${this.sql.array([...deliveryIds])}::bigint[])
|
|
147
|
+
`
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
async listFlush(
|
|
151
|
+
projectId: string,
|
|
152
|
+
datasetId: string,
|
|
153
|
+
flushId: string
|
|
154
|
+
): Promise<readonly DatasetIngestDelivery[]> {
|
|
155
|
+
const rows = await this.sql<PgDeliveryRow[]>`
|
|
156
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
157
|
+
definition_hash
|
|
158
|
+
FROM dataset_ingest_deliveries
|
|
159
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
160
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
161
|
+
`
|
|
162
|
+
return rows.map(deliveryFromRow)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
async readFlush(
|
|
166
|
+
projectId: string,
|
|
167
|
+
datasetId: string,
|
|
168
|
+
flushId: string,
|
|
169
|
+
page: { readonly afterId: string | null; readonly limit: number }
|
|
170
|
+
): Promise<readonly { readonly delivery: DatasetIngestDelivery; readonly changes: string }[]> {
|
|
171
|
+
const rows = await this.sql<(PgDeliveryRow & { readonly changes: string })[]>`
|
|
172
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
173
|
+
definition_hash, changes
|
|
174
|
+
FROM dataset_ingest_deliveries
|
|
175
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
176
|
+
AND id > ${page.afterId ?? "0"}::bigint
|
|
177
|
+
ORDER BY dataset_ingest_deliveries.id
|
|
178
|
+
LIMIT ${page.limit}
|
|
179
|
+
`
|
|
180
|
+
return rows.map((row) => ({ delivery: deliveryFromRow(row), changes: row.changes }))
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async deleteFlush(projectId: string, datasetId: string, flushId: string): Promise<void> {
|
|
184
|
+
await this.sql`
|
|
185
|
+
DELETE FROM dataset_ingest_deliveries
|
|
186
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
187
|
+
`
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async rejectDelivery(
|
|
191
|
+
projectId: string,
|
|
192
|
+
datasetId: string,
|
|
193
|
+
flushId: string,
|
|
194
|
+
deliveryId: string,
|
|
195
|
+
rejection: { readonly rejectedAt: Date; readonly reason: string }
|
|
196
|
+
): Promise<void> {
|
|
197
|
+
await this.sql`
|
|
198
|
+
UPDATE dataset_ingest_deliveries
|
|
199
|
+
SET flush_id = NULL, rejected_at = ${rejection.rejectedAt}, rejection = ${rejection.reason}
|
|
200
|
+
WHERE project_id = ${projectId} AND dataset_id = ${datasetId} AND flush_id = ${flushId}
|
|
201
|
+
AND id = ${deliveryId}::bigint
|
|
202
|
+
`
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async listRejected(
|
|
206
|
+
projectId: string,
|
|
207
|
+
datasetId: string | undefined,
|
|
208
|
+
limit: number
|
|
209
|
+
): Promise<readonly DatasetIngestRejectionRecord[]> {
|
|
210
|
+
const rows = await this.sql<PgRejectionRow[]>`
|
|
211
|
+
SELECT id::text AS id, dataset_id, execution_id, correlation_id, received_at, change_count,
|
|
212
|
+
definition_hash, changes, rejected_at, rejection
|
|
213
|
+
FROM dataset_ingest_deliveries
|
|
214
|
+
WHERE project_id = ${projectId} AND rejected_at IS NOT NULL
|
|
215
|
+
AND (${datasetId === undefined} OR dataset_id = ${datasetId ?? ""})
|
|
216
|
+
ORDER BY rejected_at DESC, dataset_ingest_deliveries.id DESC
|
|
217
|
+
LIMIT ${limit}
|
|
218
|
+
`
|
|
219
|
+
return rows.map((row) => ({
|
|
220
|
+
delivery: deliveryFromRow(row),
|
|
221
|
+
rejectedAt: new Date(row.rejected_at),
|
|
222
|
+
reason: row.rejection,
|
|
223
|
+
changes: row.changes,
|
|
224
|
+
}))
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async deleteRejected(projectId: string, before: Date): Promise<void> {
|
|
228
|
+
await this.sql`
|
|
229
|
+
DELETE FROM dataset_ingest_deliveries
|
|
230
|
+
WHERE project_id = ${projectId} AND rejected_at < ${before}
|
|
231
|
+
`
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async listStates(
|
|
235
|
+
projectId: string,
|
|
236
|
+
datasetIds: readonly string[] | undefined
|
|
237
|
+
): Promise<readonly DatasetIngestState[]> {
|
|
238
|
+
const rows = await this.sql<PgStateRow[]>`
|
|
239
|
+
SELECT dataset_id, request::text AS request, requested_at, lease_owner, lease_expires_at,
|
|
240
|
+
flush_id, flush_started_at, failure_streak, last_flushed_at
|
|
241
|
+
FROM dataset_ingest_state
|
|
242
|
+
WHERE project_id = ${projectId}
|
|
243
|
+
AND (${datasetIds === undefined} OR dataset_id = ANY(${this.sql.array([...(datasetIds ?? [])])}::text[]))
|
|
244
|
+
`
|
|
245
|
+
return rows.map(stateFromRow)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async countPending(
|
|
249
|
+
projectId: string,
|
|
250
|
+
datasetIds: readonly string[] | undefined
|
|
251
|
+
): Promise<readonly DatasetIngestPendingCounts[]> {
|
|
252
|
+
const rows = await this.sql<
|
|
253
|
+
{
|
|
254
|
+
readonly dataset_id: string
|
|
255
|
+
readonly deliveries: string
|
|
256
|
+
readonly changes: string
|
|
257
|
+
readonly oldest: Date | null
|
|
258
|
+
readonly rejected: string
|
|
259
|
+
}[]
|
|
260
|
+
>`
|
|
261
|
+
SELECT dataset_id,
|
|
262
|
+
count(*) FILTER (WHERE rejected_at IS NULL)::text AS deliveries,
|
|
263
|
+
coalesce(sum(change_count) FILTER (WHERE rejected_at IS NULL), 0)::text AS changes,
|
|
264
|
+
min(received_at) FILTER (WHERE rejected_at IS NULL) AS oldest,
|
|
265
|
+
count(*) FILTER (WHERE rejected_at IS NOT NULL)::text AS rejected
|
|
266
|
+
FROM dataset_ingest_deliveries
|
|
267
|
+
WHERE project_id = ${projectId}
|
|
268
|
+
AND (${datasetIds === undefined} OR dataset_id = ANY(${this.sql.array([...(datasetIds ?? [])])}::text[]))
|
|
269
|
+
GROUP BY dataset_id
|
|
270
|
+
`
|
|
271
|
+
return rows.map((row) => ({
|
|
272
|
+
datasetId: row.dataset_id,
|
|
273
|
+
pendingDeliveries: Number(row.deliveries),
|
|
274
|
+
pendingChanges: Number(row.changes),
|
|
275
|
+
oldestReceivedAt: row.oldest === null ? null : new Date(row.oldest),
|
|
276
|
+
rejectedDeliveries: Number(row.rejected),
|
|
277
|
+
}))
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function stateFromRow(row: PgStateRow): DatasetIngestState {
|
|
282
|
+
return {
|
|
283
|
+
datasetId: row.dataset_id,
|
|
284
|
+
request: Number(row.request),
|
|
285
|
+
requestedAt: row.requested_at === null ? null : new Date(row.requested_at),
|
|
286
|
+
leaseOwner: row.lease_owner,
|
|
287
|
+
leaseExpiresAt: row.lease_expires_at === null ? null : new Date(row.lease_expires_at),
|
|
288
|
+
flushId: row.flush_id,
|
|
289
|
+
flushStartedAt: row.flush_started_at === null ? null : new Date(row.flush_started_at),
|
|
290
|
+
failureStreak: row.failure_streak,
|
|
291
|
+
lastFlushedAt: row.last_flushed_at === null ? null : new Date(row.last_flushed_at),
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function deliveryFromRow(row: PgDeliveryRow): DatasetIngestDelivery {
|
|
296
|
+
return {
|
|
297
|
+
id: row.id,
|
|
298
|
+
datasetId: row.dataset_id,
|
|
299
|
+
executionId: row.execution_id,
|
|
300
|
+
correlationId: row.correlation_id,
|
|
301
|
+
receivedAt: new Date(row.received_at),
|
|
302
|
+
changeCount: row.change_count,
|
|
303
|
+
definitionHash: row.definition_hash,
|
|
304
|
+
}
|
|
305
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { LakeRetentionRunRecord, LakeRetentionRunStorage } from "@sixb/core/storage"
|
|
2
|
+
import type { PgStoreClient } from "./transactions"
|
|
3
|
+
|
|
4
|
+
export class PgLakeRetentionRunStorage implements LakeRetentionRunStorage {
|
|
5
|
+
constructor(private readonly sql: PgStoreClient) {}
|
|
6
|
+
|
|
7
|
+
async record(run: LakeRetentionRunRecord): Promise<void> {
|
|
8
|
+
await this.sql`
|
|
9
|
+
INSERT INTO lake_retention_runs (project_id, started_at, finished_at, failed)
|
|
10
|
+
VALUES (${run.projectId}, ${run.startedAt}, ${run.finishedAt}, ${run.failed})
|
|
11
|
+
ON CONFLICT (project_id) DO UPDATE SET
|
|
12
|
+
started_at = EXCLUDED.started_at,
|
|
13
|
+
finished_at = EXCLUDED.finished_at,
|
|
14
|
+
failed = EXCLUDED.failed
|
|
15
|
+
`
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
async getLast(input: { readonly projectId: string }): Promise<LakeRetentionRunRecord | null> {
|
|
19
|
+
const [row] = await this.sql<PgLakeRetentionRunRow[]>`
|
|
20
|
+
SELECT project_id, started_at, finished_at, failed
|
|
21
|
+
FROM lake_retention_runs
|
|
22
|
+
WHERE project_id = ${input.projectId}
|
|
23
|
+
`
|
|
24
|
+
if (!row) return null
|
|
25
|
+
|
|
26
|
+
return {
|
|
27
|
+
projectId: row.project_id,
|
|
28
|
+
startedAt: new Date(row.started_at),
|
|
29
|
+
finishedAt: row.finished_at ? new Date(row.finished_at) : null,
|
|
30
|
+
failed: row.failed,
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface PgLakeRetentionRunRow {
|
|
36
|
+
readonly project_id: string
|
|
37
|
+
readonly started_at: Date
|
|
38
|
+
readonly finished_at: Date | null
|
|
39
|
+
readonly failed: boolean
|
|
40
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parseMaintenanceHeartbeatRow,
|
|
3
|
+
serializeMaintenancePasses,
|
|
4
|
+
} from "@sixb/core/internal/storage"
|
|
5
|
+
import type { MaintenanceHeartbeatRecord, MaintenanceHeartbeatStorage } from "@sixb/core/storage"
|
|
6
|
+
import type { PgStoreClient } from "./transactions"
|
|
7
|
+
|
|
8
|
+
export class PgMaintenanceHeartbeatStorage implements MaintenanceHeartbeatStorage {
|
|
9
|
+
constructor(private readonly sql: PgStoreClient) {}
|
|
10
|
+
|
|
11
|
+
async record(heartbeat: MaintenanceHeartbeatRecord): Promise<void> {
|
|
12
|
+
const passes = serializeMaintenancePasses(heartbeat.passes)
|
|
13
|
+
await this.sql`
|
|
14
|
+
INSERT INTO maintenance_heartbeats (project_id, owner, beat_at, every_ms, passes)
|
|
15
|
+
VALUES (
|
|
16
|
+
${heartbeat.projectId}, ${heartbeat.owner}, ${heartbeat.beatAt}, ${heartbeat.everyMs},
|
|
17
|
+
${passes}::text::jsonb
|
|
18
|
+
)
|
|
19
|
+
ON CONFLICT (project_id, owner) DO UPDATE SET
|
|
20
|
+
beat_at = EXCLUDED.beat_at,
|
|
21
|
+
every_ms = EXCLUDED.every_ms,
|
|
22
|
+
passes = EXCLUDED.passes
|
|
23
|
+
`
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async list(input: {
|
|
27
|
+
readonly projectId: string
|
|
28
|
+
}): Promise<readonly MaintenanceHeartbeatRecord[]> {
|
|
29
|
+
const rows = await this.sql<PgMaintenanceHeartbeatRow[]>`
|
|
30
|
+
SELECT project_id, owner, beat_at, every_ms, passes
|
|
31
|
+
FROM maintenance_heartbeats
|
|
32
|
+
WHERE project_id = ${input.projectId}
|
|
33
|
+
`
|
|
34
|
+
|
|
35
|
+
return rows.map((row) =>
|
|
36
|
+
parseMaintenanceHeartbeatRow({
|
|
37
|
+
projectId: row.project_id,
|
|
38
|
+
owner: row.owner,
|
|
39
|
+
beatAt: row.beat_at,
|
|
40
|
+
everyMs: row.every_ms,
|
|
41
|
+
passes: row.passes,
|
|
42
|
+
})
|
|
43
|
+
)
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface PgMaintenanceHeartbeatRow {
|
|
48
|
+
readonly project_id: string
|
|
49
|
+
readonly owner: string
|
|
50
|
+
readonly beat_at: Date
|
|
51
|
+
readonly every_ms: number
|
|
52
|
+
readonly passes: unknown
|
|
53
|
+
}
|
|
@@ -4,14 +4,19 @@ import { assertPipelineRunExecution } from "@sixb/core/internal/pipeline-run-sto
|
|
|
4
4
|
import {
|
|
5
5
|
canFinishPipelineRun,
|
|
6
6
|
canRequeuePipelineRunAfterEnqueueFailure,
|
|
7
|
+
holdsRun,
|
|
8
|
+
mayFinishRun,
|
|
7
9
|
} from "@sixb/core/internal/storage"
|
|
8
10
|
import type { DatasetVersionRef } from "@sixb/core/lake-storage"
|
|
9
11
|
import type {
|
|
12
|
+
ClaimRunInput,
|
|
13
|
+
ConfirmRunExecutionInput,
|
|
10
14
|
ExecutionStorage,
|
|
11
15
|
FinishPipelineRunInput,
|
|
12
16
|
FinishPipelineStepRunInput,
|
|
13
17
|
ListLatestPipelineRunsInput,
|
|
14
18
|
ListLatestPipelineRunsResult,
|
|
19
|
+
ListLostRunsInput,
|
|
15
20
|
ListPipelineRunsInput,
|
|
16
21
|
ListPipelineRunsResult,
|
|
17
22
|
ListPipelineStepRunsInput,
|
|
@@ -20,16 +25,36 @@ import type {
|
|
|
20
25
|
PipelineRunStorage,
|
|
21
26
|
PipelineStepRunRecord,
|
|
22
27
|
QueuePipelineRunInput,
|
|
23
|
-
|
|
28
|
+
ReleaseRunExecutionInput,
|
|
29
|
+
RunClaim,
|
|
24
30
|
StartPipelineStepRunInput,
|
|
25
31
|
} from "@sixb/core/storage"
|
|
26
32
|
import { PIPELINE_RUN_FAILURE_CODES, PipelineRunError } from "@sixb/core/storage"
|
|
27
33
|
import { queryLatestRunsByOwnerId } from "./latest-run-query"
|
|
28
34
|
import type { SqlParameter } from "./pg-client"
|
|
35
|
+
import {
|
|
36
|
+
claimPgRun,
|
|
37
|
+
confirmPgRunExecution,
|
|
38
|
+
listLostPgRuns,
|
|
39
|
+
lockPgRun,
|
|
40
|
+
type PgRunExecutionColumns,
|
|
41
|
+
type PgRunTable,
|
|
42
|
+
pgRunExecutionFields,
|
|
43
|
+
releasePgRun,
|
|
44
|
+
} from "./run-claim-sql"
|
|
29
45
|
import { appendRunListFilters, hasEmptyStatuses, queryRunList } from "./run-list-query"
|
|
30
46
|
import { isUniqueViolation } from "./storage-errors"
|
|
31
47
|
import { type PgStoreClient, runPgTransaction } from "./transactions"
|
|
32
48
|
|
|
49
|
+
const PIPELINE_RUNS: PgRunTable<PipelineRunDatabaseRow, PipelineRunRecord> = {
|
|
50
|
+
name: "pipeline_runs",
|
|
51
|
+
returning: "*",
|
|
52
|
+
toRun: rowToPipelineRunRecord,
|
|
53
|
+
notFound: (projectId, id) =>
|
|
54
|
+
new PipelineRunError(`[SixbPg] Pipeline run '${id}' not found for project '${projectId}'.`),
|
|
55
|
+
staleExecution,
|
|
56
|
+
}
|
|
57
|
+
|
|
33
58
|
export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
34
59
|
constructor(
|
|
35
60
|
private readonly sql: PgStoreClient,
|
|
@@ -97,6 +122,7 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
97
122
|
const [updated] = await tx<PipelineRunDatabaseRow[]>`
|
|
98
123
|
UPDATE pipeline_runs
|
|
99
124
|
SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date()},
|
|
125
|
+
attempt = ${0}, execution_token = ${null}, execution_queue_lease_expires_at = ${null},
|
|
100
126
|
started_at = ${null}, finished_at = ${null}, output_dataset_id = ${null},
|
|
101
127
|
output_version_id = ${null}, error = ${null}
|
|
102
128
|
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
@@ -106,39 +132,25 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
106
132
|
})
|
|
107
133
|
}
|
|
108
134
|
|
|
109
|
-
async
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
SET status = ${"running"}, started_at = ${input.startedAt ?? new Date()},
|
|
113
|
-
error = ${null}
|
|
114
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"queued"}
|
|
115
|
-
RETURNING *
|
|
116
|
-
`
|
|
117
|
-
if (updated) return rowToPipelineRunRecord(updated)
|
|
135
|
+
async startOrReclaim(input: ClaimRunInput): Promise<RunClaim<PipelineRunRecord>> {
|
|
136
|
+
return runPgTransaction(this.sql, (tx) => claimPgRun(tx, PIPELINE_RUNS, input))
|
|
137
|
+
}
|
|
118
138
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
async confirmExecutionOwnership(input: ConfirmRunExecutionInput): Promise<PipelineRunRecord> {
|
|
140
|
+
return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, PIPELINE_RUNS, input))
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async listLost(input: ListLostRunsInput): Promise<readonly PipelineRunRecord[]> {
|
|
144
|
+
return listLostPgRuns(this.sql, PIPELINE_RUNS, input)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async releaseExecution(input: ReleaseRunExecutionInput): Promise<void> {
|
|
148
|
+
await releasePgRun(this.sql, PIPELINE_RUNS, input)
|
|
128
149
|
}
|
|
129
150
|
|
|
130
151
|
async finish(input: FinishPipelineRunInput): Promise<PipelineRunRecord> {
|
|
131
152
|
return runPgTransaction(this.sql, async (tx) => {
|
|
132
|
-
const
|
|
133
|
-
SELECT * FROM pipeline_runs
|
|
134
|
-
WHERE project_id = ${input.projectId} AND id = ${input.id}
|
|
135
|
-
`
|
|
136
|
-
|
|
137
|
-
if (!existing) {
|
|
138
|
-
throw new PipelineRunError(
|
|
139
|
-
`[SixbPg] Pipeline run '${input.id}' not found for project '${input.projectId}'.`
|
|
140
|
-
)
|
|
141
|
-
}
|
|
153
|
+
const existing = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, input.id)
|
|
142
154
|
|
|
143
155
|
if (!canFinishPipelineRun(existing.status, input)) {
|
|
144
156
|
throw new PipelineRunError(
|
|
@@ -146,12 +158,16 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
146
158
|
)
|
|
147
159
|
}
|
|
148
160
|
|
|
161
|
+
if (!mayFinishRun(existing, input)) throw staleExecution(input.id)
|
|
162
|
+
|
|
149
163
|
const [updated] =
|
|
150
164
|
input.status === "succeeded"
|
|
151
165
|
? await tx<PipelineRunDatabaseRow[]>`
|
|
152
166
|
UPDATE pipeline_runs
|
|
153
167
|
SET
|
|
154
168
|
status = ${input.status},
|
|
169
|
+
execution_token = ${null},
|
|
170
|
+
execution_queue_lease_expires_at = ${null},
|
|
155
171
|
finished_at = ${input.finishedAt ?? new Date()},
|
|
156
172
|
output_dataset_id = ${input.output?.datasetId ?? null},
|
|
157
173
|
output_version_id = ${input.output?.versionId ?? null},
|
|
@@ -163,6 +179,8 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
163
179
|
UPDATE pipeline_runs
|
|
164
180
|
SET
|
|
165
181
|
status = ${input.status},
|
|
182
|
+
execution_token = ${null},
|
|
183
|
+
execution_queue_lease_expires_at = ${null},
|
|
166
184
|
finished_at = ${input.finishedAt ?? new Date()},
|
|
167
185
|
output_dataset_id = ${null},
|
|
168
186
|
output_version_id = ${null},
|
|
@@ -177,16 +195,7 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
177
195
|
|
|
178
196
|
async startStep(input: StartPipelineStepRunInput): Promise<PipelineStepRunRecord> {
|
|
179
197
|
return runPgTransaction(this.sql, async (tx) => {
|
|
180
|
-
const
|
|
181
|
-
SELECT * FROM pipeline_runs
|
|
182
|
-
WHERE project_id = ${input.projectId} AND id = ${input.pipelineRunId}
|
|
183
|
-
`
|
|
184
|
-
|
|
185
|
-
if (!pipelineRun) {
|
|
186
|
-
throw new PipelineRunError(
|
|
187
|
-
`[SixbPg] Pipeline run '${input.pipelineRunId}' not found for project '${input.projectId}'.`
|
|
188
|
-
)
|
|
189
|
-
}
|
|
198
|
+
const pipelineRun = await lockPgRun(tx, PIPELINE_RUNS, input.projectId, input.pipelineRunId)
|
|
190
199
|
|
|
191
200
|
if (pipelineRun.status !== "running") {
|
|
192
201
|
throw new PipelineRunError(
|
|
@@ -194,12 +203,22 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
194
203
|
)
|
|
195
204
|
}
|
|
196
205
|
|
|
197
|
-
if (pipelineRun.
|
|
206
|
+
if (!holdsRun(pipelineRun, input.executionToken)) throw staleExecution(input.pipelineRunId)
|
|
207
|
+
|
|
208
|
+
if (pipelineRun.pipelineId !== input.pipelineId) {
|
|
198
209
|
throw new PipelineRunError(
|
|
199
|
-
`[SixbPg] Pipeline step run '${input.id}' pipeline '${input.pipelineId}' does not match pipeline run '${input.pipelineRunId}' pipeline '${pipelineRun.
|
|
210
|
+
`[SixbPg] Pipeline step run '${input.id}' pipeline '${input.pipelineId}' does not match pipeline run '${input.pipelineRunId}' pipeline '${pipelineRun.pipelineId}'.`
|
|
200
211
|
)
|
|
201
212
|
}
|
|
202
213
|
|
|
214
|
+
const [restarted] = await tx<PipelineStepRunDatabaseRow[]>`
|
|
215
|
+
UPDATE pipeline_step_runs
|
|
216
|
+
SET inputs = ${JSON.stringify(input.inputs)}::text::jsonb
|
|
217
|
+
WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
|
|
218
|
+
RETURNING *
|
|
219
|
+
`
|
|
220
|
+
if (restarted) return rowToPipelineStepRunRecord(restarted)
|
|
221
|
+
|
|
203
222
|
try {
|
|
204
223
|
const [row] = await tx<PipelineStepRunDatabaseRow[]>`
|
|
205
224
|
INSERT INTO pipeline_step_runs (
|
|
@@ -262,6 +281,16 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
262
281
|
)
|
|
263
282
|
}
|
|
264
283
|
|
|
284
|
+
const pipelineRun = await lockPgRun(
|
|
285
|
+
tx,
|
|
286
|
+
PIPELINE_RUNS,
|
|
287
|
+
input.projectId,
|
|
288
|
+
existing.pipeline_run_id
|
|
289
|
+
)
|
|
290
|
+
if (!holdsRun(pipelineRun, input.executionToken)) {
|
|
291
|
+
throw staleExecution(existing.pipeline_run_id)
|
|
292
|
+
}
|
|
293
|
+
|
|
265
294
|
if (input.status === "succeeded" && input.output.datasetId !== existing.dataset_id) {
|
|
266
295
|
throw new PipelineRunError(
|
|
267
296
|
`[SixbPg] Pipeline step run '${input.id}' output dataset '${input.output.datasetId}' does not match '${existing.dataset_id}'.`
|
|
@@ -367,6 +396,7 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
367
396
|
ownerIds: input.pipelineIds,
|
|
368
397
|
projectId: input.projectId,
|
|
369
398
|
ownerIdFor: (row) => row.pipeline_id,
|
|
399
|
+
excludedStatus: input.excludedStatus,
|
|
370
400
|
})
|
|
371
401
|
|
|
372
402
|
return {
|
|
@@ -392,6 +422,12 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
392
422
|
params.push(input.pipelineRunId)
|
|
393
423
|
}
|
|
394
424
|
|
|
425
|
+
if (input.pipelineRunIds) {
|
|
426
|
+
const placeholders = input.pipelineRunIds.map(() => `$${index++}`)
|
|
427
|
+
whereClauses.push(`pipeline_run_id IN (${placeholders.join(", ") || "NULL"})`)
|
|
428
|
+
params.push(...input.pipelineRunIds)
|
|
429
|
+
}
|
|
430
|
+
|
|
395
431
|
if (input.pipelineId) {
|
|
396
432
|
whereClauses.push(`pipeline_id = $${index++}`)
|
|
397
433
|
params.push(input.pipelineId)
|
|
@@ -429,6 +465,12 @@ export class PgPipelineRunStorage implements PipelineRunStorage {
|
|
|
429
465
|
}
|
|
430
466
|
}
|
|
431
467
|
|
|
468
|
+
function staleExecution(id: string): PipelineRunError {
|
|
469
|
+
return new PipelineRunError(
|
|
470
|
+
`[SixbPg] Execution token is no longer current on pipeline run '${id}'.`
|
|
471
|
+
)
|
|
472
|
+
}
|
|
473
|
+
|
|
432
474
|
function rowToPipelineRunRecord(row: PipelineRunDatabaseRow): PipelineRunRecord {
|
|
433
475
|
return {
|
|
434
476
|
id: row.id,
|
|
@@ -436,6 +478,7 @@ function rowToPipelineRunRecord(row: PipelineRunDatabaseRow): PipelineRunRecord
|
|
|
436
478
|
executionId: row.execution_id,
|
|
437
479
|
pipelineId: row.pipeline_id,
|
|
438
480
|
status: row.status,
|
|
481
|
+
...pgRunExecutionFields(row),
|
|
439
482
|
queuedAt: new Date(row.queued_at),
|
|
440
483
|
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
441
484
|
finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
|
|
@@ -486,7 +529,7 @@ function assertOptionalNonNegativeInteger(value: number | undefined, fieldName:
|
|
|
486
529
|
}
|
|
487
530
|
}
|
|
488
531
|
|
|
489
|
-
interface PipelineRunDatabaseRow {
|
|
532
|
+
interface PipelineRunDatabaseRow extends PgRunExecutionColumns {
|
|
490
533
|
project_id: string
|
|
491
534
|
id: string
|
|
492
535
|
execution_id: string
|