@sixb/pg 0.2.1 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +9 -8
  2. package/dist/agents/computers.d.ts +8 -2
  3. package/dist/agents/computers.d.ts.map +1 -1
  4. package/dist/auth-storage/auth-storage.d.ts +2 -1
  5. package/dist/auth-storage/auth-storage.d.ts.map +1 -1
  6. package/dist/auth-storage/identities.d.ts +4 -0
  7. package/dist/auth-storage/identities.d.ts.map +1 -1
  8. package/dist/auth-storage/rows.d.ts +2 -1
  9. package/dist/auth-storage/rows.d.ts.map +1 -1
  10. package/dist/auth-storage/users.d.ts.map +1 -1
  11. package/dist/connector-connection-storage.d.ts.map +1 -1
  12. package/dist/index.d.ts +6 -0
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +2377 -428
  15. package/dist/index.js.map +37 -30
  16. package/dist/migrations.d.ts.map +1 -1
  17. package/dist/objects/conditional-reader.d.ts +7 -2
  18. package/dist/objects/conditional-reader.d.ts.map +1 -1
  19. package/dist/objects/partitions.d.ts.map +1 -1
  20. package/dist/objects/prepare-queries.d.ts.map +1 -1
  21. package/dist/objects/query-compiler.d.ts +14 -1
  22. package/dist/objects/query-compiler.d.ts.map +1 -1
  23. package/dist/objects/query-indexes.d.ts +13 -1
  24. package/dist/objects/query-indexes.d.ts.map +1 -1
  25. package/dist/objects/read-scope.d.ts.map +1 -1
  26. package/dist/objects/reader.d.ts +14 -1
  27. package/dist/objects/reader.d.ts.map +1 -1
  28. package/dist/objects/storage.d.ts +11 -1
  29. package/dist/objects/storage.d.ts.map +1 -1
  30. package/dist/objects/visibility-indexes.d.ts +22 -0
  31. package/dist/objects/visibility-indexes.d.ts.map +1 -0
  32. package/dist/objects/visibility-statistics.d.ts +26 -0
  33. package/dist/objects/visibility-statistics.d.ts.map +1 -0
  34. package/dist/objects/visibility.d.ts +34 -0
  35. package/dist/objects/visibility.d.ts.map +1 -0
  36. package/dist/ontology-storage/materialization-session.d.ts +5 -0
  37. package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
  38. package/dist/ontology-storage/materialization-writer.d.ts +14 -0
  39. package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
  40. package/dist/ontology-storage/materializations.d.ts.map +1 -1
  41. package/dist/pg-artifact-storage.d.ts +14 -2
  42. package/dist/pg-artifact-storage.d.ts.map +1 -1
  43. package/dist/pg-client.d.ts.map +1 -1
  44. package/dist/pg-dataset-ingest-storage.d.ts +6 -0
  45. package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
  46. package/dist/pg-lake-retention-run-storage.d.ts +11 -0
  47. package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
  48. package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
  49. package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
  50. package/dist/pg-pipeline-run-storage.d.ts +5 -2
  51. package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
  52. package/dist/pg-projection-run-storage.d.ts +5 -1
  53. package/dist/pg-projection-run-storage.d.ts.map +1 -1
  54. package/dist/pg-sync-run-storage.d.ts +6 -2
  55. package/dist/pg-sync-run-storage.d.ts.map +1 -1
  56. package/dist/pg-webhook-run-storage.d.ts +2 -1
  57. package/dist/pg-webhook-run-storage.d.ts.map +1 -1
  58. package/dist/pg-workflow-run-storage.d.ts +4 -3
  59. package/dist/pg-workflow-run-storage.d.ts.map +1 -1
  60. package/dist/run-claim-sql.d.ts +37 -0
  61. package/dist/run-claim-sql.d.ts.map +1 -0
  62. package/dist/share-session-storage.d.ts +2 -1
  63. package/dist/share-session-storage.d.ts.map +1 -1
  64. package/package.json +3 -3
  65. package/src/agents/computers.ts +54 -5
  66. package/src/auth-storage/auth-storage.ts +66 -0
  67. package/src/auth-storage/identities.ts +15 -0
  68. package/src/auth-storage/rows.ts +3 -0
  69. package/src/auth-storage/users.ts +3 -0
  70. package/src/connector-connection-storage.ts +33 -16
  71. package/src/index.ts +15 -0
  72. package/src/migrations/087-agent-computer-background-commands.sql +4 -0
  73. package/src/migrations/088-web-search-model-kind.sql +12 -0
  74. package/src/migrations/089-maintenance-heartbeats.sql +10 -0
  75. package/src/migrations/090-lake-retention-runs.sql +8 -0
  76. package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
  77. package/src/migrations/092-run-recovery.sql +23 -0
  78. package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
  79. package/src/migrations/094-artifact-board-previews.sql +14 -0
  80. package/src/migrations/095-artifact-open-versions.sql +101 -0
  81. package/src/migrations/096-artifact-documents.sql +11 -0
  82. package/src/migrations/097-artifact-frame-files.sql +8 -0
  83. package/src/migrations/098-artifact-comments.sql +36 -0
  84. package/src/migrations/099-object-artifact-refs.sql +12 -0
  85. package/src/migrations/100-shared-link-guests.sql +17 -0
  86. package/src/migrations/101-connector-connection-owners.sql +38 -0
  87. package/src/migrations/102-user-suspension-source.sql +12 -0
  88. package/src/migrations.ts +46 -0
  89. package/src/objects/conditional-reader.ts +26 -4
  90. package/src/objects/partitions.ts +17 -4
  91. package/src/objects/prepare-queries.ts +8 -3
  92. package/src/objects/query-compiler.ts +64 -23
  93. package/src/objects/query-indexes.ts +93 -27
  94. package/src/objects/read-scope.ts +147 -77
  95. package/src/objects/reader.ts +46 -18
  96. package/src/objects/storage.ts +53 -0
  97. package/src/objects/visibility-indexes.ts +222 -0
  98. package/src/objects/visibility-statistics.ts +73 -0
  99. package/src/objects/visibility.ts +135 -0
  100. package/src/ontology-storage/materialization-session.ts +4 -0
  101. package/src/ontology-storage/materialization-writer.ts +68 -1
  102. package/src/ontology-storage/materializations.ts +6 -2
  103. package/src/pg-artifact-storage.ts +798 -137
  104. package/src/pg-client.ts +4 -0
  105. package/src/pg-dataset-ingest-storage.ts +305 -0
  106. package/src/pg-lake-retention-run-storage.ts +40 -0
  107. package/src/pg-maintenance-heartbeat-storage.ts +53 -0
  108. package/src/pg-pipeline-run-storage.ts +85 -42
  109. package/src/pg-projection-run-storage.ts +61 -2
  110. package/src/pg-sync-run-storage.ts +100 -37
  111. package/src/pg-webhook-run-storage.ts +37 -11
  112. package/src/pg-workflow-run-storage.ts +39 -100
  113. package/src/run-claim-sql.ts +173 -0
  114. package/src/share-session-storage.ts +20 -0
@@ -30,12 +30,15 @@ import {
30
30
  SUPERSEDING_PROJECTION_RUN_STATUSES,
31
31
  staleProjectionRunExecution,
32
32
  } from "@sixb/core/internal/projection-run-storage-provider"
33
+ import { laterLease } from "@sixb/core/internal/storage"
33
34
  import type {
34
35
  AdvanceProjectionTelemetryCheckpointInput,
36
+ ConfirmRunExecutionInput,
35
37
  ExecutionStorage,
36
38
  FailProjectionRunEnqueueInput,
37
39
  FindSupersedingProjectionRunInput,
38
40
  FinishProjectionRunInput,
41
+ ListLapsedProjectionRunsInput,
39
42
  ListLatestProjectionRunsInput,
40
43
  ListLatestProjectionRunsResult,
41
44
  ListProjectionRunsInput,
@@ -49,6 +52,7 @@ import type {
49
52
  QueueProjectionRunInput,
50
53
  RecordProjectionAttemptFailureInput,
51
54
  RecordProjectionMissingTargetInput,
55
+ ReleaseRunExecutionInput,
52
56
  StartOrReclaimProjectionRunInput,
53
57
  TelemetryProjectionRunRecord,
54
58
  UpdateProjectionRunInput,
@@ -159,11 +163,13 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
159
163
  if (!existingRow) throw projectionRunNotFound(input.projectId, input.id)
160
164
  const existing = restoreProjectionRunRow(existingRow)
161
165
  const { attempt } = planProjectionRunReclaim(existing, input)
162
- const executionToken = createFreshExecutionToken(existing.executionToken)
166
+ const executionToken =
167
+ input.execution?.token ?? createFreshExecutionToken(existing.executionToken)
163
168
  const [updated] = await tx<DatabaseRow[]>`
164
169
  UPDATE projection_runs
165
170
  SET status = ${"running"}, started_at = COALESCE(started_at, ${input.startedAt ?? new Date()}),
166
- finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken}
171
+ finished_at = ${null}, attempt = ${attempt}, execution_token = ${executionToken},
172
+ execution_queue_lease_expires_at = ${input.execution?.queueLeaseExpiresAt ?? null}
167
173
  WHERE project_id = ${input.projectId} AND id = ${input.id}
168
174
  AND (
169
175
  (status = ${"queued"} AND execution_token IS NULL)
@@ -176,6 +182,54 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
176
182
  })
177
183
  }
178
184
 
185
+ async confirmExecutionOwnership(input: ConfirmRunExecutionInput): Promise<ProjectionRunRecord> {
186
+ return runPgTransaction(this.sql, async (tx) => {
187
+ const [existingRow] = await tx<DatabaseRow[]>`
188
+ SELECT * FROM projection_runs
189
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
190
+ FOR UPDATE
191
+ `
192
+ if (!existingRow) throw projectionRunNotFound(input.projectId, input.id)
193
+
194
+ const existing = restoreProjectionRunRow(existingRow)
195
+ if (existing.status !== "running" || existing.executionToken !== input.executionToken) {
196
+ throw staleProjectionRunExecution(input.id)
197
+ }
198
+
199
+ const [updated] = await tx<DatabaseRow[]>`
200
+ UPDATE projection_runs
201
+ SET execution_queue_lease_expires_at = ${laterLease(
202
+ existing.queueLeaseExpiresAt,
203
+ input.queueLeaseExpiresAt
204
+ )}
205
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
206
+ RETURNING *
207
+ `
208
+ return rowToProjectionRunRecord(updated)
209
+ })
210
+ }
211
+
212
+ /** Starts from `(project_id, status, ...)`: running rows are few however long the history. */
213
+ async listLapsed(input: ListLapsedProjectionRunsInput): Promise<readonly ProjectionRunRecord[]> {
214
+ const rows = await this.sql<DatabaseRow[]>`
215
+ SELECT * FROM projection_runs
216
+ WHERE project_id = ${input.projectId} AND status = ${"running"}
217
+ AND execution_queue_lease_expires_at <= ${input.lapsedBy}
218
+ ORDER BY execution_queue_lease_expires_at
219
+ LIMIT ${input.limit}
220
+ `
221
+
222
+ return rows.map(rowToProjectionRunRecord)
223
+ }
224
+
225
+ async releaseExecution(input: ReleaseRunExecutionInput): Promise<void> {
226
+ await this.sql`
227
+ UPDATE projection_runs SET execution_queue_lease_expires_at = ${input.releasedAt ?? new Date()}
228
+ WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
229
+ AND execution_token = ${input.executionToken}
230
+ `
231
+ }
232
+
179
233
  async failEnqueue(input: FailProjectionRunEnqueueInput): Promise<ProjectionRunRecord> {
180
234
  return runPgTransaction(this.sql, async (tx) => {
181
235
  const [existingRow] = await tx<DatabaseRow[]>`
@@ -263,6 +317,7 @@ export class PgProjectionRunStorage implements ProjectionRunStorage {
263
317
  status = ${input.status},
264
318
  finished_at = ${plan.finishedAt},
265
319
  execution_token = ${null},
320
+ execution_queue_lease_expires_at = ${null},
266
321
  source_rows_read = ${plan.progress.sourceRowsRead},
267
322
  source_rows_skipped = ${plan.progress.sourceRowsSkipped},
268
323
  source_changes_read = ${plan.progress.sourceChangesRead ?? null},
@@ -574,6 +629,9 @@ function restoreProjectionRunRow(row: DatabaseRow): StoredProjectionRunRecord {
574
629
  finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
575
630
  attempt: databaseSafeInteger(row.attempt, "attempt"),
576
631
  executionToken: row.execution_token ?? undefined,
632
+ queueLeaseExpiresAt: row.execution_queue_lease_expires_at
633
+ ? new Date(row.execution_queue_lease_expires_at)
634
+ : undefined,
577
635
  fixedBatchSize: optionalDatabaseSafeInteger(row.fixed_batch_size, "fixedBatchSize"),
578
636
  nextBatchOrdinal: optionalDatabaseSafeInteger(row.next_batch_ordinal, "nextBatchOrdinal"),
579
637
  nextRowOffset: optionalDatabaseSafeInteger(row.next_row_offset, "nextRowOffset"),
@@ -633,6 +691,7 @@ interface DatabaseRow {
633
691
  finished_at: Date | string | null
634
692
  attempt: number | string
635
693
  execution_token: string | null
694
+ execution_queue_lease_expires_at: Date | string | null
636
695
  materialization_protocol: "replacement" | "telemetry" | null
637
696
  dataset_version_created_at: string | null
638
697
  ontology_revision: string | null
@@ -1,25 +1,54 @@
1
1
  import type { JsonValue } from "@sixb/core"
2
2
  import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
3
- import { canFinishSyncRun, canRequeueSyncRunAfterEnqueueFailure } from "@sixb/core/internal/storage"
3
+ import {
4
+ canFinishSyncRun,
5
+ canRequeueSyncRunAfterEnqueueFailure,
6
+ mayFinishRun,
7
+ } from "@sixb/core/internal/storage"
4
8
  import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider"
5
9
  import type {
10
+ ClaimRunInput,
11
+ ConfirmRunExecutionInput,
6
12
  ExecutionStorage,
7
13
  FinishSyncRunInput,
8
14
  ListLatestSyncRunsInput,
9
15
  ListLatestSyncRunsResult,
16
+ ListLostRunsInput,
10
17
  ListSyncRunsInput,
11
18
  ListSyncRunsResult,
19
+ PrepareSyncRunCommitInput,
12
20
  QueueSyncRunInput,
13
- StartSyncRunInput,
21
+ ReleaseRunExecutionInput,
22
+ RunClaim,
14
23
  SyncRunRecord,
15
24
  SyncRunStorage,
16
25
  } from "@sixb/core/storage"
17
26
  import { SYNC_RUN_FAILURE_CODES, SyncRunError } from "@sixb/core/storage"
18
27
  import { queryLatestRunsByOwnerId } from "./latest-run-query"
19
28
  import type { SqlParameter } from "./pg-client"
29
+ import {
30
+ claimPgRun,
31
+ confirmPgRunExecution,
32
+ listLostPgRuns,
33
+ lockHeldPgRun,
34
+ lockPgRun,
35
+ type PgRunExecutionColumns,
36
+ type PgRunTable,
37
+ pgRunExecutionFields,
38
+ releasePgRun,
39
+ } from "./run-claim-sql"
20
40
  import { appendRunListFilters, hasEmptyStatuses, queryRunList } from "./run-list-query"
21
41
  import { isUniqueViolation } from "./storage-errors"
22
- import { type PgStoreClient, runPgTransaction } from "./transactions"
42
+ import { lockAdvisoryKeys, type PgStoreClient, runPgTransaction } from "./transactions"
43
+
44
+ const SYNC_RUNS: PgRunTable<DatabaseRow, SyncRunRecord> = {
45
+ name: "sync_runs",
46
+ returning: "*, checkpoint IS NOT NULL AS checkpoint_present",
47
+ toRun: rowToSyncRunRecord,
48
+ notFound: (projectId, id) =>
49
+ new SyncRunError(`[SixbPg] Sync run '${id}' not found for project '${projectId}'.`),
50
+ staleExecution,
51
+ }
23
52
 
24
53
  export class PgSyncRunStorage implements SyncRunStorage {
25
54
  constructor(
@@ -91,7 +120,9 @@ export class PgSyncRunStorage implements SyncRunStorage {
91
120
  const [updated] = await tx<DatabaseRow[]>`
92
121
  UPDATE sync_runs
93
122
  SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date()},
94
- started_at = ${null}, finished_at = ${null}, rows_read = ${null},
123
+ attempt = ${0}, execution_token = ${null}, execution_queue_lease_expires_at = ${null},
124
+ commit_prepared_at = ${null}, started_at = ${null}, finished_at = ${null},
125
+ rows_read = ${null},
95
126
  output_version_id = ${null}, error = ${null}, checkpoint = ${null}
96
127
  WHERE project_id = ${input.projectId} AND id = ${input.id}
97
128
  RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
@@ -100,53 +131,70 @@ export class PgSyncRunStorage implements SyncRunStorage {
100
131
  })
101
132
  }
102
133
 
103
- async start(input: StartSyncRunInput): Promise<SyncRunRecord> {
104
- const [updated] = await this.sql<DatabaseRow[]>`
105
- UPDATE sync_runs
106
- SET status = ${"running"}, started_at = ${input.startedAt ?? new Date()},
107
- error = ${null}
108
- WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"queued"}
109
- RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
110
- `
111
- if (updated) return rowToSyncRunRecord(updated)
134
+ async startOrReclaim(input: ClaimRunInput): Promise<RunClaim<SyncRunRecord>> {
135
+ return runPgTransaction(this.sql, async (tx) => {
136
+ const { syncId } = await lockPgRun(tx, SYNC_RUNS, input.projectId, input.id)
112
137
 
113
- const existing = await this.getById({ projectId: input.projectId, id: input.id })
114
- if (!existing) {
115
- throw new SyncRunError(
116
- `[SixbPg] Sync run '${input.id}' not found for project '${input.projectId}'.`
117
- )
118
- }
119
- throw new SyncRunError(
120
- `[SixbPg] Sync run '${input.id}' cannot start from status '${existing.status}'.`
121
- )
138
+ // Two queued runs of one Sync would each see the other not running yet.
139
+ await lockAdvisoryKeys(tx, [syncRunsLockKey(input.projectId, syncId)])
140
+
141
+ const [sibling] = await tx<DatabaseRow[]>`
142
+ SELECT *, checkpoint IS NOT NULL AS checkpoint_present FROM sync_runs
143
+ WHERE project_id = ${input.projectId} AND sync_id = ${syncId}
144
+ AND status = ${"running"} AND id <> ${input.id}
145
+ LIMIT 1
146
+ `
147
+ return claimPgRun(tx, SYNC_RUNS, input, sibling && rowToSyncRunRecord(sibling))
148
+ })
122
149
  }
123
150
 
124
- async finish(input: FinishSyncRunInput): Promise<SyncRunRecord> {
151
+ async confirmExecutionOwnership(input: ConfirmRunExecutionInput): Promise<SyncRunRecord> {
152
+ return runPgTransaction(this.sql, (tx) => confirmPgRunExecution(tx, SYNC_RUNS, input))
153
+ }
154
+
155
+ async listLost(input: ListLostRunsInput): Promise<readonly SyncRunRecord[]> {
156
+ return listLostPgRuns(this.sql, SYNC_RUNS, input)
157
+ }
158
+
159
+ async releaseExecution(input: ReleaseRunExecutionInput): Promise<void> {
160
+ await releasePgRun(this.sql, SYNC_RUNS, input)
161
+ }
162
+
163
+ async prepareCommit(input: PrepareSyncRunCommitInput): Promise<SyncRunRecord> {
125
164
  return runPgTransaction(this.sql, async (tx) => {
126
- const [existing] = await tx<DatabaseRow[]>`
127
- SELECT *, checkpoint IS NOT NULL AS checkpoint_present FROM sync_runs
165
+ await lockHeldPgRun(tx, SYNC_RUNS, input)
166
+
167
+ const [updated] = await tx<DatabaseRow[]>`
168
+ UPDATE sync_runs
169
+ SET rows_read = ${input.rowsRead},
170
+ checkpoint = ${serializeCheckpoint(input.checkpoint)}::text::jsonb,
171
+ commit_prepared_at = ${input.preparedAt ?? new Date()}
128
172
  WHERE project_id = ${input.projectId} AND id = ${input.id}
173
+ RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
129
174
  `
175
+ return rowToSyncRunRecord(updated)
176
+ })
177
+ }
130
178
 
131
- if (!existing) {
132
- throw new SyncRunError(
133
- `[SixbPg] Sync run '${input.id}' not found for project '${input.projectId}'.`
134
- )
135
- }
179
+ async finish(input: FinishSyncRunInput): Promise<SyncRunRecord> {
180
+ return runPgTransaction(this.sql, async (tx) => {
181
+ const record = await lockPgRun(tx, SYNC_RUNS, input.projectId, input.id)
136
182
 
137
- if (!canFinishSyncRun(existing.status, input)) {
183
+ if (!canFinishSyncRun(record.status, input)) {
138
184
  throw new SyncRunError(
139
- `[SixbPg] Sync run '${input.id}' cannot finish from status '${existing.status}'.`
185
+ `[SixbPg] Sync run '${input.id}' cannot finish from status '${record.status}'.`
140
186
  )
141
187
  }
142
188
 
189
+ if (!mayFinishRun(record, input)) throw staleExecution(input.id)
190
+
143
191
  if (input.status === "succeeded") {
144
- if (input.output && input.output.datasetId !== existing.dataset_id) {
192
+ if (input.output && input.output.datasetId !== record.datasetId) {
145
193
  throw new SyncRunError(
146
- `[SixbPg] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${existing.dataset_id}'.`
194
+ `[SixbPg] Sync run '${input.id}' output dataset '${input.output.datasetId}' does not match '${record.datasetId}'.`
147
195
  )
148
196
  }
149
- if (!input.output && input.rowsRead !== 0 && existing.mode !== "merge") {
197
+ if (!input.output && input.rowsRead !== 0 && record.mode !== "merge") {
150
198
  throw new SyncRunError(
151
199
  `[SixbPg] Sync run '${input.id}' may omit its output with rows read only for an initial merge no-op.`
152
200
  )
@@ -159,6 +207,8 @@ export class PgSyncRunStorage implements SyncRunStorage {
159
207
  UPDATE sync_runs
160
208
  SET
161
209
  status = ${input.status},
210
+ execution_token = ${null},
211
+ execution_queue_lease_expires_at = ${null},
162
212
  finished_at = ${input.finishedAt ?? new Date()},
163
213
  rows_read = ${input.rowsRead},
164
214
  output_version_id = ${input.output?.versionId ?? null},
@@ -171,8 +221,10 @@ export class PgSyncRunStorage implements SyncRunStorage {
171
221
  UPDATE sync_runs
172
222
  SET
173
223
  status = ${input.status},
224
+ execution_token = ${null},
225
+ execution_queue_lease_expires_at = ${null},
174
226
  finished_at = ${input.finishedAt ?? new Date()},
175
- rows_read = ${input.rowsRead ?? existing.rows_read ?? null},
227
+ rows_read = ${input.rowsRead ?? record.rowsRead ?? null},
176
228
  output_version_id = ${null},
177
229
  error = ${input.error === undefined ? null : serializeSixbFailure(input.error, SYNC_RUN_FAILURE_CODES)}::text::jsonb,
178
230
  checkpoint = ${null}
@@ -266,6 +318,14 @@ export class PgSyncRunStorage implements SyncRunStorage {
266
318
  }
267
319
  }
268
320
 
321
+ function staleExecution(id: string): SyncRunError {
322
+ return new SyncRunError(`[SixbPg] Execution token is no longer current on sync run '${id}'.`)
323
+ }
324
+
325
+ function syncRunsLockKey(projectId: string, syncId: string): string {
326
+ return `sync-runs:${JSON.stringify([projectId, syncId])}`
327
+ }
328
+
269
329
  function serializeCheckpoint(checkpoint: JsonValue | undefined): string | null {
270
330
  return checkpoint !== undefined ? JSON.stringify(checkpoint) : null
271
331
  }
@@ -279,6 +339,8 @@ function rowToSyncRunRecord(row: DatabaseRow): SyncRunRecord {
279
339
  datasetId: row.dataset_id,
280
340
  mode: row.mode,
281
341
  status: row.status,
342
+ ...pgRunExecutionFields(row),
343
+ commitPreparedAt: row.commit_prepared_at ? new Date(row.commit_prepared_at) : undefined,
282
344
  queuedAt: new Date(row.queued_at),
283
345
  startedAt: row.started_at ? new Date(row.started_at) : undefined,
284
346
  finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
@@ -306,7 +368,7 @@ function hasCheckpoint(row: DatabaseRow): boolean {
306
368
  )
307
369
  }
308
370
 
309
- interface DatabaseRow {
371
+ interface DatabaseRow extends PgRunExecutionColumns {
310
372
  project_id: string
311
373
  id: string
312
374
  execution_id: string
@@ -314,6 +376,7 @@ interface DatabaseRow {
314
376
  dataset_id: string
315
377
  mode: SyncRunRecord["mode"]
316
378
  status: SyncRunRecord["status"]
379
+ commit_prepared_at: Date | string | null
317
380
  queued_at: Date | string
318
381
  started_at: Date | string | null
319
382
  finished_at: Date | string | null
@@ -6,6 +6,7 @@ import type {
6
6
  GetWebhookRunByDeliveryInput,
7
7
  ListWebhookRunsInput,
8
8
  ListWebhookRunsResult,
9
+ RenewWebhookRunInput,
9
10
  RestartWebhookRunInput,
10
11
  StartWebhookRunInput,
11
12
  WebhookRunRecord,
@@ -35,10 +36,12 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
35
36
  const [row] = await this.sql<WebhookRunDatabaseRow[]>`
36
37
  INSERT INTO webhook_runs (
37
38
  project_id, id, execution_id, connector_id, webhook_id, status, method, route,
39
+ execution_token, execution_lease_expires_at,
38
40
  started_at, request_body_bytes, request_body_sha256, idempotency_key
39
41
  ) VALUES (
40
42
  ${input.projectId}, ${input.id}, ${input.executionId}, ${input.connectorId},
41
43
  ${input.webhookId}, ${"running"}, ${input.method}, ${input.route},
44
+ ${input.execution.token}, ${input.execution.leaseExpiresAt},
42
45
  ${input.startedAt ?? new Date()}, ${input.requestBodyBytes},
43
46
  ${input.requestBodySha256}, ${input.idempotencyKey ?? null}
44
47
  )
@@ -69,7 +72,8 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
69
72
  "not_found"
70
73
  )
71
74
  }
72
- if (!canRetryWebhookRun(rowToWebhookRunRecord(existing))) {
75
+ const startedAt = input.startedAt ?? new Date()
76
+ if (!canRetryWebhookRun(rowToWebhookRunRecord(existing), startedAt)) {
73
77
  throw new WebhookRunError(
74
78
  `[SixbPg] Webhook run '${input.id}' cannot restart from status '${existing.status}'.`,
75
79
  "invalid_transition"
@@ -78,21 +82,27 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
78
82
 
79
83
  const [updated] = await tx<WebhookRunDatabaseRow[]>`
80
84
  UPDATE webhook_runs
81
- SET status = ${"running"}, started_at = ${input.startedAt ?? new Date()},
82
- finished_at = ${null}, response_status = ${null}, error = ${null}
83
- WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"failed"}
85
+ SET status = ${"running"}, execution_token = ${input.execution.token},
86
+ execution_lease_expires_at = ${input.execution.leaseExpiresAt},
87
+ started_at = ${startedAt}, finished_at = ${null}, response_status = ${null},
88
+ error = ${null}
89
+ WHERE project_id = ${input.projectId} AND id = ${input.id}
84
90
  RETURNING *
85
91
  `
86
- if (!updated) {
87
- throw new WebhookRunError(
88
- `[SixbPg] Webhook run '${input.id}' changed while restarting.`,
89
- "invalid_transition"
90
- )
91
- }
92
92
  return rowToWebhookRunRecord(updated)
93
93
  })
94
94
  }
95
95
 
96
+ async renewLease(input: RenewWebhookRunInput): Promise<boolean> {
97
+ const updated = await this.sql<WebhookRunDatabaseRow[]>`
98
+ UPDATE webhook_runs SET execution_lease_expires_at = ${input.leaseExpiresAt}
99
+ WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"running"}
100
+ AND execution_token = ${input.executionToken}
101
+ RETURNING *
102
+ `
103
+ return updated.length > 0
104
+ }
105
+
96
106
  async finish(input: FinishWebhookRunInput): Promise<WebhookRunRecord> {
97
107
  return runPgTransaction(this.sql, async (tx) => {
98
108
  const [existing] = await tx<WebhookRunDatabaseRow[]>`
@@ -112,10 +122,17 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
112
122
  "invalid_transition"
113
123
  )
114
124
  }
125
+ if (existing.execution_token !== input.executionToken) {
126
+ throw new WebhookRunError(
127
+ `[SixbPg] Execution token is no longer current on webhook run '${input.id}'.`,
128
+ "invalid_transition"
129
+ )
130
+ }
115
131
 
116
132
  const [updated] = await tx<WebhookRunDatabaseRow[]>`
117
133
  UPDATE webhook_runs
118
- SET status = ${input.status}, finished_at = ${input.finishedAt ?? new Date()},
134
+ SET status = ${input.status}, execution_token = ${null},
135
+ execution_lease_expires_at = ${null}, finished_at = ${input.finishedAt ?? new Date()},
119
136
  response_status = ${input.responseStatus ?? null},
120
137
  error = ${input.status === "succeeded" ? null : serializeSixbFailure(input.error, WEBHOOK_RUN_FAILURE_CODES)}::text::jsonb
121
138
  WHERE project_id = ${input.projectId} AND id = ${input.id}
@@ -186,6 +203,13 @@ function rowToWebhookRunRecord(row: WebhookRunDatabaseRow): WebhookRunRecord {
186
203
  connectorId: row.connector_id,
187
204
  webhookId: row.webhook_id,
188
205
  status: row.status,
206
+ execution:
207
+ row.execution_token && row.execution_lease_expires_at
208
+ ? {
209
+ token: row.execution_token,
210
+ leaseExpiresAt: new Date(row.execution_lease_expires_at),
211
+ }
212
+ : undefined,
189
213
  method: row.method,
190
214
  route: row.route,
191
215
  startedAt: new Date(row.started_at),
@@ -205,6 +229,8 @@ interface WebhookRunDatabaseRow {
205
229
  readonly connector_id: string
206
230
  readonly webhook_id: string
207
231
  readonly status: WebhookRunRecord["status"]
232
+ readonly execution_token: string | null
233
+ readonly execution_lease_expires_at: Date | string | null
208
234
  readonly method: string
209
235
  readonly route: string
210
236
  readonly started_at: Date | string