@sixb/pg 0.1.1 → 0.1.3

package/src/pg-sync-run-storage.ts

@@ -1,16 +1,23 @@
 import type { JsonValue } from "@sixb/core"
+import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
+import { assertSyncRunExecution } from "@sixb/core/internal/sync-run-storage-provider"
 import type {
+  ExecutionStorage,
   FinishSyncRunInput,
   ListLatestSyncRunsInput,
   ListLatestSyncRunsResult,
   ListSyncRunsInput,
   ListSyncRunsResult,
+  QueueSyncRunInput,
   StartSyncRunInput,
-  SyncRunFailure,
   SyncRunRecord,
   SyncRunStorage,
 } from "@sixb/core/storage"
-import { SyncRunError } from "@sixb/core/storage"
+import {
+  canRequeueSyncRunAfterEnqueueFailure,
+  SYNC_RUN_FAILURE_CODES,
+  SyncRunError,
+} from "@sixb/core/storage"
 import { queryLatestRunsByOwnerId } from "./latest-run-query"
 import type { SqlParameter } from "./pg-client"
 import { appendRunListFilters, hasEmptyStatuses, queryRunList } from "./run-list-query"
@@ -18,29 +25,43 @@ import { isUniqueViolation } from "./storage-errors"
 import { type PgStoreClient, runPgTransaction } from "./transactions"
 
 export class PgSyncRunStorage implements SyncRunStorage {
-  constructor(private readonly sql: PgStoreClient) {}
+  constructor(
+    private readonly sql: PgStoreClient,
+    private readonly executions: ExecutionStorage
+  ) {}
 
-  async start(input: StartSyncRunInput): Promise<SyncRunRecord> {
+  async queue(input: QueueSyncRunInput): Promise<SyncRunRecord> {
+    await assertSyncRunExecution({
+      executions: this.executions,
+      projectId: input.projectId,
+      executionId: input.executionId,
+      runId: input.id,
+      syncId: input.syncId,
+    })
     try {
       const [row] = await this.sql<DatabaseRow[]>`
         INSERT INTO sync_runs (
           project_id,
           id,
+          execution_id,
           sync_id,
           dataset_id,
           mode,
           status,
+          queued_at,
           started_at,
           expected_latest_version_id,
           commit_message
         ) VALUES (
           ${input.projectId},
           ${input.id},
+          ${input.executionId},
           ${input.syncId},
           ${input.datasetId},
           ${input.mode},
-          ${"running"},
-          ${input.startedAt ?? new Date()},
+          ${"queued"},
+          ${input.queuedAt ?? new Date()},
+          ${null},
           ${input.expectedLatestVersionId ?? null},
           ${input.commitMessage ?? null}
         )
@@ -50,13 +71,57 @@ export class PgSyncRunStorage implements SyncRunStorage {
       return rowToSyncRunRecord(row)
     } catch (error) {
       if (isUniqueViolation(error)) {
+        return this.requeueAfterEnqueueFailure(input)
+      }
+
+      throw error
+    }
+  }
+
+  private async requeueAfterEnqueueFailure(input: QueueSyncRunInput): Promise<SyncRunRecord> {
+    return runPgTransaction(this.sql, async (tx) => {
+      const [existing] = await tx<DatabaseRow[]>`
+        SELECT *, checkpoint IS NOT NULL AS checkpoint_present FROM sync_runs
+        WHERE project_id = ${input.projectId} AND id = ${input.id}
+        FOR UPDATE
+      `
+      if (!existing || !canRequeueSyncRunAfterEnqueueFailure(rowToSyncRunRecord(existing), input)) {
         throw new SyncRunError(
           `[SixbPg] Sync run '${input.id}' already exists for project '${input.projectId}'.`
         )
       }
 
-      throw error
+      const [updated] = await tx<DatabaseRow[]>`
+        UPDATE sync_runs
+        SET status = ${"queued"}, queued_at = ${input.queuedAt ?? new Date()},
+            started_at = ${null}, finished_at = ${null}, rows_read = ${null},
+            output_version_id = ${null}, error = ${null}, checkpoint = ${null}
+        WHERE project_id = ${input.projectId} AND id = ${input.id}
+        RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
+      `
+      return rowToSyncRunRecord(updated)
+    })
+  }
+
+  async start(input: StartSyncRunInput): Promise<SyncRunRecord> {
+    const [updated] = await this.sql<DatabaseRow[]>`
+      UPDATE sync_runs
+      SET status = ${"running"}, started_at = ${input.startedAt ?? new Date()},
+          error = ${null}
+      WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"queued"}
+      RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
+    `
+    if (updated) return rowToSyncRunRecord(updated)
+
+    const existing = await this.getById({ projectId: input.projectId, id: input.id })
+    if (!existing) {
+      throw new SyncRunError(
+        `[SixbPg] Sync run '${input.id}' not found for project '${input.projectId}'.`
+      )
     }
+    throw new SyncRunError(
+      `[SixbPg] Sync run '${input.id}' cannot start from status '${existing.status}'.`
+    )
   }
 
   async finish(input: FinishSyncRunInput): Promise<SyncRunRecord> {
@@ -72,6 +137,17 @@ export class PgSyncRunStorage implements SyncRunStorage {
         )
       }
 
+      const enqueueFailure =
+        input.status === "failed" && input.error.code === "queue.enqueue_failed"
+      if (
+        (enqueueFailure && existing.status !== "queued") ||
+        (!enqueueFailure && existing.status !== "running")
+      ) {
+        throw new SyncRunError(
+          `[SixbPg] Sync run '${input.id}' cannot finish from status '${existing.status}'.`
+        )
+      }
+
       if (input.status === "succeeded") {
         if (input.output && input.output.datasetId !== existing.dataset_id) {
           throw new SyncRunError(
@@ -94,8 +170,7 @@ export class PgSyncRunStorage implements SyncRunStorage {
                 finished_at = ${input.finishedAt ?? new Date()},
                 rows_read = ${input.rowsRead},
                 output_version_id = ${input.output?.versionId ?? null},
-                error_name = ${null},
-                error_message = ${null},
+                error = ${null},
                 checkpoint = ${serializeCheckpoint(input.checkpoint)}::text::jsonb
               WHERE project_id = ${input.projectId} AND id = ${input.id}
               RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
@@ -107,8 +182,7 @@ export class PgSyncRunStorage implements SyncRunStorage {
                 finished_at = ${input.finishedAt ?? new Date()},
                 rows_read = ${input.rowsRead ?? existing.rows_read ?? null},
                 output_version_id = ${null},
-                error_name = ${input.error?.name ?? null},
-                error_message = ${input.error?.message ?? null},
+                error = ${input.error === undefined ? null : serializeSixbFailure(input.error, SYNC_RUN_FAILURE_CODES)}::text::jsonb,
                 checkpoint = ${null}
               WHERE project_id = ${input.projectId} AND id = ${input.id}
               RETURNING *, checkpoint IS NOT NULL AS checkpoint_present
@@ -156,7 +230,13 @@ export class PgSyncRunStorage implements SyncRunStorage {
       params.push(input.datasetId)
     }
 
-    index = appendRunListFilters(whereClauses, params, index, input)
+    index = appendRunListFilters(
+      whereClauses,
+      params,
+      index,
+      input,
+      "COALESCE(started_at, queued_at)"
+    )
 
     const { rows, total, hasMore } = await queryRunList<DatabaseRow>({
       sql: this.sql,
@@ -198,26 +278,17 @@ function serializeCheckpoint(checkpoint: JsonValue | undefined): string | null {
   return checkpoint !== undefined ? JSON.stringify(checkpoint) : null
 }
 
-function toSyncRunFailure(row: DatabaseRow): SyncRunFailure | undefined {
-  if (!row.error_message) {
-    return undefined
-  }
-
-  return {
-    name: row.error_name ?? undefined,
-    message: row.error_message,
-  }
-}
-
 function rowToSyncRunRecord(row: DatabaseRow): SyncRunRecord {
   return {
     id: row.id,
     projectId: row.project_id,
+    executionId: row.execution_id,
     syncId: row.sync_id,
     datasetId: row.dataset_id,
     mode: row.mode,
     status: row.status,
-    startedAt: new Date(row.started_at),
+    queuedAt: new Date(row.queued_at),
+    startedAt: row.started_at ? new Date(row.started_at) : undefined,
     finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
     rowsRead: row.rows_read != null ? Number(row.rows_read) : undefined,
     output: row.output_version_id
@@ -228,7 +299,7 @@ function rowToSyncRunRecord(row: DatabaseRow): SyncRunRecord {
       : undefined,
     expectedLatestVersionId: row.expected_latest_version_id ?? undefined,
     commitMessage: row.commit_message ?? undefined,
-    error: toSyncRunFailure(row),
+    error: row.error === null ? undefined : parseSixbFailure(row.error, SYNC_RUN_FAILURE_CODES),
     checkpoint: hasCheckpoint(row) ? row.checkpoint : undefined,
   }
 }
@@ -246,18 +317,19 @@ function hasCheckpoint(row: DatabaseRow): boolean {
 interface DatabaseRow {
   project_id: string
   id: string
+  execution_id: string
   sync_id: string
   dataset_id: string
   mode: SyncRunRecord["mode"]
   status: SyncRunRecord["status"]
-  started_at: Date | string
+  queued_at: Date | string
+  started_at: Date | string | null
   finished_at: Date | string | null
   rows_read: number | string | null
   output_version_id: string | null
   expected_latest_version_id: string | null
   commit_message: string | null
-  error_name: string | null
-  error_message: string | null
+  error: JsonValue | null
   checkpoint: JsonValue | null
   checkpoint_present: boolean | number | string
 }

package/src/pg-webhook-run-storage.ts

@@ -1,91 +1,126 @@
+import { parseSixbFailure, serializeSixbFailure } from "@sixb/core/internal/errors"
+import { assertWebhookRunExecution } from "@sixb/core/internal/webhook-run-storage-provider"
 import type {
+  ExecutionStorage,
   FinishWebhookRunInput,
+  GetWebhookRunByDeliveryInput,
   ListWebhookRunsInput,
   ListWebhookRunsResult,
+  RestartWebhookRunInput,
   StartWebhookRunInput,
   WebhookRunRecord,
   WebhookRunStorage,
 } from "@sixb/core/storage"
-import { WebhookRunError } from "@sixb/core/storage"
+import { canRetryWebhookRun, WEBHOOK_RUN_FAILURE_CODES, WebhookRunError } from "@sixb/core/storage"
 import type { SqlParameter } from "./pg-client"
 import { appendRunListFilters, hasEmptyStatuses, queryRunList } from "./run-list-query"
 import { isUniqueViolation } from "./storage-errors"
 import { type PgStoreClient, runPgTransaction } from "./transactions"
 
 export class PgWebhookRunStorage implements WebhookRunStorage {
-  constructor(private readonly sql: PgStoreClient) {}
+  constructor(
+    private readonly sql: PgStoreClient,
+    private readonly executions: ExecutionStorage
+  ) {}
 
   async start(input: StartWebhookRunInput): Promise<WebhookRunRecord> {
+    await assertWebhookRunExecution({
+      executions: this.executions,
+      projectId: input.projectId,
+      executionId: input.executionId,
+      runId: input.id,
+      route: input.route,
+    })
     try {
       const [row] = await this.sql<WebhookRunDatabaseRow[]>`
         INSERT INTO webhook_runs (
-          project_id,
-          id,
-          connector_id,
-          webhook_id,
-          status,
-          method,
-          route,
-          started_at
+          project_id, id, execution_id, connector_id, webhook_id, status, method, route,
+          started_at, request_body_bytes, request_body_sha256, idempotency_key
         ) VALUES (
-          ${input.projectId},
-          ${input.id},
-          ${input.connectorId},
-          ${input.webhookId},
-          ${"running"},
-          ${input.method},
-          ${input.route},
-          ${input.startedAt ?? new Date()}
+          ${input.projectId}, ${input.id}, ${input.executionId}, ${input.connectorId},
+          ${input.webhookId}, ${"running"}, ${input.method}, ${input.route},
+          ${input.startedAt ?? new Date()}, ${input.requestBodyBytes},
+          ${input.requestBodySha256}, ${input.idempotencyKey ?? null}
         )
         RETURNING *
       `
-
       return rowToWebhookRunRecord(row)
     } catch (error) {
       if (isUniqueViolation(error)) {
         throw new WebhookRunError(
-          `[SixbPg] Webhook run '${input.id}' already exists for project '${input.projectId}'.`
+          `[SixbPg] Webhook run '${input.id}' or delivery key already exists for project '${input.projectId}'.`,
+          "duplicate_run"
         )
       }
-
       throw error
     }
   }
 
-  async finish(input: FinishWebhookRunInput): Promise<WebhookRunRecord> {
+  async restart(input: RestartWebhookRunInput): Promise<WebhookRunRecord> {
     return runPgTransaction(this.sql, async (tx) => {
       const [existing] = await tx<WebhookRunDatabaseRow[]>`
         SELECT * FROM webhook_runs
         WHERE project_id = ${input.projectId} AND id = ${input.id}
         FOR UPDATE
       `
-
       if (!existing) {
         throw new WebhookRunError(
-          `[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`
+          `[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`,
+          "not_found"
+        )
+      }
+      if (!canRetryWebhookRun(rowToWebhookRunRecord(existing))) {
+        throw new WebhookRunError(
+          `[SixbPg] Webhook run '${input.id}' cannot restart from status '${existing.status}'.`,
+          "invalid_transition"
+        )
+      }
+
+      const [updated] = await tx<WebhookRunDatabaseRow[]>`
+        UPDATE webhook_runs
+        SET status = ${"running"}, started_at = ${input.startedAt ?? new Date()},
+            finished_at = ${null}, response_status = ${null}, error = ${null}
+        WHERE project_id = ${input.projectId} AND id = ${input.id} AND status = ${"failed"}
+        RETURNING *
+      `
+      if (!updated) {
+        throw new WebhookRunError(
+          `[SixbPg] Webhook run '${input.id}' changed while restarting.`,
+          "invalid_transition"
         )
       }
+      return rowToWebhookRunRecord(updated)
+    })
+  }
 
+  async finish(input: FinishWebhookRunInput): Promise<WebhookRunRecord> {
+    return runPgTransaction(this.sql, async (tx) => {
+      const [existing] = await tx<WebhookRunDatabaseRow[]>`
+        SELECT * FROM webhook_runs
+        WHERE project_id = ${input.projectId} AND id = ${input.id}
+        FOR UPDATE
+      `
+      if (!existing) {
+        throw new WebhookRunError(
+          `[SixbPg] Webhook run '${input.id}' not found for project '${input.projectId}'.`,
+          "not_found"
+        )
+      }
       if (existing.status !== "running") {
         throw new WebhookRunError(
-          `[SixbPg] Webhook run '${input.id}' for project '${input.projectId}' is already terminal.`
+          `[SixbPg] Webhook run '${input.id}' for project '${input.projectId}' is already terminal.`,
+          "invalid_transition"
         )
       }
 
       const [updated] = await tx<WebhookRunDatabaseRow[]>`
         UPDATE webhook_runs
-        SET
-          status = ${input.status},
-          finished_at = ${input.finishedAt ?? new Date()},
-          request_body_bytes = ${input.requestBodyBytes ?? null},
-          response_status = ${input.responseStatus ?? null},
-          idempotency_key = ${input.idempotencyKey ?? null},
-          delivery_claim_result = ${input.deliveryClaimResult ?? null},
-          error = ${input.status === "succeeded" ? null : (input.error ?? null)}
+        SET status = ${input.status}, finished_at = ${input.finishedAt ?? new Date()},
+            response_status = ${input.responseStatus ?? null},
+            error = ${input.status === "succeeded" ? null : serializeSixbFailure(input.error, WEBHOOK_RUN_FAILURE_CODES)}::text::jsonb
         WHERE project_id = ${input.projectId} AND id = ${input.id}
         RETURNING *
       `
-
       return rowToWebhookRunRecord(updated)
     })
   }
@@ -95,38 +130,38 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
       SELECT * FROM webhook_runs
       WHERE project_id = ${params.projectId} AND id = ${params.id}
     `
+    return row ? rowToWebhookRunRecord(row) : null
+  }
 
+  async getByDelivery(input: GetWebhookRunByDeliveryInput): Promise<WebhookRunRecord | null> {
+    const [row] = await this.sql<WebhookRunDatabaseRow[]>`
+      SELECT * FROM webhook_runs
+      WHERE project_id = ${input.projectId}
+        AND connector_id = ${input.connectorId}
+        AND webhook_id = ${input.webhookId}
+        AND idempotency_key = ${input.idempotencyKey}
+    `
     return row ? rowToWebhookRunRecord(row) : null
   }
 
   async list(input: ListWebhookRunsInput): Promise<ListWebhookRunsResult> {
-    if (hasEmptyStatuses(input)) {
-      return {
-        runs: [],
-        hasMore: false,
-        total: 0,
-      }
-    }
+    if (hasEmptyStatuses(input)) return { runs: [], hasMore: false, total: 0 }
 
     const whereClauses = ["project_id = $1"]
     const params: SqlParameter[] = [input.projectId]
     let index = 2
-
     if (input.connectorId) {
       whereClauses.push(`connector_id = $${index++}`)
       params.push(input.connectorId)
     }
-
     if (input.webhookId) {
       whereClauses.push(`webhook_id = $${index++}`)
       params.push(input.webhookId)
     }
-
     if (input.idempotencyKey) {
       whereClauses.push(`idempotency_key = $${index++}`)
       params.push(input.idempotencyKey)
     }
-
     index = appendRunListFilters(whereClauses, params, index, input)
 
     const { rows, total, hasMore } = await queryRunList<WebhookRunDatabaseRow>({
@@ -139,12 +174,7 @@ export class PgWebhookRunStorage implements WebhookRunStorage {
       limit: input.limit,
       offset: input.offset,
     })
-
-    return {
-      runs: rows.map(rowToWebhookRunRecord),
-      hasMore,
-      total,
-    }
+    return { runs: rows.map(rowToWebhookRunRecord), hasMore, total }
   }
 }
 
@@ -152,6 +182,7 @@ function rowToWebhookRunRecord(row: WebhookRunDatabaseRow): WebhookRunRecord {
   return {
     id: row.id,
     projectId: row.project_id,
+    executionId: row.execution_id,
     connectorId: row.connector_id,
     webhookId: row.webhook_id,
     status: row.status,
@@ -159,17 +190,18 @@ function rowToWebhookRunRecord(row: WebhookRunDatabaseRow): WebhookRunRecord {
     route: row.route,
     startedAt: new Date(row.started_at),
     finishedAt: row.finished_at ? new Date(row.finished_at) : undefined,
-    requestBodyBytes: row.request_body_bytes != null ? Number(row.request_body_bytes) : undefined,
+    requestBodyBytes: Number(row.request_body_bytes),
+    requestBodySha256: row.request_body_sha256,
     responseStatus: row.response_status != null ? Number(row.response_status) : undefined,
     idempotencyKey: row.idempotency_key ?? undefined,
-    deliveryClaimResult: row.delivery_claim_result ?? undefined,
-    error: row.error ?? undefined,
+    error: row.error === null ? undefined : parseSixbFailure(row.error, WEBHOOK_RUN_FAILURE_CODES),
   }
 }
 
 interface WebhookRunDatabaseRow {
   readonly project_id: string
   readonly id: string
+  readonly execution_id: string
   readonly connector_id: string
   readonly webhook_id: string
   readonly status: WebhookRunRecord["status"]
@@ -177,9 +209,9 @@ interface WebhookRunDatabaseRow {
   readonly route: string
   readonly started_at: Date | string
   readonly finished_at: Date | string | null
-  readonly request_body_bytes: number | string | null
+  readonly request_body_bytes: number | string
+  readonly request_body_sha256: string
   readonly response_status: number | string | null
   readonly idempotency_key: string | null
-  readonly delivery_claim_result: WebhookRunRecord["deliveryClaimResult"] | null
-  readonly error: string | null
+  readonly error: unknown | null
 }