@sixb/pg 0.1.0 → 0.1.2

package/src/migrations/025-connector-connections.sql

@@ -0,0 +1,231 @@
+CREATE TABLE connector_connection_runs (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  kind TEXT NOT NULL CHECK (kind IN ('connect', 'reauthorize')),
+  slot TEXT NOT NULL,
+  initiated_by_execution_id TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('waiting', 'running', 'succeeded', 'failed', 'cancelled', 'expired')),
+  waiting_for TEXT CHECK (waiting_for IN ('provider_authorization', 'account_selection')),
+  processing_id TEXT,
+  callback_started_at TIMESTAMPTZ,
+  expires_at TIMESTAMPTZ,
+  authorization_id TEXT,
+  cleanup_authorization_id TEXT,
+  connections JSONB CHECK (connections IS NULL OR jsonb_typeof(connections) = 'array'),
+  error JSONB CHECK (error IS NULL OR jsonb_typeof(error) = 'object'),
+  created_at TIMESTAMPTZ NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL,
+  finished_at TIMESTAMPTZ,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  CHECK (
+    (
+      status = 'waiting'
+      AND waiting_for = 'provider_authorization'
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NOT NULL
+      AND authorization_id IS NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NULL
+    )
+    OR (
+      status = 'waiting'
+      AND waiting_for = 'account_selection'
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NOT NULL
+      AND authorization_id IS NOT NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NULL
+    )
+    OR (
+      status = 'running'
+      AND waiting_for IS NULL
+      AND processing_id IS NOT NULL
+      AND callback_started_at IS NOT NULL
+      AND expires_at IS NOT NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NULL
+    )
+    OR (
+      status = 'succeeded'
+      AND waiting_for IS NULL
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NULL
+      AND authorization_id IS NOT NULL
+      AND connections IS NOT NULL
+      AND error IS NULL
+      AND finished_at IS NOT NULL
+    )
+    OR (
+      status = 'failed'
+      AND waiting_for IS NULL
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NOT NULL
+      AND finished_at IS NOT NULL
+    )
+    OR (
+      status IN ('cancelled', 'expired')
+      AND waiting_for IS NULL
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NOT NULL
+    )
+  )
+);
+
+CREATE INDEX idx_connector_runs_status
+  ON connector_connection_runs (project_id, connector_id, status, updated_at);
+
+CREATE TABLE connector_authorization_attempts (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  slot TEXT NOT NULL,
+  initiated_by_execution_id TEXT NOT NULL,
+  state_hash TEXT NOT NULL,
+  code_verifier JSONB NOT NULL CHECK (jsonb_typeof(code_verifier) = 'object'),
+  redirect_uri TEXT NOT NULL,
+  connection_run_id TEXT,
+  return_to TEXT,
+  callback_binding_hash TEXT,
+  reauthorization_id TEXT,
+  reauthorization_revision INTEGER CHECK (reauthorization_revision IS NULL OR reauthorization_revision >= 0),
+  reauthorization_connection_ids JSONB CHECK (
+    reauthorization_connection_ids IS NULL OR jsonb_typeof(reauthorization_connection_ids) = 'array'
+  ),
+  created_at TIMESTAMPTZ NOT NULL,
+  expires_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  UNIQUE (project_id, connector_id, connection_run_id),
+  FOREIGN KEY (project_id, connector_id, connection_run_id)
+    REFERENCES connector_connection_runs (project_id, connector_id, id)
+    ON DELETE RESTRICT,
+  CHECK (
+    (connection_run_id IS NULL AND return_to IS NULL AND callback_binding_hash IS NULL)
+    OR (connection_run_id IS NOT NULL AND return_to IS NOT NULL AND callback_binding_hash IS NOT NULL)
+  ),
+  CHECK (
+    (reauthorization_id IS NULL AND reauthorization_revision IS NULL AND reauthorization_connection_ids IS NULL)
+    OR (reauthorization_id IS NOT NULL AND reauthorization_revision IS NOT NULL AND reauthorization_connection_ids IS NOT NULL)
+  )
+);
+
+CREATE INDEX idx_connector_attempts_expiry
+  ON connector_authorization_attempts (project_id, connector_id, expires_at);
+
+CREATE TABLE connector_authorizations (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  authorized_by JSONB NOT NULL CHECK (jsonb_typeof(authorized_by) = 'object'),
+  credentials JSONB CHECK (credentials IS NULL OR jsonb_typeof(credentials) = 'object'),
+  credential_expires_at TIMESTAMPTZ,
+  scopes JSONB NOT NULL CHECK (jsonb_typeof(scopes) = 'array'),
+  accounts JSONB NOT NULL CHECK (jsonb_typeof(accounts) = 'array'),
+  status TEXT NOT NULL CHECK (status IN ('pending_selection', 'active', 'needs_reauthorization', 'revocation_pending', 'revoked')),
+  selection_expires_at TIMESTAMPTZ,
+  revision INTEGER NOT NULL CHECK (revision >= 0),
+  mutation_id TEXT,
+  mutation_kind TEXT CHECK (mutation_kind IN ('refresh', 'reauthorization', 'revocation')),
+  mutation_phase TEXT CHECK (mutation_phase IN ('prepared', 'executing', 'result_staged')),
+  mutation_holder_id TEXT,
+  mutation_expires_at TIMESTAMPTZ,
+  mutation_deadline_at TIMESTAMPTZ,
+  mutation_expected_connection_ids JSONB CHECK (
+    mutation_expected_connection_ids IS NULL OR jsonb_typeof(mutation_expected_connection_ids) = 'array'
+  ),
+  staged_credentials JSONB CHECK (
+    staged_credentials IS NULL OR jsonb_typeof(staged_credentials) = 'object'
+  ),
+  staged_credential_expires_at TIMESTAMPTZ,
+  staged_scopes JSONB CHECK (staged_scopes IS NULL OR jsonb_typeof(staged_scopes) = 'array'),
+  created_at TIMESTAMPTZ NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  CHECK (
+    (mutation_id IS NULL AND mutation_kind IS NULL AND mutation_phase IS NULL AND mutation_holder_id IS NULL AND mutation_expires_at IS NULL AND mutation_deadline_at IS NULL AND mutation_expected_connection_ids IS NULL AND staged_credentials IS NULL AND staged_credential_expires_at IS NULL AND staged_scopes IS NULL)
+    OR (mutation_id IS NOT NULL AND mutation_kind IS NOT NULL AND mutation_phase IS NOT NULL AND mutation_holder_id IS NOT NULL AND mutation_expires_at IS NOT NULL AND mutation_deadline_at IS NOT NULL)
+  ),
+  CHECK (
+    (
+      mutation_phase IS NULL
+      AND staged_credentials IS NULL
+      AND staged_credential_expires_at IS NULL
+      AND staged_scopes IS NULL
+    )
+    OR (
+      mutation_phase IN ('prepared', 'executing')
+      AND staged_credentials IS NULL
+      AND staged_credential_expires_at IS NULL
+      AND staged_scopes IS NULL
+    )
+    OR (
+      mutation_phase = 'result_staged'
+      AND mutation_kind = 'revocation'
+      AND staged_credentials IS NULL
+      AND staged_credential_expires_at IS NULL
+      AND staged_scopes IS NULL
+    )
+    OR (
+      mutation_phase = 'result_staged'
+      AND mutation_kind IN ('refresh', 'reauthorization')
+      AND staged_credentials IS NOT NULL
+      AND staged_scopes IS NOT NULL
+    )
+  ),
+  CHECK ((status = 'pending_selection') = (selection_expires_at IS NOT NULL)),
+  CHECK ((status = 'revoked') = (credentials IS NULL))
+);
+
+CREATE INDEX idx_connector_authorizations_status
+  ON connector_authorizations (project_id, connector_id, status, updated_at);
+CREATE INDEX idx_connector_authorizations_mutation_expiry
+  ON connector_authorizations (project_id, connector_id, mutation_expires_at)
+  WHERE mutation_id IS NOT NULL;
+
+CREATE TABLE connector_connections (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  authorization_id TEXT NOT NULL,
+  slot TEXT NOT NULL,
+  account JSONB NOT NULL CHECK (jsonb_typeof(account) = 'object'),
+  account_id TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('connected', 'disconnected')),
+  disconnected_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  UNIQUE (project_id, connector_id, slot),
+  FOREIGN KEY (project_id, connector_id, authorization_id)
+    REFERENCES connector_authorizations (project_id, connector_id, id)
+    ON DELETE RESTRICT,
+  CHECK ((status = 'connected') = (disconnected_at IS NULL)),
+  CHECK (account ? 'id' AND account ->> 'id' = account_id)
+);
+
+CREATE INDEX idx_connector_connections_authorization
+  ON connector_connections (project_id, connector_id, authorization_id, status, id);
+CREATE INDEX idx_connector_connections_status
+  ON connector_connections (project_id, connector_id, status, id);

package/src/migrations.ts

@@ -14,6 +14,58 @@ import {
   step,
 } from "@sixb/core/storage"
 import initialSchemaSql from "./migrations/001-initial-schema.sql" with { type: "text" }
+import workflowRunOutputSql from "./migrations/002-workflow-run-output.sql" with { type: "text" }
+import mergeSyncRunsSql from "./migrations/003-merge-sync-runs.sql" with { type: "text" }
+import executionsSql from "./migrations/004-executions.sql" with { type: "text" }
+import workflowExecutionsSql from "./migrations/005-workflow-executions.sql" with { type: "text" }
+import narrowOntologySourceRootIndexSql from "./migrations/006-narrow-ontology-source-root-index.sql" with {
+  type: "text",
+}
+import splitOverridesSql from "./migrations/007-split-overrides.sql" with { type: "text" }
+import actionExecutionsSql from "./migrations/008-action-executions.sql" with { type: "text" }
+import agentExecutionsSql from "./migrations/009-agent-executions.sql" with { type: "text" }
+import aiUsageAccountingFoundationSql from "./migrations/010-ai-usage-accounting-foundation.sql" with {
+  type: "text",
+}
+import syncFailureRecordSql from "./migrations/011-sync-failure-record.sql" with { type: "text" }
+import pipelineFailureRecordSql from "./migrations/012-pipeline-failure-record.sql" with {
+  type: "text",
+}
+import workflowFailureRecordSql from "./migrations/013-workflow-failure-record.sql" with {
+  type: "text",
+}
+import agentFailureRecordSql from "./migrations/014-agent-failure-record.sql" with { type: "text" }
+import projectionFailureRecordSql from "./migrations/015-projection-failure-record.sql" with {
+  type: "text",
+}
+import webhookRunFailureRecordSql from "./migrations/016-webhook-run-failure-record.sql" with {
+  type: "text",
+}
+import actionFailureRecordSql from "./migrations/017-action-failure-record.sql" with {
+  type: "text",
+}
+import ontologyOutboxFailureRecordSql from "./migrations/018-ontology-outbox-failure-record.sql" with {
+  type: "text",
+}
+import webhookDeliveryFailureRecordSql from "./migrations/019-webhook-delivery-failure-record.sql" with {
+  type: "text",
+}
+import dropRunUsageProjectionsSql from "./migrations/020-drop-run-usage-projections.sql" with {
+  type: "text",
+}
+import syncPipelineExecutionsSql from "./migrations/021-sync-pipeline-executions.sql" with {
+  type: "text",
+}
+import projectionExecutionsSql from "./migrations/022-projection-executions.sql" with {
+  type: "text",
+}
+import webhookExecutionsSql from "./migrations/023-webhook-executions.sql" with { type: "text" }
+import ontologyCommitExecutionsSql from "./migrations/024-ontology-commit-executions.sql" with {
+  type: "text",
+}
+import connectorConnectionsSql from "./migrations/025-connector-connections.sql" with {
+  type: "text",
+}
 import type { SQL, SQLClient } from "./pg-client"
 
 export interface PostgresMigrationContext {
@@ -266,7 +318,33 @@ function checksum(value: string): string {
 
 export const postgresStorageMigrations = defineMigrations<PostgresMigrationContext>({
   adapterId: POSTGRES_STORAGE_ADAPTER_ID,
-  steps: [pgSql("001-initial-schema", initialSchemaSql)],
+  steps: [
+    pgSql("001-initial-schema", initialSchemaSql),
+    pgSql("002-workflow-run-output", workflowRunOutputSql),
+    pgSql("003-merge-sync-runs", mergeSyncRunsSql),
+    pgSql("004-executions", executionsSql),
+    pgSql("005-workflow-executions", workflowExecutionsSql),
+    pgSql("006-narrow-ontology-source-root-index", narrowOntologySourceRootIndexSql),
+    pgSql("007-split-overrides", splitOverridesSql),
+    pgSql("008-action-executions", actionExecutionsSql),
+    pgSql("009-agent-executions", agentExecutionsSql),
+    pgSql("010-ai-usage-accounting-foundation", aiUsageAccountingFoundationSql),
+    pgSql("011-sync-failure-record", syncFailureRecordSql),
+    pgSql("012-pipeline-failure-record", pipelineFailureRecordSql),
+    pgSql("013-workflow-failure-record", workflowFailureRecordSql),
+    pgSql("014-agent-failure-record", agentFailureRecordSql),
+    pgSql("015-projection-failure-record", projectionFailureRecordSql),
+    pgSql("016-webhook-run-failure-record", webhookRunFailureRecordSql),
+    pgSql("017-action-failure-record", actionFailureRecordSql),
+    pgSql("018-ontology-outbox-failure-record", ontologyOutboxFailureRecordSql),
+    pgSql("019-webhook-delivery-failure-record", webhookDeliveryFailureRecordSql),
+    pgSql("020-drop-run-usage-projections", dropRunUsageProjectionsSql),
+    pgSql("021-sync-pipeline-executions", syncPipelineExecutionsSql),
+    pgSql("022-projection-executions", projectionExecutionsSql),
+    pgSql("023-webhook-executions", webhookExecutionsSql),
+    pgSql("024-ontology-commit-executions", ontologyCommitExecutionsSql),
+    pgSql("025-connector-connections", connectorConnectionsSql),
+  ],
 })
 
 interface PostgresMigrationRow {

package/src/ontology-storage/materialization-session.ts

@@ -19,6 +19,7 @@ import type {
   StreamMaterializationWorkInput,
 } from "@sixb/core/storage"
 import type { SQLClient } from "../pg-client"
+import { isUniqueViolation } from "../storage-errors"
 import { PG_MATERIALIZATION_WORK_TABLE, PG_REPLACEMENT_WORK_TABLE } from "./materialization-state"
 import { jsonParameter } from "./shared"
 
@@ -119,21 +120,6 @@ export class PgMaterializationSessions {
     const session = this.require(input.session)
     const records = prepareMaterializationWork(session, input)
     if (records.length === 0) return
-    const keys = records.map(({ record }) => record.recordKey)
-    const uniqueKeys = records.map(({ uniqueKey }) => uniqueKey)
-
-    const [duplicate] = await this.sql<{ readonly record_key: string }[]>`
-      SELECT record_key
-      FROM ${this.sql(PG_MATERIALIZATION_WORK_TABLE)}
-      WHERE session_id = ${session.id}
-        AND (
-          record_key = ANY(${this.sql.array(keys)}::text[])
-          OR unique_key = ANY(${this.sql.array(uniqueKeys)}::text[])
-        )
-      LIMIT 1
-    `
-    if (duplicate) throw duplicateWork(duplicate.record_key)
-
     const payload = records.map(({ record, uniqueKey, columns }) => {
       return {
         recordKey: record.recordKey,
@@ -147,19 +133,30 @@ export class PgMaterializationSessions {
         record,
       }
     })
-    await this.sql`
-      WITH staged AS (
-        SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, payload)}::jsonb)
-      )
-      INSERT INTO ${this.sql(PG_MATERIALIZATION_WORK_TABLE)} (
-        session_id, record_key, unique_key, kind, lane,
-        major_order, minor_order, sort_one, sort_two, payload
-      )
-      SELECT ${session.id}, value->>'recordKey', value->>'uniqueKey', value->>'kind',
-        value->>'lane', (value->>'majorOrder')::integer, (value->>'minorOrder')::integer,
-        value->>'sortOne', value->>'sortTwo', value->'record'
-      FROM staged
-    `
+    // This session is the only writer to its transaction-local temp rows. Let the two unique
+    // constraints arbitrate cross-chunk duplicates instead of issuing a duplicate probe before
+    // every insert; a violation aborts the enclosing materialization transaction and is mapped
+    // back to the provider contract below.
+    try {
+      await this.sql`
+        WITH staged AS (
+          SELECT value FROM jsonb_array_elements(${jsonParameter(this.sql, payload)}::jsonb)
+        )
+        INSERT INTO ${this.sql(PG_MATERIALIZATION_WORK_TABLE)} (
+          session_id, record_key, unique_key, kind, lane,
+          major_order, minor_order, sort_one, sort_two, payload
+        )
+        SELECT ${session.id}, value->>'recordKey', value->>'uniqueKey', value->>'kind',
+          value->>'lane', (value->>'majorOrder')::integer, (value->>'minorOrder')::integer,
+          value->>'sortOne', value->>'sortTwo', value->'record'
+        FROM staged
+      `
+    } catch (error) {
+      if (isUniqueViolation(error)) {
+        throw duplicateWork()
+      }
+      throw error
+    }
   }
 
   async *stream(input: StreamMaterializationWorkInput): AsyncIterable<MaterializationWorkPage> {