@sixb/pg 0.1.1 → 0.1.3

package/src/index.ts

@@ -19,6 +19,8 @@ import {
   createWorkflowRunOperationScope,
 } from "@sixb/core/internal/storage-operation-scope"
 import {
+  type AiCostStorage,
+  type AiUsageStorage,
   createTransactionStorageProxy,
   type ObjectStorage,
   StorageTransactionError,
@@ -26,21 +28,25 @@ import {
 } from "@sixb/core/storage"
 import { PgAgentStorage } from "./agents"
 import { PgAuthStorage } from "./auth-storage"
+import { PgConnectorConnectionStorage } from "./connector-connection-storage"
 import { createPostgresStorageMigrators, dropSchema } from "./migrations"
 import { PgOntologyStorage, type PgOntologyTransactionContext } from "./ontology-storage"
 import { PgActionRunStorage } from "./pg-action-run-storage"
+import { PgAiCostStorage } from "./pg-ai-cost-storage"
+import { PgAiUsageStorage } from "./pg-ai-usage-storage"
 import { createPgClient, type SQL, type SQLClient } from "./pg-client"
+import { PgExecutionStorage } from "./pg-execution-storage"
 import { PgObjectStorage } from "./pg-object-storage"
 import { PgPipelineRunStorage } from "./pg-pipeline-run-storage"
 import { PgProjectionRunStorage } from "./pg-projection-run-storage"
 import { PgRulesStorage } from "./pg-rules-storage"
 import { PgSyncRunStorage } from "./pg-sync-run-storage"
 import { PgTimeseriesStorage } from "./pg-timeseries-storage"
-import { PgWebhookDeliveryStorage } from "./pg-webhook-delivery-storage"
 import { PgWebhookRunStorage } from "./pg-webhook-run-storage"
 import { PgWorkflowInterventionStorage } from "./pg-workflow-intervention-storage"
 import { PgWorkflowRunStorage } from "./pg-workflow-run-storage"
 import { isRetryableTransactionConflict } from "./storage-errors"
+import { registerPostgresStorageTestingAdapter } from "./testing"
 import { type PgStoreClient, runPgTransaction } from "./transactions"
 
 export interface PostgresStorageOptions {
@@ -106,8 +112,8 @@ export interface PostgresStorageOptions {
 /**
  * PostgreSQL storage provider for Sixb.
  *
- * Bundles Sixb storage adapters backed by a shared PostgreSQL connection pool (porsager
- * `postgres`), which reliably reclaims connections under load.
+ * Bundles Sixb storage adapters, including the AI accounting ledger, behind a shared PostgreSQL
+ * connection pool (porsager `postgres`), which reliably reclaims connections under load.
  *
  * The storage exposes a core `StorageMigrator`. Sixb CLI startup and
  * `sixb db migrate` run it automatically through `migrateStorage(storage)`.
@@ -128,7 +134,10 @@ export class PostgresStorage implements MigrationCapableStorage {
   readonly objects: ObjectStorage
   readonly ontology: PgOntologyStorage
   readonly auth: PgAuthStorage
+  readonly executions: PgExecutionStorage
   readonly agents: PgAgentStorage
+  readonly aiUsage: AiUsageStorage
+  readonly aiCosts: AiCostStorage
   readonly actionRuns: PgActionRunStorage
   readonly pipelineRuns: PgPipelineRunStorage
   readonly workflowRuns: PgWorkflowRunStorage
@@ -136,9 +145,9 @@ export class PostgresStorage implements MigrationCapableStorage {
   readonly syncRuns: PgSyncRunStorage
   readonly projectionRuns: PgProjectionRunStorage
   readonly timeseries: Storage["timeseries"]
-  readonly webhookDeliveries: PgWebhookDeliveryStorage
   readonly webhookRuns: PgWebhookRunStorage
   readonly rules: PgRulesStorage
+  readonly connectorConnections: PgConnectorConnectionStorage
   readonly migrators: readonly StorageMigrator[]
 
   private readonly sql: SQL
@@ -199,7 +208,10 @@ export class PostgresStorage implements MigrationCapableStorage {
     this.objects = createOperationScopedFacade(stores.objects, scope)
     this.ontology = createOntologyOperationScope(stores.ontology, scope)
     this.auth = createAuthOperationScope(stores.auth, scope)
+    this.executions = createOperationScopedFacade(stores.executions, scope)
     this.agents = createAgentOperationScope(stores.agents, scope)
+    this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope)
+    this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope)
     this.actionRuns = createOperationScopedFacade(stores.actionRuns, scope)
     this.pipelineRuns = createOperationScopedFacade(stores.pipelineRuns, scope)
     this.workflowRuns = createWorkflowRunOperationScope(stores.workflowRuns, scope)
@@ -207,9 +219,12 @@ export class PostgresStorage implements MigrationCapableStorage {
     this.syncRuns = createOperationScopedFacade(stores.syncRuns, scope)
     this.projectionRuns = createOperationScopedFacade(stores.projectionRuns, scope)
     this.timeseries = createOperationScopedFacade(stores.timeseries, scope)
-    this.webhookDeliveries = createOperationScopedFacade(stores.webhookDeliveries, scope)
     this.webhookRuns = createOperationScopedFacade(stores.webhookRuns, scope)
     this.rules = createOperationScopedFacade(stores.rules, scope)
+    this.connectorConnections = createOperationScopedFacade(stores.connectorConnections, scope)
+    registerPostgresStorageTestingAdapter(this, (durationMs) =>
+      stores.connectorConnections.advanceTimeForTesting(durationMs)
+    )
   }
 
   async ping(): Promise<void> {
@@ -313,6 +328,8 @@ function createPostgresStores(
     readonly transactionContext: PgOntologyTransactionContext | null
   }
 ): PostgresStoreSet {
+  const auth = new PgAuthStorage({ sql })
+  const executions = new PgExecutionStorage(sql, auth)
   return {
     objects: new PgObjectStorage(sql),
     ontology: new PgOntologyStorage({
@@ -320,18 +337,21 @@ function createPostgresStores(
       runRootOperation: options.runOntologyOperation,
       transactionContext: options.transactionContext,
     }),
-    auth: new PgAuthStorage({ sql }),
-    agents: new PgAgentStorage({ sql }),
-    actionRuns: new PgActionRunStorage(sql),
-    pipelineRuns: new PgPipelineRunStorage(sql),
-    workflowRuns: new PgWorkflowRunStorage(sql),
+    auth,
+    executions,
+    agents: new PgAgentStorage({ sql, executions }),
+    aiUsage: new PgAiUsageStorage(sql),
+    aiCosts: new PgAiCostStorage(sql),
+    actionRuns: new PgActionRunStorage(sql, executions),
+    pipelineRuns: new PgPipelineRunStorage(sql, executions),
+    workflowRuns: new PgWorkflowRunStorage(sql, executions),
     workflowInterventions: new PgWorkflowInterventionStorage(sql),
-    syncRuns: new PgSyncRunStorage(sql),
-    projectionRuns: new PgProjectionRunStorage(sql),
+    syncRuns: new PgSyncRunStorage(sql, executions),
+    projectionRuns: new PgProjectionRunStorage(sql, executions),
     timeseries: new PgTimeseriesStorage(sql),
-    webhookDeliveries: new PgWebhookDeliveryStorage(sql),
-    webhookRuns: new PgWebhookRunStorage(sql),
+    webhookRuns: new PgWebhookRunStorage(sql, executions),
     rules: new PgRulesStorage(sql),
+    connectorConnections: new PgConnectorConnectionStorage(sql),
   }
 }
 
@@ -339,7 +359,10 @@ interface PostgresStoreSet {
   readonly objects: PgObjectStorage
   readonly ontology: PgOntologyStorage
   readonly auth: PgAuthStorage
+  readonly executions: PgExecutionStorage
   readonly agents: PgAgentStorage
+  readonly aiUsage: PgAiUsageStorage
+  readonly aiCosts: PgAiCostStorage
   readonly actionRuns: PgActionRunStorage
   readonly pipelineRuns: PgPipelineRunStorage
   readonly workflowRuns: PgWorkflowRunStorage
@@ -347,9 +370,9 @@ interface PostgresStoreSet {
   readonly syncRuns: PgSyncRunStorage
   readonly projectionRuns: PgProjectionRunStorage
   readonly timeseries: PgTimeseriesStorage
-  readonly webhookDeliveries: PgWebhookDeliveryStorage
   readonly webhookRuns: PgWebhookRunStorage
   readonly rules: PgRulesStorage
+  readonly connectorConnections: PgConnectorConnectionStorage
 }
 
 function resolveTimeoutMillis(value: number | undefined, label: string): number | undefined {

package/src/latest-run-query.ts

@@ -23,12 +23,18 @@ export async function queryLatestRunsByOwnerId<TRow>(
   }
 
   const placeholders = ownerIds.map((_, index) => `$${index + 2}`).join(", ")
+  const orderColumn =
+    input.tableName === "sync_runs" ||
+    input.tableName === "pipeline_runs" ||
+    input.tableName === "projection_runs"
+      ? "COALESCE(started_at, queued_at)"
+      : "started_at"
   const rows = await input.sql.unsafe<TRow[]>(
     `
       SELECT DISTINCT ON (${input.ownerColumn}) ${input.selectList ?? "*"}
       FROM ${input.tableName}
       WHERE project_id = $1 AND ${input.ownerColumn} IN (${placeholders})
-      ORDER BY ${input.ownerColumn}, started_at DESC, id DESC
+      ORDER BY ${input.ownerColumn}, ${orderColumn} DESC, id DESC
     `,
     [input.projectId, ...ownerIds] as SqlParameter[]
   )

package/src/migrations/004-executions.sql

@@ -0,0 +1,140 @@
+CREATE TABLE executions (
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  id TEXT NOT NULL CHECK (length(trim(id)) > 0),
+
+  executor_kind TEXT NOT NULL CHECK (
+    executor_kind IN (
+      'request', 'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow',
+      'agent', 'kernel'
+    )
+  ),
+  executor_id TEXT NOT NULL CHECK (length(trim(executor_id)) > 0),
+  source_kind TEXT NOT NULL CHECK (
+    source_kind IN ('http', 'webhook', 'schedule', 'event', 'execution')
+  ),
+  source_id TEXT NOT NULL CHECK (length(trim(source_id)) > 0),
+
+  requested_by_user_id TEXT,
+  requested_by_service_account_id TEXT,
+  correlation_id TEXT NOT NULL CHECK (length(trim(correlation_id)) > 0),
+  parent_execution_id TEXT,
+
+  authority_kind TEXT NOT NULL CHECK (
+    authority_kind IN ('principal', 'trustedPrimitive', 'kernel', 'disabled')
+  ),
+  authority_user_id TEXT,
+  authority_service_account_id TEXT,
+  authority_session_id TEXT,
+  authority_access_token_id TEXT,
+  authority_primitive_kind TEXT CHECK (
+    authority_primitive_kind IS NULL
+      OR authority_primitive_kind IN (
+        'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow'
+      )
+  ),
+  authority_primitive_id TEXT,
+  authority_kernel_operation TEXT CHECK (
+    authority_kernel_operation IS NULL OR authority_kernel_operation = 'ontology.recover'
+  ),
+
+  created_at TIMESTAMPTZ NOT NULL,
+
+  PRIMARY KEY (project_id, id),
+  FOREIGN KEY (project_id, parent_execution_id)
+    REFERENCES executions (project_id, id),
+  FOREIGN KEY (project_id, requested_by_user_id)
+    REFERENCES auth_users (project_id, id),
+  FOREIGN KEY (project_id, requested_by_service_account_id)
+    REFERENCES auth_service_accounts (project_id, id),
+  FOREIGN KEY (project_id, authority_user_id)
+    REFERENCES auth_users (project_id, id),
+  FOREIGN KEY (project_id, authority_service_account_id)
+    REFERENCES auth_service_accounts (project_id, id),
+  FOREIGN KEY (project_id, authority_session_id)
+    REFERENCES auth_sessions (project_id, id),
+  FOREIGN KEY (project_id, authority_access_token_id)
+    REFERENCES auth_access_tokens (project_id, id),
+
+  CHECK (num_nonnulls(requested_by_user_id, requested_by_service_account_id) <= 1),
+  CHECK (
+    (source_kind = 'execution' AND parent_execution_id = source_id)
+      OR (source_kind <> 'execution' AND parent_execution_id IS NULL)
+  ),
+  CHECK (
+    (
+      authority_kind = 'principal'
+      AND num_nonnulls(authority_user_id, authority_service_account_id) = 1
+      AND num_nonnulls(authority_session_id, authority_access_token_id) <= 1
+      AND (authority_session_id IS NULL OR authority_user_id IS NOT NULL)
+      AND authority_primitive_kind IS NULL
+      AND authority_primitive_id IS NULL
+      AND authority_kernel_operation IS NULL
+    )
+    OR (
+      authority_kind = 'trustedPrimitive'
+      AND authority_user_id IS NULL
+      AND authority_service_account_id IS NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+      AND authority_primitive_kind IS NOT NULL
+      AND authority_primitive_id IS NOT NULL
+      AND length(trim(authority_primitive_id)) > 0
+      AND authority_kernel_operation IS NULL
+    )
+    OR (
+      authority_kind = 'kernel'
+      AND authority_user_id IS NULL
+      AND authority_service_account_id IS NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+      AND authority_primitive_kind IS NULL
+      AND authority_primitive_id IS NULL
+      AND authority_kernel_operation IS NOT NULL
+    )
+    OR (
+      authority_kind = 'disabled'
+      AND authority_user_id IS NULL
+      AND authority_service_account_id IS NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+      AND authority_primitive_kind IS NULL
+      AND authority_primitive_id IS NULL
+      AND authority_kernel_operation IS NULL
+    )
+  ),
+  CHECK (
+    (
+      executor_kind = 'request'
+      AND source_kind = 'http'
+      AND executor_id = source_id
+      AND (
+        (
+          authority_kind = 'principal'
+          AND requested_by_user_id IS NOT DISTINCT FROM authority_user_id
+          AND requested_by_service_account_id IS NOT DISTINCT FROM authority_service_account_id
+        )
+        OR (
+          authority_kind = 'disabled'
+          AND requested_by_user_id IS NULL
+          AND requested_by_service_account_id IS NULL
+        )
+      )
+    )
+    OR (
+      executor_kind IN ('action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow')
+      AND authority_kind = 'trustedPrimitive'
+      AND executor_kind = authority_primitive_kind
+    )
+    OR (
+      executor_kind = 'agent'
+      AND authority_kind = 'principal'
+      AND authority_service_account_id IS NOT NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+    )
+    OR (
+      executor_kind = 'kernel'
+      AND authority_kind = 'kernel'
+    )
+  )
+);

package/src/migrations/005-workflow-executions.sql

@@ -0,0 +1,10 @@
+ALTER TABLE workflow_runs
+  ADD COLUMN execution_id TEXT NOT NULL,
+  DROP COLUMN source,
+  DROP COLUMN requested_by_principal_type,
+  DROP COLUMN requested_by_principal_id,
+  ADD CONSTRAINT fk_workflow_runs_execution
+    FOREIGN KEY (project_id, execution_id)
+    REFERENCES executions (project_id, id)
+    ON DELETE RESTRICT,
+  ADD CONSTRAINT uq_workflow_runs_execution UNIQUE (project_id, execution_id);

package/src/migrations/006-narrow-ontology-source-root-index.sql

@@ -0,0 +1,4 @@
+DROP INDEX idx_ontology_source_rows_root;
+
+CREATE INDEX idx_ontology_source_rows_root
+  ON ontology_source_rows (project_id, source_id, materialization_id, root_sort_key);

package/src/migrations/007-split-overrides.sql

@@ -0,0 +1,153 @@
+CREATE TABLE ontology_object_overrides (
+  project_id TEXT NOT NULL,
+  object_type_id TEXT NOT NULL,
+  primary_id TEXT NOT NULL,
+  value JSONB NOT NULL CHECK (jsonb_typeof(value) = 'object'),
+  last_commit_id TEXT NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, object_type_id, primary_id),
+  CHECK (COALESCE(value->>'kind' IN ('create', 'patch', 'delete'), false))
+);
+
+CREATE TABLE ontology_link_overrides (
+  project_id TEXT NOT NULL,
+  identity_kind TEXT NOT NULL CHECK (identity_kind IN ('edge', 'slot')),
+  identity_key JSONB NOT NULL,
+  source_type_id TEXT NOT NULL,
+  source_primary_id TEXT NOT NULL,
+  link_id TEXT NOT NULL,
+  target_type_id TEXT NOT NULL,
+  target_primary_id TEXT NOT NULL,
+  value JSONB NOT NULL CHECK (jsonb_typeof(value) = 'object'),
+  last_commit_id TEXT NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, identity_kind, identity_key),
+  CHECK (
+    identity_key = CASE identity_kind
+      WHEN 'edge' THEN jsonb_build_array(
+        source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
+      )
+      WHEN 'slot' THEN jsonb_build_array(source_type_id, source_primary_id, link_id)
+    END
+  ),
+  CHECK (
+    (identity_kind = 'edge' AND COALESCE(value->>'kind' IN ('upsert', 'delete'), false))
+    OR
+    (identity_kind = 'slot'
+      AND COALESCE(value->>'kind' IN ('set', 'clear', 'legacy-conflict'), false)
+      AND (
+        value->>'kind' = 'legacy-conflict'
+        OR (
+          target_type_id = value->'target'->>'objectTypeId'
+          AND target_primary_id = value->'target'->>'primaryId'
+        )
+      ))
+  )
+);
+
+CREATE INDEX idx_ontology_link_overrides_source
+  ON ontology_link_overrides (
+    project_id, identity_kind, source_type_id, source_primary_id, link_id,
+    target_type_id, target_primary_id
+  );
+
+CREATE INDEX idx_ontology_link_overrides_target
+  ON ontology_link_overrides (project_id, target_type_id, target_primary_id);
+
+INSERT INTO ontology_object_overrides (
+  project_id, object_type_id, primary_id, value, last_commit_id, updated_at
+)
+SELECT project_id, object_type_id, primary_id, value, last_commit_id, updated_at
+FROM ontology_overrides
+WHERE entity_kind = 'object';
+
+INSERT INTO ontology_link_overrides (
+  project_id, identity_kind, identity_key,
+  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
+  value, last_commit_id, updated_at
+)
+SELECT project_id, 'edge', jsonb_build_array(
+    source_type_id, source_primary_id, link_id, target_type_id, target_primary_id
+  ),
+  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
+  value, last_commit_id, updated_at
+FROM ontology_overrides
+WHERE entity_kind = 'link';
+
+-- Storage migrations do not have access to the ontology cardinality. Derive a slot candidate for
+-- every legacy link scope; the runtime selects edge or slot authority from the registered ontology.
+-- A single upsert is an unambiguous set and a delete-only scope is an unambiguous clear. Multiple
+-- upserts are preserved as an explicit conflict instead of selecting a target arbitrarily.
+WITH scopes AS (
+  SELECT project_id, source_type_id, source_primary_id, link_id,
+    COUNT(*) FILTER (WHERE value->>'kind' = 'upsert') AS upsert_count
+  FROM ontology_overrides
+  WHERE entity_kind = 'link'
+  GROUP BY project_id, source_type_id, source_primary_id, link_id
+), selected AS (
+  SELECT scopes.*, COALESCE(upsert.entity_key, anchor.entity_key) AS selected_key
+  FROM scopes
+  LEFT JOIN LATERAL (
+    SELECT candidate.entity_key
+    FROM ontology_overrides AS candidate
+    WHERE candidate.project_id = scopes.project_id
+      AND candidate.entity_kind = 'link'
+      AND candidate.source_type_id = scopes.source_type_id
+      AND candidate.source_primary_id = scopes.source_primary_id
+      AND candidate.link_id = scopes.link_id
+      AND candidate.value->>'kind' = 'upsert'
+    ORDER BY candidate.updated_at DESC, candidate.entity_sort_key DESC
+    LIMIT 1
+  ) AS upsert ON true
+  LEFT JOIN LATERAL (
+    SELECT candidate.entity_key
+    FROM ontology_overrides AS candidate
+    WHERE candidate.project_id = scopes.project_id
+      AND candidate.entity_kind = 'link'
+      AND candidate.source_type_id = scopes.source_type_id
+      AND candidate.source_primary_id = scopes.source_primary_id
+      AND candidate.link_id = scopes.link_id
+    ORDER BY candidate.updated_at DESC, candidate.entity_sort_key DESC
+    LIMIT 1
+  ) AS anchor ON true
+)
+INSERT INTO ontology_link_overrides (
+  project_id, identity_kind, identity_key,
+  source_type_id, source_primary_id, link_id, target_type_id, target_primary_id,
+  value, last_commit_id, updated_at
+)
+SELECT selected.project_id, 'slot',
+  jsonb_build_array(selected.source_type_id, selected.source_primary_id, selected.link_id),
+  selected.source_type_id, selected.source_primary_id, selected.link_id,
+  overrides.target_type_id, overrides.target_primary_id,
+  CASE
+    WHEN selected.upsert_count > 1 THEN jsonb_build_object('kind', 'legacy-conflict')
+    WHEN selected.upsert_count = 1 THEN
+      jsonb_build_object(
+        'kind', 'set',
+        'target', jsonb_build_object(
+          'objectTypeId', overrides.target_type_id,
+          'primaryId', overrides.target_primary_id
+        )
+      ) || CASE
+        WHEN overrides.value ? 'properties'
+          THEN jsonb_build_object('properties', overrides.value->'properties')
+        ELSE '{}'::jsonb
+      END
+    ELSE jsonb_build_object(
+      'kind', 'clear',
+      'target', jsonb_build_object(
+        'objectTypeId', overrides.target_type_id,
+        'primaryId', overrides.target_primary_id
+      )
+    )
+  END,
+  overrides.last_commit_id,
+  overrides.updated_at
+FROM selected
+JOIN ontology_overrides AS overrides
+  ON overrides.project_id = selected.project_id
+ AND overrides.entity_kind = 'link'
+ AND overrides.entity_key = selected.selected_key;
+
+DROP TABLE ontology_overrides;

package/src/migrations/008-action-executions.sql

@@ -0,0 +1,7 @@
+ALTER TABLE action_runs
+  ADD COLUMN execution_id TEXT NOT NULL,
+  ADD CONSTRAINT fk_action_runs_execution
+    FOREIGN KEY (project_id, execution_id)
+    REFERENCES executions (project_id, id)
+    ON DELETE RESTRICT,
+  ADD CONSTRAINT uq_action_runs_execution UNIQUE (project_id, execution_id);

package/src/migrations/009-agent-executions.sql

@@ -0,0 +1,21 @@
+ALTER TABLE agent_runs
+  ADD COLUMN execution_id TEXT NOT NULL,
+  ADD CONSTRAINT fk_agent_runs_execution
+    FOREIGN KEY (project_id, execution_id)
+    REFERENCES executions (project_id, id)
+    ON DELETE RESTRICT,
+  ADD CONSTRAINT uq_agent_runs_execution UNIQUE (project_id, execution_id),
+  DROP COLUMN requested_by_principal_type,
+  DROP COLUMN requested_by_principal_id,
+  DROP COLUMN execution_principal_type,
+  DROP COLUMN execution_principal_id;
+
+ALTER TABLE workflow_agent_node_runs
+  ADD COLUMN execution_id TEXT NOT NULL,
+  ADD CONSTRAINT fk_workflow_agent_node_runs_execution
+    FOREIGN KEY (project_id, execution_id)
+    REFERENCES executions (project_id, id)
+    ON DELETE RESTRICT,
+  ADD CONSTRAINT uq_workflow_agent_node_runs_execution UNIQUE (project_id, execution_id),
+  DROP COLUMN execution_principal_type,
+  DROP COLUMN execution_principal_id;

package/src/migrations/010-ai-usage-accounting-foundation.sql

@@ -0,0 +1,76 @@
+ALTER TABLE agent_runs
+  ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
+ALTER TABLE agent_runs
+  ADD CONSTRAINT agent_runs_requester_group_ids_array
+  CHECK (jsonb_typeof(requester_group_ids) = 'array');
+
+ALTER TABLE workflow_runs
+  ADD COLUMN requester_group_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
+ALTER TABLE workflow_runs
+  ADD CONSTRAINT workflow_runs_requester_group_ids_array
+  CHECK (jsonb_typeof(requester_group_ids) = 'array');
+
+CREATE TABLE ai_model_call_usage (
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  id TEXT NOT NULL CHECK (length(trim(id)) > 0),
+  execution_id TEXT NOT NULL CHECK (length(trim(execution_id)) > 0),
+  attempt BIGINT NOT NULL CHECK (attempt >= 1),
+  call_id TEXT NOT NULL CHECK (length(trim(call_id)) > 0),
+  provider_id TEXT NOT NULL CHECK (length(trim(provider_id)) > 0),
+  requested_model_id TEXT NOT NULL CHECK (length(trim(requested_model_id)) > 0),
+  response_model_id TEXT CHECK (
+    response_model_id IS NULL OR length(trim(response_model_id)) > 0
+  ),
+  response_id TEXT NOT NULL CHECK (length(trim(response_id)) > 0),
+  input_tokens BIGINT CHECK (input_tokens IS NULL OR input_tokens >= 0),
+  output_tokens BIGINT CHECK (output_tokens IS NULL OR output_tokens >= 0),
+  total_tokens BIGINT CHECK (total_tokens IS NULL OR total_tokens >= 0),
+  uncached_input_tokens BIGINT CHECK (
+    uncached_input_tokens IS NULL OR uncached_input_tokens >= 0
+  ),
+  cache_read_input_tokens BIGINT CHECK (
+    cache_read_input_tokens IS NULL OR cache_read_input_tokens >= 0
+  ),
+  cache_write_input_tokens BIGINT CHECK (
+    cache_write_input_tokens IS NULL OR cache_write_input_tokens >= 0
+  ),
+  text_output_tokens BIGINT CHECK (
+    text_output_tokens IS NULL OR text_output_tokens >= 0
+  ),
+  reasoning_output_tokens BIGINT CHECK (
+    reasoning_output_tokens IS NULL OR reasoning_output_tokens >= 0
+  ),
+  reporting_status TEXT NOT NULL CHECK (
+    reporting_status IN ('complete', 'partial', 'unavailable')
+  ),
+  raw_usage JSONB CHECK (raw_usage IS NULL OR jsonb_typeof(raw_usage) = 'object'),
+  occurred_at TIMESTAMPTZ NOT NULL,
+  recorded_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, id),
+  FOREIGN KEY (project_id, execution_id)
+    REFERENCES executions(project_id, id)
+    ON DELETE RESTRICT
+);
+
+CREATE UNIQUE INDEX idx_ai_model_call_usage_idempotency
+  ON ai_model_call_usage (project_id, execution_id, attempt, call_id, response_id);
+CREATE INDEX idx_ai_model_call_usage_execution
+  ON ai_model_call_usage (project_id, execution_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_usage_project_time
+  ON ai_model_call_usage (project_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_usage_provider_response
+  ON ai_model_call_usage (project_id, provider_id, response_id);
+
+CREATE TABLE ai_model_call_usage_groups (
+  project_id TEXT NOT NULL,
+  usage_record_id TEXT NOT NULL,
+  group_id TEXT NOT NULL,
+  occurred_at TIMESTAMPTZ NOT NULL,
+  PRIMARY KEY (project_id, usage_record_id, group_id),
+  FOREIGN KEY (project_id, usage_record_id)
+    REFERENCES ai_model_call_usage(project_id, id)
+    ON DELETE CASCADE
+);
+
+CREATE INDEX idx_ai_model_call_usage_groups_group_time
+  ON ai_model_call_usage_groups (project_id, group_id, occurred_at, usage_record_id);

package/src/migrations/011-sync-failure-record.sql

@@ -0,0 +1,23 @@
+ALTER TABLE sync_runs ADD COLUMN error JSONB;
+
+UPDATE sync_runs
+SET error = jsonb_build_object(
+  'code', CASE WHEN status = 'cancelled' THEN 'runtime.cancelled' ELSE 'internal.unexpected' END,
+  'message', CASE
+    WHEN status = 'cancelled' THEN 'Execution was cancelled.'
+    ELSE 'An unexpected internal error occurred.'
+  END,
+  'retryable', false,
+  'at', to_char(COALESCE(finished_at, started_at) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
+  'details', jsonb_build_object(
+    'syncId', sync_id,
+    'runId', id,
+    'datasetId', dataset_id,
+    'migratedFromLegacyError', true
+  )
+)
+WHERE error_name IS NOT NULL OR error_message IS NOT NULL;
+
+ALTER TABLE sync_runs
+  DROP COLUMN error_name,
+  DROP COLUMN error_message;

package/src/migrations/012-pipeline-failure-record.sql

@@ -0,0 +1,47 @@
+ALTER TABLE pipeline_runs ADD COLUMN error JSONB;
+
+UPDATE pipeline_runs
+SET error = jsonb_build_object(
+  'code', CASE WHEN status = 'cancelled' THEN 'runtime.cancelled' ELSE 'internal.unexpected' END,
+  'message', CASE
+    WHEN status = 'cancelled' THEN 'Execution was cancelled.'
+    ELSE 'An unexpected internal error occurred.'
+  END,
+  'retryable', false,
+  'at', to_char(COALESCE(finished_at, started_at) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
+  'details', jsonb_build_object(
+    'pipelineId', pipeline_id,
+    'runId', id,
+    'migratedFromLegacyError', true
+  )
+)
+WHERE error_name IS NOT NULL OR error_message IS NOT NULL;
+
+ALTER TABLE pipeline_runs
+  DROP COLUMN error_name,
+  DROP COLUMN error_message;
+
+ALTER TABLE pipeline_step_runs ADD COLUMN error JSONB;
+
+UPDATE pipeline_step_runs
+SET error = jsonb_build_object(
+  'code', CASE WHEN status = 'cancelled' THEN 'runtime.cancelled' ELSE 'internal.unexpected' END,
+  'message', CASE
+    WHEN status = 'cancelled' THEN 'Execution was cancelled.'
+    ELSE 'An unexpected internal error occurred.'
+  END,
+  'retryable', false,
+  'at', to_char(COALESCE(finished_at, started_at) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
+  'details', jsonb_build_object(
+    'pipelineId', pipeline_id,
+    'pipelineRunId', pipeline_run_id,
+    'stepId', step_id,
+    'stepRunId', id,
+    'migratedFromLegacyError', true
+  )
+)
+WHERE error_name IS NOT NULL OR error_message IS NOT NULL;
+
+ALTER TABLE pipeline_step_runs
+  DROP COLUMN error_name,
+  DROP COLUMN error_message;