@sixb/sqlite 0.1.1 → 0.1.3

package/src/migrations/025-connector-connections.sql

@@ -0,0 +1,257 @@
+CREATE TABLE connector_connection_runs (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  kind TEXT NOT NULL CHECK (kind IN ('connect', 'reauthorize')),
+  slot TEXT NOT NULL,
+  initiated_by_execution_id TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('waiting', 'running', 'succeeded', 'failed', 'cancelled', 'expired')),
+  waiting_for TEXT CHECK (waiting_for IN ('provider_authorization', 'account_selection')),
+  processing_id TEXT,
+  callback_started_at TEXT,
+  expires_at TEXT,
+  authorization_id TEXT,
+  cleanup_authorization_id TEXT,
+  connections TEXT CHECK (
+    connections IS NULL OR (json_valid(connections) AND json_type(connections) = 'array')
+  ),
+  error TEXT CHECK (error IS NULL OR (json_valid(error) AND json_type(error) = 'object')),
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  finished_at TEXT,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  CHECK (
+    (
+      status = 'waiting'
+      AND waiting_for = 'provider_authorization'
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NOT NULL
+      AND authorization_id IS NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NULL
+    )
+    OR
+    (
+      status = 'waiting'
+      AND waiting_for = 'account_selection'
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NOT NULL
+      AND authorization_id IS NOT NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NULL
+    )
+    OR
+    (
+      status = 'running'
+      AND waiting_for IS NULL
+      AND processing_id IS NOT NULL
+      AND callback_started_at IS NOT NULL
+      AND expires_at IS NOT NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NULL
+    )
+    OR
+    (
+      status = 'succeeded'
+      AND waiting_for IS NULL
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NULL
+      AND authorization_id IS NOT NULL
+      AND connections IS NOT NULL
+      AND error IS NULL
+      AND finished_at IS NOT NULL
+    )
+    OR
+    (
+      status = 'failed'
+      AND waiting_for IS NULL
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NOT NULL
+      AND finished_at IS NOT NULL
+    )
+    OR
+    (
+      status IN ('cancelled', 'expired')
+      AND waiting_for IS NULL
+      AND processing_id IS NULL
+      AND callback_started_at IS NULL
+      AND expires_at IS NULL
+      AND cleanup_authorization_id IS NULL
+      AND connections IS NULL
+      AND error IS NULL
+      AND finished_at IS NOT NULL
+    )
+  )
+);
+
+CREATE INDEX idx_connector_runs_status
+  ON connector_connection_runs (project_id, connector_id, status, updated_at);
+
+CREATE TABLE connector_authorization_attempts (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  slot TEXT NOT NULL,
+  initiated_by_execution_id TEXT NOT NULL,
+  state_hash TEXT NOT NULL,
+  code_verifier TEXT NOT NULL CHECK (
+    json_valid(code_verifier) AND json_type(code_verifier) = 'object'
+  ),
+  redirect_uri TEXT NOT NULL,
+  connection_run_id TEXT,
+  return_to TEXT,
+  callback_binding_hash TEXT,
+  reauthorization_id TEXT,
+  reauthorization_revision INTEGER CHECK (
+    reauthorization_revision IS NULL OR reauthorization_revision >= 0
+  ),
+  reauthorization_connection_ids TEXT CHECK (
+    reauthorization_connection_ids IS NULL
+    OR (json_valid(reauthorization_connection_ids) AND json_type(reauthorization_connection_ids) = 'array')
+  ),
+  created_at TEXT NOT NULL,
+  expires_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  UNIQUE (project_id, connector_id, connection_run_id),
+  FOREIGN KEY (project_id, connector_id, connection_run_id)
+    REFERENCES connector_connection_runs (project_id, connector_id, id)
+    ON DELETE RESTRICT,
+  CHECK (
+    (connection_run_id IS NULL AND return_to IS NULL AND callback_binding_hash IS NULL)
+    OR
+    (connection_run_id IS NOT NULL AND return_to IS NOT NULL AND callback_binding_hash IS NOT NULL)
+  ),
+  CHECK (
+    (reauthorization_id IS NULL AND reauthorization_revision IS NULL AND reauthorization_connection_ids IS NULL)
+    OR
+    (reauthorization_id IS NOT NULL AND reauthorization_revision IS NOT NULL AND reauthorization_connection_ids IS NOT NULL)
+  )
+);
+
+CREATE INDEX idx_connector_attempts_expiry
+  ON connector_authorization_attempts (project_id, connector_id, expires_at);
+
+CREATE TABLE connector_authorizations (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  authorized_by TEXT NOT NULL CHECK (
+    json_valid(authorized_by) AND json_type(authorized_by) = 'object'
+  ),
+  credentials TEXT CHECK (
+    credentials IS NULL OR (json_valid(credentials) AND json_type(credentials) = 'object')
+  ),
+  credential_expires_at TEXT,
+  scopes TEXT NOT NULL CHECK (json_valid(scopes) AND json_type(scopes) = 'array'),
+  accounts TEXT NOT NULL CHECK (json_valid(accounts) AND json_type(accounts) = 'array'),
+  status TEXT NOT NULL CHECK (status IN ('pending_selection', 'active', 'needs_reauthorization', 'revocation_pending', 'revoked')),
+  selection_expires_at TEXT,
+  revision INTEGER NOT NULL CHECK (revision >= 0),
+  mutation_id TEXT,
+  mutation_kind TEXT CHECK (mutation_kind IN ('refresh', 'reauthorization', 'revocation')),
+  mutation_phase TEXT CHECK (mutation_phase IN ('prepared', 'executing', 'result_staged')),
+  mutation_holder_id TEXT,
+  mutation_expires_at TEXT,
+  mutation_deadline_at TEXT,
+  mutation_expected_connection_ids TEXT CHECK (
+    mutation_expected_connection_ids IS NULL
+    OR (json_valid(mutation_expected_connection_ids) AND json_type(mutation_expected_connection_ids) = 'array')
+  ),
+  staged_credentials TEXT CHECK (
+    staged_credentials IS NULL
+    OR (json_valid(staged_credentials) AND json_type(staged_credentials) = 'object')
+  ),
+  staged_credential_expires_at TEXT,
+  staged_scopes TEXT CHECK (
+    staged_scopes IS NULL OR (json_valid(staged_scopes) AND json_type(staged_scopes) = 'array')
+  ),
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  CHECK (
+    (mutation_id IS NULL AND mutation_kind IS NULL AND mutation_phase IS NULL AND mutation_holder_id IS NULL AND mutation_expires_at IS NULL AND mutation_deadline_at IS NULL AND mutation_expected_connection_ids IS NULL AND staged_credentials IS NULL AND staged_credential_expires_at IS NULL AND staged_scopes IS NULL)
+    OR
+    (mutation_id IS NOT NULL AND mutation_kind IS NOT NULL AND mutation_phase IS NOT NULL AND mutation_holder_id IS NOT NULL AND mutation_expires_at IS NOT NULL AND mutation_deadline_at IS NOT NULL)
+  ),
+  CHECK (
+    (
+      mutation_phase IS NULL
+      AND staged_credentials IS NULL
+      AND staged_credential_expires_at IS NULL
+      AND staged_scopes IS NULL
+    )
+    OR
+    (
+      mutation_phase IN ('prepared', 'executing')
+      AND staged_credentials IS NULL
+      AND staged_credential_expires_at IS NULL
+      AND staged_scopes IS NULL
+    )
+    OR
+    (
+      mutation_phase = 'result_staged'
+      AND mutation_kind = 'revocation'
+      AND staged_credentials IS NULL
+      AND staged_credential_expires_at IS NULL
+      AND staged_scopes IS NULL
+    )
+    OR
+    (
+      mutation_phase = 'result_staged'
+      AND mutation_kind IN ('refresh', 'reauthorization')
+      AND staged_credentials IS NOT NULL
+      AND staged_scopes IS NOT NULL
+    )
+  ),
+  CHECK ((status = 'pending_selection') = (selection_expires_at IS NOT NULL)),
+  CHECK ((status = 'revoked') = (credentials IS NULL))
+);
+
+CREATE INDEX idx_connector_authorizations_status
+  ON connector_authorizations (project_id, connector_id, status, updated_at);
+CREATE INDEX idx_connector_authorizations_mutation_expiry
+  ON connector_authorizations (project_id, connector_id, mutation_expires_at)
+  WHERE mutation_id IS NOT NULL;
+
+CREATE TABLE connector_connections (
+  project_id TEXT NOT NULL,
+  connector_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  authorization_id TEXT NOT NULL,
+  slot TEXT NOT NULL,
+  account TEXT NOT NULL CHECK (json_valid(account) AND json_type(account) = 'object'),
+  account_id TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('connected', 'disconnected')),
+  disconnected_at TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, connector_id, id),
+  UNIQUE (project_id, id),
+  UNIQUE (project_id, connector_id, slot),
+  FOREIGN KEY (project_id, connector_id, authorization_id)
+    REFERENCES connector_authorizations (project_id, connector_id, id)
+    ON DELETE RESTRICT,
+  CHECK ((status = 'connected') = (disconnected_at IS NULL)),
+  CHECK (json_extract(account, '$.id') IS NOT NULL AND json_extract(account, '$.id') = account_id)
+);
+
+CREATE INDEX idx_connector_connections_authorization
+  ON connector_connections (project_id, connector_id, authorization_id, status, id);
+CREATE INDEX idx_connector_connections_status
+  ON connector_connections (project_id, connector_id, status, id);

package/src/migrations/026-ai-cost-accounting.sql

@@ -0,0 +1,57 @@
+CREATE TABLE ai_model_call_valuations (
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  usage_record_id TEXT NOT NULL CHECK (length(trim(usage_record_id)) > 0),
+  status TEXT NOT NULL CHECK (status IN ('rated', 'unpriceable')),
+  provider_id TEXT CHECK (provider_id IS NULL OR length(trim(provider_id)) > 0),
+  model_id TEXT CHECK (model_id IS NULL OR length(trim(model_id)) > 0),
+  currency TEXT CHECK (
+    currency IS NULL OR (length(currency) = 3 AND currency GLOB '[A-Z][A-Z][A-Z]')
+  ),
+  amount_nanos INTEGER CHECK (
+    amount_nanos IS NULL OR (typeof(amount_nanos) = 'integer' AND amount_nanos >= 0)
+  ),
+  reason TEXT CHECK (
+    reason IS NULL OR reason IN (
+      'missingBillingIdentity',
+      'missingCatalogEntry',
+      'missingUsageMeter',
+      'unsupportedPricingDimension',
+      'invalidUsageForFormula'
+    )
+  ),
+  details TEXT NOT NULL CHECK (json_valid(details) AND json_type(details) = 'object'),
+  rated_at TEXT NOT NULL CHECK (length(trim(rated_at)) > 0),
+  PRIMARY KEY (project_id, usage_record_id),
+  FOREIGN KEY (project_id, usage_record_id)
+    REFERENCES ai_model_call_usage(project_id, id)
+    ON DELETE RESTRICT,
+  CHECK (
+    (
+      status = 'rated'
+      AND provider_id IS NOT NULL
+      AND model_id IS NOT NULL
+      AND currency IS NOT NULL
+      AND amount_nanos IS NOT NULL
+      AND reason IS NULL
+    )
+    OR
+    (
+      status = 'unpriceable'
+      AND currency IS NULL
+      AND amount_nanos IS NULL
+      AND reason IS NOT NULL
+      AND (
+        (reason = 'missingBillingIdentity' AND provider_id IS NULL AND model_id IS NULL)
+        OR
+        (reason <> 'missingBillingIdentity' AND provider_id IS NOT NULL AND model_id IS NOT NULL)
+      )
+    )
+  )
+);
+
+CREATE INDEX idx_ai_model_call_usage_provider_model_time
+  ON ai_model_call_usage (project_id, provider_id, requested_model_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_usage_model_time
+  ON ai_model_call_usage (project_id, requested_model_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_valuations_summary
+  ON ai_model_call_valuations (project_id, status, currency, usage_record_id);

package/src/migrations/027-agent-context-checkpoints.sql

@@ -0,0 +1,28 @@
+CREATE TABLE agent_context_checkpoints (
+  project_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  thread_id TEXT NOT NULL,
+  created_by_run_id TEXT NOT NULL,
+  previous_checkpoint_id TEXT,
+  reason TEXT NOT NULL CHECK (reason IN ('threshold', 'overflow')),
+  summary TEXT NOT NULL CHECK (length(trim(summary)) > 0),
+  summary_format_version INTEGER NOT NULL CHECK (summary_format_version = 1),
+  summarized_through_seq INTEGER NOT NULL CHECK (summarized_through_seq >= 1),
+  observed_head_seq INTEGER NOT NULL,
+  estimated_input_tokens_before INTEGER NOT NULL CHECK (estimated_input_tokens_before >= 0),
+  estimated_input_tokens_after INTEGER NOT NULL CHECK (estimated_input_tokens_after >= 0),
+  summary_model_id TEXT NOT NULL CHECK (length(trim(summary_model_id)) > 0),
+  created_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, id),
+  UNIQUE (project_id, created_by_run_id),
+  FOREIGN KEY (project_id, thread_id)
+    REFERENCES agent_threads(project_id, id) ON DELETE CASCADE,
+  FOREIGN KEY (project_id, created_by_run_id)
+    REFERENCES agent_runs(project_id, id) ON DELETE RESTRICT,
+  FOREIGN KEY (project_id, previous_checkpoint_id)
+    REFERENCES agent_context_checkpoints(project_id, id) ON DELETE RESTRICT,
+  CHECK (summarized_through_seq < observed_head_seq)
+);
+
+CREATE INDEX idx_agent_context_checkpoints_thread_latest
+  ON agent_context_checkpoints(project_id, thread_id, summarized_through_seq DESC, created_at DESC, id DESC);

package/src/migrations/028-object-override-edit-times.sql

@@ -0,0 +1,3 @@
+ALTER TABLE ontology_object_overrides
+ADD COLUMN edited_at TEXT NOT NULL DEFAULT '{}'
+CHECK (json_valid(edited_at) AND json_type(edited_at) = 'object');

package/src/migrations.ts

@@ -19,6 +19,63 @@ import {
 import initialSchemaSql from "./migrations/001-initial-schema.sql" with { type: "text" }
 import workflowRunOutputSql from "./migrations/002-workflow-run-output.sql" with { type: "text" }
 import mergeSyncRunsSql from "./migrations/003-merge-sync-runs.sql" with { type: "text" }
+import executionsSql from "./migrations/004-executions.sql" with { type: "text" }
+import workflowExecutionsSql from "./migrations/005-workflow-executions.sql" with { type: "text" }
+import narrowOntologySourceRootIndexSql from "./migrations/006-narrow-ontology-source-root-index.sql" with {
+  type: "text",
+}
+import splitOverridesSql from "./migrations/007-split-overrides.sql" with { type: "text" }
+import actionExecutionsSql from "./migrations/008-action-executions.sql" with { type: "text" }
+import agentExecutionsSql from "./migrations/009-agent-executions.sql" with { type: "text" }
+import aiUsageAccountingFoundationSql from "./migrations/010-ai-usage-accounting-foundation.sql" with {
+  type: "text",
+}
+import syncFailureRecordSql from "./migrations/011-sync-failure-record.sql" with { type: "text" }
+import pipelineFailureRecordSql from "./migrations/012-pipeline-failure-record.sql" with {
+  type: "text",
+}
+import workflowFailureRecordSql from "./migrations/013-workflow-failure-record.sql" with {
+  type: "text",
+}
+import agentFailureRecordSql from "./migrations/014-agent-failure-record.sql" with { type: "text" }
+import projectionFailureRecordSql from "./migrations/015-projection-failure-record.sql" with {
+  type: "text",
+}
+import webhookRunFailureRecordSql from "./migrations/016-webhook-run-failure-record.sql" with {
+  type: "text",
+}
+import actionFailureRecordSql from "./migrations/017-action-failure-record.sql" with {
+  type: "text",
+}
+import ontologyOutboxFailureRecordSql from "./migrations/018-ontology-outbox-failure-record.sql" with {
+  type: "text",
+}
+import webhookDeliveryFailureRecordSql from "./migrations/019-webhook-delivery-failure-record.sql" with {
+  type: "text",
+}
+import dropRunUsageProjectionsSql from "./migrations/020-drop-run-usage-projections.sql" with {
+  type: "text",
+}
+import syncPipelineExecutionsSql from "./migrations/021-sync-pipeline-executions.sql" with {
+  type: "text",
+}
+import projectionExecutionsSql from "./migrations/022-projection-executions.sql" with {
+  type: "text",
+}
+import webhookExecutionsSql from "./migrations/023-webhook-executions.sql" with { type: "text" }
+import ontologyCommitExecutionsSql from "./migrations/024-ontology-commit-executions.sql" with {
+  type: "text",
+}
+import connectorConnectionsSql from "./migrations/025-connector-connections.sql" with {
+  type: "text",
+}
+import aiCostAccountingSql from "./migrations/026-ai-cost-accounting.sql" with { type: "text" }
+import agentContextCheckpointsSql from "./migrations/027-agent-context-checkpoints.sql" with {
+  type: "text",
+}
+import objectOverrideEditTimesSql from "./migrations/028-object-override-edit-times.sql" with {
+  type: "text",
+}
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -42,6 +99,66 @@ export const sqliteStorageMigrations = defineMigrations({
     sqliteSql("001-initial-schema", initialSchemaSql),
     sqliteSql("002-workflow-run-output", workflowRunOutputSql),
     sqliteSql("003-merge-sync-runs", mergeSyncRunsSql),
+    sqliteSql("004-executions", executionsSql),
+    sqliteSql("005-workflow-executions", workflowExecutionsSql),
+    sqliteSql("006-narrow-ontology-source-root-index", narrowOntologySourceRootIndexSql),
+    sqliteSql("007-split-overrides", splitOverridesSql),
+    sqliteSql("008-action-executions", actionExecutionsSql),
+    sqliteSql("009-agent-executions", agentExecutionsSql),
+    sqliteSql("010-ai-usage-accounting-foundation", aiUsageAccountingFoundationSql),
+    sqliteSql("011-sync-failure-record", syncFailureRecordSql),
+    sqliteSql("012-pipeline-failure-record", pipelineFailureRecordSql),
+    sqliteSql("013-workflow-failure-record", workflowFailureRecordSql),
+    sqliteSql("014-agent-failure-record", agentFailureRecordSql),
+    sqliteSql("015-projection-failure-record", projectionFailureRecordSql),
+    sqliteSql("016-webhook-run-failure-record", webhookRunFailureRecordSql),
+    sqliteSql("017-action-failure-record", actionFailureRecordSql),
+    sqliteSql("018-ontology-outbox-failure-record", ontologyOutboxFailureRecordSql),
+    sqliteSql("019-webhook-delivery-failure-record", webhookDeliveryFailureRecordSql),
+    sqliteSql("020-drop-run-usage-projections", dropRunUsageProjectionsSql),
+    sqliteSql("021-sync-pipeline-executions", syncPipelineExecutionsSql),
+    sqliteStep(
+      "022-projection-executions",
+      (db) => {
+        if (db.query("SELECT 1 FROM projection_runs LIMIT 1").get()) {
+          throw new Error(
+            "[SixbSqliteStorage] Projection execution migration cannot preserve legacy runs with unknown authority."
+          )
+        }
+        db.run(projectionExecutionsSql)
+      },
+      { checksum: checksum(projectionExecutionsSql) }
+    ),
+    sqliteStep(
+      "023-webhook-executions",
+      (db) => {
+        const hasRuns = db.query("SELECT 1 FROM webhook_runs LIMIT 1").get()
+        const hasDeliveries = db.query("SELECT 1 FROM webhook_deliveries LIMIT 1").get()
+        if (hasRuns || hasDeliveries) {
+          throw new Error(
+            "[SixbSqliteStorage] Webhook execution migration cannot preserve legacy runs or deliveries with unknown authority."
+          )
+        }
+        db.run(webhookExecutionsSql)
+      },
+      { checksum: checksum(webhookExecutionsSql) }
+    ),
+    sqliteStep(
+      "024-ontology-commit-executions",
+      (db) => {
+        if (db.query("SELECT 1 FROM ontology_commits LIMIT 1").get()) {
+          throw new Error(
+            "[SixbSqliteStorage] Ontology commit execution migration cannot preserve commits with unknown authority."
+          )
+        }
+        db.run(ontologyCommitExecutionsSql)
+      },
+      { checksum: checksum(ontologyCommitExecutionsSql) }
+    ),
+    sqliteSql("025-connector-connections", connectorConnectionsSql),
+    sqliteSql("026-ai-cost-accounting", aiCostAccountingSql),
+    sqliteSql("027-agent-context-checkpoints", agentContextCheckpointsSql),
+    sqliteSql("028-object-override-edit-times", objectOverrideEditTimesSql),
   ],
 })
 
@@ -120,10 +237,6 @@ export async function migrateSqliteStorage(basePath: string): Promise<void> {
   }
 }
 
-export async function migrateSqliteDatabase(path: string): Promise<void> {
-  await createSqliteMigrator({ path, migrations: sqliteStorageMigrations }).migrate()
-}
-
 function sqliteMigrationHistoryStore(db: Database): MigrationHistoryStore {
   return {
     ensure() {
@@ -156,15 +269,27 @@ function sqliteMigrationHistoryStore(db: Database): MigrationHistoryStore {
       `).run(at, adapterId, migration.version)
     },
     async transaction(run) {
+      // SQLite cannot rebuild a referenced parent table while foreign-key enforcement is active,
+      // even when the replacement satisfies every deferred reference. Migrations therefore run
+      // with enforcement paused, validate the complete schema before commit, then restore it.
+      db.run("PRAGMA foreign_keys = OFF")
       db.run("BEGIN")
 
       try {
         const result = await run()
+        const violations = db.query("PRAGMA foreign_key_check").all()
+        if (violations.length > 0) {
+          throw new Error(
+            `[${SQLITE_STORAGE_ADAPTER_ID}] Migration produced ${violations.length} foreign-key violation(s).`
+          )
+        }
         db.run("COMMIT")
         return result
       } catch (error) {
         rollbackQuietly(db)
         throw error
+      } finally {
+        db.run("PRAGMA foreign_keys = ON")
       }
     },
   }