@sixb/sqlite 0.1.1 → 0.1.3

package/src/execution-storage.ts

@@ -0,0 +1,158 @@
+import type { Database } from "bun:sqlite"
+import {
+  type ExecutionStorageRow,
+  executionRecordFromStorageRow,
+  executionRecordToStorageRow,
+  normalizeExecutionRecord,
+  validateExecutionRecordReferences,
+} from "@sixb/core/internal/execution-storage"
+import type { CreateExecutionInput, ExecutionRecord, ExecutionStorage } from "@sixb/core/storage"
+import { ExecutionStorageError } from "@sixb/core/storage"
+import type { SqliteAuthStorage } from "./auth-storage"
+
+export class SqliteExecutionStorage implements ExecutionStorage {
+  constructor(
+    private readonly db: Database,
+    private readonly auth: SqliteAuthStorage
+  ) {}
+
+  async create(input: CreateExecutionInput): Promise<ExecutionRecord> {
+    const record = normalizeExecutionRecord(input)
+    await validateExecutionRecordReferences(record, {
+      auth: this.auth,
+      getExecution: (params) => this.getById(params),
+    })
+    const row = executionRecordToStorageRow(record)
+
+    try {
+      this.db
+        .query(
+          `
+          INSERT INTO executions (
+            project_id,
+            id,
+            executor_kind,
+            executor_id,
+            source_kind,
+            source_id,
+            requested_by_user_id,
+            requested_by_service_account_id,
+            correlation_id,
+            parent_execution_id,
+            authority_kind,
+            authority_user_id,
+            authority_service_account_id,
+            authority_session_id,
+            authority_access_token_id,
+            authority_primitive_kind,
+            authority_primitive_id,
+            authority_kernel_operation,
+            created_at
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `
+        )
+        .run(
+          row.projectId,
+          row.id,
+          row.executorKind,
+          row.executorId,
+          row.sourceKind,
+          row.sourceId,
+          row.requestedByUserId,
+          row.requestedByServiceAccountId,
+          row.correlationId,
+          row.parentExecutionId,
+          row.authorityKind,
+          row.authorityUserId,
+          row.authorityServiceAccountId,
+          row.authoritySessionId,
+          row.authorityAccessTokenId,
+          row.authorityPrimitiveKind,
+          row.authorityPrimitiveId,
+          row.authorityKernelOperation,
+          row.createdAt.toISOString()
+        )
+    } catch (error) {
+      if (isDuplicateExecutionError(error)) {
+        throw new ExecutionStorageError(
+          "duplicate_execution",
+          `[SixbSqlite] Execution '${record.id}' already exists in project '${record.projectId}'.`,
+          { cause: error }
+        )
+      }
+      throw error
+    }
+
+    return executionRecordFromStorageRow(row)
+  }
+
+  async getById(params: {
+    readonly projectId: string
+    readonly id: string
+  }): Promise<ExecutionRecord | null> {
+    const row = this.db
+      .query(
+        `
+        SELECT *
+        FROM executions
+        WHERE project_id = ? AND id = ?
+      `
+      )
+      .get(params.projectId, params.id) as SqliteExecutionRow | null
+
+    return row ? executionRecordFromStorageRow(toStorageRow(row)) : null
+  }
+}
+
+function toStorageRow(row: SqliteExecutionRow): ExecutionStorageRow {
+  return {
+    projectId: row.project_id,
+    id: row.id,
+    executorKind: row.executor_kind,
+    executorId: row.executor_id,
+    sourceKind: row.source_kind,
+    sourceId: row.source_id,
+    requestedByUserId: row.requested_by_user_id,
+    requestedByServiceAccountId: row.requested_by_service_account_id,
+    correlationId: row.correlation_id,
+    parentExecutionId: row.parent_execution_id,
+    authorityKind: row.authority_kind,
+    authorityUserId: row.authority_user_id,
+    authorityServiceAccountId: row.authority_service_account_id,
+    authoritySessionId: row.authority_session_id,
+    authorityAccessTokenId: row.authority_access_token_id,
+    authorityPrimitiveKind: row.authority_primitive_kind,
+    authorityPrimitiveId: row.authority_primitive_id,
+    authorityKernelOperation: row.authority_kernel_operation,
+    createdAt: new Date(row.created_at),
+  }
+}
+
+function isDuplicateExecutionError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    error.message.includes("UNIQUE constraint failed: executions.project_id, executions.id")
+  )
+}
+
+interface SqliteExecutionRow {
+  readonly project_id: string
+  readonly id: string
+  readonly executor_kind: ExecutionStorageRow["executorKind"]
+  readonly executor_id: string
+  readonly source_kind: ExecutionStorageRow["sourceKind"]
+  readonly source_id: string
+  readonly requested_by_user_id: string | null
+  readonly requested_by_service_account_id: string | null
+  readonly correlation_id: string
+  readonly parent_execution_id: string | null
+  readonly authority_kind: ExecutionStorageRow["authorityKind"]
+  readonly authority_user_id: string | null
+  readonly authority_service_account_id: string | null
+  readonly authority_session_id: string | null
+  readonly authority_access_token_id: string | null
+  readonly authority_primitive_kind: ExecutionStorageRow["authorityPrimitiveKind"]
+  readonly authority_primitive_id: string | null
+  readonly authority_kernel_operation: ExecutionStorageRow["authorityKernelOperation"]
+  readonly created_at: string
+}

package/src/index.ts

@@ -19,13 +19,19 @@ import {
   createWorkflowRunOperationScope,
 } from "@sixb/core/internal/storage-operation-scope"
 import {
+  type AiCostStorage,
+  type AiUsageStorage,
   createTransactionStorageProxy,
   StorageTransactionError,
   throwNestedStorageTransaction,
 } from "@sixb/core/storage"
 import { SqliteActionRunStorage } from "./action-run-storage"
 import { SqliteAgentStorage } from "./agents"
+import { SqliteAiCostStorage } from "./ai-cost-storage"
+import { SqliteAiUsageStorage } from "./ai-usage-storage"
 import { SqliteAuthStorage } from "./auth-storage"
+import { SqliteConnectorConnectionStorage } from "./connector-connection-storage"
+import { SqliteExecutionStorage } from "./execution-storage"
 import {
   createSqliteStorageMigrators,
   installFreshSqliteSchema,
@@ -37,6 +43,7 @@ import { SqlitePipelineRunStorage } from "./pipeline-run-storage"
 import { SqliteProjectionRunStorage } from "./projection-run-storage"
 import { SqliteRulesStorage } from "./rules-storage"
 import { SqliteSyncRunStorage } from "./sync-run-storage"
+import { registerSqliteStorageTestingAdapter } from "./testing"
 import { SqliteTimeseriesStorage } from "./timeseries-storage"
 import {
   closeSqliteStoreConnection,
@@ -44,7 +51,6 @@ import {
   runImmediateTransactionAsync,
   type SqliteStoreConnection,
 } from "./transactions"
-import { SqliteWebhookDeliveryStorage } from "./webhook-delivery-storage"
 import { SqliteWebhookRunStorage } from "./webhook-run-storage"
 import { SqliteWorkflowInterventionStorage } from "./workflow-intervention-storage"
 import { SqliteWorkflowRunStorage } from "./workflow-run-storage"
@@ -57,8 +63,8 @@ export interface SqliteStorageOptions {
 /**
  * SQLite storage provider for Sixb.
  *
- * Bundles object, timeseries, auth, sync run, pipeline run, projection run, workflow run, webhook
- * run, and webhook delivery storage backed by SQLite.
+ * Bundles object, timeseries, auth, execution, AI accounting, sync run, pipeline run, projection
+ * run, workflow run, webhook run, and connector connection storage backed by SQLite.
  *
  * Usage:
  * ```ts
@@ -74,7 +80,10 @@ export class SqliteStorage implements MigrationCapableStorage {
   readonly objects: Storage["objects"]
   readonly ontology: SqliteOntologyStorage
   readonly auth: SqliteAuthStorage
+  readonly executions: SqliteExecutionStorage
   readonly agents: SqliteAgentStorage
+  readonly aiUsage: AiUsageStorage
+  readonly aiCosts: AiCostStorage
   readonly actionRuns: SqliteActionRunStorage
   readonly pipelineRuns: SqlitePipelineRunStorage
   readonly syncRuns: SqliteSyncRunStorage
@@ -82,18 +91,25 @@ export class SqliteStorage implements MigrationCapableStorage {
   readonly workflowRuns: SqliteWorkflowRunStorage
   readonly workflowInterventions: SqliteWorkflowInterventionStorage
   readonly timeseries: Storage["timeseries"]
-  readonly webhookDeliveries: SqliteWebhookDeliveryStorage
   readonly webhookRuns: SqliteWebhookRunStorage
   readonly rules: SqliteRulesStorage
+  readonly connectorConnections: SqliteConnectorConnectionStorage
   readonly migrators: readonly StorageMigrator[]
 
   private readonly connection: SqliteStoreConnection
+  private readonly readConnection: SqliteStoreConnection
   private readonly transactionScope = new AsyncLocalStorage<{ active: boolean }>()
   private transactionTail: Promise<void> = Promise.resolve()
 
   constructor(options: SqliteStorageOptions = {}) {
     const path = options.path ? sqliteStoragePath(options.path) : undefined
     this.connection = openSqliteStoreConnection({ path })
+    if (path) {
+      this.connection.db.run("PRAGMA journal_mode = WAL")
+      this.readConnection = openSqliteStoreConnection({ path, readonly: true })
+    } else {
+      this.readConnection = this.connection
+    }
 
     if (this.connection.installFreshSchema) {
       installFreshSqliteSchema(this.connection.db)
@@ -112,35 +128,57 @@ export class SqliteStorage implements MigrationCapableStorage {
       (run) => this.runRootOntologyOperation(run),
       assertRootOperationAvailable
     )
-    this.objects = createOperationScopedFacade(stores.objects, scope)
+    const readScope = path
+      ? createStorageOperationScope(async (run) => {
+          this.assertRootOperationAvailable()
+          return run()
+        }, assertRootOperationAvailable)
+      : scope
+    const readObjects = path
+      ? new SqliteObjectStorage({ connection: this.readConnection })
+      : stores.objects
+    const readTimeseries = path
+      ? new SqliteTimeseriesStorage({ connection: this.readConnection })
+      : stores.timeseries
+    this.objects = createOperationScopedFacade(readObjects, readScope)
     this.ontology = createOntologyOperationScope(stores.ontology, ontologyScope)
     this.auth = createAuthOperationScope(stores.auth, scope)
+    this.executions = createOperationScopedFacade(stores.executions, scope)
     this.agents = createAgentOperationScope(stores.agents, scope)
+    this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope)
+    this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope)
     this.actionRuns = createOperationScopedFacade(stores.actionRuns, scope)
     this.pipelineRuns = createOperationScopedFacade(stores.pipelineRuns, scope)
-    this.timeseries = createOperationScopedFacade(stores.timeseries, scope)
+    this.timeseries = createOperationScopedFacade(readTimeseries, readScope)
     this.syncRuns = createOperationScopedFacade(stores.syncRuns, scope)
     this.projectionRuns = createOperationScopedFacade(stores.projectionRuns, scope)
     this.workflowRuns = createWorkflowRunOperationScope(stores.workflowRuns, scope)
     this.workflowInterventions = createOperationScopedFacade(stores.workflowInterventions, scope)
-    this.webhookDeliveries = createOperationScopedFacade(stores.webhookDeliveries, scope)
     this.webhookRuns = createOperationScopedFacade(stores.webhookRuns, scope)
     this.rules = createOperationScopedFacade(stores.rules, scope)
+    this.connectorConnections = createOperationScopedFacade(stores.connectorConnections, scope)
     this.migrators = options.path ? createSqliteStorageMigrators(options.path) : []
+    registerSqliteStorageTestingAdapter(this, (durationMs) =>
+      stores.connectorConnections.advanceTimeForTesting(durationMs)
+    )
   }
 
   async ping(): Promise<void> {
+    this.assertRootOperationAvailable()
+    if (this.readConnection !== this.connection) {
+      this.readConnection.db.query("SELECT 1").get()
+      return
+    }
     await this.runRootStorageOperation(() => {
       this.connection.db.query("SELECT 1").get()
     })
   }
 
   /**
-   * The `isolation` option is intentionally ignored. SQLite runs every transaction through one
-   * shared connection serialized by {@link withTransactionLock} and a `BEGIN IMMEDIATE`, so there is
-   * no concurrent transaction to isolate against — the lock already provides serializable semantics
-   * for transaction-vs-transaction races. Postgres, which has true concurrent connections, is where
-   * `isolation: "serializable"` translates to a real isolation level.
+   * The `isolation` option is intentionally ignored. SQLite runs every write transaction through
+   * one connection serialized by {@link withTransactionLock} and a `BEGIN IMMEDIATE`, so there is no
+   * concurrent writer to isolate against. File-backed object and timeseries reads use a separate WAL
+   * snapshot connection; Postgres is where `isolation: "serializable"` maps to a real isolation level.
    */
   async transaction<T>(
     run: (tx: Storage) => Promise<T> | T,
@@ -183,6 +221,7 @@ export class SqliteStorage implements MigrationCapableStorage {
   }
 
   close(): void {
+    if (this.readConnection !== this.connection) closeSqliteStoreConnection(this.readConnection)
     closeSqliteStoreConnection(this.connection)
   }
 
@@ -250,6 +289,8 @@ function createSqliteStores(
     readonly transactionContext: SqliteOntologyTransactionContext | null
   }
 ): SqliteStoreSet {
+  const auth = new SqliteAuthStorage({ connection })
+  const executions = new SqliteExecutionStorage(connection.db, auth)
   return {
     objects: new SqliteObjectStorage({ connection }),
     ontology: new SqliteOntologyStorage({
@@ -257,18 +298,21 @@ function createSqliteStores(
       runRootOperation: options.runOntologyOperation,
       transactionContext: options.transactionContext,
     }),
-    auth: new SqliteAuthStorage({ connection }),
-    agents: new SqliteAgentStorage({ connection }),
-    actionRuns: new SqliteActionRunStorage({ connection }),
-    pipelineRuns: new SqlitePipelineRunStorage({ connection }),
+    auth,
+    executions,
+    agents: new SqliteAgentStorage({ connection, executions }),
+    aiUsage: new SqliteAiUsageStorage({ connection }),
+    aiCosts: new SqliteAiCostStorage(connection),
+    actionRuns: new SqliteActionRunStorage({ connection, executions }),
+    pipelineRuns: new SqlitePipelineRunStorage({ connection, executions }),
     timeseries: new SqliteTimeseriesStorage({ connection }),
-    syncRuns: new SqliteSyncRunStorage({ connection }),
-    projectionRuns: new SqliteProjectionRunStorage({ connection }),
-    workflowRuns: new SqliteWorkflowRunStorage({ connection }),
+    syncRuns: new SqliteSyncRunStorage({ connection, executions }),
+    projectionRuns: new SqliteProjectionRunStorage({ connection, executions }),
+    workflowRuns: new SqliteWorkflowRunStorage({ connection, executions }),
     workflowInterventions: new SqliteWorkflowInterventionStorage({ connection }),
-    webhookDeliveries: new SqliteWebhookDeliveryStorage({ connection }),
-    webhookRuns: new SqliteWebhookRunStorage({ connection }),
+    webhookRuns: new SqliteWebhookRunStorage({ connection, executions }),
     rules: new SqliteRulesStorage({ connection }),
+    connectorConnections: new SqliteConnectorConnectionStorage(connection),
   }
 }
 
@@ -276,7 +320,10 @@ interface SqliteStoreSet {
   readonly objects: SqliteObjectStorage
   readonly ontology: SqliteOntologyStorage
   readonly auth: SqliteAuthStorage
+  readonly executions: SqliteExecutionStorage
   readonly agents: SqliteAgentStorage
+  readonly aiUsage: SqliteAiUsageStorage
+  readonly aiCosts: SqliteAiCostStorage
   readonly actionRuns: SqliteActionRunStorage
   readonly pipelineRuns: SqlitePipelineRunStorage
   readonly syncRuns: SqliteSyncRunStorage
@@ -284,9 +331,9 @@ interface SqliteStoreSet {
   readonly workflowRuns: SqliteWorkflowRunStorage
   readonly workflowInterventions: SqliteWorkflowInterventionStorage
   readonly timeseries: SqliteTimeseriesStorage
-  readonly webhookDeliveries: SqliteWebhookDeliveryStorage
   readonly webhookRuns: SqliteWebhookRunStorage
   readonly rules: SqliteRulesStorage
+  readonly connectorConnections: SqliteConnectorConnectionStorage
 }
 
 export { migrateSqliteStorage } from "./migrations"

package/src/latest-run-query.ts

@@ -21,6 +21,12 @@ export function queryLatestRunsByOwnerId<TRow>(
   }
 
   const placeholders = ownerIds.map(() => "?").join(", ")
+  const orderColumn =
+    input.tableName === "sync_runs" ||
+    input.tableName === "pipeline_runs" ||
+    input.tableName === "projection_runs"
+      ? "COALESCE(started_at, queued_at)"
+      : "started_at"
   const rows = input.db
     .query(
       `
@@ -30,7 +36,7 @@ export function queryLatestRunsByOwnerId<TRow>(
             *,
             ROW_NUMBER() OVER (
               PARTITION BY ${input.ownerColumn}
-              ORDER BY started_at DESC, id DESC
+              ORDER BY ${orderColumn} DESC, id DESC
             ) AS sixb_latest_rank
           FROM ${input.tableName}
           WHERE project_id = ? AND ${input.ownerColumn} IN (${placeholders})

package/src/migrations/004-executions.sql

@@ -0,0 +1,142 @@
+CREATE TABLE executions (
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  id TEXT NOT NULL CHECK (length(trim(id)) > 0),
+
+  executor_kind TEXT NOT NULL CHECK (
+    executor_kind IN (
+      'request', 'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow',
+      'agent', 'kernel'
+    )
+  ),
+  executor_id TEXT NOT NULL CHECK (length(trim(executor_id)) > 0),
+  source_kind TEXT NOT NULL CHECK (
+    source_kind IN ('http', 'webhook', 'schedule', 'event', 'execution')
+  ),
+  source_id TEXT NOT NULL CHECK (length(trim(source_id)) > 0),
+
+  requested_by_user_id TEXT,
+  requested_by_service_account_id TEXT,
+  correlation_id TEXT NOT NULL CHECK (length(trim(correlation_id)) > 0),
+  parent_execution_id TEXT,
+
+  authority_kind TEXT NOT NULL CHECK (
+    authority_kind IN ('principal', 'trustedPrimitive', 'kernel', 'disabled')
+  ),
+  authority_user_id TEXT,
+  authority_service_account_id TEXT,
+  authority_session_id TEXT,
+  authority_access_token_id TEXT,
+  authority_primitive_kind TEXT CHECK (
+    authority_primitive_kind IS NULL
+      OR authority_primitive_kind IN (
+        'action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow'
+      )
+  ),
+  authority_primitive_id TEXT,
+  authority_kernel_operation TEXT CHECK (
+    authority_kernel_operation IS NULL OR authority_kernel_operation = 'ontology.recover'
+  ),
+
+  created_at TEXT NOT NULL,
+
+  PRIMARY KEY (project_id, id),
+  FOREIGN KEY (project_id, parent_execution_id)
+    REFERENCES executions (project_id, id),
+  FOREIGN KEY (project_id, requested_by_user_id)
+    REFERENCES auth_users (project_id, id),
+  FOREIGN KEY (project_id, requested_by_service_account_id)
+    REFERENCES auth_service_accounts (project_id, id),
+  FOREIGN KEY (project_id, authority_user_id)
+    REFERENCES auth_users (project_id, id),
+  FOREIGN KEY (project_id, authority_service_account_id)
+    REFERENCES auth_service_accounts (project_id, id),
+  FOREIGN KEY (project_id, authority_session_id)
+    REFERENCES auth_sessions (project_id, id),
+  FOREIGN KEY (project_id, authority_access_token_id)
+    REFERENCES auth_access_tokens (project_id, id),
+
+  CHECK (
+    (requested_by_user_id IS NOT NULL) + (requested_by_service_account_id IS NOT NULL) <= 1
+  ),
+  CHECK (
+    (source_kind = 'execution' AND parent_execution_id = source_id)
+      OR (source_kind <> 'execution' AND parent_execution_id IS NULL)
+  ),
+  CHECK (
+    (
+      authority_kind = 'principal'
+      AND (authority_user_id IS NOT NULL) + (authority_service_account_id IS NOT NULL) = 1
+      AND (authority_session_id IS NOT NULL) + (authority_access_token_id IS NOT NULL) <= 1
+      AND (authority_session_id IS NULL OR authority_user_id IS NOT NULL)
+      AND authority_primitive_kind IS NULL
+      AND authority_primitive_id IS NULL
+      AND authority_kernel_operation IS NULL
+    )
+    OR (
+      authority_kind = 'trustedPrimitive'
+      AND authority_user_id IS NULL
+      AND authority_service_account_id IS NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+      AND authority_primitive_kind IS NOT NULL
+      AND authority_primitive_id IS NOT NULL
+      AND length(trim(authority_primitive_id)) > 0
+      AND authority_kernel_operation IS NULL
+    )
+    OR (
+      authority_kind = 'kernel'
+      AND authority_user_id IS NULL
+      AND authority_service_account_id IS NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+      AND authority_primitive_kind IS NULL
+      AND authority_primitive_id IS NULL
+      AND authority_kernel_operation IS NOT NULL
+    )
+    OR (
+      authority_kind = 'disabled'
+      AND authority_user_id IS NULL
+      AND authority_service_account_id IS NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+      AND authority_primitive_kind IS NULL
+      AND authority_primitive_id IS NULL
+      AND authority_kernel_operation IS NULL
+    )
+  ),
+  CHECK (
+    (
+      executor_kind = 'request'
+      AND source_kind = 'http'
+      AND executor_id = source_id
+      AND (
+        (
+          authority_kind = 'principal'
+          AND requested_by_user_id IS authority_user_id
+          AND requested_by_service_account_id IS authority_service_account_id
+        )
+        OR (
+          authority_kind = 'disabled'
+          AND requested_by_user_id IS NULL
+          AND requested_by_service_account_id IS NULL
+        )
+      )
+    )
+    OR (
+      executor_kind IN ('action', 'pipeline', 'projection', 'rule', 'sync', 'webhook', 'workflow')
+      AND authority_kind = 'trustedPrimitive'
+      AND executor_kind = authority_primitive_kind
+    )
+    OR (
+      executor_kind = 'agent'
+      AND authority_kind = 'principal'
+      AND authority_service_account_id IS NOT NULL
+      AND authority_session_id IS NULL
+      AND authority_access_token_id IS NULL
+    )
+    OR (
+      executor_kind = 'kernel'
+      AND authority_kind = 'kernel'
+    )
+  )
+);

package/src/migrations/005-workflow-executions.sql

@@ -0,0 +1,7 @@
+ALTER TABLE workflow_runs ADD COLUMN execution_id TEXT NOT NULL;
+ALTER TABLE workflow_runs DROP COLUMN source;
+ALTER TABLE workflow_runs DROP COLUMN requested_by_principal_type;
+ALTER TABLE workflow_runs DROP COLUMN requested_by_principal_id;
+
+CREATE UNIQUE INDEX idx_workflow_runs_project_execution
+  ON workflow_runs(project_id, execution_id);

package/src/migrations/006-narrow-ontology-source-root-index.sql

@@ -0,0 +1,4 @@
+DROP INDEX idx_ontology_source_rows_root;
+
+CREATE INDEX idx_ontology_source_rows_root
+  ON ontology_source_rows(project_id, source_id, materialization_id, root_sort_key);