@sixb/sqlite 0.1.12 → 0.1.13

package/src/migrations/047-ontology-commit-attribution.sql

@@ -0,0 +1,67 @@
+-- Commits and their events name who requested the write and what ran it, replacing the
+-- authority-only `actor`. Both are copied from the commit's execution, which stays canonical and
+-- is immutable, so the backfill is exact. SQLite cannot add a NOT NULL column without a default;
+-- every insert supplies `executor`, and the UPDATE below replaces the placeholder on every row.
+ALTER TABLE ontology_commits ADD COLUMN requested_by TEXT
+  CHECK (requested_by IS NULL OR json_valid(requested_by));
+ALTER TABLE ontology_commits ADD COLUMN executor TEXT NOT NULL DEFAULT '{}'
+  CHECK (json_valid(executor));
+
+UPDATE ontology_commits SET
+  requested_by = (
+    SELECT CASE
+      WHEN e.requested_by_user_id IS NOT NULL
+        THEN json_object('id', e.requested_by_user_id, 'type', 'user')
+      WHEN e.requested_by_service_account_id IS NOT NULL
+        THEN json_object('id', e.requested_by_service_account_id, 'type', 'serviceAccount')
+    END
+    FROM executions e
+    WHERE e.project_id = ontology_commits.project_id AND e.id = ontology_commits.execution_id
+  ),
+  executor = (
+    SELECT CASE e.executor_kind
+      WHEN 'request' THEN json_object('requestId', e.executor_id, 'type', 'request')
+      WHEN 'agent' THEN json_object('runId', e.executor_id, 'type', 'agent')
+      WHEN 'kernel' THEN json_object(
+        'operation',
+        CASE e.authority_kernel_operation
+          WHEN 'ontology.recover'
+            THEN json_object('recoveryId', e.executor_id, 'type', 'ontology.recover')
+          ELSE json_object('indexingId', e.executor_id, 'type', 'ontology.indexVectors')
+        END,
+        'type', 'kernel'
+      )
+      ELSE json_object(
+        'id', e.authority_primitive_id,
+        'kind', e.executor_kind,
+        'runId', e.executor_id,
+        'type', 'primitive'
+      )
+    END
+    FROM executions e
+    WHERE e.project_id = ontology_commits.project_id AND e.id = ontology_commits.execution_id
+  );
+
+-- Events not yet published would otherwise reach consumers in the retired shape.
+UPDATE ontology_outbox SET envelope = (
+  SELECT json_remove(
+    CASE
+      WHEN c.requested_by IS NULL
+        THEN json_set(ontology_outbox.envelope, '$.executor', json(c.executor))
+      ELSE json_set(
+        ontology_outbox.envelope,
+        '$.executor', json(c.executor),
+        '$.requestedBy', json(c.requested_by)
+      )
+    END,
+    '$.actor'
+  )
+  FROM ontology_commits c
+  WHERE c.project_id = ontology_outbox.project_id AND c.id = ontology_outbox.commit_id
+)
+WHERE EXISTS (
+  SELECT 1 FROM ontology_commits c
+  WHERE c.project_id = ontology_outbox.project_id AND c.id = ontology_outbox.commit_id
+);
+
+ALTER TABLE ontology_commits DROP COLUMN actor;

package/src/migrations/048-file-upload-sessions.sql

@@ -0,0 +1,35 @@
+-- Durable staged-upload sessions, so an upload survives restarts and spans API replicas.
+-- Timestamps are ISO-8601 UTC text and compare lexicographically.
+CREATE TABLE file_upload_sessions (
+  id TEXT PRIMARY KEY CHECK (length(trim(id)) > 0),
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  owner_principal_type TEXT NOT NULL
+    CHECK (owner_principal_type IN ('user', 'serviceAccount', 'system')),
+  owner_principal_id TEXT NOT NULL CHECK (length(owner_principal_id) > 0),
+  strategy TEXT NOT NULL CHECK (strategy IN ('server', 'direct-put', 'multipart')),
+  status TEXT NOT NULL CHECK (status IN ('pending', 'completed', 'aborted')),
+  file_name TEXT,
+  media_type TEXT,
+  logical_path TEXT,
+  expected_size_bytes INTEGER CHECK (expected_size_bytes IS NULL OR expected_size_bytes >= 0),
+  expected_digest TEXT,
+  provider_upload TEXT CHECK (provider_upload IS NULL OR json_type(provider_upload) = 'object'),
+  signed_parts TEXT NOT NULL CHECK (json_type(signed_parts) = 'array'),
+  file_ref TEXT CHECK (file_ref IS NULL OR json_type(file_ref) = 'object'),
+  created_at TEXT NOT NULL,
+  expires_at TEXT NOT NULL,
+  completed_at TEXT,
+  aborted_at TEXT,
+  -- Derived by core (fileUploadSessionReapAt); NULL while an abandoned upload awaits its abort.
+  reap_at TEXT,
+  CHECK ((status = 'completed') = (completed_at IS NOT NULL)),
+  CHECK ((status = 'aborted') = (aborted_at IS NOT NULL))
+);
+
+CREATE INDEX idx_file_upload_sessions_reap
+  ON file_upload_sessions (reap_at)
+  WHERE reap_at IS NOT NULL;
+
+CREATE INDEX idx_file_upload_sessions_abandoned
+  ON file_upload_sessions (project_id, expires_at)
+  WHERE status = 'pending' AND provider_upload IS NOT NULL;

package/src/migrations/049-native-sessions.sql

@@ -0,0 +1,12 @@
+-- Native clients, such as the CLI, hold their session as bearer tokens: a
+-- short-lived access token, whose hash is `token_hash`, and a rotating refresh token. Browser
+-- sessions leave these columns NULL; `refresh_token_hash IS NULL` is what marks a cookie session.
+ALTER TABLE auth_sessions ADD COLUMN client_name TEXT;
+ALTER TABLE auth_sessions ADD COLUMN access_expires_at TEXT;
+ALTER TABLE auth_sessions ADD COLUMN refresh_token_hash TEXT;
+ALTER TABLE auth_sessions ADD COLUMN previous_refresh_token_hash TEXT;
+ALTER TABLE auth_sessions ADD COLUMN refreshed_at TEXT;
+
+-- The device flow now starts a session instead of issuing a personal access token.
+ALTER TABLE auth_device_authorizations DROP COLUMN token_name;
+ALTER TABLE auth_device_authorizations DROP COLUMN token_expires_at;

package/src/migrations.ts

@@ -105,6 +105,13 @@ import vectorBatchingSql from "./migrations/044-vector-batching.sql" with { type
 import agentThreadSandboxStateSql from "./migrations/045-agent-thread-sandbox-state.sql" with {
   type: "text",
 }
+import workflowInterventionPrincipalsSql from "./migrations/046-workflow-intervention-principals.sql" with {
+  type: "text",
+}
+import ontologyCommitAttributionSql from "./migrations/047-ontology-commit-attribution.sql" with {
+  type: "text",
+}
+import nativeSessionsSql from "./migrations/049-native-sessions.sql" with { type: "text" }
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -205,6 +212,10 @@ export const sqliteStorageMigrations = defineMigrations({
     sqliteSql("043-vector-indexing", vectorIndexingSql),
     sqliteSql("044-vector-batching", vectorBatchingSql),
     sqliteSql("045-agent-thread-sandbox-state", agentThreadSandboxStateSql),
+    sqliteSql("046-workflow-intervention-principals", workflowInterventionPrincipalsSql),
+    sqliteSql("047-ontology-commit-attribution", ontologyCommitAttributionSql),
+    sqliteSql("048-file-upload-sessions", fileUploadSessionsSql),
+    sqliteSql("049-native-sessions", nativeSessionsSql),
   ],
 })
 
@@ -428,3 +439,4 @@ import executionRequesterGroupsSql from "./migrations/037-execution-requester-gr
 import connectorOptionalPkceSql from "./migrations/039-connector-optional-pkce.sql" with {
   type: "text",
 }
+import fileUploadSessionsSql from "./migrations/048-file-upload-sessions.sql" with { type: "text" }

package/src/objects/query-compiler.ts

@@ -963,9 +963,13 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
       return negatePredicate(item)
     }
     case "eq":
-      return compileEqualityPredicate(predicate.propertyId, "eq", predicate.value)
+      return predicate.scalarKind === "userRef"
+        ? compileUserRefInPredicate(predicate.propertyId, [predicate.value])
+        : compileEqualityPredicate(predicate.propertyId, "eq", predicate.value)
     case "neq":
-      return compileEqualityPredicate(predicate.propertyId, "neq", predicate.value)
+      return predicate.scalarKind === "userRef"
+        ? negatePredicate(compileUserRefInPredicate(predicate.propertyId, [predicate.value]))
+        : compileEqualityPredicate(predicate.propertyId, "neq", predicate.value)
     case "lt":
     case "lte":
     case "gt":
@@ -977,17 +981,68 @@ function compilePredicate(predicate: ObjectQueryPredicate): CompiledPredicate {
       }
     }
     case "in":
-      return compileInPredicate(predicate.propertyId, predicate.values)
+      return predicate.scalarKind === "userRef"
+        ? compileUserRefInPredicate(predicate.propertyId, predicate.values)
+        : compileInPredicate(predicate.propertyId, predicate.values)
     case "exists": {
       const sql = `${jsonTypeExpression()} IS NOT NULL`
       const item: CompiledPredicate = { sql, args: [jsonPath(predicate.propertyId)] }
       return predicate.value ? item : negatePredicate(item)
     }
     case "contains":
-      return compileContainsPredicate(predicate.propertyId, predicate.value)
+      return predicate.scalarKind === "userRef"
+        ? compileUserRefContainsPredicate(predicate.propertyId, predicate.value)
+        : compileContainsPredicate(predicate.propertyId, predicate.value)
   }
 }
 
+/**
+ * User references are JSON objects, so they match on both fields rather than on serialized text,
+ * which would depend on key order.
+ */
+function compileUserRefInPredicate(
+  propertyId: string,
+  values: readonly unknown[]
+): CompiledPredicate {
+  const path = jsonPath(propertyId)
+  const ids = values.flatMap((value) => {
+    const id = userRefId(value)
+    return id === undefined ? [] : [id]
+  })
+  const clauses: string[] = []
+  const args: SqliteValue[] = []
+
+  if (values.includes(null)) {
+    clauses.push(`${jsonTypeExpression()} = 'null'`)
+    args.push(path)
+  }
+  if (ids.length > 0) {
+    clauses.push(
+      `(${jsonValueExpression()} = 'user' AND ${jsonValueExpression()} IN (${ids.map(() => "?").join(", ")}))`
+    )
+    args.push(`${path}.type`, `${path}.id`, ...ids)
+  }
+  if (clauses.length === 0) return { sql: "0 = 1", args: [] }
+
+  return { sql: `(${clauses.join(" OR ")})`, args }
+}
+
+function compileUserRefContainsPredicate(propertyId: string, value: unknown): CompiledPredicate {
+  const id = userRefId(value)
+  if (id === undefined) return { sql: "0 = 1", args: [] }
+  const path = jsonPath(propertyId)
+  return {
+    sql: `(${jsonTypeExpression()} = 'array' AND EXISTS (SELECT 1 FROM json_each(properties, ?) AS item WHERE item.type = 'object' AND json_extract(item.value, '$.type') = 'user' AND json_extract(item.value, '$.id') = ?))`,
+    args: [path, path, id],
+  }
+}
+
+function userRefId(value: unknown): string | undefined {
+  if (typeof value !== "object" || value === null) return undefined
+  const { type, id } = value as { type?: unknown; id?: unknown }
+  return type === "user" && typeof id === "string" ? id : undefined
+}
+
 function negatePredicate(predicate: CompiledPredicate): CompiledPredicate {
   return {
     sql: `(NOT COALESCE((${predicate.sql}), 0))`,

package/src/objects/reader.ts

@@ -512,6 +512,10 @@ function sqliteFacetValue(valueType: string | null, value: unknown): unknown {
   if (valueType === "true") return true
   if (valueType === "false") return false
   if (valueType === "null") return null
+  // json_extract returns structured values, such as user references, as JSON text.
+  if ((valueType === "object" || valueType === "array") && typeof value === "string") {
+    return JSON.parse(value)
+  }
   return value
 }
 

package/src/objects/storage.ts

@@ -95,6 +95,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
     decimal: { equality: true },
     date: { equality: true, ordering: true },
     timestamp: { equality: true, ordering: true },
+    userRef: { equality: true },
   },
   limits: {
     totalCount: true,

package/src/ontology-storage/materializations.ts

@@ -1,8 +1,4 @@
 import type { Database } from "bun:sqlite"
-import type {
-  ExpectedLinkRevision,
-  ExpectedObjectRevision,
-} from "@sixb/core/internal/materialization"
 import {
   assertPinnedDatasetWatermark,
   linkRefKey,
@@ -16,6 +12,9 @@ import {
   telemetryPointSortKey,
 } from "@sixb/core/internal/materialization"
 import {
+  assertExpectedLinkRevision,
+  assertExpectedLinkScopeRevision,
+  assertExpectedObjectRevision,
   assertMaterializationFinalizationCorrelation,
   assertMaterializationHeader,
   assertMaterializationLaneCompletion,
@@ -100,16 +99,15 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
       const reader = new SqliteMaterializationStateReader(this.db, input.commit.projectId)
       for (const expected of input.expected.sources)
         this.assertSource(expected, input.commit.projectId)
-      for (const expected of input.expected.objects) this.assertObject(reader, expected)
-      for (const expected of input.expected.links) this.assertLink(reader, expected)
+      for (const expected of input.expected.objects) {
+        assertExpectedObjectRevision(reader.effectiveObjectRevision(expected.ref), expected)
+      }
+      for (const expected of input.expected.links) {
+        assertExpectedLinkRevision(reader.effectiveLinkLastCommit(expected.ref), expected)
+      }
       const linkScopeRevisions = reader.linkScopeRevisions(input.expected.linkScopes)
       for (const [index, expected] of input.expected.linkScopes.entries()) {
-        if (linkScopeRevisions[index]?.fingerprint !== expected.fingerprint) {
-          throw new MaterializationConflictError(
-            "effective-state",
-            `Expected link scope changed for ${expected.source.objectTypeId}:${expected.source.primaryId}.${expected.linkId}.`
-          )
-        }
+        assertExpectedLinkScopeRevision(linkScopeRevisions[index]?.fingerprint, expected)
       }
       for (const expected of input.expected.points) {
         const point = reader.exactPoint(expected.series, expected.at)
@@ -449,50 +447,6 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
     }
   }
 
-  private assertObject(
-    reader: SqliteMaterializationStateReader,
-    expected: ExpectedObjectRevision
-  ): void {
-    const row = reader.effectiveObjectRevision(expected.ref)
-    if (!expected.exists) {
-      if (row) {
-        throw new MaterializationConflictError(
-          "effective-state",
-          `Expected object ${objectRefKey(expected.ref)} to be absent.`
-        )
-      }
-      return
-    }
-    if (!row || row.version !== expected.version || row.lastCommitId !== expected.lastCommitId) {
-      throw new MaterializationConflictError(
-        "effective-state",
-        `Expected object ${objectRefKey(expected.ref)} changed.`
-      )
-    }
-  }
-
-  private assertLink(
-    reader: SqliteMaterializationStateReader,
-    expected: ExpectedLinkRevision
-  ): void {
-    const lastCommitId = reader.effectiveLinkLastCommit(expected.ref)
-    if (!expected.exists) {
-      if (lastCommitId !== undefined) {
-        throw new MaterializationConflictError(
-          "effective-state",
-          `Expected link ${linkRefKey(expected.ref)} to be absent.`
-        )
-      }
-      return
-    }
-    if (lastCommitId === undefined || lastCommitId !== expected.lastCommitId) {
-      throw new MaterializationConflictError(
-        "effective-state",
-        `Expected link ${linkRefKey(expected.ref)} changed.`
-      )
-    }
-  }
-
   private async assertFinalization(
     session: SqliteMaterializationSessionState,
     input: FinalizeMaterializationInput
@@ -719,10 +673,12 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
           `
             INSERT INTO ontology_commits (
               project_id, id, idempotency_key, request_hash, execution_id,
-              origin_kind, origin_run_id, origin_batch_ordinal, origin, actor,
+              origin_kind, origin_run_id, origin_batch_ordinal, origin, requested_by, executor,
               ontology_revision, projection_revision, ownership_hash,
               intent, result, committed_at
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), ?, ?, ?, json(?), json(?), ?)
+            ) VALUES (
+              ?, ?, ?, ?, ?, ?, ?, ?, json(?), json(?), json(?), ?, ?, ?, json(?), json(?), ?
+            )
           `
         )
         .run(
@@ -735,7 +691,8 @@ export class SqliteOntologyMaterializationStorage implements OntologyMaterializa
           origin.runId,
           origin.batchOrdinal,
           canonicalJson(commit.origin),
-          commit.actor === undefined ? null : canonicalJson(commit.actor),
+          commit.requestedBy === undefined ? null : canonicalJson(commit.requestedBy),
+          canonicalJson(commit.executor),
           commit.ontologyRevision,
           commit.projectionRevision ?? null,
           commit.ownershipHash ?? null,

package/src/ontology-storage/shared.ts

@@ -37,7 +37,8 @@ export interface SqliteOntologyCommitRow {
   readonly origin_run_id: string | null
   readonly origin_batch_ordinal: number | null
   readonly origin: string
-  readonly actor: string | null
+  readonly requested_by: string | null
+  readonly executor: string
   readonly ontology_revision: string
   readonly projection_revision: string | null
   readonly ownership_hash: string | null
@@ -162,9 +163,12 @@ export function commitRecord(row: SqliteOntologyCommitRow): OntologyCommitRecord
     requestHash: row.request_hash,
     executionId: row.execution_id,
     origin: parseJson<OntologyCommitWrite["origin"]>(row.origin),
-    ...(row.actor === null
+    ...(row.requested_by === null
       ? {}
-      : { actor: parseJson<NonNullable<OntologyCommitWrite["actor"]>>(row.actor) }),
+      : {
+          requestedBy: parseJson<NonNullable<OntologyCommitWrite["requestedBy"]>>(row.requested_by),
+        }),
+    executor: parseJson<OntologyCommitWrite["executor"]>(row.executor),
     ontologyRevision: row.ontology_revision,
     ...(row.projection_revision === null ? {} : { projectionRevision: row.projection_revision }),
     ...(row.ownership_hash === null ? {} : { ownershipHash: row.ownership_hash }),

package/src/workflow-intervention-storage.ts

@@ -6,8 +6,8 @@ import type {
   ExpireWorkflowInterventionInput,
   ListWorkflowInterventionsInput,
   ListWorkflowInterventionsResult,
+  Principal,
   SubmitWorkflowInterventionInput,
-  WorkflowInterventionActor,
   WorkflowInterventionRecord,
   WorkflowInterventionStorage,
 } from "@sixb/core/storage"
@@ -113,7 +113,7 @@ export class SqliteWorkflowInterventionStorage implements WorkflowInterventionSt
         .run(
           "submitted",
           (input.submittedAt ?? new Date()).toISOString(),
-          serializeActor(input.submittedBy),
+          serializePrincipal(input.submittedBy),
           serializeRecord(input.response),
           existing.project_id,
           existing.id
@@ -141,7 +141,7 @@ export class SqliteWorkflowInterventionStorage implements WorkflowInterventionSt
         .run(
           "cancelled",
           (input.cancelledAt ?? new Date()).toISOString(),
-          serializeActor(input.cancelledBy),
+          serializePrincipal(input.cancelledBy),
           existing.project_id,
           existing.id
         )
@@ -331,12 +331,12 @@ function parseRecord(value: string): WorkflowIOSnapshot {
   return JSON.parse(value) as WorkflowIOSnapshot
 }
 
-function serializeActor(actor: WorkflowInterventionActor | undefined): string | null {
-  return actor ? JSON.stringify(actor) : null
+function serializePrincipal(principal: Principal | undefined): string | null {
+  return principal ? JSON.stringify(principal) : null
 }
 
-function parseActor(value: string | null): WorkflowInterventionActor | undefined {
-  return value ? (JSON.parse(value) as WorkflowInterventionActor) : undefined
+function parsePrincipal(value: string | null): Principal | undefined {
+  return value ? (JSON.parse(value) as Principal) : undefined
 }
 
 function rowToWorkflowInterventionRecord(
@@ -358,10 +358,10 @@ function rowToWorkflowInterventionRecord(
     requestedAt: new Date(row.requested_at),
     expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,
     submittedAt: row.submitted_at ? new Date(row.submitted_at) : undefined,
-    submittedBy: parseActor(row.submitted_by),
+    submittedBy: parsePrincipal(row.submitted_by),
     response: row.response ? parseRecord(row.response) : undefined,
     cancelledAt: row.cancelled_at ? new Date(row.cancelled_at) : undefined,
-    cancelledBy: parseActor(row.cancelled_by),
+    cancelledBy: parsePrincipal(row.cancelled_by),
     expiredAt: row.expired_at ? new Date(row.expired_at) : undefined,
   }
 }