@sixb/sqlite 0.1.2 → 0.1.4

package/src/ai-usage-storage.ts

@@ -7,6 +7,7 @@ import {
   AiUsageStorageError,
   aggregateAiModelCallUsage,
   assertAiUsageExecutionId,
+  type GetLatestAiModelCallForExecutionInput,
   normalizeAiModelCallRecord,
   type RecordAiModelCallInput,
   type RecordAiModelCallResult,
@@ -84,6 +85,24 @@ export class SqliteAiUsageStorage implements AiUsageStorage {
     return summary!
   }
 
+  async getLatestForExecution(
+    input: GetLatestAiModelCallForExecutionInput
+  ): Promise<AiModelCallUsageRecord | null> {
+    assertNonBlankProjectId(input.projectId)
+    assertAiUsageExecutionId(input.executionId)
+    const row = this.connection.db
+      .query(
+        `
+          SELECT * FROM ai_model_call_usage
+          WHERE project_id = ? AND execution_id = ?
+          ORDER BY occurred_at DESC, id DESC
+          LIMIT 1
+        `
+      )
+      .get(input.projectId, input.executionId) as AiUsageRow | null
+    return row ? this.rowToRecord(row) : null
+  }
+
   async summarizeExecutions(
     input: SummarizeAiUsageExecutionsInput
   ): Promise<readonly AiUsageExecutionSummary[]> {

package/src/index.ts

@@ -19,6 +19,7 @@ import {
   createWorkflowRunOperationScope,
 } from "@sixb/core/internal/storage-operation-scope"
 import {
+  type AiCostStorage,
   type AiUsageStorage,
   createTransactionStorageProxy,
   StorageTransactionError,
@@ -26,6 +27,7 @@ import {
 } from "@sixb/core/storage"
 import { SqliteActionRunStorage } from "./action-run-storage"
 import { SqliteAgentStorage } from "./agents"
+import { SqliteAiCostStorage } from "./ai-cost-storage"
 import { SqliteAiUsageStorage } from "./ai-usage-storage"
 import { SqliteAuthStorage } from "./auth-storage"
 import { SqliteConnectorConnectionStorage } from "./connector-connection-storage"
@@ -61,8 +63,8 @@ export interface SqliteStorageOptions {
 /**
  * SQLite storage provider for Sixb.
  *
- * Bundles object, timeseries, auth, execution, AI usage, sync run, pipeline run, projection run,
- * workflow run, webhook run, and connector connection storage backed by SQLite.
+ * Bundles object, timeseries, auth, execution, AI accounting, sync run, pipeline run, projection
+ * run, workflow run, webhook run, and connector connection storage backed by SQLite.
  *
  * Usage:
  * ```ts
@@ -81,6 +83,7 @@ export class SqliteStorage implements MigrationCapableStorage {
   readonly executions: SqliteExecutionStorage
   readonly agents: SqliteAgentStorage
   readonly aiUsage: AiUsageStorage
+  readonly aiCosts: AiCostStorage
   readonly actionRuns: SqliteActionRunStorage
   readonly pipelineRuns: SqlitePipelineRunStorage
   readonly syncRuns: SqliteSyncRunStorage
@@ -143,6 +146,7 @@ export class SqliteStorage implements MigrationCapableStorage {
     this.executions = createOperationScopedFacade(stores.executions, scope)
     this.agents = createAgentOperationScope(stores.agents, scope)
     this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope)
+    this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope)
     this.actionRuns = createOperationScopedFacade(stores.actionRuns, scope)
     this.pipelineRuns = createOperationScopedFacade(stores.pipelineRuns, scope)
     this.timeseries = createOperationScopedFacade(readTimeseries, readScope)
@@ -298,6 +302,7 @@ function createSqliteStores(
     executions,
     agents: new SqliteAgentStorage({ connection, executions }),
     aiUsage: new SqliteAiUsageStorage({ connection }),
+    aiCosts: new SqliteAiCostStorage(connection),
     actionRuns: new SqliteActionRunStorage({ connection, executions }),
     pipelineRuns: new SqlitePipelineRunStorage({ connection, executions }),
     timeseries: new SqliteTimeseriesStorage({ connection }),
@@ -318,6 +323,7 @@ interface SqliteStoreSet {
   readonly executions: SqliteExecutionStorage
   readonly agents: SqliteAgentStorage
   readonly aiUsage: SqliteAiUsageStorage
+  readonly aiCosts: SqliteAiCostStorage
   readonly actionRuns: SqliteActionRunStorage
   readonly pipelineRuns: SqlitePipelineRunStorage
   readonly syncRuns: SqliteSyncRunStorage

package/src/migrations/026-ai-cost-accounting.sql

@@ -0,0 +1,57 @@
+CREATE TABLE ai_model_call_valuations (
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  usage_record_id TEXT NOT NULL CHECK (length(trim(usage_record_id)) > 0),
+  status TEXT NOT NULL CHECK (status IN ('rated', 'unpriceable')),
+  provider_id TEXT CHECK (provider_id IS NULL OR length(trim(provider_id)) > 0),
+  model_id TEXT CHECK (model_id IS NULL OR length(trim(model_id)) > 0),
+  currency TEXT CHECK (
+    currency IS NULL OR (length(currency) = 3 AND currency GLOB '[A-Z][A-Z][A-Z]')
+  ),
+  amount_nanos INTEGER CHECK (
+    amount_nanos IS NULL OR (typeof(amount_nanos) = 'integer' AND amount_nanos >= 0)
+  ),
+  reason TEXT CHECK (
+    reason IS NULL OR reason IN (
+      'missingBillingIdentity',
+      'missingCatalogEntry',
+      'missingUsageMeter',
+      'unsupportedPricingDimension',
+      'invalidUsageForFormula'
+    )
+  ),
+  details TEXT NOT NULL CHECK (json_valid(details) AND json_type(details) = 'object'),
+  rated_at TEXT NOT NULL CHECK (length(trim(rated_at)) > 0),
+  PRIMARY KEY (project_id, usage_record_id),
+  FOREIGN KEY (project_id, usage_record_id)
+    REFERENCES ai_model_call_usage(project_id, id)
+    ON DELETE RESTRICT,
+  CHECK (
+    (
+      status = 'rated'
+      AND provider_id IS NOT NULL
+      AND model_id IS NOT NULL
+      AND currency IS NOT NULL
+      AND amount_nanos IS NOT NULL
+      AND reason IS NULL
+    )
+    OR
+    (
+      status = 'unpriceable'
+      AND currency IS NULL
+      AND amount_nanos IS NULL
+      AND reason IS NOT NULL
+      AND (
+        (reason = 'missingBillingIdentity' AND provider_id IS NULL AND model_id IS NULL)
+        OR
+        (reason <> 'missingBillingIdentity' AND provider_id IS NOT NULL AND model_id IS NOT NULL)
+      )
+    )
+  )
+);
+
+CREATE INDEX idx_ai_model_call_usage_provider_model_time
+  ON ai_model_call_usage (project_id, provider_id, requested_model_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_usage_model_time
+  ON ai_model_call_usage (project_id, requested_model_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_valuations_summary
+  ON ai_model_call_valuations (project_id, status, currency, usage_record_id);

package/src/migrations/027-agent-context-checkpoints.sql

@@ -0,0 +1,28 @@
+CREATE TABLE agent_context_checkpoints (
+  project_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  thread_id TEXT NOT NULL,
+  created_by_run_id TEXT NOT NULL,
+  previous_checkpoint_id TEXT,
+  reason TEXT NOT NULL CHECK (reason IN ('threshold', 'overflow')),
+  summary TEXT NOT NULL CHECK (length(trim(summary)) > 0),
+  summary_format_version INTEGER NOT NULL CHECK (summary_format_version = 1),
+  summarized_through_seq INTEGER NOT NULL CHECK (summarized_through_seq >= 1),
+  observed_head_seq INTEGER NOT NULL,
+  estimated_input_tokens_before INTEGER NOT NULL CHECK (estimated_input_tokens_before >= 0),
+  estimated_input_tokens_after INTEGER NOT NULL CHECK (estimated_input_tokens_after >= 0),
+  summary_model_id TEXT NOT NULL CHECK (length(trim(summary_model_id)) > 0),
+  created_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, id),
+  UNIQUE (project_id, created_by_run_id),
+  FOREIGN KEY (project_id, thread_id)
+    REFERENCES agent_threads(project_id, id) ON DELETE CASCADE,
+  FOREIGN KEY (project_id, created_by_run_id)
+    REFERENCES agent_runs(project_id, id) ON DELETE RESTRICT,
+  FOREIGN KEY (project_id, previous_checkpoint_id)
+    REFERENCES agent_context_checkpoints(project_id, id) ON DELETE RESTRICT,
+  CHECK (summarized_through_seq < observed_head_seq)
+);
+
+CREATE INDEX idx_agent_context_checkpoints_thread_latest
+  ON agent_context_checkpoints(project_id, thread_id, summarized_through_seq DESC, created_at DESC, id DESC);

package/src/migrations/028-object-override-edit-times.sql

@@ -0,0 +1,3 @@
+ALTER TABLE ontology_object_overrides
+ADD COLUMN edited_at TEXT NOT NULL DEFAULT '{}'
+CHECK (json_valid(edited_at) AND json_type(edited_at) = 'object');

package/src/migrations.ts

@@ -69,6 +69,13 @@ import ontologyCommitExecutionsSql from "./migrations/024-ontology-commit-execut
 import connectorConnectionsSql from "./migrations/025-connector-connections.sql" with {
   type: "text",
 }
+import aiCostAccountingSql from "./migrations/026-ai-cost-accounting.sql" with { type: "text" }
+import agentContextCheckpointsSql from "./migrations/027-agent-context-checkpoints.sql" with {
+  type: "text",
+}
+import objectOverrideEditTimesSql from "./migrations/028-object-override-edit-times.sql" with {
+  type: "text",
+}
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -149,6 +156,9 @@ export const sqliteStorageMigrations = defineMigrations({
       { checksum: checksum(ontologyCommitExecutionsSql) }
     ),
     sqliteSql("025-connector-connections", connectorConnectionsSql),
+    sqliteSql("026-ai-cost-accounting", aiCostAccountingSql),
+    sqliteSql("027-agent-context-checkpoints", agentContextCheckpointsSql),
+    sqliteSql("028-object-override-edit-times", objectOverrideEditTimesSql),
   ],
 })
 

package/src/object-query-compiler.ts

@@ -98,6 +98,8 @@ function compileObjectQueryInternal(
   switch (query.kind) {
     case "start":
       return compileStart(projectId, query)
+    case "refs":
+      return compileRefs(projectId, query)
     case "filter":
       return compileFilter(projectId, query.input, query.predicate)
     case "sort":
@@ -162,6 +164,54 @@ function compileStart(
   }
 }
 
+function compileRefs(
+  projectId: string,
+  query: Extract<ObjectQuery, { kind: "refs" }>
+): CompiledObjectQuery {
+  if (query.refs.length === 0) {
+    throw new Error("[Sixb] SQLite object storage requires at least one ref")
+  }
+
+  const order = compileOrder(identityOrderFields())
+  const selectedOrder = compileOrder(identityOrderFields(), "selected")
+  const refsJson = JSON.stringify(query.refs)
+  const requested = `
+    SELECT DISTINCT
+      json_extract(ref.value, '$.objectTypeId') AS object_type_id,
+      json_extract(ref.value, '$.primaryId') AS primary_id
+    FROM json_each(?) AS ref
+  `
+  const sql = `
+    WITH requested AS (${requested})
+    SELECT selected.*, selected.properties AS _cursor_properties
+    FROM requested
+    JOIN objects AS selected
+      ON selected.project_id = ?
+     AND selected.object_type_id = requested.object_type_id
+     AND selected.primary_id = requested.primary_id
+    ORDER BY ${selectedOrder.sql}
+  `
+
+  return {
+    sql,
+    args: [refsJson, projectId, ...selectedOrder.args],
+    totalSql: `
+      WITH requested AS (${requested})
+      SELECT COUNT(*) AS total
+      FROM requested
+      JOIN objects AS selected
+        ON selected.project_id = ?
+       AND selected.object_type_id = requested.object_type_id
+       AND selected.primary_id = requested.primary_id
+    `,
+    totalArgs: [refsJson, projectId],
+    order,
+    hasMore: () => false,
+    trimRows: identityRows,
+    nextPageToken: () => undefined,
+  }
+}
+
 function compileFilter(
   projectId: string,
   inputQuery: ObjectQuery,

package/src/object-storage.ts

@@ -9,16 +9,21 @@ import type {
   ExpandedObjectRow,
   FacetObjectsInput,
   FacetObjectsResult,
+  LinkBatchKey,
   LinkDirection,
+  ObjectBatchKey,
   ObjectFacetRequest,
   ObjectLinkRow,
   ObjectQueryCapabilities,
   ObjectRow,
   ObjectRowLinks,
   ObjectStorage,
+  QueryObjectLinksInput,
+  QueryObjectLinksResult,
   QueryObjectsInput,
   QueryObjectsResult,
 } from "@sixb/core/storage"
+import { linkBatchKey, objectBatchKey } from "@sixb/core/storage"
 import { installFreshSqliteSchema } from "./migrations"
 import { type CompiledObjectQuery, compileObjectQuery } from "./object-query-compiler"
 import {
@@ -41,6 +46,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
   facetObjects: true,
   nodes: {
     start: true,
+    refs: true,
     filter: true,
     text: true,
     sort: true,
@@ -92,7 +98,7 @@ const SQLITE_OBJECT_QUERY_CAPABILITIES: ObjectQueryCapabilities = {
     stablePageTokens: true,
   },
   notes: [
-    "SQLite object query pushdown supports start/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
+    "SQLite object query pushdown supports start/refs/filter/text/sort/limit/page/traverse/set/project/expand over JSON properties and object links.",
     "expand hydrates linked objects in-database (top-N per parent via row_number() + json_group_array); core resolves each expansion's cardinality before pushdown, and a mixed/unresolved one stays on the fallback.",
     "Ordered decimal predicates and sorting use the bounded core fallback because SQLite has no native exact decimal type; canonical decimal equality remains pushdown-safe.",
     "Relevance sorting, vector search, and unresolved start.includeSubtypes remain planner fallback or rejection cases.",
@@ -219,22 +225,24 @@ export class SqliteObjectStorage implements ObjectStorage {
   async getByPrimaryIdBatch(params: {
     projectId: string
     items: readonly { objectTypeId: string; primaryId: string }[]
-  }): Promise<Map<string, ObjectRow>> {
-    const result = new Map<string, ObjectRow>()
+  }): Promise<Map<ObjectBatchKey, ObjectRow>> {
+    const result = new Map<ObjectBatchKey, ObjectRow>()
     if (params.items.length === 0) return result
 
-    const stmt = this.db.query(
-      "SELECT * FROM objects WHERE project_id = ? AND object_type_id = ? AND primary_id = ?"
-    )
-    for (const item of params.items) {
-      const row = stmt.get(
-        params.projectId,
-        item.objectTypeId,
-        item.primaryId
-      ) as DatabaseRow | null
-      if (row) {
-        result.set(`${item.objectTypeId}:${item.primaryId}`, this.rowToObject(row))
-      }
+    const rows = this.db
+      .query(
+        `
+          SELECT object.*
+          FROM json_each(?) AS requested
+          JOIN objects AS object
+            ON object.project_id = ?
+           AND object.object_type_id = json_extract(requested.value, '$.objectTypeId')
+           AND object.primary_id = json_extract(requested.value, '$.primaryId')
+        `
+      )
+      .all(JSON.stringify(params.items), params.projectId) as DatabaseRow[]
+    for (const row of rows) {
+      result.set(objectBatchKey(row.object_type_id, row.primary_id), this.rowToObject(row))
     }
     return result
   }
@@ -242,8 +250,8 @@ export class SqliteObjectStorage implements ObjectStorage {
   async listLinksBatch(params: {
     projectId: string
     items: readonly { objectTypeId: string; objectId: string; linkId: string }[]
-  }): Promise<Map<string, ObjectLinkRow[]>> {
-    const result = new Map<string, ObjectLinkRow[]>()
+  }): Promise<Map<LinkBatchKey, ObjectLinkRow[]>> {
+    const result = new Map<LinkBatchKey, ObjectLinkRow[]>()
     if (params.items.length === 0) return result
 
     const stmt = this.db.query(
@@ -258,7 +266,7 @@ export class SqliteObjectStorage implements ObjectStorage {
       ) as LinkDatabaseRow[]
       if (rows.length > 0) {
         result.set(
-          `${item.objectTypeId}:${item.objectId}:${item.linkId}`,
+          linkBatchKey(item.objectTypeId, item.objectId, item.linkId),
           rows.map((r) => this.rowToLink(r))
         )
       }
@@ -266,26 +274,116 @@ export class SqliteObjectStorage implements ObjectStorage {
     return result
   }
 
+  async queryLinks(params: QueryObjectLinksInput): Promise<QueryObjectLinksResult> {
+    assertLinkQueryLimit(params.limit)
+    if (params.objectRefs.length === 0 || params.endpointObjectTypeIds?.length === 0) {
+      return { links: [], hasMore: false }
+    }
+
+    const sourceJoin = `
+      SELECT link.*
+      FROM links AS link
+      JOIN requested
+        ON requested.object_type_id = link.source_type_id
+       AND requested.object_id = link.source_id
+      WHERE link.project_id = ?
+    `
+    const targetJoin = `
+      SELECT link.*
+      FROM links AS link
+      JOIN requested
+        ON requested.object_type_id = link.target_type_id
+       AND requested.object_id = link.target_id
+      WHERE link.project_id = ?
+    `
+    const incidentSql =
+      params.direction === "outgoing"
+        ? sourceJoin
+        : params.direction === "incoming"
+          ? targetJoin
+          : `${sourceJoin} UNION ${targetJoin}`
+    const args: (string | number)[] = [JSON.stringify(params.objectRefs), params.projectId]
+    if (params.direction === "both") args.push(params.projectId)
+
+    const predicates: string[] = []
+    if (params.linkId !== undefined) {
+      predicates.push("link_id = ?")
+      args.push(params.linkId)
+    }
+    if (params.endpointObjectTypeIds !== undefined) {
+      const allowedTypes = JSON.stringify([...new Set(params.endpointObjectTypeIds)])
+      predicates.push(
+        "source_type_id IN (SELECT value FROM json_each(?))",
+        "target_type_id IN (SELECT value FROM json_each(?))"
+      )
+      args.push(allowedTypes, allowedTypes)
+    }
+    if (params.after) {
+      predicates.push(
+        "(source_type_id, source_id, link_id, target_type_id, target_id) > (?, ?, ?, ?, ?)"
+      )
+      args.push(...params.after)
+    }
+    args.push(params.limit + 1)
+
+    const whereSql = predicates.length > 0 ? `WHERE ${predicates.join(" AND ")}` : ""
+    const rows = this.db
+      .query(
+        `
+          WITH requested AS (
+            SELECT DISTINCT
+              json_extract(value, '$.objectTypeId') AS object_type_id,
+              json_extract(value, '$.primaryId') AS object_id
+            FROM json_each(?)
+          ), incident AS (${incidentSql})
+          SELECT *
+          FROM incident
+          ${whereSql}
+          ORDER BY source_type_id, source_id, link_id, target_type_id, target_id
+          LIMIT ?
+        `
+      )
+      .all(...args) as LinkDatabaseRow[]
+
+    return {
+      links: rows.slice(0, params.limit).map((row) => this.rowToLink(row)),
+      hasMore: rows.length > params.limit,
+    }
+  }
+
   async listIncidentLinksBatch(params: {
     projectId: string
     items: readonly { objectTypeId: string; objectId: string }[]
   }): Promise<readonly ObjectLinkRow[]> {
-    const deduped = new Map<string, ObjectLinkRow>()
-    for (const item of params.items) {
-      const rows = await this.listLinks({
-        projectId: params.projectId,
-        objectTypeId: item.objectTypeId,
-        objectId: item.objectId,
-        direction: "both",
-      })
-      for (const row of rows) {
-        deduped.set(
-          `${row.sourceTypeId}:${row.sourceId}:${row.linkId}:${row.targetTypeId}:${row.targetId}`,
-          row
-        )
-      }
-    }
-    return [...deduped.values()]
+    if (params.items.length === 0) return []
+
+    const rows = this.db
+      .query(
+        `
+          WITH requested AS (
+            SELECT
+              json_extract(value, '$.objectTypeId') AS object_type_id,
+              json_extract(value, '$.objectId') AS object_id
+            FROM json_each(?)
+          )
+          SELECT link.*
+          FROM links AS link
+          JOIN requested
+            ON requested.object_type_id = link.source_type_id
+           AND requested.object_id = link.source_id
+          WHERE link.project_id = ?
+          UNION
+          SELECT link.*
+          FROM links AS link
+          JOIN requested
+            ON requested.object_type_id = link.target_type_id
+           AND requested.object_id = link.target_id
+          WHERE link.project_id = ?
+        `
+      )
+      .all(JSON.stringify(params.items), params.projectId, params.projectId) as LinkDatabaseRow[]
+
+    return rows.map((row) => this.rowToLink(row))
   }
 
   async listByPrimaryIdPage(params: {
@@ -457,6 +555,12 @@ function assertReconciliationPageLimit(limit: number): void {
   }
 }
 
+function assertLinkQueryLimit(limit: number): void {
+  if (!Number.isSafeInteger(limit) || limit <= 0) {
+    throw new Error("Object link query limit must be a positive safe integer.")
+  }
+}
+
 function readTotal(db: Database, compiled: CompiledObjectQuery): number {
   const row = db.query(compiled.totalSql).get(...compiled.totalArgs) as { total: number }
   return row.total

package/src/ontology-storage/materialization-state.ts

@@ -86,6 +86,7 @@ interface LinkScopeRow extends EffectiveLinkRow {
 interface ObjectOverrideRow extends SqliteStoredOverrideRow {
   readonly object_type_id: string
   readonly primary_id: string
+  readonly edited_at: string
 }
 
 interface LinkOverrideRow extends SqliteStoredOverrideRow {
@@ -1028,6 +1029,7 @@ function storedObjectOverride(row: ObjectOverrideRow): StoredObjectOverride {
   return {
     ref,
     value: parseJson<StoredObjectOverride["value"]>(row.value),
+    editedAt: parseJson<Readonly<Record<string, string>>>(row.edited_at),
     lastCommitId: row.last_commit_id,
     updatedAt: row.updated_at,
   }

package/src/ontology-storage/materialization-writer.ts

@@ -41,14 +41,16 @@ export class SqliteMaterializationWriter {
           this.db
             .query(
               `INSERT INTO ontology_object_overrides (
-                 project_id, object_type_id, primary_id, value, last_commit_id, updated_at
-               ) VALUES (?, ?, ?, json(?), ?, ?)`
+                 project_id, object_type_id, primary_id, value, edited_at,
+                 last_commit_id, updated_at
+               ) VALUES (?, ?, ?, json(?), json(?), ?, ?)`
             )
             .run(
               projectId,
               item.ref.objectTypeId,
               item.ref.primaryId,
               canonicalJson(item.value),
+              canonicalJson(item.editedAt),
               item.lastCommitId,
               item.updatedAt
             )
@@ -64,12 +66,13 @@ export class SqliteMaterializationWriter {
         this.db
           .query(
             `UPDATE ontology_object_overrides
-             SET value = json(?), last_commit_id = ?, updated_at = ?
+             SET value = json(?), edited_at = json(?), last_commit_id = ?, updated_at = ?
              WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
                AND last_commit_id = ?`
           )
           .run(
             canonicalJson(item.value),
+            canonicalJson(item.editedAt),
             item.lastCommitId,
             item.updatedAt,
             projectId,