@sixb/sqlite 0.1.2 → 0.1.3

package/src/index.ts

@@ -19,6 +19,7 @@ import {
   createWorkflowRunOperationScope,
 } from "@sixb/core/internal/storage-operation-scope"
 import {
+  type AiCostStorage,
   type AiUsageStorage,
   createTransactionStorageProxy,
   StorageTransactionError,
@@ -26,6 +27,7 @@ import {
 } from "@sixb/core/storage"
 import { SqliteActionRunStorage } from "./action-run-storage"
 import { SqliteAgentStorage } from "./agents"
+import { SqliteAiCostStorage } from "./ai-cost-storage"
 import { SqliteAiUsageStorage } from "./ai-usage-storage"
 import { SqliteAuthStorage } from "./auth-storage"
 import { SqliteConnectorConnectionStorage } from "./connector-connection-storage"
@@ -61,8 +63,8 @@ export interface SqliteStorageOptions {
 /**
  * SQLite storage provider for Sixb.
  *
- * Bundles object, timeseries, auth, execution, AI usage, sync run, pipeline run, projection run,
- * workflow run, webhook run, and connector connection storage backed by SQLite.
+ * Bundles object, timeseries, auth, execution, AI accounting, sync run, pipeline run, projection
+ * run, workflow run, webhook run, and connector connection storage backed by SQLite.
  *
  * Usage:
  * ```ts
@@ -81,6 +83,7 @@ export class SqliteStorage implements MigrationCapableStorage {
   readonly executions: SqliteExecutionStorage
   readonly agents: SqliteAgentStorage
   readonly aiUsage: AiUsageStorage
+  readonly aiCosts: AiCostStorage
   readonly actionRuns: SqliteActionRunStorage
   readonly pipelineRuns: SqlitePipelineRunStorage
   readonly syncRuns: SqliteSyncRunStorage
@@ -143,6 +146,7 @@ export class SqliteStorage implements MigrationCapableStorage {
     this.executions = createOperationScopedFacade(stores.executions, scope)
     this.agents = createAgentOperationScope(stores.agents, scope)
     this.aiUsage = createOperationScopedFacade(stores.aiUsage, scope)
+    this.aiCosts = createOperationScopedFacade(stores.aiCosts, scope)
     this.actionRuns = createOperationScopedFacade(stores.actionRuns, scope)
     this.pipelineRuns = createOperationScopedFacade(stores.pipelineRuns, scope)
     this.timeseries = createOperationScopedFacade(readTimeseries, readScope)
@@ -298,6 +302,7 @@ function createSqliteStores(
     executions,
     agents: new SqliteAgentStorage({ connection, executions }),
     aiUsage: new SqliteAiUsageStorage({ connection }),
+    aiCosts: new SqliteAiCostStorage(connection),
     actionRuns: new SqliteActionRunStorage({ connection, executions }),
     pipelineRuns: new SqlitePipelineRunStorage({ connection, executions }),
     timeseries: new SqliteTimeseriesStorage({ connection }),
@@ -318,6 +323,7 @@ interface SqliteStoreSet {
   readonly executions: SqliteExecutionStorage
   readonly agents: SqliteAgentStorage
   readonly aiUsage: SqliteAiUsageStorage
+  readonly aiCosts: SqliteAiCostStorage
   readonly actionRuns: SqliteActionRunStorage
   readonly pipelineRuns: SqlitePipelineRunStorage
   readonly syncRuns: SqliteSyncRunStorage

package/src/migrations/026-ai-cost-accounting.sql

@@ -0,0 +1,57 @@
+CREATE TABLE ai_model_call_valuations (
+  project_id TEXT NOT NULL CHECK (length(trim(project_id)) > 0),
+  usage_record_id TEXT NOT NULL CHECK (length(trim(usage_record_id)) > 0),
+  status TEXT NOT NULL CHECK (status IN ('rated', 'unpriceable')),
+  provider_id TEXT CHECK (provider_id IS NULL OR length(trim(provider_id)) > 0),
+  model_id TEXT CHECK (model_id IS NULL OR length(trim(model_id)) > 0),
+  currency TEXT CHECK (
+    currency IS NULL OR (length(currency) = 3 AND currency GLOB '[A-Z][A-Z][A-Z]')
+  ),
+  amount_nanos INTEGER CHECK (
+    amount_nanos IS NULL OR (typeof(amount_nanos) = 'integer' AND amount_nanos >= 0)
+  ),
+  reason TEXT CHECK (
+    reason IS NULL OR reason IN (
+      'missingBillingIdentity',
+      'missingCatalogEntry',
+      'missingUsageMeter',
+      'unsupportedPricingDimension',
+      'invalidUsageForFormula'
+    )
+  ),
+  details TEXT NOT NULL CHECK (json_valid(details) AND json_type(details) = 'object'),
+  rated_at TEXT NOT NULL CHECK (length(trim(rated_at)) > 0),
+  PRIMARY KEY (project_id, usage_record_id),
+  FOREIGN KEY (project_id, usage_record_id)
+    REFERENCES ai_model_call_usage(project_id, id)
+    ON DELETE RESTRICT,
+  CHECK (
+    (
+      status = 'rated'
+      AND provider_id IS NOT NULL
+      AND model_id IS NOT NULL
+      AND currency IS NOT NULL
+      AND amount_nanos IS NOT NULL
+      AND reason IS NULL
+    )
+    OR
+    (
+      status = 'unpriceable'
+      AND currency IS NULL
+      AND amount_nanos IS NULL
+      AND reason IS NOT NULL
+      AND (
+        (reason = 'missingBillingIdentity' AND provider_id IS NULL AND model_id IS NULL)
+        OR
+        (reason <> 'missingBillingIdentity' AND provider_id IS NOT NULL AND model_id IS NOT NULL)
+      )
+    )
+  )
+);
+
+CREATE INDEX idx_ai_model_call_usage_provider_model_time
+  ON ai_model_call_usage (project_id, provider_id, requested_model_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_usage_model_time
+  ON ai_model_call_usage (project_id, requested_model_id, occurred_at, id);
+CREATE INDEX idx_ai_model_call_valuations_summary
+  ON ai_model_call_valuations (project_id, status, currency, usage_record_id);

package/src/migrations/027-agent-context-checkpoints.sql

@@ -0,0 +1,28 @@
+CREATE TABLE agent_context_checkpoints (
+  project_id TEXT NOT NULL,
+  id TEXT NOT NULL,
+  thread_id TEXT NOT NULL,
+  created_by_run_id TEXT NOT NULL,
+  previous_checkpoint_id TEXT,
+  reason TEXT NOT NULL CHECK (reason IN ('threshold', 'overflow')),
+  summary TEXT NOT NULL CHECK (length(trim(summary)) > 0),
+  summary_format_version INTEGER NOT NULL CHECK (summary_format_version = 1),
+  summarized_through_seq INTEGER NOT NULL CHECK (summarized_through_seq >= 1),
+  observed_head_seq INTEGER NOT NULL,
+  estimated_input_tokens_before INTEGER NOT NULL CHECK (estimated_input_tokens_before >= 0),
+  estimated_input_tokens_after INTEGER NOT NULL CHECK (estimated_input_tokens_after >= 0),
+  summary_model_id TEXT NOT NULL CHECK (length(trim(summary_model_id)) > 0),
+  created_at TEXT NOT NULL,
+  PRIMARY KEY (project_id, id),
+  UNIQUE (project_id, created_by_run_id),
+  FOREIGN KEY (project_id, thread_id)
+    REFERENCES agent_threads(project_id, id) ON DELETE CASCADE,
+  FOREIGN KEY (project_id, created_by_run_id)
+    REFERENCES agent_runs(project_id, id) ON DELETE RESTRICT,
+  FOREIGN KEY (project_id, previous_checkpoint_id)
+    REFERENCES agent_context_checkpoints(project_id, id) ON DELETE RESTRICT,
+  CHECK (summarized_through_seq < observed_head_seq)
+);
+
+CREATE INDEX idx_agent_context_checkpoints_thread_latest
+  ON agent_context_checkpoints(project_id, thread_id, summarized_through_seq DESC, created_at DESC, id DESC);

package/src/migrations/028-object-override-edit-times.sql

@@ -0,0 +1,3 @@
+ALTER TABLE ontology_object_overrides
+ADD COLUMN edited_at TEXT NOT NULL DEFAULT '{}'
+CHECK (json_valid(edited_at) AND json_type(edited_at) = 'object');

package/src/migrations.ts

@@ -69,6 +69,13 @@ import ontologyCommitExecutionsSql from "./migrations/024-ontology-commit-execut
 import connectorConnectionsSql from "./migrations/025-connector-connections.sql" with {
   type: "text",
 }
+import aiCostAccountingSql from "./migrations/026-ai-cost-accounting.sql" with { type: "text" }
+import agentContextCheckpointsSql from "./migrations/027-agent-context-checkpoints.sql" with {
+  type: "text",
+}
+import objectOverrideEditTimesSql from "./migrations/028-object-override-edit-times.sql" with {
+  type: "text",
+}
 
 const MIGRATIONS_TABLE_SQL = `
   CREATE TABLE IF NOT EXISTS sixb_migrations (
@@ -149,6 +156,9 @@ export const sqliteStorageMigrations = defineMigrations({
       { checksum: checksum(ontologyCommitExecutionsSql) }
     ),
     sqliteSql("025-connector-connections", connectorConnectionsSql),
+    sqliteSql("026-ai-cost-accounting", aiCostAccountingSql),
+    sqliteSql("027-agent-context-checkpoints", agentContextCheckpointsSql),
+    sqliteSql("028-object-override-edit-times", objectOverrideEditTimesSql),
   ],
 })
 

package/src/ontology-storage/materialization-state.ts

@@ -86,6 +86,7 @@ interface LinkScopeRow extends EffectiveLinkRow {
 interface ObjectOverrideRow extends SqliteStoredOverrideRow {
   readonly object_type_id: string
   readonly primary_id: string
+  readonly edited_at: string
 }
 
 interface LinkOverrideRow extends SqliteStoredOverrideRow {
@@ -1028,6 +1029,7 @@ function storedObjectOverride(row: ObjectOverrideRow): StoredObjectOverride {
   return {
     ref,
     value: parseJson<StoredObjectOverride["value"]>(row.value),
+    editedAt: parseJson<Readonly<Record<string, string>>>(row.edited_at),
     lastCommitId: row.last_commit_id,
     updatedAt: row.updated_at,
   }

package/src/ontology-storage/materialization-writer.ts

@@ -41,14 +41,16 @@ export class SqliteMaterializationWriter {
           this.db
             .query(
               `INSERT INTO ontology_object_overrides (
-                 project_id, object_type_id, primary_id, value, last_commit_id, updated_at
-               ) VALUES (?, ?, ?, json(?), ?, ?)`
+                 project_id, object_type_id, primary_id, value, edited_at,
+                 last_commit_id, updated_at
+               ) VALUES (?, ?, ?, json(?), json(?), ?, ?)`
             )
             .run(
               projectId,
               item.ref.objectTypeId,
               item.ref.primaryId,
               canonicalJson(item.value),
+              canonicalJson(item.editedAt),
               item.lastCommitId,
               item.updatedAt
             )
@@ -64,12 +66,13 @@ export class SqliteMaterializationWriter {
         this.db
           .query(
             `UPDATE ontology_object_overrides
-             SET value = json(?), last_commit_id = ?, updated_at = ?
+             SET value = json(?), edited_at = json(?), last_commit_id = ?, updated_at = ?
              WHERE project_id = ? AND object_type_id = ? AND primary_id = ?
                AND last_commit_id = ?`
           )
           .run(
             canonicalJson(item.value),
+            canonicalJson(item.editedAt),
             item.lastCommitId,
             item.updatedAt,
             projectId,