or3-provider-sqlite 0.0.13 → 0.0.14

package/README.md

@@ -227,6 +227,13 @@ bun run type-check  # TypeScript validation
 bun run build       # build for distribution
 ```
 
+The normal test suite runs in a provider-only checkout using hash-verified,
+byte-for-byte host runtime fixtures. `bun run test:host-integration` reruns the
+same background-job, canonical-history and registration cases against a prepared
+sibling `../or3-chat` at the exact reviewed source tree. It fails on missing or
+mismatched source rather than falling back to fixtures. See the
+[fixture provenance and prerequisites](test/fixtures/host/README.md).
+
 ## Compatibility
 
 - Works with multiple auth providers (`basic-auth`, `clerk`, or custom)
@@ -274,6 +281,14 @@ or failed builds fall back to installed packages with a warning.
 `OR3_LOCAL_PROVIDERS=false` disables local selection. Production builds use the
 installed package, so local development does not publish these changes.
 
+### Canonical compaction history reader (unreleased)
+
+`canonicalChatHistory: 'v1'` and `readChatHistory` read current `s_threads`/`s_messages`, with explicit workspace membership (owner/editor/viewer), bounded by-ID queries and canonical `(index, order_key, id)` keyset pages. Migration 022 adds transactional per-thread revision triggers. Migration 023 matches the index to legacy missing-order-key normalization and preserves the existing orphan-row cleanup lifecycle; revision bookkeeping does not add a foreign key to otherwise independent sync rows. Current-chat tool result writes do not invalidate ancestor search cursors. The reader never consults retained sync logs. D1 omits the capability.
+
+The additive local qualification command is `bun run build` followed by `bunx vitest run --config vitest.canonical-history.config.ts`. It checks committed current host contract bytes without replacing the older pinned fixture. The current-host lane passes 119 cases across three files, including the real gateway owner and source-built artifact reads after change-log deletion, legacy tie paging in both directions, bounded queries, revision changes, cancellation, revoked membership and an index plan without a temporary sort. These checks do not qualify a deployed rollout, second-client/background reconciliation or released pins. This compaction capability is not yet included in a published package.
+
+The canonical-history qualification lane defaults to a sibling `../or3-chat` checkout. For an isolated host worktree, set `OR3_CANONICAL_HOST_ROOT=/absolute/path/to/or3-chat` before the command. Install the host dependencies too; the two-client fixture resolves its IndexedDB test dependency from that checkout. Its contract-byte checks fail closed on uncommitted changes; this does not replace the pinned host fixture or certify deployed/released provider behavior.
+
 ## Workspace Files and Trash admission
 
 Native synchronous adapters advertise workspace-item capability v1. Canonical

package/dist/module.json

@@ -1,7 +1,7 @@
 {
   "name": "or3-provider-sqlite",
   "configKey": "or3-provider-sqlite",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "builder": {
     "@nuxt/module-builder": "1.0.2",
     "unbuild": "3.6.1"

package/dist/runtime/server/background-jobs/sqlite-provider.js

@@ -1,4 +1,5 @@
 import { getJobConfig } from "~~/server/utils/background-jobs/store";
+import { readRequestUsage } from "~~/shared/chat/compaction";
 import { randomUUID } from "node:crypto";
 import { d1All, d1Run } from "../db/d1.js";
 import { getRawDb, isD1Driver } from "../db/kysely.js";
@@ -9,6 +10,13 @@ function parseJson(value) {
 function json(value) {
   return value === void 0 ? null : JSON.stringify(value);
 }
+function readStoredUsage(value) {
+  try {
+    return readRequestUsage(parseJson(value));
+  } catch {
+    return void 0;
+  }
+}
 function toJob(row) {
   return {
     id: row.id,
@@ -29,6 +37,7 @@ function toJob(row) {
     completedAt: row.completed_at ?? void 0,
     error: row.error ?? void 0,
     tool_calls: parseJson(row.tool_calls_json),
+    usage: readStoredUsage(row.usage_json),
     workflow_state: parseJson(
       row.workflow_state_json
     ),
@@ -326,12 +335,19 @@ export class SqliteBackgroundJobProvider {
       sets.push("workflow_state_json = ?");
       values.push(json(update.workflow_state));
     }
+    const usage = readRequestUsage(update.usage);
+    if (usage) {
+      sets.push("usage_json = ?");
+      values.push(json(usage));
+    }
     values.push(jobId);
     let where = "id = ? AND status = 'streaming'";
     if (update.leaseOwner) {
       where += " AND lease_owner = ? AND lease_expires_at > ?";
       values.push(update.leaseOwner);
       values.push(Date.now());
+    } else {
+      where += " AND lease_owner IS NULL";
     }
     const changes = await run(
       `UPDATE background_jobs SET ${sets.join(", ")} WHERE ${where}`,
@@ -353,8 +369,10 @@ export class SqliteBackgroundJobProvider {
     values.push(jobId);
     let where = "id = ? AND status = 'streaming'";
     if (leaseOwner) {
-      where += " AND lease_owner = ?";
-      values.push(leaseOwner);
+      where += " AND lease_owner = ? AND lease_expires_at > ?";
+      values.push(leaseOwner, now);
+    } else {
+      where += " AND lease_owner IS NULL";
     }
     const changes = await run(
       `UPDATE background_jobs SET status = ?, completed_at = ?, last_activity_at = ?,
@@ -403,6 +421,11 @@ export class SqliteBackgroundJobProvider {
                 reasoning = CASE WHEN attempts > 0
                     THEN COALESCE(json_extract(execution_json, '$.reasoningBase'), '')
                     ELSE reasoning END,
+                usage_json = CASE WHEN attempts > 0
+                    THEN CASE WHEN json_type(execution_json, '$.normalizedToolState.requestUsage') = 'object'
+                        THEN json_extract(execution_json, '$.normalizedToolState.requestUsage')
+                        ELSE NULL END
+                    ELSE usage_json END,
                 chunks_received = CASE WHEN attempts > 0 THEN 0 ELSE chunks_received END,
                 attempts = attempts + 1
              WHERE id = ? AND status = 'streaming' AND execution_json IS NOT NULL
@@ -430,6 +453,11 @@ export class SqliteBackgroundJobProvider {
                 reasoning = CASE WHEN attempts > 0
                     THEN COALESCE(json_extract(execution_json, '$.reasoningBase'), '')
                     ELSE reasoning END,
+                usage_json = CASE WHEN attempts > 0
+                    THEN CASE WHEN json_type(execution_json, '$.normalizedToolState.requestUsage') = 'object'
+                        THEN json_extract(execution_json, '$.normalizedToolState.requestUsage')
+                        ELSE NULL END
+                    ELSE usage_json END,
                 chunks_received = CASE WHEN attempts > 0 THEN 0 ELSE chunks_received END,
                 attempts = attempts + 1
              WHERE id = (
@@ -462,11 +490,13 @@ export class SqliteBackgroundJobProvider {
   async updateJobExecution(jobId, execution, leaseOwner) {
     return await run(
       `UPDATE background_jobs SET execution_json = ?, last_activity_at = ?
-             WHERE id = ? AND status = 'streaming' AND lease_owner = ?`,
+             WHERE id = ? AND status = 'streaming' AND lease_owner = ?
+               AND lease_expires_at > ?`,
       json(execution),
       Date.now(),
       jobId,
-      leaseOwner
+      leaseOwner,
+      Date.now()
     ) > 0;
   }
   async claimClientToolCall(jobId, userId, callId, claimToken, claimExpiresAt) {
@@ -521,6 +551,7 @@ export class SqliteBackgroundJobProvider {
       snapshot.content,
       snapshot.reasoning,
       json(snapshot.toolCalls),
+      json(readRequestUsage(snapshot.usage)),
       snapshot.error ?? null,
       snapshot.completedAt,
       snapshot.completedAt
@@ -530,10 +561,12 @@ export class SqliteBackgroundJobProvider {
     if (leaseOwner) {
       where += " AND lease_owner = ? AND lease_expires_at > ?";
       values.push(leaseOwner, Date.now());
+    } else {
+      where += " AND lease_owner IS NULL";
     }
     return await run(
       `UPDATE background_jobs SET status = ?, content = ?, reasoning = ?,
-                    tool_calls_json = ?, error = ?, completed_at = ?,
+                    tool_calls_json = ?, usage_json = COALESCE(?, usage_json), error = ?, completed_at = ?,
                     history_phase = 'finalization_pending',
                     last_activity_at = ?, lease_owner = NULL, lease_expires_at = NULL
                  WHERE ${where}`,

package/dist/runtime/server/db/migrate.js

@@ -19,6 +19,9 @@ import * as m017 from "./migrations/017_background_jobs.js";
 import * as m018 from "./migrations/018_background_admission_cancels.js";
 import * as m019 from "./migrations/019_background_generation_history.js";
 import * as m020 from "./migrations/020_background_generation_receipts.js";
+import * as m021 from "./migrations/021_background_request_usage.js";
+import * as m022 from "./migrations/022_canonical_chat_reader.js";
+import * as m023 from "./migrations/023_legacy_chat_seek_index.js";
 const REQUIRED_SCHEMA_TABLES = [
   "users",
   "auth_accounts",
@@ -68,7 +71,10 @@ const migrations = {
   "017_background_jobs": m017,
   "018_background_admission_cancels": m018,
   "019_background_generation_history": m019,
-  "020_background_generation_receipts": m020
+  "020_background_generation_receipts": m020,
+  "021_background_request_usage": m021,
+  "022_canonical_chat_reader": m022,
+  "023_legacy_chat_seek_index": m023
 };
 class StaticMigrationProvider {
   async getMigrations() {

package/dist/runtime/server/db/migrations/021_background_request_usage.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+/** Optional last measured request; existing jobs remain unmeasured. */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;

package/dist/runtime/server/db/migrations/021_background_request_usage.js

@@ -0,0 +1,7 @@
+import { sql } from "kysely";
+export async function up(db) {
+  await sql`ALTER TABLE background_jobs ADD COLUMN usage_json TEXT`.execute(db);
+}
+export async function down(db) {
+  await sql`ALTER TABLE background_jobs DROP COLUMN usage_json`.execute(db);
+}

package/dist/runtime/server/db/migrations/022_canonical_chat_reader.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+/** Workspace-scoped canonical message keysets, independent of retained logs. */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;

package/dist/runtime/server/db/migrations/022_canonical_chat_reader.js

@@ -0,0 +1,26 @@
+import { sql } from "kysely";
+export async function up(db) {
+  await sql`CREATE TABLE IF NOT EXISTS chat_history_revisions (
+        workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
+        thread_id TEXT NOT NULL, value INTEGER NOT NULL DEFAULT 0,
+        PRIMARY KEY (workspace_id, thread_id))`.execute(db);
+  await sql`CREATE INDEX IF NOT EXISTS s_messages_history_order ON s_messages
+        (workspace_id, json_extract(data_json, '$.thread_id'), json_extract(data_json, '$.index'), json_extract(data_json, '$.order_key'), id)`.execute(db);
+  for (const table of ["s_threads", "s_messages"]) for (const event of ["INSERT", "UPDATE", "DELETE"]) {
+    const rowNames = event === "UPDATE" ? ["OLD", "NEW"] : [event === "DELETE" ? "OLD" : "NEW"];
+    const statements = rowNames.map((row) => {
+      const thread = table === "s_threads" ? `${row}.id` : `json_extract(${row}.data_json, '$.thread_id')`;
+      return `INSERT INTO chat_history_revisions (workspace_id, thread_id, value)
+                SELECT ${row}.workspace_id, ${thread}, 1 WHERE ${thread} IS NOT NULL
+                ON CONFLICT (workspace_id, thread_id) DO UPDATE SET value = chat_history_revisions.value + 1;`;
+    }).join("\n");
+    await sql.raw(`CREATE TRIGGER IF NOT EXISTS ${table}_history_${event.toLowerCase()} AFTER ${event} ON ${table} BEGIN ${statements} END`).execute(db);
+  }
+}
+export async function down(db) {
+  for (const table of ["s_threads", "s_messages"]) for (const event of ["insert", "update", "delete"]) {
+    await sql.raw(`DROP TRIGGER IF EXISTS ${table}_history_${event}`).execute(db);
+  }
+  await sql`DROP INDEX IF EXISTS s_messages_history_order`.execute(db);
+  await sql`DROP TABLE IF EXISTS chat_history_revisions`.execute(db);
+}

package/dist/runtime/server/db/migrations/023_legacy_chat_seek_index.d.ts

@@ -0,0 +1,4 @@
+import { type Kysely } from 'kysely';
+/** Match the reader's legacy missing-order-key normalization exactly. */
+export declare function up(db: Kysely<unknown>): Promise<void>;
+export declare function down(db: Kysely<unknown>): Promise<void>;

package/dist/runtime/server/db/migrations/023_legacy_chat_seek_index.js

@@ -0,0 +1,24 @@
+import { sql } from "kysely";
+import { up as installCanonicalHistory } from "./022_canonical_chat_reader.js";
+export async function up(db) {
+  for (const table of ["s_threads", "s_messages"]) for (const event of ["insert", "update", "delete"]) {
+    await sql.raw(`DROP TRIGGER IF EXISTS ${table}_history_${event}`).execute(db);
+  }
+  await sql`CREATE TABLE chat_history_revisions_v2 (
+        workspace_id TEXT NOT NULL, thread_id TEXT NOT NULL, value INTEGER NOT NULL DEFAULT 0,
+        PRIMARY KEY (workspace_id, thread_id))`.execute(db);
+  await sql`INSERT INTO chat_history_revisions_v2 SELECT workspace_id, thread_id, value FROM chat_history_revisions`.execute(db);
+  await sql`DROP TABLE chat_history_revisions`.execute(db);
+  await sql`ALTER TABLE chat_history_revisions_v2 RENAME TO chat_history_revisions`.execute(db);
+  await installCanonicalHistory(db);
+  await sql`DROP INDEX IF EXISTS s_messages_history_order`.execute(db);
+  await sql`CREATE INDEX s_messages_history_order ON s_messages
+        (workspace_id, json_extract(data_json, '$.thread_id'), json_extract(data_json, '$.index'),
+        COALESCE(json_extract(data_json, '$.order_key'), ''), id)`.execute(db);
+}
+export async function down(db) {
+  await sql`DROP INDEX IF EXISTS s_messages_history_order`.execute(db);
+  await sql`CREATE INDEX s_messages_history_order ON s_messages
+        (workspace_id, json_extract(data_json, '$.thread_id'), json_extract(data_json, '$.index'),
+        json_extract(data_json, '$.order_key'), id)`.execute(db);
+}

package/dist/runtime/server/db/schema.d.ts

@@ -207,6 +207,7 @@ export interface BackgroundJobsTable {
     completed_at: number | null;
     error: string | null;
     tool_calls_json: string | null;
+    usage_json: string | null;
     workflow_state_json: string | null;
     execution_json: string | null;
     idempotency_key: string | null;

package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.d.ts

@@ -9,20 +9,24 @@ import type { H3Event } from 'h3';
 import type { CanonicalStorageQueryRequest, CanonicalStorageQueryResponse, SyncGatewayAdapter, SyncMaintenanceState, UploadIntentConsumptionRequest, UploadIntentReservationRequest } from '~~/server/sync/gateway/types';
 import type { AdmitChatGenerationResult, CanonicalGenerationSnapshot, CanonicalHistoryActor, ChatGenerationAdmissionEnvelope, FinalizeChatGenerationResult } from '~~/shared/chat/background-history';
 import type { PullRequest, PullResponse, PushBatch, PushResult, SnapshotRequest, SnapshotResponse } from '~~/shared/sync/types';
+import { type CanonicalChatQuery, type CanonicalChatReadResult } from '~~/shared/chat/history-reader';
 export declare class SqliteSyncGatewayAdapter implements SyncGatewayAdapter {
     id: string;
     get capabilities(): {
         readonly snapshotBootstrap: "snapshot-v1";
         readonly historyRetention: "snapshot-v1";
         readonly backgroundGenerationHistory?: undefined;
+        readonly canonicalChatHistory?: undefined;
         readonly workspaceItems?: undefined;
     } | {
         readonly snapshotBootstrap: "snapshot-v1";
         readonly historyRetention: "snapshot-v1";
         readonly backgroundGenerationHistory: "v1";
+        readonly canonicalChatHistory: "v1";
         readonly workspaceItems: "v1";
     };
     private get db();
+    readChatHistory(actor: CanonicalHistoryActor, query: CanonicalChatQuery, signal?: AbortSignal): Promise<CanonicalChatReadResult>;
     private assertCanonicalActor;
     admitChatGeneration(actor: CanonicalHistoryActor, input: ChatGenerationAdmissionEnvelope): Promise<AdmitChatGenerationResult>;
     finalizeChatGeneration(actor: CanonicalHistoryActor, input: {

package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.js

@@ -20,6 +20,8 @@ import { SYNCED_TABLE_MAP, ALLOWED_SYNC_TABLES } from "../db/schema.js";
 import { emitWebhookSystemHook } from "~~/server/utils/webhooks/runtime";
 import { incomingRevisionWins } from "~~/shared/sync/revision";
 import { sanitizePayloadForSync } from "~~/shared/sync/sanitize";
+import { readRequestUsage } from "~~/shared/chat/compaction";
+import { validateCanonicalChatQuery, encodeCanonicalChatSeek, parseCanonicalChatSeek } from "~~/shared/chat/history-reader";
 import { computePullRetention } from "./history-gc-policy.js";
 const DEFAULT_PULL_LIMIT = 100;
 const MAX_PULL_LIMIT = 1e3;
@@ -430,12 +432,58 @@ export class SqliteSyncGatewayAdapter {
       snapshotBootstrap: "snapshot-v1",
       historyRetention: "snapshot-v1",
       backgroundGenerationHistory: "v1",
+      canonicalChatHistory: "v1",
       workspaceItems: "v1"
     };
   }
   get db() {
     return getSqliteDb();
   }
+  async readChatHistory(actor, query, signal) {
+    validateCanonicalChatQuery(query);
+    signal?.throwIfAborted();
+    if (isD1Driver()) return { status: "scope_incomplete" };
+    const raw = getRawDb();
+    const result = raw.transaction(() => {
+      const membership = raw.prepare(`SELECT m.role FROM workspace_members m JOIN workspaces w ON w.id = m.workspace_id
+                WHERE m.workspace_id = ? AND m.user_id = ? AND COALESCE(w.deleted, 0) = 0`).get(actor.workspaceId, actor.userId);
+      if (!membership || !["owner", "editor", "viewer"].includes(membership.role)) throw new Error("Forbidden canonical history actor");
+      const counter = raw.prepare("SELECT value FROM server_version_counter WHERE workspace_id = ?").get(actor.workspaceId);
+      const revision = String(counter?.value ?? 0);
+      const parse = (row) => {
+        const payload = JSON.parse(row.data_json);
+        if (payload.id !== row.id) throw new Error("Canonical history identity mismatch");
+        return { ...payload, id: row.id, clock: row.clock, deleted: row.deleted === 1 };
+      };
+      if (query.kind === "thread") {
+        const threadRevision = raw.prepare("SELECT value FROM chat_history_revisions WHERE workspace_id = ? AND thread_id = ?").get(actor.workspaceId, query.thread_id);
+        const row = raw.prepare("SELECT id, data_json, clock, deleted FROM s_threads WHERE workspace_id = ? AND id = ?").get(actor.workspaceId, query.thread_id);
+        return { status: "ok", revision: String(threadRevision?.value ?? 0), thread: row ? parse(row) : void 0 };
+      }
+      if (query.kind === "messages") {
+        if (!query.message_ids.length) return { status: "ok", revision, messages: [] };
+        const rows2 = raw.prepare(`SELECT id, data_json, clock, deleted FROM s_messages WHERE workspace_id = ? AND id IN (${query.message_ids.map(() => "?").join(",")})`).all(actor.workspaceId, ...query.message_ids);
+        return { status: "ok", revision, messages: rows2.map(parse) };
+      }
+      let after;
+      const seek = parseCanonicalChatSeek(query.cursor, query.thread_id);
+      if (seek) after = seek.key;
+      else if (query.cursor) {
+        const value = JSON.parse(query.cursor);
+        if (!Array.isArray(value) || value.length !== 4 || value[0] !== query.thread_id || !Number.isSafeInteger(value[1]) || typeof value[2] !== "string" || typeof value[3] !== "string") throw new Error("Invalid canonical history cursor");
+        after = [value[1], value[2], value[3]];
+      }
+      const rows = raw.prepare(`SELECT id, data_json, clock, deleted FROM s_messages
+                WHERE workspace_id = ? AND json_extract(data_json, '$.thread_id') = ?
+                ${after ? `AND (json_extract(data_json, '$.index'), COALESCE(json_extract(data_json, '$.order_key'), ''), id) ${seek?.backward ? "<" : ">"} (?, ?, ?)` : ""}
+                ORDER BY json_extract(data_json, '$.index') ${seek?.backward ? "DESC" : "ASC"}, COALESCE(json_extract(data_json, '$.order_key'), '') ${seek?.backward ? "DESC" : "ASC"}, id ${seek?.backward ? "DESC" : "ASC"} LIMIT ?`).all(actor.workspaceId, query.thread_id, ...after ?? [], query.limit);
+      const messages = rows.map(parse);
+      const last = messages.at(-1);
+      return { status: "ok", revision, messages, next_cursor: messages.length === query.limit && last ? seek ? encodeCanonicalChatSeek({ ...seek, key: [Number(last.index), String(last.order_key ?? ""), last.id] }) : JSON.stringify([query.thread_id, last.index, last.order_key ?? "", last.id]) : void 0 };
+    })();
+    signal?.throwIfAborted();
+    return result;
+  }
   assertCanonicalActor(actor) {
     if (isD1Driver()) {
       throw new Error("SQLite D1 does not support background generation history v1");
@@ -641,6 +689,7 @@ export class SqliteSyncGatewayAdapter {
       const hlc = `${Date.now().toString(36).padStart(9, "0")}:000:background`;
       const opId = backgroundHistoryOpId(admission, "finalize");
       const terminalState = input.snapshot.status === "complete" ? "complete" : input.snapshot.status === "aborted" ? "aborted" : "failed";
+      const usage = readRequestUsage(input.snapshot.usage);
       const payload = {
         ...current,
         pending: false,
@@ -651,6 +700,7 @@ export class SqliteSyncGatewayAdapter {
         op_id: opId,
         data: {
           ...data,
+          ...usage ? { usage } : {},
           content: input.snapshot.content,
           reasoning_text: input.snapshot.reasoning || null,
           tool_calls: input.snapshot.toolCalls ?? null,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "or3-provider-sqlite",
-    "version": "0.0.13",
+    "version": "0.0.14",
     "repository": {
         "type": "git",
         "url": "git+https://github.com/Saluana/or3-provider-sqlite.git"
@@ -38,7 +38,8 @@
     "lint": "eslint src --max-warnings 0",
     "type-check": "tsc --noEmit",
     "type-check:standalone": "tsc -p tsconfig.release.json --noEmit",
-    "test": "vitest run"
+    "test": "vitest run",
+    "test:host-integration": "vitest run --config vitest.host-integration.config.ts"
     },
     "publishConfig": {
         "access": "public"
@@ -65,6 +66,7 @@
         "h3": "^1.15.4",
         "pathe": "^2.0.3",
         "typescript": "^5.9.3",
-        "vitest": "^4.0.18"
+        "vitest": "^4.0.18",
+        "zod": "4.6.5"
     }
 }