@aperdomoll90/ledger-ai 1.3.0 → 1.4.2

package/dist/lib/config.js

@@ -1,10 +1,13 @@
 import dotenv from 'dotenv';
 import { createClient } from '@supabase/supabase-js';
 import OpenAI from 'openai';
+import { observeOpenAI } from '@langfuse/openai';
 import { resolve } from 'path';
 import { homedir } from 'os';
 import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
 import { fatal, ExitCode } from './errors.js';
+import { openaiLimiter, updateLimitsFromHeaders } from './rate-limiter.js';
+import { initObservability } from './observability.js';
 // --- Defaults ---
 const LEDGER_DIR = resolve(homedir(), '.ledger');
 const LEDGER_DOTENV = resolve(LEDGER_DIR, '.env');
@@ -45,13 +48,24 @@ export function getDefaultConfig() {
         },
     };
 }
+// --- Custom fetch for rate limit header interception ---
+// Wraps the global fetch to read OpenAI's rate limit headers on every response.
+// This works below both the OpenAI SDK and the Langfuse wrapper, so header
+// reading survives regardless of how the client is wrapped.
+const openaiHeaderFetch = async (input, init) => {
+    const response = await fetch(input, init);
+    await updateLimitsFromHeaders(openaiLimiter, response.headers);
+    return response;
+};
 // --- Load Config ---
-export function loadConfig() {
+export function loadConfig(options) {
     // Priority: env vars > DOTENV_CONFIG_PATH > ~/.ledger/.env
     const dotenvPath = process.env.DOTENV_CONFIG_PATH
         || (existsSync(LEDGER_DOTENV) ? LEDGER_DOTENV : undefined);
     if (dotenvPath)
         dotenv.config({ path: dotenvPath, quiet: true });
+    // Init observability after dotenv loads (Langfuse env vars are now available)
+    initObservability();
     if (!process.env.SUPABASE_URL || !process.env.SUPABASE_SERVICE_ROLE_KEY) {
         fatal('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY. Run `ledger init` or check your .env file.', ExitCode.GENERAL_ERROR);
     }
@@ -63,6 +77,9 @@ export function loadConfig() {
         memoryDir: process.env.LEDGER_MEMORY_DIR || fileConfig.memoryDir || DEFAULT_MEMORY_DIR,
         claudeMdPath: process.env.LEDGER_CLAUDE_MD_PATH || fileConfig.claudeMdPath || DEFAULT_CLAUDE_MD_PATH,
         supabase: createClient(process.env.SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY),
-        openai: new OpenAI({ apiKey: process.env.OPENAI_API_KEY }),
+        openai: observeOpenAI(new OpenAI({ apiKey: process.env.OPENAI_API_KEY, maxRetries: 5, fetch: openaiHeaderFetch })),
+        cohereApiKey: process.env.COHERE_API_KEY || undefined,
+        sessionId: options?.sessionId,
+        observabilityEnvironment: options?.observabilityEnvironment,
     };
 }

package/dist/lib/document-classification.js

@@ -0,0 +1,5 @@
+// document-classification.ts
+// Types and interfaces matching the database schema.
+// Pure data definitions — no logic, no I/O, no dependencies.
+// Every other file imports from here.
+export {};

package/dist/lib/document-fetching.js

@@ -0,0 +1,77 @@
+// document-fetching.ts
+// Read documents from the database. No writes, no search — just SELECT queries.
+// Every query filters deleted_at IS NULL so soft-deleted documents are invisible.
+/**
+ * Get a single document by its database ID.
+ * Returns null if the document doesn't exist or is soft-deleted.
+ */
+export async function getDocumentById(supabase, id) {
+    const { data, error } = await supabase
+        .from('documents')
+        .select('*')
+        .eq('id', id)
+        .is('deleted_at', null)
+        .single();
+    if (error || !data)
+        return null;
+    return data;
+}
+/**
+ * Get a single document by its unique name.
+ * Returns null if no document has this name or it's soft-deleted.
+ */
+export async function getDocumentByName(supabase, name) {
+    const { data, error } = await supabase
+        .from('documents')
+        .select('*')
+        .eq('name', name)
+        .is('deleted_at', null)
+        .single();
+    if (error || !data)
+        return null;
+    return data;
+}
+/**
+ * List documents with optional filters. Returns newest first.
+ * All filters are optional — no filters = list all active documents.
+ *
+ * Uses indexed columns: domain, document_type, project, created_at DESC.
+ * The deleted_at IS NULL filter uses the index_documents_active partial index.
+ */
+export async function listDocuments(supabase, filters = {}) {
+    let query = supabase
+        .from('documents')
+        .select('*')
+        .is('deleted_at', null)
+        .order('created_at', { ascending: false })
+        .limit(filters.limit ?? 20);
+    if (filters.domain)
+        query = query.eq('domain', filters.domain);
+    if (filters.document_type)
+        query = query.eq('document_type', filters.document_type);
+    if (filters.project)
+        query = query.eq('project', filters.project);
+    const { data, error } = await query;
+    if (error || !data)
+        return [];
+    return data;
+}
+/**
+ * Fetch all documents that should sync to every machine.
+ * Sync is driven by is_auto_load, not domain — a document syncs locally because
+ * it needs to be in the AI's context every session (CLAUDE.md, MEMORY.md,
+ * personality, behavioral rules). Everything else stays in the database and is
+ * accessed via search on demand, regardless of domain.
+ *
+ * Uses the index_documents_is_auto_load partial index.
+ */
+export async function fetchSyncableDocuments(supabase) {
+    const { data, error } = await supabase
+        .from('documents')
+        .select('*')
+        .eq('is_auto_load', true)
+        .is('deleted_at', null);
+    if (error || !data)
+        return [];
+    return data;
+}

package/dist/lib/document-operations.js

@@ -0,0 +1,150 @@
+// document-operations.ts
+// Write operations — create, update, delete, restore documents.
+// Each function prepares data (chunk, embed, hash) then calls a Postgres RPC function.
+// The database handles transactions (document + chunks + audit = atomic).
+import { contentHash, chunkText, generateEmbedding, toVectorString } from './embeddings.js';
+const DEFAULT_EMBEDDING_MODEL = 'openai/text-embedding-3-small';
+/**
+ * Create a new document.
+ *
+ * What happens:
+ * 1. Hash the content (for change detection)
+ * 2. Split content into chunks (for better search)
+ * 3. Generate an embedding for each chunk (calls OpenAI — costs money)
+ * 4. Format embeddings as Postgres vector strings
+ * 5. Call document_create RPC (Postgres inserts document + chunks + audit in one transaction)
+ * 6. Return the new document's ID
+ */
+export async function createDocument(clients, props) {
+    // Always compute hash from actual content — never accept a pre-computed hash
+    const hash = contentHash(props.content);
+    // Chunk and embed
+    const chunks = chunkText(props.content);
+    const chunkContents = chunks.map(chunk => chunk.content);
+    const chunkEmbeddings = [];
+    for (const chunk of chunks) {
+        const embedding = await generateEmbedding(clients.openai, chunk.content);
+        chunkEmbeddings.push(toVectorString(embedding));
+    }
+    const { data, error } = await clients.supabase.rpc('document_create', {
+        p_name: props.name,
+        p_domain: props.domain,
+        p_document_type: props.document_type,
+        p_project: props.project ?? null,
+        p_protection: props.protection ?? 'open',
+        p_owner_type: props.owner_type ?? 'user',
+        p_owner_id: props.owner_id ?? null,
+        p_is_auto_load: props.is_auto_load ?? false,
+        p_content: props.content,
+        p_description: props.description ?? null,
+        p_content_hash: hash,
+        p_source_type: props.source_type ?? 'text',
+        p_source_url: props.source_url ?? null,
+        p_file_path: props.file_path ?? null,
+        p_file_permissions: props.file_permissions ?? null,
+        p_agent: props.agent ?? null,
+        p_status: props.status ?? null,
+        p_skill_ref: props.skill_ref ?? null,
+        p_embedding_model_id: props.embedding_model_id ?? DEFAULT_EMBEDDING_MODEL,
+        p_chunk_contents: chunkContents,
+        p_chunk_embeddings: chunkEmbeddings,
+        p_chunk_strategy: chunks[0]?.strategy ?? 'paragraph',
+    });
+    if (error)
+        throw new Error(`Failed to create document: ${error.message}`);
+    return data;
+}
+/**
+ * Update a document's content. Triggers re-chunking and re-embedding.
+ *
+ * What happens:
+ * 1. Hash the new content
+ * 2. Split new content into chunks
+ * 3. Generate new embeddings for each chunk (calls OpenAI)
+ * 4. Call document_update RPC — Postgres handles:
+ *    - Save old content to document_versions (version snapshot)
+ *    - Update the document row
+ *    - Delete old chunks, insert new chunks
+ *    - Write audit entry
+ */
+export async function updateDocument(clients, props) {
+    const hash = contentHash(props.content);
+    const chunks = chunkText(props.content);
+    const chunkContents = chunks.map(chunk => chunk.content);
+    const chunkEmbeddings = [];
+    for (const chunk of chunks) {
+        const embedding = await generateEmbedding(clients.openai, chunk.content);
+        chunkEmbeddings.push(toVectorString(embedding));
+    }
+    const { error } = await clients.supabase.rpc('document_update', {
+        p_id: props.id,
+        p_content: props.content,
+        p_content_hash: hash,
+        p_agent: props.agent ?? null,
+        p_description: props.description ?? null,
+        p_status: props.status ?? null,
+        p_embedding_model_id: props.embedding_model_id ?? DEFAULT_EMBEDDING_MODEL,
+        p_chunk_contents: chunkContents,
+        p_chunk_embeddings: chunkEmbeddings,
+        p_chunk_strategy: chunks[0]?.strategy ?? 'paragraph',
+    });
+    if (error)
+        throw new Error(`Failed to update document: ${error.message}`);
+}
+/**
+ * Update document fields without changing content. No re-embedding needed.
+ *
+ * This is cheap (no OpenAI calls) — just passes the fields to Postgres.
+ * Postgres handles: update columns, sync domain to chunks if changed, write audit.
+ */
+export async function updateDocumentFields(clients, props) {
+    const { error } = await clients.supabase.rpc('document_update_fields', {
+        p_id: props.id,
+        p_agent: props.agent ?? null,
+        p_name: props.name ?? null,
+        p_domain: props.domain ?? null,
+        p_document_type: props.document_type ?? null,
+        p_project: props.project ?? null,
+        p_protection: props.protection ?? null,
+        p_owner_type: props.owner_type ?? null,
+        p_owner_id: props.owner_id ?? null,
+        p_is_auto_load: props.is_auto_load ?? null,
+        p_description: props.description ?? null,
+        p_source_type: props.source_type ?? null,
+        p_source_url: props.source_url ?? null,
+        p_file_path: props.file_path ?? null,
+        p_file_permissions: props.file_permissions ?? null,
+        p_status: props.status ?? null,
+        p_skill_ref: props.skill_ref ?? null,
+        p_embedding_model_id: props.embedding_model_id ?? null,
+    });
+    if (error)
+        throw new Error(`Failed to update document fields: ${error.message}`);
+}
+/**
+ * Soft delete a document. The document stays in the database with deleted_at set.
+ * Chunks are removed (search shouldn't find deleted documents).
+ * Can be restored within 30 days via restoreDocument().
+ * After 30 days, document_purge() permanently removes it.
+ */
+export async function deleteDocument(clients, id, agent) {
+    const { error } = await clients.supabase.rpc('document_delete', {
+        p_id: id,
+        p_agent: agent,
+    });
+    if (error)
+        throw new Error(`Failed to delete document: ${error.message}`);
+}
+/**
+ * Undo a soft delete. The document becomes active again.
+ * Note: chunks were removed during delete — they need to be regenerated
+ * by calling updateDocument() with the same content (which re-chunks and re-embeds).
+ */
+export async function restoreDocument(clients, id, agent) {
+    const { error } = await clients.supabase.rpc('document_restore', {
+        p_id: id,
+        p_agent: agent,
+    });
+    if (error)
+        throw new Error(`Failed to restore document: ${error.message}`);
+}

package/dist/lib/documents/classification.js

@@ -0,0 +1,5 @@
+// document-classification.ts
+// Types and interfaces matching the database schema.
+// Pure data definitions — no logic, no I/O, no dependencies.
+// Every other file imports from here.
+export {};

package/dist/lib/documents/fetching.js

@@ -0,0 +1,89 @@
+// document-fetching.ts
+// Read documents from the database. No writes, no search — just SELECT queries.
+// Every query filters deleted_at IS NULL so soft-deleted documents are invisible.
+/**
+ * Get a single document by its database ID.
+ * Returns null if the document doesn't exist or is soft-deleted.
+ */
+export async function getDocumentById(supabase, id) {
+    const { data, error } = await supabase
+        .from('documents')
+        .select('*')
+        .eq('id', id)
+        .is('deleted_at', null)
+        .single();
+    if (error) {
+        if (error.code !== 'PGRST116') {
+            process.stderr.write(`[ledger] getDocumentById(${id}) failed: ${error.message}\n`);
+        }
+        return null;
+    }
+    return data ?? null;
+}
+/**
+ * Get a single document by its unique name.
+ * Returns null if no document has this name or it's soft-deleted.
+ */
+export async function getDocumentByName(supabase, name) {
+    const { data, error } = await supabase
+        .from('documents')
+        .select('*')
+        .eq('name', name)
+        .is('deleted_at', null)
+        .single();
+    if (error) {
+        if (error.code !== 'PGRST116') {
+            process.stderr.write(`[ledger] getDocumentByName("${name}") failed: ${error.message}\n`);
+        }
+        return null;
+    }
+    return data ?? null;
+}
+/**
+ * List documents with optional filters. Returns newest first.
+ * All filters are optional — no filters = list all active documents.
+ *
+ * Uses indexed columns: domain, document_type, project, created_at DESC.
+ * The deleted_at IS NULL filter uses the index_documents_active partial index.
+ */
+export async function listDocuments(supabase, filters = {}) {
+    let query = supabase
+        .from('documents')
+        .select('*')
+        .is('deleted_at', null)
+        .order('created_at', { ascending: false })
+        .limit(filters.limit ?? 20);
+    if (filters.domain)
+        query = query.eq('domain', filters.domain);
+    if (filters.document_type)
+        query = query.eq('document_type', filters.document_type);
+    if (filters.project)
+        query = query.eq('project', filters.project);
+    const { data, error } = await query;
+    if (error) {
+        process.stderr.write(`[ledger] listDocuments failed: ${error.message}\n`);
+        return [];
+    }
+    return data ?? [];
+}
+/**
+ * Fetch all documents that should sync to every machine.
+ * Sync is driven by is_auto_load, not domain — a document syncs locally because
+ * it needs to be in the AI's context every session (CLAUDE.md, MEMORY.md,
+ * personality, behavioral rules). Everything else stays in the database and is
+ * accessed via search on demand, regardless of domain.
+ *
+ * Uses the index_documents_is_auto_load partial index.
+ */
+export async function fetchSyncableDocuments(supabase) {
+    const { data, error } = await supabase
+        .from('documents')
+        .select('*')
+        .eq('is_auto_load', true)
+        .is('deleted_at', null);
+    if (error) {
+        process.stderr.write(`[ledger] fetchSyncableDocuments failed: ${error.message}\n`);
+        return [];
+    }
+    return data ?? [];
+}

package/dist/lib/documents/operations.js

@@ -0,0 +1,304 @@
+// document-operations.ts
+// Write operations — create, update, delete, restore documents.
+// Each function prepares data (chunk, embed, hash) then calls a Postgres RPC function.
+// The database handles transactions (document + chunks + audit = atomic).
+import { readFileSync } from 'fs';
+import { contentHash, chunkText, generateEmbeddingsBatch, toVectorString } from '../search/embeddings.js';
+import { generateContextSummaries } from '../search/chunk-context-enrichment.js';
+import { getDocumentById } from './fetching.js';
+import { startTrace, startSpan } from '../observability.js';
+const DEFAULT_EMBEDDING_MODEL = 'openai/text-embedding-3-small';
+const DEFAULT_CHUNK_CONFIG = {
+    maxChunkSize: 1000,
+    overlapChars: 200,
+    strategy: 'recursive',
+};
+/**
+ * Create a new document.
+ *
+ * Pipeline:
+ * 1. Hash the content (change detection)
+ * 2. Chunk with recursive splitter
+ * 3. Generate context summaries per chunk (LLM call — chunk context enrichment)
+ * 4. Embed summary + chunk content (OpenAI embedding call per chunk)
+ * 5. Call document_create RPC (atomic: document + chunks + audit)
+ */
+export async function createDocument(clients, props, chunkConfig) {
+    const config = { ...DEFAULT_CHUNK_CONFIG, ...chunkConfig };
+    const hash = contentHash(props.content);
+    const trace = startTrace('document-ingestion', {
+        tags: ['ingestion', 'create'],
+        metadata: { documentName: props.name, domain: props.domain, documentType: props.document_type },
+        input: { contentLength: props.content.length },
+    });
+    // Chunk
+    const chunkSpan = startSpan('chunking', { input: { contentLength: props.content.length } });
+    const chunks = chunkText(props.content, config);
+    const chunkContents = chunks.map(chunk => chunk.content);
+    chunkSpan.update({ output: { chunkCount: chunks.length, avgChunkSize: Math.round(props.content.length / chunks.length) } });
+    chunkSpan.end();
+    // Enrich — generate context summaries per chunk (LLM calls auto-traced by wrapped client)
+    const enrichSpan = startSpan('context-enrichment', { metadata: { chunkCount: chunks.length, model: 'gpt-4o-mini' } });
+    const enrichmentResults = await generateContextSummaries(clients.openai, chunks, props.content);
+    const chunkSummaries = enrichmentResults.map(result => result.summary);
+    const chunkTokenCounts = enrichmentResults.map(result => result.tokenCount);
+    enrichSpan.end();
+    // Embed — summary + "\n\n" + chunk content (batch: one API call per 100 chunks, auto-traced)
+    const embedSpan = startSpan('batch-embedding', { metadata: { chunkCount: chunks.length, model: 'text-embedding-3-small' } });
+    const embeddingInputs = chunks.map((chunk, index) => chunkSummaries[index] + '\n\n' + chunk.content);
+    const embeddings = await generateEmbeddingsBatch(clients.openai, embeddingInputs);
+    const chunkEmbeddings = embeddings.map(toVectorString);
+    embedSpan.end();
+    // DB write
+    const dbSpan = startSpan('db-write', { input: { chunkCount: chunks.length } });
+    const { data, error } = await clients.supabase.rpc('document_create', {
+        p_name: props.name,
+        p_domain: props.domain,
+        p_document_type: props.document_type,
+        p_project: props.project ?? null,
+        p_protection: props.protection ?? 'open',
+        p_owner_type: props.owner_type ?? 'user',
+        p_owner_id: props.owner_id ?? null,
+        p_is_auto_load: props.is_auto_load ?? false,
+        p_content: props.content,
+        p_description: props.description ?? null,
+        p_content_hash: hash,
+        p_source_type: props.source_type ?? 'text',
+        p_source_url: props.source_url ?? null,
+        p_file_path: props.file_path ?? null,
+        p_file_permissions: props.file_permissions ?? null,
+        p_agent: props.agent ?? null,
+        p_status: props.status ?? null,
+        p_skill_ref: props.skill_ref ?? null,
+        p_embedding_model_id: props.embedding_model_id ?? DEFAULT_EMBEDDING_MODEL,
+        p_chunk_contents: chunkContents,
+        p_chunk_embeddings: chunkEmbeddings,
+        p_chunk_strategy: chunks[0]?.strategy ?? config.strategy,
+        p_chunk_summaries: chunkSummaries,
+        p_chunk_token_counts: chunkTokenCounts,
+        p_chunk_overlap: config.overlapChars,
+    });
+    dbSpan.update({ output: { documentId: data } });
+    dbSpan.end();
+    trace.end();
+    if (error)
+        throw new Error(`Failed to create document "${props.name}" (${props.domain}/${props.document_type}): ${error.message}`);
+    return data;
+}
+/**
+ * Update a document's content. Triggers re-chunking, re-enrichment, and re-embedding.
+ *
+ * Same pipeline as createDocument — hash, chunk, enrich, embed — then calls
+ * document_update RPC which versions old content before overwriting.
+ */
+export async function updateDocument(clients, props, chunkConfig) {
+    const config = { ...DEFAULT_CHUNK_CONFIG, ...chunkConfig };
+    const hash = contentHash(props.content);
+    const trace = startTrace('document-ingestion', {
+        tags: ['ingestion', 'update'],
+        metadata: { documentId: props.id },
+        input: { contentLength: props.content.length },
+    });
+    // Chunk
+    const chunkSpan = startSpan('chunking', { input: { contentLength: props.content.length } });
+    const chunks = chunkText(props.content, config);
+    const chunkContents = chunks.map(chunk => chunk.content);
+    chunkSpan.update({ output: { chunkCount: chunks.length, avgChunkSize: Math.round(props.content.length / chunks.length) } });
+    chunkSpan.end();
+    // Enrich (LLM calls auto-traced)
+    const enrichSpan = startSpan('context-enrichment', { metadata: { chunkCount: chunks.length, model: 'gpt-4o-mini' } });
+    const enrichmentResults = await generateContextSummaries(clients.openai, chunks, props.content);
+    const chunkSummaries = enrichmentResults.map(result => result.summary);
+    const chunkTokenCounts = enrichmentResults.map(result => result.tokenCount);
+    enrichSpan.end();
+    // Embed (auto-traced)
+    const embedSpan = startSpan('batch-embedding', { metadata: { chunkCount: chunks.length, model: 'text-embedding-3-small' } });
+    const embeddingInputs = chunks.map((chunk, index) => chunkSummaries[index] + '\n\n' + chunk.content);
+    const embeddings = await generateEmbeddingsBatch(clients.openai, embeddingInputs);
+    const chunkEmbeddings = embeddings.map(toVectorString);
+    embedSpan.end();
+    // DB write
+    const dbSpan = startSpan('db-write', { input: { chunkCount: chunks.length } });
+    const { error } = await clients.supabase.rpc('document_update', {
+        p_id: props.id,
+        p_content: props.content,
+        p_content_hash: hash,
+        p_agent: props.agent ?? null,
+        p_description: props.description ?? null,
+        p_status: props.status ?? null,
+        p_embedding_model_id: props.embedding_model_id ?? DEFAULT_EMBEDDING_MODEL,
+        p_chunk_contents: chunkContents,
+        p_chunk_embeddings: chunkEmbeddings,
+        p_chunk_strategy: chunks[0]?.strategy ?? config.strategy,
+        p_chunk_summaries: chunkSummaries,
+        p_chunk_token_counts: chunkTokenCounts,
+        p_chunk_overlap: config.overlapChars,
+    });
+    dbSpan.end();
+    trace.end();
+    if (error)
+        throw new Error(`Failed to update document #${props.id}: ${error.message}`);
+}
+/**
+ * Update document fields without changing content. No re-embedding needed.
+ *
+ * This is cheap (no OpenAI calls) — just passes the fields to Postgres.
+ * Postgres handles: update columns, sync domain to chunks if changed, write audit.
+ */
+export async function updateDocumentFields(clients, props) {
+    const { error } = await clients.supabase.rpc('document_update_fields', {
+        p_id: props.id,
+        p_agent: props.agent ?? null,
+        p_name: props.name ?? null,
+        p_domain: props.domain ?? null,
+        p_document_type: props.document_type ?? null,
+        p_project: props.project ?? null,
+        p_protection: props.protection ?? null,
+        p_owner_type: props.owner_type ?? null,
+        p_owner_id: props.owner_id ?? null,
+        p_is_auto_load: props.is_auto_load ?? null,
+        p_description: props.description ?? null,
+        p_source_type: props.source_type ?? null,
+        p_source_url: props.source_url ?? null,
+        p_file_path: props.file_path ?? null,
+        p_file_permissions: props.file_permissions ?? null,
+        p_status: props.status ?? null,
+        p_skill_ref: props.skill_ref ?? null,
+        p_embedding_model_id: props.embedding_model_id ?? null,
+    });
+    if (error)
+        throw new Error(`Failed to update fields on document #${props.id}: ${error.message}`);
+}
+/**
+ * Soft delete a document. The document stays in the database with deleted_at set.
+ * Chunks are removed (search shouldn't find deleted documents).
+ * Can be restored within 30 days via restoreDocument().
+ * After 30 days, document_purge() permanently removes it.
+ */
+export async function deleteDocument(clients, id, agent) {
+    const { error } = await clients.supabase.rpc('document_delete', {
+        p_id: id,
+        p_agent: agent,
+    });
+    if (error)
+        throw new Error(`Failed to delete document #${id}: ${error.message}`);
+}
+/**
+ * Undo a soft delete. The document becomes active again.
+ * Note: chunks were removed during delete — they need to be regenerated
+ * by calling updateDocument() with the same content (which re-chunks and re-embeds).
+ */
+export async function restoreDocument(clients, id, agent) {
+    const { error } = await clients.supabase.rpc('document_restore', {
+        p_id: id,
+        p_agent: agent,
+    });
+    if (error)
+        throw new Error(`Failed to restore document #${id}: ${error.message}`);
+}
+/**
+ * Thrown when the post-push verify pull-back does not byte-match the file we sent.
+ * Carries the document id, both byte counts, and a single-line diff preview that
+ * locates the first divergence (line / col / expected snippet / actual snippet).
+ *
+ * Caller decides how to surface this. CLI prints to stderr and exits non-zero;
+ * MCP returns it as an error result. Neither path attempts to rollback.
+ */
+export class VerifyMismatchError extends Error {
+    id;
+    expectedLength;
+    actualLength;
+    diffPreview;
+    constructor(id, expectedLength, actualLength, diffPreview) {
+        super(`Verify mismatch on document ${id}: pushed ${expectedLength} bytes, pulled ${actualLength} bytes. ${diffPreview}`);
+        this.id = id;
+        this.expectedLength = expectedLength;
+        this.actualLength = actualLength;
+        this.diffPreview = diffPreview;
+        this.name = 'VerifyMismatchError';
+    }
+}
+// Locate the first byte at which two strings differ and produce a one-line preview
+// in the form: `line L, col C: expected '<snippet>' but got '<snippet>'`.
+// Returns null when the strings are byte-identical.
+function buildDiffPreview(expected, actual) {
+    if (expected === actual)
+        return null;
+    const minLength = Math.min(expected.length, actual.length);
+    let diffIndex = minLength;
+    for (let cursor = 0; cursor < minLength; cursor++) {
+        if (expected[cursor] !== actual[cursor]) {
+            diffIndex = cursor;
+            break;
+        }
+    }
+    let line = 1;
+    let col = 1;
+    for (let cursor = 0; cursor < diffIndex; cursor++) {
+        if (expected[cursor] === '\n') {
+            line++;
+            col = 1;
+        }
+        else {
+            col++;
+        }
+    }
+    const SNIPPET_LENGTH = 40;
+    const escape = (snippet) => snippet.replace(/\n/g, '\\n').replace(/\t/g, '\\t');
+    const expectedSnippet = escape(expected.slice(diffIndex, diffIndex + SNIPPET_LENGTH));
+    const actualSnippet = escape(actual.slice(diffIndex, diffIndex + SNIPPET_LENGTH));
+    return `line ${line}, col ${col}: expected '${expectedSnippet}' but got '${actualSnippet}'`;
+}
+// Pull the document back and byte-compare against the bytes we just wrote.
+// Throws VerifyMismatchError if the DB returned different bytes than we sent
+// (drift / pipeline transformation / concurrent write all manifest the same way).
+async function verifyAfterWrite(clients, id, expectedBody) {
+    const pulled = await getDocumentById(clients.supabase, id);
+    if (!pulled) {
+        throw new VerifyMismatchError(id, expectedBody.length, 0, 'document not found during verify (deleted between write and verify, or wrong id)');
+    }
+    const diffPreview = buildDiffPreview(expectedBody, pulled.content);
+    if (diffPreview !== null) {
+        throw new VerifyMismatchError(id, expectedBody.length, pulled.content.length, diffPreview);
+    }
+}
+/**
+ * Update a document by reading its new content from a file on disk, then verify the write.
+ *
+ * Pipeline:
+ * 1. Read file from `filePath` (utf8). Surfaces fs errors (ENOENT, EACCES) as-is.
+ * 2. Call updateDocument() with the file bytes as content.
+ * 3. Pull the document back and byte-compare against what we wrote.
+ * 4. On match: return { id, verified, bytes }. On mismatch: throw VerifyMismatchError.
+ *
+ * The file is never trimmed, normalized, or transformed — bytes-in equals bytes-out.
+ */
+export async function updateDocumentFromFile(clients, props) {
+    const body = readFileSync(props.filePath, 'utf8');
+    await updateDocument(clients, { id: props.id, content: body, agent: props.agent });
+    await verifyAfterWrite(clients, props.id, body);
+    return { id: props.id, verified: true, bytes: body.length };
+}
+/**
+ * Create a new document by reading its content from a file on disk, then verify the write.
+ *
+ * Same shape as updateDocumentFromFile: read file, call createDocument(), pull-back, byte-compare.
+ * On mismatch the new document still exists; caller decides whether to delete it (audit_log
+ * preserves the create event for manual cleanup).
+ */
+export async function createDocumentFromFile(clients, props) {
+    const body = readFileSync(props.filePath, 'utf8');
+    const id = await createDocument(clients, {
+        name: props.name,
+        domain: props.domain,
+        document_type: props.document_type,
+        content: body,
+        description: props.description,
+        project: props.project,
+        agent: props.agent,
+        status: props.status,
+        protection: props.protection,
+    });
+    await verifyAfterWrite(clients, id, body);
+    return { id, verified: true, bytes: body.length };
+}