@aperdomoll90/ledger-ai 1.3.0 → 1.4.2

package/dist/cli.js

@@ -1,110 +1,209 @@
 #!/usr/bin/env node
+// cli.ts — Ledger CLI entry point.
+// 14 commands for managing the RAG knowledge base.
 import { Command } from 'commander';
 import { createRequire } from 'module';
+import { randomUUID } from 'node:crypto';
 import { loadConfig } from './lib/config.js';
+import { shutdownObservability } from './lib/observability.js';
+// Per-invocation session ID so all searches from this CLI run group together
+// in the Langfuse dashboard.
+const CLI_SESSION_ID = `cli-${randomUUID()}`;
+const CLI_ENVIRONMENT = process.env.NODE_ENV ?? 'development';
+const CLI_CONTEXT = { sessionId: CLI_SESSION_ID, observabilityEnvironment: CLI_ENVIRONMENT };
 const require = createRequire(import.meta.url);
 const { version } = require('../package.json');
-import { pull } from './commands/pull.js';
-import { push } from './commands/push.js';
-import { check, checkChunks } from './commands/check.js';
-import { sync } from './commands/sync.js';
-import { show } from './commands/show.js';
-import { exportNote } from './commands/export.js';
-import { ingest } from './commands/ingest.js';
+// Commands
 import { init } from './commands/init.js';
-import { wizard } from './commands/wizard.js';
-import { setupClaudeCode, setupOpenclaw, setupChatgpt } from './commands/setup.js';
-import { backup, enableBackupCron, disableBackupCron } from './commands/backup.js';
-import { restore } from './commands/restore.js';
-import { onboard } from './commands/onboard.js';
-import { configGet, configSet, configUnset, configList } from './commands/config.js';
-import { migrate } from './commands/migrate.js';
-import { add } from './commands/add.js';
-import { update } from './commands/update.js';
-import { deleteNote } from './commands/delete.js';
+import { addDocumentFromFile } from './commands/add.js';
 import { list } from './commands/list.js';
+import { show } from './commands/show.js';
+import { get } from './commands/get.js';
+import { exportDocument } from './commands/export.js';
+import { push } from './commands/push.js';
+import { updateFromFile } from './commands/update.js';
+import { removeDocument } from './commands/delete.js';
 import { tag } from './commands/tag.js';
+import { check } from './commands/check.js';
+import { backup, enableBackupCron, disableBackupCron } from './commands/backup.js';
+import { restore } from './commands/restore.js';
 import { lint } from './commands/lint.js';
-process.on('unhandledRejection', (err) => {
-    console.error(err instanceof Error ? err.message : String(err));
+import { evalSearch, sweepThreshold, showEvalRun } from './commands/eval.js';
+import { evalJudge } from './commands/eval-judge.js';
+process.on('unhandledRejection', (rejection) => {
+    console.error(rejection instanceof Error ? rejection.message : String(rejection));
     process.exit(1);
 });
+// Flush pending Langfuse traces before exit
+const shutdown = async () => {
+    await shutdownObservability();
+    process.exit(0);
+};
+process.on('SIGINT', shutdown);
+process.on('SIGTERM', shutdown);
 const program = new Command();
 program
     .name('ledger')
-    .description('AI identity and memory system — sync knowledge across agents and devices')
+    .description('AI identity and memory system — RAG-powered knowledge base for agents')
     .version(version);
+// =============================================================================
+// Document management
+// =============================================================================
 program
-    .command('pull')
-    .description('Download notes from Ledger to local cache')
-    .option('-q, --quiet', 'suppress non-conflict output')
-    .option('-f, --force', 'overwrite local changes without conflict check')
+    .command('add')
+    .description('Add a new document to Ledger by reading content from a file on disk (auto-verified after create)')
+    .requiredOption('-f, --file <file>', 'absolute or relative path to the file containing the document content')
+    .requiredOption('-n, --name <name>', 'unique document name (lowercase, hyphens)')
+    .option('-d, --domain <domain>', 'domain: system, persona, workspace, project, general', 'general')
+    .option('-t, --type <type>', 'document type (architecture, reference, knowledge, etc.)', 'knowledge')
+    .option('-p, --project <project>', 'project name')
+    .option('--description <text>', 'one-line description')
+    .option('-a, --agent <agent>', 'agent name', 'cli')
+    .option('-s, --status <status>', 'status: idea, planning, active, done')
+    .option('--protection <level>', 'protection: open, guarded, protected, immutable')
     .action(async (options) => {
-    const config = loadConfig();
-    await pull(config, { quiet: options.quiet ?? false, force: options.force ?? false });
-});
-program
-    .command('push <file>')
-    .description('Upload a local file to Ledger')
-    .action(async (file) => {
-    const config = loadConfig();
-    await push(config, file);
-});
-program
-    .command('check')
-    .description('Compare local files vs Ledger, report sync status')
-    .option('--chunks', 'Check chunk group integrity')
-    .action(async (opts) => {
-    const config = loadConfig();
-    if (opts.chunks) {
-        await checkChunks(config);
-    }
-    else {
-        await check(config);
-    }
+    const config = loadConfig(CLI_CONTEXT);
+    await addDocumentFromFile(config, {
+        filePath: options.file,
+        name: options.name,
+        domain: options.domain,
+        documentType: options.type,
+        project: options.project,
+        description: options.description,
+        agent: options.agent,
+        status: options.status,
+        protection: options.protection,
+    });
 });
 program
-    .command('sync')
-    .description('Bidirectional sync of persona files between Ledger and local cache')
-    .option('-q, --quiet', 'suppress output unless conflicts (for hooks)')
-    .option('-f, --force', 'overwrite local with Ledger version')
-    .option('-n, --dry-run', 'show what would happen without doing it')
+    .command('list')
+    .description('List recent documents from Ledger')
+    .option('-n, --limit <number>', 'number of documents', '20')
+    .option('-t, --type <type>', 'filter by document type')
+    .option('-p, --project <project>', 'filter by project name')
     .action(async (options) => {
-    const config = loadConfig();
-    await sync(config, {
-        quiet: options.quiet ?? false,
-        force: options.force ?? false,
-        dryRun: options.dryRun ?? false,
+    const config = loadConfig(CLI_CONTEXT);
+    await list(config, {
+        limit: parseInt(options.limit, 10),
+        type: options.type,
+        project: options.project,
     });
 });
+program
+    .command('get <name>')
+    .description('Fetch a document by exact name and print its content')
+    .action(async (name) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await get(config, name);
+});
 program
     .command('show <query...>')
-    .description('Search Ledger by meaning, open matching note')
-    .option('-t, --type <type>', 'filter by note type (e.g. feedback, reference)')
+    .description('Search Ledger by meaning, open matching document')
+    .option('-t, --type <type>', 'filter by document type')
     .option('-p, --project <project>', 'filter by project name')
     .action(async (queryParts, options) => {
-    const config = loadConfig();
+    const config = loadConfig(CLI_CONTEXT);
     await show(config, queryParts.join(' '), { type: options.type, project: options.project });
 });
 program
     .command('export <query...>')
-    .description('Download a note to a custom location (untracked)')
+    .description('Download a document to a file')
     .option('-o, --output <path>', 'output directory (default: current directory)')
     .action(async (queryParts, options) => {
-    const config = loadConfig();
-    await exportNote(config, queryParts.join(' '), options.output);
+    const config = loadConfig(CLI_CONTEXT);
+    await exportDocument(config, queryParts.join(' '), options.output);
+});
+program
+    .command('push <file>')
+    .description('Upload a local file to Ledger')
+    .action(async (file) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await push(config, file);
+});
+program
+    .command('update <id>')
+    .description('Update a document by ID by reading new content from a file on disk (auto-verified after push)')
+    .requiredOption('-f, --file <file>', 'absolute or relative path to the file containing the new content')
+    .option('-y, --yes', 'skip the interactive confirmation prompt (for non-interactive scripts)')
+    .action(async (documentId, options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await updateFromFile(config, parseInt(documentId, 10), options.file, { yes: options.yes });
+});
+program
+    .command('delete <id>')
+    .description('Soft-delete a document by ID (restorable within 30 days)')
+    .option('-y, --yes', 'skip the interactive confirmation prompt (for non-interactive scripts)')
+    .action(async (documentId, options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await removeDocument(config, parseInt(documentId, 10), { yes: options.yes });
+});
+program
+    .command('tag <id>')
+    .description('Update metadata on a document (description, project, domain)')
+    .option('-d, --description <text>', 'document description')
+    .option('-p, --project <name>', 'project name')
+    .option('--domain <domain>', 'domain: system, persona, workspace, project, general')
+    .action(async (documentId, options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await tag(config, parseInt(documentId, 10), {
+        description: options.description,
+        project: options.project,
+        domain: options.domain,
+    });
+});
+// =============================================================================
+// Search quality
+// =============================================================================
+program
+    .command('eval')
+    .description('Run search quality evaluation against golden dataset')
+    .option('--dry-run', 'print report without saving to database')
+    .action(async (options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await evalSearch(config, { dryRun: options.dryRun ?? false });
+});
+program
+    .command('eval:sweep')
+    .description('Test multiple similarity thresholds to find optimal value')
+    .option('--thresholds <values>', 'comma-separated thresholds to test', '0.15,0.20,0.25,0.30,0.35,0.40')
+    .action(async (options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await sweepThreshold(config, { thresholds: options.thresholds });
+});
+program
+    .command('eval:show <runId>')
+    .description('Inspect a saved eval run — summary metrics and missed queries with doc names')
+    .option('--full', 'also dump per-query results as JSON', false)
+    .action(async (runIdArg, options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    const runId = parseInt(runIdArg, 10);
+    if (isNaN(runId)) {
+        process.stderr.write(`Invalid run id: ${runIdArg}\n`);
+        process.exit(1);
+    }
+    await showEvalRun(config, runId, { full: options.full ?? false });
+});
+program
+    .command('eval:judge')
+    .description('Resumable rejudging walkthrough — grade top-10 search results per query using TREC 4-level scale')
+    .option('--query <id>', 'start at a specific golden query id', (value) => parseInt(value, 10))
+    .action(async (options) => {
+    const config = loadConfig(CLI_CONTEXT);
+    await evalJudge(config, { query: options.query });
 });
+// =============================================================================
+// Sync and maintenance
+// =============================================================================
 program
-    .command('ingest [file]')
-    .description('Scan for unknown files and add them to Ledger with duplicate detection')
-    .option('-a, --auto', 'auto-ingest without prompts (for hooks)')
-    .action(async (file, options) => {
-    const config = loadConfig();
-    await ingest(config, { file, auto: options.auto ?? false });
+    .command('check')
+    .description('Compare local files vs Ledger, report sync status')
+    .action(async () => {
+    const config = loadConfig(CLI_CONTEXT);
+    await check(config);
 });
 program
     .command('backup')
-    .description('Backup all notes to ~/.ledger/backups/')
+    .description('Backup all documents to ~/.ledger/backups/')
     .option('-q, --quiet', 'suppress output unless error')
     .option('--enable-cron', 'enable daily backup at 1am')
     .option('--disable-cron', 'disable daily backup cron')
@@ -117,167 +216,24 @@ program
         disableBackupCron();
         return;
     }
-    const config = loadConfig();
+    const config = loadConfig(CLI_CONTEXT);
     await backup(config, { quiet: options.quiet ?? false });
 });
 program
     .command('restore <file>')
-    .description('Restore notes from a backup JSON file')
+    .description('Restore documents from a backup JSON file')
     .action(async (file) => {
-    const config = loadConfig();
+    const config = loadConfig(CLI_CONTEXT);
     await restore(config, file);
 });
-const configCmd = program
-    .command('config')
-    .description('View or change Ledger settings');
-configCmd
-    .command('get <key>')
-    .description('Get a config value (or "all" for full config)')
-    .action(async (key) => {
-    await configGet(key);
-});
-configCmd
-    .command('set <key> <value>')
-    .description('Set a config value')
-    .action(async (key, value) => {
-    if (key.startsWith('types.')) {
-        const config = loadConfig();
-        await configSet(key, value, { supabase: config.supabase, openai: config.openai });
-    }
-    else {
-        await configSet(key, value);
-    }
-});
-configCmd
-    .command('unset <key>')
-    .description('Remove a config override (types.* keys only)')
-    .action(async (key) => {
-    if (key.startsWith('types.')) {
-        const config = loadConfig();
-        await configUnset(key, { supabase: config.supabase, openai: config.openai });
-    }
-    else {
-        await configUnset(key);
-    }
-});
-configCmd
-    .command('list')
-    .description('Show all settings')
-    .action(async () => {
-    await configList();
-});
-program
-    .command('onboard')
-    .description('Create your AI persona (profile, communication style, rules)')
-    .action(async () => {
-    const config = loadConfig();
-    await onboard(config);
-});
+// =============================================================================
+// Setup and configuration
+// =============================================================================
 program
     .command('init')
-    .description('Guided setup wizard (credentials, database, persona, platforms, sync)')
-    .option('--legacy', 'run legacy init (credentials + database only)')
-    .action(async (options) => {
-    if (options.legacy) {
-        await init();
-    }
-    else {
-        await wizard();
-    }
-});
-const setupCmd = program
-    .command('setup')
-    .description('Configure an agent platform to use Ledger');
-setupCmd
-    .command('claude-code')
-    .description('Register MCP, install hooks, pull cache (live sync)')
-    .action(async () => {
-    await setupClaudeCode();
-});
-setupCmd
-    .command('openclaw [path]')
-    .description('Generate persona files for OpenClaw (live sync via CLI)')
-    .action(async (path) => {
-    await setupOpenclaw(path);
-});
-setupCmd
-    .command('chatgpt')
-    .description('Generate system prompt for ChatGPT (static snapshot)')
-    .action(async () => {
-    await setupChatgpt();
-});
-program
-    .command('migrate')
-    .description('Safely migrate local files to Ledger (backup, compare, upload)')
+    .description('Set up Ledger credentials and database')
     .action(async () => {
-    const config = loadConfig();
-    await migrate(config);
-});
-program
-    .command('add')
-    .description('Add a new note to Ledger (with duplicate detection)')
-    .requiredOption('-c, --content <content>', 'note content (or use stdin)')
-    .option('-t, --type <type>', 'note type (feedback, reference, event, etc.)')
-    .option('-a, --agent <agent>', 'agent name', 'cli')
-    .option('-p, --project <project>', 'project name')
-    .option('-k, --upsert-key <key>', 'upsert key for dedup')
-    .option('-d, --description <text>', 'one-line description of the note')
-    .option('-s, --status <status>', 'note status (idea, planning, active, done)')
-    .option('-f, --force', 'skip duplicate check and interactive prompts')
-    .action(async (options) => {
-    const config = loadConfig();
-    await add(config, options.content, {
-        type: options.type,
-        agent: options.agent,
-        project: options.project,
-        upsertKey: options.upsertKey,
-        description: options.description,
-        status: options.status,
-        force: options.force ?? false,
-    });
-});
-program
-    .command('update <id>')
-    .description('Update an existing note by ID (with confirmation)')
-    .requiredOption('-c, --content <content>', 'new content')
-    .action(async (id, options) => {
-    const config = loadConfig();
-    await update(config, parseInt(id, 10), options.content, {});
-});
-program
-    .command('delete <id>')
-    .description('Delete a note by ID (with confirmation)')
-    .action(async (id) => {
-    const config = loadConfig();
-    await deleteNote(config, parseInt(id, 10));
-});
-program
-    .command('list')
-    .description('List recent notes from Ledger')
-    .option('-n, --limit <number>', 'number of notes', '20')
-    .option('-t, --type <type>', 'filter by note type')
-    .option('-p, --project <project>', 'filter by project name')
-    .action(async (options) => {
-    const config = loadConfig();
-    await list(config, {
-        limit: parseInt(options.limit, 10),
-        type: options.type,
-        project: options.project,
-    });
-});
-program
-    .command('tag <id>')
-    .description('Update metadata on a note (description, project, scope)')
-    .option('-d, --description <text>', 'note description/purpose')
-    .option('-p, --project <name>', 'project name')
-    .option('-s, --scope <scope>', 'scope (user, system, general)')
-    .action(async (id, options) => {
-    const config = loadConfig();
-    await tag(config, parseInt(id, 10), {
-        description: options.description,
-        project: options.project,
-        scope: options.scope,
-    });
+    await init();
 });
 program
     .command('lint')
@@ -287,4 +243,4 @@ program
     .action(async (options) => {
     await lint({ personal: options.personal ?? false, diff: options.diff ?? false });
 });
-program.parse();
+program.parseAsync().then(() => shutdownObservability());

package/dist/commands/add.js

@@ -1,103 +1,54 @@
-import { loadConfigFile } from '../lib/config.js';
-import { opAddNote, getRegisteredTypes, isRegisteredType, registerType, validateTypeName, inferDelivery, NOTE_STATUSES } from '../lib/notes.js';
-import { ask, confirm, choose } from '../lib/prompt.js';
-export async function add(config, content, options) {
-    const configFile = loadConfigFile();
-    const interactive = configFile.naming?.interactive !== false;
-    let type = options.type || '';
-    const metadata = {};
-    if (options.project)
-        metadata.project = options.project;
-    if (options.upsertKey)
-        metadata.upsert_key = options.upsertKey;
-    if (options.description)
-        metadata.description = options.description;
-    if (options.status)
-        metadata.status = options.status;
-    // Interactive prompting for missing fields (CLI only)
-    if (interactive && !options.force) {
-        // Type
-        if (!type) {
-            const registeredTypes = getRegisteredTypes();
-            const typeChoice = await choose('What type of note is this?', [
-                ...registeredTypes,
-                'skip — use default (general)',
-            ]);
-            type = typeChoice.startsWith('skip') ? 'general' : typeChoice;
-        }
-        // Handle unknown type from --type flag
-        if (type && !isRegisteredType(type)) {
-            console.error(`\nType "${type}" is not registered.`);
-            const action = await choose('What would you like to do?', [
-                'register — register it now (pick a delivery tier)',
-                'existing — use an existing type instead',
-                'proceed — save anyway (defaults to "knowledge" delivery)',
-            ]);
-            if (action.startsWith('register')) {
-                const nameError = validateTypeName(type);
-                if (nameError) {
-                    console.error(nameError);
-                    process.exit(1);
-                }
-                const deliveryChoice = await choose('Delivery tier?', ['persona', 'project', 'knowledge', 'protected']);
-                registerType(type, deliveryChoice);
-                console.error(`Registered type "${type}" with delivery "${deliveryChoice}".`);
-            }
-            else if (action.startsWith('existing')) {
-                const registeredTypes = getRegisteredTypes();
-                type = await choose('Choose a type:', registeredTypes);
-            }
-            // 'proceed' — use the type as-is, will default to knowledge delivery
-        }
-        // Description
-        if (!metadata.description) {
-            const desc = await ask('One-line description (what is this note for?): ');
-            if (desc)
-                metadata.description = desc;
-        }
-        // upsert_key
-        if (!metadata.upsert_key) {
-            const key = await ask('Unique key for this note (lowercase-hyphenated, or Enter to auto-generate): ');
-            if (key)
-                metadata.upsert_key = key;
-        }
-        // Project
-        if (!metadata.project) {
-            const proj = await ask('Project name (or Enter to skip): ');
-            if (proj)
-                metadata.project = proj;
-        }
-        // Status (only for project-scoped types)
-        if (inferDelivery(type) === 'project' && !metadata.status) {
-            const statusChoice = await choose('What stage is this?', [
-                ...NOTE_STATUSES,
-                'skip — no status',
-            ]);
-            if (!statusChoice.startsWith('skip')) {
-                metadata.status = statusChoice;
-            }
-        }
+import { resolve } from 'path';
+import { existsSync } from 'fs';
+import { createDocumentFromFile, VerifyMismatchError } from '../lib/documents/operations.js';
+import { fatal, ExitCode } from '../lib/errors.js';
+// =============================================================================
+// Command
+// =============================================================================
+/**
+ * Add a new document by reading content from a file on disk. Auto-verified after create.
+ *
+ * Bytes flow file -> createDocumentFromFile() -> Postgres without retyping.
+ * The composed-string path (`-c`) was removed in Phase 4 of the
+ * file-based-write-api rollout to close the drift class of bug.
+ *
+ * Drift surfaces as VerifyMismatchError on stderr with exit code VERIFY_MISMATCH.
+ * The document still exists on verify failure (audit_log preserves the create
+ * event for manual cleanup if needed).
+ */
+export async function addDocumentFromFile(config, options) {
+    const absPath = resolve(options.filePath);
+    if (!existsSync(absPath)) {
+        fatal(`File not found: ${absPath}`, ExitCode.FILE_NOT_FOUND);
     }
-    // Default type if still empty
-    if (!type)
-        type = 'general';
-    // Mark as having gone through interactive (or skipped it)
-    // so opAddNote doesn't re-prompt via MCP confirm flow
-    metadata.interactive_skip = true;
-    const result = await opAddNote({ supabase: config.supabase, openai: config.openai }, content, type, options.agent || 'cli', metadata, options.force ?? false);
-    if (result.status === 'confirm') {
-        console.error(result.message);
-        const proceed = await confirm('\nCreate new note anyway?');
-        if (proceed) {
-            const forced = await opAddNote({ supabase: config.supabase, openai: config.openai }, content, type, options.agent || 'cli', { ...metadata, interactive_skip: true }, true);
-            console.error(forced.message);
-        }
-        else {
-            console.error('Cancelled.');
-        }
-        return;
+    const clients = {
+        supabase: config.supabase,
+        openai: config.openai,
+    };
+    process.stderr.write(`Adding document "${options.name}" from ${absPath} (${options.domain}/${options.documentType})...\n`);
+    try {
+        const result = await createDocumentFromFile(clients, {
+            filePath: absPath,
+            name: options.name,
+            domain: options.domain,
+            document_type: options.documentType,
+            description: options.description,
+            project: options.project,
+            agent: options.agent,
+            status: options.status,
+            protection: options.protection,
+        });
+        process.stdout.write(`${result.id}\n`);
+        process.stderr.write(`Document created (id: ${result.id}, ${result.bytes} bytes, verified)\n`);
+    }
+    catch (error) {
+        if (error instanceof VerifyMismatchError) {
+            process.stderr.write(`\nVerify failed on document ${error.id}.\n`);
+            process.stderr.write(`Pushed ${error.expectedLength} bytes, pulled ${error.actualLength} bytes.\n`);
+            process.stderr.write(`${error.diffPreview}\n`);
+            process.stderr.write(`The document was created but the round-trip diff caught drift. Inspect the doc and re-run if needed.\n`);
+            process.exit(ExitCode.VERIFY_MISMATCH);
+        }
+        throw error;
     }
-    console.error(result.message);
-    if (result.status === 'error')
-        process.exit(1);
 }