@aperdomoll90/ledger-ai 1.4.0 → 1.4.2

package/dist/scripts/hunter-cron.js

@@ -0,0 +1,63 @@
+import { loadConfig, loadConfigFile } from '../lib/config.js';
+import { fetchAllFeeds } from '../lib/hunter-rss.js';
+import { getExistingUrls, insertRawListings } from '../lib/hunter-db.js';
+import { deduplicateListings } from '../lib/hunter-rss.js';
+import { DEFAULT_HUNTER_CONFIG } from '../lib/hunter-types.js';
+export function buildCronSummary(stats) {
+    return `[hunter-cron] ${stats.feedsChecked} feeds checked, ${stats.totalFound} listings found, ${stats.newStored} new stored, ${stats.skippedDuplicates} duplicates skipped`;
+}
+export async function runHunterCron() {
+    const config = loadConfig();
+    const fileConfig = loadConfigFile();
+    const hunterConfig = {
+        ...DEFAULT_HUNTER_CONFIG,
+        ...fileConfig.hunter,
+    };
+    const feeds = hunterConfig.feeds;
+    if (feeds.length === 0) {
+        console.error('[hunter-cron] No feeds configured. Run `ledger config set hunter.feeds`.');
+        return { feedsChecked: 0, totalFound: 0, newStored: 0, skippedDuplicates: 0 };
+    }
+    const allListings = await fetchAllFeeds(feeds);
+    const feedsChecked = feeds.length;
+    const totalFound = allListings.length;
+    if (totalFound === 0) {
+        const stats = { feedsChecked, totalFound: 0, newStored: 0, skippedDuplicates: 0 };
+        console.error(buildCronSummary(stats));
+        return stats;
+    }
+    const urls = allListings.map(l => l.url);
+    const existingUrls = await getExistingUrls(config.supabase, urls);
+    const newListings = deduplicateListings(allListings, existingUrls);
+    let newStored = 0;
+    if (newListings.length > 0) {
+        for (const feed of feeds) {
+            const feedListings = newListings.filter(l => l.source === feed.name);
+            if (feedListings.length > 0) {
+                const result = await insertRawListings(config.supabase, feedListings, feed.track);
+                newStored += result.inserted;
+            }
+        }
+    }
+    const stats = {
+        feedsChecked,
+        totalFound,
+        newStored,
+        skippedDuplicates: totalFound - newListings.length,
+    };
+    console.error(buildCronSummary(stats));
+    return stats;
+}
+const isDirectExecution = process.argv[1]?.endsWith('hunter-cron.js')
+    || process.argv[1]?.endsWith('hunter-cron.ts');
+if (isDirectExecution) {
+    runHunterCron()
+        .then(stats => {
+        if (stats.newStored > 0)
+            console.log(JSON.stringify(stats));
+    })
+        .catch(err => {
+        console.error(`[hunter-cron] Error: ${err.message}`);
+        process.exit(1);
+    });
+}

package/dist/scripts/hunter-purge.js

@@ -0,0 +1,25 @@
+import { loadConfig, loadConfigFile } from '../lib/config.js';
+import { purgeOldRejected } from '../lib/hunter-db.js';
+import { DEFAULT_HUNTER_CONFIG } from '../lib/hunter-types.js';
+export async function runPurge(purgeDays) {
+    const config = loadConfig();
+    const fileConfig = loadConfigFile();
+    const hunterConfig = {
+        ...DEFAULT_HUNTER_CONFIG,
+        ...fileConfig.hunter,
+    };
+    const days = purgeDays ?? hunterConfig.purge_after_days;
+    console.error(`[hunter-purge] Purging rejected opportunities older than ${days} days...`);
+    const count = await purgeOldRejected(config.supabase, days);
+    console.error(`[hunter-purge] Purged ${count} rejected opportunities.`);
+    return count;
+}
+const isDirectExecution = process.argv[1]?.endsWith('hunter-purge.js')
+    || process.argv[1]?.endsWith('hunter-purge.ts');
+if (isDirectExecution) {
+    const days = process.argv[2] ? parseInt(process.argv[2], 10) : undefined;
+    runPurge(days).catch(err => {
+        console.error(`[hunter-purge] Error: ${err.message}`);
+        process.exit(1);
+    });
+}

package/dist/scripts/migrate-v2.js

@@ -0,0 +1,140 @@
+// migrate-v2.ts
+// One-time migration: read from old `notes` table, write to new `documents` table.
+// Each note becomes a document via createDocument() RPC (chunk + embed + audit).
+//
+// Run: npx tsx src/scripts/migrate-v2.ts
+// Dry run: npx tsx src/scripts/migrate-v2.ts --dry-run
+import 'dotenv/config';
+import { createClient } from '@supabase/supabase-js';
+import OpenAI from 'openai';
+import { createDocument } from '../lib/document-operations.js';
+// =============================================================================
+// Setup
+// =============================================================================
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+const openaiKey = process.env.OPENAI_API_KEY;
+if (!supabaseUrl || !supabaseKey || !openaiKey) {
+    console.error('Missing SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, or OPENAI_API_KEY');
+    process.exit(1);
+}
+const clients = {
+    supabase: createClient(supabaseUrl, supabaseKey),
+    openai: new OpenAI({ apiKey: openaiKey }),
+};
+const DRY_RUN = process.argv.includes('--dry-run');
+// Valid values for constrained fields — reject anything outside these
+const VALID_DOMAINS = new Set(['system', 'persona', 'workspace', 'project', 'general']);
+const VALID_PROTECTIONS = new Set(['open', 'guarded', 'protected', 'immutable']);
+const VALID_OWNER_TYPES = new Set(['system', 'user', 'team']);
+const VALID_STATUSES = new Set(['idea', 'planning', 'active', 'done']);
+const VALID_SOURCE_TYPES = new Set(['text', 'pdf', 'docx', 'spreadsheet', 'code', 'image', 'audio', 'video', 'web', 'email', 'slides', 'handwriting']);
+// =============================================================================
+// Field mapping
+// =============================================================================
+function mapNoteToDocument(note) {
+    const meta = note.metadata;
+    const domain = typeof meta.domain === 'string' && VALID_DOMAINS.has(meta.domain) ? meta.domain : 'general';
+    const protection = typeof meta.protection === 'string' && VALID_PROTECTIONS.has(meta.protection) ? meta.protection : 'open';
+    const ownerType = typeof meta.owner_type === 'string' && VALID_OWNER_TYPES.has(meta.owner_type) ? meta.owner_type : 'user';
+    const status = typeof meta.status === 'string' && VALID_STATUSES.has(meta.status) ? meta.status : undefined;
+    const sourceType = typeof meta.source_type === 'string' && VALID_SOURCE_TYPES.has(meta.source_type) ? meta.source_type : 'text';
+    return {
+        name: meta.upsert_key ?? `note-${note.id}`,
+        domain,
+        document_type: meta.type ?? 'reference',
+        content: note.content,
+        description: meta.description,
+        project: meta.project,
+        protection,
+        owner_type: ownerType,
+        owner_id: meta.owner_id,
+        is_auto_load: meta.auto_load === true,
+        source_type: sourceType,
+        file_path: meta.file_path,
+        file_permissions: meta.file_permissions,
+        agent: 'migrate-v2',
+        status,
+        skill_ref: meta.skill_ref,
+        embedding_model_id: 'openai/text-embedding-3-small',
+    };
+}
+// =============================================================================
+// Migration
+// =============================================================================
+async function migrate() {
+    console.log(`\n${'='.repeat(60)}`);
+    console.log(`Ledger v2 Migration${DRY_RUN ? ' (DRY RUN)' : ''}`);
+    console.log(`${'='.repeat(60)}\n`);
+    // Read all notes
+    const { data: notes, error } = await clients.supabase
+        .from('notes')
+        .select('id, content, metadata, created_at')
+        .order('id', { ascending: true });
+    if (error) {
+        console.error('Failed to read notes:', error.message);
+        process.exit(1);
+    }
+    if (!notes || notes.length === 0) {
+        console.log('No notes found. Nothing to migrate.');
+        return;
+    }
+    console.log(`Found ${notes.length} notes to migrate.\n`);
+    const results = [];
+    let succeeded = 0;
+    let failed = 0;
+    let skipped = 0;
+    for (const note of notes) {
+        const props = mapNoteToDocument(note);
+        const index = notes.indexOf(note);
+        const total = notes.length;
+        const pct = Math.round(((index + 1) / total) * 100);
+        const filled = Math.round(pct / 2.5);
+        const bar = '█'.repeat(filled) + '░'.repeat(40 - filled);
+        if (DRY_RUN) {
+            process.stdout.write(`\r[${bar}] ${pct}% (${index + 1}/${total}) ${props.name.slice(0, 40).padEnd(40)}`);
+            results.push({ oldId: note.id, name: props.name });
+            skipped++;
+            continue;
+        }
+        try {
+            const newId = await createDocument(clients, props);
+            process.stdout.write(`\r[${bar}] ${pct}% (${index + 1}/${total}) ${props.name.slice(0, 40).padEnd(40)}`);
+            results.push({ oldId: note.id, name: props.name, newId });
+            succeeded++;
+        }
+        catch (err) {
+            const message = err.message;
+            process.stdout.write(`\r[${bar}] ${pct}% (${index + 1}/${total}) FAIL: ${props.name.slice(0, 30)}\n`);
+            results.push({ oldId: note.id, name: props.name, error: message });
+            failed++;
+        }
+    }
+    // Clear progress bar line
+    process.stdout.write('\n');
+    // Summary
+    console.log(`\n${'='.repeat(60)}`);
+    console.log('Migration Summary');
+    console.log(`${'='.repeat(60)}`);
+    console.log(`Total notes:  ${notes.length}`);
+    if (DRY_RUN) {
+        console.log(`Dry run:      ${skipped} would be migrated`);
+    }
+    else {
+        console.log(`Succeeded:    ${succeeded}`);
+        console.log(`Failed:       ${failed}`);
+    }
+    console.log();
+    if (failed > 0) {
+        console.log('Failed notes:');
+        for (const result of results) {
+            if (result.error) {
+                console.log(`  note ${result.oldId} "${result.name}": ${result.error}`);
+            }
+        }
+    }
+}
+migrate().catch((err) => {
+    console.error('Migration crashed:', err);
+    process.exit(1);
+});

package/dist/scripts/reindex.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env npx tsx
+// reindex.ts
+// Bulk re-index all documents through the new chunking + enrichment pipeline.
+// Reads all active documents, re-chunks with recursive splitter, generates
+// context summaries via gpt-4o-mini, re-embeds with enriched vectors, and
+// calls document_update RPC (which versions old content before overwriting).
+//
+// Usage:
+//   npx tsx src/scripts/reindex.ts              # dry-run (default — shows what would change)
+//   npx tsx src/scripts/reindex.ts --execute    # actually re-index all documents
+//   npx tsx src/scripts/reindex.ts --id 42      # re-index one document (dry-run)
+//   npx tsx src/scripts/reindex.ts --id 42 --execute  # re-index one document (write)
+import 'dotenv/config';
+import { loadConfig } from '../lib/config.js';
+import { updateDocument } from '../lib/documents/operations.js';
+async function main() {
+    const dryRun = !process.argv.includes('--execute');
+    const singleIdFlag = process.argv.indexOf('--id');
+    const singleId = singleIdFlag !== -1 ? Number(process.argv[singleIdFlag + 1]) : null;
+    const config = loadConfig();
+    const clients = {
+        supabase: config.supabase,
+        openai: config.openai,
+    };
+    console.error(dryRun ? '=== DRY RUN (pass --execute to write) ===' : '=== EXECUTING RE-INDEX ===');
+    // Fetch documents
+    let query = config.supabase
+        .from('documents')
+        .select('id, name, content, chunk_count')
+        .is('deleted_at', null)
+        .order('id', { ascending: true });
+    if (singleId !== null) {
+        query = query.eq('id', singleId);
+    }
+    const { data: documents, error } = await query;
+    if (error)
+        throw new Error(`Failed to fetch documents: ${error.message}`);
+    const documentList = documents;
+    console.error(`Found ${documentList.length} documents to re-index\n`);
+    let successCount = 0;
+    let failureCount = 0;
+    for (const document of documentList) {
+        const contentLength = document.content?.length ?? 0;
+        const estimatedChunks = Math.max(1, Math.ceil(contentLength / 1000));
+        if (dryRun) {
+            console.error(`[DRY] #${document.id} ${document.name} — ${contentLength} chars, ${document.chunk_count} chunks → ~${estimatedChunks} chunks`);
+            successCount++;
+            continue;
+        }
+        try {
+            console.error(`[${successCount + failureCount + 1}/${documentList.length}] #${document.id} ${document.name} — re-indexing...`);
+            await updateDocument(clients, {
+                id: document.id,
+                content: document.content,
+                agent: 'reindex-script',
+            });
+            successCount++;
+            console.error(`  done (${contentLength} chars → ~${estimatedChunks} chunks)`);
+        }
+        catch (reindexError) {
+            failureCount++;
+            console.error(`  FAILED: ${reindexError instanceof Error ? reindexError.message : String(reindexError)}`);
+        }
+    }
+    console.error(`\n=== Summary ===`);
+    console.error(`Success: ${successCount} | Failed: ${failureCount} | Total: ${documentList.length}`);
+    if (dryRun) {
+        console.error('This was a dry run. Pass --execute to actually re-index.');
+    }
+}
+main().catch((error) => {
+    console.error('Fatal error:', error);
+    process.exit(1);
+});

package/dist/scripts/sync-local-docs.js

@@ -0,0 +1,153 @@
+// sync-local-docs.ts
+// Push local docs/* files to their matching Ledger documents.
+// Handles any file size — chunks, embeds, and updates via the proper
+// updateDocument() pipeline (not the MCP tool).
+//
+// Usage:
+//   npx tsx src/scripts/sync-local-docs.ts                    # sync all known docs
+//   npx tsx src/scripts/sync-local-docs.ts --file docs/foo.md # sync one file
+//   npx tsx src/scripts/sync-local-docs.ts --dry-run          # show what would sync
+import 'dotenv/config';
+import { createClient } from '@supabase/supabase-js';
+import OpenAI from 'openai';
+import { readFileSync, existsSync } from 'fs';
+import { createDocument, updateDocument } from '../lib/documents/operations.js';
+const supabaseUrl = process.env.SUPABASE_URL;
+const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY;
+const openaiKey = process.env.OPENAI_API_KEY;
+if (!supabaseUrl || !supabaseKey || !openaiKey) {
+    console.error('Missing SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, or OPENAI_API_KEY');
+    process.exit(1);
+}
+const supabase = createClient(supabaseUrl, supabaseKey);
+const openai = new OpenAI({ apiKey: openaiKey });
+const clients = { supabase, openai, cohereApiKey: undefined };
+const dryRun = process.argv.includes('--dry-run');
+const fileArgIdx = process.argv.indexOf('--file');
+const singleFile = fileArgIdx >= 0 ? process.argv[fileArgIdx + 1] : null;
+// Map of local file paths to Ledger document names.
+// Add entries here as docs are created.
+const FILE_TO_DOC = {
+    'docs/ledger-architecture.md': 'ledger-architecture',
+    'docs/ledger-architecture-database.md': 'ledger-architecture-database',
+    'docs/ledger-architecture-database-tables.md': 'ledger-architecture-database-tables',
+    'docs/ledger-architecture-database-schemas.md': 'ledger-architecture-database-schemas',
+    'docs/ledger-architecture-database-indexes.md': 'ledger-architecture-database-indexes',
+    'docs/ledger-architecture-database-functions.md': 'ledger-architecture-database-functions',
+    'docs/reference-rag-system-architecture.md': 'reference-rag-system-architecture',
+    'docs/reference-rag-core-ingestion.md': 'reference-rag-core-ingestion',
+    'docs/reference-rag-core-query-pipeline.md': 'reference-rag-core-query-pipeline',
+    'docs/reference-rag-core-database-schemas.md': 'reference-rag-core-database-schemas',
+    'docs/reference-rag-quality-evaluation.md': 'reference-rag-quality-evaluation',
+    'docs/reference-rag-quality-improvement.md': 'reference-rag-quality-improvement',
+    'docs/reference-rag-security-access-control.md': 'reference-rag-security-access-control',
+    'docs/reference-rag-security-defenses.md': 'reference-rag-security-defenses',
+    'docs/reference-rag-operations-observability.md': 'reference-rag-operations-observability',
+    'docs/reference-rag-operations-scaling.md': 'reference-rag-operations-scaling',
+    'docs/reference-rag-operations-deployment.md': 'reference-rag-operations-deployment',
+    'docs/reference-rag-interface-api.md': 'reference-rag-interface-api',
+};
+// Metadata for new docs that don't exist in Ledger yet.
+// Only used when creating; updates skip this.
+const NEW_DOC_META = {
+    'reference-rag-core-ingestion': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: ingestion pipeline (extraction, chunking, enrichment, embedding, storage)' },
+    'reference-rag-core-query-pipeline': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: query pipeline (embedding, retrieval, reranking, generation)' },
+    'reference-rag-core-database-schemas': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: production database schemas (SQL for all tables, indexes, functions, triggers, RLS)' },
+    'reference-rag-quality-evaluation': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: evaluation framework (metrics, golden datasets, graded relevance, eval runners)' },
+    'reference-rag-quality-improvement': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: quality optimization levers (reranker, enrichment, chunking, thresholds)' },
+    'reference-rag-security-access-control': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: access control (RLS, RBAC, JWT auth, document permissions)' },
+    'reference-rag-security-defenses': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: security defenses (prompt injection, PII, content sanitization)' },
+    'reference-rag-operations-observability': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: observability (latency tracking, cost monitoring, cache analytics)' },
+    'reference-rag-operations-scaling': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: scaling (connection pooling, index tuning, sharding, rate limiting)' },
+    'reference-rag-operations-deployment': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: deployment (CI/CD, migrations, rollback, monitoring)' },
+    'reference-rag-interface-api': { domain: 'general', documentType: 'knowledge-guide', description: 'RAG reference: API layer (MCP tools, REST endpoints, SDK design)' },
+};
+async function main() {
+    const filesToSync = singleFile
+        ? { [singleFile]: FILE_TO_DOC[singleFile] }
+        : FILE_TO_DOC;
+    let synced = 0;
+    let skipped = 0;
+    let errors = 0;
+    for (const [filePath, docName] of Object.entries(filesToSync)) {
+        if (!docName) {
+            console.error(`  [SKIP] ${filePath} — not in FILE_TO_DOC mapping`);
+            skipped++;
+            continue;
+        }
+        if (!existsSync(filePath)) {
+            console.error(`  [SKIP] ${filePath} — file not found`);
+            skipped++;
+            continue;
+        }
+        // Look up the document id by name
+        const { data: doc, error: lookupError } = await supabase
+            .from('documents')
+            .select('id, content_hash')
+            .eq('name', docName)
+            .single();
+        const localContent = readFileSync(filePath, 'utf8');
+        if (lookupError || !doc) {
+            // Document doesn't exist in Ledger yet. Create it if we have metadata.
+            const meta = NEW_DOC_META[docName];
+            if (!meta) {
+                console.error(`  [SKIP] ${filePath} — no Ledger document named "${docName}" and no creation metadata`);
+                skipped++;
+                continue;
+            }
+            if (dryRun) {
+                console.log(`  [DRY-CREATE] ${filePath} → NEW "${docName}" (${localContent.length} chars)`);
+                synced++;
+                continue;
+            }
+            console.log(`  [CREATE] ${filePath} → "${docName}" (${localContent.length} chars)...`);
+            try {
+                const newId = await createDocument(clients, {
+                    name: docName,
+                    domain: meta.domain,
+                    document_type: meta.documentType,
+                    content: localContent,
+                    description: meta.description,
+                    project: meta.project,
+                    agent: 'sync-local-docs',
+                });
+                console.log(`           created #${newId}.`);
+                synced++;
+            }
+            catch (createError) {
+                const message = createError instanceof Error ? createError.message : String(createError);
+                console.error(`  [ERR] ${filePath}: ${message}`);
+                errors++;
+            }
+            continue;
+        }
+        if (dryRun) {
+            console.log(`  [DRY] ${filePath} → #${doc.id} ${docName} (${localContent.length} chars)`);
+            synced++;
+            continue;
+        }
+        console.log(`  [SYNC] ${filePath} → #${doc.id} ${docName} (${localContent.length} chars)...`);
+        try {
+            await updateDocument(clients, {
+                id: doc.id,
+                content: localContent,
+                agent: 'sync-local-docs',
+            });
+            console.log(`         done.`);
+            synced++;
+        }
+        catch (syncError) {
+            const message = syncError instanceof Error ? syncError.message : String(syncError);
+            console.error(`  [ERR] ${filePath}: ${message}`);
+            errors++;
+        }
+    }
+    console.log('');
+    console.log(`Synced: ${synced}, Skipped: ${skipped}, Errors: ${errors}`);
+    if (dryRun)
+        console.log('[DRY RUN] No writes performed.');
+}
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aperdomoll90/ledger-ai",
-  "version": "1.4.0",
+  "version": "1.4.2",
   "description": "AI identity and memory system — portable persona, knowledge sync, semantic search across agents and devices",
   "type": "module",
   "bin": {
@@ -47,8 +47,14 @@
     "node": ">=20"
   },
   "dependencies": {
+    "@langfuse/openai": "^5.1.0",
+    "@langfuse/otel": "^5.1.0",
+    "@langfuse/tracing": "^5.1.0",
     "@modelcontextprotocol/sdk": "^1.27.1",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/sdk-trace-node": "^2.6.1",
     "@supabase/supabase-js": "^2.99.1",
+    "bottleneck": "^2.19.5",
     "commander": "^14.0.3",
     "dotenv": "^17.3.1",
     "openai": "^6.29.0",