@rbalchii/anchor-engine 4.7.0

package/engine/dist/services/ingest/ingest.js

@@ -0,0 +1,230 @@
+/**
+ * Ingest Service - Memory Ingestion with Provenance Tracking
+ *
+ * Implements the Data Provenance feature by adding a 'provenance' column
+ * to distinguish between "Sovereign" (User-Created) and "Ancillary" (External) data.
+ */
+import { db } from '../../core/db.js';
+import crypto from 'crypto';
+import { config } from '../../config/index.js';
+import { cleanContent, getCleaningStats } from './content-cleaner.js';
+/**
+ * Determines the provenance of content based on its source
+ */
+function determineProvenance(source, type) {
+    const normalizedSource = source.replace(/\\/g, '/');
+    // 1. Explicit Trusted Inbox (or default 'inbox' folder)
+    // Matches "inbox/..." or ".../inbox/..."
+    if (normalizedSource.includes('/inbox/') || normalizedSource.startsWith('inbox/') ||
+        normalizedSource.includes('/internal-inbox/') || normalizedSource.startsWith('internal-inbox/') ||
+        normalizedSource.includes('/sovereign/') ||
+        type === 'user') {
+        return 'internal';
+    }
+    // 2. Explicit External Inbox
+    // Matches "external-inbox/..." or ".../external-inbox/..."
+    if (normalizedSource.includes('/external-inbox/') || normalizedSource.startsWith('external-inbox/') ||
+        normalizedSource.includes('web_scrape') ||
+        normalizedSource.includes('news_agent') ||
+        type === 'external') {
+        return 'external';
+    }
+    // Default to external only if it didn't match the explicitly internal folders above
+    // Note: We flipped the order to prioritize the known 'inbox' check which was failing before (falling through to default)
+    return 'external';
+}
+/**
+ * Ingest content into the memory database with provenance tracking
+ */
+export async function ingestContent(content, source, type = 'text', buckets = ['core'], tags = [], options = {}) {
+    if (!content) {
+        throw new Error('Content is required for ingestion');
+    }
+    // --- DATA REFINERY: Clean content ---
+    let processedContent = content;
+    const processedTags = [...tags];
+    if (!options.skipCleaning) {
+        const startTime = Date.now();
+        // Configure cleaning based on content type
+        const cleanOptions = {
+            stripHtml: type === 'html' || type === 'web_page',
+            decodeHtml: true,
+            normalizeUnicode: true,
+            removeControlChars: true,
+            normalizeWhitespace: true,
+            removeBoilerplate: type === 'web_page',
+            normalizeLineEndings: true,
+            collapseBlankLines: true
+        };
+        processedContent = cleanContent(content, cleanOptions);
+        const stats = getCleaningStats(content, processedContent);
+        if (stats.charsRemoved > 0) {
+            console.log(`[Ingest] 🏭 Content cleaned: -${stats.charsRemoved} chars (${stats.reductionPercent}% reduction) in ${Date.now() - startTime}ms`);
+        }
+    }
+    // Auto-assign provenance based on source
+    const provenance = determineProvenance(source, type);
+    // Generate hash for content deduplication (using processed content)
+    const hash = crypto.createHash('md5').update(processedContent).digest('hex');
+    // Check if content with same hash already exists
+    const existingQuery = `SELECT id FROM atoms WHERE simhash = $1`;
+    const existingResult = await db.run(existingQuery, [BigInt(hash)]);
+    if (existingResult.rows && existingResult.rows.length > 0) {
+        return {
+            status: 'skipped',
+            id: existingResult.rows[0][0],
+            message: 'Content with same hash already exists'
+        };
+    }
+    // Generate unique ID
+    const id = `mem_${Date.now()}_${crypto.randomBytes(8).toString('hex').substring(0, 16)}`;
+    const timestamp = Date.now();
+    // Process content into atomic structure using AtomizerService (Legacy Pipeline)
+    const { AtomizerService } = await import('./atomizer-service.js');
+    const { AtomicIngestService } = await import('./ingest-atomic.js');
+    const atomizer = new AtomizerService();
+    const atomicIngest = new AtomicIngestService();
+    // Ensure provenance matches expected type for atomizer
+    const atomizerProvenance = (provenance === 'system') ? 'internal' : provenance;
+    const atomizeResult = await atomizer.atomize(processedContent, source, atomizerProvenance, timestamp);
+    // Skip ingestion if transient data was detected
+    if (!atomizeResult) {
+        console.log(`[Ingest] ⚠️ SKIP: ${source} - Transient data detected, skipping ingestion`);
+        return {
+            status: 'skipped',
+            message: 'Content skipped (transient data)'
+        };
+    }
+    const { compound, molecules, atoms } = atomizeResult;
+    // Ingest result using AtomicIngestService
+    await atomicIngest.ingestResult(compound, molecules, atoms, buckets);
+    // Return success (ID is compound ID)
+    return {
+        status: 'success',
+        id: compound.id,
+        message: 'Content ingested successfully with provenance tracking'
+    };
+}
+/**
+ * Ingest pre-processed atoms
+ */
+/**
+ * Ingest pre-processed atoms (Batched)
+ */
+export async function ingestAtoms(atoms, source, buckets = ['core'], tags = [], // Batch-level tags (e.g., "inbox")
+fileTimestamp) {
+    if (atoms.length === 0)
+        return 0;
+    const BATCH_SIZE = 50;
+    let inserted = 0;
+    // Process in chunks
+    for (let i = 0; i < atoms.length; i += BATCH_SIZE) {
+        const chunk = atoms.slice(i, i + BATCH_SIZE);
+        // --- 1. Prepare Atoms Batch ---
+        const atomValuePlaceholders = [];
+        const atomParams = [];
+        let paramIndex = 1;
+        for (const atom of chunk) {
+            // Standard 096: Timestamp Assignment
+            let finalTimestamp = atom.timestamp;
+            if (!finalTimestamp || finalTimestamp <= 0 || isNaN(finalTimestamp)) {
+                finalTimestamp = (fileTimestamp != null) ? fileTimestamp : Date.now();
+            }
+            // Simhash to BigInt
+            let simhashBigInt = null;
+            if (atom.simhash) {
+                try {
+                    simhashBigInt = BigInt(atom.simhash);
+                }
+                catch (e) { /* ignore */ }
+            }
+            // Embedding
+            let embeddingArray = new Array(config.MODELS.EMBEDDING_DIM).fill(0.1);
+            if (atom.embedding && atom.embedding.length === config.MODELS.EMBEDDING_DIM) {
+                embeddingArray = atom.embedding;
+            }
+            // Payload
+            const payloadJson = atom.payload ? JSON.stringify(atom.payload) : '{}';
+            atomValuePlaceholders.push(`($${paramIndex}, $${paramIndex + 1}, $${paramIndex + 2}, $${paramIndex + 3}, $${paramIndex + 4}, $${paramIndex + 5}, $${paramIndex + 6}, $${paramIndex + 7})`);
+            atomParams.push(atom.id, atom.content, atom.sourcePath, finalTimestamp, simhashBigInt || 0n, embeddingArray, atom.provenance, payloadJson);
+            paramIndex += 8;
+        }
+        const atomInsertQuery = `
+      INSERT INTO atoms (id, content, source_path, timestamp, simhash, embedding, provenance, payload)
+      VALUES ${atomValuePlaceholders.join(', ')}
+      ON CONFLICT (id) DO UPDATE SET
+        content = EXCLUDED.content,
+        source_path = EXCLUDED.source_path,
+        timestamp = EXCLUDED.timestamp,
+        simhash = EXCLUDED.simhash,
+        embedding = EXCLUDED.embedding,
+        provenance = EXCLUDED.provenance,
+        payload = EXCLUDED.payload
+    `;
+        try {
+            await db.run(atomInsertQuery, atomParams);
+        }
+        catch (e) {
+            console.error(`[Ingest] Batch insert failed for chunk starting at index ${i}:`, e.message);
+            continue; // Skip tags if atoms fail
+        }
+        // --- 2. Prepare Tags Batch ---
+        const tagValuePlaceholders = [];
+        const tagParams = [];
+        let tagParamIndex = 1;
+        for (const atom of chunk) {
+            const atomSpecificTags = atom.tags || [];
+            const finalTags = [...new Set([...tags, ...atomSpecificTags])];
+            for (const tag of finalTags) {
+                tagValuePlaceholders.push(`($${tagParamIndex}, $${tagParamIndex + 1}, $${tagParamIndex + 2})`);
+                tagParams.push(atom.id, tag, buckets[0]);
+                tagParamIndex += 3;
+            }
+        }
+        if (tagParams.length > 0) {
+            const tagInsertQuery = `
+        INSERT INTO tags (atom_id, tag, bucket)
+        VALUES ${tagValuePlaceholders.join(', ')}
+        ON CONFLICT (atom_id, tag, bucket) DO UPDATE SET
+          bucket = EXCLUDED.bucket
+      `;
+            try {
+                await db.run(tagInsertQuery, tagParams);
+            }
+            catch (e) {
+                console.warn(`[Ingest] Batch tag insert failed for chunk ${i}:`, e.message);
+            }
+        }
+        inserted += chunk.length;
+    }
+    return inserted;
+}
+/**
+ * Bulk import YAML content with provenance tracking
+ */
+export async function importYamlContent(yamlContent) {
+    let imported = 0;
+    let skipped = 0;
+    let errors = 0;
+    for (const record of yamlContent) {
+        try {
+            if (!record.content) {
+                errors++;
+                continue;
+            }
+            const result = await ingestContent(record.content, record.source || 'yaml_import', record.type || 'text', record.buckets || ['imported'], record.tags || []);
+            if (result.status === 'success') {
+                imported++;
+            }
+            else if (result.status === 'skipped') {
+                skipped++;
+            }
+        }
+        catch (error) {
+            console.error('YAML import error for record:', record, error);
+            errors++;
+        }
+    }
+    return { imported, skipped, errors };
+}
+//# sourceMappingURL=ingest.js.map

package/engine/dist/services/ingest/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../src/services/ingest/ingest.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAOtE;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAc,EAAE,IAAa;IACxD,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAEpD,wDAAwD;IACxD,yCAAyC;IACzC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,gBAAgB,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC/E,gBAAgB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,gBAAgB,CAAC,UAAU,CAAC,iBAAiB,CAAC;QAC/F,gBAAgB,CAAC,QAAQ,CAAC,aAAa,CAAC;QACxC,IAAI,KAAK,MAAM,EAAE,CAAC;QAClB,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,6BAA6B;IAC7B,2DAA2D;IAC3D,IAAI,gBAAgB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,gBAAgB,CAAC,UAAU,CAAC,iBAAiB,CAAC;QACjG,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC;QACvC,gBAAgB,CAAC,QAAQ,CAAC,YAAY,CAAC;QACvC,IAAI,KAAK,UAAU,EAAE,CAAC;QACtB,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,oFAAoF;IACpF,yHAAyH;IACzH,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,MAAc,EACd,OAAe,MAAM,EACrB,UAAoB,CAAC,MAAM,CAAC,EAC5B,OAAiB,EAAE,EACnB,UAAyB,EAAE;IAG3B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,uCAAuC;IACvC,IAAI,gBAAgB,GAAG,OAAO,CAAC;IAC/B,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IAEhC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,2CAA2C;QAC3C,MAAM,YAAY,GAAG;YACnB,SAAS,EAAE,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,UAAU;YACjD,UAAU,EAAE,IAAI;YAChB,gBAAgB,EAAE,IAAI;YACtB,kBAAkB,EAAE,IAAI;YACxB,mBAAmB,EAAE,IAAI;YACzB,iBAAiB,EAAE,IAAI,KAAK,UAAU;YACtC,oBAAoB,EAAE,IAAI;YAC1B,kBAAkB,EAAE,IAAI;SACzB,CAAC;QAEF,gBAAgB,GAAG,YAAY,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAE1D,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,iCAAiC,KAAK,CAAC,YAAY,WAAW,KAAK,CAAC,gBAAgB,mBAAmB,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,IAAI,CAAC,CAAC;QACjJ,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAErD,oEAAoE;IACpE,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE7E,iDAAiD;IACjD,MAAM,aAAa,GAAG,yCAAyC,CAAC;IAChE,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEnE,IAAI,cAAc,CAAC,IAAI,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,EAAE,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,OAAO,EAAE,uCAAuC;SACjD,CAAC;IACJ,CAAC;IAED,qBAAqB;IACrB,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACzF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,gFAAgF;IAChF,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAClE,MAAM,EAAE,mBAAmB,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAEnE,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;IACvC,MAAM,YAAY,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAE/C,uDAAuD;IACvD,MAAM,kBAAkB,GAAG,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;IAE/E,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,OAAO,CAC1C,gBAAgB,EAChB,MAAM,EACN,kBAAkB,EAClB,SAAS,CACV,CAAC;IAEF,gDAAgD;IAChD,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,gDAAgD,CAAC,CAAC;QACzF,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,kCAAkC;SAC5C,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC;IAErD,0CAA0C;IAC1C,MAAM,YAAY,CAAC,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAErE,qCAAqC;IACrC,OAAO;QACL,MAAM,EAAE,SAAS;QACjB,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,OAAO,EAAE,wDAAwD;KAClE,CAAC;AACJ,CAAC;AAiBD;;GAEG;AACH;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAmB,EACnB,MAAc,EACd,UAAoB,CAAC,MAAM,CAAC,EAC5B,OAAiB,EAAE,EAAE,mCAAmC;AACxD,aAAsB;IAGtB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,oBAAoB;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAE7C,iCAAiC;QACjC,MAAM,qBAAqB,GAAa,EAAE,CAAC;QAC3C,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,qCAAqC;YACrC,IAAI,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC;YACpC,IAAI,CAAC,cAAc,IAAI,cAAc,IAAI,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;gBACpE,cAAc,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACxE,CAAC;YAED,oBAAoB;YACpB,IAAI,aAAa,GAAkB,IAAI,CAAC;YACxC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC;oBAAC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAAC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1E,CAAC;YAED,YAAY;YACZ,IAAI,cAAc,GAAa,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChF,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC5E,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC;YAClC,CAAC;YAED,UAAU;YACV,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEvE,qBAAqB,CAAC,IAAI,CAAC,KAAK,UAAU,MAAM,UAAU,GAAG,CAAC,MAAM,UAAU,GAAG,CAAC,MAAM,UAAU,GAAG,CAAC,MAAM,UAAU,GAAG,CAAC,MAAM,UAAU,GAAG,CAAC,MAAM,UAAU,GAAG,CAAC,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3L,UAAU,CAAC,IAAI,CACb,IAAI,CAAC,EAAE,EACP,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,UAAU,EACf,cAAc,EACd,aAAa,IAAI,EAAE,EACnB,cAAc,EACd,IAAI,CAAC,UAAU,EACf,WAAW,CACZ,CAAC;YACF,UAAU,IAAI,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,eAAe,GAAG;;eAEb,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;KAS1C,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,GAAG,CAAC,eAAe,EAAE,UAAU,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3F,SAAS,CAAC,0BAA0B;QACtC,CAAC;QAED,gCAAgC;QAChC,MAAM,oBAAoB,GAAa,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAU,EAAE,CAAC;QAC5B,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAE/D,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,oBAAoB,CAAC,IAAI,CAAC,KAAK,aAAa,MAAM,aAAa,GAAG,CAAC,MAAM,aAAa,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC/F,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzC,aAAa,IAAI,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,cAAc,GAAG;;iBAEZ,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC;;;OAGzC,CAAC;YACF,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAC1C,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,8CAA8C,CAAC,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;QAED,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,WAAkB;IACxD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,EAAE,CAAC;gBACT,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,aAAa,CAChC,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,MAAM,IAAI,aAAa,EAC9B,MAAM,CAAC,IAAI,IAAI,MAAM,EACrB,MAAM,CAAC,OAAO,IAAI,CAAC,UAAU,CAAC,EAC9B,MAAM,CAAC,IAAI,IAAI,EAAE,CAClB,CAAC;YAEF,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,QAAQ,EAAE,CAAC;YACb,CAAC;iBAAM,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvC,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YAC9D,MAAM,EAAE,CAAC;QACX,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC"}

package/engine/dist/services/ingest/watchdog.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Watchdog Service
+ *
+ * Scans the Notebook directory for changes and ingests new content.
+ * Uses 'chokidar' for efficient file watching.
+ */
+export declare function startWatchdog(): Promise<void>;
+export declare function getWatchedPaths(): string[];
+export declare function addWatchPath(newPath: string): Promise<boolean>;
+export declare function removeWatchPath(pathToRemove: string): Promise<boolean>;
+/**
+ * Stop the watchdog service
+ */
+export declare function stopWatchdog(): Promise<void>;
+/**
+ * Get watchdog status
+ */
+export declare function getWatcherStatus(): {
+    isRunning: boolean;
+    watchedPaths: string[];
+};
+/**
+ * Trigger manual ingestion scan
+ */
+export declare function triggerManualIngest(): Promise<{
+    status: string;
+    message: string;
+    filesProcessed?: number;
+    filesIngested?: number;
+}>;
+//# sourceMappingURL=watchdog.d.ts.map

package/engine/dist/services/ingest/watchdog.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"watchdog.d.ts","sourceRoot":"","sources":["../../../src/services/ingest/watchdog.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA8CH,wBAAsB,aAAa,kBAkDlC;AAGD,wBAAgB,eAAe,IAAI,MAAM,EAAE,CAY1C;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAwCpE;AAED,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAgC5E;AAED;;GAEG;AACH,wBAAsB,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC,CAMlD;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI;IAAE,SAAS,EAAE,OAAO,CAAC;IAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CASjF;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAkEzI"}

package/engine/dist/services/ingest/watchdog.js

@@ -0,0 +1,400 @@
+/**
+ * Watchdog Service
+ *
+ * Scans the Notebook directory for changes and ingests new content.
+ * Uses 'chokidar' for efficient file watching.
+ */
+import * as chokidar from 'chokidar';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { db } from '../../core/db.js';
+import { NOTEBOOK_DIR, PROJECT_ROOT } from '../../config/paths.js';
+import { config } from '../../config/index.js';
+import { pathManager } from '../../utils/path-manager.js';
+import { systemStatus } from '../system-status.js';
+let watcher = null;
+const IGNORE_PATTERNS = /(^|[\/\\])\../; // Ignore dotfiles
+// Post-ingestion synonym generation
+let ingestionTimeout = null;
+const INGESTION_DEBOUNCE_MS = 30000; // Wait 30 seconds after last ingestion
+async function triggerPostIngestionSynonyms() {
+    // Clear any pending timeout
+    if (ingestionTimeout) {
+        clearTimeout(ingestionTimeout);
+    }
+    // Set new timeout to generate synonyms after ingestion stops
+    ingestionTimeout = setTimeout(async () => {
+        console.log('[Watchdog] Post-ingestion synonym generation starting...');
+        try {
+            const { AutoSynonymGenerator } = await import('../synonyms/auto-synonym-generator.js');
+            const generator = new AutoSynonymGenerator();
+            const synonyms = await generator.generateSynonymRings();
+            const synonymDir = path.join(pathManager.getDatabasePath(), 'synonyms');
+            if (!fs.existsSync(synonymDir)) {
+                fs.mkdirSync(synonymDir, { recursive: true });
+            }
+            const synonymPath = path.join(synonymDir, 'synonym-ring-auto.json');
+            await generator.saveSynonymRings(synonyms, synonymPath);
+            console.log(`[Watchdog] ✅ Post-ingestion synonym rings saved to ${synonymPath}`);
+        }
+        catch (error) {
+            console.warn('[Watchdog] Post-ingestion synonym generation failed:', error.message);
+        }
+    }, INGESTION_DEBOUNCE_MS);
+}
+export async function startWatchdog() {
+    if (watcher)
+        return;
+    if (!fs.existsSync(NOTEBOOK_DIR)) {
+        console.warn(`[Watchdog] Notebook directory not found: ${NOTEBOOK_DIR}. Skipping watch.`);
+        return;
+    }
+    const inbox = path.join(PROJECT_ROOT, 'inbox');
+    const externalInbox = path.join(PROJECT_ROOT, 'external-inbox');
+    // Auto-create inbox directories if missing (Standard 051: Ephemeral Index)
+    // These are gitignored and should be created on-demand
+    if (!fs.existsSync(inbox)) {
+        fs.mkdirSync(inbox, { recursive: true });
+        console.log(`[Watchdog] Created inbox directory: ${inbox}`);
+    }
+    if (!fs.existsSync(externalInbox)) {
+        fs.mkdirSync(externalInbox, { recursive: true });
+        console.log(`[Watchdog] Created external-inbox directory: ${externalInbox}`);
+    }
+    console.log(`[Watchdog] Starting watch on: ${inbox} and ${externalInbox}`);
+    // Load extra paths from config
+    const extraPaths = config.WATCHER_EXTRA_PATHS || [];
+    const validExtraPaths = extraPaths.filter((p) => {
+        if (fs.existsSync(p))
+            return true;
+        console.warn(`[Watchdog] Extra path not found: ${p}`);
+        return false;
+    });
+    const pathsToWatch = [inbox, externalInbox, ...validExtraPaths];
+    watcher = chokidar.watch(pathsToWatch, {
+        ignored: IGNORE_PATTERNS,
+        persistent: true,
+        ignoreInitial: false, // Force scan on start to ingest existing files
+        awaitWriteFinish: {
+            stabilityThreshold: config.WATCHER_STABILITY_THRESHOLD_MS,
+            pollInterval: 100
+        }
+    });
+    watcher
+        .on('add', (path) => processFile(path, 'add'))
+        .on('change', (path) => processFile(path, 'change'))
+        .on('addDir', (path) => console.log(`[Watchdog] Detected new directory: ${path}`));
+    // .on('unlink', (path) => deleteFile(path)); // Implement delete logic later
+}
+// Dynamic Path Management
+export function getWatchedPaths() {
+    if (!watcher)
+        return [];
+    // chokidar.getWatched() returns an object where keys are paths
+    // But it returns all subdirectories too. We mainly want the roots we added.
+    // For simplicity, we can return the configured roots + static roots.
+    // Better approach: Return the paths explicitly tracked
+    const inbox = path.join(PROJECT_ROOT, 'inbox');
+    const externalInbox = path.join(PROJECT_ROOT, 'external-inbox');
+    const extraPaths = config.WATCHER_EXTRA_PATHS || [];
+    return [inbox, externalInbox, ...extraPaths];
+}
+export async function addWatchPath(newPath) {
+    if (!fs.existsSync(newPath)) {
+        throw new Error(`Path does not exist: ${newPath}`);
+    }
+    // Add to watcher if it's running
+    if (watcher) {
+        watcher.add(newPath);
+        console.log(`[Watchdog] Added dynamic watch path: ${newPath}`);
+    }
+    else {
+        console.log(`[Watchdog] Path saved for later (watchdog not running): ${newPath}`);
+    }
+    // Update Config (In-Memory)
+    if (!config.WATCHER_EXTRA_PATHS)
+        config.WATCHER_EXTRA_PATHS = [];
+    if (!config.WATCHER_EXTRA_PATHS.includes(newPath)) {
+        config.WATCHER_EXTRA_PATHS.push(newPath);
+        // Persist to user_settings.json (always do this, even if watchdog isn't running)
+        try {
+            const settingsPath = path.join(process.cwd(), 'user_settings.json');
+            if (fs.existsSync(settingsPath)) {
+                const settingsRequest = await fs.promises.readFile(settingsPath, 'utf8');
+                const settings = JSON.parse(settingsRequest);
+                if (!settings.watcher)
+                    settings.watcher = {};
+                if (!settings.watcher.extra_paths)
+                    settings.watcher.extra_paths = [];
+                if (!settings.watcher.extra_paths.includes(newPath)) {
+                    settings.watcher.extra_paths.push(newPath);
+                    await fs.promises.writeFile(settingsPath, JSON.stringify(settings, null, 4));
+                    console.log(`[Watchdog] Persisted path to user_settings.json`);
+                }
+            }
+        }
+        catch (e) {
+            console.error(`[Watchdog] Failed to persist settings: ${e.message}`);
+        }
+    }
+    return true;
+}
+export async function removeWatchPath(pathToRemove) {
+    // Remove from chokidar watcher if it exists (watchdog is running)
+    if (watcher) {
+        watcher.unwatch(pathToRemove);
+        console.log(`[Watchdog] Removed watch path: ${pathToRemove}`);
+    }
+    else {
+        console.log(`[Watchdog] Path marked for removal (watchdog not running): ${pathToRemove}`);
+    }
+    // Update Config (In-Memory)
+    if (config.WATCHER_EXTRA_PATHS && config.WATCHER_EXTRA_PATHS.includes(pathToRemove)) {
+        config.WATCHER_EXTRA_PATHS = config.WATCHER_EXTRA_PATHS.filter((p) => p !== pathToRemove);
+        // Persist to user_settings.json
+        try {
+            const settingsPath = path.join(process.cwd(), 'user_settings.json');
+            if (fs.existsSync(settingsPath)) {
+                const settingsRequest = await fs.promises.readFile(settingsPath, 'utf8');
+                const settings = JSON.parse(settingsRequest);
+                if (settings.watcher && settings.watcher.extra_paths) {
+                    settings.watcher.extra_paths = settings.watcher.extra_paths.filter((p) => p !== pathToRemove);
+                    await fs.promises.writeFile(settingsPath, JSON.stringify(settings, null, 4));
+                    console.log(`[Watchdog] Persisted path removal to user_settings.json`);
+                }
+            }
+        }
+        catch (e) {
+            console.error(`[Watchdog] Failed to persist settings removal: ${e.message}`);
+        }
+    }
+    return true;
+}
+/**
+ * Stop the watchdog service
+ */
+export async function stopWatchdog() {
+    if (watcher) {
+        await watcher.close();
+        watcher = null;
+        console.log('[Watchdog] Stopped watching files');
+    }
+}
+/**
+ * Get watchdog status
+ */
+export function getWatcherStatus() {
+    const inbox = path.join(PROJECT_ROOT, 'inbox');
+    const externalInbox = path.join(PROJECT_ROOT, 'external-inbox');
+    const extraPaths = config.WATCHER_EXTRA_PATHS || [];
+    return {
+        isRunning: watcher !== null,
+        watchedPaths: [inbox, externalInbox, ...extraPaths]
+    };
+}
+/**
+ * Trigger manual ingestion scan
+ */
+export async function triggerManualIngest() {
+    try {
+        const inbox = path.join(PROJECT_ROOT, 'inbox');
+        const externalInbox = path.join(PROJECT_ROOT, 'external-inbox');
+        if (!fs.existsSync(inbox)) {
+            return { status: 'error', message: 'Inbox directory not found' };
+        }
+        let filesProcessed = 0;
+        let filesIngested = 0;
+        // Scan inbox directory
+        const files = fs.readdirSync(inbox, { recursive: true });
+        for (const file of files) {
+            const filePath = path.join(inbox, file);
+            // Skip directories and ignored patterns
+            if (fs.statSync(filePath).isDirectory())
+                continue;
+            if (IGNORE_PATTERNS.test(file))
+                continue;
+            filesProcessed++;
+            // Trigger actual ingestion by calling processFile
+            try {
+                await processFile(filePath, 'manual');
+                filesIngested++;
+            }
+            catch (error) {
+                console.error(`[ManualIngest] Failed to process ${file}:`, error.message);
+            }
+        }
+        // Also scan external-inbox if it exists
+        if (fs.existsSync(externalInbox)) {
+            const externalFiles = fs.readdirSync(externalInbox, { recursive: true });
+            for (const file of externalFiles) {
+                const filePath = path.join(externalInbox, file);
+                if (fs.statSync(filePath).isDirectory())
+                    continue;
+                if (IGNORE_PATTERNS.test(file))
+                    continue;
+                filesProcessed++;
+                try {
+                    await processFile(filePath, 'manual');
+                    filesIngested++;
+                }
+                catch (error) {
+                    console.error(`[ManualIngest] Failed to process ${file}:`, error.message);
+                }
+            }
+        }
+        return {
+            status: 'success',
+            message: `Manual ingest complete: ${filesIngested}/${filesProcessed} files processed`,
+            filesProcessed,
+            filesIngested
+        };
+    }
+    catch (error) {
+        return {
+            status: 'error',
+            message: `Manual ingest failed: ${error.message}`
+        };
+    }
+}
+// Revert to AtomizerService for performance
+// import { SemanticIngestionService } from '../semantic/semantic-ingestion-service.js';
+import { AtomizerService } from './atomizer-service.js';
+import { AtomicIngestService } from './ingest-atomic.js';
+// import { ingestAtoms } from './ingest.js'; // Already imported at top of file
+// Singleton Services
+// const semanticIngest = new SemanticIngestionService();
+const atomizer = new AtomizerService();
+const atomicIngest = new AtomicIngestService();
+async function processFile(filePath, event) {
+    // Accept markdown, text, YAML, CSV, JSON, and HTML files
+    if (!filePath.endsWith('.md') && !filePath.endsWith('.txt') && !filePath.endsWith('.yaml') &&
+        !filePath.endsWith('.csv') && !filePath.endsWith('.json') &&
+        !filePath.endsWith('.html') && !filePath.endsWith('.htm'))
+        return;
+    if (filePath.includes('mirrored_brain'))
+        return;
+    console.log(`[Watchdog] Detected ${event}: ${filePath}`);
+    // Set system status to ingesting
+    systemStatus.setState('ingesting', `Processing: ${path.basename(filePath)}`);
+    try {
+        const buffer = await fs.promises.readFile(filePath);
+        if (buffer.length === 0)
+            return;
+        // 1. Calculate File Hash (Raw)
+        const fileHash = crypto.createHash('sha256').update(buffer).digest('hex');
+        const relativePath = path.relative(NOTEBOOK_DIR, filePath);
+        const content = buffer.toString('utf8');
+        // 2. Check Source Table (Change Detection)
+        const sourceQuery = `SELECT path, hash FROM sources WHERE path = $1`;
+        const sourceResult = await db.run(sourceQuery, [relativePath]);
+        // Handle potential null result
+        if (!sourceResult || !sourceResult.rows) {
+            console.log(`[Watchdog] No existing record for path: ${relativePath}`);
+        }
+        if (sourceResult && sourceResult.rows && sourceResult.rows.length > 0) {
+            const row = sourceResult.rows[0];
+            // Handle both array and object formats that PGlite might return
+            let existingHash;
+            if (Array.isArray(row)) {
+                // Row is in array format [path, hash]
+                existingHash = row[1];
+            }
+            else {
+                // Row is in object format {path, hash}
+                existingHash = row.hash;
+            }
+            if (existingHash === fileHash) {
+                console.log(`[Watchdog] File unchanged (hash match): ${relativePath}`);
+                systemStatus.setState('idle');
+                return;
+            }
+        }
+        console.log(`[Watchdog] Processing Pipeline: ${relativePath}`);
+        systemStatus.setProgress(0, 100, 'Starting ingestion...');
+        // 3. DETERMINE METADATA
+        // Determine buckets
+        const parts = relativePath.split(path.sep);
+        let bucket = 'notebook';
+        // logic: if inside a root folder (inbox/external-inbox) and has a subfolder, use subfolder as bucket
+        // otherwise use the root folder
+        if (parts.length >= 2) {
+            const root = parts[0];
+            if ((root === 'inbox' || root === 'external-inbox') && parts.length > 2) {
+                bucket = parts[1];
+            }
+            else {
+                bucket = root;
+            }
+        }
+        // Determine type (auto-detect HTML for cleaning)
+        const ext = path.extname(filePath).replace('.', '');
+        let type = ext || 'text';
+        // Auto-detect HTML content for cleaning pipeline
+        if (ext === 'html' || ext === 'htm') {
+            type = 'web_page'; // Triggers full HTML cleaning
+        }
+        // Determine Provenance
+        let provenance = 'internal';
+        if (relativePath.includes('external-inbox') || relativePath.includes('web_scrape')) {
+            provenance = 'external';
+        }
+        // 4. ATOMIZE (Legacy Pipeline)
+        // This is the fast, regex-based splitter that respects token limits and semantics without heavy NLP
+        const atomizeResult = await atomizer.atomize(content, relativePath, provenance);
+        // Skip ingestion if transient data was detected
+        if (!atomizeResult) {
+            console.log(`[Watchdog] ⚠️ SKIP: ${relativePath} - Transient data, skipping ingestion`);
+            return; // Exit early, no ingestion
+        }
+        const { compound, molecules, atoms } = atomizeResult;
+        // 5. INGEST (Atomic)
+        // Use the specialized AtomicIngestService for efficiency
+        await atomicIngest.ingestResult(compound, molecules, atoms, [bucket]);
+        // 6. Update Source Table
+        await db.run(`INSERT INTO sources (path, hash, total_atoms, last_ingest)
+             VALUES ($1, $2, $3, $4)
+             ON CONFLICT (path) DO UPDATE SET
+               hash = EXCLUDED.hash,
+               total_atoms = EXCLUDED.total_atoms,
+               last_ingest = EXCLUDED.last_ingest`, [
+            relativePath,
+            fileHash,
+            atoms.length,
+            Date.now()
+        ]);
+        console.log(`[Watchdog] Sync Complete: ${relativePath}`);
+        // Trigger Mirror: write cleaned content directly (O(1) vs full rebuild)
+        console.log(`[Watchdog] Preparing mirror write...`);
+        console.log(`[Watchdog] compound exists: ${!!compound}`);
+        console.log(`[Watchdog] compound.compound_body exists: ${!!compound?.compound_body}`);
+        console.log(`[Watchdog] compound.compound_body length: ${compound?.compound_body?.length || 0}`);
+        console.log(`[Watchdog] provenance: ${provenance}`);
+        try {
+            console.log(`[Watchdog] Importing mirror module...`);
+            const { writeMirroredFile } = await import('../mirror/mirror.js');
+            console.log(`[Watchdog] Mirror module imported, calling writeMirroredFile...`);
+            await writeMirroredFile(relativePath, compound.compound_body, provenance);
+            console.log(`[Watchdog] ✓ Mirror write completed successfully`);
+        }
+        catch (e) {
+            console.error(`[Watchdog] ✗ Mirror write failed:`, e.message);
+            console.error(`[Watchdog] Stack trace:`, e.stack);
+        }
+        // Trigger post-ingestion synonym generation (debounced)
+        triggerPostIngestionSynonyms();
+        // Reset system status to idle after ingestion completes
+        if (typeof global.gc === 'function')
+            global.gc();
+        systemStatus.setState('idle');
+        systemStatus.clearProgress();
+        console.log(`[SystemStatus] Ingestion complete, system ready for search`);
+    }
+    catch (error) {
+        console.error(`[Watchdog] Error processing ${filePath}:`, error.message);
+        systemStatus.setState('idle');
+        systemStatus.clearProgress();
+        throw error;
+    }
+}
+//# sourceMappingURL=watchdog.js.map