@rbalchii/anchor-engine 4.7.0

package/engine/dist/services/tags/discovery.js

@@ -0,0 +1,206 @@
+import { db } from '../../core/db.js';
+import { extractEntitiesWithGLiNER } from './gliner.js';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..');
+const MASTER_TAGS_PATH = path.join(PROJECT_ROOT, 'engine', 'context', 'internal_tags.json');
+/**
+ * Discovery Service (The Teacher)
+ *
+ * Implements "Tag Walker" Strategy (Standard 068 Phase B):
+ * 1. Pick a seed tag from the master list.
+ * 2. Find atoms that contain this tag ("Walking the Graph").
+ * 3. Use BERT NER to find NEW entities in these specific contexts.
+ * 4. Add new entities to the master list (Expansion).
+ */
+export async function runDiscovery(sampleSize = 30) {
+    const masterTags = getMasterTags();
+    let query = '';
+    let strategy = 'random';
+    let seedTag = '';
+    // Strategy 1: The Walker (80% chance if we have tags)
+    // "We have data specific tags... identified entities beyond the basic list that accommodate specific intricacies."
+    if (masterTags.length > 0 && Math.random() > 0.2) {
+        seedTag = masterTags[Math.floor(Math.random() * masterTags.length)];
+        // Find atoms that contain the seed tag (Simulating Graph Walk)
+        // We use the '~' operator for "contains text", which is efficient enough for now.
+        // We limit to sampleSize to keep it fast.
+        query = `
+            SELECT content
+            FROM atoms
+            WHERE content ILIKE $1
+            LIMIT ${sampleSize}
+        `;
+        strategy = 'walker';
+        console.log(`[Discovery] Teacher Mode (Walker): Expanding on seed tag '${seedTag}'...`);
+    }
+    // Strategy 2: The Explorer (Fallback / Initial Boot)
+    else {
+        query = `SELECT content FROM atoms LIMIT ${sampleSize}`;
+        strategy = 'explorer';
+        console.log(`[Discovery] Teacher Mode (Explorer): Random Sampling ${sampleSize} atoms...`);
+    }
+    let result;
+    try {
+        if (strategy === 'walker' && seedTag) {
+            result = await db.run(query, [`%${seedTag}%`]);
+        }
+        else {
+            result = await db.run(query);
+        }
+    }
+    catch (e) {
+        console.warn(`[Discovery] Query failed for strategy '${strategy}' (Seed: ${seedTag}):`, e.message);
+        console.warn(`[Discovery] Falling back to safe Explorer mode.`);
+        query = `SELECT content FROM atoms LIMIT ${sampleSize}`;
+        result = await db.run(query);
+    }
+    if (!result.rows || result.rows.length === 0) {
+        if (strategy === 'walker') {
+            console.log(`[Discovery] Walker found no atoms for tag '${seedTag}'. It might be rare.`);
+        }
+        else {
+            console.warn('[Discovery] No atoms found for learning.');
+        }
+        return [];
+    }
+    const sampledContent = result.rows.map((r) => {
+        // Handle both array and object formats that PGlite might return
+        let content;
+        if (Array.isArray(r)) {
+            content = String(r[0]); // If array format, content is at index 0
+        }
+        else {
+            content = String(r.content); // If object format, use content property
+        }
+        // Truncate to keep BERT fast
+        return content.length > 500 ? content.substring(0, 500) : content;
+    }).join('\n---\n');
+    console.log(`[Discovery] Teacher analyzing ${result.rows.length} atoms via BERT...`);
+    try {
+        // 2a. Attempt Zero-Shot/BERT Extraction
+        // We ask BERT to look for standard entities, but since the context is specific (seeded),
+        // it is more likely to find domain-specific co-occurrences.
+        const discoveredTags = await extractEntitiesWithGLiNER(sampledContent, [
+            'person', 'organization', 'technology', 'project', 'software', 'location', 'concept'
+        ]);
+        console.log(`[Discovery] BERT found ${discoveredTags.length} potential tags.`);
+        if (discoveredTags.length > 0) {
+            // Filter out the seed tag so we don't just rediscover it
+            const newTags = discoveredTags.filter(t => t.toLowerCase() !== seedTag.toLowerCase());
+            if (newTags.length > 0) {
+                console.log(`[Discovery] Expansion Successful! '${seedTag}' led to: ${newTags.slice(0, 5).join(', ')}...`);
+                await updateMasterTags(newTags);
+            }
+            return newTags;
+        }
+        else {
+            throw new Error("BERT found no entities.");
+        }
+    }
+    catch (e) {
+        console.warn(`[Discovery] Teacher (BERT) passed. Error: ${e.message}`);
+        // Optional: LLM Fallback (Slow, but very smart)
+        // For now, we return empty to stay fast/CPU-specific as requested.
+        return [];
+    }
+}
+/**
+ * Updates the JSON master list with new findings.
+ */
+async function updateMasterTags(newTags) {
+    try {
+        let currentTags = { keywords: [] };
+        // Ensure directory exists
+        const contextDir = path.dirname(MASTER_TAGS_PATH);
+        if (!fs.existsSync(contextDir)) {
+            fs.mkdirSync(contextDir, { recursive: true });
+        }
+        // Read existing
+        if (fs.existsSync(MASTER_TAGS_PATH)) {
+            const content = fs.readFileSync(MASTER_TAGS_PATH, 'utf8');
+            try {
+                currentTags = JSON.parse(content);
+                // Handle if it's just an array vs object
+                if (Array.isArray(currentTags)) {
+                    currentTags = { keywords: currentTags };
+                }
+            }
+            catch (jsonErr) {
+                console.warn('[Discovery] Corrupt tags file, starting fresh.');
+            }
+        }
+        // Merge
+        const existingSet = new Set(currentTags.keywords.map((t) => t.toLowerCase()));
+        const added = [];
+        newTags.forEach(tag => {
+            const normalized = tag.toLowerCase().trim();
+            if (normalized.length > 2 && !existingSet.has(normalized)) {
+                // Basic filtering
+                if (!['the', 'and', 'for', 'with'].includes(normalized)) {
+                    currentTags.keywords.push(tag); // Keep original case
+                    existingSet.add(normalized);
+                    added.push(tag);
+                }
+            }
+        });
+        if (added.length > 0) {
+            fs.writeFileSync(MASTER_TAGS_PATH, JSON.stringify(currentTags, null, 2));
+            console.log(`[Discovery] Learned ${added.length} new tags:`, added.join(', '));
+            // Explicitly invalidate cache just to be safe (though watcher usually catches it)
+            cachedMasterTags = null;
+        }
+    }
+    catch (e) {
+        console.error('[Discovery] Failed to update master list:', e);
+    }
+}
+let cachedMasterTags = null;
+let watcherInitialized = false;
+/**
+ * Reads the master list for the Infector (and Walker).
+ * Uses an in-memory cache updated via file watcher.
+ */
+export function getMasterTags() {
+    if (cachedMasterTags !== null) {
+        return cachedMasterTags;
+    }
+    try {
+        if (fs.existsSync(MASTER_TAGS_PATH)) {
+            const content = fs.readFileSync(MASTER_TAGS_PATH, 'utf8');
+            const data = JSON.parse(content);
+            let tags = [];
+            if (Array.isArray(data))
+                tags = data;
+            else if (data.keywords && Array.isArray(data.keywords))
+                tags = data.keywords;
+            cachedMasterTags = tags;
+            // Initialize watcher on first successful read
+            if (!watcherInitialized) {
+                watcherInitialized = true;
+                try {
+                    fs.watch(MASTER_TAGS_PATH, (eventType) => {
+                        if (eventType === 'change' || eventType === 'rename') {
+                            // Invalidate cache
+                            cachedMasterTags = null;
+                        }
+                    });
+                }
+                catch (watchErr) {
+                    console.warn('[Discovery] Failed to set up file watcher for tags:', watchErr);
+                    // If watch fails, we fallback to just reading it occasionally or next startup,
+                    // but the error is logged. We still consider it "initialized" so we don't spam watch attempts.
+                }
+            }
+            return tags;
+        }
+    }
+    catch (e) {
+        console.error('[Discovery] Failed to load internal_tags.json:', e);
+    }
+    return [];
+}
+//# sourceMappingURL=discovery.js.map

package/engine/dist/services/tags/discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.js","sourceRoot":"","sources":["../../../src/services/tags/discovery.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACtC,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAC/D,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC;AAE5F;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,aAAqB,EAAE;IACtD,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;IACnC,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,QAAQ,GAAG,QAAQ,CAAC;IACxB,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,sDAAsD;IACtD,mHAAmH;IACnH,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;QAC/C,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpE,+DAA+D;QAC/D,kFAAkF;QAClF,0CAA0C;QAC1C,KAAK,GAAG;;;;oBAII,UAAU;SACrB,CAAC;QACF,QAAQ,GAAG,QAAQ,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,6DAA6D,OAAO,MAAM,CAAC,CAAC;IAC5F,CAAC;IACD,qDAAqD;SAChD,CAAC;QACF,KAAK,GAAG,mCAAmC,UAAU,EAAE,CAAC;QACxD,QAAQ,GAAG,UAAU,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,wDAAwD,UAAU,WAAW,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACD,IAAI,QAAQ,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC;YACnC,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACJ,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,0CAA0C,QAAQ,YAAY,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACnG,OAAO,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAChE,KAAK,GAAG,mCAAmC,UAAU,EAAE,CAAC;QACxD,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,8CAA8C,OAAO,sBAAsB,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,EAAE,CAAC;IACd,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE;QAC9C,gEAAgE;QAChE,IAAI,OAAO,CAAC;QACZ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,yCAAyC;QACrE,CAAC;aAAM,CAAC;YACJ,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yCAAyC;QAC1E,CAAC;QACD,6BAA6B;QAC7B,OAAO,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACtE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEnB,OAAO,CAAC,GAAG,CAAC,iCAAiC,MAAM,CAAC,IAAI,CAAC,MAAM,oBAAoB,CAAC,CAAC;IAErF,IAAI,CAAC;QACD,wCAAwC;QACxC,yFAAyF;QACzF,4DAA4D;QAC5D,MAAM,cAAc,GAAG,MAAM,yBAAyB,CAAC,cAAc,EAAE;YACnE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS;SACvF,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,0BAA0B,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;QAE/E,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,yDAAyD;YACzD,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;YAEtF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,GAAG,CAAC,sCAAsC,OAAO,aAAa,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3G,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;YACD,OAAO,OAAO,CAAC;QACnB,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,gDAAgD;QAChD,mEAAmE;QACnE,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAAC,OAAiB;IAC7C,IAAI,CAAC;QACD,IAAI,WAAW,GAAQ,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAExC,0BAA0B;QAC1B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAClD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,gBAAgB;QAChB,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACD,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAClC,yCAAyC;gBACzC,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC7B,WAAW,GAAG,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;gBAC5C,CAAC;YACL,CAAC;YAAC,OAAO,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;YACnE,CAAC;QACL,CAAC;QAED,QAAQ;QACR,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACtF,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;YAClB,MAAM,UAAU,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBACxD,kBAAkB;gBAClB,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;oBACtD,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,qBAAqB;oBACrD,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpB,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,CAAC,MAAM,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAE/E,kFAAkF;YAClF,gBAAgB,GAAG,IAAI,CAAC;QAC5B,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;AACL,CAAC;AAED,IAAI,gBAAgB,GAAoB,IAAI,CAAC;AAC7C,IAAI,kBAAkB,GAAG,KAAK,CAAC;AAE/B;;;GAGG;AACH,MAAM,UAAU,aAAa;IACzB,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;QAC5B,OAAO,gBAAgB,CAAC;IAC5B,CAAC;IAED,IAAI,CAAC;QACD,IAAI,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAEjC,IAAI,IAAI,GAAa,EAAE,CAAC;YACxB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,IAAI,GAAG,IAAI,CAAC;iBAChC,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;gBAAE,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC;YAE7E,gBAAgB,GAAG,IAAI,CAAC;YAExB,8CAA8C;YAC9C,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBACtB,kBAAkB,GAAG,IAAI,CAAC;gBAC1B,IAAI,CAAC;oBACD,EAAE,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,SAAS,EAAE,EAAE;wBACrC,IAAI,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;4BACnD,mBAAmB;4BACnB,gBAAgB,GAAG,IAAI,CAAC;wBAC5B,CAAC;oBACL,CAAC,CAAC,CAAC;gBACP,CAAC;gBAAC,OAAO,QAAQ,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC,qDAAqD,EAAE,QAAQ,CAAC,CAAC;oBAC9E,+EAA+E;oBAC/E,+FAA+F;gBACnG,CAAC;YACL,CAAC;YAED,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,EAAE,CAAC;AACd,CAAC"}

package/engine/dist/services/tags/gliner.d.ts

@@ -0,0 +1,18 @@
+/**
+ * NER Teacher Service (BERT-based)
+ *
+ * Uses an ONNX-optimized BERT model to perform Named Entity Recognition.
+ * Switched from GLiNER (unsupported architecture) to standard BERT NER.
+ * Implements lazy loading and automatic unloading to manage memory usage.
+ */
+/**
+ * Unload the NER model to free memory
+ */
+export declare function unloadModel(): Promise<void>;
+/**
+ * Check if model is currently loaded
+ */
+export declare function isModelLoadedStatus(): boolean;
+export declare function extractEntitiesWithGLiNER(text: string, _entities?: string[]): Promise<string[]>;
+export declare function unloadNerModel(): Promise<void>;
+//# sourceMappingURL=gliner.d.ts.map

package/engine/dist/services/tags/gliner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gliner.d.ts","sourceRoot":"","sources":["../../../src/services/tags/gliner.ts"],"names":[],"mappings":"AACA;;;;;;GAMG;AAuBH;;GAEG;AACH,wBAAsB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAwDD,wBAAsB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,GAAE,MAAM,EAAO,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAgCzG;AAGD,wBAAsB,cAAc,kBAEnC"}

package/engine/dist/services/tags/gliner.js

@@ -0,0 +1,119 @@
+/**
+ * NER Teacher Service (BERT-based)
+ *
+ * Uses an ONNX-optimized BERT model to perform Named Entity Recognition.
+ * Switched from GLiNER (unsupported architecture) to standard BERT NER.
+ * Implements lazy loading and automatic unloading to manage memory usage.
+ */
+let nerPipeline = null;
+let lastUsed = 0;
+let idleTimeout = null;
+const UNLOAD_TIMEOUT = 5 * 60 * 1000; // 5 minutes (consistent with NLP service)
+const isModelLoaded = false;
+/**
+ * Schedule model unload after idle period
+ */
+function scheduleModelUnload() {
+    if (idleTimeout) {
+        clearTimeout(idleTimeout);
+    }
+    idleTimeout = setTimeout(() => {
+        cleanupPipeline();
+    }, UNLOAD_TIMEOUT);
+}
+/**
+ * Unload the NER model to free memory
+ */
+export async function unloadModel() {
+    await cleanupPipeline();
+}
+/**
+ * Check if model is currently loaded
+ */
+export function isModelLoadedStatus() {
+    return nerPipeline !== null;
+}
+async function initializePipeline() {
+    if (nerPipeline) {
+        lastUsed = Date.now();
+        scheduleModelUnload(); // Reset idle timer on use
+        return nerPipeline;
+    }
+    console.log('[NER] Dynamically loading Transformers.js...');
+    const { pipeline, env } = await import('@xenova/transformers');
+    // Disable native dependencies that might cause crashes on Windows
+    env.allowLocalModels = true;
+    // Disable ONNX native backend that requires sharp
+    env.backends.onnx['native'] = false;
+    env.backends.onnx.wasm.proxy = false;
+    env.backends.onnx.wasm.numThreads = 1;
+    // Additional settings to avoid sharp
+    env.useFS = false;
+    env.useBrowserCache = false;
+    console.log('[NER] Loading BERT NER model (Xenova/bert-base-NER)...');
+    try {
+        nerPipeline = await pipeline('token-classification', 'Xenova/bert-base-NER', {
+            quantized: true
+        });
+    }
+    catch (e) {
+        console.warn('[NER] Primary model failed. Trying fallback (Xenova/bert-base-multilingual-cased-ner-hrl)...');
+        nerPipeline = await pipeline('token-classification', 'Xenova/bert-base-multilingual-cased-ner-hrl', {
+            quantized: true
+        });
+    }
+    console.log('[NER] Model loaded successfully.');
+    lastUsed = Date.now();
+    scheduleModelUnload(); // Start idle timer
+    return nerPipeline;
+}
+async function cleanupPipeline() {
+    if (nerPipeline) {
+        try {
+            // Attempt to clean up the pipeline if it has a dispose method
+            if (nerPipeline.dispose) {
+                await nerPipeline.dispose();
+            }
+            nerPipeline = null;
+            console.log('[NER] Model unloaded to free memory.');
+        }
+        catch (e) {
+            console.warn('[NER] Error during pipeline cleanup:', e);
+        }
+    }
+}
+export async function extractEntitiesWithGLiNER(text, _entities = []) {
+    try {
+        const pipelineInstance = await initializePipeline();
+        // BERT NER returns entities with labels like B-PER, I-ORG, B-LOC, B-MISC
+        // We extract the actual text (word) from each recognized entity
+        const results = await pipelineInstance(text);
+        const discovered = new Set();
+        for (const res of results) {
+            // Filter by confidence score and entity type
+            // B- prefix means "Beginning of entity", I- means "Inside entity"
+            if (res.score > 0.7 && res.entity && res.word) {
+                // Clean up subword tokens (BERT uses ## prefix for subwords)
+                const word = res.word.replace(/^##/, '').trim();
+                if (word.length > 1) {
+                    discovered.add(word);
+                }
+            }
+        }
+        console.log(`[NER] Discovered ${discovered.size} entities.`);
+        lastUsed = Date.now();
+        return Array.from(discovered);
+    }
+    catch (e) {
+        console.warn('[NER] Service Initialization Failed:', e.message);
+        console.log('[NER] Falling back gracefully to LLM...');
+        // Unload the pipeline on error to free memory
+        await cleanupPipeline();
+        return [];
+    }
+}
+// Export a function to manually trigger cleanup if needed
+export async function unloadNerModel() {
+    await cleanupPipeline();
+}
+//# sourceMappingURL=gliner.js.map

package/engine/dist/services/tags/gliner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gliner.js","sourceRoot":"","sources":["../../../src/services/tags/gliner.ts"],"names":[],"mappings":"AACA;;;;;;GAMG;AAIH,IAAI,WAAW,GAAQ,IAAI,CAAC;AAC5B,IAAI,QAAQ,GAAW,CAAC,CAAC;AACzB,IAAI,WAAW,GAA0B,IAAI,CAAC;AAC9C,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,0CAA0C;AAChF,MAAM,aAAa,GAAG,KAAK,CAAC;AAE5B;;GAEG;AACH,SAAS,mBAAmB;IAC1B,IAAI,WAAW,EAAE,CAAC;QAChB,YAAY,CAAC,WAAW,CAAC,CAAC;IAC5B,CAAC;IAED,WAAW,GAAG,UAAU,CAAC,GAAG,EAAE;QAC5B,eAAe,EAAE,CAAC;IACpB,CAAC,EAAE,cAAc,CAAC,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,eAAe,EAAE,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB;IACjC,OAAO,WAAW,KAAK,IAAI,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,kBAAkB;IAC7B,IAAI,WAAW,EAAE,CAAC;QACd,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,mBAAmB,EAAE,CAAC,CAAC,0BAA0B;QACjD,OAAO,WAAW,CAAC;IACvB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC5D,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAE/D,kEAAkE;IAClE,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAC5B,kDAAkD;IAClD,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,KAAK,CAAC;IACpC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;IAEtC,qCAAqC;IACrC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,GAAG,CAAC,eAAe,GAAG,KAAK,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,wDAAwD,CAAC,CAAC;IACtE,IAAI,CAAC;QACD,WAAW,GAAG,MAAM,QAAQ,CAAC,sBAAsB,EAAE,sBAAsB,EAAE;YACzE,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;IACP,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,CAAC,IAAI,CAAC,8FAA8F,CAAC,CAAC;QAC7G,WAAW,GAAG,MAAM,QAAQ,CAAC,sBAAsB,EAAE,6CAA6C,EAAE;YAChG,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;IACP,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAEhD,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACtB,mBAAmB,EAAE,CAAC,CAAC,mBAAmB;IAC1C,OAAO,WAAW,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,eAAe;IAC1B,IAAI,WAAW,EAAE,CAAC;QACd,IAAI,CAAC;YACD,8DAA8D;YAC9D,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;gBACtB,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;YAChC,CAAC;YACD,WAAW,GAAG,IAAI,CAAC;YACnB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,CAAC;QAC5D,CAAC;IACL,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,IAAY,EAAE,YAAsB,EAAE;IAClF,IAAI,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEpD,yEAAyE;QACzE,gEAAgE;QAChE,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QAErC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YACxB,6CAA6C;YAC7C,kEAAkE;YAClE,IAAI,GAAG,CAAC,KAAK,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC5C,6DAA6D;gBAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,CAAC,IAAI,YAAY,CAAC,CAAC;QAC7D,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,OAAO,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,CAAM,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QAEvD,8CAA8C;QAC9C,MAAM,eAAe,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,CAAC,KAAK,UAAU,cAAc;IAChC,MAAM,eAAe,EAAE,CAAC;AAC5B,CAAC"}

package/engine/dist/services/tags/infector.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Tag Infection Service (The "Student")
+ *
+ * Implements Standard 068: Weak Supervision via High-Speed Pattern Matching.
+ * Implements Standard 069: Functional Flow (Generators) for infinite scaling.
+ */
+/**
+ * 2. The Processor (Transform)
+ * Applies "Viral Tags" to a single atom.
+ */
+export declare function infectAtom(atom: {
+    id: string;
+    content: string;
+    tags: string[];
+}, patterns: any): string[] | null;
+/**
+ * 3. The Orchestrator (Sink)
+ * Connects the Stream to the Processor.
+ */
+export declare function runInfectionLoop(): Promise<void>;
+//# sourceMappingURL=infector.d.ts.map

package/engine/dist/services/tags/infector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"infector.d.ts","sourceRoot":"","sources":["../../../src/services/tags/infector.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA0EH;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE,EAAE,QAAQ,EAAE,GAAG,GAAG,MAAM,EAAE,GAAG,IAAI,CA2DhH;AAED;;;GAGG;AACH,wBAAsB,gBAAgB,kBA+CrC"}

package/engine/dist/services/tags/infector.js

@@ -0,0 +1,168 @@
+/**
+ * Tag Infection Service (The "Student")
+ *
+ * Implements Standard 068: Weak Supervision via High-Speed Pattern Matching.
+ * Implements Standard 069: Functional Flow (Generators) for infinite scaling.
+ */
+import wink from 'wink-nlp';
+import model from 'wink-eng-lite-web-model';
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { db } from '../../core/db.js';
+// Initialize the "Reflex" Engine (Fast CPU NLP)
+// Cast to any to avoid strict typing issues with wink-nlp generic models
+const nlp = wink(model);
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PROJECT_ROOT = path.resolve(__dirname, '..', '..', '..'); // engine/src/services/tags -> engine/src/services -> engine/src -> engine
+const TAGS_FILE = path.join(PROJECT_ROOT, 'engine', 'context', 'internal_tags.json');
+/**
+ * 1. The Generator (Source)
+ * Lazily fetches atoms from the database in batches to prevent RAM spikes.
+ * This replaces the need for recursion or massive array loading.
+ */
+async function* atomStream(batchSize = 50) {
+    let lastId = '';
+    let batchCount = 0;
+    while (true) {
+        // Fetch next batch where ID > lastId
+        const query = `
+            SELECT id, content, tags
+            FROM atoms
+            WHERE id > $1
+            ORDER BY id
+            LIMIT $2
+        `;
+        const result = await db.run(query, [lastId, batchSize]);
+        batchCount++;
+        if (result.rows && result.rows.length > 0 && batchCount % 50 === 0) {
+            console.log(`[Infector] Stream fetched batch of ${result.rows.length} atoms... (Batch ${batchCount})`);
+        }
+        if (!result.rows || result.rows.length === 0) {
+            break; // Stream exhausted
+        }
+        // Yield one atom at a time (Functional Flow)
+        for (const row of result.rows) {
+            // Handle both array and object formats that PGlite might return
+            let id, content, tags;
+            if (Array.isArray(row)) {
+                // Row is in array format [id, content, tags]
+                [id, content, tags] = row;
+            }
+            else {
+                // Row is in object format {id, content, tags}
+                id = row.id;
+                content = row.content;
+                tags = row.tags;
+            }
+            lastId = id; // Move cursor for next batch
+            yield {
+                id: id,
+                content: content,
+                tags: tags || []
+            };
+        }
+    }
+}
+/**
+ * 2. The Processor (Transform)
+ * Applies "Viral Tags" to a single atom.
+ */
+export function infectAtom(atom, patterns) {
+    if (!atom.content)
+        return null;
+    const currentTags = new Set(atom.tags);
+    let changed = false;
+    // Use Wink-NLP to normalize text (case folding, tokenization)
+    const doc = nlp.readDoc(atom.content);
+    const text = doc.out(nlp.its.text).toLowerCase();
+    // Regex check with smart boundaries
+    patterns.keywords.forEach((keyword) => {
+        if (currentTags.has(keyword))
+            return;
+        // Escape specialregex characters
+        const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+        // Apply boundary only if the keyword starts/ends with a word character
+        // This handles "C++" correctly (no boundary after +) vs "Java" (boundary after a)
+        const startBoundary = /^\w/.test(keyword) ? '\\b' : '';
+        const endBoundary = /\w$/.test(keyword) ? '\\b' : '';
+        const regex = new RegExp(`${startBoundary}${escaped}${endBoundary}`, 'i');
+        if (regex.test(text)) {
+            currentTags.add(keyword); // Infection!
+            changed = true;
+        }
+    });
+    // --- ENHANCEMENT: Temporal Auto-Tagging ---
+    // 1. Years (1900 - 2099)
+    // Regex matches 4 digits starting with 19 or 20, surrounded by boundaries
+    const yearMatches = text.match(/\b((?:19|20)\d{2})\b/g);
+    if (yearMatches) {
+        yearMatches.forEach(year => (!currentTags.has(year)) && (currentTags.add(year), changed = true));
+    }
+    // 2. Months (Full Names)
+    const months = [
+        "january", "february", "march", "april", "may", "june",
+        "july", "august", "september", "october", "november", "december"
+    ];
+    // Simple inclusion check for months (since we normalized text to lowercase)
+    // We check for word boundaries to avoid matching "may" inside "maybe"
+    months.forEach(month => {
+        // Create regex for word boundary match
+        const regex = new RegExp(`\\b${month}\\b`, 'i');
+        if (!currentTags.has(month) && regex.test(text)) {
+            // Capitalize first letter for the tag
+            const tag = month.charAt(0).toUpperCase() + month.slice(1);
+            currentTags.add(tag);
+            changed = true;
+        }
+    });
+    return changed ? Array.from(currentTags) : null;
+}
+/**
+ * 3. The Orchestrator (Sink)
+ * Connects the Stream to the Processor.
+ */
+export async function runInfectionLoop() {
+    console.log('🦠 Infection Protocol: Initializing...');
+    // Load the "Virus" (Master Tag List)
+    if (!fs.existsSync(TAGS_FILE)) {
+        // Fallback check for alternate location (if running from dist/)
+        console.warn(`🦠 No tag definitions found at ${TAGS_FILE}. Checking common paths...`);
+        return;
+    }
+    const viralPatterns = JSON.parse(fs.readFileSync(TAGS_FILE, 'utf-8'));
+    let infectedCount = 0;
+    // The Loop (Looks clean, acts efficient)
+    for await (const atom of atomStream()) {
+        const newTags = infectAtom(atom, viralPatterns);
+        if (newTags) {
+            // Persist the infection
+            // We update the 'tags' column. In Cozo, :update needs keys.
+            // Using a retry loop to handle potential lock contention with Ingest
+            let attempts = 0;
+            const maxAttempts = 3;
+            while (attempts < maxAttempts) {
+                try {
+                    await db.run(`UPDATE atoms SET tags = $1 WHERE id = $2`, [newTags, atom.id]);
+                    infectedCount++;
+                    if (infectedCount % 100 === 0)
+                        process.stdout.write(`.`);
+                    break; // Success
+                }
+                catch (error) {
+                    attempts++;
+                    if (attempts >= maxAttempts) {
+                        console.warn(`[Infector] Failed to update atom ${atom.id} after ${maxAttempts} attempts:`, error.message);
+                    }
+                    else {
+                        // Small backoff
+                        await new Promise(r => setTimeout(r, 100 * attempts));
+                    }
+                }
+            }
+        }
+    }
+    console.log(`\n🦠 Infection Complete. ${infectedCount} atoms infected with new context.`);
+}
+//# sourceMappingURL=infector.js.map

package/engine/dist/services/tags/infector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"infector.js","sourceRoot":"","sources":["../../../src/services/tags/infector.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,IAAI,MAAM,UAAU,CAAC;AAC5B,OAAO,KAAK,MAAM,yBAAyB,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,IAAI,CAAC;AACzB,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AAEtC,gDAAgD;AAChD,yEAAyE;AACzE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAQ,CAAC;AAE/B,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,0EAA0E;AAC1I,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAC;AAErF;;;;GAIG;AACH,KAAK,SAAS,CAAC,CAAC,UAAU,CAAC,SAAS,GAAG,EAAE;IACrC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,IAAI,EAAE,CAAC;QACV,qCAAqC;QACrC,MAAM,KAAK,GAAG;;;;;;SAMb,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;QACxD,UAAU,EAAE,CAAC;QAEb,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,sCAAsC,MAAM,CAAC,IAAI,CAAC,MAAM,oBAAoB,UAAU,GAAG,CAAC,CAAC;QAC3G,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,mBAAmB;QAC9B,CAAC;QAED,6CAA6C;QAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5B,gEAAgE;YAChE,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC;YAEtB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,6CAA6C;gBAC7C,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACJ,8CAA8C;gBAC9C,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;gBACtB,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YACpB,CAAC;YAED,MAAM,GAAG,EAAY,CAAC,CAAC,6BAA6B;YAEpD,MAAM;gBACF,EAAE,EAAE,EAAY;gBAChB,OAAO,EAAE,OAAiB;gBAC1B,IAAI,EAAG,IAAiB,IAAI,EAAE;aACjC,CAAC;QACN,CAAC;IACL,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,IAAqD,EAAE,QAAa;IAC3F,IAAI,CAAC,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE/B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,8DAA8D;IAC9D,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,IAAI,GAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAY,CAAC,WAAW,EAAE,CAAC;IAE7D,oCAAoC;IACpC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAe,EAAE,EAAE;QAC1C,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,OAAO;QAErC,iCAAiC;QACjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;QAE/D,uEAAuE;QACvE,kFAAkF;QAClF,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAErD,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,aAAa,GAAG,OAAO,GAAG,WAAW,EAAE,EAAE,GAAG,CAAC,CAAC;QAE1E,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa;YACvC,OAAO,GAAG,IAAI,CAAC;QACnB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6CAA6C;IAE7C,yBAAyB;IACzB,0EAA0E;IAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACxD,IAAI,WAAW,EAAE,CAAC;QACd,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IACrG,CAAC;IAED,yBAAyB;IACzB,MAAM,MAAM,GAAG;QACX,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;QACtD,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU;KACnE,CAAC;IAEF,4EAA4E;IAC5E,sEAAsE;IACtE,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QACnB,uCAAuC;QACvC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,sCAAsC;YACtC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC3D,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACnB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IAClC,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IAEtD,qCAAqC;IACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC5B,gEAAgE;QAChE,OAAO,CAAC,IAAI,CAAC,kCAAkC,SAAS,4BAA4B,CAAC,CAAC;QACtF,OAAO;IACX,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,yCAAyC;IACzC,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,UAAU,EAAE,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAEhD,IAAI,OAAO,EAAE,CAAC;YACV,wBAAwB;YACxB,4DAA4D;YAC5D,qEAAqE;YACrE,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,MAAM,WAAW,GAAG,CAAC,CAAC;YACtB,OAAO,QAAQ,GAAG,WAAW,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACD,MAAM,EAAE,CAAC,GAAG,CACR,0CAA0C,EAC1C,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CACrB,CAAC;oBAEF,aAAa,EAAE,CAAC;oBAChB,IAAI,aAAa,GAAG,GAAG,KAAK,CAAC;wBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBACzD,MAAM,CAAC,UAAU;gBACrB,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBAClB,QAAQ,EAAE,CAAC;oBACX,IAAI,QAAQ,IAAI,WAAW,EAAE,CAAC;wBAC1B,OAAO,CAAC,IAAI,CAAC,oCAAoC,IAAI,CAAC,EAAE,UAAU,WAAW,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC9G,CAAC;yBAAM,CAAC;wBACJ,gBAAgB;wBAChB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC;oBAC1D,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,4BAA4B,aAAa,mCAAmC,CAAC,CAAC;AAC9F,CAAC"}

package/engine/dist/services/tags/tag-auditor.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Tag Quality Auditor
+ *
+ * Analyzes tagging patterns to identify:
+ * 1. Under-tagged atoms (high content, few tags)
+ * 2. Over-tagged atoms (noise)
+ * 3. Orphan tags (used once, never retrieved)
+ * 4. Tag clusters (tags that always appear together)
+ * 5. Missing tags (common terms not used as tags)
+ *
+ * Usage:
+ *   const auditor = new TagAuditor();
+ *   const report = await auditor.generateAuditReport();
+ *   console.log(report);
+ */
+interface AuditReport {
+    totalAtoms: number;
+    totalTags: number;
+    underTagged: UnderTaggedAtom[];
+    orphanTags: string[];
+    tagClusters: string[][];
+    suggestions: TagSuggestion[];
+    statistics: TagStatistics;
+}
+interface UnderTaggedAtom {
+    id: string;
+    source: string;
+    contentLength: number;
+    tagCount: number;
+    suggestedTags: string[];
+}
+interface TagSuggestion {
+    atomId: string;
+    suggestedTags: string[];
+    confidence: number;
+}
+interface TagStatistics {
+    avgTagsPerAtom: number;
+    medianTagsPerAtom: number;
+    maxTagsInAtom: number;
+    uniqueTags: number;
+    tagsUsedOnce: number;
+}
+export declare class TagAuditor {
+    /**
+     * Find under-tagged content
+     */
+    findUnderTaggedAtoms(minContentLength?: number, maxTags?: number): Promise<UnderTaggedAtom[]>;
+    /**
+     * Find orphan tags (used only once)
+     */
+    findOrphanTags(minAtoms?: number): Promise<string[]>;
+    /**
+     * Find tag clusters (tags that always appear together)
+     */
+    findTagClusters(minSupport?: number): Promise<string[][]>;
+    /**
+     * Suggest tags for an atom based on content
+     */
+    suggestTagsForAtom(atomId: string, limit?: number, context?: {
+        content: string;
+        existingTags: string[];
+        allTags: string[];
+    }): Promise<string[]>;
+    /**
+     * Get tag statistics
+     */
+    getTagStatistics(): Promise<TagStatistics>;
+    /**
+     * Generate comprehensive audit report
+     */
+    generateAuditReport(): Promise<AuditReport>;
+    private getTotalAtoms;
+    private getTotalTags;
+}
+export {};
+//# sourceMappingURL=tag-auditor.d.ts.map

package/engine/dist/services/tags/tag-auditor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag-auditor.d.ts","sourceRoot":"","sources":["../../../src/services/tags/tag-auditor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,UAAU,WAAW;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,eAAe,EAAE,CAAC;IAC/B,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,EAAE,CAAC;IACxB,WAAW,EAAE,aAAa,EAAE,CAAC;IAC7B,UAAU,EAAE,aAAa,CAAC;CAC3B;AAED,UAAU,eAAe;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,UAAU,aAAa;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,aAAa;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,UAAU;IACrB;;OAEG;IACG,oBAAoB,CAAC,gBAAgB,GAAE,MAAY,EAAE,OAAO,GAAE,MAAU,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiD3G;;OAEG;IACG,cAAc,CAAC,QAAQ,GAAE,MAAY,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAyB/D;;OAEG;IACG,eAAe,CAAC,UAAU,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAkEnE;;OAEG;IACG,kBAAkB,CACtB,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,MAAU,EACjB,OAAO,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,GACvE,OAAO,CAAC,MAAM,EAAE,CAAC;IA8DpB;;OAEG;IACG,gBAAgB,IAAI,OAAO,CAAC,aAAa,CAAC;IAiDhD;;OAEG;IACG,mBAAmB,IAAI,OAAO,CAAC,WAAW,CAAC;YAoCnC,aAAa;YAKb,YAAY;CAK3B"}