@rbalchii/anchor-engine 4.7.0

package/engine/dist/utils/structured-logger.js

@@ -0,0 +1,332 @@
+/**
+ * Structured Logger for Anchor Engine
+ *
+ * Implements standardized, structured logging with metrics collection
+ */
+import winston from 'winston';
+import DailyRotateFile from 'winston-daily-rotate-file';
+import path from 'path';
+import fs from 'fs';
+import { format } from 'winston';
+import { fileURLToPath } from 'url';
+// Get absolute path to project root (anchor-os directory)
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PROJECT_ROOT = path.resolve(__dirname, '../../..');
+// Define log levels with numerical values
+const logLevels = {
+    error: 0,
+    warn: 1,
+    info: 2,
+    verbose: 3,
+    debug: 4,
+    silly: 5
+};
+// Create logs directory at project root
+const LOGS_DIR = path.join(PROJECT_ROOT, 'logs');
+if (!fs.existsSync(LOGS_DIR)) {
+    fs.mkdirSync(LOGS_DIR, { recursive: true });
+}
+/**
+ * Truncate log files to last N lines to prevent unbounded growth
+ * Runs at startup to clean up existing logs
+ */
+function truncateLogFiles(maxLines = 10000) {
+    try {
+        const files = fs.readdirSync(LOGS_DIR);
+        let totalTruncated = 0;
+        for (const file of files) {
+            if (!file.endsWith('.log') && !file.includes('.log.'))
+                continue;
+            const filePath = path.join(LOGS_DIR, file);
+            const stats = fs.statSync(filePath);
+            // Skip if file is small (< 1MB)
+            if (stats.size < 1024 * 1024)
+                continue;
+            // Read file and truncate to last N lines
+            const content = fs.readFileSync(filePath, 'utf-8');
+            const lines = content.split('\n');
+            if (lines.length > maxLines) {
+                const truncated = lines.slice(-maxLines).join('\n');
+                fs.writeFileSync(filePath, truncated, 'utf-8');
+                totalTruncated++;
+                console.log(`[Logger] Truncated ${file} from ${lines.length} to ${maxLines} lines`);
+            }
+        }
+        if (totalTruncated > 0) {
+            console.log(`[Logger] Truncated ${totalTruncated} log files to ${maxLines} lines max`);
+        }
+    }
+    catch (e) {
+        console.error('[Logger] Failed to truncate logs:', e);
+    }
+}
+// Truncate logs at startup
+truncateLogFiles(10000);
+// Custom format for structured logging
+const structuredFormat = winston.format.combine(winston.format.timestamp(), winston.format.errors({ stack: true }), winston.format.splat(), winston.format.json());
+// Create logger instance
+const logger = winston.createLogger({
+    levels: logLevels,
+    level: 'silly', // Capture all log levels including debug
+    format: structuredFormat,
+    transports: [
+        // Main anchor_engine.log file with size-based rotation (10KB)
+        new DailyRotateFile({
+            filename: path.join(LOGS_DIR, 'anchor_engine.log'),
+            datePattern: 'YYYY-MM-DD',
+            zippedArchive: false,
+            maxSize: '10k',
+            maxFiles: '7d',
+            format: format.combine(format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }), format.errors({ stack: true }), format.splat(), format.printf(({ timestamp, level, message, ...metadata }) => {
+                const metaStr = Object.keys(metadata).length > 0 ? ` ${JSON.stringify(metadata)}` : '';
+                return `[${timestamp}] [${level.toUpperCase()}] ${message}${metaStr}`;
+            }))
+        }),
+        // Separate error file
+        new DailyRotateFile({
+            level: 'error',
+            filename: path.join(LOGS_DIR, 'anchor_engine_error-%DATE%.log'),
+            datePattern: 'YYYY-MM-DD',
+            zippedArchive: true,
+            maxSize: '10k',
+            maxFiles: '14d',
+            format: format.combine(format.timestamp(), format.errors({ stack: true }), format.splat(), format.json())
+        }),
+        // Console transport for development
+        new winston.transports.Console({
+            format: format.combine(format.colorize(), format.printf(({ level, message, timestamp, ...metadata }) => {
+                let msg = `${timestamp} [${level}] ${message}`;
+                if (Object.keys(metadata).length > 0) {
+                    msg += ` ${JSON.stringify(metadata)}`;
+                }
+                return msg;
+            }))
+        })
+    ]
+});
+// Performance metrics tracker
+class MetricsTracker {
+    metrics;
+    startTime;
+    constructor() {
+        this.metrics = new Map();
+        this.startTime = Date.now();
+    }
+    startTimer(operation) {
+        const startTime = Date.now();
+        return () => {
+            const duration = Date.now() - startTime;
+            this.recordMetric(operation, duration);
+            return duration;
+        };
+    }
+    recordMetric(operation, duration) {
+        if (!this.metrics.has(operation)) {
+            this.metrics.set(operation, { count: 0, total: 0, min: Infinity, max: -Infinity, last: 0 });
+        }
+        const metric = this.metrics.get(operation);
+        metric.count++;
+        metric.total += duration;
+        metric.min = Math.min(metric.min, duration);
+        metric.max = Math.max(metric.max, duration);
+        metric.last = duration;
+    }
+    getMetrics(operation) {
+        return this.metrics.get(operation);
+    }
+    getAllMetrics() {
+        const result = {};
+        const now = Date.now();
+        for (const [name, data] of this.metrics.entries()) {
+            result[name] = {
+                count: data.count,
+                total: data.total,
+                average: data.count > 0 ? data.total / data.count : 0,
+                min: data.min === Infinity ? 0 : data.min,
+                max: data.max === -Infinity ? 0 : data.max,
+                last: data.last,
+                uptime: now - this.startTime
+            };
+        }
+        return result;
+    }
+    reset() {
+        this.metrics.clear();
+    }
+    /**
+     * Prune old metrics to free memory (called during idle cleanup)
+     * Removes metrics older than TTL and enforces maximum size limit
+     */
+    pruneOldMetrics() {
+        const METRIC_TTL_MS = 10 * 60 * 1000; // 10 minutes TTL
+        const MAX_METRICS = 500; // Maximum number of metrics to keep
+        const now = Date.now();
+        // Remove metrics that haven't been updated in TTL period
+        let prunedCount = 0;
+        for (const [key, metric] of this.metrics.entries()) {
+            // If last operation was more than TTL ago, remove it
+            if (metric.last && (now - metric.last) > METRIC_TTL_MS) {
+                this.metrics.delete(key);
+                prunedCount++;
+            }
+        }
+        // Enforce hard limit if still too many metrics
+        if (this.metrics.size > MAX_METRICS) {
+            const keys = Array.from(this.metrics.keys());
+            const toDelete = keys.slice(0, keys.length - MAX_METRICS);
+            for (const key of toDelete) {
+                this.metrics.delete(key);
+                prunedCount++;
+            }
+        }
+        if (prunedCount > 0) {
+            console.log(`[StructuredLogger] Pruned ${prunedCount} old metrics (${this.metrics.size} remaining)`);
+        }
+    }
+}
+// Initialize metrics tracker
+const metricsTracker = new MetricsTracker();
+// Enhanced logging functions with metrics
+export const logWithContext = {
+    /**
+     * Log an info message with context and metrics
+     */
+    info: (message, context) => {
+        if (context && context.metrics) {
+            for (const [name, value] of Object.entries(context.metrics)) {
+                metricsTracker.recordMetric(name, value);
+            }
+        }
+        logger.info(message, {
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log a warning with context
+     */
+    warn: (message, context) => {
+        logger.warn(message, {
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log an error with context
+     */
+    error: (message, error, context) => {
+        logger.error(message, {
+            error: error instanceof Error ? { message: error.message, stack: error.stack } : error,
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log a debug message with context
+     */
+    debug: (message, context) => {
+        logger.debug(message, {
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log a silly/verbose message with context
+     */
+    silly: (message, context) => {
+        logger.silly(message, {
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log a performance metric
+     */
+    performance: (operation, duration, context) => {
+        metricsTracker.recordMetric(operation, duration);
+        logger.info('PERFORMANCE_METRIC', {
+            operation,
+            duration_ms: duration,
+            average: metricsTracker.getMetrics(operation)?.total / metricsTracker.getMetrics(operation)?.count,
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Start a timed operation
+     */
+    startTimer: (operation) => {
+        return metricsTracker.startTimer(operation);
+    },
+    /**
+     * Get current metrics
+     */
+    getMetrics: () => {
+        return metricsTracker.getAllMetrics();
+    },
+    /**
+     * Log ingestion event
+     */
+    ingestion: (status, details) => {
+        metricsTracker.recordMetric('ingestion_attempts', 1);
+        if (status === 'success') {
+            metricsTracker.recordMetric('ingestion_successes', 1);
+        }
+        else if (status === 'failed') {
+            metricsTracker.recordMetric('ingestion_failures', 1);
+        }
+        logger.info('INGESTION_EVENT', {
+            event: 'ingestion',
+            status,
+            details,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log search event
+     */
+    search: (query, resultCount, duration, context) => {
+        metricsTracker.recordMetric('search_queries', 1);
+        metricsTracker.recordMetric('search_results', resultCount);
+        metricsTracker.recordMetric('search_duration', duration);
+        logger.info('SEARCH_EVENT', {
+            event: 'search',
+            query: query.substring(0, 100), // Truncate long queries
+            resultCount,
+            duration_ms: duration,
+            context,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    },
+    /**
+     * Log system health event
+     */
+    health: (status, details) => {
+        logger.info('HEALTH_EVENT', {
+            event: 'health-check',
+            status,
+            details,
+            pid: process.pid,
+            module: 'structured-logger'
+        });
+    }
+};
+// Export the base logger as well
+export { logger, metricsTracker };
+// Export a function to get formatted metrics for monitoring endpoints
+export function getFormattedMetrics() {
+    const metrics = metricsTracker.getAllMetrics();
+    return JSON.stringify(metrics, null, 2);
+}
+// Export alias for backward compatibility
+export const StructuredLogger = logWithContext;
+//# sourceMappingURL=structured-logger.js.map

package/engine/dist/utils/structured-logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-logger.js","sourceRoot":"","sources":["../../src/utils/structured-logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,eAAe,MAAM,2BAA2B,CAAC;AACxD,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,0DAA0D;AAC1D,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAEzD,0CAA0C;AAC1C,MAAM,SAAS,GAAG;IAChB,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,OAAO,EAAE,CAAC;IACV,KAAK,EAAE,CAAC;IACR,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,wCAAwC;AACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;AACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC7B,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,WAAmB,KAAK;IAChD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAEhE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC3C,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAEpC,gCAAgC;YAChC,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI;gBAAE,SAAS;YAEvC,yCAAyC;YACzC,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAElC,IAAI,KAAK,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;gBAC/C,cAAc,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,SAAS,KAAK,CAAC,MAAM,OAAO,QAAQ,QAAQ,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;QAED,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,sBAAsB,cAAc,iBAAiB,QAAQ,YAAY,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,mCAAmC,EAAE,CAAC,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,gBAAgB,CAAC,KAAK,CAAC,CAAC;AAExB,uCAAuC;AACvC,MAAM,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAC7C,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,EAC1B,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EACtC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EACtB,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CACtB,CAAC;AAEF,yBAAyB;AACzB,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,MAAM,EAAE,SAAS;IACjB,KAAK,EAAE,OAAO,EAAE,yCAAyC;IACzD,MAAM,EAAE,gBAAgB;IACxB,UAAU,EAAE;QACV,8DAA8D;QAC9D,IAAI,eAAe,CAAC;YAClB,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC;YAClD,WAAW,EAAE,YAAY;YACzB,aAAa,EAAE,KAAK;YACpB,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,MAAM,CAAC,OAAO,CACpB,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,EACnD,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAC9B,MAAM,CAAC,KAAK,EAAE,EACd,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;gBAC3D,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvF,OAAO,IAAI,SAAS,MAAM,KAAK,CAAC,WAAW,EAAE,KAAK,OAAO,GAAG,OAAO,EAAE,CAAC;YACxE,CAAC,CAAC,CACH;SACF,CAAC;QACF,sBAAsB;QACtB,IAAI,eAAe,CAAC;YAClB,KAAK,EAAE,OAAO;YACd,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,gCAAgC,CAAC;YAC/D,WAAW,EAAE,YAAY;YACzB,aAAa,EAAE,IAAI;YACnB,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE,KAAK;YACf,MAAM,EAAE,MAAM,CAAC,OAAO,CACpB,MAAM,CAAC,SAAS,EAAE,EAClB,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,EAC9B,MAAM,CAAC,KAAK,EAAE,EACd,MAAM,CAAC,IAAI,EAAE,CACd;SACF,CAAC;QACF,oCAAoC;QACpC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,OAAO,CACpB,MAAM,CAAC,QAAQ,EAAE,EACjB,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,QAAQ,EAAE,EAAE,EAAE;gBAC3D,IAAI,GAAG,GAAG,GAAG,SAAS,KAAK,KAAK,KAAK,OAAO,EAAE,CAAC;gBAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrC,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC,CAAC,CACH;SACF,CAAC;KACH;CACF,CAAC,CAAC;AAEH,8BAA8B;AAC9B,MAAM,cAAc;IACV,OAAO,CAAwF;IAC/F,SAAS,CAAS;IAE1B;QACE,IAAI,CAAC,OAAO,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,UAAU,CAAC,SAAiB;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,OAAO,GAAG,EAAE;YACV,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACxC,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACvC,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,SAAiB,EAAE,QAAgB;QAC9C,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;QAC5C,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC;QACzB,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,GAAG,QAAQ,CAAC;IACzB,CAAC;IAED,UAAU,CAAC,SAAiB;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACrC,CAAC;IAED,aAAa;QACX,MAAM,MAAM,GAAwB,EAAE,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,MAAM,CAAC,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,OAAO,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrD,GAAG,EAAE,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG;gBACzC,GAAG,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG;gBAC1C,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC,SAAS;aAC7B,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,eAAe;QACb,MAAM,aAAa,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,iBAAiB;QACvD,MAAM,WAAW,GAAG,GAAG,CAAC,CAAC,oCAAoC;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,yDAAyD;QACzD,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,qDAAqD;YACrD,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,EAAE,CAAC;gBACvD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,WAAW,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;YAC1D,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;QAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,6BAA6B,WAAW,iBAAiB,IAAI,CAAC,OAAO,CAAC,IAAI,aAAa,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;CACF;AAED,6BAA6B;AAC7B,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAC;AAE5C,0CAA0C;AAC1C,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B;;OAEG;IACH,IAAI,EAAE,CAAC,OAAe,EAAE,OAA6B,EAAE,EAAE;QACvD,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5D,cAAc,CAAC,YAAY,CAAC,IAAc,EAAE,KAAe,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;YACnB,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI,EAAE,CAAC,OAAe,EAAE,OAA6B,EAAE,EAAE;QACvD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;YACnB,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,EAAE,CAAC,OAAe,EAAE,KAAsB,EAAE,OAA6B,EAAE,EAAE;QAChF,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;YACpB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK;YACtF,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,EAAE,CAAC,OAAe,EAAE,OAA6B,EAAE,EAAE;QACxD,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;YACpB,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,EAAE,CAAC,OAAe,EAAE,OAA6B,EAAE,EAAE;QACxD,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE;YACpB,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,WAAW,EAAE,CAAC,SAAiB,EAAE,QAAgB,EAAE,OAA6B,EAAE,EAAE;QAClF,cAAc,CAAC,YAAY,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAEjD,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE;YAChC,SAAS;YACT,WAAW,EAAE,QAAQ;YACrB,OAAO,EAAE,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,KAAM,GAAG,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,KAAM;YACpG,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,UAAU,EAAE,CAAC,SAAiB,EAAE,EAAE;QAChC,OAAO,cAAc,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,UAAU,EAAE,GAAG,EAAE;QACf,OAAO,cAAc,CAAC,aAAa,EAAE,CAAC;IACxC,CAAC;IAED;;OAEG;IACH,SAAS,EAAE,CAAC,MAAwC,EAAE,OAA4B,EAAE,EAAE;QACpF,cAAc,CAAC,YAAY,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;QACrD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,cAAc,CAAC,YAAY,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,cAAc,CAAC,YAAY,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAC7B,KAAK,EAAE,WAAW;YAClB,MAAM;YACN,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,EAAE,CAAC,KAAa,EAAE,WAAmB,EAAE,QAAgB,EAAE,OAA6B,EAAE,EAAE;QAC9F,cAAc,CAAC,YAAY,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC;QACjD,cAAc,CAAC,YAAY,CAAC,gBAAgB,EAAE,WAAW,CAAC,CAAC;QAC3D,cAAc,CAAC,YAAY,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC;QAEzD,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE;YAC1B,KAAK,EAAE,QAAQ;YACf,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,wBAAwB;YACxD,WAAW;YACX,WAAW,EAAE,QAAQ;YACrB,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,EAAE,CAAC,MAA4C,EAAE,OAA4B,EAAE,EAAE;QACrF,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE;YAC1B,KAAK,EAAE,cAAc;YACrB,MAAM;YACN,OAAO;YACP,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,mBAAmB;SAC5B,CAAC,CAAC;IACL,CAAC;CACF,CAAC;AAEF,iCAAiC;AACjC,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;AAElC,sEAAsE;AACtE,MAAM,UAAU,mBAAmB;IACjC,MAAM,OAAO,GAAG,cAAc,CAAC,aAAa,EAAE,CAAC;IAC/C,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,0CAA0C;AAC1C,MAAM,CAAC,MAAM,gBAAgB,GAAG,cAAc,CAAC"}

package/engine/dist/utils/tag-cleanup.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Tag Cleanup Script - Run on startup to prune blacklisted tags
+ *
+ * Removes all blacklisted tags from existing atoms in the database.
+ * This ensures the tag space stays clean even for previously ingested content.
+ */
+/**
+ * Main cleanup function - run on startup
+ */
+export declare function cleanupBlacklistedTags(): Promise<void>;
+//# sourceMappingURL=tag-cleanup.d.ts.map

package/engine/dist/utils/tag-cleanup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag-cleanup.d.ts","sourceRoot":"","sources":["../../src/utils/tag-cleanup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA0FH;;GAEG;AACH,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC,CAoC5D"}

package/engine/dist/utils/tag-cleanup.js

@@ -0,0 +1,111 @@
+/**
+ * Tag Cleanup Script - Run on startup to prune blacklisted tags
+ *
+ * Removes all blacklisted tags from existing atoms in the database.
+ * This ensures the tag space stays clean even for previously ingested content.
+ */
+import { db } from '../core/db.js';
+import { filterTags } from '../utils/tag-filter.js';
+const BATCH_SIZE = 1000;
+/**
+ * Get all distinct tags from the database
+ */
+async function getAllTags() {
+    const result = await db.run('SELECT DISTINCT tag FROM tags');
+    return result.rows?.map((row) => row.tag) || [];
+}
+/**
+ * Find blacklisted tags
+ */
+async function findBlacklistedTags() {
+    const allTags = await getAllTags();
+    const blacklisted = [];
+    for (const tag of allTags) {
+        const filtered = filterTags([tag]);
+        if (filtered.length === 0) {
+            blacklisted.push(tag);
+        }
+    }
+    return blacklisted;
+}
+/**
+ * Remove a tag from all atoms
+ */
+async function removeTag(tag) {
+    const result = await db.run('DELETE FROM tags WHERE tag = $1', [tag]);
+    return result.changes || 0;
+}
+/**
+ * Cleanup atoms with blacklisted tags
+ * Updates the tags TEXT[] array in the atoms table
+ */
+async function cleanupAtomTags() {
+    let totalCleaned = 0;
+    try {
+        // Get atoms that have tags (tags is TEXT[] not JSONB)
+        const result = await db.run(`
+      SELECT id, tags 
+      FROM atoms 
+      WHERE tags IS NOT NULL 
+      AND array_length(tags, 1) > 0
+    `);
+        if (!result.rows || result.rows.length === 0) {
+            return 0;
+        }
+        console.log(`[TagCleanup] Checking ${result.rows.length} atoms for blacklisted tags...`);
+        for (const row of result.rows) {
+            const atomId = row.id;
+            const currentTags = row.tags || [];
+            // Filter out blacklisted tags
+            const filteredTags = filterTags(currentTags);
+            // If tags were removed, update the atom
+            if (filteredTags.length < currentTags.length) {
+                await db.run('UPDATE atoms SET tags = $1 WHERE id = $2', [filteredTags, atomId]);
+                totalCleaned++;
+                if (totalCleaned % 100 === 0) {
+                    console.log(`[TagCleanup] Cleaned ${totalCleaned} atoms...`);
+                }
+            }
+        }
+        return totalCleaned;
+    }
+    catch (error) {
+        console.error('[TagCleanup] Error cleaning atom tags:', error);
+        return 0;
+    }
+}
+/**
+ * Main cleanup function - run on startup
+ */
+export async function cleanupBlacklistedTags() {
+    const startTime = Date.now();
+    console.log('[TagCleanup] Starting tag cleanup on startup...');
+    try {
+        // Step 1: Remove blacklisted tags from tags table
+        const blacklistedTags = await findBlacklistedTags();
+        if (blacklistedTags.length > 0) {
+            console.log(`[TagCleanup] Found ${blacklistedTags.length} blacklisted tags to remove:`);
+            console.log(`[TagCleanup]   ${blacklistedTags.slice(0, 20).join(', ')}${blacklistedTags.length > 20 ? ` ... and ${blacklistedTags.length - 20} more` : ''}`);
+            let totalDeleted = 0;
+            for (const tag of blacklistedTags) {
+                const deleted = await removeTag(tag);
+                totalDeleted += deleted;
+            }
+            console.log(`[TagCleanup] Removed ${totalDeleted} tag entries from ${blacklistedTags.length} blacklisted tags`);
+        }
+        else {
+            console.log('[TagCleanup] No blacklisted tags found in tags table');
+        }
+        // Step 2: Cleanup tags in atoms table
+        const atomsCleaned = await cleanupAtomTags();
+        if (atomsCleaned > 0) {
+            console.log(`[TagCleanup] Updated ${atomsCleaned} atoms with filtered tags`);
+        }
+        const duration = ((Date.now() - startTime) / 1000).toFixed(2);
+        console.log(`[TagCleanup] ✅ Cleanup complete in ${duration}s`);
+    }
+    catch (error) {
+        console.error('[TagCleanup] ❌ Cleanup failed:', error);
+    }
+}
+//# sourceMappingURL=tag-cleanup.js.map

package/engine/dist/utils/tag-cleanup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag-cleanup.js","sourceRoot":"","sources":["../../src/utils/tag-cleanup.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,EAAE,EAAE,MAAM,eAAe,CAAC;AACnC,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB;;GAEG;AACH,KAAK,UAAU,UAAU;IACvB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7D,OAAO,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AACvD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,mBAAmB;IAChC,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;IACnC,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,SAAS,CAAC,GAAW;IAClC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,iCAAiC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,OAAO,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,eAAe;IAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,IAAI,CAAC;QACH,sDAAsD;QACtD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC;;;;;KAK3B,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,CAAC;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,IAAI,CAAC,MAAM,gCAAgC,CAAC,CAAC;QAEzF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;YAEnC,8BAA8B;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;YAE7C,wCAAwC;YACxC,IAAI,YAAY,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;gBAC7C,MAAM,EAAE,CAAC,GAAG,CACV,0CAA0C,EAC1C,CAAC,YAAY,EAAE,MAAM,CAAC,CACvB,CAAC;gBACF,YAAY,EAAE,CAAC;gBAEf,IAAI,YAAY,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,wBAAwB,YAAY,WAAW,CAAC,CAAC;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;QAC/D,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,OAAO,CAAC,GAAG,CAAC,iDAAiD,CAAC,CAAC;IAE/D,IAAI,CAAC;QACH,kDAAkD;QAClD,MAAM,eAAe,GAAG,MAAM,mBAAmB,EAAE,CAAC;QAEpD,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,sBAAsB,eAAe,CAAC,MAAM,8BAA8B,CAAC,CAAC;YACxF,OAAO,CAAC,GAAG,CAAC,kBAAkB,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,YAAY,eAAe,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAE7J,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;gBACrC,YAAY,IAAI,OAAO,CAAC;YAC1B,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,YAAY,qBAAqB,eAAe,CAAC,MAAM,mBAAmB,CAAC,CAAC;QAClH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACtE,CAAC;QAED,sCAAsC;QACtC,MAAM,YAAY,GAAG,MAAM,eAAe,EAAE,CAAC;QAC7C,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,wBAAwB,YAAY,2BAA2B,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,sCAAsC,QAAQ,GAAG,CAAC,CAAC;IAEjE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IACzD,CAAC;AACH,CAAC"}

package/engine/dist/utils/tag-filter.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Tag Filter - Blacklist for noise/system tags
+ *
+ * Prevents low-value tags from being stored in the database.
+ * Applied at ingestion time to keep the tag space clean.
+ */
+/**
+ * Check if a tag should be filtered out
+ */
+export declare function isTagBlacklisted(tag: string): boolean;
+/**
+ * Filter an array of tags, removing blacklisted ones
+ */
+export declare function filterTags(tags: string[]): string[];
+/**
+ * Filter tags and log what was removed (for debugging)
+ */
+export declare function filterTagsWithLogging(tags: string[], context?: string): string[];
+//# sourceMappingURL=tag-filter.d.ts.map

package/engine/dist/utils/tag-filter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag-filter.d.ts","sourceRoot":"","sources":["../../src/utils/tag-filter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAqGH;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAkBrD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAOnD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAmBhF"}

package/engine/dist/utils/tag-filter.js

@@ -0,0 +1,147 @@
+/**
+ * Tag Filter - Blacklist for noise/system tags
+ *
+ * Prevents low-value tags from being stored in the database.
+ * Applied at ingestion time to keep the tag space clean.
+ */
+// Blacklist patterns - tags matching these will be rejected
+const TAG_BLACKLIST_PATTERNS = [
+    // Color codes (hex)
+    /^#[0-9a-fA-F]{3,8}$/,
+    // Pure numbers or too short
+    /^#\d{1,3}$/,
+    /^#_\w*$/,
+    /^#__[\w\d_]+$/, // __lottie_element_, etc.
+    // HTML/DOM artifacts
+    /^#btn\b/,
+    /^#class\b/,
+    /^#div\b/,
+    /^#id\b/,
+    /^#span\b/,
+    /^#href\b/,
+    /^#src\b/,
+    // Code artifacts
+    /^#fn\b/,
+    /^#elif\b/,
+    /^#else\b/,
+    /^#endif\b/,
+    /^#ifdef\b/,
+    /^#ifndef\b/,
+    /^#include\b/,
+    /^#define\b/,
+    /^#pragma\b/,
+    // Scraping artifacts (Wikipedia, etc.)
+    /^#cite_note/,
+    /^#cite_ref/,
+    /^#amp_tf/,
+    /^#details_of_atom/,
+    /^#entry_lin/,
+    /^#entry_links/,
+    /^#entry_metadata/,
+    /^#feed_metadata/,
+    /^#opensearch_extension/,
+    /^#extension_elements/,
+    /^#simple_examples/,
+    /^#query_interface/,
+    /^#api_response/,
+    /^#response_example/,
+    /^#examples?$/,
+    /^#overview$/,
+    /^#preface$/,
+    /^#appendix/,
+    /^#appendices$/,
+    /^#bib\b/,
+    /^#ref\b/,
+    /^#fn\b/,
+    // Error/artifact tags
+    /^#incorrect_/,
+    /^#error_/,
+    /^#null\b/,
+    /^#undefined\b/,
+    /^#nan\b/,
+    // Too generic / low signal
+    /^#slow_pickup$/,
+    /^#late_night$/,
+    /^#early_morning$/,
+    /^#monday\b/,
+    /^#tuesday\b/,
+    /^#wednesday\b/,
+    /^#thursday\b/,
+    /^#friday\b/,
+    /^#saturday\b/,
+    /^#sunday\b/,
+    /^#manual\b/,
+    /^#manually_/,
+    /^#test_/,
+    /^#tmp\b/,
+    /^#temp\b/,
+    /^#untagged$/,
+    // Old project names / deprecated
+    /^#agentgpt$/,
+    /^#babyagi$/,
+    /^#autogen$/,
+    /^#chimaera$/,
+    // System tags (not user-meaningful)
+    /^#manually_quarantined$/,
+    /^#quarantined$/,
+    /^#system$/,
+    /^#internal$/,
+    /^#external$/,
+];
+// Also maintain a simple blacklist for exact matches
+const TAG_BLACKLIST_EXACT = new Set([
+    '#_', '#0', '#1', '#2', '#3', '#4', '#5', '#6', '#7', '#8', '#9',
+    '#00', '#000', '#0000', '#00000', '#000000',
+]);
+/**
+ * Check if a tag should be filtered out
+ */
+export function isTagBlacklisted(tag) {
+    if (!tag || typeof tag !== 'string')
+        return true;
+    const normalizedTag = tag.trim();
+    // Check exact blacklist
+    if (TAG_BLACKLIST_EXACT.has(normalizedTag)) {
+        return true;
+    }
+    // Check pattern blacklist
+    for (const pattern of TAG_BLACKLIST_PATTERNS) {
+        if (pattern.test(normalizedTag)) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Filter an array of tags, removing blacklisted ones
+ */
+export function filterTags(tags) {
+    if (!tags || !Array.isArray(tags))
+        return [];
+    return tags
+        .filter(tag => !isTagBlacklisted(tag))
+        .map(tag => tag.trim())
+        .filter(tag => tag.length > 0);
+}
+/**
+ * Filter tags and log what was removed (for debugging)
+ */
+export function filterTagsWithLogging(tags, context) {
+    if (!tags || !Array.isArray(tags))
+        return [];
+    const filtered = tags.filter(tag => {
+        if (isTagBlacklisted(tag)) {
+            if (context) {
+                console.debug(`[TagFilter] ${context}: Filtered out "${tag}"`);
+            }
+            return false;
+        }
+        return true;
+    });
+    const removedCount = tags.length - filtered.length;
+    if (removedCount > 0 && context) {
+        console.debug(`[TagFilter] ${context}: Removed ${removedCount}/${tags.length} blacklisted tags`);
+    }
+    return filtered.map(tag => tag.trim());
+}
+//# sourceMappingURL=tag-filter.js.map

package/engine/dist/utils/tag-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tag-filter.js","sourceRoot":"","sources":["../../src/utils/tag-filter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,4DAA4D;AAC5D,MAAM,sBAAsB,GAAG;IAC7B,oBAAoB;IACpB,qBAAqB;IAErB,4BAA4B;IAC5B,YAAY;IACZ,SAAS;IACT,eAAe,EAAG,0BAA0B;IAE5C,qBAAqB;IACrB,SAAS;IACT,WAAW;IACX,SAAS;IACT,QAAQ;IACR,UAAU;IACV,UAAU;IACV,SAAS;IAET,iBAAiB;IACjB,QAAQ;IACR,UAAU;IACV,UAAU;IACV,WAAW;IACX,WAAW;IACX,YAAY;IACZ,aAAa;IACb,YAAY;IACZ,YAAY;IAEZ,uCAAuC;IACvC,aAAa;IACb,YAAY;IACZ,UAAU;IACV,mBAAmB;IACnB,aAAa;IACb,eAAe;IACf,kBAAkB;IAClB,iBAAiB;IACjB,wBAAwB;IACxB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,gBAAgB;IAChB,oBAAoB;IACpB,cAAc;IACd,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,eAAe;IACf,SAAS;IACT,SAAS;IACT,QAAQ;IAER,sBAAsB;IACtB,cAAc;IACd,UAAU;IACV,UAAU;IACV,eAAe;IACf,SAAS;IAET,2BAA2B;IAC3B,gBAAgB;IAChB,eAAe;IACf,kBAAkB;IAClB,YAAY;IACZ,aAAa;IACb,eAAe;IACf,cAAc;IACd,YAAY;IACZ,cAAc;IACd,YAAY;IACZ,YAAY;IACZ,aAAa;IACb,SAAS;IACT,SAAS;IACT,UAAU;IACV,aAAa;IAEb,iCAAiC;IACjC,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,aAAa;IAEb,oCAAoC;IACpC,yBAAyB;IACzB,gBAAgB;IAChB,WAAW;IACX,aAAa;IACb,aAAa;CACd,CAAC;AAEF,qDAAqD;AACrD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IAChE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS;CAC5C,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjD,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAEjC,wBAAwB;IACxB,IAAI,mBAAmB,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0BAA0B;IAC1B,KAAK,MAAM,OAAO,IAAI,sBAAsB,EAAE,CAAC;QAC7C,IAAI,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAc;IACvC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7C,OAAO,IAAI;SACR,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;SACrC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;SACtB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAc,EAAE,OAAgB;IACpE,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;QACjC,IAAI,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,eAAe,OAAO,mBAAmB,GAAG,GAAG,CAAC,CAAC;YACjE,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IACnD,IAAI,YAAY,GAAG,CAAC,IAAI,OAAO,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,eAAe,OAAO,aAAa,YAAY,IAAI,IAAI,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACnG,CAAC;IAED,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;AACzC,CAAC"}