@eeshans/howiprompt 2.1.0

package/dist/pipeline/config.d.ts

@@ -0,0 +1,21 @@
+export interface BackendToggle {
+    enabled: boolean;
+    exclusions: string[];
+}
+export interface Config {
+    dataDir: string;
+    claudeCodeSource: string;
+    codexHistorySource: string;
+    codexSessionsSource: string;
+    copilotChatSource: string;
+    cursorSource: string;
+    lmStudioSource: string;
+    engagementThreshold: number;
+    politenessThreshold: number;
+    agentCwds: string[];
+    backends: Record<string, BackendToggle>;
+    hasCompletedSetup: boolean;
+}
+export declare function loadConfig(dataDir?: string): Config;
+export declare function saveConfig(dataDir: string, updates: Record<string, any>): void;
+export declare function loadBranding(projectRoot?: string): Record<string, string> | null;

package/dist/pipeline/config.js

@@ -0,0 +1,79 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+function defaultBackends(agentCwds) {
+    return {
+        claude_code: { enabled: true, exclusions: agentCwds },
+        codex: { enabled: true, exclusions: [] },
+        copilot_chat: { enabled: false, exclusions: [] },
+        cursor: { enabled: false, exclusions: [] },
+        lmstudio: { enabled: false, exclusions: [] },
+    };
+}
+function mergeBackends(defaults, configured = {}) {
+    const merged = {};
+    const ids = new Set([...Object.keys(defaults), ...Object.keys(configured)]);
+    for (const id of ids) {
+        merged[id] = {
+            enabled: configured[id]?.enabled ?? defaults[id]?.enabled ?? false,
+            exclusions: Array.isArray(configured[id]?.exclusions)
+                ? configured[id].exclusions
+                : (defaults[id]?.exclusions ?? []),
+        };
+    }
+    return merged;
+}
+export function loadConfig(dataDir) {
+    const dd = dataDir ?? path.join(os.homedir(), ".howiprompt");
+    // Load user config if it exists
+    let userConfig = {};
+    const configPath = path.join(dd, "config.json");
+    try {
+        userConfig = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    }
+    catch {
+        // No config file or invalid — that's fine
+    }
+    const agentCwds = Array.isArray(userConfig.agentCwds) ? userConfig.agentCwds : [];
+    // Migrate legacy agentCwds → backends.claude_code.exclusions
+    const backends = mergeBackends(defaultBackends(agentCwds), userConfig.backends);
+    return {
+        dataDir: dd,
+        claudeCodeSource: path.join(dd, "raw", "claude_code"),
+        codexHistorySource: path.join(dd, "raw", "codex", "history.jsonl"),
+        codexSessionsSource: path.join(os.homedir(), ".codex", "sessions"),
+        copilotChatSource: path.join(dd, "raw", "copilot_chat"),
+        cursorSource: path.join(dd, "raw", "cursor"),
+        lmStudioSource: path.join(dd, "raw", "lmstudio"),
+        engagementThreshold: 12.0,
+        politenessThreshold: 4.5,
+        agentCwds,
+        backends,
+        hasCompletedSetup: userConfig.hasCompletedSetup ?? false,
+    };
+}
+export function saveConfig(dataDir, updates) {
+    const configPath = path.join(dataDir, "config.json");
+    let existing = {};
+    try {
+        existing = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    }
+    catch {
+        // Start fresh
+    }
+    const merged = { ...existing, ...updates };
+    const agentCwds = Array.isArray(merged.agentCwds) ? merged.agentCwds : [];
+    merged.backends = mergeBackends(defaultBackends(agentCwds), merged.backends);
+    fs.mkdirSync(dataDir, { recursive: true });
+    fs.writeFileSync(configPath, JSON.stringify(merged, null, 2));
+}
+export function loadBranding(projectRoot) {
+    const brandingPath = path.join(projectRoot ?? process.cwd(), "branding.json");
+    try {
+        return JSON.parse(fs.readFileSync(brandingPath, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=config.js.map

package/dist/pipeline/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/pipeline/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAsB7B,SAAS,eAAe,CAAC,SAAmB;IAC1C,OAAO;QACL,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE;QACrD,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE;QACxC,YAAY,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE;QAChD,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE;QAC1C,QAAQ,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE;KAC7C,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CACpB,QAAuC,EACvC,aAA4C,EAAE;IAE9C,MAAM,MAAM,GAAkC,EAAE,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAE5E,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,CAAC,EAAE,CAAC,GAAG;YACX,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC,EAAE,OAAO,IAAI,QAAQ,CAAC,EAAE,CAAC,EAAE,OAAO,IAAI,KAAK;YAClE,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC;gBACnD,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,UAAU;gBAC3B,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,UAAU,IAAI,EAAE,CAAC;SACrC,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAgB;IACzC,MAAM,EAAE,GAAG,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;IAE7D,gCAAgC;IAChC,IAAI,UAAU,GAAwB,EAAE,CAAC;IACzC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;IAChD,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,0CAA0C;IAC5C,CAAC;IAED,MAAM,SAAS,GAAa,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IAE5F,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,aAAa,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEhF,OAAO;QACL,OAAO,EAAE,EAAE;QACX,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,aAAa,CAAC;QACrD,kBAAkB,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC;QAClE,mBAAmB,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC;QAClE,iBAAiB,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC;QACvD,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC;QAC5C,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC;QAChD,mBAAmB,EAAE,IAAI;QACzB,mBAAmB,EAAE,GAAG;QACxB,SAAS;QACT,QAAQ;QACR,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,IAAI,KAAK;KACzD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAe,EAAE,OAA4B;IACtE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACrD,IAAI,QAAQ,GAAwB,EAAE,CAAC;IACvC,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;IAChB,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAa,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACpF,MAAM,CAAC,QAAQ,GAAG,aAAa,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC7E,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,WAAoB;IAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC,CAAC;IAC9E,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/pipeline/db.d.ts

@@ -0,0 +1,41 @@
+import { type Client } from "@libsql/client";
+import { Message, Platform, Role } from "./models.js";
+export declare function createDbClient(dbPath: string): Client;
+export declare function messageHash(m: Message): string;
+export declare function insertMessages(client: Client, messages: Message[]): Promise<{
+    inserted: number;
+    skipped: number;
+}>;
+export declare function insertNlpEnrichments(client: Client, enrichments: Array<{
+    messageId: number;
+    intent: string;
+    intentConfidence: number;
+    complexityScore: number;
+    complexityConfidence: number;
+    iterationScore: number;
+    iterationConfidence: number;
+}>): Promise<void>;
+export interface MessageRow {
+    id: number;
+    timestamp: string;
+    platform: string;
+    role: string;
+    content: string;
+    conversationId: string;
+    wordCount: number;
+    modelId: string | null;
+    modelProvider: string | null;
+}
+export declare function queryMessages(client: Client, opts?: {
+    role?: Role;
+    platform?: Platform;
+}): Promise<MessageRow[]>;
+export declare function platformFilter(platform?: Platform): {
+    clause: string;
+    args: any[];
+};
+export declare function getLastSync(client: Client, source: string): Promise<{
+    lastTimestamp: string | null;
+    lastFile: string | null;
+}>;
+export declare function logSync(client: Client, source: string, lastFile: string | null, lastTimestamp: string | null, messageCount: number): Promise<void>;

package/dist/pipeline/db.js

@@ -0,0 +1,130 @@
+import { createHash } from "node:crypto";
+import { createClient } from "@libsql/client";
+export function createDbClient(dbPath) {
+    return createClient({ url: `file:${dbPath}` });
+}
+export function messageHash(m) {
+    const data = `${m.platform}|${m.conversationId}|${m.timestamp.toISOString()}|${m.content.slice(0, 200)}`;
+    return createHash("sha256").update(data).digest("hex");
+}
+export async function insertMessages(client, messages) {
+    let inserted = 0;
+    let skipped = 0;
+    const batchSize = 500;
+    for (let i = 0; i < messages.length; i += batchSize) {
+        const batch = messages.slice(i, i + batchSize);
+        const stmts = batch.map((m) => {
+            const local = localTime(m.timestamp);
+            return {
+                sql: `INSERT OR IGNORE INTO messages
+              (hash, timestamp, platform, role, content, conversation_id,
+               word_count, model_id, model_provider,
+               local_hour, local_weekday, local_date, source_file, synced_at)
+              VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
+                args: [
+                    messageHash(m),
+                    m.timestamp.toISOString(),
+                    m.platform,
+                    m.role,
+                    m.content,
+                    m.conversationId,
+                    m.wordCount,
+                    m.modelId ?? null,
+                    m.modelProvider ?? null,
+                    local.hour,
+                    local.weekday,
+                    local.dateStr,
+                    m.sourceFile ?? null,
+                ],
+            };
+        });
+        const results = await client.batch(stmts, "write");
+        for (const r of results) {
+            if (r.rowsAffected > 0)
+                inserted++;
+            else
+                skipped++;
+        }
+    }
+    return { inserted, skipped };
+}
+function localTime(d) {
+    return {
+        hour: d.getHours(),
+        weekday: ((d.getDay() + 6) % 7), // JS: 0=Sun → Python-like: 0=Mon
+        dateStr: `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`,
+    };
+}
+export async function insertNlpEnrichments(client, enrichments) {
+    const batchSize = 500;
+    for (let i = 0; i < enrichments.length; i += batchSize) {
+        const batch = enrichments.slice(i, i + batchSize);
+        await client.batch(batch.map((e) => ({
+            sql: `INSERT OR IGNORE INTO nlp_enrichments
+              (message_id, intent, intent_confidence, complexity_score,
+               complexity_confidence, iteration_score, iteration_confidence)
+              VALUES (?, ?, ?, ?, ?, ?, ?)`,
+            args: [
+                e.messageId,
+                e.intent,
+                e.intentConfidence,
+                e.complexityScore,
+                e.complexityConfidence,
+                e.iterationScore,
+                e.iterationConfidence,
+            ],
+        })), "write");
+    }
+}
+export async function queryMessages(client, opts) {
+    let sql = "SELECT id, timestamp, platform, role, content, conversation_id, word_count, model_id, model_provider FROM messages WHERE 1=1";
+    const args = [];
+    if (opts?.role) {
+        sql += " AND role = ?";
+        args.push(opts.role);
+        if (opts.role === "human")
+            sql += " AND is_excluded = 0";
+    }
+    if (opts?.platform) {
+        sql += " AND platform = ?";
+        args.push(opts.platform);
+    }
+    sql += " ORDER BY timestamp";
+    const result = await client.execute({ sql, args });
+    return result.rows.map((r) => ({
+        id: Number(r.id),
+        timestamp: String(r.timestamp),
+        platform: String(r.platform),
+        role: String(r.role),
+        content: String(r.content),
+        conversationId: String(r.conversation_id),
+        wordCount: Number(r.word_count),
+        modelId: r.model_id ? String(r.model_id) : null,
+        modelProvider: r.model_provider ? String(r.model_provider) : null,
+    }));
+}
+export function platformFilter(platform) {
+    if (platform)
+        return { clause: " AND platform = ?", args: [platform] };
+    return { clause: "", args: [] };
+}
+export async function getLastSync(client, source) {
+    const result = await client.execute({
+        sql: "SELECT last_timestamp, last_file FROM sync_log WHERE source = ? ORDER BY synced_at DESC LIMIT 1",
+        args: [source],
+    });
+    if (result.rows.length === 0)
+        return { lastTimestamp: null, lastFile: null };
+    const row = result.rows[0];
+    return {
+        lastTimestamp: row.last_timestamp ? String(row.last_timestamp) : null,
+        lastFile: row.last_file ? String(row.last_file) : null,
+    };
+}
+export async function logSync(client, source, lastFile, lastTimestamp, messageCount) {
+    await client.execute({
+        sql: "INSERT INTO sync_log (source, last_file, last_timestamp, message_count, synced_at) VALUES (?, ?, ?, ?, datetime('now'))",
+        args: [source, lastFile, lastTimestamp, messageCount],
+    });
+}
+//# sourceMappingURL=db.js.map

package/dist/pipeline/db.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/pipeline/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAiC,MAAM,gBAAgB,CAAC;AAG7E,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,OAAO,YAAY,CAAC,EAAE,GAAG,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,CAAU;IACpC,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IACzG,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAc,EACd,QAAmB;IAEnB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,SAAS,GAAG,GAAG,CAAC;IAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAkB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACrC,OAAO;gBACL,GAAG,EAAE;;;;8EAIiE;gBACtE,IAAI,EAAE;oBACJ,WAAW,CAAC,CAAC,CAAC;oBACd,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE;oBACzB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,OAAO;oBACT,CAAC,CAAC,cAAc;oBAChB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,OAAO,IAAI,IAAI;oBACjB,CAAC,CAAC,aAAa,IAAI,IAAI;oBACvB,KAAK,CAAC,IAAI;oBACV,KAAK,CAAC,OAAO;oBACb,KAAK,CAAC,OAAO;oBACb,CAAC,CAAC,UAAU,IAAI,IAAI;iBACrB;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC;gBAAE,QAAQ,EAAE,CAAC;;gBAC9B,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,SAAS,CAAC,CAAO;IACxB,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE;QAClB,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,iCAAiC;QAClE,OAAO,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KACnH,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAc,EACd,WAQE;IAEF,MAAM,SAAS,GAAG,GAAG,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAClD,MAAM,MAAM,CAAC,KAAK,CAChB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChB,GAAG,EAAE;;;2CAG8B;YACnC,IAAI,EAAE;gBACJ,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,gBAAgB;gBAClB,CAAC,CAAC,eAAe;gBACjB,CAAC,CAAC,oBAAoB;gBACtB,CAAC,CAAC,cAAc;gBAChB,CAAC,CAAC,mBAAmB;aACtB;SACF,CAAC,CAAC,EACH,OAAO,CACR,CAAC;IACJ,CAAC;AACH,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,IAA2C;IAE3C,IAAI,GAAG,GACL,8HAA8H,CAAC;IACjI,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,IAAI,IAAI,EAAE,IAAI,EAAE,CAAC;QACf,GAAG,IAAI,eAAe,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO;YAAE,GAAG,IAAI,sBAAsB,CAAC;IAC3D,CAAC;IACD,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;QACnB,GAAG,IAAI,mBAAmB,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IACD,GAAG,IAAI,qBAAqB,CAAC;IAE7B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC9B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC5B,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QACpB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAC1B,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC;QACzC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/B,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;QAC/C,aAAa,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI;KAClE,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAAmB;IAChD,IAAI,QAAQ;QAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;IACvE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAc,EACd,MAAc;IAEd,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE,iGAAiG;QACtG,IAAI,EAAE,CAAC,MAAM,CAAC;KACf,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC7E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,OAAO;QACL,aAAa,EAAE,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI;QACrE,QAAQ,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI;KACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAAc,EACd,QAAuB,EACvB,aAA4B,EAC5B,YAAoB;IAEpB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE,yHAAyH;QAC9H,IAAI,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,CAAC;KACtD,CAAC,CAAC;AACL,CAAC"}

package/dist/pipeline/embeddings.d.ts

@@ -0,0 +1,15 @@
+import type { Client } from "@libsql/client";
+import type { MlConfig } from "./ml-config.js";
+export declare function resetEmbedderForTests(): void;
+export declare function initEmbedder(mlConfig: MlConfig, cacheDir: string, onProgress?: (progress: {
+    status: string;
+    file?: string;
+    progress?: number;
+}) => void): Promise<void>;
+export declare function embedTexts(texts: string[], mlConfig: MlConfig): Promise<Float32Array[]>;
+export declare function embedSingle(text: string, mlConfig: MlConfig): Promise<Float32Array>;
+export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
+export declare function enrichEmbeddings(client: Client, mlConfig: MlConfig, dataDir: string, onProgress?: (progress: {
+    status: string;
+    progress?: number;
+}) => void, onBatchProgress?: (embedded: number, total: number) => void): Promise<number>;

package/dist/pipeline/embeddings.js

@@ -0,0 +1,82 @@
+import path from "node:path";
+let extractor = null;
+export function resetEmbedderForTests() {
+    extractor = null;
+}
+export async function initEmbedder(mlConfig, cacheDir, onProgress) {
+    if (extractor)
+        return;
+    const { env, pipeline: createPipeline } = await import("@huggingface/transformers");
+    env.cacheDir = cacheDir;
+    // Silence unnecessary logs
+    env.allowRemoteModels = true;
+    extractor = await createPipeline("feature-extraction", mlConfig.embedding.model, {
+        dtype: mlConfig.embedding.dtype,
+        revision: "main",
+        progress_callback: onProgress,
+    });
+}
+export async function embedTexts(texts, mlConfig) {
+    if (!extractor)
+        throw new Error("Embedder not initialized — call initEmbedder() first");
+    const results = [];
+    const batchSize = mlConfig.embedding.batchSize;
+    for (let i = 0; i < texts.length; i += batchSize) {
+        const batch = texts.slice(i, i + batchSize);
+        const output = await extractor(batch, { pooling: "cls", normalize: true });
+        for (let j = 0; j < batch.length; j++) {
+            const vec = output[j].data;
+            results.push(new Float32Array(vec));
+        }
+    }
+    return results;
+}
+export async function embedSingle(text, mlConfig) {
+    const results = await embedTexts([text], mlConfig);
+    return results[0];
+}
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+export async function enrichEmbeddings(client, mlConfig, dataDir, onProgress, onBatchProgress) {
+    // Find human messages without embeddings
+    const result = await client.execute("SELECT id, content FROM messages WHERE role = 'human' AND is_excluded = 0 AND embedding IS NULL");
+    if (result.rows.length === 0)
+        return 0;
+    // Init model (downloads on first run)
+    const modelCacheDir = path.join(dataDir, "models");
+    await initEmbedder(mlConfig, modelCacheDir, onProgress);
+    const ids = result.rows.map((r) => Number(r.id));
+    const texts = result.rows.map((r) => String(r.content));
+    // Embed in small batches with brief pauses to avoid sustained CPU spike
+    const batchSize = Math.min(mlConfig.embedding.batchSize, 32);
+    let embedded = 0;
+    const total = texts.length;
+    for (let i = 0; i < total; i += batchSize) {
+        const batchTexts = texts.slice(i, i + batchSize);
+        const batchIds = ids.slice(i, i + batchSize);
+        const vectors = await embedTexts(batchTexts, mlConfig);
+        // Store vectors in DB
+        const stmts = batchIds.map((id, j) => ({
+            sql: "UPDATE messages SET embedding = vector32(?) WHERE id = ?",
+            args: [JSON.stringify(Array.from(vectors[j])), id],
+        }));
+        await client.batch(stmts, "write");
+        embedded += batchTexts.length;
+        onBatchProgress?.(embedded, total);
+        // Brief yield every batch so the event loop breathes
+        if (i + batchSize < total) {
+            await new Promise((r) => setTimeout(r, 10));
+        }
+    }
+    return embedded;
+}
+//# sourceMappingURL=embeddings.js.map

package/dist/pipeline/embeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../src/pipeline/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAI7B,IAAI,SAAS,GAAQ,IAAI,CAAC;AAE1B,MAAM,UAAU,qBAAqB;IACnC,SAAS,GAAG,IAAI,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,QAAkB,EAClB,QAAgB,EAChB,UAAqF;IAErF,IAAI,SAAS;QAAE,OAAO;IAEtB,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IAEpF,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACxB,2BAA2B;IAC3B,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAE7B,SAAS,GAAG,MAAM,cAAc,CAAC,oBAAoB,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE;QAC/E,KAAK,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAY;QACtC,QAAQ,EAAE,MAAM;QAChB,iBAAiB,EAAE,UAAU;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,QAAkB;IAElB,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAExF,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,QAAkB;IAElB,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;IACnD,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,CAAe,EAAE,CAAe;IAC/D,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,QAAkB,EAClB,OAAe,EACf,UAAsE,EACtE,eAA2D;IAE3D,yCAAyC;IACzC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CACjC,iGAAiG,CAClG,CAAC;IAEF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEvC,sCAAsC;IACtC,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC;IAExD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAExD,wEAAwE;IACxE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7D,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAEvD,sBAAsB;QACtB,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,GAAG,EAAE,0DAA0D;YAC/D,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;SACnD,CAAC,CAAC,CAAC;QAEJ,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACnC,QAAQ,IAAI,UAAU,CAAC,MAAM,CAAC;QAC9B,eAAe,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEnC,qDAAqD;QACrD,IAAI,CAAC,GAAG,SAAS,GAAG,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/pipeline/exclusions.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Unified exclusion rules engine.
+ *
+ * All message filtering rules live in the `exclusion_rules` DB table.
+ * The parser and flagging system read from this table — no hardcoded filters.
+ *
+ * Rule types:
+ *   - content_prefix:    message content starts with pattern
+ *   - cwd_pattern:       JSONL entry cwd field contains pattern
+ *   - dir_name:          source file directory name matches pattern
+ *   - skill_invocation:  message matches a skill invocation syntax (regex)
+ *
+ * Match modes:
+ *   - starts_with:  content.startsWith(pattern)
+ *   - contains:     content.includes(pattern)
+ *   - exact:        content === pattern
+ *   - regex:        new RegExp(pattern).test(content)
+ */
+import type { Client } from "@libsql/client";
+import type { Backend } from "./backends.js";
+export interface ExclusionRule {
+    id: number;
+    platform: string;
+    ruleType: string;
+    pattern: string;
+    matchMode: string;
+    description: string;
+    source: string;
+    templateContent: string;
+    isActive: boolean;
+}
+export interface CompiledRules {
+    contentPrefix: Array<{
+        rule: ExclusionRule;
+        test: (content: string) => boolean;
+    }>;
+    cwdPattern: Array<{
+        rule: ExclusionRule;
+        test: (cwd: string) => boolean;
+    }>;
+    dirName: Array<{
+        rule: ExclusionRule;
+        test: (dirName: string) => boolean;
+    }>;
+    skillInvocation: Array<{
+        rule: ExclusionRule;
+        test: (content: string, platform: string) => boolean;
+    }>;
+}
+export declare function loadExclusionRules(client: Client): Promise<ExclusionRule[]>;
+export declare function compileRules(rules: ExclusionRule[]): CompiledRules;
+/** Check if a message content should be excluded (content_prefix rules). */
+export declare function shouldExcludeContent(compiled: CompiledRules, content: string, platform: string): {
+    excluded: boolean;
+    ruleId: number | null;
+};
+/** Check if a CWD indicates a programmatic (non-human) session. */
+export declare function shouldExcludeCwd(compiled: CompiledRules, cwd: string): {
+    excluded: boolean;
+    ruleId: number | null;
+};
+/** Check if a source directory name should be excluded. */
+export declare function shouldExcludeDir(compiled: CompiledRules, dirName: string): {
+    excluded: boolean;
+    ruleId: number | null;
+};
+/**
+ * Flag existing messages that match exclusion rules.
+ * Checks skill_invocation and content_prefix rules against unflagged messages.
+ * Also checks template_content via hash comparison.
+ */
+export declare function flagExcludedMessages(client: Client): Promise<number>;
+/**
+ * Discover skills from all backends and upsert as exclusion rules.
+ * Also upserts into skills table for template content storage.
+ */
+export declare function discoverAndSyncRules(client: Client, backends: Backend[]): Promise<{
+    skillsFound: number;
+    rulesUpserted: number;
+}>;
+/**
+ * Seed system-level exclusion rules.
+ * These are platform-inherent rules that don't come from skill discovery.
+ * Uses INSERT OR IGNORE so they're only added once.
+ */
+export declare function seedSystemRules(client: Client): Promise<number>;