@luckydraw/cumulus 0.5.1

package/dist/lib/migrate.js

@@ -0,0 +1,196 @@
+/**
+ * Migration utilities for upgrading old threads to support new features.
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { embeddingsAvailable, generateMissingEmbeddings } from './embeddings.js';
+import { HistoryStore } from './history.js';
+import { SessionManager } from './session.js';
+/**
+ * Migrate a thread to support RAG features.
+ * - Generates embeddings for all messages
+ * - Creates a legacy session document from old conversations
+ */
+export async function migrateThread(threadPath, onProgress) {
+    const threadName = path.basename(threadPath, '.jsonl');
+    const sessionsPath = threadPath.replace(/\.jsonl$/, '.sessions');
+    const result = {
+        threadName,
+        messagesTotal: 0,
+        embeddingsGenerated: 0,
+        sessionCreated: false,
+        sessionExchanges: 0,
+        errors: [],
+    };
+    // Phase 1: Load messages
+    onProgress?.({
+        phase: 'loading',
+        current: 0,
+        total: 1,
+        message: 'Loading messages...',
+    });
+    const historyStore = new HistoryStore(threadPath);
+    let messages;
+    try {
+        messages = await historyStore.getAll();
+    }
+    catch (err) {
+        result.errors.push(`Failed to load messages: ${err}`);
+        return result;
+    }
+    // Filter out session documents (we only want user/assistant messages)
+    const conversationMessages = messages.filter(m => m.role !== 'session');
+    result.messagesTotal = conversationMessages.length;
+    if (conversationMessages.length === 0) {
+        onProgress?.({
+            phase: 'done',
+            current: 1,
+            total: 1,
+            message: 'No messages to migrate',
+        });
+        return result;
+    }
+    // Phase 2: Generate embeddings
+    onProgress?.({
+        phase: 'embeddings',
+        current: 0,
+        total: conversationMessages.length,
+        message: 'Checking embedding availability...',
+    });
+    if (await embeddingsAvailable()) {
+        onProgress?.({
+            phase: 'embeddings',
+            current: 0,
+            total: conversationMessages.length,
+            message: 'Generating embeddings (this may take a while)...',
+        });
+        try {
+            // Count existing embeddings before
+            const { loadEmbeddings } = await import('./embeddings.js');
+            const existingBefore = await loadEmbeddings(threadPath);
+            const countBefore = existingBefore.size;
+            // Generate missing embeddings
+            await generateMissingEmbeddings(threadPath, conversationMessages.map(m => ({ id: m.id, content: m.content })));
+            // Count after
+            const existingAfter = await loadEmbeddings(threadPath);
+            result.embeddingsGenerated = existingAfter.size - countBefore;
+            onProgress?.({
+                phase: 'embeddings',
+                current: conversationMessages.length,
+                total: conversationMessages.length,
+                message: `Generated ${result.embeddingsGenerated} embeddings`,
+            });
+        }
+        catch (err) {
+            result.errors.push(`Embedding generation error: ${err}`);
+        }
+    }
+    else {
+        onProgress?.({
+            phase: 'embeddings',
+            current: conversationMessages.length,
+            total: conversationMessages.length,
+            message: 'Embeddings not available (transformers.js not installed)',
+        });
+    }
+    // Phase 3: Create legacy session document
+    onProgress?.({
+        phase: 'sessions',
+        current: 0,
+        total: 1,
+        message: 'Creating legacy session document...',
+    });
+    try {
+        // Check if any session documents already exist
+        let hasExistingSessions = false;
+        try {
+            const sessionFiles = await fs.readdir(sessionsPath);
+            hasExistingSessions = sessionFiles.some(f => f.endsWith('.md'));
+        }
+        catch {
+            // Directory doesn't exist, no existing sessions
+        }
+        if (hasExistingSessions) {
+            onProgress?.({
+                phase: 'sessions',
+                current: 1,
+                total: 1,
+                message: 'Session documents already exist, skipping',
+            });
+        }
+        else {
+            // Create a legacy session with all old exchanges
+            const legacySession = new SessionManager(sessionsPath, threadName, 'ses_legacy');
+            await legacySession.initialize();
+            // Group messages into user/assistant exchanges
+            let exchangeCount = 0;
+            for (let i = 0; i < conversationMessages.length - 1; i++) {
+                const msg = conversationMessages[i];
+                const nextMsg = conversationMessages[i + 1];
+                if (msg.role === 'user' && nextMsg.role === 'assistant') {
+                    await legacySession.appendExchange(msg.content, nextMsg.content);
+                    exchangeCount++;
+                    i++; // Skip the assistant message we just processed
+                }
+            }
+            result.sessionCreated = true;
+            result.sessionExchanges = exchangeCount;
+            // Generate embeddings for the session
+            if (await embeddingsAvailable()) {
+                await legacySession.generateMissingEmbeddings();
+            }
+            onProgress?.({
+                phase: 'sessions',
+                current: 1,
+                total: 1,
+                message: `Created legacy session with ${exchangeCount} exchanges`,
+            });
+        }
+    }
+    catch (err) {
+        result.errors.push(`Session creation error: ${err}`);
+    }
+    onProgress?.({
+        phase: 'done',
+        current: 1,
+        total: 1,
+        message: 'Migration complete',
+    });
+    return result;
+}
+/**
+ * Check if a thread needs migration.
+ * Returns true if embeddings or sessions are missing.
+ */
+export async function needsMigration(threadPath) {
+    const sessionsPath = threadPath.replace(/\.jsonl$/, '.sessions');
+    const historyStore = new HistoryStore(threadPath);
+    const messages = await historyStore.getAll();
+    const conversationMessages = messages.filter(m => m.role !== 'session');
+    // Check embeddings
+    let embeddingCount = 0;
+    try {
+        const { loadEmbeddings } = await import('./embeddings.js');
+        const embeddings = await loadEmbeddings(threadPath);
+        embeddingCount = embeddings.size;
+    }
+    catch {
+        // No embeddings file
+    }
+    // Check sessions
+    let hasSessions = false;
+    try {
+        const sessionFiles = await fs.readdir(sessionsPath);
+        hasSessions = sessionFiles.some(f => f.endsWith('.md'));
+    }
+    catch {
+        // No sessions directory
+    }
+    return {
+        needsEmbeddings: embeddingCount < conversationMessages.length,
+        needsSessions: !hasSessions && conversationMessages.length > 0,
+        messageCount: conversationMessages.length,
+        embeddingCount,
+    };
+}
+//# sourceMappingURL=migrate.js.map

package/dist/lib/migrate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrate.js","sourceRoot":"","sources":["../../src/lib/migrate.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,mBAAmB,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAW,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAoB9C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAkB,EAClB,UAA6B;IAE7B,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACvD,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAEjE,MAAM,MAAM,GAAoB;QAC9B,UAAU;QACV,aAAa,EAAE,CAAC;QAChB,mBAAmB,EAAE,CAAC;QACtB,cAAc,EAAE,KAAK;QACrB,gBAAgB,EAAE,CAAC;QACnB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,yBAAyB;IACzB,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,qBAAqB;KAC/B,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;IAClD,IAAI,QAAmB,CAAC;IAExB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,CAAC;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;QACtD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,sEAAsE;IACtE,MAAM,oBAAoB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IACxE,MAAM,CAAC,aAAa,GAAG,oBAAoB,CAAC,MAAM,CAAC;IAEnD,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,wBAAwB;SAClC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,+BAA+B;IAC/B,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,YAAY;QACnB,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,oBAAoB,CAAC,MAAM;QAClC,OAAO,EAAE,oCAAoC;KAC9C,CAAC,CAAC;IAEH,IAAI,MAAM,mBAAmB,EAAE,EAAE,CAAC;QAChC,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,YAAY;YACnB,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,oBAAoB,CAAC,MAAM;YAClC,OAAO,EAAE,kDAAkD;SAC5D,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,mCAAmC;YACnC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAC3D,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;YACxD,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC;YAExC,8BAA8B;YAC9B,MAAM,yBAAyB,CAC7B,UAAU,EACV,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAClE,CAAC;YAEF,cAAc;YACd,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;YACvD,MAAM,CAAC,mBAAmB,GAAG,aAAa,CAAC,IAAI,GAAG,WAAW,CAAC;YAE9D,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,YAAY;gBACnB,OAAO,EAAE,oBAAoB,CAAC,MAAM;gBACpC,KAAK,EAAE,oBAAoB,CAAC,MAAM;gBAClC,OAAO,EAAE,aAAa,MAAM,CAAC,mBAAmB,aAAa;aAC9D,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,UAAU,EAAE,CAAC;YACX,KAAK,EAAE,YAAY;YACnB,OAAO,EAAE,oBAAoB,CAAC,MAAM;YACpC,KAAK,EAAE,oBAAoB,CAAC,MAAM;YAClC,OAAO,EAAE,0DAA0D;SACpE,CAAC,CAAC;IACL,CAAC;IAED,0CAA0C;IAC1C,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,UAAU;QACjB,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,qCAAqC;KAC/C,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,+CAA+C;QAC/C,IAAI,mBAAmB,GAAG,KAAK,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YACpD,mBAAmB,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;QAED,IAAI,mBAAmB,EAAE,CAAC;YACxB,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,2CAA2C;aACrD,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,iDAAiD;YACjD,MAAM,aAAa,GAAG,IAAI,cAAc,CAAC,YAAY,EAAE,UAAU,EAAE,YAAY,CAAC,CAAC;YACjF,MAAM,aAAa,CAAC,UAAU,EAAE,CAAC;YAEjC,+CAA+C;YAC/C,IAAI,aAAa,GAAG,CAAC,CAAC;YACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzD,MAAM,GAAG,GAAG,oBAAoB,CAAC,CAAC,CAAE,CAAC;gBACrC,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC;gBAE7C,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBACxD,MAAM,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;oBACjE,aAAa,EAAE,CAAC;oBAChB,CAAC,EAAE,CAAC,CAAC,+CAA+C;gBACtD,CAAC;YACH,CAAC;YAED,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;YAC7B,MAAM,CAAC,gBAAgB,GAAG,aAAa,CAAC;YAExC,sCAAsC;YACtC,IAAI,MAAM,mBAAmB,EAAE,EAAE,CAAC;gBAChC,MAAM,aAAa,CAAC,yBAAyB,EAAE,CAAC;YAClD,CAAC;YAED,UAAU,EAAE,CAAC;gBACX,KAAK,EAAE,UAAU;gBACjB,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,+BAA+B,aAAa,YAAY;aAClE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,UAAU,EAAE,CAAC;QACX,KAAK,EAAE,MAAM;QACb,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,oBAAoB;KAC9B,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,UAAkB;IAMrD,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IAEjE,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,CAAC;IAC7C,MAAM,oBAAoB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAExE,mBAAmB;IACnB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,CAAC;QACH,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC3D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAC;QACpD,cAAc,GAAG,UAAU,CAAC,IAAI,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,qBAAqB;IACvB,CAAC;IAED,iBAAiB;IACjB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACpD,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;IAED,OAAO;QACL,eAAe,EAAE,cAAc,GAAG,oBAAoB,CAAC,MAAM;QAC7D,aAAa,EAAE,CAAC,WAAW,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC;QAC9D,YAAY,EAAE,oBAAoB,CAAC,MAAM;QACzC,cAAc;KACf,CAAC;AACJ,CAAC"}

package/dist/lib/retriever.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Automatic RAG retriever for injecting relevant context before Claude invocation.
+ * Runs hybrid search (keyword + semantic) on history and content stores.
+ */
+import { ContentStore } from './content-store.js';
+import { HistoryStore } from './history.js';
+/** Default budget fraction for retrieved context (25% of effective limit) */
+export declare const RETRIEVAL_BUDGET_FRACTION = 0.25;
+export interface RetrievalResult {
+    /** Formatted context string ready for injection into system prompt */
+    context: string;
+    /** Number of history messages included */
+    historyCount: number;
+    /** Number of content items included */
+    contentCount: number;
+    /** Total tokens used by retrieved context */
+    tokensUsed: number;
+    /** Debug info about what was retrieved */
+    debug: {
+        historyResults: Array<{
+            index: number;
+            role: string;
+            score: number;
+            snippet: string;
+        }>;
+        contentResults: Array<{
+            id: string;
+            score: number;
+            snippet: string;
+        }>;
+        sessionResults: Array<{
+            sessionId: string;
+            score: number;
+            snippet: string;
+        }>;
+    };
+}
+export interface RetrieverOptions {
+    /** Token budget for retrieved context */
+    budgetTokens: number;
+    /** Current session ID (to boost current session results) */
+    currentSessionId?: string;
+    /** Sessions path for session document search */
+    sessionsPath?: string;
+    /** Recent conversation messages for query expansion */
+    recentMessages?: Array<{
+        role: string;
+        content: string;
+    }>;
+}
+/**
+ * Retrieve relevant context for a user query.
+ * Runs hybrid search on history, content, and session stores.
+ */
+export declare function retrieve(query: string, historyStore: HistoryStore, contentStore: ContentStore | null, options: RetrieverOptions): Promise<RetrievalResult>;
+//# sourceMappingURL=retriever.d.ts.map

package/dist/lib/retriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retriever.d.ts","sourceRoot":"","sources":["../../src/lib/retriever.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAuB,MAAM,oBAAoB,CAAC;AASvE,OAAO,EAAE,YAAY,EAAW,MAAM,cAAc,CAAC;AAGrD,6EAA6E;AAC7E,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAiB9C,MAAM,WAAW,eAAe;IAC9B,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,YAAY,EAAE,MAAM,CAAC;IACrB,uCAAuC;IACvC,YAAY,EAAE,MAAM,CAAC;IACrB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,KAAK,EAAE;QACL,cAAc,EAAE,KAAK,CAAC;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACvF,cAAc,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;QACtE,cAAc,EAAE,KAAK,CAAC;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAC9E,CAAC;CACH;AAED,MAAM,WAAW,gBAAgB;IAC/B,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,4DAA4D;IAC5D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,cAAc,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC3D;AAqpBD;;;GAGG;AACH,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY,GAAG,IAAI,EACjC,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,eAAe,CAAC,CA+D1B"}