jfl 0.4.4 → 0.5.0

package/dist/lib/text-preprocessing.js

@@ -0,0 +1,261 @@
+/**
+ * Text Preprocessing Module
+ *
+ * Advanced text preprocessing for document indexing and search.
+ * Includes stopword removal, phrase detection, and stemming-light normalization.
+ *
+ * @purpose Improve search quality through better text preprocessing
+ */
+// ============================================================================
+// Stopwords
+// ============================================================================
+/**
+ * English stopwords commonly filtered in IR systems.
+ * Based on NLTK + Lucene stopword lists, tuned for technical/journal content.
+ *
+ * We intentionally KEEP some words that might be stopwords elsewhere but
+ * carry meaning in code/development context (e.g., "not", "new", "same").
+ */
+export const STOPWORDS = new Set([
+    // Articles
+    'a', 'an', 'the',
+    // Pronouns (but keep technical pronouns like "it" which often refers to specific things)
+    'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves',
+    'you', 'your', 'yours', 'yourself', 'yourselves',
+    'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself',
+    'they', 'them', 'their', 'theirs', 'themselves',
+    'what', 'which', 'who', 'whom',
+    // Common verbs (forms that rarely carry search meaning)
+    'is', 'am', 'are', 'was', 'were', 'be', 'been', 'being',
+    'have', 'has', 'had', 'having',
+    'do', 'does', 'did', 'doing',
+    'would', 'should', 'could', 'might', 'must', 'shall', 'will', 'can',
+    // Prepositions
+    'in', 'on', 'at', 'by', 'for', 'with', 'about', 'against',
+    'between', 'into', 'through', 'during', 'before', 'after',
+    'above', 'below', 'to', 'from', 'up', 'down', 'out', 'off',
+    'over', 'under', 'again', 'further', 'then', 'once',
+    // Conjunctions
+    'and', 'but', 'or', 'nor', 'so', 'yet', 'both', 'either', 'neither',
+    'if', 'because', 'as', 'until', 'while', 'of', 'than',
+    // Determiners/quantifiers
+    'this', 'that', 'these', 'those',
+    'each', 'few', 'more', 'most', 'other', 'some', 'such',
+    'any', 'all', 'only', 'own', 'very',
+    // Misc high-frequency low-signal words
+    'here', 'there', 'when', 'where', 'why', 'how',
+    'just', 'also', 'now', 'too', 'very', 'really',
+    // Technical context - keep "not" as it's semantically important
+    // Keep "new", "same", "first", "last" as they carry meaning
+]);
+/**
+ * Domain-specific terms that should NEVER be filtered, even if they look like stopwords.
+ * These are common in development/technical contexts.
+ */
+export const PROTECTED_TERMS = new Set([
+    // Programming concepts that might look like stopwords
+    'null', 'void', 'true', 'false', 'not', 'and', 'or', // logical operators
+    'new', 'class', 'type', 'interface', 'function', 'return',
+    'get', 'set', 'let', 'const', 'var',
+    'api', 'cli', 'sql', 'css', 'html', 'jsx', 'tsx',
+    'use', 'run', 'add', 'fix', 'bug', 'test',
+]);
+// ============================================================================
+// Phrase Detection
+// ============================================================================
+/**
+ * Common multi-word phrases in development/product contexts.
+ * These should be kept together as single tokens for better matching.
+ */
+export const COMMON_PHRASES = [
+    // Technical phrases
+    ['machine', 'learning'],
+    ['deep', 'learning'],
+    ['neural', 'network'],
+    ['natural', 'language'],
+    ['language', 'model'],
+    ['large', 'language'],
+    ['text', 'embedding'],
+    ['semantic', 'search'],
+    ['vector', 'search'],
+    ['document', 'indexing'],
+    ['information', 'retrieval'],
+    // Product/business phrases
+    ['product', 'rank'],
+    ['user', 'experience'],
+    ['user', 'interface'],
+    ['pull', 'request'],
+    ['code', 'review'],
+    ['unit', 'test'],
+    ['integration', 'test'],
+    ['smoke', 'test'],
+    ['regression', 'test'],
+    // Metrics
+    ['precision', 'recall'],
+    ['true', 'positive'],
+    ['false', 'positive'],
+    ['true', 'negative'],
+    ['false', 'negative'],
+    // Common compound concepts
+    ['real', 'time'],
+    ['open', 'source'],
+    ['end', 'point'],
+    ['data', 'base'],
+    ['front', 'end'],
+    ['back', 'end'],
+    ['full', 'stack'],
+];
+/**
+ * Build a phrase lookup map for efficient bigram detection.
+ */
+const PHRASE_MAP = new Map();
+for (const [first, second] of COMMON_PHRASES) {
+    PHRASE_MAP.set(`${first}_${second}`, `${first}_${second}`);
+}
+const DEFAULT_OPTIONS = {
+    removeStopwords: true,
+    detectPhrases: true,
+    minLength: 2,
+    lowercase: true,
+    preserveNumbers: true,
+};
+/**
+ * Basic tokenization: split text into tokens.
+ * Does NOT apply stopword removal or phrase detection.
+ */
+export function basicTokenize(text, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    let normalized = text;
+    if (opts.lowercase) {
+        normalized = normalized.toLowerCase();
+    }
+    // Replace non-alphanumeric with spaces (preserve underscores for snake_case)
+    normalized = normalized.replace(/[^a-z0-9_\s]/gi, ' ');
+    // Split on whitespace
+    const tokens = normalized.split(/\s+/).filter(Boolean);
+    // Filter by minimum length
+    return tokens.filter(token => {
+        // Numbers are allowed if preserveNumbers is true
+        if (opts.preserveNumbers && /^\d+$/.test(token)) {
+            return token.length >= 1;
+        }
+        return token.length >= opts.minLength;
+    });
+}
+/**
+ * Detect and merge bigram phrases in token list.
+ * Converts adjacent tokens that form known phrases into single underscore-joined tokens.
+ */
+export function detectPhrases(tokens) {
+    if (tokens.length < 2)
+        return tokens;
+    const result = [];
+    let i = 0;
+    while (i < tokens.length) {
+        if (i < tokens.length - 1) {
+            const bigram = `${tokens[i]}_${tokens[i + 1]}`;
+            if (PHRASE_MAP.has(bigram)) {
+                result.push(bigram);
+                i += 2;
+                continue;
+            }
+        }
+        result.push(tokens[i]);
+        i++;
+    }
+    return result;
+}
+/**
+ * Remove stopwords from token list, preserving protected terms.
+ */
+export function removeStopwords(tokens) {
+    return tokens.filter(token => {
+        // Never filter protected terms
+        if (PROTECTED_TERMS.has(token))
+            return true;
+        // Filter stopwords
+        return !STOPWORDS.has(token);
+    });
+}
+/**
+ * Advanced tokenization with all preprocessing steps.
+ *
+ * Pipeline:
+ * 1. Basic tokenization (lowercase, split, clean)
+ * 2. Phrase detection (merge known bigrams)
+ * 3. Stopword removal (filter common words)
+ *
+ * @param text - Raw text to tokenize
+ * @param options - Preprocessing options
+ * @returns Array of processed tokens
+ */
+export function tokenize(text, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    // Step 1: Basic tokenization
+    let tokens = basicTokenize(text, opts);
+    // Step 2: Phrase detection (before stopword removal to catch phrases like "pull request")
+    if (opts.detectPhrases) {
+        tokens = detectPhrases(tokens);
+    }
+    // Step 3: Stopword removal
+    if (opts.removeStopwords) {
+        tokens = removeStopwords(tokens);
+    }
+    return tokens;
+}
+/**
+ * Tokenize a query with different defaults than document tokenization.
+ * For queries, we're more conservative about filtering.
+ */
+export function tokenizeQuery(query) {
+    // For queries, use slightly different settings:
+    // - Still detect phrases
+    // - Be more conservative with stopwords (only remove very common ones)
+    // - Keep shorter tokens (might be abbreviations like "UI", "JS")
+    return tokenize(query, {
+        removeStopwords: true,
+        detectPhrases: true,
+        minLength: 2,
+    });
+}
+/**
+ * Tokenize a document for indexing.
+ * More aggressive preprocessing for index efficiency.
+ */
+export function tokenizeDocument(text) {
+    return tokenize(text, {
+        removeStopwords: true,
+        detectPhrases: true,
+        minLength: 2,
+    });
+}
+// ============================================================================
+// Utility Functions
+// ============================================================================
+/**
+ * Extract unique terms from text (deduped).
+ */
+export function extractTerms(text) {
+    const tokens = tokenize(text);
+    return [...new Set(tokens)];
+}
+/**
+ * Count term frequencies in text.
+ */
+export function countTermFrequencies(text) {
+    const tokens = tokenize(text);
+    const counts = new Map();
+    for (const token of tokens) {
+        counts.set(token, (counts.get(token) || 0) + 1);
+    }
+    return counts;
+}
+/**
+ * Check if a token is a stopword (considering protected terms).
+ */
+export function isStopword(token) {
+    if (PROTECTED_TERMS.has(token.toLowerCase()))
+        return false;
+    return STOPWORDS.has(token.toLowerCase());
+}
+//# sourceMappingURL=text-preprocessing.js.map

package/dist/lib/text-preprocessing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-preprocessing.js","sourceRoot":"","sources":["../../src/lib/text-preprocessing.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IAC/B,WAAW;IACX,GAAG,EAAE,IAAI,EAAE,KAAK;IAEhB,yFAAyF;IACzF,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW;IAC3D,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY;IAChD,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS;IAC9D,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY;IAC/C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM;IAE9B,wDAAwD;IACxD,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACvD,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ;IAC9B,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAC5B,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK;IAEnE,eAAe;IACf,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IACzD,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IACzD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IAC1D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM;IAEnD,eAAe;IACf,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS;IACnE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM;IAErD,0BAA0B;IAC1B,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAChC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IACtD,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAEnC,uCAAuC;IACvC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK;IAC9C,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ;IAE9C,gEAAgE;IAChE,4DAA4D;CAC7D,CAAC,CAAA;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IACrC,sDAAsD;IACtD,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,oBAAoB;IACzE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ;IACzD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK;IACnC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK;IAChD,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;CAC1C,CAAC,CAAA;AAEF,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAuB;IAChD,oBAAoB;IACpB,CAAC,SAAS,EAAE,UAAU,CAAC;IACvB,CAAC,MAAM,EAAE,UAAU,CAAC;IACpB,CAAC,QAAQ,EAAE,SAAS,CAAC;IACrB,CAAC,SAAS,EAAE,UAAU,CAAC;IACvB,CAAC,UAAU,EAAE,OAAO,CAAC;IACrB,CAAC,OAAO,EAAE,UAAU,CAAC;IACrB,CAAC,MAAM,EAAE,WAAW,CAAC;IACrB,CAAC,UAAU,EAAE,QAAQ,CAAC;IACtB,CAAC,QAAQ,EAAE,QAAQ,CAAC;IACpB,CAAC,UAAU,EAAE,UAAU,CAAC;IACxB,CAAC,aAAa,EAAE,WAAW,CAAC;IAE5B,2BAA2B;IAC3B,CAAC,SAAS,EAAE,MAAM,CAAC;IACnB,CAAC,MAAM,EAAE,YAAY,CAAC;IACtB,CAAC,MAAM,EAAE,WAAW,CAAC;IACrB,CAAC,MAAM,EAAE,SAAS,CAAC;IACnB,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,aAAa,EAAE,MAAM,CAAC;IACvB,CAAC,OAAO,EAAE,MAAM,CAAC;IACjB,CAAC,YAAY,EAAE,MAAM,CAAC;IAEtB,UAAU;IACV,CAAC,WAAW,EAAE,QAAQ,CAAC;IACvB,CAAC,MAAM,EAAE,UAAU,CAAC;IACpB,CAAC,OAAO,EAAE,UAAU,CAAC;IACrB,CAAC,MAAM,EAAE,UAAU,CAAC;IACpB,CAAC,OAAO,EAAE,UAAU,CAAC;IAErB,2BAA2B;IAC3B,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,MAAM,EAAE,QAAQ,CAAC;IAClB,CAAC,KAAK,EAAE,OAAO,CAAC;IAChB,CAAC,MAAM,EAAE,MAAM,CAAC;IAChB,CAAC,OAAO,EAAE,KAAK,CAAC;IAChB,CAAC,MAAM,EAAE,KAAK,CAAC;IACf,CAAC,MAAM,EAAE,OAAO,CAAC;CAClB,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;AAC5C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,cAAc,EAAE,CAAC;IAC7C,UAAU,CAAC,GAAG,CAAC,GAAG,KAAK,IAAI,MAAM,EAAE,EAAE,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC,CAAA;AAC5D,CAAC;AAcD,MAAM,eAAe,GAA8B;IACjD,eAAe,EAAE,IAAI;IACrB,aAAa,EAAE,IAAI;IACnB,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,IAAI;IACf,eAAe,EAAE,IAAI;CACtB,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,UAA2B,EAAE;IACvE,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAA;IAE/C,IAAI,UAAU,GAAG,IAAI,CAAA;IAErB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnB,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAA;IACvC,CAAC;IAED,6EAA6E;IAC7E,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,CAAA;IAEtD,sBAAsB;IACtB,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAEtD,2BAA2B;IAC3B,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QAC3B,iDAAiD;QACjD,IAAI,IAAI,CAAC,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAChD,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAA;QAC1B,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAA;IACvC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,MAAgB;IAC5C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAA;IAEpC,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAA;YAC9C,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBACnB,CAAC,IAAI,CAAC,CAAA;gBACN,SAAQ;YACV,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QACtB,CAAC,EAAE,CAAA;IACL,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,MAAgB;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;QAC3B,+BAA+B;QAC/B,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAA;QAE3C,mBAAmB;QACnB,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC9B,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,UAA2B,EAAE;IAClE,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAA;IAE/C,6BAA6B;IAC7B,IAAI,MAAM,GAAG,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAEtC,0FAA0F;IAC1F,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAA;IAChC,CAAC;IAED,2BAA2B;IAC3B,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;QACzB,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,gDAAgD;IAChD,yBAAyB;IACzB,uEAAuE;IACvE,iEAAiE;IACjE,OAAO,QAAQ,CAAC,KAAK,EAAE;QACrB,eAAe,EAAE,IAAI;QACrB,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,CAAC;KACb,CAAC,CAAA;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,QAAQ,CAAC,IAAI,EAAE;QACpB,eAAe,EAAE,IAAI;QACrB,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,CAAC;KACb,CAAC,CAAA;AACJ,CAAC;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC7B,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAA;AAC7B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IACjD,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAAE,OAAO,KAAK,CAAA;IAC1D,OAAO,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAA;AAC3C,CAAC"}

package/dist/lib/training-buffer.d.ts

@@ -0,0 +1,86 @@
+/**
+ * @purpose Write (state, action, reward) training tuples for policy head training — generalized for any RL agent
+ */
+export interface RLState {
+    composite_score: number;
+    dimension_scores: Record<string, number>;
+    tests_passing: number;
+    tests_total: number;
+    trajectory_length: number;
+    recent_deltas: number[];
+    agent: string;
+}
+export interface RLAction {
+    type: "fix" | "refactor" | "feature" | "test" | "config" | "experiment";
+    description: string;
+    files_affected: string[];
+    scope: "small" | "medium" | "large";
+    branch: string;
+    code_diff?: string;
+}
+export interface RLReward {
+    composite_delta: number;
+    dimension_deltas: Record<string, number>;
+    tests_added: number;
+    quality_score: number;
+    improved: boolean;
+    prediction_error?: number;
+}
+export interface TrainingBufferEntry {
+    id: string;
+    v: 1;
+    ts: string;
+    agent: string;
+    wave?: string;
+    state: RLState;
+    action: RLAction;
+    reward: RLReward;
+    metadata: {
+        pr_number?: number;
+        branch: string;
+        eval_run_id?: string;
+        prediction_id?: string;
+        autoresearch_round?: number;
+        source: "ci" | "autoresearch" | "experiment" | "manual" | "mined";
+        mine_source?: string;
+        scopes?: string[];
+        changed_files?: string[];
+        session_id?: string;
+        round?: number;
+        hypothesis?: string;
+    };
+}
+export declare function hashEntry(state: RLState, action: RLAction): string;
+export declare class TrainingBuffer {
+    private bufferPath;
+    private projectRoot;
+    constructor(projectRoot?: string);
+    append(entry: Omit<TrainingBufferEntry, "id" | "v" | "ts">): TrainingBufferEntry;
+    read(): TrainingBufferEntry[];
+    /**
+     * Read training buffer entries from all registered services.
+     * Use this from GTM hub to get cross-project training data.
+     */
+    readAll(): TrainingBufferEntry[];
+    private static readFromPath;
+    stats(): {
+        total: number;
+        byAgent: Record<string, number>;
+        bySource: Record<string, number>;
+        avgReward: number;
+        improvedRate: number;
+    };
+    /**
+     * Export in format ready for policy head training:
+     * Each entry becomes a (state_text, action_text, reward) triple
+     * that can be embedded via Stratus for the policy head
+     */
+    exportForTraining(): Array<{
+        state_text: string;
+        action_text: string;
+        reward: number;
+        agent: string;
+        ts: string;
+    }>;
+}
+//# sourceMappingURL=training-buffer.d.ts.map

package/dist/lib/training-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-buffer.d.ts","sourceRoot":"","sources":["../../src/lib/training-buffer.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,MAAM,WAAW,OAAO;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxC,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,iBAAiB,EAAE,MAAM,CAAA;IACzB,aAAa,EAAE,MAAM,EAAE,CAAA;IACvB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,KAAK,GAAG,UAAU,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,YAAY,CAAA;IACvE,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,KAAK,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAA;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxC,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,OAAO,CAAA;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,CAAC,EAAE,CAAC,CAAA;IACJ,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,QAAQ,CAAA;IAChB,MAAM,EAAE,QAAQ,CAAA;IAChB,QAAQ,EAAE;QACR,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,MAAM,EAAE,MAAM,CAAA;QACd,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,aAAa,CAAC,EAAE,MAAM,CAAA;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,MAAM,EAAE,IAAI,GAAG,cAAc,GAAG,YAAY,GAAG,QAAQ,GAAG,OAAO,CAAA;QACjE,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;QACjB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;QACxB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;CACF;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,GAAG,MAAM,CAGlE;AAYD,qBAAa,cAAc;IACzB,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,WAAW,CAAQ;gBAEf,WAAW,CAAC,EAAE,MAAM;IAKhC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,mBAAmB;IAehF,IAAI,IAAI,mBAAmB,EAAE;IAI7B;;;OAGG;IACH,OAAO,IAAI,mBAAmB,EAAE;IA4BhC,OAAO,CAAC,MAAM,CAAC,YAAY;IAa3B,KAAK,IAAI;QACP,KAAK,EAAE,MAAM,CAAA;QACb,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAChC,SAAS,EAAE,MAAM,CAAA;QACjB,YAAY,EAAE,MAAM,CAAA;KACrB;IAuBD;;;;OAIG;IACH,iBAAiB,IAAI,KAAK,CAAC;QACzB,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,EAAE,EAAE,MAAM,CAAA;KACX,CAAC;CAsBH"}

package/dist/lib/training-buffer.js

@@ -0,0 +1,139 @@
+/**
+ * @purpose Write (state, action, reward) training tuples for policy head training — generalized for any RL agent
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "fs";
+import { join, dirname } from "path";
+import { createHash } from "crypto";
+export function hashEntry(state, action) {
+    const content = JSON.stringify({ s: state.composite_score, a: action.description, t: action.type });
+    return createHash("sha256").update(content).digest("hex").slice(0, 12);
+}
+function findProjectRoot() {
+    let dir = process.cwd();
+    while (dir !== dirname(dir)) {
+        if (existsSync(join(dir, ".jfl", "config.json")))
+            return dir;
+        if (existsSync(join(dir, ".jfl")))
+            return dir;
+        dir = dirname(dir);
+    }
+    return process.cwd();
+}
+export class TrainingBuffer {
+    bufferPath;
+    projectRoot;
+    constructor(projectRoot) {
+        this.projectRoot = projectRoot || findProjectRoot();
+        this.bufferPath = join(this.projectRoot, ".jfl", "training-buffer.jsonl");
+    }
+    append(entry) {
+        const full = {
+            id: `tb_${hashEntry(entry.state, entry.action)}`,
+            v: 1,
+            ts: new Date().toISOString(),
+            ...entry,
+        };
+        const dir = dirname(this.bufferPath);
+        if (!existsSync(dir))
+            mkdirSync(dir, { recursive: true });
+        appendFileSync(this.bufferPath, JSON.stringify(full) + "\n");
+        return full;
+    }
+    read() {
+        return TrainingBuffer.readFromPath(this.bufferPath);
+    }
+    /**
+     * Read training buffer entries from all registered services.
+     * Use this from GTM hub to get cross-project training data.
+     */
+    readAll() {
+        const entries = this.read();
+        // Check if this project is a GTM or portfolio — aggregate from services
+        const configPath = join(this.projectRoot, ".jfl", "config.json");
+        if (existsSync(configPath)) {
+            try {
+                const config = JSON.parse(readFileSync(configPath, "utf-8"));
+                if ((config.type === "gtm" || config.type === "portfolio") && config.registered_services) {
+                    const seen = new Set(entries.map(e => e.id));
+                    for (const svc of config.registered_services) {
+                        if (!svc.path)
+                            continue;
+                        const svcBufferPath = join(svc.path, ".jfl", "training-buffer.jsonl");
+                        const svcEntries = TrainingBuffer.readFromPath(svcBufferPath);
+                        for (const entry of svcEntries) {
+                            if (!seen.has(entry.id)) {
+                                entries.push(entry);
+                                seen.add(entry.id);
+                            }
+                        }
+                    }
+                }
+            }
+            catch { }
+        }
+        return entries;
+    }
+    static readFromPath(bufferPath) {
+        if (!existsSync(bufferPath))
+            return [];
+        const entries = [];
+        const lines = readFileSync(bufferPath, "utf-8").split("\n");
+        for (const line of lines) {
+            if (!line.trim())
+                continue;
+            try {
+                entries.push(JSON.parse(line));
+            }
+            catch { }
+        }
+        return entries;
+    }
+    stats() {
+        const entries = this.read();
+        const byAgent = {};
+        const bySource = {};
+        let rewardSum = 0;
+        let improvedCount = 0;
+        for (const e of entries) {
+            byAgent[e.agent] = (byAgent[e.agent] || 0) + 1;
+            bySource[e.metadata.source] = (bySource[e.metadata.source] || 0) + 1;
+            rewardSum += e.reward.composite_delta;
+            if (e.reward.improved)
+                improvedCount++;
+        }
+        return {
+            total: entries.length,
+            byAgent,
+            bySource,
+            avgReward: entries.length > 0 ? rewardSum / entries.length : 0,
+            improvedRate: entries.length > 0 ? improvedCount / entries.length : 0,
+        };
+    }
+    /**
+     * Export in format ready for policy head training:
+     * Each entry becomes a (state_text, action_text, reward) triple
+     * that can be embedded via Stratus for the policy head
+     */
+    exportForTraining() {
+        const entries = this.read();
+        return entries.map(e => ({
+            state_text: [
+                `Composite: ${e.state.composite_score.toFixed(4)}`,
+                `Tests: ${e.state.tests_passing}/${e.state.tests_total}`,
+                `Dimensions: ${Object.entries(e.state.dimension_scores).map(([k, v]) => `${k}=${v.toFixed(4)}`).join(", ")}`,
+                `Trajectory length: ${e.state.trajectory_length}`,
+                `Recent deltas: ${e.state.recent_deltas.map(d => (d >= 0 ? "+" : "") + d.toFixed(4)).join(", ")}`,
+            ].join("\n"),
+            action_text: [
+                `Type: ${e.action.type}`,
+                `Description: ${e.action.description}`,
+                `Scope: ${e.action.scope}`,
+                `Files: ${e.action.files_affected.join(", ")}`,
+            ].join("\n"),
+            reward: e.reward.composite_delta,
+            agent: e.agent,
+            ts: e.ts,
+        }));
+    }
+}
+//# sourceMappingURL=training-buffer.js.map

package/dist/lib/training-buffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-buffer.js","sourceRoot":"","sources":["../../src/lib/training-buffer.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,IAAI,CAAA;AACxE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAuDnC,MAAM,UAAU,SAAS,CAAC,KAAc,EAAE,MAAgB;IACxD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,eAAe,EAAE,CAAC,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAA;IACnG,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AACxE,CAAC;AAED,SAAS,eAAe;IACtB,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;IACvB,OAAO,GAAG,KAAK,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;YAAE,OAAO,GAAG,CAAA;QAC5D,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAAE,OAAO,GAAG,CAAA;QAC7C,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;IACpB,CAAC;IACD,OAAO,OAAO,CAAC,GAAG,EAAE,CAAA;AACtB,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,UAAU,CAAQ;IAClB,WAAW,CAAQ;IAE3B,YAAY,WAAoB;QAC9B,IAAI,CAAC,WAAW,GAAG,WAAW,IAAI,eAAe,EAAE,CAAA;QACnD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAA;IAC3E,CAAC;IAED,MAAM,CAAC,KAAmD;QACxD,MAAM,IAAI,GAAwB;YAChC,EAAE,EAAE,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE;YAChD,CAAC,EAAE,CAAC;YACJ,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,GAAG,KAAK;SACT,CAAA;QAED,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACpC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAEzD,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;QAC5D,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI;QACF,OAAO,cAAc,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACrD,CAAC;IAED;;;OAGG;IACH,OAAO;QACL,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,wEAAwE;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;QAChE,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAA;gBAC5D,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,CAAC,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;oBACzF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;oBAC5C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,mBAAmB,EAAE,CAAC;wBAC7C,IAAI,CAAC,GAAG,CAAC,IAAI;4BAAE,SAAQ;wBACvB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAA;wBACrE,MAAM,UAAU,GAAG,cAAc,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;wBAC7D,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;4BAC/B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAC;gCACxB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gCACnB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;4BACpB,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,UAAkB;QAC5C,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;YAAE,OAAO,EAAE,CAAA;QACtC,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,MAAM,KAAK,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAQ;YAC1B,IAAI,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAwB,CAAC,CAAA;YACvD,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;QACZ,CAAC;QACD,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,KAAK;QAOH,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,MAAM,OAAO,GAA2B,EAAE,CAAA;QAC1C,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAC3C,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,aAAa,GAAG,CAAC,CAAA;QAErB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;YAC9C,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;YACpE,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,eAAe,CAAA;YACrC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ;gBAAE,aAAa,EAAE,CAAA;QACxC,CAAC;QAED,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,OAAO;YACP,QAAQ;YACR,SAAS,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC9D,YAAY,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;SACtE,CAAA;IACH,CAAC;IAED;;;;OAIG;IACH,iBAAiB;QAOf,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACvB,UAAU,EAAE;gBACV,cAAc,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAClD,UAAU,CAAC,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE;gBACxD,eAAe,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBAC5G,sBAAsB,CAAC,CAAC,KAAK,CAAC,iBAAiB,EAAE;gBACjD,kBAAkB,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aAClG,CAAC,IAAI,CAAC,IAAI,CAAC;YACZ,WAAW,EAAE;gBACX,SAAS,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE;gBACxB,gBAAgB,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE;gBACtC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE;gBAC1B,UAAU,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aAC/C,CAAC,IAAI,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,eAAe;YAChC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,EAAE,EAAE,CAAC,CAAC,EAAE;SACT,CAAC,CAAC,CAAA;IACL,CAAC;CACF"}

package/dist/lib/tuple-miner.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @purpose Mine training tuples from existing journals, MAP events, telemetry, and eval history
+ */
+import type { TrainingBufferEntry } from "./training-buffer.js";
+export type MinedTuple = Omit<TrainingBufferEntry, "id" | "v" | "ts">;
+export interface MineStats {
+    journalTuples: number;
+    sessionTuples: number;
+    flowTuples: number;
+    evalTuples: number;
+    telemetryTuples: number;
+    directories: string[];
+    totalMined: number;
+    skippedTypes: number;
+}
+export interface MineOptions {
+    dirs?: string[];
+    all?: boolean;
+    telemetry?: boolean;
+}
+export declare function mineJournalTuples(projectDir: string): MinedTuple[];
+export declare function mineFlowTuples(projectDir: string): MinedTuple[];
+export declare function mineSessionTuples(projectDir: string): MinedTuple[];
+export declare function mineEvalTuples(projectDir: string): MinedTuple[];
+export declare function mineTelemetryTuples(): MinedTuple[];
+export declare function mineAll(options: MineOptions): {
+    tuples: MinedTuple[];
+    stats: MineStats;
+};
+//# sourceMappingURL=tuple-miner.d.ts.map

package/dist/lib/tuple-miner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tuple-miner.d.ts","sourceRoot":"","sources":["../../src/lib/tuple-miner.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAA+B,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AA4B5F,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,mBAAmB,EAAE,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,CAAA;AAErE,MAAM,WAAW,SAAS;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,GAAG,CAAC,EAAE,OAAO,CAAA;IACb,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AA2ED,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,EAAE,CAsDlE;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,EAAE,CAqD/D;AAED,wBAAgB,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,EAAE,CAwElE;AAED,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,UAAU,EAAE,CAgD/D;AAED,wBAAgB,mBAAmB,IAAI,UAAU,EAAE,CA0ElD;AA0BD,wBAAgB,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG;IAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,CAiDxF"}