distill-mcp 0.6.0-beta

package/dist/utils/segment-scorer.js

@@ -0,0 +1,148 @@
+/**
+ * Segment Scorer
+ *
+ * Combines multiple scoring signals to determine segment importance:
+ * - TF-IDF: Unique/rare terms are more important
+ * - Position: Beginning and end of content are more important
+ * - Keywords: Errors, instructions, code blocks get priority
+ */
+import { countTokens } from "./token-counter.js";
+/**
+ * Default scoring weights (must sum to 1.0)
+ */
+export const DEFAULT_WEIGHTS = {
+    tfidf: 0.4, // Unique content
+    position: 0.3, // Location in document
+    keyword: 0.3, // Important terms
+};
+/**
+ * Calculate position weight using a U-shaped curve
+ * Beginning (0-10%) and end (90-100%) get highest weight
+ * Middle content gets lower weight
+ *
+ * @param position - Normalized position (0-1)
+ * @returns Position weight (0.6-1.0)
+ */
+export function calculatePositionWeight(position) {
+    // U-shaped curve: high at edges, low in middle
+    if (position <= 0.1 || position >= 0.9) {
+        return 1.0; // First/last 10% are most important
+    }
+    if (position <= 0.2 || position >= 0.8) {
+        return 0.85; // Next 10% still important
+    }
+    if (position <= 0.3 || position >= 0.7) {
+        return 0.7; // Transitional zones
+    }
+    return 0.6; // Middle content baseline
+}
+/**
+ * Keyword patterns for importance detection
+ */
+const KEYWORD_PATTERNS = {
+    // Errors and failures (highest priority)
+    errors: /\b(error|Error|ERROR|fail|Fail|FAIL|failed|exception|Exception|EXCEPTION|panic|crash|fatal|critical)\b/,
+    // Instructions and requirements
+    instructions: /\b(must|MUST|should|SHOULD|required|Required|REQUIRED|important|Important|IMPORTANT|note|Note|NOTE|warning|Warning|WARNING|todo|TODO|fixme|FIXME)\b/,
+    // Code blocks (markdown)
+    codeBlocks: /```[\s\S]*?```|`[^`]+`/,
+    // Technical terms (programming)
+    technical: /\b(function|class|interface|type|const|let|var|async|await|return|import|export|def|fn|struct|impl|pub|private|public|protected)\b/,
+    // Structural markers (headers, lists)
+    structural: /^(#{1,6}\s|[-*+]\s|\d+\.\s|>\s)/m,
+    // Questions (often need answers preserved)
+    questions: /\?[\s]*$/m,
+    // URLs and references
+    references: /https?:\/\/[^\s]+|@\w+|#\w+/,
+};
+/**
+ * Calculate keyword boost based on content patterns
+ * Multiple matches compound additively up to 1.0
+ *
+ * @param text - Segment text to analyze
+ * @returns Keyword boost (0-1)
+ */
+export function calculateKeywordBoost(text) {
+    let boost = 0;
+    if (KEYWORD_PATTERNS.errors.test(text)) {
+        boost += 0.4; // Errors are critical
+    }
+    if (KEYWORD_PATTERNS.instructions.test(text)) {
+        boost += 0.3; // Instructions are important
+    }
+    if (KEYWORD_PATTERNS.codeBlocks.test(text)) {
+        boost += 0.2; // Code should be preserved
+    }
+    if (KEYWORD_PATTERNS.structural.test(text)) {
+        boost += 0.15; // Structure helps comprehension
+    }
+    if (KEYWORD_PATTERNS.technical.test(text)) {
+        boost += 0.1; // Technical content is usually relevant
+    }
+    if (KEYWORD_PATTERNS.questions.test(text)) {
+        boost += 0.15; // Questions need context
+    }
+    if (KEYWORD_PATTERNS.references.test(text)) {
+        boost += 0.1; // References are informational
+    }
+    return Math.min(boost, 1.0); // Cap at 1.0
+}
+/**
+ * Create a segment from text
+ *
+ * @param text - Segment text
+ * @param startLine - Starting line number
+ * @param endLine - Ending line number
+ * @param type - Segment type
+ * @param totalLines - Total lines in document (for position calculation)
+ * @returns Segment object
+ */
+export function createSegment(text, startLine, endLine, type, totalLines) {
+    return {
+        text,
+        startLine,
+        endLine,
+        type,
+        position: totalLines > 0 ? startLine / totalLines : 0,
+        tokens: countTokens(text),
+        isPreserved: false,
+    };
+}
+/**
+ * Score a segment combining all signals
+ *
+ * @param segment - Segment to score
+ * @param tfidfScore - Pre-calculated TF-IDF score (0-1)
+ * @param weights - Scoring weights
+ * @returns Scored segment with importance
+ */
+export function scoreSegment(segment, tfidfScore, weights = DEFAULT_WEIGHTS) {
+    const positionScore = calculatePositionWeight(segment.position);
+    const keywordScore = calculateKeywordBoost(segment.text);
+    // Weighted combination
+    const combined = weights.tfidf * tfidfScore +
+        weights.position * positionScore +
+        weights.keyword * keywordScore;
+    return {
+        ...segment,
+        importance: combined,
+        scores: {
+            tfidf: tfidfScore,
+            position: positionScore,
+            keyword: keywordScore,
+            combined,
+        },
+    };
+}
+/**
+ * Check if text contains error indicators
+ */
+export function hasErrorIndicators(text) {
+    return KEYWORD_PATTERNS.errors.test(text);
+}
+/**
+ * Check if text contains instruction indicators
+ */
+export function hasInstructionIndicators(text) {
+    return KEYWORD_PATTERNS.instructions.test(text);
+}

package/dist/utils/signature-grouper.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Signature Grouper
+ *
+ * Groups error lines by their normalized signature for deduplication.
+ */
+export interface GroupOptions {
+    /** Minimum occurrences to consider as duplicate (default: 2) */
+    threshold: number;
+    /** Number of first occurrences to keep in full (default: 1) */
+    keepFirst: number;
+    /** Custom regex pattern to identify errors */
+    customPattern?: RegExp;
+    /** Maximum samples to keep per group */
+    maxSamples?: number;
+}
+export interface DeduplicatedErrorGroup {
+    /** Normalized signature for this error type */
+    signature: string;
+    /** Number of occurrences */
+    count: number;
+    /** First full occurrence (raw line) */
+    firstOccurrence: string;
+    /** Locations where this error occurred */
+    locations: string[];
+    /** Sample raw lines (up to maxSamples) */
+    samples: string[];
+    /** Error code if available */
+    code?: string;
+    /** Cleaned message without location info */
+    message: string;
+}
+export interface GroupingResult {
+    /** Grouped errors by signature */
+    groups: Map<string, DeduplicatedErrorGroup>;
+    /** Lines that don't match error patterns */
+    nonErrorLines: string[];
+    /** Total error lines processed */
+    totalErrorLines: number;
+}
+/**
+ * Group error lines by their normalized signature
+ */
+export declare function groupBySignature(lines: string[], options?: Partial<GroupOptions>): GroupingResult;
+/**
+ * Format grouped errors as a readable string
+ */
+export declare function formatGroups(result: GroupingResult, format?: "plain" | "markdown", options?: Partial<GroupOptions>): string;
+/**
+ * Calculate deduplication statistics
+ */
+export declare function calculateStats(result: GroupingResult): {
+    originalLines: number;
+    deduplicatedLines: number;
+    uniqueErrors: number;
+    totalDuplicates: number;
+    reductionPercent: number;
+};
+//# sourceMappingURL=signature-grouper.d.ts.map

package/dist/utils/signature-grouper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"signature-grouper.d.ts","sourceRoot":"","sources":["../../src/utils/signature-grouper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH,MAAM,WAAW,YAAY;IAC3B,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,8CAA8C;IAC9C,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,sBAAsB;IACrC,+CAA+C;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,0CAA0C;IAC1C,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,kCAAkC;IAClC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;IAC5C,4CAA4C;IAC5C,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,kCAAkC;IAClC,eAAe,EAAE,MAAM,CAAC;CACzB;AAQD;;GAEG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAClC,cAAc,CAsChB;AAgED;;GAEG;AACH,wBAAgB,YAAY,CAC1B,MAAM,EAAE,cAAc,EACtB,MAAM,GAAE,OAAO,GAAG,UAAoB,EACtC,OAAO,GAAE,OAAO,CAAC,YAAY,CAAM,GAClC,MAAM,CA2DR;AAUD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,cAAc,GAAG;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CAC1B,CAeA"}

package/dist/utils/signature-grouper.js

@@ -0,0 +1,185 @@
+/**
+ * Signature Grouper
+ *
+ * Groups error lines by their normalized signature for deduplication.
+ */
+import { normalizeErrorLine, extractErrorParts, createSignature, formatLocation, isLikelyError, } from "./error-normalizer.js";
+const DEFAULT_OPTIONS = {
+    threshold: 2,
+    keepFirst: 1,
+    maxSamples: 3,
+};
+/**
+ * Group error lines by their normalized signature
+ */
+export function groupBySignature(lines, options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    const groups = new Map();
+    const nonErrorLines = [];
+    let totalErrorLines = 0;
+    for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        // Check custom pattern first
+        if (opts.customPattern) {
+            if (opts.customPattern.test(trimmed)) {
+                processErrorLine(trimmed, groups, opts);
+                totalErrorLines++;
+                continue;
+            }
+        }
+        // Try to extract error parts
+        const parts = extractErrorParts(trimmed);
+        if (parts) {
+            processErrorLineWithParts(trimmed, parts, groups, opts);
+            totalErrorLines++;
+        }
+        else if (isLikelyError(trimmed)) {
+            // Line looks like an error but doesn't match patterns
+            processErrorLine(trimmed, groups, opts);
+            totalErrorLines++;
+        }
+        else {
+            nonErrorLines.push(trimmed);
+        }
+    }
+    return {
+        groups,
+        nonErrorLines,
+        totalErrorLines,
+    };
+}
+/**
+ * Process an error line with extracted parts
+ */
+function processErrorLineWithParts(rawLine, parts, groups, options) {
+    const signature = createSignature(parts);
+    const location = formatLocation(parts);
+    if (groups.has(signature)) {
+        const group = groups.get(signature);
+        group.count++;
+        if (location && !group.locations.includes(location)) {
+            group.locations.push(location);
+        }
+        if (group.samples.length < (options.maxSamples ?? 3)) {
+            group.samples.push(rawLine);
+        }
+    }
+    else {
+        groups.set(signature, {
+            signature,
+            count: 1,
+            firstOccurrence: rawLine,
+            locations: location ? [location] : [],
+            samples: [rawLine],
+            code: parts.code,
+            message: parts.message,
+        });
+    }
+}
+/**
+ * Process an error line without structured parts
+ */
+function processErrorLine(rawLine, groups, options) {
+    const signature = normalizeErrorLine(rawLine);
+    if (groups.has(signature)) {
+        const group = groups.get(signature);
+        group.count++;
+        if (group.samples.length < (options.maxSamples ?? 3)) {
+            group.samples.push(rawLine);
+        }
+    }
+    else {
+        groups.set(signature, {
+            signature,
+            count: 1,
+            firstOccurrence: rawLine,
+            locations: [],
+            samples: [rawLine],
+            message: rawLine,
+        });
+    }
+}
+/**
+ * Format grouped errors as a readable string
+ */
+export function formatGroups(result, format = "plain", options = {}) {
+    const opts = { ...DEFAULT_OPTIONS, ...options };
+    const parts = [];
+    const md = format === "markdown";
+    // Sort groups by count (most frequent first)
+    const sortedGroups = Array.from(result.groups.values()).sort((a, b) => b.count - a.count);
+    // Separate duplicates from unique errors
+    const duplicates = sortedGroups.filter((g) => g.count >= opts.threshold);
+    const unique = sortedGroups.filter((g) => g.count < opts.threshold);
+    // Format duplicated errors
+    if (duplicates.length > 0) {
+        parts.push(md ? "## Deduplicated Errors\n" : "DEDUPLICATED ERRORS:");
+        for (const [i, group] of duplicates.entries()) {
+            const code = group.code ? `${group.code}: ` : "";
+            const msg = truncateMessage(group.message, 80);
+            if (md) {
+                parts.push(`### ${i + 1}. ${code}${msg}`);
+                parts.push(`**Occurrences:** ${group.count}`);
+                parts.push(`**First:** \`${truncateMessage(group.firstOccurrence, 100)}\``);
+            }
+            else {
+                parts.push(`${i + 1}. ${code}${msg}`);
+                parts.push(`   Occurrences: ${group.count}`);
+                parts.push(`   First: ${truncateMessage(group.firstOccurrence, 100)}`);
+            }
+            // Show locations if available
+            if (group.locations.length > 1) {
+                const otherLocations = group.locations.slice(1, 6);
+                const remaining = group.locations.length - 6;
+                const suffix = remaining > 0 ? ` (+${remaining} more)` : "";
+                if (md) {
+                    parts.push(`**Also in:** ${otherLocations.join(", ")}${suffix}`);
+                }
+                else {
+                    parts.push(`   Also in: ${otherLocations.join(", ")}${suffix}`);
+                }
+            }
+            parts.push("");
+        }
+    }
+    // Show unique errors if any (below threshold)
+    if (unique.length > 0 && opts.keepFirst > 0) {
+        parts.push(md ? "## Unique Errors\n" : "UNIQUE ERRORS:");
+        for (const group of unique.slice(0, opts.keepFirst * 5)) {
+            parts.push(md ? `- ${group.firstOccurrence}` : `  ${group.firstOccurrence}`);
+        }
+        if (unique.length > opts.keepFirst * 5) {
+            const more = unique.length - opts.keepFirst * 5;
+            parts.push(md ? `\n*...and ${more} more unique errors*` : `  ...and ${more} more unique errors`);
+        }
+        parts.push("");
+    }
+    return parts.join("\n");
+}
+/**
+ * Truncate a message to a maximum length
+ */
+function truncateMessage(message, maxLength) {
+    if (message.length <= maxLength)
+        return message;
+    return message.slice(0, maxLength - 3) + "...";
+}
+/**
+ * Calculate deduplication statistics
+ */
+export function calculateStats(result) {
+    const originalLines = result.totalErrorLines + result.nonErrorLines.length;
+    const uniqueErrors = result.groups.size;
+    const totalDuplicates = result.totalErrorLines - uniqueErrors;
+    const deduplicatedLines = uniqueErrors + result.nonErrorLines.length;
+    const reductionPercent = originalLines > 0 ? Math.round(((originalLines - deduplicatedLines) / originalLines) * 100) : 0;
+    return {
+        originalLines,
+        deduplicatedLines,
+        uniqueErrors,
+        totalDuplicates,
+        reductionPercent,
+    };
+}

package/dist/utils/tfidf.d.ts

@@ -0,0 +1,45 @@
+/**
+ * TF-IDF (Term Frequency - Inverse Document Frequency) Utilities
+ *
+ * Provides local TF-IDF calculation for semantic importance scoring.
+ * No external ML models required - pure algorithmic implementation.
+ */
+/**
+ * TF-IDF result for a single term
+ */
+export interface TFIDFScore {
+    term: string;
+    tf: number;
+    idf: number;
+    tfidf: number;
+}
+/**
+ * TF-IDF scores for all terms in all segments
+ */
+export type TFIDFMap = Map<number, TFIDFScore[]>;
+/**
+ * Calculate TF-IDF scores for all segments
+ *
+ * @param segments - Array of text segments to analyze
+ * @returns Map of segment index to TF-IDF scores for each term
+ */
+export declare function calculateTFIDF(segments: string[]): TFIDFMap;
+/**
+ * Get the average TF-IDF score for a segment
+ * Higher score = more unique/important content
+ *
+ * @param segmentIndex - Index of the segment
+ * @param tfidfMap - Pre-calculated TF-IDF scores
+ * @returns Average TF-IDF score (0-1 normalized)
+ */
+export declare function getSegmentTFIDFScore(segmentIndex: number, tfidfMap: TFIDFMap): number;
+/**
+ * Get top terms for a segment (useful for debugging/display)
+ *
+ * @param segmentIndex - Index of the segment
+ * @param tfidfMap - Pre-calculated TF-IDF scores
+ * @param topN - Number of top terms to return
+ * @returns Array of top terms with scores
+ */
+export declare function getTopTerms(segmentIndex: number, tfidfMap: TFIDFMap, topN?: number): TFIDFScore[];
+//# sourceMappingURL=tfidf.d.ts.map

package/dist/utils/tfidf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.d.ts","sourceRoot":"","sources":["../../src/utils/tfidf.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;AAyIjD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,QAAQ,CAmC3D;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,QAAQ,GACjB,MAAM,CAYR;AAED;;;;;;;GAOG;AACH,wBAAgB,WAAW,CACzB,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,QAAQ,EAClB,IAAI,GAAE,MAAU,GACf,UAAU,EAAE,CAMd"}

package/dist/utils/tfidf.js

@@ -0,0 +1,204 @@
+/**
+ * TF-IDF (Term Frequency - Inverse Document Frequency) Utilities
+ *
+ * Provides local TF-IDF calculation for semantic importance scoring.
+ * No external ML models required - pure algorithmic implementation.
+ */
+// Common stopwords to filter out (they don't carry semantic meaning)
+const STOPWORDS = new Set([
+    "a",
+    "an",
+    "the",
+    "and",
+    "or",
+    "but",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "of",
+    "with",
+    "by",
+    "from",
+    "as",
+    "is",
+    "was",
+    "are",
+    "were",
+    "been",
+    "be",
+    "have",
+    "has",
+    "had",
+    "do",
+    "does",
+    "did",
+    "will",
+    "would",
+    "could",
+    "should",
+    "may",
+    "might",
+    "must",
+    "shall",
+    "can",
+    "need",
+    "dare",
+    "ought",
+    "used",
+    "it",
+    "its",
+    "this",
+    "that",
+    "these",
+    "those",
+    "i",
+    "you",
+    "he",
+    "she",
+    "we",
+    "they",
+    "what",
+    "which",
+    "who",
+    "whom",
+    "when",
+    "where",
+    "why",
+    "how",
+    "all",
+    "each",
+    "every",
+    "both",
+    "few",
+    "more",
+    "most",
+    "other",
+    "some",
+    "such",
+    "no",
+    "nor",
+    "not",
+    "only",
+    "own",
+    "same",
+    "so",
+    "than",
+    "too",
+    "very",
+    "just",
+    "also",
+]);
+/**
+ * Tokenize text into words for TF-IDF analysis
+ * - Lowercase
+ * - Remove punctuation
+ * - Filter stopwords
+ * - Keep words with 2+ characters
+ */
+function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s]/g, " ") // Remove punctuation
+        .split(/\s+/)
+        .filter((word) => word.length >= 2 && !STOPWORDS.has(word));
+}
+/**
+ * Calculate term frequency for a segment
+ * TF = count of term / total terms in segment
+ */
+function calculateTF(tokens) {
+    const counts = new Map();
+    for (const token of tokens) {
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+    const tf = new Map();
+    const total = tokens.length || 1;
+    for (const [term, count] of counts) {
+        tf.set(term, count / total);
+    }
+    return tf;
+}
+/**
+ * Calculate document frequency for all terms
+ * DF = number of segments containing the term
+ */
+function calculateDF(segmentTokens) {
+    const df = new Map();
+    for (const tokens of segmentTokens) {
+        const uniqueTerms = new Set(tokens);
+        for (const term of uniqueTerms) {
+            df.set(term, (df.get(term) ?? 0) + 1);
+        }
+    }
+    return df;
+}
+/**
+ * Calculate TF-IDF scores for all segments
+ *
+ * @param segments - Array of text segments to analyze
+ * @returns Map of segment index to TF-IDF scores for each term
+ */
+export function calculateTFIDF(segments) {
+    const result = new Map();
+    if (segments.length === 0) {
+        return result;
+    }
+    // Tokenize all segments
+    const segmentTokens = segments.map(tokenize);
+    // Calculate document frequency
+    const df = calculateDF(segmentTokens);
+    const numSegments = segments.length;
+    // Calculate TF-IDF for each segment
+    for (let i = 0; i < segments.length; i++) {
+        const tokens = segmentTokens[i];
+        const tf = calculateTF(tokens);
+        const scores = [];
+        for (const [term, tfScore] of tf) {
+            const dfScore = df.get(term) ?? 1;
+            // IDF = log(N / df) where N is total segments
+            const idf = Math.log(numSegments / dfScore);
+            const tfidf = tfScore * idf;
+            scores.push({ term, tf: tfScore, idf, tfidf });
+        }
+        // Sort by TF-IDF score descending
+        scores.sort((a, b) => b.tfidf - a.tfidf);
+        result.set(i, scores);
+    }
+    return result;
+}
+/**
+ * Get the average TF-IDF score for a segment
+ * Higher score = more unique/important content
+ *
+ * @param segmentIndex - Index of the segment
+ * @param tfidfMap - Pre-calculated TF-IDF scores
+ * @returns Average TF-IDF score (0-1 normalized)
+ */
+export function getSegmentTFIDFScore(segmentIndex, tfidfMap) {
+    const scores = tfidfMap.get(segmentIndex);
+    if (!scores || scores.length === 0) {
+        return 0;
+    }
+    // Calculate average TF-IDF
+    const sum = scores.reduce((acc, s) => acc + s.tfidf, 0);
+    const avg = sum / scores.length;
+    // Normalize to 0-1 range (typical TF-IDF values are 0-2)
+    return Math.min(avg / 2, 1);
+}
+/**
+ * Get top terms for a segment (useful for debugging/display)
+ *
+ * @param segmentIndex - Index of the segment
+ * @param tfidfMap - Pre-calculated TF-IDF scores
+ * @param topN - Number of top terms to return
+ * @returns Array of top terms with scores
+ */
+export function getTopTerms(segmentIndex, tfidfMap, topN = 5) {
+    const scores = tfidfMap.get(segmentIndex);
+    if (!scores) {
+        return [];
+    }
+    return scores.slice(0, topN);
+}

package/dist/utils/tfidf.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * TF-IDF Utilities Tests
+ */
+export {};
+//# sourceMappingURL=tfidf.test.d.ts.map

package/dist/utils/tfidf.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.test.d.ts","sourceRoot":"","sources":["../../src/utils/tfidf.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}