@hassangameryt/freeflow 0.1.0 → 0.3.0

package/router/dist/evidence/evidence-search.js

@@ -0,0 +1,466 @@
+import { selectEvidenceRangeForChunk } from "./evidence-range-selector.js";
+const DEFAULT_CONTEXT_LINES = 2;
+const QUERY_COVERAGE_MAX_LINES = 80;
+const STOPWORDS = new Set([
+    "a",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "by",
+    "for",
+    "from",
+    "has",
+    "in",
+    "is",
+    "it",
+    "of",
+    "on",
+    "or",
+    "that",
+    "the",
+    "this",
+    "to",
+    "with",
+    "use",
+]);
+export function searchRepoEvidenceCandidates(options) {
+    const tokens = uniqueTokens(tokenize(options.query));
+    const normalizedQueryPhrase = normalizePhraseSequence(options.query);
+    const chunks = options.files.flatMap((file) => candidateChunksForFile(file));
+    const matchingChunks = chunks.filter((chunk) => chunkMightMatch(chunk, tokens));
+    const stats = chunkStats(matchingChunks);
+    const scored = [];
+    for (const chunk of matchingChunks) {
+        const candidate = scoreCandidateChunk(chunk, tokens, normalizedQueryPhrase, stats, {
+            defaultContextLines: options.defaultContextLines ?? DEFAULT_CONTEXT_LINES,
+            queryCoverageMaxLines: options.queryCoverageMaxLines ?? QUERY_COVERAGE_MAX_LINES,
+        });
+        if (candidate) {
+            scored.push(candidate);
+        }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const selected = [];
+    const seen = new Set();
+    for (const candidate of scored) {
+        const key = candidate.file.path;
+        if (seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        selected.push(candidate);
+        if (selected.length >= options.topK) {
+            break;
+        }
+    }
+    return selected;
+}
+export function findBestLineForQuery(lines, query) {
+    const tokens = tokenize(query);
+    let bestLineIndex = null;
+    let bestScore = 0;
+    lines.forEach((line, lineIndex) => {
+        const score = scoreText(line, tokens);
+        if (score > bestScore) {
+            bestLineIndex = lineIndex;
+            bestScore = score;
+        }
+    });
+    return bestLineIndex;
+}
+function candidateChunksForFile(file) {
+    const structuralChunks = [];
+    structuralChunks.push(...markdownPreambleChunk(file));
+    structuralChunks.push(...markdownSectionChunks(file));
+    structuralChunks.push(...codeSymbolChunks(file));
+    return structuralChunks.length ? structuralChunks : lineWindowChunks(file);
+}
+function markdownPreambleChunk(file) {
+    const firstHeadingIndex = file.lines.findIndex((line) => line.trimStart().startsWith("#"));
+    if (firstHeadingIndex <= 0) {
+        return [];
+    }
+    const range = { start: 1, end: firstHeadingIndex };
+    return [{
+            file,
+            range,
+            text: file.lines.slice(range.start - 1, range.end).join("\n"),
+            heading: "",
+            kind: "section",
+        }];
+}
+function markdownSectionChunks(file) {
+    const headingIndexes = file.lines
+        .map((line, index) => ({ line, index }))
+        .filter(({ line }) => line.trimStart().startsWith("#"));
+    if (!headingIndexes.length) {
+        return [];
+    }
+    return headingIndexes.map(({ line, index }, headingOrdinal) => {
+        const nextHeading = headingIndexes[headingOrdinal + 1]?.index ?? file.lines.length;
+        const range = { start: index + 1, end: Math.max(index + 1, nextHeading) };
+        return {
+            file,
+            range,
+            text: file.lines.slice(range.start - 1, range.end).join("\n"),
+            heading: line,
+            kind: "section",
+        };
+    });
+}
+function codeSymbolChunks(file) {
+    const symbolIndexes = file.lines
+        .map((line, index) => ({ line, index, match: codeSymbolMatch(line) }))
+        .filter((entry) => Boolean(entry.match));
+    if (!symbolIndexes.length) {
+        return [];
+    }
+    return symbolIndexes.map(({ line, index }, symbolOrdinal) => {
+        const nextSymbol = nextSymbolBoundary(file.lines, symbolIndexes, symbolOrdinal);
+        const end = Math.min(file.lines.length, Math.max(index + 1, Math.min(nextSymbol, index + 80)));
+        const range = { start: index + 1, end };
+        return {
+            file,
+            range,
+            text: file.lines.slice(range.start - 1, range.end).join("\n"),
+            heading: line,
+            kind: "symbol",
+        };
+    });
+}
+function nextSymbolBoundary(lines, symbolIndexes, symbolOrdinal) {
+    const current = symbolIndexes[symbolOrdinal];
+    if (!current) {
+        return lines.length;
+    }
+    if (/^\s*impl\b/.test(current.line)) {
+        const nextTopLevel = symbolIndexes.slice(symbolOrdinal + 1).find((entry) => isTopLevelCodeSymbol(entry.line));
+        return nextTopLevel?.index ?? lines.length;
+    }
+    return symbolIndexes[symbolOrdinal + 1]?.index ?? lines.length;
+}
+function isTopLevelCodeSymbol(line) {
+    return /^(?:pub\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|class|interface|type|const|def)\s+([A-Za-z_][A-Za-z0-9_]*)\b/.test(line);
+}
+function codeSymbolMatch(line) {
+    return /^\s*(?:pub\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|class|interface|type|const|def)\s+([A-Za-z_][A-Za-z0-9_]*)\b/.exec(line);
+}
+function lineWindowChunks(file) {
+    return file.lines.map((_, lineIndex) => {
+        const start = Math.max(0, lineIndex - DEFAULT_CONTEXT_LINES);
+        const end = Math.min(file.lines.length - 1, lineIndex + DEFAULT_CONTEXT_LINES);
+        return {
+            file,
+            range: { start: start + 1, end: end + 1 },
+            text: file.lines.slice(start, end + 1).join("\n"),
+            kind: "window",
+        };
+    });
+}
+function chunkStats(chunks) {
+    const documentFrequency = new Map();
+    let totalLength = 0;
+    for (const chunk of chunks) {
+        const tokens = tokensForChunk(chunk);
+        totalLength += Math.max(1, tokens.length);
+        for (const token of uniqueTokens(tokens)) {
+            documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
+        }
+    }
+    return {
+        averageLength: chunks.length ? totalLength / chunks.length : 1,
+        documentFrequency,
+        chunkCount: chunks.length,
+    };
+}
+function scoreCandidateChunk(chunk, queryTokens, normalizedQueryPhrase, stats, options) {
+    if (!queryTokens.length) {
+        return null;
+    }
+    const chunkTokens = tokensForChunk(chunk);
+    const tokenCounts = tokenCountMap(chunkTokens);
+    const matchingTokens = queryTokens.filter((token) => (tokenCounts.get(token) ?? 0) > 0);
+    const pathScore = scoreText(chunk.file.path, queryTokens);
+    if (!matchingTokens.length && pathScore <= 0) {
+        return null;
+    }
+    const exactPhrase = hasExactNormalizedPhraseInChunk(chunk, normalizedQueryPhrase);
+    const chunkLength = Math.max(1, chunkTokens.length);
+    const score = exactPhraseBoost(exactPhrase, chunk.file.path, queryTokens) +
+        bm25Score(queryTokens, tokenCounts, chunkLength, stats) * 10 +
+        coverageRatio(matchingTokens, queryTokens) * 120 +
+        contentCoverageBoost(matchingTokens, queryTokens, chunk.kind) +
+        completeCoverageBoost(matchingTokens, queryTokens, chunk.kind) -
+        missingCoveragePenalty(matchingTokens, queryTokens, chunk.kind) -
+        lowContentCoveragePenalty(matchingTokens, queryTokens, pathScore) +
+        headingCoverageBoost(chunk.heading ?? "", queryTokens) +
+        identifierBoost(chunk.text, queryTokens) +
+        (matchingTokens.length >= 4 ? orderedPhraseBoost(chunkTokens, queryTokens) : 0) +
+        (chunk.kind === "symbol" ? codeDefinitionBoost(chunk.text, queryTokens) : 0) +
+        chunkKindBoost(chunk.kind) +
+        pathIntentBoost(chunk.file.path, queryTokens) +
+        sourceTestPrior(chunk.file.path, queryTokens) +
+        pathScore -
+        Math.log1p(chunkLength) * 10;
+    const selection = selectEvidenceRangeForChunk({
+        lines: chunk.file.lines,
+        chunkRange: chunk.range,
+        chunkKind: chunk.kind,
+        queryTokens,
+        normalizedQueryPhrase,
+        chunkHasExactPhrase: exactPhrase,
+        defaultContextLines: options.defaultContextLines,
+        queryCoverageMaxLines: options.queryCoverageMaxLines,
+    });
+    const reason = selection.matchKind === "exact-phrase"
+        ? `matched exact normalized query phrase in ${chunk.kind} chunk`
+        : `BM25-style scored ${chunk.kind} chunk with ${matchingTokens.length}/${queryTokens.length} query-token coverage`;
+    return {
+        file: chunk.file,
+        lineIndex: selection.anchorLine - 1,
+        range: selection.range,
+        score,
+        reason,
+        ...(selection.matchKind === "exact-phrase" ? { exactNormalizedPhrase: normalizedQueryPhrase } : {}),
+    };
+}
+function chunkMightMatch(chunk, queryTokens) {
+    const path = chunk.file.path.toLowerCase();
+    const heading = chunk.heading?.toLowerCase() ?? "";
+    const text = chunk.text.toLowerCase();
+    return queryTokens.some((token) => path.includes(token) || heading.includes(token) || text.includes(token));
+}
+function bm25Score(queryTokens, tokenCounts, chunkLength, stats) {
+    const k1 = 1.2;
+    const b = 0.75;
+    return queryTokens.reduce((score, token) => {
+        const termFrequency = tokenCounts.get(token) ?? 0;
+        if (termFrequency <= 0) {
+            return score;
+        }
+        const documentFrequency = stats.documentFrequency.get(token) ?? 0;
+        const idf = Math.log(1 + (stats.chunkCount - documentFrequency + 0.5) / (documentFrequency + 0.5));
+        const normalizedLength = k1 * (1 - b + b * (chunkLength / Math.max(1, stats.averageLength)));
+        const saturatedFrequency = (termFrequency * (k1 + 1)) / (termFrequency + normalizedLength);
+        return score + idf * saturatedFrequency;
+    }, 0);
+}
+function coverageRatio(matchingTokens, queryTokens) {
+    return queryTokens.length ? matchingTokens.length / queryTokens.length : 0;
+}
+function contentCoverageBoost(matchingTokens, queryTokens, chunkKind) {
+    if (queryTokens.length === 0) {
+        return 0;
+    }
+    const perTokenBoost = chunkKind === "window" ? 180 : 350;
+    return matchingTokens.length * perTokenBoost + coverageRatio(matchingTokens, queryTokens) * 100;
+}
+function completeCoverageBoost(matchingTokens, queryTokens, chunkKind) {
+    if (queryTokens.length === 0 || matchingTokens.length !== queryTokens.length) {
+        return 0;
+    }
+    if (chunkKind === "symbol") {
+        return 240;
+    }
+    return queryTokens.length >= 4 ? 1_200 : 240;
+}
+function missingCoveragePenalty(matchingTokens, queryTokens, chunkKind) {
+    if (chunkKind === "symbol" || queryTokens.length < 4 || matchingTokens.length >= queryTokens.length) {
+        return 0;
+    }
+    return (queryTokens.length - matchingTokens.length) * 90;
+}
+function lowContentCoveragePenalty(matchingTokens, queryTokens, pathScore) {
+    if (queryTokens.length < 4 || matchingTokens.length >= 2) {
+        return 0;
+    }
+    const pathOnlyPenalty = matchingTokens.length === 0 && pathScore > 0 ? 1_000 : 0;
+    return (2 - matchingTokens.length) * 600 + pathOnlyPenalty;
+}
+function exactPhraseBoost(exactPhrase, path, queryTokens) {
+    if (!exactPhrase) {
+        return 0;
+    }
+    const testPath = isTestPath(path);
+    const testIntent = queryTokens.some((token) => ["test", "tests", "expect", "expected", "should", "emitted"].includes(token));
+    return testPath && !testIntent ? 1_000 : 100_000;
+}
+function headingCoverageBoost(heading, queryTokens) {
+    const headingTokens = new Set(tokenize(heading));
+    const coveredTokens = queryTokens.filter((token) => headingTokens.has(token)).length;
+    return coveredTokens * 8;
+}
+function identifierBoost(text, queryTokens) {
+    if (!text.includes("`")) {
+        return 0;
+    }
+    const identifiers = Array.from(text.matchAll(/`([^`]+)`/g), (match) => tokenize(match[1] ?? "")).flat();
+    if (!identifiers.length) {
+        return 0;
+    }
+    const identifierSet = new Set(identifiers);
+    return queryTokens.filter((token) => identifierSet.has(token)).length * 10;
+}
+function orderedPhraseBoost(chunkTokens, queryTokens) {
+    const normalizedText = ` ${chunkTokens.join(" ")} `;
+    let best = 0;
+    for (let start = 0; start < queryTokens.length; start += 1) {
+        const phrase = [];
+        for (let end = start; end < queryTokens.length; end += 1) {
+            phrase.push(queryTokens[end] ?? "");
+            if (phrase.length <= best) {
+                continue;
+            }
+            if (normalizedText.includes(` ${phrase.join(" ")} `)) {
+                best = phrase.length;
+            }
+        }
+    }
+    return best >= 3 ? best * best * 20 : 0;
+}
+function codeDefinitionBoost(text, queryTokens) {
+    const definitionMatches = Array.from(text.matchAll(/^\s*(?:pub\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|class|interface|type|const|def)\s+([A-Za-z_][A-Za-z0-9_]*)\b/gm), (match) => match[1] ?? "");
+    if (!definitionMatches.length) {
+        return 0;
+    }
+    let bestBoost = 0;
+    for (const name of definitionMatches) {
+        const symbolTokens = new Set(tokenize(name));
+        const coverage = queryTokens.filter((token) => symbolTokens.has(token)).length;
+        const exactCompoundCoverage = queryTokens.filter((token) => token.length >= 8 && symbolTokens.has(token) && !/[._/-]/.test(token)).length;
+        if (coverage > 0) {
+            bestBoost = Math.max(bestBoost, 120 + coverage * 80 + exactCompoundCoverage * 520);
+        }
+    }
+    return bestBoost;
+}
+function chunkKindBoost(kind) {
+    if (kind === "symbol") {
+        return 500;
+    }
+    if (kind === "section") {
+        return 12;
+    }
+    return 0;
+}
+function pathIntentBoost(path, queryTokens) {
+    const pathTokens = new Set(tokenize(path));
+    const coveredTokens = queryTokens.filter((token) => pathTokens.has(token)).length;
+    return coveredTokens * 140 + (coveredTokens >= 2 ? 600 : 0);
+}
+function sourceTestPrior(path, queryTokens) {
+    const testPath = isTestPath(path);
+    const testIntent = queryTokens.some((token) => ["test", "tests", "expect", "expected", "should", "emitted"].includes(token));
+    if (testPath && !testIntent) {
+        return -2_000;
+    }
+    if (testPath && testIntent) {
+        return 35;
+    }
+    if (!testPath && testIntent) {
+        return -8;
+    }
+    return 0;
+}
+function isTestPath(path) {
+    const lower = path.toLowerCase();
+    return /(^|\/)(tests?|fixtures?)(\/|$)/.test(lower) || lower.endsWith("_tests.rs") || lower.endsWith(".test.ts") || lower.endsWith(".test.js");
+}
+function hasExactNormalizedPhraseInChunk(chunk, normalizedQueryPhrase) {
+    return normalizedQueryPhrase !== "" && phraseSequenceForChunk(chunk).includes(normalizedQueryPhrase);
+}
+function tokensForChunk(chunk) {
+    chunk.tokens ??= tokenize(chunk.text);
+    return chunk.tokens;
+}
+function phraseSequenceForChunk(chunk) {
+    chunk.phraseSequence ??= normalizePhraseSequence(chunk.text);
+    return chunk.phraseSequence;
+}
+function normalizePhraseSequence(text) {
+    const tokens = tokenizeForPhrase(text);
+    return tokens.length >= 2 ? tokens.join(" ") : "";
+}
+function tokenCountMap(tokens) {
+    const counts = new Map();
+    for (const token of tokens) {
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+    return counts;
+}
+function uniqueTokens(tokens) {
+    return Array.from(new Set(tokens));
+}
+function scoreText(text, tokens) {
+    const lower = text.toLowerCase();
+    return tokens.reduce((score, token) => score + countOccurrences(lower, token), 0);
+}
+function countOccurrences(text, token) {
+    let count = 0;
+    let index = text.indexOf(token);
+    while (index !== -1) {
+        count += 1;
+        index = text.indexOf(token, index + token.length);
+    }
+    return count;
+}
+function tokenize(query) {
+    return query
+        .split(/[^A-Za-z0-9_./-]+/)
+        .flatMap((token) => expandedIdentifierTokens(token.trim()))
+        .filter((token) => token.length >= 2 && !STOPWORDS.has(token));
+}
+function tokenizeForPhrase(query) {
+    return query
+        .split(/[^A-Za-z0-9_./-]+/)
+        .flatMap((token) => expandedIdentifierTokens(token.trim()))
+        .filter((token) => token.length >= 2);
+}
+function expandedIdentifierTokens(token) {
+    if (!token) {
+        return [];
+    }
+    const lower = token.toLowerCase();
+    if (!needsIdentifierSplit(token)) {
+        return [lower];
+    }
+    const variants = new Set();
+    addTokenVariant(variants, lower);
+    for (const part of splitIdentifierToken(token)) {
+        addTokenVariant(variants, part.toLowerCase());
+    }
+    return Array.from(variants);
+}
+function addTokenVariant(variants, token) {
+    if (token) {
+        variants.add(token);
+    }
+}
+function needsIdentifierSplit(token) {
+    if (/[._/-]/.test(token)) {
+        return true;
+    }
+    for (let index = 1; index < token.length; index += 1) {
+        const previous = token.charCodeAt(index - 1);
+        const current = token.charCodeAt(index);
+        const next = index + 1 < token.length ? token.charCodeAt(index + 1) : 0;
+        const previousIsLowerOrDigit = (previous >= 97 && previous <= 122) || (previous >= 48 && previous <= 57);
+        const previousIsUpper = previous >= 65 && previous <= 90;
+        const currentIsUpper = current >= 65 && current <= 90;
+        const nextIsLower = next >= 97 && next <= 122;
+        if ((previousIsLowerOrDigit && currentIsUpper) || (previousIsUpper && currentIsUpper && nextIsLower)) {
+            return true;
+        }
+    }
+    return false;
+}
+function splitIdentifierToken(token) {
+    const separated = token
+        .replace(/[._/-]+/g, " ")
+        .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+    return separated.split(/\s+/).filter(Boolean);
+}

package/router/dist/evidence/evidence.d.ts

@@ -0,0 +1,38 @@
+import type { ImportantLine } from "../config/types.js";
+export type ImportantStream = ImportantLine["stream"];
+export interface LineEntry {
+    line: string;
+    lineNumber: number;
+}
+export interface EvidenceCaps {
+    maxLines?: number;
+    maxExcerptBytes?: number;
+    maxLineBytes?: number;
+}
+export interface BoundedEvidence {
+    importantLines: ImportantLine[];
+    fidelity: "exact" | "lossy";
+    compressed: boolean;
+    selectedLineCount: number;
+    sourceLineCount: number;
+}
+export declare const DEFAULT_MAX_IMPORTANT_LINES = 8;
+export declare const DEFAULT_MAX_IMPORTANT_EXCERPT_BYTES: number;
+export declare const DEFAULT_MAX_IMPORTANT_LINE_BYTES: number;
+export declare const TRUNCATION_MARKER = "\u2026 [truncated; recover exact output from vault]";
+export declare function assembleImportantLines(options: {
+    stream: ImportantStream;
+    entries: readonly LineEntry[];
+    sourceText: string;
+    caps?: EvidenceCaps;
+}): BoundedEvidence;
+export declare function assembleTextEvidence(options: {
+    stream: ImportantStream;
+    text: string;
+    caps?: EvidenceCaps;
+}): BoundedEvidence;
+export declare function lineEntries(text: string): LineEntry[];
+export declare function nonEmptyLineEntries(text: string): LineEntry[];
+export declare function splitLines(text: string): string[];
+export declare function countLines(text: string): number;
+export declare function byteLength(text: string): number;

package/router/dist/evidence/evidence.js

@@ -0,0 +1,196 @@
+export const DEFAULT_MAX_IMPORTANT_LINES = 8;
+export const DEFAULT_MAX_IMPORTANT_EXCERPT_BYTES = 8 * 1024;
+export const DEFAULT_MAX_IMPORTANT_LINE_BYTES = 2 * 1024;
+export const TRUNCATION_MARKER = "… [truncated; recover exact output from vault]";
+export function assembleImportantLines(options) {
+    const caps = normalizeCaps(options.caps);
+    const sourceLineCount = countLines(options.sourceText);
+    const sortedEntries = uniqueSortedEntries(options.entries).slice(0, caps.maxLines);
+    const sourceEntries = lineEntries(options.sourceText);
+    if (sortedEntries.length === 0) {
+        return {
+            importantLines: [],
+            fidelity: "exact",
+            compressed: sourceLineCount > 0,
+            selectedLineCount: 0,
+            sourceLineCount,
+        };
+    }
+    const selectedLineNumbers = new Set(sortedEntries.map((entry) => entry.lineNumber));
+    const omittedSelectedEntries = uniqueSortedEntries(options.entries).length > sortedEntries.length;
+    const omittedSourceEntries = sourceEntries.some((entry) => !selectedLineNumbers.has(entry.lineNumber));
+    const groups = contiguousGroups(sortedEntries);
+    const importantLines = [];
+    let remainingBytes = caps.maxExcerptBytes;
+    let lossy = omittedSelectedEntries;
+    let selectedLineCount = 0;
+    for (const group of groups) {
+        if (remainingBytes <= 0) {
+            lossy = true;
+            break;
+        }
+        const built = buildBoundedExcerpt(group, remainingBytes, caps.maxLineBytes);
+        if (built.excerpt.length === 0 && group.length > 0) {
+            lossy = true;
+            break;
+        }
+        importantLines.push({
+            stream: options.stream,
+            lines: lineRange(built.entries),
+            excerpt: built.excerpt,
+        });
+        remainingBytes -= byteLength(built.excerpt);
+        selectedLineCount += built.selectedLineCount;
+        lossy = lossy || built.lossy;
+    }
+    return {
+        importantLines,
+        fidelity: lossy ? "lossy" : "exact",
+        compressed: lossy || omittedSourceEntries,
+        selectedLineCount,
+        sourceLineCount,
+    };
+}
+export function assembleTextEvidence(options) {
+    const caps = normalizeCaps(options.caps);
+    const entries = lineEntries(options.text);
+    const sourceLineCount = entries.length;
+    if (entries.length === 0) {
+        return {
+            importantLines: [],
+            fidelity: "exact",
+            compressed: false,
+            selectedLineCount: 0,
+            sourceLineCount,
+        };
+    }
+    if (entries.length <= caps.maxLines &&
+        byteLength(options.text) <= caps.maxExcerptBytes &&
+        entries.every((entry) => byteLength(entry.line) <= caps.maxLineBytes)) {
+        return {
+            importantLines: [
+                {
+                    stream: options.stream,
+                    lines: lineRange(entries),
+                    excerpt: options.text,
+                },
+            ],
+            fidelity: "exact",
+            compressed: false,
+            selectedLineCount: entries.length,
+            sourceLineCount,
+        };
+    }
+    return assembleImportantLines({ stream: options.stream, entries, sourceText: options.text, caps });
+}
+export function lineEntries(text) {
+    return splitLines(text).map((line, index) => ({ line, lineNumber: index + 1 }));
+}
+export function nonEmptyLineEntries(text) {
+    return lineEntries(text).filter((entry) => entry.line.trim().length > 0);
+}
+export function splitLines(text) {
+    if (text.length === 0) {
+        return [];
+    }
+    return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n");
+}
+export function countLines(text) {
+    return splitLines(text).length;
+}
+export function byteLength(text) {
+    return Buffer.byteLength(text, "utf8");
+}
+function normalizeCaps(caps) {
+    return {
+        maxLines: caps?.maxLines ?? DEFAULT_MAX_IMPORTANT_LINES,
+        maxExcerptBytes: caps?.maxExcerptBytes ?? DEFAULT_MAX_IMPORTANT_EXCERPT_BYTES,
+        maxLineBytes: caps?.maxLineBytes ?? DEFAULT_MAX_IMPORTANT_LINE_BYTES,
+    };
+}
+function uniqueSortedEntries(entries) {
+    const byLineNumber = new Map();
+    for (const entry of entries) {
+        if (entry.lineNumber <= 0 || byLineNumber.has(entry.lineNumber)) {
+            continue;
+        }
+        byLineNumber.set(entry.lineNumber, entry);
+    }
+    return [...byLineNumber.values()].sort((a, b) => a.lineNumber - b.lineNumber);
+}
+function contiguousGroups(entries) {
+    const groups = [];
+    let current = [];
+    for (const entry of entries) {
+        const previous = current[current.length - 1];
+        if (!previous || entry.lineNumber === previous.lineNumber + 1) {
+            current.push(entry);
+            continue;
+        }
+        groups.push(current);
+        current = [entry];
+    }
+    if (current.length > 0) {
+        groups.push(current);
+    }
+    return groups;
+}
+function buildBoundedExcerpt(entries, maxExcerptBytes, maxLineBytes) {
+    const parts = [];
+    const includedEntries = [];
+    let remainingBytes = maxExcerptBytes;
+    let lossy = false;
+    let selectedLineCount = 0;
+    for (const entry of entries) {
+        const separatorBytes = parts.length === 0 ? 0 : 1;
+        const remainingForLine = remainingBytes - separatorBytes;
+        if (remainingForLine <= 0) {
+            lossy = true;
+            break;
+        }
+        const line = truncateLine(entry.line, Math.min(maxLineBytes, remainingForLine));
+        if (parts.length > 0) {
+            remainingBytes -= 1;
+        }
+        parts.push(line.text);
+        includedEntries.push(entry);
+        remainingBytes -= byteLength(line.text);
+        selectedLineCount += 1;
+        lossy = lossy || line.lossy;
+    }
+    return { excerpt: parts.join("\n"), entries: includedEntries, lossy, selectedLineCount };
+}
+function truncateLine(line, maxBytes) {
+    if (byteLength(line) <= maxBytes) {
+        return { text: line, lossy: false };
+    }
+    if (maxBytes <= 0) {
+        return { text: "", lossy: true };
+    }
+    const markerBytes = byteLength(TRUNCATION_MARKER);
+    if (maxBytes <= markerBytes) {
+        return { text: truncateUtf8(TRUNCATION_MARKER, maxBytes), lossy: true };
+    }
+    return {
+        text: `${truncateUtf8(line, maxBytes - markerBytes)}${TRUNCATION_MARKER}`,
+        lossy: true,
+    };
+}
+function truncateUtf8(text, maxBytes) {
+    if (byteLength(text) <= maxBytes) {
+        return text;
+    }
+    let end = Math.min(text.length, maxBytes);
+    while (end > 0 && byteLength(text.slice(0, end)) > maxBytes) {
+        end -= 1;
+    }
+    return text.slice(0, end);
+}
+function lineRange(entries) {
+    const first = entries[0];
+    const last = entries[entries.length - 1];
+    if (!first || !last) {
+        return "1-1";
+    }
+    return first.lineNumber === last.lineNumber ? `${first.lineNumber}-${first.lineNumber}` : `${first.lineNumber}-${last.lineNumber}`;
+}