@hassangameryt/freeflow 0.1.0 → 0.3.0

package/router/dist/evidence/bounded-evidence.js

@@ -0,0 +1,310 @@
+const TRUNCATION_SUFFIX = " … [truncated; expand or retrieve exact lines for recovery]";
+const TRUNCATION_PREFIX = "[truncated head; expand or retrieve exact lines for recovery] … ";
+export function buildBoundedEdgeChunks(options) {
+    return overCapEdgeRanges(options.lines, options.range, options.caps).map((range) => {
+        const excerpt = range.edge === "tail"
+            ? truncateTailToUtf8Bytes(linesForRange(options.lines, range), options.caps.exactChunkMaxBytes)
+            : truncateToUtf8Bytes(linesForRange(options.lines, range), options.caps.exactChunkMaxBytes);
+        return {
+            range: { start: range.start, end: range.end },
+            linesLabel: `${range.start}-${range.end}`,
+            edge: range.edge,
+            excerpt,
+        };
+    });
+}
+export function buildBoundedExcerpt(options) {
+    const capped = capLineRangeForWindow(options.range, options.window, options.caps);
+    const selected = options.lines.slice(capped.range.start - 1, capped.range.end);
+    const maxBytes = maxExcerptBytesForWindow(options.window, options.caps);
+    if (maxBytes !== null && options.exactNormalizedPhrase !== undefined) {
+        const exactExcerpt = excerptAroundNormalizedPhrase(selected.join("\n"), options.exactNormalizedPhrase, maxBytes);
+        if (exactExcerpt !== null) {
+            return {
+                range: capped.range,
+                linesLabel: `${capped.range.start}-${capped.range.end}`,
+                excerpt: exactExcerpt,
+                truncatedByLineCap: capped.truncated,
+                truncatedByByteCap: byteLength(selected.join("\n")) > maxBytes,
+            };
+        }
+    }
+    const previewedLines = maxBytes !== null
+        ? selected.map((line) => truncateLinePreview(line, options.caps.linePreviewMaxBytes, options.exactNormalizedPhrase))
+        : selected;
+    const uncappedExcerpt = previewedLines.join("\n");
+    const excerpt = maxBytes === null ? uncappedExcerpt : truncateToUtf8Bytes(uncappedExcerpt, maxBytes);
+    return {
+        range: capped.range,
+        linesLabel: `${capped.range.start}-${capped.range.end}`,
+        excerpt,
+        truncatedByLineCap: capped.truncated,
+        truncatedByByteCap: maxBytes !== null && byteLength(uncappedExcerpt) > maxBytes,
+    };
+}
+function overCapEdgeRanges(lines, range, caps) {
+    const lineCount = range.end - range.start + 1;
+    const chunkLineCount = Math.min(10, Math.max(1, Math.floor(lineCount / 2)));
+    const headRange = shrinkRangeToMaxBytes(lines, { start: range.start, end: Math.min(range.end, range.start + chunkLineCount - 1) }, "head", caps);
+    const tailRange = shrinkRangeToMaxBytes(lines, { start: Math.max(range.start, range.end - chunkLineCount + 1), end: range.end }, "tail", caps);
+    const head = { ...headRange, edge: "head" };
+    const tail = { ...tailRange, edge: "tail" };
+    if (tail.start <= head.end) {
+        return byteLength(linesForRange(lines, head)) > caps.exactChunkMaxBytes ? [head, tail] : [head];
+    }
+    return [head, tail];
+}
+function shrinkRangeToMaxBytes(lines, range, edge, caps) {
+    let current = range;
+    while (current.end > current.start && byteLength(linesForRange(lines, current)) > caps.exactChunkMaxBytes) {
+        current = edge === "head" ? { start: current.start, end: current.end - 1 } : { start: current.start + 1, end: current.end };
+    }
+    return current;
+}
+function linesForRange(lines, range) {
+    return lines.slice(range.start - 1, range.end).join("\n");
+}
+function capLineRangeForWindow(range, window, caps) {
+    const maxLines = maxLinesForWindow(window, caps);
+    const lineCount = range.end - range.start + 1;
+    if (maxLines === null || lineCount <= maxLines) {
+        return { range, truncated: false };
+    }
+    return {
+        range: { start: range.start, end: range.start + maxLines - 1 },
+        truncated: true,
+    };
+}
+function maxLinesForWindow(window, caps) {
+    if (window === "lines_30") {
+        return caps.expandLines30MaxLines;
+    }
+    if (window === "lines_80") {
+        return caps.expandLines80MaxLines;
+    }
+    return null;
+}
+function maxExcerptBytesForWindow(window, caps) {
+    if (window === "small" || window === "section") {
+        return caps.queryExcerptMaxBytes;
+    }
+    if (window === "lines_30") {
+        return caps.expandLines30MaxBytes;
+    }
+    if (window === "lines_80") {
+        return caps.expandLines80MaxBytes;
+    }
+    return null;
+}
+function excerptAroundNormalizedPhrase(text, exactNormalizedPhrase, maxBytes) {
+    const span = normalizedPhraseRawSpan(text, exactNormalizedPhrase);
+    if (span === null) {
+        return null;
+    }
+    return truncateToUtf8BytesAroundSpan(text, span, maxBytes);
+}
+function truncateLinePreview(text, maxBytes, exactNormalizedPhrase) {
+    if (exactNormalizedPhrase === undefined || byteLength(text) <= maxBytes) {
+        return truncateToUtf8Bytes(text, maxBytes);
+    }
+    const span = normalizedPhraseRawSpan(text, exactNormalizedPhrase);
+    if (span === null) {
+        return truncateToUtf8Bytes(text, maxBytes);
+    }
+    return truncateToUtf8BytesAroundSpan(text, span, maxBytes);
+}
+function truncateToUtf8Bytes(text, maxBytes) {
+    if (byteLength(text) <= maxBytes) {
+        return text;
+    }
+    const suffixBytes = byteLength(TRUNCATION_SUFFIX);
+    const contentBytes = Math.max(0, maxBytes - suffixBytes);
+    return `${truncateHeadToUtf8Bytes(text, contentBytes)}${TRUNCATION_SUFFIX}`;
+}
+function truncateToUtf8BytesAroundSpan(text, span, maxBytes) {
+    if (byteLength(text) <= maxBytes) {
+        return text;
+    }
+    const prefix = span.start > 0 ? TRUNCATION_PREFIX : "";
+    const suffix = span.end < text.length ? TRUNCATION_SUFFIX : "";
+    const budget = maxBytes - byteLength(prefix) - byteLength(suffix);
+    const phrase = text.slice(span.start, span.end);
+    const phraseBytes = byteLength(phrase);
+    if (budget <= 0 || phraseBytes >= budget) {
+        return truncateToUtf8Bytes(text.slice(span.start), maxBytes);
+    }
+    const contextBudget = budget - phraseBytes;
+    const beforeBudget = Math.floor(contextBudget / 2);
+    const afterBudget = contextBudget - beforeBudget;
+    let start = span.start;
+    while (start > 0 && byteLength(text.slice(start - 1, span.start)) <= beforeBudget) {
+        start -= 1;
+    }
+    if (byteLength(text.slice(start, span.start)) > beforeBudget) {
+        start += 1;
+    }
+    let end = span.end;
+    while (end < text.length && byteLength(text.slice(span.end, end + 1)) <= afterBudget) {
+        end += 1;
+    }
+    if (byteLength(text.slice(span.end, end)) > afterBudget) {
+        end -= 1;
+    }
+    start = moveToCodePointStartBoundary(text, start);
+    end = moveToCodePointEndBoundary(text, end);
+    const actualPrefix = start > 0 ? prefix : "";
+    const actualSuffix = end < text.length ? suffix : "";
+    return `${actualPrefix}${text.slice(start, end)}${actualSuffix}`;
+}
+function normalizedPhraseRawSpan(text, normalizedPhrase) {
+    const phraseTokens = normalizedPhrase.split(/\s+/).filter(Boolean);
+    if (phraseTokens.length === 0) {
+        return null;
+    }
+    const spans = tokenSpansForPhrase(text);
+    for (let startIndex = 0; startIndex <= spans.length - phraseTokens.length; startIndex += 1) {
+        let matched = true;
+        for (let offset = 0; offset < phraseTokens.length; offset += 1) {
+            if (spans[startIndex + offset]?.token !== phraseTokens[offset]) {
+                matched = false;
+                break;
+            }
+        }
+        if (matched) {
+            const first = spans[startIndex];
+            const last = spans[startIndex + phraseTokens.length - 1];
+            if (first !== undefined && last !== undefined) {
+                return { start: first.start, end: last.end };
+            }
+        }
+    }
+    return null;
+}
+function tokenSpansForPhrase(text) {
+    const spans = [];
+    for (const match of text.matchAll(/[A-Za-z0-9_./-]+/g)) {
+        const rawToken = match[0];
+        const start = match.index ?? 0;
+        const end = start + rawToken.length;
+        for (const token of expandedIdentifierTokens(rawToken)) {
+            if (token.length >= 2) {
+                spans.push({ token, start, end });
+            }
+        }
+    }
+    return spans;
+}
+function expandedIdentifierTokens(token) {
+    if (!token) {
+        return [];
+    }
+    const lower = token.toLowerCase();
+    if (!needsIdentifierSplit(token)) {
+        return [lower];
+    }
+    const variants = new Set();
+    addTokenVariant(variants, lower);
+    for (const part of splitIdentifierToken(token)) {
+        addTokenVariant(variants, part.toLowerCase());
+    }
+    return Array.from(variants);
+}
+function addTokenVariant(variants, token) {
+    if (token) {
+        variants.add(token);
+    }
+}
+function needsIdentifierSplit(token) {
+    if (/[._/-]/.test(token)) {
+        return true;
+    }
+    for (let index = 1; index < token.length; index += 1) {
+        const previous = token.charCodeAt(index - 1);
+        const current = token.charCodeAt(index);
+        const next = index + 1 < token.length ? token.charCodeAt(index + 1) : 0;
+        const previousIsLowerOrDigit = (previous >= 97 && previous <= 122) || (previous >= 48 && previous <= 57);
+        const previousIsUpper = previous >= 65 && previous <= 90;
+        const currentIsUpper = current >= 65 && current <= 90;
+        const nextIsLower = next >= 97 && next <= 122;
+        if ((previousIsLowerOrDigit && currentIsUpper) || (previousIsUpper && currentIsUpper && nextIsLower)) {
+            return true;
+        }
+    }
+    return false;
+}
+function splitIdentifierToken(token) {
+    const separated = token
+        .replace(/[._/-]+/g, " ")
+        .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
+    return separated.split(/\s+/).filter(Boolean);
+}
+function truncateTailToUtf8Bytes(text, maxBytes) {
+    if (byteLength(text) <= maxBytes) {
+        return text;
+    }
+    const prefixBytes = byteLength(TRUNCATION_PREFIX);
+    const contentBytes = Math.max(0, maxBytes - prefixBytes);
+    return `${TRUNCATION_PREFIX}${truncateTailToUtf8BytesContent(text, contentBytes)}`;
+}
+function truncateHeadToUtf8Bytes(text, maxBytes) {
+    if (maxBytes <= 0) {
+        return "";
+    }
+    if (byteLength(text) <= maxBytes) {
+        return text;
+    }
+    let low = 0;
+    let high = text.length;
+    while (low < high) {
+        const mid = Math.ceil((low + high) / 2);
+        if (byteLength(text.slice(0, mid)) <= maxBytes) {
+            low = mid;
+        }
+        else {
+            high = mid - 1;
+        }
+    }
+    return text.slice(0, moveToCodePointEndBoundary(text, low));
+}
+function truncateTailToUtf8BytesContent(text, maxBytes) {
+    if (maxBytes <= 0) {
+        return "";
+    }
+    if (byteLength(text) <= maxBytes) {
+        return text;
+    }
+    let low = 0;
+    let high = text.length;
+    while (low < high) {
+        const mid = Math.floor((low + high) / 2);
+        if (byteLength(text.slice(mid)) <= maxBytes) {
+            high = mid;
+        }
+        else {
+            low = mid + 1;
+        }
+    }
+    return text.slice(moveToCodePointStartBoundary(text, low));
+}
+function moveToCodePointStartBoundary(text, index) {
+    if (index > 0 && index < text.length && isLowSurrogate(text.charCodeAt(index))) {
+        return index + 1;
+    }
+    return index;
+}
+function moveToCodePointEndBoundary(text, index) {
+    if (index > 0 && isHighSurrogate(text.charCodeAt(index - 1))) {
+        return index - 1;
+    }
+    return index;
+}
+function isHighSurrogate(code) {
+    return code >= 0xd800 && code <= 0xdbff;
+}
+function isLowSurrogate(code) {
+    return code >= 0xdc00 && code <= 0xdfff;
+}
+function byteLength(text) {
+    return Buffer.byteLength(text, "utf8");
+}

package/router/dist/evidence/evidence-range-selector.d.ts

@@ -0,0 +1,22 @@
+export interface LineRange {
+    start: number;
+    end: number;
+}
+export type EvidenceChunkKind = "section" | "symbol" | "window";
+export type EvidenceRangeMatchKind = "exact-phrase" | "coverage" | "best-line";
+export interface SelectEvidenceRangeForChunkOptions {
+    lines: readonly string[];
+    chunkRange: LineRange;
+    chunkKind: EvidenceChunkKind;
+    queryTokens: readonly string[];
+    normalizedQueryPhrase: string;
+    chunkHasExactPhrase: boolean;
+    defaultContextLines: number;
+    queryCoverageMaxLines: number;
+}
+export interface EvidenceRangeSelection {
+    range: LineRange;
+    anchorLine: number;
+    matchKind: EvidenceRangeMatchKind;
+}
+export declare function selectEvidenceRangeForChunk(options: SelectEvidenceRangeForChunkOptions): EvidenceRangeSelection;

package/router/dist/evidence/evidence-range-selector.js

@@ -0,0 +1,231 @@
+export function selectEvidenceRangeForChunk(options) {
+    const exactRange = options.chunkHasExactPhrase ? exactPhraseRangeForChunk(options) : null;
+    const chunkLength = options.chunkRange.end - options.chunkRange.start + 1;
+    const anchorLineIndex = exactRange ? exactRange.start - 1 : bestLineIndexInChunk(options);
+    if (exactRange) {
+        const range = expandSectionRangeToNearbyFencedCodeBlocks(options, {
+            start: Math.max(options.chunkRange.start, exactRange.start - options.defaultContextLines),
+            end: Math.min(options.chunkRange.end, exactRange.end + options.defaultContextLines),
+        });
+        return {
+            range,
+            anchorLine: exactRange.start,
+            matchKind: "exact-phrase",
+        };
+    }
+    if (options.chunkKind === "symbol" || chunkLength <= options.defaultContextLines * 2 + 4) {
+        return {
+            range: options.chunkRange,
+            anchorLine: anchorLineIndex + 1,
+            matchKind: exactRange ? "exact-phrase" : "best-line",
+        };
+    }
+    const coverageRange = queryCoverageRangeForChunk(options);
+    if (coverageRange) {
+        return {
+            range: expandSectionRangeToNearbyFencedCodeBlocks(options, coverageRange),
+            anchorLine: coverageRange.start,
+            matchKind: "coverage",
+        };
+    }
+    const range = expandSectionRangeToNearbyFencedCodeBlocks(options, {
+        start: Math.max(options.chunkRange.start, anchorLineIndex + 1 - options.defaultContextLines),
+        end: Math.min(options.chunkRange.end, anchorLineIndex + 1 + options.defaultContextLines),
+    });
+    return {
+        range,
+        anchorLine: anchorLineIndex + 1,
+        matchKind: "best-line",
+    };
+}
+function exactPhraseRangeForChunk(options) {
+    if (options.normalizedQueryPhrase === "") {
+        return null;
+    }
+    for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
+        const line = options.lines[lineIndex] ?? "";
+        if (hasExactNormalizedPhrase(line, options.normalizedQueryPhrase)) {
+            return { start: lineIndex + 1, end: lineIndex + 1 };
+        }
+    }
+    for (let startIndex = options.chunkRange.start - 1; startIndex < options.chunkRange.end; startIndex += 1) {
+        let text = "";
+        for (let endIndex = startIndex; endIndex < options.chunkRange.end; endIndex += 1) {
+            const line = options.lines[endIndex] ?? "";
+            text = text === "" ? line : `${text}\n${line}`;
+            if (hasExactNormalizedPhrase(text, options.normalizedQueryPhrase)) {
+                return { start: startIndex + 1, end: endIndex + 1 };
+            }
+        }
+    }
+    return null;
+}
+function bestLineIndexInChunk(options) {
+    let bestIndex = options.chunkRange.start - 1;
+    let bestScore = 0;
+    for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
+        const line = options.lines[lineIndex] ?? "";
+        const score = scoreText(line, options.queryTokens) * 4 + (line.trimStart().startsWith("#") ? 2 : 0);
+        if (score > bestScore) {
+            bestIndex = lineIndex;
+            bestScore = score;
+        }
+    }
+    return bestIndex;
+}
+function queryCoverageRangeForChunk(options) {
+    if (options.chunkKind !== "section" || options.queryTokens.length < 2) {
+        return null;
+    }
+    const remainingTokens = new Set(options.queryTokens);
+    const matchingLines = [];
+    for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
+        const line = options.lines[lineIndex] ?? "";
+        const lineTokens = new Set(tokenizeForSearch(line));
+        let matchedLine = false;
+        for (const token of options.queryTokens) {
+            if (lineTokens.has(token)) {
+                remainingTokens.delete(token);
+                matchedLine = true;
+            }
+        }
+        if (matchedLine) {
+            matchingLines.push(lineIndex + 1);
+        }
+        if (remainingTokens.size === 0) {
+            break;
+        }
+    }
+    if (remainingTokens.size > 0 || matchingLines.length === 0) {
+        return null;
+    }
+    return boundedCoverageRange(options, matchingLines);
+}
+function boundedCoverageRange(options, matchingLines) {
+    const firstMatch = Math.min(...matchingLines);
+    const lastMatch = Math.max(...matchingLines);
+    const start = Math.max(options.chunkRange.start, firstMatch - options.defaultContextLines);
+    const end = Math.min(options.chunkRange.end, lastMatch + options.defaultContextLines);
+    if (end - start + 1 <= options.queryCoverageMaxLines) {
+        return { start, end };
+    }
+    const cappedStart = start;
+    const cappedEnd = Math.min(options.chunkRange.end, cappedStart + options.queryCoverageMaxLines - 1);
+    return { start: cappedStart, end: cappedEnd };
+}
+function expandSectionRangeToNearbyFencedCodeBlocks(options, range) {
+    if (options.chunkKind !== "section") {
+        return range;
+    }
+    let expanded = range;
+    for (const block of fencedCodeBlocksInChunk(options)) {
+        const adjacentOrOverlapping = block.start <= expanded.end + options.defaultContextLines && block.end >= expanded.start - options.defaultContextLines;
+        if (!adjacentOrOverlapping) {
+            continue;
+        }
+        const candidate = {
+            start: Math.min(expanded.start, block.start),
+            end: Math.max(expanded.end, block.end),
+        };
+        if (candidate.end - candidate.start + 1 <= options.queryCoverageMaxLines) {
+            expanded = candidate;
+        }
+    }
+    return expanded;
+}
+function fencedCodeBlocksInChunk(options) {
+    const blocks = [];
+    let openStart = null;
+    for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
+        const lineNumber = lineIndex + 1;
+        const line = options.lines[lineIndex] ?? "";
+        if (!/^\s*```/.test(line)) {
+            continue;
+        }
+        if (openStart === null) {
+            openStart = lineNumber;
+        }
+        else {
+            blocks.push({ start: openStart, end: lineNumber });
+            openStart = null;
+        }
+    }
+    if (openStart !== null) {
+        blocks.push({ start: openStart, end: options.chunkRange.end });
+    }
+    return blocks;
+}
+function hasExactNormalizedPhrase(text, normalizedQueryPhrase) {
+    return normalizedQueryPhrase !== "" && normalizePhraseSequence(text).includes(normalizedQueryPhrase);
+}
+function normalizePhraseSequence(text) {
+    const tokens = tokenizeForPhrase(text);
+    return tokens.length >= 2 ? tokens.join(" ") : "";
+}
+function tokenizeForSearch(text) {
+    return text
+        .split(/[^A-Za-z0-9_./-]+/)
+        .flatMap((token) => expandedIdentifierTokens(token.trim()))
+        .filter((token) => token.length >= 2);
+}
+function tokenizeForPhrase(text) {
+    return tokenizeForSearch(text);
+}
+function scoreText(text, tokens) {
+    const lower = text.toLowerCase();
+    return tokens.reduce((score, token) => score + countOccurrences(lower, token), 0);
+}
+function countOccurrences(text, token) {
+    let count = 0;
+    let index = text.indexOf(token);
+    while (index !== -1) {
+        count += 1;
+        index = text.indexOf(token, index + token.length);
+    }
+    return count;
+}
+function expandedIdentifierTokens(token) {
+    if (!token) {
+        return [];
+    }
+    const lower = token.toLowerCase();
+    if (!needsIdentifierSplit(token)) {
+        return [lower];
+    }
+    const variants = new Set();
+    addTokenVariant(variants, lower);
+    for (const part of splitIdentifierToken(token)) {
+        addTokenVariant(variants, part.toLowerCase());
+    }
+    return Array.from(variants);
+}
+function addTokenVariant(variants, token) {
+    if (token) {
+        variants.add(token);
+    }
+}
+function needsIdentifierSplit(token) {
+    if (/[._/-]/.test(token)) {
+        return true;
+    }
+    for (let index = 1; index < token.length; index += 1) {
+        const previous = token.charCodeAt(index - 1);
+        const current = token.charCodeAt(index);
+        const next = index + 1 < token.length ? token.charCodeAt(index + 1) : 0;
+        const previousIsLowerOrDigit = (previous >= 97 && previous <= 122) || (previous >= 48 && previous <= 57);
+        const previousIsUpper = previous >= 65 && previous <= 90;
+        const currentIsUpper = current >= 65 && current <= 90;
+        const nextIsLower = next >= 97 && next <= 122;
+        if ((previousIsLowerOrDigit && currentIsUpper) || (previousIsUpper && currentIsUpper && nextIsLower)) {
+            return true;
+        }
+    }
+    return false;
+}
+function splitIdentifierToken(token) {
+    return token
+        .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
+        .split(/[._/\-\s]+/)
+        .filter((part) => part.length > 0);
+}

package/router/dist/evidence/evidence-search.d.ts

@@ -0,0 +1,25 @@
+export interface EvidenceSearchLineRange {
+    start: number;
+    end: number;
+}
+export interface EvidenceSearchFile {
+    path: string;
+    lines: readonly string[];
+}
+export interface EvidenceSearchCandidate<TFile extends EvidenceSearchFile = EvidenceSearchFile> {
+    file: TFile;
+    lineIndex: number;
+    score: number;
+    reason: string;
+    range?: EvidenceSearchLineRange;
+    exactNormalizedPhrase?: string;
+}
+export interface SearchRepoEvidenceCandidatesOptions<TFile extends EvidenceSearchFile = EvidenceSearchFile> {
+    files: readonly TFile[];
+    query: string;
+    topK: number;
+    defaultContextLines?: number;
+    queryCoverageMaxLines?: number;
+}
+export declare function searchRepoEvidenceCandidates<TFile extends EvidenceSearchFile>(options: SearchRepoEvidenceCandidatesOptions<TFile>): EvidenceSearchCandidate<TFile>[];
+export declare function findBestLineForQuery(lines: readonly string[], query: string): number | null;