@hassangameryt/freeflow 0.4.0 → 0.6.0

package/router/dist/evidence/bounded-evidence.js

@@ -1,323 +0,0 @@
-const TRUNCATION_SUFFIX = " … [truncated; expand or retrieve exact lines for recovery]";
-const TRUNCATION_PREFIX = "[truncated head; expand or retrieve exact lines for recovery] … ";
-export function buildBoundedEdgeChunks(options) {
-  return overCapEdgeRanges(options.lines, options.range, options.caps).map((range) => {
-    const excerpt =
-      range.edge === "tail"
-        ? truncateTailToUtf8Bytes(linesForRange(options.lines, range), options.caps.exactChunkMaxBytes)
-        : truncateToUtf8Bytes(linesForRange(options.lines, range), options.caps.exactChunkMaxBytes);
-    return {
-      range: { start: range.start, end: range.end },
-      linesLabel: `${range.start}-${range.end}`,
-      edge: range.edge,
-      excerpt,
-    };
-  });
-}
-export function buildBoundedExcerpt(options) {
-  const capped = capLineRangeForWindow(options.range, options.window, options.caps);
-  const selected = options.lines.slice(capped.range.start - 1, capped.range.end);
-  const maxBytes = maxExcerptBytesForWindow(options.window, options.caps);
-  if (maxBytes !== null && options.exactNormalizedPhrase !== undefined) {
-    const exactExcerpt = excerptAroundNormalizedPhrase(selected.join("\n"), options.exactNormalizedPhrase, maxBytes);
-    if (exactExcerpt !== null) {
-      return {
-        range: capped.range,
-        linesLabel: `${capped.range.start}-${capped.range.end}`,
-        excerpt: exactExcerpt,
-        truncatedByLineCap: capped.truncated,
-        truncatedByByteCap: byteLength(selected.join("\n")) > maxBytes,
-      };
-    }
-  }
-  const previewedLines =
-    maxBytes !== null
-      ? selected.map((line) =>
-          truncateLinePreview(line, options.caps.linePreviewMaxBytes, options.exactNormalizedPhrase),
-        )
-      : selected;
-  const uncappedExcerpt = previewedLines.join("\n");
-  const excerpt = maxBytes === null ? uncappedExcerpt : truncateToUtf8Bytes(uncappedExcerpt, maxBytes);
-  return {
-    range: capped.range,
-    linesLabel: `${capped.range.start}-${capped.range.end}`,
-    excerpt,
-    truncatedByLineCap: capped.truncated,
-    truncatedByByteCap: maxBytes !== null && byteLength(uncappedExcerpt) > maxBytes,
-  };
-}
-function overCapEdgeRanges(lines, range, caps) {
-  const lineCount = range.end - range.start + 1;
-  const chunkLineCount = Math.min(10, Math.max(1, Math.floor(lineCount / 2)));
-  const headRange = shrinkRangeToMaxBytes(
-    lines,
-    { start: range.start, end: Math.min(range.end, range.start + chunkLineCount - 1) },
-    "head",
-    caps,
-  );
-  const tailRange = shrinkRangeToMaxBytes(
-    lines,
-    { start: Math.max(range.start, range.end - chunkLineCount + 1), end: range.end },
-    "tail",
-    caps,
-  );
-  const head = { ...headRange, edge: "head" };
-  const tail = { ...tailRange, edge: "tail" };
-  if (tail.start <= head.end) {
-    return byteLength(linesForRange(lines, head)) > caps.exactChunkMaxBytes ? [head, tail] : [head];
-  }
-  return [head, tail];
-}
-function shrinkRangeToMaxBytes(lines, range, edge, caps) {
-  let current = range;
-  while (current.end > current.start && byteLength(linesForRange(lines, current)) > caps.exactChunkMaxBytes) {
-    current =
-      edge === "head" ? { start: current.start, end: current.end - 1 } : { start: current.start + 1, end: current.end };
-  }
-  return current;
-}
-function linesForRange(lines, range) {
-  return lines.slice(range.start - 1, range.end).join("\n");
-}
-function capLineRangeForWindow(range, window, caps) {
-  const maxLines = maxLinesForWindow(window, caps);
-  const lineCount = range.end - range.start + 1;
-  if (maxLines === null || lineCount <= maxLines) {
-    return { range, truncated: false };
-  }
-  return {
-    range: { start: range.start, end: range.start + maxLines - 1 },
-    truncated: true,
-  };
-}
-function maxLinesForWindow(window, caps) {
-  if (window === "lines_30") {
-    return caps.expandLines30MaxLines;
-  }
-  if (window === "lines_80") {
-    return caps.expandLines80MaxLines;
-  }
-  return null;
-}
-function maxExcerptBytesForWindow(window, caps) {
-  if (window === "small" || window === "section") {
-    return caps.queryExcerptMaxBytes;
-  }
-  if (window === "lines_30") {
-    return caps.expandLines30MaxBytes;
-  }
-  if (window === "lines_80") {
-    return caps.expandLines80MaxBytes;
-  }
-  return null;
-}
-function excerptAroundNormalizedPhrase(text, exactNormalizedPhrase, maxBytes) {
-  const span = normalizedPhraseRawSpan(text, exactNormalizedPhrase);
-  if (span === null) {
-    return null;
-  }
-  return truncateToUtf8BytesAroundSpan(text, span, maxBytes);
-}
-function truncateLinePreview(text, maxBytes, exactNormalizedPhrase) {
-  if (exactNormalizedPhrase === undefined || byteLength(text) <= maxBytes) {
-    return truncateToUtf8Bytes(text, maxBytes);
-  }
-  const span = normalizedPhraseRawSpan(text, exactNormalizedPhrase);
-  if (span === null) {
-    return truncateToUtf8Bytes(text, maxBytes);
-  }
-  return truncateToUtf8BytesAroundSpan(text, span, maxBytes);
-}
-function truncateToUtf8Bytes(text, maxBytes) {
-  if (byteLength(text) <= maxBytes) {
-    return text;
-  }
-  const suffixBytes = byteLength(TRUNCATION_SUFFIX);
-  const contentBytes = Math.max(0, maxBytes - suffixBytes);
-  return `${truncateHeadToUtf8Bytes(text, contentBytes)}${TRUNCATION_SUFFIX}`;
-}
-function truncateToUtf8BytesAroundSpan(text, span, maxBytes) {
-  if (byteLength(text) <= maxBytes) {
-    return text;
-  }
-  const prefix = span.start > 0 ? TRUNCATION_PREFIX : "";
-  const suffix = span.end < text.length ? TRUNCATION_SUFFIX : "";
-  const budget = maxBytes - byteLength(prefix) - byteLength(suffix);
-  const phrase = text.slice(span.start, span.end);
-  const phraseBytes = byteLength(phrase);
-  if (budget <= 0 || phraseBytes >= budget) {
-    return truncateToUtf8Bytes(text.slice(span.start), maxBytes);
-  }
-  const contextBudget = budget - phraseBytes;
-  const beforeBudget = Math.floor(contextBudget / 2);
-  const afterBudget = contextBudget - beforeBudget;
-  let start = span.start;
-  while (start > 0 && byteLength(text.slice(start - 1, span.start)) <= beforeBudget) {
-    start -= 1;
-  }
-  if (byteLength(text.slice(start, span.start)) > beforeBudget) {
-    start += 1;
-  }
-  let end = span.end;
-  while (end < text.length && byteLength(text.slice(span.end, end + 1)) <= afterBudget) {
-    end += 1;
-  }
-  if (byteLength(text.slice(span.end, end)) > afterBudget) {
-    end -= 1;
-  }
-  start = moveToCodePointStartBoundary(text, start);
-  end = moveToCodePointEndBoundary(text, end);
-  const actualPrefix = start > 0 ? prefix : "";
-  const actualSuffix = end < text.length ? suffix : "";
-  return `${actualPrefix}${text.slice(start, end)}${actualSuffix}`;
-}
-function normalizedPhraseRawSpan(text, normalizedPhrase) {
-  const phraseTokens = normalizedPhrase.split(/\s+/).filter(Boolean);
-  if (phraseTokens.length === 0) {
-    return null;
-  }
-  const spans = tokenSpansForPhrase(text);
-  for (let startIndex = 0; startIndex <= spans.length - phraseTokens.length; startIndex += 1) {
-    let matched = true;
-    for (let offset = 0; offset < phraseTokens.length; offset += 1) {
-      if (spans[startIndex + offset]?.token !== phraseTokens[offset]) {
-        matched = false;
-        break;
-      }
-    }
-    if (matched) {
-      const first = spans[startIndex];
-      const last = spans[startIndex + phraseTokens.length - 1];
-      if (first !== undefined && last !== undefined) {
-        return { start: first.start, end: last.end };
-      }
-    }
-  }
-  return null;
-}
-function tokenSpansForPhrase(text) {
-  const spans = [];
-  for (const match of text.matchAll(/[A-Za-z0-9_./-]+/g)) {
-    const rawToken = match[0];
-    const start = match.index ?? 0;
-    const end = start + rawToken.length;
-    for (const token of expandedIdentifierTokens(rawToken)) {
-      if (token.length >= 2) {
-        spans.push({ token, start, end });
-      }
-    }
-  }
-  return spans;
-}
-function expandedIdentifierTokens(token) {
-  if (!token) {
-    return [];
-  }
-  const lower = token.toLowerCase();
-  if (!needsIdentifierSplit(token)) {
-    return [lower];
-  }
-  const variants = new Set();
-  addTokenVariant(variants, lower);
-  for (const part of splitIdentifierToken(token)) {
-    addTokenVariant(variants, part.toLowerCase());
-  }
-  return Array.from(variants);
-}
-function addTokenVariant(variants, token) {
-  if (token) {
-    variants.add(token);
-  }
-}
-function needsIdentifierSplit(token) {
-  if (/[._/-]/.test(token)) {
-    return true;
-  }
-  for (let index = 1; index < token.length; index += 1) {
-    const previous = token.charCodeAt(index - 1);
-    const current = token.charCodeAt(index);
-    const next = index + 1 < token.length ? token.charCodeAt(index + 1) : 0;
-    const previousIsLowerOrDigit = (previous >= 97 && previous <= 122) || (previous >= 48 && previous <= 57);
-    const previousIsUpper = previous >= 65 && previous <= 90;
-    const currentIsUpper = current >= 65 && current <= 90;
-    const nextIsLower = next >= 97 && next <= 122;
-    if ((previousIsLowerOrDigit && currentIsUpper) || (previousIsUpper && currentIsUpper && nextIsLower)) {
-      return true;
-    }
-  }
-  return false;
-}
-function splitIdentifierToken(token) {
-  const separated = token
-    .replace(/[._/-]+/g, " ")
-    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
-    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
-  return separated.split(/\s+/).filter(Boolean);
-}
-function truncateTailToUtf8Bytes(text, maxBytes) {
-  if (byteLength(text) <= maxBytes) {
-    return text;
-  }
-  const prefixBytes = byteLength(TRUNCATION_PREFIX);
-  const contentBytes = Math.max(0, maxBytes - prefixBytes);
-  return `${TRUNCATION_PREFIX}${truncateTailToUtf8BytesContent(text, contentBytes)}`;
-}
-function truncateHeadToUtf8Bytes(text, maxBytes) {
-  if (maxBytes <= 0) {
-    return "";
-  }
-  if (byteLength(text) <= maxBytes) {
-    return text;
-  }
-  let low = 0;
-  let high = text.length;
-  while (low < high) {
-    const mid = Math.ceil((low + high) / 2);
-    if (byteLength(text.slice(0, mid)) <= maxBytes) {
-      low = mid;
-    } else {
-      high = mid - 1;
-    }
-  }
-  return text.slice(0, moveToCodePointEndBoundary(text, low));
-}
-function truncateTailToUtf8BytesContent(text, maxBytes) {
-  if (maxBytes <= 0) {
-    return "";
-  }
-  if (byteLength(text) <= maxBytes) {
-    return text;
-  }
-  let low = 0;
-  let high = text.length;
-  while (low < high) {
-    const mid = Math.floor((low + high) / 2);
-    if (byteLength(text.slice(mid)) <= maxBytes) {
-      high = mid;
-    } else {
-      low = mid + 1;
-    }
-  }
-  return text.slice(moveToCodePointStartBoundary(text, low));
-}
-function moveToCodePointStartBoundary(text, index) {
-  if (index > 0 && index < text.length && isLowSurrogate(text.charCodeAt(index))) {
-    return index + 1;
-  }
-  return index;
-}
-function moveToCodePointEndBoundary(text, index) {
-  if (index > 0 && isHighSurrogate(text.charCodeAt(index - 1))) {
-    return index - 1;
-  }
-  return index;
-}
-function isHighSurrogate(code) {
-  return code >= 0xd800 && code <= 0xdbff;
-}
-function isLowSurrogate(code) {
-  return code >= 0xdc00 && code <= 0xdfff;
-}
-function byteLength(text) {
-  return Buffer.byteLength(text, "utf8");
-}

package/router/dist/evidence/evidence-range-selector.d.ts

@@ -1,24 +0,0 @@
-export interface LineRange {
-  start: number;
-  end: number;
-}
-export type EvidenceChunkKind = "section" | "symbol" | "window";
-export type EvidenceRangeMatchKind = "exact-phrase" | "coverage" | "best-line";
-export interface SelectEvidenceRangeForChunkOptions {
-  lines: readonly string[];
-  chunkRange: LineRange;
-  chunkKind: EvidenceChunkKind;
-  queryTokens: readonly string[];
-  normalizedQueryPhrase: string;
-  chunkHasExactPhrase: boolean;
-  defaultContextLines: number;
-  queryCoverageMaxLines: number;
-}
-export interface EvidenceRangeSelection {
-  range: LineRange;
-  anchorLine: number;
-  matchKind: EvidenceRangeMatchKind;
-}
-export declare function selectEvidenceRangeForChunk(
-  options: SelectEvidenceRangeForChunkOptions,
-): EvidenceRangeSelection;

package/router/dist/evidence/evidence-range-selector.js

@@ -1,232 +0,0 @@
-export function selectEvidenceRangeForChunk(options) {
-  const exactRange = options.chunkHasExactPhrase ? exactPhraseRangeForChunk(options) : null;
-  const chunkLength = options.chunkRange.end - options.chunkRange.start + 1;
-  const anchorLineIndex = exactRange ? exactRange.start - 1 : bestLineIndexInChunk(options);
-  if (exactRange) {
-    const range = expandSectionRangeToNearbyFencedCodeBlocks(options, {
-      start: Math.max(options.chunkRange.start, exactRange.start - options.defaultContextLines),
-      end: Math.min(options.chunkRange.end, exactRange.end + options.defaultContextLines),
-    });
-    return {
-      range,
-      anchorLine: exactRange.start,
-      matchKind: "exact-phrase",
-    };
-  }
-  if (options.chunkKind === "symbol" || chunkLength <= options.defaultContextLines * 2 + 4) {
-    return {
-      range: options.chunkRange,
-      anchorLine: anchorLineIndex + 1,
-      matchKind: exactRange ? "exact-phrase" : "best-line",
-    };
-  }
-  const coverageRange = queryCoverageRangeForChunk(options);
-  if (coverageRange) {
-    return {
-      range: expandSectionRangeToNearbyFencedCodeBlocks(options, coverageRange),
-      anchorLine: coverageRange.start,
-      matchKind: "coverage",
-    };
-  }
-  const range = expandSectionRangeToNearbyFencedCodeBlocks(options, {
-    start: Math.max(options.chunkRange.start, anchorLineIndex + 1 - options.defaultContextLines),
-    end: Math.min(options.chunkRange.end, anchorLineIndex + 1 + options.defaultContextLines),
-  });
-  return {
-    range,
-    anchorLine: anchorLineIndex + 1,
-    matchKind: "best-line",
-  };
-}
-function exactPhraseRangeForChunk(options) {
-  if (options.normalizedQueryPhrase === "") {
-    return null;
-  }
-  for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
-    const line = options.lines[lineIndex] ?? "";
-    if (hasExactNormalizedPhrase(line, options.normalizedQueryPhrase)) {
-      return { start: lineIndex + 1, end: lineIndex + 1 };
-    }
-  }
-  for (let startIndex = options.chunkRange.start - 1; startIndex < options.chunkRange.end; startIndex += 1) {
-    let text = "";
-    for (let endIndex = startIndex; endIndex < options.chunkRange.end; endIndex += 1) {
-      const line = options.lines[endIndex] ?? "";
-      text = text === "" ? line : `${text}\n${line}`;
-      if (hasExactNormalizedPhrase(text, options.normalizedQueryPhrase)) {
-        return { start: startIndex + 1, end: endIndex + 1 };
-      }
-    }
-  }
-  return null;
-}
-function bestLineIndexInChunk(options) {
-  let bestIndex = options.chunkRange.start - 1;
-  let bestScore = 0;
-  for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
-    const line = options.lines[lineIndex] ?? "";
-    const score = scoreText(line, options.queryTokens) * 4 + (line.trimStart().startsWith("#") ? 2 : 0);
-    if (score > bestScore) {
-      bestIndex = lineIndex;
-      bestScore = score;
-    }
-  }
-  return bestIndex;
-}
-function queryCoverageRangeForChunk(options) {
-  if (options.chunkKind !== "section" || options.queryTokens.length < 2) {
-    return null;
-  }
-  const remainingTokens = new Set(options.queryTokens);
-  const matchingLines = [];
-  for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
-    const line = options.lines[lineIndex] ?? "";
-    const lineTokens = new Set(tokenizeForSearch(line));
-    let matchedLine = false;
-    for (const token of options.queryTokens) {
-      if (lineTokens.has(token)) {
-        remainingTokens.delete(token);
-        matchedLine = true;
-      }
-    }
-    if (matchedLine) {
-      matchingLines.push(lineIndex + 1);
-    }
-    if (remainingTokens.size === 0) {
-      break;
-    }
-  }
-  if (remainingTokens.size > 0 || matchingLines.length === 0) {
-    return null;
-  }
-  return boundedCoverageRange(options, matchingLines);
-}
-function boundedCoverageRange(options, matchingLines) {
-  const firstMatch = Math.min(...matchingLines);
-  const lastMatch = Math.max(...matchingLines);
-  const start = Math.max(options.chunkRange.start, firstMatch - options.defaultContextLines);
-  const end = Math.min(options.chunkRange.end, lastMatch + options.defaultContextLines);
-  if (end - start + 1 <= options.queryCoverageMaxLines) {
-    return { start, end };
-  }
-  const cappedStart = start;
-  const cappedEnd = Math.min(options.chunkRange.end, cappedStart + options.queryCoverageMaxLines - 1);
-  return { start: cappedStart, end: cappedEnd };
-}
-function expandSectionRangeToNearbyFencedCodeBlocks(options, range) {
-  if (options.chunkKind !== "section") {
-    return range;
-  }
-  let expanded = range;
-  for (const block of fencedCodeBlocksInChunk(options)) {
-    const adjacentOrOverlapping =
-      block.start <= expanded.end + options.defaultContextLines &&
-      block.end >= expanded.start - options.defaultContextLines;
-    if (!adjacentOrOverlapping) {
-      continue;
-    }
-    const candidate = {
-      start: Math.min(expanded.start, block.start),
-      end: Math.max(expanded.end, block.end),
-    };
-    if (candidate.end - candidate.start + 1 <= options.queryCoverageMaxLines) {
-      expanded = candidate;
-    }
-  }
-  return expanded;
-}
-function fencedCodeBlocksInChunk(options) {
-  const blocks = [];
-  let openStart = null;
-  for (let lineIndex = options.chunkRange.start - 1; lineIndex < options.chunkRange.end; lineIndex += 1) {
-    const lineNumber = lineIndex + 1;
-    const line = options.lines[lineIndex] ?? "";
-    if (!/^\s*```/.test(line)) {
-      continue;
-    }
-    if (openStart === null) {
-      openStart = lineNumber;
-    } else {
-      blocks.push({ start: openStart, end: lineNumber });
-      openStart = null;
-    }
-  }
-  if (openStart !== null) {
-    blocks.push({ start: openStart, end: options.chunkRange.end });
-  }
-  return blocks;
-}
-function hasExactNormalizedPhrase(text, normalizedQueryPhrase) {
-  return normalizedQueryPhrase !== "" && normalizePhraseSequence(text).includes(normalizedQueryPhrase);
-}
-function normalizePhraseSequence(text) {
-  const tokens = tokenizeForPhrase(text);
-  return tokens.length >= 2 ? tokens.join(" ") : "";
-}
-function tokenizeForSearch(text) {
-  return text
-    .split(/[^A-Za-z0-9_./-]+/)
-    .flatMap((token) => expandedIdentifierTokens(token.trim()))
-    .filter((token) => token.length >= 2);
-}
-function tokenizeForPhrase(text) {
-  return tokenizeForSearch(text);
-}
-function scoreText(text, tokens) {
-  const lower = text.toLowerCase();
-  return tokens.reduce((score, token) => score + countOccurrences(lower, token), 0);
-}
-function countOccurrences(text, token) {
-  let count = 0;
-  let index = text.indexOf(token);
-  while (index !== -1) {
-    count += 1;
-    index = text.indexOf(token, index + token.length);
-  }
-  return count;
-}
-function expandedIdentifierTokens(token) {
-  if (!token) {
-    return [];
-  }
-  const lower = token.toLowerCase();
-  if (!needsIdentifierSplit(token)) {
-    return [lower];
-  }
-  const variants = new Set();
-  addTokenVariant(variants, lower);
-  for (const part of splitIdentifierToken(token)) {
-    addTokenVariant(variants, part.toLowerCase());
-  }
-  return Array.from(variants);
-}
-function addTokenVariant(variants, token) {
-  if (token) {
-    variants.add(token);
-  }
-}
-function needsIdentifierSplit(token) {
-  if (/[._/-]/.test(token)) {
-    return true;
-  }
-  for (let index = 1; index < token.length; index += 1) {
-    const previous = token.charCodeAt(index - 1);
-    const current = token.charCodeAt(index);
-    const next = index + 1 < token.length ? token.charCodeAt(index + 1) : 0;
-    const previousIsLowerOrDigit = (previous >= 97 && previous <= 122) || (previous >= 48 && previous <= 57);
-    const previousIsUpper = previous >= 65 && previous <= 90;
-    const currentIsUpper = current >= 65 && current <= 90;
-    const nextIsLower = next >= 97 && next <= 122;
-    if ((previousIsLowerOrDigit && currentIsUpper) || (previousIsUpper && currentIsUpper && nextIsLower)) {
-      return true;
-    }
-  }
-  return false;
-}
-function splitIdentifierToken(token) {
-  return token
-    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
-    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
-    .split(/[._/\-\s]+/)
-    .filter((part) => part.length > 0);
-}

package/router/dist/evidence/evidence-search.d.ts

@@ -1,27 +0,0 @@
-export interface EvidenceSearchLineRange {
-  start: number;
-  end: number;
-}
-export interface EvidenceSearchFile {
-  path: string;
-  lines: readonly string[];
-}
-export interface EvidenceSearchCandidate<TFile extends EvidenceSearchFile = EvidenceSearchFile> {
-  file: TFile;
-  lineIndex: number;
-  score: number;
-  reason: string;
-  range?: EvidenceSearchLineRange;
-  exactNormalizedPhrase?: string;
-}
-export interface SearchRepoEvidenceCandidatesOptions<TFile extends EvidenceSearchFile = EvidenceSearchFile> {
-  files: readonly TFile[];
-  query: string;
-  topK: number;
-  defaultContextLines?: number;
-  queryCoverageMaxLines?: number;
-}
-export declare function searchRepoEvidenceCandidates<TFile extends EvidenceSearchFile>(
-  options: SearchRepoEvidenceCandidatesOptions<TFile>,
-): EvidenceSearchCandidate<TFile>[];
-export declare function findBestLineForQuery(lines: readonly string[], query: string): number | null;