@hassangameryt/freeflow 0.4.0 → 0.6.0

package/router/dist/evidence/evidence-search.js

@@ -1,490 +0,0 @@
-import { selectEvidenceRangeForChunk } from "./evidence-range-selector.js";
-const DEFAULT_CONTEXT_LINES = 2;
-const QUERY_COVERAGE_MAX_LINES = 80;
-const STOPWORDS = new Set([
-  "a",
-  "an",
-  "and",
-  "are",
-  "as",
-  "at",
-  "be",
-  "by",
-  "for",
-  "from",
-  "has",
-  "in",
-  "is",
-  "it",
-  "of",
-  "on",
-  "or",
-  "that",
-  "the",
-  "this",
-  "to",
-  "with",
-  "use",
-]);
-export function searchRepoEvidenceCandidates(options) {
-  const tokens = uniqueTokens(tokenize(options.query));
-  const normalizedQueryPhrase = normalizePhraseSequence(options.query);
-  const chunks = options.files.flatMap((file) => candidateChunksForFile(file));
-  const matchingChunks = chunks.filter((chunk) => chunkMightMatch(chunk, tokens));
-  const stats = chunkStats(matchingChunks);
-  const scored = [];
-  for (const chunk of matchingChunks) {
-    const candidate = scoreCandidateChunk(chunk, tokens, normalizedQueryPhrase, stats, {
-      defaultContextLines: options.defaultContextLines ?? DEFAULT_CONTEXT_LINES,
-      queryCoverageMaxLines: options.queryCoverageMaxLines ?? QUERY_COVERAGE_MAX_LINES,
-    });
-    if (candidate) {
-      scored.push(candidate);
-    }
-  }
-  scored.sort((a, b) => b.score - a.score);
-  const selected = [];
-  const seen = new Set();
-  for (const candidate of scored) {
-    const key = candidate.file.path;
-    if (seen.has(key)) {
-      continue;
-    }
-    seen.add(key);
-    selected.push(candidate);
-    if (selected.length >= options.topK) {
-      break;
-    }
-  }
-  return selected;
-}
-export function findBestLineForQuery(lines, query) {
-  const tokens = tokenize(query);
-  let bestLineIndex = null;
-  let bestScore = 0;
-  lines.forEach((line, lineIndex) => {
-    const score = scoreText(line, tokens);
-    if (score > bestScore) {
-      bestLineIndex = lineIndex;
-      bestScore = score;
-    }
-  });
-  return bestLineIndex;
-}
-function candidateChunksForFile(file) {
-  const structuralChunks = [];
-  structuralChunks.push(...markdownPreambleChunk(file));
-  structuralChunks.push(...markdownSectionChunks(file));
-  structuralChunks.push(...codeSymbolChunks(file));
-  return structuralChunks.length ? structuralChunks : lineWindowChunks(file);
-}
-function markdownPreambleChunk(file) {
-  const firstHeadingIndex = file.lines.findIndex((line) => line.trimStart().startsWith("#"));
-  if (firstHeadingIndex <= 0) {
-    return [];
-  }
-  const range = { start: 1, end: firstHeadingIndex };
-  return [
-    {
-      file,
-      range,
-      text: file.lines.slice(range.start - 1, range.end).join("\n"),
-      heading: "",
-      kind: "section",
-    },
-  ];
-}
-function markdownSectionChunks(file) {
-  const headingIndexes = file.lines
-    .map((line, index) => ({ line, index }))
-    .filter(({ line }) => line.trimStart().startsWith("#"));
-  if (!headingIndexes.length) {
-    return [];
-  }
-  return headingIndexes.map(({ line, index }, headingOrdinal) => {
-    const nextHeading = headingIndexes[headingOrdinal + 1]?.index ?? file.lines.length;
-    const range = { start: index + 1, end: Math.max(index + 1, nextHeading) };
-    return {
-      file,
-      range,
-      text: file.lines.slice(range.start - 1, range.end).join("\n"),
-      heading: line,
-      kind: "section",
-    };
-  });
-}
-function codeSymbolChunks(file) {
-  const symbolIndexes = file.lines
-    .map((line, index) => ({ line, index, match: codeSymbolMatch(line) }))
-    .filter((entry) => Boolean(entry.match));
-  if (!symbolIndexes.length) {
-    return [];
-  }
-  return symbolIndexes.map(({ line, index }, symbolOrdinal) => {
-    const nextSymbol = nextSymbolBoundary(file.lines, symbolIndexes, symbolOrdinal);
-    const end = Math.min(file.lines.length, Math.max(index + 1, Math.min(nextSymbol, index + 80)));
-    const range = { start: index + 1, end };
-    return {
-      file,
-      range,
-      text: file.lines.slice(range.start - 1, range.end).join("\n"),
-      heading: line,
-      kind: "symbol",
-    };
-  });
-}
-function nextSymbolBoundary(lines, symbolIndexes, symbolOrdinal) {
-  const current = symbolIndexes[symbolOrdinal];
-  if (!current) {
-    return lines.length;
-  }
-  if (/^\s*impl\b/.test(current.line)) {
-    const nextTopLevel = symbolIndexes.slice(symbolOrdinal + 1).find((entry) => isTopLevelCodeSymbol(entry.line));
-    return nextTopLevel?.index ?? lines.length;
-  }
-  return symbolIndexes[symbolOrdinal + 1]?.index ?? lines.length;
-}
-function isTopLevelCodeSymbol(line) {
-  return /^(?:pub\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|class|interface|type|const|def)\s+([A-Za-z_][A-Za-z0-9_]*)\b/.test(
-    line,
-  );
-}
-function codeSymbolMatch(line) {
-  return /^\s*(?:pub\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|class|interface|type|const|def)\s+([A-Za-z_][A-Za-z0-9_]*)\b/.exec(
-    line,
-  );
-}
-function lineWindowChunks(file) {
-  return file.lines.map((_, lineIndex) => {
-    const start = Math.max(0, lineIndex - DEFAULT_CONTEXT_LINES);
-    const end = Math.min(file.lines.length - 1, lineIndex + DEFAULT_CONTEXT_LINES);
-    return {
-      file,
-      range: { start: start + 1, end: end + 1 },
-      text: file.lines.slice(start, end + 1).join("\n"),
-      kind: "window",
-    };
-  });
-}
-function chunkStats(chunks) {
-  const documentFrequency = new Map();
-  let totalLength = 0;
-  for (const chunk of chunks) {
-    const tokens = tokensForChunk(chunk);
-    totalLength += Math.max(1, tokens.length);
-    for (const token of uniqueTokens(tokens)) {
-      documentFrequency.set(token, (documentFrequency.get(token) ?? 0) + 1);
-    }
-  }
-  return {
-    averageLength: chunks.length ? totalLength / chunks.length : 1,
-    documentFrequency,
-    chunkCount: chunks.length,
-  };
-}
-function scoreCandidateChunk(chunk, queryTokens, normalizedQueryPhrase, stats, options) {
-  if (!queryTokens.length) {
-    return null;
-  }
-  const chunkTokens = tokensForChunk(chunk);
-  const tokenCounts = tokenCountMap(chunkTokens);
-  const matchingTokens = queryTokens.filter((token) => (tokenCounts.get(token) ?? 0) > 0);
-  const pathScore = scoreText(chunk.file.path, queryTokens);
-  if (!matchingTokens.length && pathScore <= 0) {
-    return null;
-  }
-  const exactPhrase = hasExactNormalizedPhraseInChunk(chunk, normalizedQueryPhrase);
-  const chunkLength = Math.max(1, chunkTokens.length);
-  const score =
-    exactPhraseBoost(exactPhrase, chunk.file.path, queryTokens) +
-    bm25Score(queryTokens, tokenCounts, chunkLength, stats) * 10 +
-    coverageRatio(matchingTokens, queryTokens) * 120 +
-    contentCoverageBoost(matchingTokens, queryTokens, chunk.kind) +
-    completeCoverageBoost(matchingTokens, queryTokens, chunk.kind) -
-    missingCoveragePenalty(matchingTokens, queryTokens, chunk.kind) -
-    lowContentCoveragePenalty(matchingTokens, queryTokens, pathScore) +
-    headingCoverageBoost(chunk.heading ?? "", queryTokens) +
-    identifierBoost(chunk.text, queryTokens) +
-    (matchingTokens.length >= 4 ? orderedPhraseBoost(chunkTokens, queryTokens) : 0) +
-    (chunk.kind === "symbol" ? codeDefinitionBoost(chunk.text, queryTokens) : 0) +
-    chunkKindBoost(chunk.kind) +
-    pathIntentBoost(chunk.file.path, queryTokens) +
-    sourceTestPrior(chunk.file.path, queryTokens) +
-    pathScore -
-    Math.log1p(chunkLength) * 10;
-  const selection = selectEvidenceRangeForChunk({
-    lines: chunk.file.lines,
-    chunkRange: chunk.range,
-    chunkKind: chunk.kind,
-    queryTokens,
-    normalizedQueryPhrase,
-    chunkHasExactPhrase: exactPhrase,
-    defaultContextLines: options.defaultContextLines,
-    queryCoverageMaxLines: options.queryCoverageMaxLines,
-  });
-  const reason =
-    selection.matchKind === "exact-phrase"
-      ? `matched exact normalized query phrase in ${chunk.kind} chunk`
-      : `BM25-style scored ${chunk.kind} chunk with ${matchingTokens.length}/${queryTokens.length} query-token coverage`;
-  return {
-    file: chunk.file,
-    lineIndex: selection.anchorLine - 1,
-    range: selection.range,
-    score,
-    reason,
-    ...(selection.matchKind === "exact-phrase" ? { exactNormalizedPhrase: normalizedQueryPhrase } : {}),
-  };
-}
-function chunkMightMatch(chunk, queryTokens) {
-  const path = chunk.file.path.toLowerCase();
-  const heading = chunk.heading?.toLowerCase() ?? "";
-  const text = chunk.text.toLowerCase();
-  return queryTokens.some((token) => path.includes(token) || heading.includes(token) || text.includes(token));
-}
-function bm25Score(queryTokens, tokenCounts, chunkLength, stats) {
-  const k1 = 1.2;
-  const b = 0.75;
-  return queryTokens.reduce((score, token) => {
-    const termFrequency = tokenCounts.get(token) ?? 0;
-    if (termFrequency <= 0) {
-      return score;
-    }
-    const documentFrequency = stats.documentFrequency.get(token) ?? 0;
-    const idf = Math.log(1 + (stats.chunkCount - documentFrequency + 0.5) / (documentFrequency + 0.5));
-    const normalizedLength = k1 * (1 - b + b * (chunkLength / Math.max(1, stats.averageLength)));
-    const saturatedFrequency = (termFrequency * (k1 + 1)) / (termFrequency + normalizedLength);
-    return score + idf * saturatedFrequency;
-  }, 0);
-}
-function coverageRatio(matchingTokens, queryTokens) {
-  return queryTokens.length ? matchingTokens.length / queryTokens.length : 0;
-}
-function contentCoverageBoost(matchingTokens, queryTokens, chunkKind) {
-  if (queryTokens.length === 0) {
-    return 0;
-  }
-  const perTokenBoost = chunkKind === "window" ? 180 : 350;
-  return matchingTokens.length * perTokenBoost + coverageRatio(matchingTokens, queryTokens) * 100;
-}
-function completeCoverageBoost(matchingTokens, queryTokens, chunkKind) {
-  if (queryTokens.length === 0 || matchingTokens.length !== queryTokens.length) {
-    return 0;
-  }
-  if (chunkKind === "symbol") {
-    return 240;
-  }
-  return queryTokens.length >= 4 ? 1_200 : 240;
-}
-function missingCoveragePenalty(matchingTokens, queryTokens, chunkKind) {
-  if (chunkKind === "symbol" || queryTokens.length < 4 || matchingTokens.length >= queryTokens.length) {
-    return 0;
-  }
-  return (queryTokens.length - matchingTokens.length) * 90;
-}
-function lowContentCoveragePenalty(matchingTokens, queryTokens, pathScore) {
-  if (queryTokens.length < 4 || matchingTokens.length >= 2) {
-    return 0;
-  }
-  const pathOnlyPenalty = matchingTokens.length === 0 && pathScore > 0 ? 1_000 : 0;
-  return (2 - matchingTokens.length) * 600 + pathOnlyPenalty;
-}
-function exactPhraseBoost(exactPhrase, path, queryTokens) {
-  if (!exactPhrase) {
-    return 0;
-  }
-  const testPath = isTestPath(path);
-  const testIntent = queryTokens.some((token) =>
-    ["test", "tests", "expect", "expected", "should", "emitted"].includes(token),
-  );
-  return testPath && !testIntent ? 1_000 : 100_000;
-}
-function headingCoverageBoost(heading, queryTokens) {
-  const headingTokens = new Set(tokenize(heading));
-  const coveredTokens = queryTokens.filter((token) => headingTokens.has(token)).length;
-  return coveredTokens * 8;
-}
-function identifierBoost(text, queryTokens) {
-  if (!text.includes("`")) {
-    return 0;
-  }
-  const identifiers = Array.from(text.matchAll(/`([^`]+)`/g), (match) => tokenize(match[1] ?? "")).flat();
-  if (!identifiers.length) {
-    return 0;
-  }
-  const identifierSet = new Set(identifiers);
-  return queryTokens.filter((token) => identifierSet.has(token)).length * 10;
-}
-function orderedPhraseBoost(chunkTokens, queryTokens) {
-  const normalizedText = ` ${chunkTokens.join(" ")} `;
-  let best = 0;
-  for (let start = 0; start < queryTokens.length; start += 1) {
-    const phrase = [];
-    for (let end = start; end < queryTokens.length; end += 1) {
-      phrase.push(queryTokens[end] ?? "");
-      if (phrase.length <= best) {
-        continue;
-      }
-      if (normalizedText.includes(` ${phrase.join(" ")} `)) {
-        best = phrase.length;
-      }
-    }
-  }
-  return best >= 3 ? best * best * 20 : 0;
-}
-function codeDefinitionBoost(text, queryTokens) {
-  const definitionMatches = Array.from(
-    text.matchAll(
-      /^\s*(?:pub\s+)?(?:async\s+)?(?:fn|struct|enum|trait|impl|mod|class|interface|type|const|def)\s+([A-Za-z_][A-Za-z0-9_]*)\b/gm,
-    ),
-    (match) => match[1] ?? "",
-  );
-  if (!definitionMatches.length) {
-    return 0;
-  }
-  let bestBoost = 0;
-  for (const name of definitionMatches) {
-    const symbolTokens = new Set(tokenize(name));
-    const coverage = queryTokens.filter((token) => symbolTokens.has(token)).length;
-    const exactCompoundCoverage = queryTokens.filter(
-      (token) => token.length >= 8 && symbolTokens.has(token) && !/[._/-]/.test(token),
-    ).length;
-    if (coverage > 0) {
-      bestBoost = Math.max(bestBoost, 120 + coverage * 80 + exactCompoundCoverage * 520);
-    }
-  }
-  return bestBoost;
-}
-function chunkKindBoost(kind) {
-  if (kind === "symbol") {
-    return 500;
-  }
-  if (kind === "section") {
-    return 12;
-  }
-  return 0;
-}
-function pathIntentBoost(path, queryTokens) {
-  const pathTokens = new Set(tokenize(path));
-  const coveredTokens = queryTokens.filter((token) => pathTokens.has(token)).length;
-  return coveredTokens * 140 + (coveredTokens >= 2 ? 600 : 0);
-}
-function sourceTestPrior(path, queryTokens) {
-  const testPath = isTestPath(path);
-  const testIntent = queryTokens.some((token) =>
-    ["test", "tests", "expect", "expected", "should", "emitted"].includes(token),
-  );
-  if (testPath && !testIntent) {
-    return -2_000;
-  }
-  if (testPath && testIntent) {
-    return 35;
-  }
-  if (!testPath && testIntent) {
-    return -8;
-  }
-  return 0;
-}
-function isTestPath(path) {
-  const lower = path.toLowerCase();
-  return (
-    /(^|\/)(tests?|fixtures?)(\/|$)/.test(lower) ||
-    lower.endsWith("_tests.rs") ||
-    lower.endsWith(".test.ts") ||
-    lower.endsWith(".test.js")
-  );
-}
-function hasExactNormalizedPhraseInChunk(chunk, normalizedQueryPhrase) {
-  return normalizedQueryPhrase !== "" && phraseSequenceForChunk(chunk).includes(normalizedQueryPhrase);
-}
-function tokensForChunk(chunk) {
-  chunk.tokens ??= tokenize(chunk.text);
-  return chunk.tokens;
-}
-function phraseSequenceForChunk(chunk) {
-  chunk.phraseSequence ??= normalizePhraseSequence(chunk.text);
-  return chunk.phraseSequence;
-}
-function normalizePhraseSequence(text) {
-  const tokens = tokenizeForPhrase(text);
-  return tokens.length >= 2 ? tokens.join(" ") : "";
-}
-function tokenCountMap(tokens) {
-  const counts = new Map();
-  for (const token of tokens) {
-    counts.set(token, (counts.get(token) ?? 0) + 1);
-  }
-  return counts;
-}
-function uniqueTokens(tokens) {
-  return Array.from(new Set(tokens));
-}
-function scoreText(text, tokens) {
-  const lower = text.toLowerCase();
-  return tokens.reduce((score, token) => score + countOccurrences(lower, token), 0);
-}
-function countOccurrences(text, token) {
-  let count = 0;
-  let index = text.indexOf(token);
-  while (index !== -1) {
-    count += 1;
-    index = text.indexOf(token, index + token.length);
-  }
-  return count;
-}
-function tokenize(query) {
-  return query
-    .split(/[^A-Za-z0-9_./-]+/)
-    .flatMap((token) => expandedIdentifierTokens(token.trim()))
-    .filter((token) => token.length >= 2 && !STOPWORDS.has(token));
-}
-function tokenizeForPhrase(query) {
-  return query
-    .split(/[^A-Za-z0-9_./-]+/)
-    .flatMap((token) => expandedIdentifierTokens(token.trim()))
-    .filter((token) => token.length >= 2);
-}
-function expandedIdentifierTokens(token) {
-  if (!token) {
-    return [];
-  }
-  const lower = token.toLowerCase();
-  if (!needsIdentifierSplit(token)) {
-    return [lower];
-  }
-  const variants = new Set();
-  addTokenVariant(variants, lower);
-  for (const part of splitIdentifierToken(token)) {
-    addTokenVariant(variants, part.toLowerCase());
-  }
-  return Array.from(variants);
-}
-function addTokenVariant(variants, token) {
-  if (token) {
-    variants.add(token);
-  }
-}
-function needsIdentifierSplit(token) {
-  if (/[._/-]/.test(token)) {
-    return true;
-  }
-  for (let index = 1; index < token.length; index += 1) {
-    const previous = token.charCodeAt(index - 1);
-    const current = token.charCodeAt(index);
-    const next = index + 1 < token.length ? token.charCodeAt(index + 1) : 0;
-    const previousIsLowerOrDigit = (previous >= 97 && previous <= 122) || (previous >= 48 && previous <= 57);
-    const previousIsUpper = previous >= 65 && previous <= 90;
-    const currentIsUpper = current >= 65 && current <= 90;
-    const nextIsLower = next >= 97 && next <= 122;
-    if ((previousIsLowerOrDigit && currentIsUpper) || (previousIsUpper && currentIsUpper && nextIsLower)) {
-      return true;
-    }
-  }
-  return false;
-}
-function splitIdentifierToken(token) {
-  const separated = token
-    .replace(/[._/-]+/g, " ")
-    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
-    .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2");
-  return separated.split(/\s+/).filter(Boolean);
-}

package/router/dist/evidence/evidence.d.ts

@@ -1,38 +0,0 @@
-import type { ImportantLine } from "../config/types.js";
-export type ImportantStream = ImportantLine["stream"];
-export interface LineEntry {
-  line: string;
-  lineNumber: number;
-}
-export interface EvidenceCaps {
-  maxLines?: number;
-  maxExcerptBytes?: number;
-  maxLineBytes?: number;
-}
-export interface BoundedEvidence {
-  importantLines: ImportantLine[];
-  fidelity: "exact" | "lossy";
-  compressed: boolean;
-  selectedLineCount: number;
-  sourceLineCount: number;
-}
-export declare const DEFAULT_MAX_IMPORTANT_LINES = 8;
-export declare const DEFAULT_MAX_IMPORTANT_EXCERPT_BYTES: number;
-export declare const DEFAULT_MAX_IMPORTANT_LINE_BYTES: number;
-export declare const TRUNCATION_MARKER = "\u2026 [truncated; recover exact output from vault]";
-export declare function assembleImportantLines(options: {
-  stream: ImportantStream;
-  entries: readonly LineEntry[];
-  sourceText: string;
-  caps?: EvidenceCaps;
-}): BoundedEvidence;
-export declare function assembleTextEvidence(options: {
-  stream: ImportantStream;
-  text: string;
-  caps?: EvidenceCaps;
-}): BoundedEvidence;
-export declare function lineEntries(text: string): LineEntry[];
-export declare function nonEmptyLineEntries(text: string): LineEntry[];
-export declare function splitLines(text: string): string[];
-export declare function countLines(text: string): number;
-export declare function byteLength(text: string): number;