@oscharko-dev/keiko-workspace 0.2.0

package/dist/repoSearchMatchers.js

@@ -0,0 +1,414 @@
+// Pure match strategies and the query fingerprint used by the repo-search facade (Issue #179).
+// Kept separate from repoSearch.ts to hold the file-length cap and to make every matcher
+// independently testable.
+import { createHash } from "node:crypto";
+import { RepoSearchInvalidQueryError } from "./errors.js";
+import { regexSafetyIssue } from "./repoSearchRegexSafety.js";
+export function fingerprintFor(query) {
+    const canonical = JSON.stringify({
+        kind: query.kind,
+        text: query.text,
+        caseSensitive: query.caseSensitive,
+        maxResults: query.maxResults,
+    });
+    return createHash("sha256").update(canonical).digest("hex").slice(0, 16);
+}
+// Issue #177 retrieval correctness: a natural-language question carries function words ("the",
+// "to", "are", "based", "on", ...) that appear on nearly every prose line. Scoring the raw
+// whitespace tokens let those stop words match almost everything, so the global
+// `maxMatchesReturned` budget was exhausted on the first alphabetically-scanned files and the
+// rest of a multi-file scope was never read (a `docs/` connect would only ever surface its
+// first file, never the file the question was actually about). We mirror the exploration
+// planner's fixed English stop-word policy (planner/anchors.ts in keiko-workflows - duplicated
+// here rather than imported because the architecture forbids keiko-workspace depending on the
+// higher-level keiko-workflows package): strip surrounding punctuation, drop single-character and
+// stop-word tokens, and keep `adr-0022`/`file.ts`-style hyphenated and dotted identifiers intact.
+const NL_STOP_WORDS = new Set([
+    "the",
+    "and",
+    "for",
+    "with",
+    "from",
+    "this",
+    "that",
+    "what",
+    "where",
+    "when",
+    "which",
+    "have",
+    "has",
+    "had",
+    "are",
+    "was",
+    "were",
+    "is",
+    "be",
+    "been",
+    "being",
+    "do",
+    "does",
+    "did",
+    "doing",
+    "of",
+    "in",
+    "on",
+    "at",
+    "to",
+    "an",
+    "as",
+    "or",
+    "but",
+    "not",
+    "no",
+    "yes",
+    "if",
+    "by",
+    "it",
+    "its",
+    "you",
+    "your",
+    "we",
+    "our",
+    "they",
+    "their",
+    "them",
+    "he",
+    "she",
+    "his",
+    "her",
+    "my",
+    "me",
+    "i",
+    "us",
+    "how",
+    "why",
+    "who",
+    "whom",
+    "whose",
+    "than",
+    "then",
+    "there",
+    "can",
+    "could",
+    "would",
+    "should",
+    "may",
+    "might",
+    "must",
+    "will",
+    "so",
+    "such",
+    "any",
+    "all",
+    "some",
+    "every",
+    "each",
+    "about",
+    "into",
+    "only",
+    "based",
+    "answer",
+    "aber",
+    "alle",
+    "als",
+    "am",
+    "an",
+    "auch",
+    "auf",
+    "aus",
+    "bei",
+    "bin",
+    "bis",
+    "bitte",
+    "da",
+    "das",
+    "dass",
+    "dein",
+    "deine",
+    "dem",
+    "den",
+    "der",
+    "des",
+    "die",
+    "dir",
+    "du",
+    "durch",
+    "ein",
+    "eine",
+    "einem",
+    "einen",
+    "einer",
+    "es",
+    "für",
+    "habe",
+    "haben",
+    "hat",
+    "ich",
+    "im",
+    "ist",
+    "kann",
+    "kannst",
+    "kein",
+    "keine",
+    "mit",
+    "mir",
+    "nach",
+    "nicht",
+    "noch",
+    "oder",
+    "sagen",
+    "sind",
+    "und",
+    "uns",
+    "von",
+    "war",
+    "was",
+    "welche",
+    "welchen",
+    "welcher",
+    "welches",
+    "wenn",
+    "wer",
+    "wie",
+    "wir",
+    "wird",
+    "wo",
+    "zu",
+    "zum",
+    "zur",
+]);
+const DEFINITION_INTENT_TOKENS = new Set([
+    "define",
+    "defined",
+    "definition",
+    "declare",
+    "declared",
+    "declaration",
+    "implement",
+    "implements",
+    "implemented",
+    "implementation",
+]);
+const HTTP_METHOD_TOKENS = new Set([
+    "get",
+    "post",
+    "put",
+    "patch",
+    "delete",
+    "head",
+    "options",
+]);
+const TECHNICAL_PHRASES = [
+    { pattern: /\btype[\s_-]?script\b/iu, term: "typescript" },
+    { pattern: /\bjava[\s_-]?script\b/iu, term: "javascript" },
+    { pattern: /\bnode(?:\.js)?\b/iu, term: "node" },
+    { pattern: /\bnext(?:\.js)?\b/iu, term: "nextjs" },
+    { pattern: /\bpackage\.json\b/iu, term: "package.json" },
+    { pattern: /\bpackage[\s_-]?manager\b|\bpaket[\s_-]?manager\b/iu, term: "package-manager" },
+    { pattern: /\btest[\s_-]?runner\b|\btestumgebung\b/iu, term: "test-runner" },
+];
+// Strip leading/trailing non-alphanumeric characters (Unicode-aware) while preserving internal
+// punctuation such as the hyphen in "ADR-0022" or the dot in "file.ts". Anchored, single
+// character-class quantifiers only - linear in input length (ReDoS-safe).
+function normalizeNaturalLanguageToken(raw) {
+    return raw.replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}]+$/u, "");
+}
+function naturalLanguageNormalizedTokens(rawTokens) {
+    return rawTokens.map(normalizeNaturalLanguageToken).filter((t) => t.length > 0);
+}
+// Extract the content tokens a relevance score should be computed over. Falls back to the
+// normalized-but-unfiltered tokens when filtering removes everything (a degenerate single-char
+// or stop-word-only query), so the matcher never silently scores nothing.
+function naturalLanguageContentTokens(rawTokens, caseSensitive) {
+    const normalized = naturalLanguageNormalizedTokens(rawTokens).map((t) => caseSensitive ? t : t.toLowerCase());
+    const content = normalized.filter((t) => t.length >= 2 && !NL_STOP_WORDS.has(t.toLowerCase()));
+    return content.length > 0 ? content : normalized;
+}
+function technicalPhraseTerms(queryText, caseSensitive) {
+    const terms = new Set();
+    for (const entry of TECHNICAL_PHRASES) {
+        if (entry.pattern.test(queryText)) {
+            terms.add(caseSensitive ? entry.term : entry.term.toLowerCase());
+        }
+    }
+    return [...terms];
+}
+function uniqueStrings(values) {
+    return [...new Set(values)];
+}
+function isDefinitionIntentToken(token) {
+    return DEFINITION_INTENT_TOKENS.has(token.toLowerCase());
+}
+function isSymbolLikeToken(token) {
+    return /[A-Z_]/u.test(token) || token.includes("-");
+}
+function analyzeNaturalLanguageIntent(normalizedTokens, caseSensitive) {
+    const lowered = normalizedTokens.map((t) => t.toLowerCase());
+    const definitionIntent = lowered.some(isDefinitionIntentToken);
+    const symbolTokens = uniqueStrings(normalizedTokens
+        .filter((t) => isSymbolLikeToken(t) && !DEFINITION_INTENT_TOKENS.has(t.toLowerCase()))
+        .map((t) => (caseSensitive ? t : t.toLowerCase())));
+    const routeTokens = uniqueStrings(normalizedTokens
+        .filter((t) => t.includes("/"))
+        .map((t) => (caseSensitive ? t : t.toLowerCase())));
+    const httpMethods = uniqueStrings(lowered
+        .filter((t) => HTTP_METHOD_TOKENS.has(t))
+        .map((t) => (caseSensitive ? t : t.toLowerCase())));
+    return { definitionIntent, symbolTokens, routeTokens, httpMethods };
+}
+function escapeRegExp(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+}
+function lineLooksLikeImport(line) {
+    return /^\s*import\b/u.test(line) || /^\s*export\s*\{/u.test(line);
+}
+function lineLooksLikeSymbolDefinition(line, symbolToken, caseSensitive) {
+    const escaped = escapeRegExp(symbolToken);
+    const flags = caseSensitive ? "u" : "iu";
+    const patterns = [
+        new RegExp(`\\b(?:export\\s+)?(?:async\\s+)?function\\s+${escaped}\\b`, flags),
+        new RegExp(`\\b(?:export\\s+)?(?:const|let|var)\\s+${escaped}\\b`, flags),
+        new RegExp(`\\b(?:export\\s+)?(?:class|interface|type|enum)\\s+${escaped}\\b`, flags),
+        new RegExp(`\\b${escaped}\\s*[:=]\\s*(?:async\\s*)?\\(`, flags),
+    ];
+    return patterns.some((pattern) => pattern.test(line));
+}
+function lineLooksLikeRouteDeclaration(line, haystack, intent) {
+    const routeHit = intent.routeTokens.some((token) => haystack.includes(token));
+    const methodHit = intent.httpMethods.some((method) => haystack.includes(`"${method}"`));
+    if (!routeHit || !methodHit) {
+        return false;
+    }
+    return line.includes("method:") || line.includes("pattern:");
+}
+function adjustedDefinitionIntentScore(line, haystack, baseScore, intent, caseSensitive) {
+    if (!intent.definitionIntent) {
+        return baseScore;
+    }
+    let bonus = 0;
+    let penalty = 0;
+    for (const symbolToken of intent.symbolTokens) {
+        if (!haystack.includes(symbolToken)) {
+            continue;
+        }
+        if (lineLooksLikeSymbolDefinition(line, symbolToken, caseSensitive)) {
+            bonus = Math.max(bonus, 0.75);
+        }
+        else if (lineLooksLikeImport(line)) {
+            penalty = Math.max(penalty, 0.2);
+        }
+    }
+    if (lineLooksLikeRouteDeclaration(line, haystack, intent)) {
+        bonus = Math.max(bonus, 0.65);
+    }
+    return Math.max(0, Math.min(1, baseScore + bonus - penalty));
+}
+function buildNaturalLanguageMatcher(query) {
+    const rawTokens = query.text.split(/\s+/).filter((t) => t.length > 0);
+    const normalizedTokens = naturalLanguageNormalizedTokens(rawTokens);
+    // GRD-033: dedupe content tokens (as symbol/route/method tokens already are) so a repeated
+    // query word does not double-count in `hits/total`, which over-rewarded prose-heavy scopes.
+    const tokens = uniqueStrings([
+        ...naturalLanguageContentTokens(rawTokens, query.caseSensitive),
+        ...technicalPhraseTerms(query.text, query.caseSensitive),
+    ]);
+    const intent = analyzeNaturalLanguageIntent(normalizedTokens, query.caseSensitive);
+    const total = tokens.length;
+    return {
+        match: (line) => {
+            if (total === 0) {
+                return 0;
+            }
+            const haystack = query.caseSensitive ? line : line.toLowerCase();
+            let hits = 0;
+            for (const token of tokens) {
+                if (haystack.includes(token)) {
+                    hits += 1;
+                }
+            }
+            if (hits === 0) {
+                return 0;
+            }
+            return adjustedDefinitionIntentScore(line, haystack, hits / total, intent, query.caseSensitive);
+        },
+    };
+}
+function buildExactSymbolMatcher(query) {
+    if (/\s/.test(query.text)) {
+        throw new RepoSearchInvalidQueryError("exact-symbol query must not contain whitespace");
+    }
+    const needle = query.caseSensitive ? query.text : query.text.toLowerCase();
+    return {
+        match: (line) => {
+            const haystack = query.caseSensitive ? line : line.toLowerCase();
+            return haystack.includes(needle) ? 1 : 0;
+        },
+    };
+}
+function buildRegexMatcher(query) {
+    const issue = regexSafetyIssue(query.text);
+    if (issue !== undefined) {
+        throw new RepoSearchInvalidQueryError(issue);
+    }
+    let regex;
+    try {
+        regex = new RegExp(query.text, query.caseSensitive ? "g" : "gi");
+    }
+    catch {
+        throw new RepoSearchInvalidQueryError(`invalid regex: ${query.text}`);
+    }
+    const cap = 100;
+    return {
+        match: (line) => {
+            regex.lastIndex = 0;
+            let count = 0;
+            while (regex.exec(line) !== null && count < cap) {
+                count += 1;
+                if (regex.lastIndex === 0) {
+                    break;
+                }
+            }
+            return count === 0 ? 0 : count / cap;
+        },
+    };
+}
+export function buildMatcher(query) {
+    if (query.kind === "natural-language") {
+        return buildNaturalLanguageMatcher(query);
+    }
+    if (query.kind === "exact-symbol") {
+        return buildExactSymbolMatcher(query);
+    }
+    if (query.kind === "regex") {
+        return buildRegexMatcher(query);
+    }
+    throw new RepoSearchInvalidQueryError(`unsupported query kind: ${query.kind}`);
+}
+// Anchored-glob compilation for findFiles. Supports `*`, `**`, `?`, and literal characters.
+// Brace expansion and extglob patterns are intentionally not supported.
+export function compileGlob(pattern, caseSensitive = true) {
+    let body = "";
+    let i = 0;
+    while (i < pattern.length) {
+        const ch = pattern.charAt(i);
+        if (ch === "*" && pattern.charAt(i + 1) === "*") {
+            body += ".*";
+            i += pattern.charAt(i + 2) === "/" ? 3 : 2;
+            continue;
+        }
+        if (ch === "*") {
+            body += "[^/]*";
+        }
+        else if (ch === "?") {
+            body += "[^/]";
+        }
+        else if (/[.+^${}()|[\]\\]/.test(ch)) {
+            body += `\\${ch}`;
+        }
+        else {
+            body += ch;
+        }
+        i += 1;
+    }
+    return new RegExp(`^${body}$`, caseSensitive ? "u" : "iu");
+}

package/dist/repoSearchPolicy.d.ts

@@ -0,0 +1,34 @@
+import type { CandidateOmissionReason, RetrievalQuery } from "@oscharko-dev/keiko-contracts/connected-context";
+import type { DiscoveredFile } from "./types.js";
+export type SearchIntent = "project-metadata" | "repository-overview" | "targeted-code-search" | "diagnostic-search" | "clarification-needed" | "generic";
+export type SearchPolicyMode = "workspace-root-default" | "explicit-scope";
+export type CandidateBucket = "canonical-metadata" | "overview-doc" | "exact-path" | "symbol-source" | "source" | "test" | "docs" | "lockfile" | "low-value" | "other";
+export interface SearchHints {
+    readonly retrievalIntent?: SearchIntent | undefined;
+}
+export interface SearchPolicy {
+    readonly mode: SearchPolicyMode;
+    readonly intent: SearchIntent;
+    readonly applyGitignore: boolean;
+    readonly omitLowValueWorkspaceFiles: boolean;
+}
+export interface SearchDiagnostics {
+    readonly policyMode: SearchPolicyMode;
+    readonly intent: SearchIntent;
+    readonly filesDiscovered: number;
+    readonly filesAfterPolicy: number;
+    readonly ignoredByDiscovery: number;
+    readonly deniedByDiscovery: number;
+    readonly candidateBuckets: Readonly<Record<CandidateBucket, number>>;
+}
+export interface CandidateOrderingResult {
+    readonly files: readonly DiscoveredFile[];
+    readonly diagnostics: SearchDiagnostics;
+}
+export declare function resolveSearchPolicy(hasExplicitRelativePaths: boolean, hints: SearchHints | undefined): SearchPolicy;
+export declare function legacyDiscoveryPolicy(hasExplicitRelativePaths: boolean): SearchPolicy;
+export declare function policyOmissionReason(scopePath: string, policy: SearchPolicy): CandidateOmissionReason | undefined;
+export declare function extraIgnoreLinesForSearch(policy: SearchPolicy): readonly string[];
+export declare function orderCandidatesForSearch(files: readonly DiscoveredFile[], query: RetrievalQuery, policy: SearchPolicy, ignoredByDiscovery: number, deniedByDiscovery: number): CandidateOrderingResult;
+export declare function shouldScoreContent(query: RetrievalQuery, text: string, policy: SearchPolicy): boolean;
+//# sourceMappingURL=repoSearchPolicy.d.ts.map

package/dist/repoSearchPolicy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"repoSearchPolicy.d.ts","sourceRoot":"","sources":["../src/repoSearchPolicy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,uBAAuB,EACvB,cAAc,EACf,MAAM,iDAAiD,CAAC;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,MAAM,YAAY,GACpB,kBAAkB,GAClB,qBAAqB,GACrB,sBAAsB,GACtB,mBAAmB,GACnB,sBAAsB,GACtB,SAAS,CAAC;AAEd,MAAM,MAAM,gBAAgB,GAAG,wBAAwB,GAAG,gBAAgB,CAAC;AAE3E,MAAM,MAAM,eAAe,GACvB,oBAAoB,GACpB,cAAc,GACd,YAAY,GACZ,eAAe,GACf,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,GACV,WAAW,GACX,OAAO,CAAC;AAEZ,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,eAAe,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;CACrD;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;IACjC,QAAQ,CAAC,0BAA0B,EAAE,OAAO,CAAC;CAC9C;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;IAC1C,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC;CACzC;AA4TD,wBAAgB,mBAAmB,CACjC,wBAAwB,EAAE,OAAO,EACjC,KAAK,EAAE,WAAW,GAAG,SAAS,GAC7B,YAAY,CASd;AAOD,wBAAgB,qBAAqB,CAAC,wBAAwB,EAAE,OAAO,GAAG,YAAY,CAOrF;AAED,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,YAAY,GACnB,uBAAuB,GAAG,SAAS,CAQrC;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,YAAY,GAAG,SAAS,MAAM,EAAE,CAEjF;AAED,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,YAAY,EACpB,kBAAkB,EAAE,MAAM,EAC1B,iBAAiB,EAAE,MAAM,GACxB,uBAAuB,CAczB;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,cAAc,EACrB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,YAAY,GACnB,OAAO,CAUT"}