agentera 3.0.0-dev.8 → 3.0.0-dev.81

package/dist/analytics/personalGlossaryExplicitSegments.js

@@ -0,0 +1,418 @@
+import { EXPLICIT_SEGMENT_REASONS, explicitSegmentTransition, } from "../registries/explicitSegmentGrammarContract.js";
+import { stableGlossaryTermIdentity } from "../registries/glossaryTermIdentity.js";
+import { EXPLICIT_GLOSSARY_REASONS, } from "./personalGlossaryExplicitTypes.js";
+const LETTER_OR_NUMBER_RE = /[\p{L}\p{N}]/u;
+const EXAMPLE_RE = /\b(?:for\s+example|e\.g\.?|example|such\s+as)\b/iu;
+const HYPOTHETICAL_RE = /\b(?:if|when|unless|whenever|suppose|assuming|hypothetically|imagine|would|could|might|may)\b/iu;
+const QUESTION_RE = /\?|^\s*(?:does|do|did|can|could|would|should|what|why|how)\b/iu;
+const INDIRECT_QUESTION_RE = /\b(?:i\s+(?:wonder|ask|want\s+to\s+know)|tell\s+me|whether|what|why|how)\b/iu;
+const FUTURE_RE = /\b(?:will|shall|going\s+to|plan\s+to|intend\s+to|in\s+the\s+future|from\s+now\s+on|later)\b/iu;
+const FOLLOWING_CUE_REFERENCE_RE = /\b(?:the\s+)?following\s+(?:definition|term|meaning)\b/iu;
+const PREVIOUS_CUE_REFERENCE_RE = /\bthis\s+(?:definition|term|meaning)\b/iu;
+function lineIndexAt(lines, position) {
+    let low = 0;
+    let high = lines.length - 1;
+    while (low <= high) {
+        const middle = Math.floor((low + high) / 2);
+        const line = lines[middle];
+        if (position < line.start)
+            high = middle - 1;
+        else if (position > line.end)
+            low = middle + 1;
+        else
+            return middle;
+    }
+    return Math.max(0, Math.min(lines.length - 1, low));
+}
+function hasListMarker(value) {
+    return /^\s*(?:[-*+]\s+|\d+[.)]\s+)/u.test(value) || /^\s*---\s*$/u.test(value);
+}
+function hasBlockScalarHeader(value) {
+    return /^\s*[^:#\n][^:\n]*:\s*[|>](?:[1-9])?(?:[+-])?\s*$/u.test(value);
+}
+function hasHeadingMarker(value) {
+    return /^\s*#{1,6}(?:\s|$)/u.test(value);
+}
+function safeMarkerPrefix(value) {
+    return /^\s*(?:[-*+]\s+)?(?:definition|term|correction|sarcasm)\s*:\s*$/iu.test(value);
+}
+function linePlans(text, raws, ranges, deps) {
+    const lines = deps.lineRanges(text);
+    const byLine = lines.map(() => []);
+    for (const raw of raws) {
+        if (raw.termStart === raw.termEnd && raw.meaningStart === raw.meaningEnd)
+            continue;
+        byLine[lineIndexAt(lines, raw.termStart)].push(raw);
+    }
+    return lines.map((line, index) => {
+        const comment = deps.commentStart(text, line, ranges);
+        const scanEnd = comment < 0 ? line.end : comment;
+        const value = text.slice(line.start, scanEnd);
+        const trimmed = value.trim();
+        const rawsOnLine = byLine[index].sort((left, right) => deps.rawCueOrder(left, right, text));
+        const firstColon = deps.colonOutsideRanges(text, line.start, scanEnd, ranges);
+        const firstRaw = rawsOnLine[0];
+        const markerPrefix = firstColon >= 0 && firstRaw !== undefined
+            ? safeMarkerPrefix(text.slice(line.start, firstColon + 1))
+            : false;
+        const unsafePrefix = firstColon >= 0 && firstRaw !== undefined && firstColon < firstRaw.termStart && !markerPrefix;
+        const leadingWhitespace = /^\s+\S/u.test(value);
+        const commentOnly = text.slice(line.start, line.end).trim().startsWith("#");
+        const listMarker = hasListMarker(value);
+        const heading = hasHeadingMarker(value);
+        const blockScalar = hasBlockScalarHeader(value);
+        const approved = rawsOnLine.length > 0 &&
+            !leadingWhitespace &&
+            !commentOnly &&
+            !heading &&
+            !blockScalar &&
+            !unsafePrefix;
+        let kind;
+        if (commentOnly)
+            kind = "comment";
+        else if (trimmed.length === 0)
+            kind = "blank";
+        else if (leadingWhitespace)
+            kind = "indentation";
+        else if (approved)
+            kind = "approved_cue";
+        else if (listMarker)
+            kind = "list_boundary";
+        else if (heading || blockScalar || firstColon >= 0)
+            kind = "structural_fragment";
+        else
+            kind = "prose";
+        const input = kind === "blank"
+            ? "blank_line"
+            : kind === "comment"
+                ? "comment"
+                : kind === "indentation"
+                    ? "indentation"
+                    : kind === "list_boundary"
+                        ? "list_boundary"
+                        : kind === "structural_fragment"
+                            ? blockScalar
+                                ? "block_scalar"
+                                : heading || trimmed.startsWith("#")
+                                    ? "comment"
+                                    : "structural_fragment"
+                            : kind === "approved_cue"
+                                ? "approved_cue"
+                                : "prose";
+        return { ...line, scanEnd, kind, input, raws: rawsOnLine };
+    });
+}
+function structuralRejection(line, reason) {
+    return {
+        kind: "structural_fragment",
+        termStart: line.start,
+        termEnd: line.start,
+        meaningStart: line.start,
+        meaningEnd: line.start,
+        sentenceStart: line.start,
+        sentenceEnd: line.end,
+        rejectionReason: reason,
+    };
+}
+function loadedStructuralReason(transition) {
+    return transition.reason === EXPLICIT_SEGMENT_REASONS.structuralFragment
+        ? transition.reason
+        : EXPLICIT_GLOSSARY_REASONS.structuralFragment;
+}
+function segmentRawCues(text, initial, ranges, grammar, deps) {
+    const plans = linePlans(text, initial, ranges, deps);
+    const accepted = [];
+    let state = grammar.initialState;
+    for (const plan of plans) {
+        if (state === "structural_config") {
+            const transition = explicitSegmentTransition(grammar, state, plan.input);
+            if (plan.kind === "prose") {
+                accepted.push(structuralRejection(plan, loadedStructuralReason(transition)));
+                state = transition.to;
+                continue;
+            }
+            if (plan.kind !== "blank" && plan.kind !== "comment") {
+                accepted.push(structuralRejection(plan, loadedStructuralReason(transition)));
+            }
+            continue;
+        }
+        if (plan.kind === "approved_cue") {
+            const transition = explicitSegmentTransition(grammar, state, "approved_cue");
+            if (transition.to === "cue")
+                accepted.push(...plan.raws);
+            continue;
+        }
+        if (plan.kind === "structural_fragment" || plan.kind === "list_boundary") {
+            const transition = explicitSegmentTransition(grammar, state, plan.input);
+            if (transition.to === "structural_config") {
+                state = transition.to;
+                accepted.push(structuralRejection(plan, loadedStructuralReason(transition)));
+            }
+        }
+        else if (plan.kind === "comment" && /:\s*/u.test(text.slice(plan.start, plan.end))) {
+            const transition = explicitSegmentTransition(grammar, state, "structural_fragment");
+            accepted.push(structuralRejection(plan, loadedStructuralReason(transition)));
+        }
+    }
+    return { raws: accepted, plans };
+}
+function firstRawAfter(plan, position, text, deps) {
+    return plan.raws
+        .filter((raw) => raw.termStart > position)
+        .sort((left, right) => deps.rawCueOrder(left, right, text))[0];
+}
+function cueBoundaryStart(text, cue, closeToOpen) {
+    if (cue.kind === "definition_list") {
+        const floor = Math.max(text.lastIndexOf("\n", cue.termStart - 1), text.lastIndexOf(";", cue.termStart - 1)) + 1;
+        const prefix = text.slice(floor, cue.termStart);
+        if (/^\s*(?:[-*+]\s+)?(?:definition|term)\s*:\s*$/iu.test(prefix)) {
+            const first = prefix.search(/\S/u);
+            if (first >= 0)
+                return floor + first;
+        }
+    }
+    const opening = closeToOpen.get(text[cue.termEnd] ?? "");
+    if (opening !== undefined) {
+        const start = text.lastIndexOf(opening, cue.termStart);
+        if (start >= 0)
+            return start;
+    }
+    return cue.termStart;
+}
+function isBoundaryLine(plan) {
+    return ["blank", "comment", "indentation", "list_boundary", "structural_fragment"].includes(plan.kind);
+}
+function sentenceRanges(text, ranges, deps) {
+    const boundaries = deps.sentenceBoundaries(text, ranges);
+    return [0, ...boundaries].map((start, index, starts) => ({
+        start,
+        end: starts[index + 1] ?? text.length,
+    }));
+}
+function nearMissTermOccurs(text, term, start, end) {
+    const literal = escapedTerm(term);
+    if (!literal)
+        return false;
+    const identifierContinue = "\\p{ID_Continue}$\\u200C\\u200D";
+    return new RegExp(`(?:(?<![${identifierContinue}])${literal}(?=[${identifierContinue}])|(?<=[${identifierContinue}])${literal}(?![${identifierContinue}]))`, "iu").test(text.slice(start, end));
+}
+function referenceLikeQualifierStart(text, line, raws, ranges, deps) {
+    for (const sentence of sentenceRanges(text, ranges, deps)) {
+        if (sentence.end <= line.start || sentence.start > line.end)
+            continue;
+        const value = text.slice(sentence.start, sentence.end);
+        if (!referenceQualifier(value))
+            continue;
+        const following = value.match(new RegExp(FOLLOWING_CUE_REFERENCE_RE.source, "iu"));
+        if (following?.index !== undefined)
+            return Math.max(sentence.start, line.start);
+        const previous = value.match(new RegExp(PREVIOUS_CUE_REFERENCE_RE.source, "iu"));
+        if (previous?.index !== undefined)
+            return Math.max(sentence.start, line.start);
+        if (raws.some((raw) => raw.rejectionReason === undefined &&
+            raw.termStart >= sentence.start &&
+            raw.termStart < sentence.end))
+            continue;
+        for (const raw of raws) {
+            if (raw.rejectionReason !== undefined || raw.termEnd <= raw.termStart)
+                continue;
+            const term = text.slice(raw.termStart, raw.termEnd).trim();
+            const occurrence = exactTermOccurrences(text, term, sentence.start, sentence.end)[0];
+            if (occurrence !== undefined ||
+                nearMissTermOccurs(text, term, sentence.start, sentence.end)) {
+                return Math.max(sentence.start, line.start);
+            }
+        }
+    }
+    return null;
+}
+function boundedSegmentCues(text, raws, plans, ranges, grammar, deps) {
+    const lines = deps.lineRanges(text);
+    const structural = raws.filter((raw) => raw.termStart === raw.termEnd && raw.meaningStart === raw.meaningEnd);
+    const usable = raws.filter((raw) => !(raw.termStart === raw.termEnd && raw.meaningStart === raw.meaningEnd));
+    const bounded = usable.map((raw) => {
+        if (raw.kind === "acronym_parenthetical")
+            return raw;
+        if (raw.kind !== "definition_list") {
+            const line = plans[lineIndexAt(lines, raw.termStart)];
+            const next = firstRawAfter(line, raw.termStart, text, deps);
+            const boundary = next === undefined ? raw.sentenceEnd : cueBoundaryStart(text, next, deps.closeToOpen);
+            const end = boundary < raw.sentenceEnd ? boundary : raw.sentenceEnd;
+            const [meaningStart, meaningEnd] = deps.unwrapMeaning(text, ...deps.trimMeaning(text, raw.meaningStart, end));
+            return { ...raw, meaningStart, meaningEnd };
+        }
+        const lineIndex = lineIndexAt(lines, raw.termStart);
+        const line = plans[lineIndex];
+        const nextOnLine = firstRawAfter(line, raw.termStart, text, deps);
+        let end = nextOnLine ? cueBoundaryStart(text, nextOnLine, deps.closeToOpen) : line.scanEnd;
+        const referenceStart = referenceLikeQualifierStart(text, line, raws, ranges, deps);
+        const sameLineReference = referenceStart !== null && referenceStart > raw.meaningStart ? referenceStart : null;
+        if (sameLineReference !== null) {
+            end = Math.min(end, sameLineReference);
+        }
+        let state = "cue";
+        const inline = deps.trimMeaning(text, raw.meaningStart, end);
+        const inlineNonEmpty = LETTER_OR_NUMBER_RE.test(text.slice(...inline));
+        const cueTransition = explicitSegmentTransition(grammar, state, inlineNonEmpty ? "non_empty_inline_meaning" : nextOnLine ? "next_cue" : "blank_line");
+        if (!inlineNonEmpty) {
+            return {
+                ...raw,
+                meaningStart: inline[0],
+                meaningEnd: inline[1],
+                ...(raw.rejectionReason === undefined
+                    ? { rejectionReason: EXPLICIT_GLOSSARY_REASONS.emptyMeaning }
+                    : {}),
+            };
+        }
+        state = cueTransition.to;
+        let lastIncludedEnd = end;
+        for (let index = lineIndex + 1; sameLineReference === null && index < plans.length; index += 1) {
+            const nextPlan = plans[index];
+            const nextCue = nextPlan.raws.find((candidate) => candidate.termStart > raw.termStart);
+            const transition = explicitSegmentTransition(grammar, state, nextPlan.input);
+            if (isBoundaryLine(nextPlan)) {
+                end = nextPlan.start;
+                break;
+            }
+            if (nextCue !== undefined) {
+                end = cueBoundaryStart(text, nextCue, deps.closeToOpen);
+                break;
+            }
+            const referenceStart = referenceLikeQualifierStart(text, nextPlan, raws, ranges, deps);
+            if (referenceStart !== null) {
+                end = referenceStart;
+                break;
+            }
+            state = transition.to;
+            if (state !== "continuation") {
+                end = nextPlan.start;
+                break;
+            }
+            lastIncludedEnd = nextPlan.end;
+            end = lastIncludedEnd;
+            if (nextPlan.kind === "approved_cue") {
+                end = nextPlan.start;
+                break;
+            }
+        }
+        const [meaningStart, meaningEnd] = deps.unwrapMeaning(text, ...deps.trimMeaning(text, raw.meaningStart, end));
+        const overBound = Buffer.byteLength(text.slice(meaningStart, meaningEnd), "utf8") >
+            grammar.continuation.maximumUtf8Bytes;
+        return {
+            ...raw,
+            meaningStart,
+            meaningEnd,
+            ...(overBound && raw.rejectionReason === undefined
+                ? { rejectionReason: EXPLICIT_GLOSSARY_REASONS.meaningBoundExceeded }
+                : {}),
+        };
+    });
+    return [...bounded, ...structural];
+}
+function escapedTerm(term) {
+    return term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function exactTermOccurrences(text, term, start, end) {
+    const literal = escapedTerm(term);
+    if (!literal)
+        return [];
+    const expression = new RegExp(`(?<![\\p{ID_Continue}$\\u200C\\u200D])${literal}(?![\\p{ID_Continue}$\\u200C\\u200D])`, "giu");
+    return [...text.slice(start, end).matchAll(expression)].map((match) => start + (match.index ?? 0));
+}
+function occurrenceInsideMeaning(position, raw) {
+    return position >= raw.meaningStart && position < raw.meaningEnd;
+}
+function referenceQualifier(text) {
+    if (EXAMPLE_RE.test(text))
+        return EXPLICIT_GLOSSARY_REASONS.exampleContext;
+    if (HYPOTHETICAL_RE.test(text))
+        return EXPLICIT_GLOSSARY_REASONS.hypotheticalDefinition;
+    if (INDIRECT_QUESTION_RE.test(text) || QUESTION_RE.test(text)) {
+        return EXPLICIT_GLOSSARY_REASONS.indirectQuestion;
+    }
+    if (FUTURE_RE.test(text))
+        return EXPLICIT_GLOSSARY_REASONS.futureDefinition;
+    return null;
+}
+function applyReferenceBindings(text, raws, ranges, deps) {
+    const sentences = sentenceRanges(text, ranges, deps);
+    const cues = raws
+        .filter((raw) => raw.rejectionReason === undefined &&
+        raw.termEnd > raw.termStart &&
+        raw.meaningEnd > raw.meaningStart)
+        .sort((left, right) => deps.rawCueOrder(left, right, text));
+    const rejected = new Map();
+    const missing = [];
+    const reject = (target, reason) => {
+        if (!rejected.has(target))
+            rejected.set(target, reason);
+    };
+    const cueSentence = (cue) => Math.max(0, sentences.findIndex((sentence) => cue.termStart >= sentence.start && cue.termStart < sentence.end));
+    const cuesBySentence = sentences.map((_, index) => cues.filter((cue) => cueSentence(cue) === index));
+    for (const [sentenceIndex, sentence] of sentences.entries()) {
+        const value = text.slice(sentence.start, sentence.end);
+        const qualifier = referenceQualifier(value);
+        if (!qualifier)
+            continue;
+        for (const match of value.matchAll(new RegExp(FOLLOWING_CUE_REFERENCE_RE.source, "giu"))) {
+            const position = sentence.start + (match.index ?? 0) + match[0].length;
+            const target = (cuesBySentence[sentenceIndex + 1] ?? []).find((cue) => cue.termStart >= position);
+            if (target)
+                reject(target, qualifier);
+            else
+                missing.push(structuralRejection({ start: sentence.start, end: sentence.end }, EXPLICIT_GLOSSARY_REASONS.ambiguousReference));
+        }
+        for (const match of value.matchAll(new RegExp(PREVIOUS_CUE_REFERENCE_RE.source, "giu"))) {
+            const position = sentence.start + (match.index ?? 0);
+            const target = [...(cuesBySentence[sentenceIndex - 1] ?? [])]
+                .reverse()
+                .find((cue) => cue.termStart < position);
+            if (target)
+                reject(target, qualifier);
+            else
+                missing.push(structuralRejection({ start: sentence.start, end: sentence.end }, EXPLICIT_GLOSSARY_REASONS.ambiguousReference));
+        }
+        if (cues.some((cue) => cue.termStart >= sentence.start && cue.termStart < sentence.end))
+            continue;
+        const directExactReference = QUESTION_RE.test(value) ||
+            EXAMPLE_RE.test(value) ||
+            HYPOTHETICAL_RE.test(value) ||
+            FUTURE_RE.test(value);
+        const nearbyCues = cues.filter((cue) => Math.abs(cueSentence(cue) - sentenceIndex) <= 1);
+        const matches = new Map();
+        for (const cue of nearbyCues) {
+            const term = text.slice(cue.termStart, cue.termEnd).trim();
+            const occurrences = exactTermOccurrences(text, term, sentence.start, sentence.end).filter((position) => !nearbyCues.some((other) => other !== cue &&
+                other.rejectionReason === undefined &&
+                occurrenceInsideMeaning(position, other) &&
+                !directExactReference));
+            if (occurrences.length > 0) {
+                const identity = stableGlossaryTermIdentity(term);
+                matches.set(identity, [...(matches.get(identity) ?? []), cue]);
+            }
+        }
+        for (const group of matches.values()) {
+            const unique = [...new Map(group.map((cue) => [cue.termStart, cue])).values()];
+            if (unique.length > 1) {
+                for (const cue of unique)
+                    reject(cue, EXPLICIT_GLOSSARY_REASONS.ambiguousReference);
+            }
+            else if (unique[0])
+                reject(unique[0], qualifier);
+        }
+    }
+    return [
+        ...raws.map((raw) => {
+            const reason = rejected.get(raw);
+            return reason === undefined || raw.rejectionReason !== undefined
+                ? raw
+                : { ...raw, rejectionReason: reason };
+        }),
+        ...missing,
+    ];
+}
+export function segmentAndBindExplicitCues(text, initial, ranges, grammar, deps) {
+    const segmented = segmentRawCues(text, initial, ranges, grammar, deps);
+    const bounded = boundedSegmentCues(text, segmented.raws, segmented.plans, ranges, grammar, deps);
+    return applyReferenceBindings(text, bounded, ranges, deps).sort((left, right) => deps.rawCueOrder(left, right, text));
+}

package/dist/analytics/personalGlossaryExplicitTypes.js

@@ -0,0 +1,26 @@
+export const EXPLICIT_GLOSSARY_REASONS = {
+    attributedQuotation: "attributed_quotation",
+    ambiguousReference: "ambiguous_reference",
+    conflictingMeaning: "conflicting_meaning",
+    emptyMeaning: "empty_meaning",
+    emptyTerm: "empty_term",
+    exampleContext: "example_context",
+    hypotheticalDefinition: "hypothetical_definition",
+    malformedSpan: "malformed_span",
+    meaningBoundExceeded: "meaning_bound_exceeded",
+    negatedDefinition: "negated_definition",
+    indirectQuestion: "indirect_question",
+    futureDefinition: "future_definition",
+    projectOnlyScope: "project_only_scope",
+    questionDefinition: "question_definition",
+    retractedDefinition: "retracted_definition",
+    sarcasmMarker: "sarcasm_marker",
+    staleAnchor: "stale_anchor",
+    structuralFragment: "structural_fragment",
+    termBoundExceeded: "term_bound_exceeded",
+    unsafeSyntax: "unsafe_syntax",
+    uncertainScope: "uncertain_scope",
+    unresolvedAnchor: "unresolved_anchor",
+    provenanceIncomplete: "provenance_incomplete",
+    userAuthorshipRequired: "user_authorship_required",
+};

package/dist/analytics/personalGlossaryLexicalClassifier.js

@@ -0,0 +1,175 @@
+const COMMON_GRAMMAR_TERMS = new Set([
+    "a",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "by",
+    "can",
+    "could",
+    "do",
+    "each",
+    "for",
+    "from",
+    "has",
+    "have",
+    "how",
+    "if",
+    "in",
+    "into",
+    "is",
+    "it",
+    "must",
+    "no",
+    "not",
+    "of",
+    "on",
+    "one",
+    "only",
+    "or",
+    "should",
+    "that",
+    "the",
+    "then",
+    "this",
+    "to",
+    "use",
+    "version",
+    "when",
+    "which",
+    "why",
+    "with",
+    "would",
+    "yes",
+    "two",
+    "three",
+]);
+const ROUTINE_FREQUENCY_TERMS = new Set([
+    "always",
+    "frequently",
+    "generally",
+    "never",
+    "normally",
+    "occasionally",
+    "often",
+    "rarely",
+    "regularly",
+    "routinely",
+    "seldom",
+    "sometimes",
+    "typically",
+    "usually",
+]);
+const GENERIC_WORKFLOW_TERMS = new Set([
+    "change",
+    "check",
+    "choose",
+    "command",
+    "configuration",
+    "continue",
+    "correction",
+    "decision",
+    "file",
+    "keep",
+    "make",
+    "name",
+    "package",
+    "personal",
+    "project",
+    "repository",
+    "run",
+    "script",
+    "scripts",
+]);
+const KNOWN_COMMAND_WORDS = new Set([
+    "agentera",
+    "bash",
+    "build",
+    "cargo",
+    "cat",
+    "cd",
+    "check",
+    "ci",
+    "curl",
+    "git",
+    "install",
+    "lint",
+    "ls",
+    "make",
+    "node",
+    "npm",
+    "npx",
+    "pnpm",
+    "python",
+    "sh",
+    "tcsh",
+    "test",
+    "tsc",
+    "tsx",
+    "yarn",
+]);
+const KNOWN_COMMAND_INVOCATIONS = {
+    agentera: ["build", "check"],
+    cargo: ["build", "test"],
+    git: ["add", "branch", "checkout", "commit", "diff", "fetch", "log", "push", "status", "tag"],
+    npm: ["install", "test"],
+    pnpm: ["build", "install", "lint", "test", "typecheck"],
+    yarn: ["build", "install", "lint", "test"],
+};
+const KNOWN_COMMAND_SPELLINGS = new Set(Object.entries(KNOWN_COMMAND_INVOCATIONS).flatMap(([command, actions]) => actions.map((action) => `${command}${action}`)));
+const KNOWN_COMMAND_COMPONENTS = new Set([
+    ...KNOWN_COMMAND_WORDS,
+    ...Object.keys(KNOWN_COMMAND_INVOCATIONS),
+    ...Object.values(KNOWN_COMMAND_INVOCATIONS).flat(),
+]);
+const PATH_COMPONENTS = new Set([
+    "dist",
+    "home",
+    "lib",
+    "modules",
+    "node",
+    "node_modules",
+    "path",
+    "repo",
+    "src",
+    "tmp",
+]);
+function isKnownCommandSpelling(value) {
+    const lower = value.toLowerCase();
+    if (KNOWN_COMMAND_COMPONENTS.has(lower))
+        return true;
+    const compact = lower.replace(/[\s\-_/\\.:]+/gu, "");
+    return KNOWN_COMMAND_SPELLINGS.has(compact);
+}
+function isDirectPathToken(value, text, start, end) {
+    if (/[\\/]/u.test(value))
+        return true;
+    const previous = text[start - 1] ?? "";
+    const next = text[end] ?? "";
+    const dotIsPathPunctuation = previous === "." || (next === "." && /[\p{L}\p{N}]/u.test(text[end + 1] ?? ""));
+    return ["/", "\\"].includes(previous) || ["/", "\\"].includes(next) || dotIsPathPunctuation;
+}
+function isApprovedDerivedPathSpelling(value) {
+    const segments = value
+        .toLowerCase()
+        .split(/[-_.:]+/u)
+        .filter(Boolean);
+    return segments.length >= 2 && segments.every((segment) => PATH_COMPONENTS.has(segment));
+}
+/** Classify one complete cue without rewriting it or matching arbitrary substrings. */
+export function classifyRecurringLexicalToken(value, text, start, end) {
+    const lower = value.toLowerCase();
+    if (COMMON_GRAMMAR_TERMS.has(lower) ||
+        ROUTINE_FREQUENCY_TERMS.has(lower) ||
+        GENERIC_WORKFLOW_TERMS.has(lower)) {
+        return "common_term";
+    }
+    if (isKnownCommandSpelling(value))
+        return "known_command";
+    if (isDirectPathToken(value, text, start, end) || isApprovedDerivedPathSpelling(value)) {
+        return "path_like";
+    }
+    return null;
+}