@pugi/cli 0.1.0-beta.5 → 0.1.0-beta.51

package/dist/core/engine/intent.js

@@ -0,0 +1,260 @@
+/**
+ * β5a P1+P6 — CLI-side intent classifier: definitional vs operational.
+ *
+ * Background (CEO 2026-05-26 quality gate ≥80% vs Claude Code): the
+ * α7.X Phase 2 comparative eval surfaced a high-frequency Pugi loss
+ * mode where the model interpreted EVERY user message as an operational
+ * task and reached for tools. Concrete reproducer: when an operator
+ * asks "What is grep?", pre-β5a Pugi would issue a `bash` tool call
+ * (`man grep`, `grep --help`, sometimes a `glob` over `**`) instead
+ * of answering explanatorily. Claude Code answers in one paragraph,
+ * no tool calls. That single failure mode costs ~3 of the 23
+ * comparative fixtures.
+ *
+ * Root cause: the system prompt advertises tools as the primary way
+ * to make progress; without a counter-instruction, the model defaults
+ * to tools even for pure-knowledge questions. Prompt v2 (P6) adds
+ * definitional EXAMPLES in the prompt; this classifier is the
+ * deterministic safety net that flags definitional questions BEFORE
+ * the engine loop fires so we can:
+ *
+ *   1. Inject a one-line `<intent>definitional</intent>` marker into
+ *      the user message that the prompt explicitly teaches the model
+ *      to honour ("when you see this marker, answer in prose, do not
+ *      call tools").
+ *   2. Make the marker visible in the SSE stream so cabinet UI / eval
+ *      reports can show why a tool-free answer was correct.
+ *
+ * The classifier is intentionally simple — pure regex/keyword scoring,
+ * sub-millisecond, no LLM call, no fs reads. It must be:
+ *
+ *   - Deterministic (same input always → same output, the eval relies
+ *     on this for stable comparison runs).
+ *   - Conservative on the operational side. False-positive
+ *     definitional is mostly harmless (the model can still call tools
+ *     if needed — the marker is a hint, not a hard gate). False-
+ *     positive operational on a definitional question is what we are
+ *     fixing — biggest loss surface.
+ *
+ * Coverage: EN + RU (the two primary CLI languages per
+ * `feedback_chat_always_russian_no_drift.md`). Uses lowercase compare
+ * for keyword matches, anchored regex for prefix patterns. Stop-word
+ * agnostic — the question "what is X?" is the dominant template and
+ * survives every common stop-word filter.
+ *
+ * Out of scope: deciding WHICH tool to call when operational. That
+ * is the model's job. We only decide whether to suggest "no tools
+ * needed" up front.
+ */
+/**
+ * EN + RU definitional question openers. Anchored at start-of-string
+ * (after trim + lower) so a sentence-medial "what is" inside a larger
+ * operational sentence does not flip the verdict. Each entry is
+ * either a literal prefix or a RegExp matched against the lowercase
+ * trimmed input.
+ *
+ * Boundary policy: JS `\b` is ASCII-only — Cyrillic letters do not
+ * count as word characters, so `^объясни\b` matches "объясни" against
+ * a space follower BUT fails when the next char is a Cyrillic letter
+ * (which it never is at this position) and surprisingly fails on
+ * "объясни X" because `\b` between `и` (non-word in ASCII regex) and
+ * ` ` (non-word) is never a boundary. We use an explicit
+ * `(?=\s|$|[!?.,])` lookahead so Cyrillic prefixes match the same
+ * way ASCII ones do.
+ */
+const WB = '(?=\\s|$|[!?.,;:])';
+const DEFINITIONAL_PATTERNS = [
+    // EN
+    new RegExp(`^what\\s+(is|are|does|do)${WB}`),
+    new RegExp(`^who\\s+(is|are|was|were)${WB}`),
+    new RegExp(`^when\\s+(is|was|does|did)${WB}`),
+    new RegExp(`^where\\s+(is|are|does|did)${WB}`),
+    new RegExp(`^why\\s+(is|are|does|did)${WB}`),
+    new RegExp(`^how\\s+(does|do|did)\\s+\\S+\\s+work${WB}`),
+    new RegExp(`^explain\\s+(what|why|how)${WB}`),
+    new RegExp(`^define${WB}`),
+    new RegExp(`^tell\\s+me\\s+(what|about)${WB}`),
+    new RegExp(`^can\\s+you\\s+(tell\\s+me|explain|describe)${WB}`),
+    // RU
+    new RegExp(`^что\\s+(такое|это|значит)${WB}`),
+    new RegExp(`^кто\\s+(такой|такая|такие|это)${WB}`),
+    new RegExp(`^как\\s+работает${WB}`),
+    new RegExp(`^зачем${WB}`),
+    new RegExp(`^почему${WB}`),
+    new RegExp(`^объясни${WB}`),
+    new RegExp(`^расскажи\\s+(про|о|об)${WB}`),
+];
+/**
+ * Operational verbs that strongly imply a tool-use task. A match
+ * here overrides a weak definitional signal (e.g. "explain the bug
+ * and fix it" -> operational, not definitional).
+ */
+const OPERATIONAL_KEYWORDS = [
+    // EN
+    'fix', 'patch', 'add', 'remove', 'delete', 'create', 'build',
+    'implement', 'refactor', 'rename', 'migrate', 'deploy', 'install',
+    'run', 'test', 'debug', 'investigate', 'open a pr', 'commit',
+    'push', 'merge', 'review my', 'write a', 'write the', 'replace',
+    'update the', 'modify', 'edit', 'change the',
+    // RU
+    'почини', 'исправь', 'добавь', 'удали', 'создай', 'собери',
+    'реализуй', 'отрефактори', 'переименуй', 'мигрируй', 'задеплой',
+    'установи', 'запусти', 'протестируй', 'отладь', 'разберись',
+    'закоммить', 'смержи', 'напиши', 'замени', 'обнови', 'измени',
+    'отредактируй',
+];
+/**
+ * File-path / identifier signals that, when combined with any verb,
+ * push us toward operational regardless of question framing. A bare
+ * "what is" question that names a file (`what is in apps/admin-api/
+ * src/main.ts?`) reads as operational — the operator wants the file
+ * inspected, not a dictionary definition.
+ */
+const PATH_SIGNAL = /[./][a-z0-9_-]+\.(ts|tsx|js|jsx|json|md|yaml|yml|sh|py|prisma|css|scss|html|sql|env)\b/i;
+const CODE_SYMBOL_SIGNAL = /\b[A-Z][a-zA-Z0-9]+\.(prototype|find|create|run|build|register)\b/;
+/**
+ * Classify a single user message. Returns one of:
+ *
+ *   - `definitional` — high confidence the operator wants knowledge,
+ *     not edits. Prompt the model to answer in prose, no tools.
+ *   - `operational` — verbs / paths / explicit "fix X" framing.
+ *     Model decides tools as normal.
+ *   - `ambiguous` — neither side won decisively. The model gets no
+ *     extra hint; default behaviour applies. The marker is NOT
+ *     injected for ambiguous to keep the prompt cache stable.
+ *
+ * Confidence: rough heuristic — `1.0` when a pattern matched cleanly
+ * with no opposing signal; lower when both sides matched.
+ *
+ * Pure function, no side effects.
+ */
+export function classifyIntent(rawMessage) {
+    const trimmed = rawMessage.trim();
+    if (trimmed.length === 0) {
+        return {
+            intent: 'ambiguous',
+            confidence: 0,
+            matched: [],
+            rationale: 'empty input',
+        };
+    }
+    const lower = trimmed.toLowerCase();
+    const matched = [];
+    let definitionalScore = 0;
+    let operationalScore = 0;
+    let hasDefinitionalOpener = false;
+    for (const pattern of DEFINITIONAL_PATTERNS) {
+        if (pattern.test(lower)) {
+            // Three points — heavier than a single operational keyword
+            // match (two) so a definitional opener cannot be flipped by
+            // ONE incidental verb appearing as part of a noun phrase
+            // (e.g. "what does prisma MIGRATE do" — the verb "migrate"
+            // here names a subcommand, not an action the operator wants
+            // executed). Operational still wins when multiple operational
+            // signals stack (verb + path + imperative).
+            definitionalScore += 3;
+            matched.push(`def:${pattern.source}`);
+            hasDefinitionalOpener = true;
+            break; // a single anchored prefix is enough
+        }
+    }
+    // ?-terminated short utterance with no verbs is a strong
+    // definitional signal even without a what/who/why opener.
+    if (lower.endsWith('?') && lower.split(/\s+/).length <= 6) {
+        definitionalScore += 1;
+        matched.push('def:short-question');
+    }
+    // A definitional opener PLUS an explicit `?` terminator together
+    // form a pure question. Verbs mentioned inside ("what does prisma
+    // MIGRATE do?") are sub-noun-phrase references, not commands —
+    // boost definitional so the verb-keyword scoring below cannot tie.
+    if (hasDefinitionalOpener && lower.endsWith('?')) {
+        definitionalScore += 2;
+        matched.push('def:opener+question');
+    }
+    for (const keyword of OPERATIONAL_KEYWORDS) {
+        if (lower.includes(keyword)) {
+            operationalScore += 2;
+            matched.push(`op:${keyword}`);
+            break;
+        }
+    }
+    if (PATH_SIGNAL.test(trimmed)) {
+        operationalScore += 1;
+        matched.push('op:path-signal');
+    }
+    if (CODE_SYMBOL_SIGNAL.test(trimmed)) {
+        operationalScore += 1;
+        matched.push('op:code-symbol');
+    }
+    // Imperative ("do X") without any question framing is operational.
+    if (!lower.endsWith('?') && /^(please\s+)?(do|run|make|let'?s)\b/.test(lower)) {
+        operationalScore += 1;
+        matched.push('op:imperative');
+    }
+    // Decision.
+    if (definitionalScore === 0 && operationalScore === 0) {
+        return {
+            intent: 'ambiguous',
+            confidence: 0,
+            matched,
+            rationale: 'no rules matched',
+        };
+    }
+    if (definitionalScore > operationalScore + 1) {
+        return {
+            intent: 'definitional',
+            confidence: clamp01(definitionalScore / (definitionalScore + operationalScore + 1)),
+            matched,
+            rationale: `definitional ${definitionalScore} > operational ${operationalScore}`,
+        };
+    }
+    if (operationalScore >= definitionalScore) {
+        return {
+            intent: 'operational',
+            confidence: clamp01(operationalScore / (definitionalScore + operationalScore + 1)),
+            matched,
+            rationale: `operational ${operationalScore} >= definitional ${definitionalScore}`,
+        };
+    }
+    // Definitional won but margin is tight — keep but lower confidence.
+    return {
+        intent: 'definitional',
+        confidence: clamp01(definitionalScore / (definitionalScore + operationalScore + 1)) * 0.7,
+        matched,
+        rationale: `definitional ${definitionalScore} marginal over operational ${operationalScore}`,
+    };
+}
+/**
+ * Marker the engine wraps the user message in when the classifier
+ * picked `definitional`. The system prompt v2 teaches the model to
+ * answer in prose when it sees this marker.
+ *
+ * Format choice: angle-bracket pseudo-XML, matches the existing
+ * `<context>` / `<privacy-mode>` markers in the prompt — consistent
+ * marker grammar across the codebase keeps the model attentive.
+ */
+export const DEFINITIONAL_MARKER_OPEN = '<intent kind="definitional">';
+export const DEFINITIONAL_MARKER_CLOSE = '</intent>';
+/**
+ * Wrap a user message with the intent marker when the classifier
+ * fires `definitional`. Returns the original message verbatim for
+ * the `operational` and `ambiguous` cases — we never lie to the
+ * model about ambiguity, and we never restrict the model's tool
+ * use on an unrelated operational task.
+ */
+export function applyIntentMarker(message, intent) {
+    if (intent !== 'definitional')
+        return message;
+    return `${DEFINITIONAL_MARKER_OPEN}\n${message}\n${DEFINITIONAL_MARKER_CLOSE}`;
+}
+function clamp01(value) {
+    if (!Number.isFinite(value))
+        return 0;
+    if (value < 0)
+        return 0;
+    if (value > 1)
+        return 1;
+    return value;
+}
+//# sourceMappingURL=intent.js.map