notoken-core 1.0.0

package/dist/nlp/multiClassifier.js

@@ -0,0 +1,181 @@
+import { loadIntents, loadRules } from "../utils/config.js";
+import { semanticParse, fuzzyMatch } from "./semantic.js";
+import { parseByRules } from "./ruleParser.js";
+const CLASSIFIER_WEIGHTS = {
+    synonym: 1.0,
+    semantic: 0.8,
+    context: 0.6,
+    fuzzy: 0.5,
+};
+/**
+ * Run all classifiers and merge results.
+ */
+export function classifyMulti(rawText, recentIntents) {
+    const votes = [];
+    // 1. Synonym classifier (existing rule parser)
+    votes.push(...classifySynonym(rawText));
+    // 2. Semantic classifier (compromise-powered)
+    votes.push(...classifySemantic(rawText));
+    // 3. Context classifier (recent history)
+    if (recentIntents && recentIntents.length > 0) {
+        votes.push(...classifyContext(rawText, recentIntents));
+    }
+    // 4. Fuzzy classifier (keyboard distance)
+    votes.push(...classifyFuzzy(rawText));
+    // Merge votes into weighted scores
+    const scoreMap = new Map();
+    for (const vote of votes) {
+        const weight = CLASSIFIER_WEIGHTS[vote.classifier] ?? 1.0;
+        const existing = scoreMap.get(vote.intent) ?? { total: 0, count: 0 };
+        existing.total += vote.confidence * weight;
+        existing.count += 1;
+        scoreMap.set(vote.intent, existing);
+    }
+    const scores = Array.from(scoreMap.entries())
+        .map(([intent, { total, count }]) => ({
+        intent,
+        score: total / count,
+        votes: count,
+    }))
+        .sort((a, b) => b.score - a.score);
+    const best = scores[0] ?? null;
+    const second = scores[1];
+    const ambiguous = !!(best && second && best.score - second.score < 0.15);
+    return {
+        votes,
+        scores: scores.map((s) => ({ intent: s.intent, score: Math.round(s.score * 100) / 100, votes: s.votes })),
+        best: best ? { intent: best.intent, score: Math.round(best.score * 100) / 100 } : null,
+        ambiguous,
+    };
+}
+// ─── Individual Classifiers ──────────────────────────────────────────────────
+function classifySynonym(rawText) {
+    const result = parseByRules(rawText);
+    if (!result || result.intent === "unknown")
+        return [];
+    return [{
+            classifier: "synonym",
+            intent: result.intent,
+            confidence: result.confidence,
+            reason: "Matched synonym in rules",
+        }];
+}
+function classifySemantic(rawText) {
+    const rules = loadRules();
+    const intents = loadIntents();
+    const services = Object.keys(rules.serviceAliases);
+    const envs = Object.keys(rules.environmentAliases);
+    const parse = semanticParse(rawText, services, envs);
+    const votes = [];
+    if (!parse.action)
+        return votes;
+    // Match action verb to intent
+    for (const def of intents) {
+        const actionScore = scoreActionMatch(parse.action, def);
+        if (actionScore > 0) {
+            // Boost if entities also match expected fields
+            const entityBoost = scoreEntityMatch(parse, def);
+            const confidence = Math.min(0.95, actionScore * 0.6 + entityBoost * 0.4);
+            votes.push({
+                classifier: "semantic",
+                intent: def.name,
+                confidence,
+                reason: `Verb "${parse.action}" matches ${def.name}`,
+            });
+        }
+    }
+    return votes;
+}
+function classifyContext(rawText, recentIntents) {
+    const votes = [];
+    // If recent intents are heavily weighted toward one area, boost it slightly
+    const intentCounts = new Map();
+    for (const i of recentIntents) {
+        intentCounts.set(i, (intentCounts.get(i) ?? 0) + 1);
+    }
+    // Get the domain prefix of recent intents (e.g., "service", "git", "logs")
+    const domainCounts = new Map();
+    for (const [intent, count] of intentCounts) {
+        const domain = intent.split(".")[0];
+        domainCounts.set(domain, (domainCounts.get(domain) ?? 0) + count);
+    }
+    // If the raw text matches a recent domain, give a small boost
+    const intents = loadIntents();
+    for (const def of intents) {
+        const domain = def.name.split(".")[0];
+        const domainFreq = domainCounts.get(domain) ?? 0;
+        if (domainFreq > 0) {
+            // Check if any synonym partially matches
+            const hasPartialMatch = def.synonyms.some((s) => rawText.toLowerCase().includes(s.split(" ")[0]));
+            if (hasPartialMatch) {
+                votes.push({
+                    classifier: "context",
+                    intent: def.name,
+                    confidence: Math.min(0.7, 0.3 + domainFreq * 0.1),
+                    reason: `Recent context favors ${domain} domain`,
+                });
+            }
+        }
+    }
+    return votes;
+}
+function classifyFuzzy(rawText) {
+    const intents = loadIntents();
+    const votes = [];
+    const words = rawText.toLowerCase().split(/\s+/);
+    for (const def of intents) {
+        for (const word of words) {
+            for (const synonym of def.synonyms) {
+                const synonymWords = synonym.split(" ");
+                for (const sw of synonymWords) {
+                    if (sw.length < 3)
+                        continue;
+                    const match = fuzzyMatch(word, [sw], 1.5);
+                    if (match && match.distance > 0 && match.distance <= 1.5) {
+                        votes.push({
+                            classifier: "fuzzy",
+                            intent: def.name,
+                            confidence: Math.max(0.3, 0.7 - match.distance * 0.3),
+                            reason: `Fuzzy: "${word}" ≈ "${sw}" (dist: ${match.distance})`,
+                        });
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    return votes;
+}
+// ─── Scoring Helpers ─────────────────────────────────────────────────────────
+function scoreActionMatch(action, def) {
+    const actionLower = action.toLowerCase();
+    // Direct synonym match
+    if (def.synonyms.some((s) => s.includes(actionLower)))
+        return 0.9;
+    // Check if action is part of the intent name
+    if (def.name.includes(actionLower))
+        return 0.7;
+    // Check examples
+    if (def.examples.some((e) => e.toLowerCase().includes(actionLower)))
+        return 0.5;
+    return 0;
+}
+function scoreEntityMatch(parse, def) {
+    let matches = 0;
+    let total = Object.keys(def.fields).length;
+    if (total === 0)
+        return 0.5;
+    for (const [fieldName, fieldDef] of Object.entries(def.fields)) {
+        if (fieldDef.type === "environment" && parse.location)
+            matches++;
+        if (fieldDef.type === "service" && parse.entities.some((e) => e.type === "SERVICE"))
+            matches++;
+        if (fieldDef.type === "number" && parse.quantity !== undefined)
+            matches++;
+        if (fieldName === "destination" && parse.destination)
+            matches++;
+        if (fieldName === "source" && parse.source)
+            matches++;
+    }
+    return matches / total;
+}

package/dist/nlp/parseIntent.d.ts

@@ -0,0 +1,2 @@
+import type { ParsedCommand } from "../types/intent.js";
+export declare function parseIntent(rawText: string): Promise<ParsedCommand>;

package/dist/nlp/parseIntent.js

@@ -0,0 +1,34 @@
+import { parseByRules } from "./ruleParser.js";
+import { parseByLLM } from "./llmParser.js";
+import { disambiguate } from "./disambiguate.js";
+import { logFailure } from "../utils/logger.js";
+export async function parseIntent(rawText) {
+    // Stage 1: deterministic rule parser
+    const ruleResult = parseByRules(rawText);
+    if (ruleResult && ruleResult.confidence >= 0.7) {
+        return disambiguate(ruleResult);
+    }
+    // Stage 2: LLM fallback
+    const llmResult = await parseByLLM(rawText);
+    if (llmResult && llmResult.confidence >= 0.5) {
+        return disambiguate(llmResult);
+    }
+    // Stage 3: if rule parser had a low-confidence result, use it anyway
+    if (ruleResult) {
+        return disambiguate(ruleResult);
+    }
+    // Stage 4: unknown — log the failure for auto-learning
+    logFailure({
+        rawText,
+        timestamp: new Date().toISOString(),
+        parsedIntent: null,
+        confidence: 0,
+        error: "No parser matched",
+    });
+    return disambiguate({
+        intent: "unknown",
+        rawText,
+        confidence: 0,
+        fields: { reason: "No supported intent matched" },
+    });
+}

package/dist/nlp/ruleParser.d.ts

@@ -0,0 +1,2 @@
+import type { DynamicIntent } from "../types/intent.js";
+export declare function parseByRules(rawText: string): DynamicIntent | null;

package/dist/nlp/ruleParser.js

@@ -0,0 +1,234 @@
+import { loadRules, loadIntents } from "../utils/config.js";
+import { normalizePath } from "../utils/wslPaths.js";
+export function parseByRules(rawText) {
+    const rules = loadRules();
+    const intents = loadIntents();
+    const text = rawText.trim().toLowerCase();
+    // Match intent by synonyms defined in intents.json
+    const matched = matchIntent(text, intents);
+    if (!matched)
+        return null;
+    const { def, matchedPhrase } = matched;
+    // Extract fields based on the intent's field definitions
+    const fields = {};
+    let allRequiredFound = true;
+    // First pass: extract typed fields (environment, service, number, branch)
+    for (const [fieldName, fieldDef] of Object.entries(def.fields)) {
+        let value = undefined;
+        switch (fieldDef.type) {
+            case "environment":
+                value = extractEnvironment(text, rules.environmentAliases);
+                break;
+            case "service":
+                value = extractService(text, rules.serviceAliases, matchedPhrase);
+                break;
+            case "number":
+                value = extractNumber(text);
+                break;
+            case "branch":
+                value = extractBranch(text);
+                break;
+        }
+        if (value !== undefined) {
+            fields[fieldName] = value;
+        }
+    }
+    // Second pass: extract string fields using context-aware extraction
+    const stringFields = Object.entries(def.fields).filter(([, fd]) => fd.type === "string");
+    if (stringFields.length > 0) {
+        const extracted = extractStringFields(rawText, text, matchedPhrase, stringFields.map(([n]) => n), fields);
+        for (const [k, v] of Object.entries(extracted)) {
+            if (v !== undefined)
+                fields[k] = v;
+        }
+    }
+    // Apply defaults for missing fields
+    for (const [fieldName, fieldDef] of Object.entries(def.fields)) {
+        if (fields[fieldName] === undefined && fieldDef.default !== undefined) {
+            fields[fieldName] = fieldDef.default;
+        }
+        if (fields[fieldName] === undefined && fieldDef.required) {
+            allRequiredFound = false;
+        }
+    }
+    // Resolve logPaths if the intent uses them
+    if (def.logPaths && fields.service) {
+        const logPath = def.logPaths[fields.service];
+        if (logPath)
+            fields.logPath = logPath;
+    }
+    // Confidence scoring
+    let confidence = 0.7;
+    if (allRequiredFound)
+        confidence += 0.15;
+    if (matchedPhrase.length > 4)
+        confidence += 0.05;
+    confidence = Math.min(confidence, 0.95);
+    return {
+        intent: def.name,
+        confidence,
+        rawText,
+        fields,
+    };
+}
+/**
+ * Extract string fields from natural language using preposition patterns.
+ *
+ * Handles patterns like:
+ *   "copy nginx.conf to /root"       → source=nginx.conf, destination=/root
+ *   "move app.log to /backup"        → source=app.log, destination=/backup
+ *   "grep error in /var/log"         → query=error, path=/var/log
+ *   "find *.conf in /etc"            → pattern=*.conf, path=/etc
+ */
+function extractStringFields(rawText, lowerText, matchedPhrase, fieldNames, alreadyExtracted) {
+    const result = {};
+    // Remove the matched intent phrase and known extracted values from text
+    let remaining = lowerText.replace(matchedPhrase, " ");
+    for (const [, v] of Object.entries(alreadyExtracted)) {
+        if (typeof v === "string") {
+            remaining = remaining.replace(v, " ");
+        }
+    }
+    remaining = remaining.replace(/\s+/g, " ").trim();
+    // Check for quoted strings first
+    const quoted = rawText.match(/["']([^"']+)["']/g);
+    if (quoted) {
+        for (let i = 0; i < Math.min(quoted.length, fieldNames.length); i++) {
+            result[fieldNames[i]] = quoted[i].replace(/["']/g, "");
+        }
+        return result;
+    }
+    // For source/destination patterns (copy X to Y, move X to Y)
+    if (fieldNames.includes("source") && fieldNames.includes("destination")) {
+        const toMatch = remaining.match(/(.+?)\s+to\s+(.+)/);
+        if (toMatch) {
+            result.source = extractPathOrFilename(toMatch[1].trim());
+            result.destination = extractPathOrFilename(toMatch[2].trim());
+            return result;
+        }
+    }
+    // For query/path patterns (grep X in Y, search X in Y, Y for X)
+    if (fieldNames.includes("query") && fieldNames.includes("path")) {
+        // "X in Y"
+        const inMatch = remaining.match(/(.+?)\s+in\s+(.+)/);
+        if (inMatch) {
+            result.query = inMatch[1].trim().replace(/^(for|the)\s+/, "");
+            result.path = extractPathOrFilename(inMatch[2].trim());
+            return result;
+        }
+        // "Y for X" (path first, query second)
+        const forMatch = remaining.match(/(.+?)\s+for\s+(.+)/);
+        if (forMatch) {
+            const left = forMatch[1].trim();
+            const right = forMatch[2].trim();
+            // If left looks like a path, it's path+query. Otherwise query+path.
+            if (left.includes("/") || left.includes(".")) {
+                result.path = extractPathOrFilename(left);
+                result.query = right.replace(/^(the)\s+/, "");
+            }
+            else {
+                result.query = left.replace(/^(the)\s+/, "");
+                result.path = extractPathOrFilename(right);
+            }
+            return result;
+        }
+        // Split by path-like token: anything with / or . is path, rest is query
+        const words = remaining.split(/\s+/).filter((w) => !isStopWord(w));
+        const pathWord = words.find((w) => w.includes("/") || (w.includes(".") && w.length > 3));
+        if (pathWord) {
+            result.path = pathWord;
+            result.query = words.filter((w) => w !== pathWord).join(" ") || undefined;
+            return result;
+        }
+        if (words.length > 0) {
+            result.query = words[0];
+        }
+        return result;
+    }
+    // For pattern/path patterns (find X in Y)
+    if (fieldNames.includes("pattern") && fieldNames.includes("path")) {
+        const inMatch = remaining.match(/(.+?)\s+in\s+(.+)/);
+        if (inMatch) {
+            result.pattern = extractPathOrFilename(inMatch[1].trim());
+            result.path = extractPathOrFilename(inMatch[2].trim());
+            return result;
+        }
+    }
+    // For single target field (delete X, kill X)
+    if (fieldNames.length === 1) {
+        const words = remaining.split(/\s+/).filter((w) => !isStopWord(w));
+        const pathLike = words.find((w) => w.includes("/") || w.includes("."));
+        result[fieldNames[0]] = pathLike ?? words[0];
+        return result;
+    }
+    // Generic: assign remaining words to fields in order
+    const words = remaining.split(/\s+/).filter((w) => !isStopWord(w));
+    for (let i = 0; i < Math.min(words.length, fieldNames.length); i++) {
+        result[fieldNames[i]] = words[i];
+    }
+    return result;
+}
+function extractPathOrFilename(text) {
+    const cleaned = text.replace(/^(the|this|that|a|an|file|directory|dir|folder)\s+/gi, "").trim();
+    const words = cleaned.split(/\s+/);
+    // Find the most path-like word (Linux or Windows paths)
+    const pathWord = words.find((w) => w.includes("/") || w.includes("\\") || w.includes(".") || /^[A-Za-z]:/.test(w));
+    const result = pathWord ?? words[0] ?? cleaned;
+    // Normalize Windows paths to Linux in WSL
+    return normalizePath(result);
+}
+function isStopWord(word) {
+    return ["the", "a", "an", "this", "that", "on", "in", "at", "for", "from", "with", "of", "file", "files"].includes(word);
+}
+function matchIntent(text, intents) {
+    let best = null;
+    for (const def of intents) {
+        for (const phrase of def.synonyms) {
+            if (text.includes(phrase)) {
+                if (!best || phrase.length > best.length) {
+                    best = { def, matchedPhrase: phrase, length: phrase.length };
+                }
+            }
+        }
+    }
+    return best ? { def: best.def, matchedPhrase: best.matchedPhrase } : null;
+}
+function extractEnvironment(text, aliases) {
+    for (const [canonical, aliasList] of Object.entries(aliases)) {
+        for (const alias of aliasList) {
+            const pattern = new RegExp(`\\b${escapeRegex(alias)}\\b`);
+            if (pattern.test(text))
+                return canonical;
+        }
+    }
+    return undefined;
+}
+function extractService(text, aliases, intentPhrase) {
+    const cleaned = text.replace(intentPhrase, " ").trim();
+    for (const [canonical, aliasList] of Object.entries(aliases)) {
+        for (const alias of aliasList) {
+            const pattern = new RegExp(`\\b${escapeRegex(alias)}\\b`);
+            if (pattern.test(cleaned))
+                return canonical;
+        }
+    }
+    for (const [canonical, aliasList] of Object.entries(aliases)) {
+        for (const alias of aliasList) {
+            const pattern = new RegExp(`\\b${escapeRegex(alias)}\\b`);
+            if (pattern.test(text))
+                return canonical;
+        }
+    }
+    return undefined;
+}
+function extractNumber(text) {
+    const match = text.match(/\b(\d+)\b/);
+    return match ? Number(match[1]) : undefined;
+}
+function extractBranch(text) {
+    const match = text.match(/\b(main|master|develop|release\/[a-z0-9._-]+)\b/);
+    return match?.[1];
+}
+function escapeRegex(str) {
+    return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/dist/nlp/semantic.d.ts

@@ -0,0 +1,104 @@
+/**
+ * Semantic NLP layer — powered by compromise.
+ *
+ * Uses compromise for:
+ * - Tokenization
+ * - POS tagging (Verb, Noun, Adjective, Preposition, etc.)
+ * - Normalization and root form extraction
+ * - Sentence/clause structure
+ *
+ * Layers on top:
+ * - Domain entity recognition (services, environments, paths)
+ * - Adjacent keyboard typo correction
+ * - Dependency parsing (SVO + prepositional)
+ * - Concept graph builder
+ * - Knowledge graph for entity relationships
+ */
+export declare function keyboardDistance(a: string, b: string): number;
+export declare function fuzzyMatch(word: string, candidates: string[], maxDistance?: number): {
+    match: string;
+    distance: number;
+} | null;
+export type TokenTag = "VERB" | "NOUN" | "PATH" | "ENV" | "SERVICE" | "NUMBER" | "PREP" | "DET" | "ADJ" | "ADV" | "CONJ" | "UNKNOWN";
+export interface Token {
+    text: string;
+    tag: TokenTag;
+    index: number;
+    normalized?: string;
+    /** POS tags from compromise */
+    compromiseTags?: string[];
+    /** Root/infinitive form from compromise */
+    root?: string;
+}
+/**
+ * Use compromise to get rich POS tagging and normalization,
+ * then overlay domain-specific entity detection and typo correction.
+ *
+ * Pre-processes input to protect path-like tokens (e.g. /var/log)
+ * from being split by compromise's tokenizer.
+ */
+export declare function tokenize(text: string, knownServices: string[], knownEnvironments: string[]): Token[];
+export interface Dependency {
+    head: Token;
+    dependent: Token;
+    relation: "subject" | "object" | "modifier" | "location" | "destination" | "source" | "quantity";
+}
+export declare function parseDependencies(tokens: Token[]): Dependency[];
+export interface ConceptNode {
+    id: string;
+    label: string;
+    type: "action" | "entity" | "property" | "location" | "quantity";
+    aliases: string[];
+}
+export interface ConceptEdge {
+    from: string;
+    to: string;
+    relation: "acts_on" | "located_at" | "has_property" | "quantity_of" | "destination" | "source";
+}
+export interface ConceptGraph {
+    nodes: ConceptNode[];
+    edges: ConceptEdge[];
+}
+export declare function buildConceptGraph(tokens: Token[], deps: Dependency[]): ConceptGraph;
+/**
+ * Extract verbs in their root/infinitive form using compromise.
+ */
+export declare function extractVerbs(text: string): string[];
+/**
+ * Extract nouns using compromise.
+ */
+export declare function extractNouns(text: string): string[];
+/**
+ * Normalize text — lowercase, expand contractions, normalize whitespace.
+ */
+export declare function normalizeText(text: string): string;
+/**
+ * Get sentence structure analysis from compromise.
+ */
+export declare function analyzeSentence(text: string): {
+    verbs: string[];
+    nouns: string[];
+    adjectives: string[];
+    prepositions: string[];
+    isQuestion: boolean;
+    isNegative: boolean;
+    tense: string;
+};
+export interface SemanticParse {
+    tokens: Token[];
+    dependencies: Dependency[];
+    graph: ConceptGraph;
+    action?: string;
+    actionRoot?: string;
+    entities: Array<{
+        text: string;
+        type: string;
+        normalized?: string;
+    }>;
+    location?: string;
+    destination?: string;
+    source?: string;
+    quantity?: number;
+    sentence: ReturnType<typeof analyzeSentence>;
+}
+export declare function semanticParse(text: string, knownServices: string[], knownEnvironments: string[]): SemanticParse;