notoken-core 1.0.0

package/dist/nlp/semantic.js

@@ -0,0 +1,419 @@
+/**
+ * Semantic NLP layer — powered by compromise.
+ *
+ * Uses compromise for:
+ * - Tokenization
+ * - POS tagging (Verb, Noun, Adjective, Preposition, etc.)
+ * - Normalization and root form extraction
+ * - Sentence/clause structure
+ *
+ * Layers on top:
+ * - Domain entity recognition (services, environments, paths)
+ * - Adjacent keyboard typo correction
+ * - Dependency parsing (SVO + prepositional)
+ * - Concept graph builder
+ * - Knowledge graph for entity relationships
+ */
+import nlp from "compromise";
+// ─── Adjacent Keyboard Map ──────────────────────────────────────────────────
+const KEYBOARD_ADJACENCY = {
+    q: ["w", "a", "s"],
+    w: ["q", "e", "a", "s", "d"],
+    e: ["w", "r", "s", "d", "f"],
+    r: ["e", "t", "d", "f", "g"],
+    t: ["r", "y", "f", "g", "h"],
+    y: ["t", "u", "g", "h", "j"],
+    u: ["y", "i", "h", "j", "k"],
+    i: ["u", "o", "j", "k", "l"],
+    o: ["i", "p", "k", "l"],
+    p: ["o", "l"],
+    a: ["q", "w", "s", "z", "x"],
+    s: ["q", "w", "e", "a", "d", "z", "x", "c"],
+    d: ["w", "e", "r", "s", "f", "x", "c", "v"],
+    f: ["e", "r", "t", "d", "g", "c", "v", "b"],
+    g: ["r", "t", "y", "f", "h", "v", "b", "n"],
+    h: ["t", "y", "u", "g", "j", "b", "n", "m"],
+    j: ["y", "u", "i", "h", "k", "n", "m"],
+    k: ["u", "i", "o", "j", "l", "m"],
+    l: ["i", "o", "p", "k"],
+    z: ["a", "s", "x"],
+    x: ["a", "s", "d", "z", "c"],
+    c: ["s", "d", "f", "x", "v"],
+    v: ["d", "f", "g", "c", "b"],
+    b: ["f", "g", "h", "v", "n"],
+    n: ["g", "h", "j", "b", "m"],
+    m: ["h", "j", "k", "n"],
+};
+export function keyboardDistance(a, b) {
+    const la = a.toLowerCase();
+    const lb = b.toLowerCase();
+    const m = la.length;
+    const n = lb.length;
+    const dp = Array.from({ length: m + 1 }, () => Array(n + 1).fill(0));
+    for (let i = 0; i <= m; i++)
+        dp[i][0] = i;
+    for (let j = 0; j <= n; j++)
+        dp[0][j] = j;
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            if (la[i - 1] === lb[j - 1]) {
+                dp[i][j] = dp[i - 1][j - 1];
+            }
+            else {
+                const isAdjacent = KEYBOARD_ADJACENCY[la[i - 1]]?.includes(lb[j - 1]);
+                const subCost = isAdjacent ? 0.5 : 1;
+                dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + subCost);
+            }
+        }
+    }
+    return dp[m][n];
+}
+export function fuzzyMatch(word, candidates, maxDistance = 2) {
+    let best = null;
+    for (const candidate of candidates) {
+        const dist = keyboardDistance(word, candidate);
+        if (dist <= maxDistance && (!best || dist < best.distance)) {
+            best = { match: candidate, distance: dist };
+        }
+    }
+    return best;
+}
+// Words that compromise misclassifies for CLI usage
+const FORCE_PREPOSITION = new Set(["to", "from", "into", "onto"]);
+// Domain verbs that compromise may tag as nouns
+const DOMAIN_VERBS = new Set([
+    "restart", "bounce", "reload", "recycle", "kick", "deploy", "release", "ship",
+    "push", "promote", "rollback", "revert", "undo", "tail", "watch", "show",
+    "check", "search", "grep", "find", "locate", "list", "kill", "stop",
+    "terminate", "copy", "move", "delete", "remove", "create", "add", "modify",
+    "ping", "connect", "login", "ssh", "stash", "commit", "stage", "fetch",
+    "merge", "rebase", "reset", "checkout", "pull", "clone", "diff", "status",
+]);
+/**
+ * Use compromise to get rich POS tagging and normalization,
+ * then overlay domain-specific entity detection and typo correction.
+ *
+ * Pre-processes input to protect path-like tokens (e.g. /var/log)
+ * from being split by compromise's tokenizer.
+ */
+export function tokenize(text, knownServices, knownEnvironments) {
+    // Pre-process: extract path-like tokens before compromise gets them
+    const pathMap = new Map();
+    let processed = text;
+    const pathRegex = /\/[a-zA-Z0-9_.\/\-]+/g;
+    let pathIdx = 0;
+    for (const match of text.matchAll(pathRegex)) {
+        const placeholder = `PATHPLACEHOLDER${pathIdx}`;
+        pathMap.set(placeholder.toLowerCase(), match[0]);
+        processed = processed.replace(match[0], placeholder);
+        pathIdx++;
+    }
+    // Also protect dotfiles like nginx.conf, app.log
+    const dotfileRegex = /\b([a-zA-Z0-9_-]+\.[a-zA-Z0-9]+)\b/g;
+    let dotIdx = 0;
+    for (const match of processed.matchAll(dotfileRegex)) {
+        // Don't protect common words with periods (e.g., "Mr.", "Dr.")
+        if (match[1].includes("/"))
+            continue;
+        const placeholder = `DOTFILE${dotIdx}`;
+        pathMap.set(placeholder.toLowerCase(), match[1]);
+        processed = processed.replace(match[1], placeholder);
+        dotIdx++;
+    }
+    const doc = nlp(processed);
+    const json = doc.json();
+    if (!json.length || !json[0].terms)
+        return [];
+    const terms = json[0].terms;
+    const tokens = [];
+    // Also get verb root forms from compromise
+    const verbs = doc.verbs().toInfinitive().json();
+    const verbRoots = new Map();
+    for (const sentence of verbs) {
+        for (const term of sentence.terms) {
+            if (term.tags.includes("Verb")) {
+                verbRoots.set(term.normal, term.text.toLowerCase());
+            }
+        }
+    }
+    for (let i = 0; i < terms.length; i++) {
+        const term = terms[i];
+        let word = term.normal || term.text.toLowerCase();
+        const compromiseTags = term.tags;
+        // Restore path placeholders
+        const restoredPath = pathMap.get(word);
+        if (restoredPath) {
+            word = restoredPath.toLowerCase();
+            tokens.push({ text: word, tag: "PATH", index: i, compromiseTags });
+            continue;
+        }
+        // Determine our tag using compromise POS + domain knowledge
+        let tag;
+        let normalized;
+        let root;
+        // Force common CLI prepositions that compromise misclassifies
+        if (FORCE_PREPOSITION.has(word)) {
+            tag = "PREP";
+        }
+        else if (word.includes("/") || (word.includes(".") && !compromiseTags.includes("Verb"))) {
+            // Domain-specific overrides first (paths, services, envs)
+            tag = "PATH";
+        }
+        else if (knownEnvironments.includes(word)) {
+            tag = "ENV";
+        }
+        else if (knownServices.includes(word)) {
+            tag = "SERVICE";
+        }
+        else if (/^\d+$/.test(word)) {
+            tag = "NUMBER";
+        }
+        else if (DOMAIN_VERBS.has(word)) {
+            // Domain verbs override compromise — "tail", "grep", etc. are verbs in our context
+            tag = "VERB";
+            root = word;
+        }
+        else if (compromiseTags.includes("Verb") || compromiseTags.includes("Infinitive") || compromiseTags.includes("Imperative")) {
+            tag = "VERB";
+            root = verbRoots.get(word) ?? word;
+        }
+        else if (compromiseTags.includes("Preposition")) {
+            tag = "PREP";
+        }
+        else if (compromiseTags.includes("Determiner")) {
+            tag = "DET";
+        }
+        else if (compromiseTags.includes("Conjunction")) {
+            tag = "CONJ";
+        }
+        else if (compromiseTags.includes("Adjective") || compromiseTags.includes("Comparable") || compromiseTags.includes("Superlative")) {
+            tag = "ADJ";
+        }
+        else if (compromiseTags.includes("Adverb")) {
+            tag = "ADV";
+        }
+        else if (compromiseTags.includes("Noun") || compromiseTags.includes("Singular") || compromiseTags.includes("Plural")) {
+            // It's a noun — check if it fuzzy-matches a service or env
+            const serviceMatch = fuzzyMatch(word, knownServices, 1.5);
+            if (serviceMatch) {
+                tag = "SERVICE";
+                normalized = serviceMatch.match;
+            }
+            else {
+                const envMatch = fuzzyMatch(word, knownEnvironments, 1.5);
+                if (envMatch) {
+                    tag = "ENV";
+                    normalized = envMatch.match;
+                }
+                else {
+                    tag = "NOUN";
+                }
+            }
+        }
+        else {
+            // Unknown POS — try fuzzy matching
+            const serviceMatch = fuzzyMatch(word, knownServices, 1.5);
+            if (serviceMatch) {
+                tag = "SERVICE";
+                normalized = serviceMatch.match;
+            }
+            else {
+                const envMatch = fuzzyMatch(word, knownEnvironments, 1.5);
+                if (envMatch) {
+                    tag = "ENV";
+                    normalized = envMatch.match;
+                }
+                else {
+                    tag = "UNKNOWN";
+                }
+            }
+        }
+        tokens.push({
+            text: word,
+            tag,
+            index: i,
+            normalized,
+            compromiseTags,
+            root,
+        });
+    }
+    return tokens;
+}
+export function parseDependencies(tokens) {
+    const deps = [];
+    const verb = tokens.find((t) => t.tag === "VERB");
+    if (!verb)
+        return deps;
+    const afterVerb = tokens.filter((t) => t.index > verb.index);
+    const beforeVerb = tokens.filter((t) => t.index < verb.index);
+    for (const t of beforeVerb) {
+        if (t.tag === "NOUN" || t.tag === "SERVICE") {
+            deps.push({ head: verb, dependent: t, relation: "subject" });
+        }
+    }
+    for (let i = 0; i < afterVerb.length; i++) {
+        const token = afterVerb[i];
+        const next = afterVerb[i + 1];
+        if (token.tag === "PREP" && next) {
+            if (token.text === "to" || token.text === "into") {
+                deps.push({ head: verb, dependent: next, relation: "destination" });
+                i++;
+            }
+            else if (token.text === "from") {
+                deps.push({ head: verb, dependent: next, relation: "source" });
+                i++;
+            }
+            else if (token.text === "on" || token.text === "in" || token.text === "at") {
+                deps.push({ head: verb, dependent: next, relation: "location" });
+                i++;
+            }
+        }
+        else if (token.tag === "SERVICE" || token.tag === "NOUN" || token.tag === "PATH") {
+            deps.push({ head: verb, dependent: token, relation: "object" });
+        }
+        else if (token.tag === "NUMBER") {
+            deps.push({ head: verb, dependent: token, relation: "quantity" });
+        }
+        else if (token.tag === "ADJ" && next) {
+            deps.push({ head: next, dependent: token, relation: "modifier" });
+        }
+    }
+    return deps;
+}
+export function buildConceptGraph(tokens, deps) {
+    const nodes = [];
+    const edges = [];
+    const seen = new Set();
+    function addNode(token, type) {
+        const id = `${token.tag}_${token.normalized ?? token.text}`;
+        if (!seen.has(id)) {
+            seen.add(id);
+            nodes.push({
+                id,
+                label: token.normalized ?? token.text,
+                type,
+                aliases: token.normalized ? [token.text] : [],
+            });
+        }
+        return id;
+    }
+    const verb = tokens.find((t) => t.tag === "VERB");
+    if (verb)
+        addNode(verb, "action");
+    for (const token of tokens) {
+        if (token.tag === "SERVICE" || token.tag === "NOUN")
+            addNode(token, "entity");
+        if (token.tag === "ENV")
+            addNode(token, "location");
+        if (token.tag === "PATH")
+            addNode(token, "entity");
+        if (token.tag === "NUMBER")
+            addNode(token, "quantity");
+        if (token.tag === "ADJ")
+            addNode(token, "property");
+    }
+    for (const dep of deps) {
+        const fromId = `${dep.head.tag}_${dep.head.normalized ?? dep.head.text}`;
+        const toId = `${dep.dependent.tag}_${dep.dependent.normalized ?? dep.dependent.text}`;
+        let relation;
+        switch (dep.relation) {
+            case "object":
+            case "subject":
+                relation = "acts_on";
+                break;
+            case "location":
+                relation = "located_at";
+                break;
+            case "destination":
+                relation = "destination";
+                break;
+            case "source":
+                relation = "source";
+                break;
+            case "quantity":
+                relation = "quantity_of";
+                break;
+            case "modifier":
+                relation = "has_property";
+                break;
+        }
+        edges.push({ from: fromId, to: toId, relation });
+    }
+    return { nodes, edges };
+}
+// ─── High-level NLP Utilities ────────────────────────────────────────────────
+/**
+ * Extract verbs in their root/infinitive form using compromise.
+ */
+export function extractVerbs(text) {
+    const doc = nlp(text);
+    const verbs = doc.verbs().toInfinitive();
+    return verbs.out("array");
+}
+/**
+ * Extract nouns using compromise.
+ */
+export function extractNouns(text) {
+    const doc = nlp(text);
+    return doc.nouns().out("array");
+}
+/**
+ * Normalize text — lowercase, expand contractions, normalize whitespace.
+ */
+export function normalizeText(text) {
+    const doc = nlp(text);
+    // Expand contractions: don't → do not
+    doc.contractions().expand();
+    return doc.text("normal");
+}
+/**
+ * Get sentence structure analysis from compromise.
+ */
+export function analyzeSentence(text) {
+    const doc = nlp(text);
+    return {
+        verbs: doc.verbs().toInfinitive().out("array"),
+        nouns: doc.nouns().out("array"),
+        adjectives: doc.adjectives().out("array"),
+        prepositions: doc.match("#Preposition").out("array"),
+        isQuestion: doc.questions().length > 0,
+        isNegative: doc.has("#Negative"),
+        tense: detectTense(doc),
+    };
+}
+function detectTense(doc) {
+    if (doc.has("#PastTense"))
+        return "past";
+    if (doc.has("#FutureTense"))
+        return "future";
+    if (doc.has("#Imperative"))
+        return "imperative";
+    return "present";
+}
+export function semanticParse(text, knownServices, knownEnvironments) {
+    const tokens = tokenize(text, knownServices, knownEnvironments);
+    const dependencies = parseDependencies(tokens);
+    const graph = buildConceptGraph(tokens, dependencies);
+    const sentence = analyzeSentence(text);
+    const actionToken = tokens.find((t) => t.tag === "VERB");
+    const entities = tokens
+        .filter((t) => ["SERVICE", "NOUN", "PATH"].includes(t.tag))
+        .map((t) => ({ text: t.text, type: t.tag, normalized: t.normalized }));
+    const locationDep = dependencies.find((d) => d.relation === "location");
+    const destDep = dependencies.find((d) => d.relation === "destination");
+    const srcDep = dependencies.find((d) => d.relation === "source");
+    const qtyDep = dependencies.find((d) => d.relation === "quantity");
+    return {
+        tokens,
+        dependencies,
+        graph,
+        action: actionToken?.normalized ?? actionToken?.text,
+        actionRoot: actionToken?.root ?? sentence.verbs[0],
+        entities,
+        location: locationDep?.dependent.normalized ?? locationDep?.dependent.text,
+        destination: destDep?.dependent.normalized ?? destDep?.dependent.text,
+        source: srcDep?.dependent.normalized ?? srcDep?.dependent.text,
+        quantity: qtyDep ? Number(qtyDep.dependent.text) : undefined,
+        sentence,
+    };
+}

package/dist/nlp/uncertainty.d.ts

@@ -0,0 +1,42 @@
+import type { Token } from "./semantic.js";
+import type { DynamicIntent } from "../types/intent.js";
+import type { UncertaintyReport } from "../conversation/store.js";
+export interface UncertaintyEntry {
+    timestamp: string;
+    rawText: string;
+    intent: string;
+    overallConfidence: number;
+    unknownTokens: string[];
+    lowConfidenceFields: Array<{
+        field: string;
+        value: string;
+        confidence: number;
+    }>;
+    /** Which parts of the sentence had no coverage */
+    uncoveredSpans: string[];
+}
+/**
+ * Analyze a parse result for uncertainty.
+ *
+ * Returns a report of:
+ * - Tokens that weren't classified (UNKNOWN tag)
+ * - Fields that were filled with low confidence
+ * - Parts of the sentence that weren't used by any field
+ */
+export declare function analyzeUncertainty(rawText: string, tokens: Token[], intent: DynamicIntent): UncertaintyReport;
+/**
+ * Get the uncovered spans — parts of the sentence we didn't understand.
+ */
+export declare function getUncoveredSpans(rawText: string, tokens: Token[]): string[];
+/**
+ * Log uncertainty for later analysis by the auto-learning system.
+ */
+export declare function logUncertainty(entry: UncertaintyEntry): void;
+export declare function loadUncertaintyLog(): UncertaintyEntry[];
+/**
+ * Get a summary of most common uncertain tokens across all logged entries.
+ */
+export declare function getUncertaintySummary(): Array<{
+    token: string;
+    count: number;
+}>;

package/dist/nlp/uncertainty.js

@@ -0,0 +1,103 @@
+import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import { resolve } from "node:path";
+import { LOG_DIR } from "../utils/paths.js";
+const UNCERTAINTY_LOG = resolve(LOG_DIR, "uncertainty.json");
+/**
+ * Analyze a parse result for uncertainty.
+ *
+ * Returns a report of:
+ * - Tokens that weren't classified (UNKNOWN tag)
+ * - Fields that were filled with low confidence
+ * - Parts of the sentence that weren't used by any field
+ */
+export function analyzeUncertainty(rawText, tokens, intent) {
+    const unknownTokens = tokens
+        .filter((t) => t.tag === "UNKNOWN")
+        .map((t) => t.text);
+    // Find tokens that weren't consumed by any field
+    const usedWords = new Set();
+    for (const value of Object.values(intent.fields)) {
+        if (typeof value === "string") {
+            for (const word of value.toLowerCase().split(/\s+/)) {
+                usedWords.add(word);
+            }
+        }
+    }
+    const uncoveredTokens = tokens.filter((t) => t.tag !== "DET" &&
+        t.tag !== "PREP" &&
+        t.tag !== "CONJ" &&
+        !usedWords.has(t.text) &&
+        t.text !== intent.intent.split(".")[0] // Don't count the intent verb as uncovered
+    );
+    // Estimate field confidence based on extraction method
+    const lowConfidenceFields = [];
+    for (const [field, value] of Object.entries(intent.fields)) {
+        if (typeof value !== "string" || !value)
+            continue;
+        // Fields filled by defaults are lower confidence
+        const wasExplicit = rawText.toLowerCase().includes(value.toLowerCase());
+        if (!wasExplicit) {
+            lowConfidenceFields.push({ field, value, confidence: 0.4 });
+        }
+    }
+    return {
+        unknownTokens,
+        lowConfidenceFields,
+        overallConfidence: intent.confidence,
+    };
+}
+/**
+ * Get the uncovered spans — parts of the sentence we didn't understand.
+ */
+export function getUncoveredSpans(rawText, tokens) {
+    const spans = [];
+    let currentSpan = [];
+    for (const token of tokens) {
+        if (token.tag === "UNKNOWN") {
+            currentSpan.push(token.text);
+        }
+        else {
+            if (currentSpan.length > 0) {
+                spans.push(currentSpan.join(" "));
+                currentSpan = [];
+            }
+        }
+    }
+    if (currentSpan.length > 0) {
+        spans.push(currentSpan.join(" "));
+    }
+    return spans;
+}
+/**
+ * Log uncertainty for later analysis by the auto-learning system.
+ */
+export function logUncertainty(entry) {
+    if (!existsSync(LOG_DIR))
+        mkdirSync(LOG_DIR, { recursive: true });
+    const existing = loadUncertaintyLog();
+    existing.push(entry);
+    // Keep last 500 entries
+    if (existing.length > 500)
+        existing.splice(0, existing.length - 500);
+    writeFileSync(UNCERTAINTY_LOG, JSON.stringify(existing, null, 2));
+}
+export function loadUncertaintyLog() {
+    if (!existsSync(UNCERTAINTY_LOG))
+        return [];
+    return JSON.parse(readFileSync(UNCERTAINTY_LOG, "utf-8"));
+}
+/**
+ * Get a summary of most common uncertain tokens across all logged entries.
+ */
+export function getUncertaintySummary() {
+    const log = loadUncertaintyLog();
+    const counts = new Map();
+    for (const entry of log) {
+        for (const token of entry.unknownTokens) {
+            counts.set(token, (counts.get(token) ?? 0) + 1);
+        }
+    }
+    return Array.from(counts.entries())
+        .map(([token, count]) => ({ token, count }))
+        .sort((a, b) => b.count - a.count);
+}

package/dist/parsers/apacheParser.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Apache config parser.
+ *
+ * Parses httpd.conf / apache2.conf and vhost configs.
+ * Handles: <VirtualHost>, <Directory>, <Location>, directives.
+ */
+export interface ApacheDirective {
+    name: string;
+    value: string;
+    line: number;
+}
+export interface ApacheBlock {
+    tag: string;
+    args: string;
+    directives: ApacheDirective[];
+    blocks: ApacheBlock[];
+    line: number;
+}
+export interface ApacheConfig {
+    directives: ApacheDirective[];
+    blocks: ApacheBlock[];
+    vhosts: ApacheVHost[];
+}
+export interface ApacheVHost {
+    address: string;
+    serverName?: string;
+    serverAlias: string[];
+    documentRoot?: string;
+    errorLog?: string;
+    customLog?: string;
+    ssl: boolean;
+    sslCertificate?: string;
+    sslCertificateKey?: string;
+    locations: Array<{
+        path: string;
+        directives: ApacheDirective[];
+    }>;
+    directories: Array<{
+        path: string;
+        directives: ApacheDirective[];
+    }>;
+}
+/**
+ * Parse Apache config content.
+ */
+export declare function parseApache(content: string): ApacheConfig;
+/**
+ * Format an Apache config summary for display.
+ */
+export declare function formatApacheSummary(config: ApacheConfig): string;