@crewhaus/tool-text 0.7.0

package/dist/lib/regex.js

@@ -0,0 +1,36 @@
+import { lineStarts, offsetToLineCol } from "./locate";
+/**
+ * Run a regex over text and return located matches.
+ *
+ * Always compiles with `g` so the scan advances, and a zero-width match bumps
+ * `lastIndex` by one so a pattern like `a*` terminates instead of spinning
+ * forever at the same offset.
+ */
+export function regexExtractAll(text, pattern, flags, maxMatches) {
+    const withGlobal = flags.includes("g") ? flags : `${flags}g`;
+    const re = new RegExp(pattern, withGlobal);
+    const starts = lineStarts(text);
+    const matches = [];
+    let truncated = false;
+    for (;;) {
+        const m = re.exec(text);
+        if (m === null)
+            break;
+        if (matches.length >= maxMatches) {
+            truncated = true;
+            break;
+        }
+        const { line, column } = offsetToLineCol(starts, m.index);
+        matches.push({
+            match: m[0],
+            index: m.index,
+            line,
+            column,
+            groups: { ...(m.groups ?? {}) },
+            captures: m.slice(1),
+        });
+        if (m[0].length === 0)
+            re.lastIndex += 1;
+    }
+    return { matches, truncated };
+}

package/dist/lib/similarity.d.ts

@@ -0,0 +1,33 @@
+/** Classic Levenshtein edit distance, two-row variant. */
+export declare function levenshtein(a: string, b: string): number;
+/** Edit distance expressed as a 0..1 similarity. */
+export declare function levenshteinRatio(a: string, b: string): number;
+/** Jaro-Winkler, which rewards a shared prefix — good for names and ids. */
+export declare function jaroWinkler(a: string, b: string): number;
+/** Character n-grams, the basis for trigram similarity. */
+export declare function nGrams(text: string, n: number): Set<string>;
+export declare function jaccard(a: ReadonlySet<string>, b: ReadonlySet<string>): number;
+export type SimilarityMethod = "levenshtein" | "jaro" | "trigram" | "tokenJaccard";
+export declare function similarity(a: string, b: string, method: SimilarityMethod): number;
+export type FuzzyHit = {
+    readonly candidate: string;
+    readonly score: number;
+    readonly index: number;
+};
+/** Rank candidates against a query, best first, ties broken by input order. */
+export declare function fuzzyRank(query: string, candidates: ReadonlyArray<string>, method: SimilarityMethod, minScore: number, limit: number): FuzzyHit[];
+/** Very common words carry no signal; excluding them is what makes a keyword
+ *  list readable rather than a list of "the, and, of". */
+export declare const STOP_WORDS: ReadonlySet<string>;
+export type Keyword = {
+    readonly term: string;
+    readonly count: number;
+    readonly score: number;
+};
+/**
+ * Rank terms by frequency with a length bonus, so multi-character terms beat
+ * short noise. Single-document term frequency only: there is no corpus here
+ * to compute a real inverse document frequency against, and inventing one
+ * would make the score depend on hidden state.
+ */
+export declare function extractKeywords(text: string, limit: number, minLength: number): Keyword[];

package/dist/lib/similarity.js

@@ -0,0 +1,216 @@
+import { tokenize } from "./locate";
+/** Classic Levenshtein edit distance, two-row variant. */
+export function levenshtein(a, b) {
+    if (a === b)
+        return 0;
+    if (a.length === 0)
+        return b.length;
+    if (b.length === 0)
+        return a.length;
+    let prev = Array.from({ length: b.length + 1 }, (_, i) => i);
+    for (let i = 1; i <= a.length; i++) {
+        const cur = new Array(b.length + 1);
+        cur[0] = i;
+        for (let j = 1; j <= b.length; j++) {
+            const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+            cur[j] = Math.min(cur[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost);
+        }
+        prev = cur;
+    }
+    return prev[b.length];
+}
+/** Edit distance expressed as a 0..1 similarity. */
+export function levenshteinRatio(a, b) {
+    const longest = Math.max(a.length, b.length);
+    if (longest === 0)
+        return 1;
+    return 1 - levenshtein(a, b) / longest;
+}
+/** Jaro-Winkler, which rewards a shared prefix — good for names and ids. */
+export function jaroWinkler(a, b) {
+    if (a === b)
+        return 1;
+    if (a.length === 0 || b.length === 0)
+        return 0;
+    const window = Math.max(0, Math.floor(Math.max(a.length, b.length) / 2) - 1);
+    const aFlags = new Array(a.length).fill(false);
+    const bFlags = new Array(b.length).fill(false);
+    let matches = 0;
+    for (let i = 0; i < a.length; i++) {
+        const lo = Math.max(0, i - window);
+        const hi = Math.min(b.length - 1, i + window);
+        for (let j = lo; j <= hi; j++) {
+            if (bFlags[j] === true || a[i] !== b[j])
+                continue;
+            aFlags[i] = true;
+            bFlags[j] = true;
+            matches++;
+            break;
+        }
+    }
+    if (matches === 0)
+        return 0;
+    let transpositions = 0;
+    let k = 0;
+    for (let i = 0; i < a.length; i++) {
+        if (aFlags[i] !== true)
+            continue;
+        while (bFlags[k] !== true)
+            k++;
+        if (a[i] !== b[k])
+            transpositions++;
+        k++;
+    }
+    const t = transpositions / 2;
+    const jaro = (matches / a.length + matches / b.length + (matches - t) / matches) / 3;
+    let prefix = 0;
+    while (prefix < 4 && prefix < a.length && prefix < b.length && a[prefix] === b[prefix])
+        prefix++;
+    return jaro + prefix * 0.1 * (1 - jaro);
+}
+/** Character n-grams, the basis for trigram similarity. */
+export function nGrams(text, n) {
+    const padded = ` ${text.toLowerCase()} `;
+    const out = new Set();
+    for (let i = 0; i + n <= padded.length; i++)
+        out.add(padded.slice(i, i + n));
+    return out;
+}
+export function jaccard(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    let intersection = 0;
+    for (const item of a)
+        if (b.has(item))
+            intersection++;
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+export function similarity(a, b, method) {
+    if (method === "levenshtein")
+        return levenshteinRatio(a, b);
+    if (method === "jaro")
+        return jaroWinkler(a, b);
+    if (method === "trigram")
+        return jaccard(nGrams(a, 3), nGrams(b, 3));
+    return jaccard(new Set(tokenize(a)), new Set(tokenize(b)));
+}
+/** Rank candidates against a query, best first, ties broken by input order. */
+export function fuzzyRank(query, candidates, method, minScore, limit) {
+    return candidates
+        .map((candidate, index) => ({ candidate, index, score: similarity(query, candidate, method) }))
+        .filter((h) => h.score >= minScore)
+        .sort((a, b) => b.score - a.score || a.index - b.index)
+        .slice(0, limit);
+}
+/** Very common words carry no signal; excluding them is what makes a keyword
+ *  list readable rather than a list of "the, and, of". */
+export const STOP_WORDS = new Set([
+    "a",
+    "an",
+    "and",
+    "are",
+    "as",
+    "at",
+    "be",
+    "but",
+    "by",
+    "for",
+    "from",
+    "has",
+    "have",
+    "he",
+    "her",
+    "his",
+    "if",
+    "in",
+    "into",
+    "is",
+    "it",
+    "its",
+    "of",
+    "on",
+    "or",
+    "she",
+    "that",
+    "the",
+    "their",
+    "them",
+    "then",
+    "there",
+    "these",
+    "they",
+    "this",
+    "to",
+    "was",
+    "were",
+    "which",
+    "who",
+    "will",
+    "with",
+    "you",
+    "your",
+    "we",
+    "our",
+    "i",
+    "not",
+    "no",
+    "can",
+    "could",
+    "should",
+    "would",
+    "do",
+    "does",
+    "did",
+    "been",
+    "being",
+    "had",
+    "how",
+    "what",
+    "when",
+    "where",
+    "why",
+    "all",
+    "any",
+    "each",
+    "more",
+    "most",
+    "other",
+    "some",
+    "such",
+    "than",
+    "too",
+    "very",
+    "just",
+    "also",
+    "may",
+    "might",
+    "must",
+    "one",
+    "two",
+    "get",
+    "got",
+    "make",
+    "made",
+    "use",
+]);
+/**
+ * Rank terms by frequency with a length bonus, so multi-character terms beat
+ * short noise. Single-document term frequency only: there is no corpus here
+ * to compute a real inverse document frequency against, and inventing one
+ * would make the score depend on hidden state.
+ */
+export function extractKeywords(text, limit, minLength) {
+    const counts = new Map();
+    for (const token of tokenize(text)) {
+        if (token.length < minLength)
+            continue;
+        if (STOP_WORDS.has(token))
+            continue;
+        counts.set(token, (counts.get(token) ?? 0) + 1);
+    }
+    return [...counts.entries()]
+        .map(([term, count]) => ({ term, count, score: count * Math.log2(term.length + 1) }))
+        .sort((a, b) => b.score - a.score || a.term.localeCompare(b.term))
+        .slice(0, limit);
+}

package/dist/lib/template.d.ts

@@ -0,0 +1,42 @@
+export declare class TemplateError extends Error {
+    readonly name = "TemplateError";
+}
+/** Walk a dotted path (`user.name`, `items.0.id`) through plain data. */
+export declare function lookupPath(data: unknown, path: string): unknown;
+/**
+ * Render `{{path}}` placeholders against `data`.
+ *
+ * Deliberately not a general template language: no loops, no conditionals,
+ * no partials, no arbitrary expressions. A harness needing those should
+ * build the string it wants and pass it in. What this guarantees is the part
+ * that matters — the same data and template always produce the same bytes,
+ * and under `strict` a typo in a placeholder is an error rather than an
+ * empty string silently shipped to a customer.
+ *
+ * `{{{path}}}` is accepted as a synonym for `{{path}}`; this renders text,
+ * not HTML, so there is no escaping distinction to draw.
+ */
+export declare function renderTemplateString(template: string, data: unknown, strict: boolean): {
+    text: string;
+    missing: string[];
+};
+export type ClassifyRule = {
+    readonly label: string;
+    readonly patterns: ReadonlyArray<string>;
+    readonly weight?: number;
+    readonly regex?: boolean;
+};
+export type ClassifyResult = {
+    readonly label: string | null;
+    readonly score: number;
+    readonly matched: ReadonlyArray<string>;
+    readonly scores: Readonly<Record<string, number>>;
+};
+/**
+ * Score text against operator-written rules and return the best label.
+ *
+ * Lexical only — phrase and regex matching with per-rule weights. No model
+ * and no embedding, so the same message always gets the same label, which is
+ * what makes it safe to route on.
+ */
+export declare function classifyByRules(text: string, rules: ReadonlyArray<ClassifyRule>, threshold: number, defaultLabel: string | null): ClassifyResult;

package/dist/lib/template.js

@@ -0,0 +1,88 @@
+export class TemplateError extends Error {
+    name = "TemplateError";
+}
+/** Walk a dotted path (`user.name`, `items.0.id`) through plain data. */
+export function lookupPath(data, path) {
+    if (path === ".")
+        return data;
+    let cur = data;
+    for (const part of path.split(".")) {
+        if (cur === null || cur === undefined)
+            return undefined;
+        if (Array.isArray(cur)) {
+            const i = Number(part);
+            if (!Number.isInteger(i))
+                return undefined;
+            cur = cur[i];
+            continue;
+        }
+        if (typeof cur !== "object")
+            return undefined;
+        cur = cur[part];
+    }
+    return cur;
+}
+/**
+ * Render `{{path}}` placeholders against `data`.
+ *
+ * Deliberately not a general template language: no loops, no conditionals,
+ * no partials, no arbitrary expressions. A harness needing those should
+ * build the string it wants and pass it in. What this guarantees is the part
+ * that matters — the same data and template always produce the same bytes,
+ * and under `strict` a typo in a placeholder is an error rather than an
+ * empty string silently shipped to a customer.
+ *
+ * `{{{path}}}` is accepted as a synonym for `{{path}}`; this renders text,
+ * not HTML, so there is no escaping distinction to draw.
+ */
+export function renderTemplateString(template, data, strict) {
+    const missing = [];
+    const text = template.replace(/\{\{\{?\s*([\w.]+)\s*\}?\}\}/g, (_full, path) => {
+        const value = lookupPath(data, path);
+        if (value === undefined || value === null) {
+            missing.push(path);
+            return "";
+        }
+        if (typeof value === "object")
+            return JSON.stringify(value);
+        return String(value);
+    });
+    const unique = [...new Set(missing)];
+    if (strict && unique.length > 0) {
+        throw new TemplateError(`template references ${unique.map((p) => `"${p}"`).join(", ")}, which the data does not provide`);
+    }
+    return { text, missing: unique };
+}
+/**
+ * Score text against operator-written rules and return the best label.
+ *
+ * Lexical only — phrase and regex matching with per-rule weights. No model
+ * and no embedding, so the same message always gets the same label, which is
+ * what makes it safe to route on.
+ */
+export function classifyByRules(text, rules, threshold, defaultLabel) {
+    const haystack = text.toLowerCase();
+    const scores = {};
+    const matched = [];
+    for (const rule of rules) {
+        const weight = rule.weight ?? 1;
+        let hits = 0;
+        for (const pattern of rule.patterns) {
+            const found = rule.regex === true
+                ? new RegExp(pattern, "i").test(text)
+                : haystack.includes(pattern.toLowerCase());
+            if (found) {
+                hits += 1;
+                matched.push(pattern);
+            }
+        }
+        if (hits > 0)
+            scores[rule.label] = (scores[rule.label] ?? 0) + hits * weight;
+    }
+    const ranked = Object.entries(scores).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+    const top = ranked[0];
+    if (top === undefined || top[1] < threshold) {
+        return { label: defaultLabel, score: top?.[1] ?? 0, matched, scores };
+    }
+    return { label: top[0], score: top[1], matched, scores };
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@crewhaus/tool-text",
+  "version": "0.7.0",
+  "type": "module",
+  "description": "Deterministic text tools: regex extraction, diffing, unified-diff parsing, budget truncation, log compaction, outlining, templating, rule classification, normalization, similarity, and escaping",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.7.0",
+    "@crewhaus/tool-builder": "0.7.0",
+    "@crewhaus/tool-catalog": "0.7.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@crewhaus/tool-executor": "workspace:*"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@crewhaus.ai",
+    "url": "https://crewhaus.ai"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/tool-text"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/tool-text#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": ["dist", "README.md", "LICENSE", "NOTICE"]
+}