@inflexa-ai/harness 0.28.1 → 0.29.0

package/dist/input-scan/templates.js

@@ -0,0 +1,369 @@
+/**
+ * Stage 3 — name-template mining within one directory context.
+ *
+ * Literal versus slot is decided by per-position distinct counts: a position whose
+ * token is the same in every member is literal text, and one that varies is a slot.
+ * Groups that differ in exactly one position, or only in extension, merge — variance
+ * inside a set is a property of the set, never a partition key.
+ *
+ * Proliferation is bounded three ways, because a template per file is the same failure
+ * as no templates at all: a prefix trie claims residual singletons that share literal
+ * text, a family collapse claims singletons that share only a token-kind sequence
+ * (one template for the family costs less than one template each), and a catch-all
+ * claims the rest where the repetition across directories is itself the structure.
+ */
+import { DATE_SYMBOL, DIGIT_SYMBOL, ID_SYMBOL, classifyValues, commonAffixes, isDateToken, maskAffix, skeletonOf, tokenizeName } from "./tokens.js";
+import { MAX_SLOT_SAMPLE_VALUES, MIN_AFFIX_LENGTH, MIN_CATCH_ALL_MEMBERS, MIN_FAMILY_SET_MEMBERS, MIN_PREFIX_SET_MEMBERS } from "./tuning.js";
+function toItem(file) {
+    const { tokens, suffix } = tokenizeName(file.name);
+    return { file, tokens, suffix, skeleton: skeletonOf(tokens) };
+}
+function newGroup(kind, seed) {
+    return { kind, tokenKinds: [], prefixTokens: [], suffixes: new Map(), items: [], ...seed };
+}
+function firstPass(items) {
+    const groups = new Map();
+    for (const item of items) {
+        const key = `${item.skeleton} ${item.suffix}`;
+        let group = groups.get(key);
+        if (!group) {
+            group = newGroup("tokens", { tokenKinds: item.tokens.map((token) => token.kind), suffix: item.suffix });
+            groups.set(key, group);
+        }
+        group.items.push(item);
+        group.suffixes.set(item.suffix, (group.suffixes.get(item.suffix) ?? 0) + 1);
+    }
+    return [...groups.values()];
+}
+function distinctCounts(group) {
+    if (!group.distinct) {
+        group.distinct = group.tokenKinds.map((_, position) => {
+            const seen = new Set();
+            for (const item of group.items)
+                seen.add(item.tokens[position]?.value ?? "");
+            return seen.size;
+        });
+    }
+    return group.distinct;
+}
+function isVariable(group, position) {
+    if (group.tokenKinds[position] === "id")
+        return true;
+    return distinctCounts(group)[position] > 1;
+}
+function mergeGroups(groups) {
+    const head = groups[0];
+    const suffixes = new Map();
+    const items = [];
+    for (const group of groups) {
+        items.push(...group.items);
+        for (const [suffix, count] of group.suffixes)
+            suffixes.set(suffix, (suffixes.get(suffix) ?? 0) + count);
+    }
+    return { kind: head.kind, tokenKinds: head.tokenKinds, prefixTokens: head.prefixTokens, suffix: head.suffix, suffixes, items };
+}
+/**
+ * Merge groups differing in exactly one alphabetic or numeric position to a fixpoint,
+ * then fold groups whose stems agree once their slots are wildcarded.
+ *
+ * Without the first pass a categorical position reads as unrelated groups, and the
+ * agent is handed the very split it exists to decide against. Without the second,
+ * members that differ only in compression become two sets that a reader has to notice
+ * are one.
+ */
+function mergeVariants(groups) {
+    let current = [...groups];
+    for (;;) {
+        const families = new Map();
+        for (const group of current) {
+            const key = `${group.tokenKinds.join(",")} ${group.suffix}`;
+            const bucket = families.get(key);
+            if (bucket)
+                bucket.push(group);
+            else
+                families.set(key, [group]);
+        }
+        const next = [];
+        let merged = false;
+        for (const family of families.values()) {
+            if (family.length === 1) {
+                next.push(family[0]);
+                continue;
+            }
+            const positions = family[0].tokenKinds.length;
+            let mergedHere = false;
+            for (let p = 0; p < positions && !mergedHere; p++) {
+                const kind = family[0].tokenKinds[p];
+                if (kind !== "alpha" && kind !== "digit")
+                    continue;
+                const buckets = new Map();
+                for (const group of family) {
+                    // Every member of a group shares its literal tokens, so the first row
+                    // identifies the whole group at this position.
+                    const row = group.items[0].tokens;
+                    const key = row.map((token, i) => (i === p || isVariable(group, i) ? "" : token.value)).join(" ");
+                    const bucket = buckets.get(key);
+                    if (bucket)
+                        bucket.push(group);
+                    else
+                        buckets.set(key, [group]);
+                }
+                if ([...buckets.values()].every((bucket) => bucket.length === 1))
+                    continue;
+                for (const bucket of buckets.values())
+                    next.push(bucket.length === 1 ? bucket[0] : mergeGroups(bucket));
+                mergedHere = true;
+                merged = true;
+            }
+            if (!mergedHere)
+                next.push(...family);
+        }
+        current = next;
+        if (!merged)
+            break;
+    }
+    const byStem = new Map();
+    for (const group of current) {
+        const row = group.items[0].tokens;
+        const key = `${group.tokenKinds.join(",")} ${row.map((token, i) => (isVariable(group, i) ? "" : token.value)).join(" ")}`;
+        const bucket = byStem.get(key);
+        if (bucket)
+            bucket.push(group);
+        else
+            byStem.set(key, [group]);
+    }
+    return [...byStem.values()].map((bucket) => (bucket.length === 1 ? bucket[0] : mergeGroups(bucket)));
+}
+function symbolOf(token) {
+    if (token.kind === "id")
+        return ID_SYMBOL;
+    if (token.kind === "digit")
+        return DIGIT_SYMBOL;
+    if (token.kind === "date")
+        return DATE_SYMBOL;
+    return token.value;
+}
+/**
+ * Claim residual singletons by shared literal prefix, deepest node first.
+ *
+ * Children are claimed before their parent, so a specific prefix wins over the general
+ * one it extends, and a prefix earns a set only once it covers enough files to be
+ * cheaper than the singletons it replaces.
+ */
+function minePrefixes(residue) {
+    const bySuffix = new Map();
+    for (const item of residue) {
+        const bucket = bySuffix.get(item.suffix);
+        if (bucket)
+            bucket.push(item);
+        else
+            bySuffix.set(item.suffix, [item]);
+    }
+    const sets = [];
+    const rest = [];
+    for (const [suffix, items] of bySuffix) {
+        if (items.length < MIN_PREFIX_SET_MEMBERS) {
+            rest.push(...items);
+            continue;
+        }
+        const root = { children: new Map(), items: [] };
+        for (const item of items) {
+            let node = root;
+            node.items.push(item);
+            for (const token of item.tokens) {
+                const key = symbolOf(token);
+                let child = node.children.get(key);
+                if (!child) {
+                    child = { children: new Map(), items: [], token };
+                    node.children.set(key, child);
+                }
+                child.items.push(item);
+                node = child;
+            }
+        }
+        const taken = new Set();
+        const claim = (node, prefixTokens) => {
+            for (const child of node.children.values())
+                claim(child, [...prefixTokens, child.token]);
+            const free = node.items.filter((item) => !taken.has(item));
+            const literals = prefixTokens.filter((token) => token.kind === "alpha").length;
+            if (free.length < MIN_PREFIX_SET_MEMBERS || literals < 1)
+                return;
+            for (const item of free)
+                taken.add(item);
+            const suffixes = new Map([[suffix, free.length]]);
+            sets.push(newGroup("prefix", { prefixTokens, suffix, suffixes, items: free }));
+        };
+        claim(root, []);
+        rest.push(...items.filter((item) => !taken.has(item)));
+    }
+    return { sets, rest };
+}
+/**
+ * Collapse singletons that share a token-kind sequence into one set.
+ *
+ * A flat directory of differently-named tables has one shape even though no two names
+ * differ in a single position. One template for the family is a shorter description
+ * than one template each, which is the only reason to prefer it — so it applies only
+ * once the family is large enough to pay for the generality it gives up.
+ */
+function collapseFamilies(residue) {
+    const families = new Map();
+    for (const item of residue) {
+        const key = `${item.tokens.map((token) => token.kind).join(",")} ${item.suffix}`;
+        const bucket = families.get(key);
+        if (bucket)
+            bucket.push(item);
+        else
+            families.set(key, [item]);
+    }
+    const sets = [];
+    const rest = [];
+    for (const items of families.values()) {
+        if (items.length < MIN_FAMILY_SET_MEMBERS) {
+            rest.push(...items);
+            continue;
+        }
+        const head = items[0];
+        sets.push(newGroup("family", {
+            tokenKinds: head.tokens.map((token) => token.kind),
+            suffix: head.suffix,
+            suffixes: new Map([[head.suffix, items.length]]),
+            items: [...items],
+        }));
+    }
+    return { sets, rest };
+}
+/**
+ * Fold what is left of a cross-directory context into one set.
+ *
+ * Where the same directory shape repeats, that repetition is the structure, and the
+ * residual names inside it are one set whatever they are called. Extension
+ * disagreement is carried as the format census, never used to split.
+ */
+export function catchAll(residue, hasDirectorySlot) {
+    if (!hasDirectorySlot || residue.length < MIN_CATCH_ALL_MEMBERS)
+        return { sets: [], rest: [...residue] };
+    const suffixes = new Map();
+    for (const item of residue)
+        suffixes.set(item.suffix, (suffixes.get(item.suffix) ?? 0) + 1);
+    return {
+        sets: [newGroup("catch-all", { suffix: [...suffixes.keys()].sort((a, b) => a.localeCompare(b, "en")).join(","), suffixes, items: [...residue] })],
+        rest: [],
+    };
+}
+/** Mine one directory context: every file whose directory path instantiates one template. */
+export function mineContext(files) {
+    const groups = mergeVariants(firstPass(files.map(toItem)));
+    const sets = [];
+    const singletons = [];
+    for (const group of groups) {
+        if (group.items.length >= 2)
+            sets.push(group);
+        else
+            singletons.push(group.items[0]);
+    }
+    const prefixes = minePrefixes(singletons);
+    sets.push(...prefixes.sets);
+    const families = collapseFamilies(prefixes.rest);
+    sets.push(...families.sets);
+    return { sets, residue: families.rest };
+}
+/** Describe one varying position: its class, its cardinality, and a bounded value sample. */
+export function describeSlot(rawValues, location, index) {
+    const distinct = [...new Set(rawValues)];
+    let { prefix, suffix } = commonAffixes(distinct);
+    if (prefix.length < MIN_AFFIX_LENGTH)
+        prefix = "";
+    if (suffix.length < MIN_AFFIX_LENGTH)
+        suffix = "";
+    // A number and a date are each one value: slicing a shared year off them would
+    // report a date as a literal prefix plus a pattern.
+    if (distinct.every((value) => /^[0-9]*$/.test(value)) || distinct.every((value) => isDateToken(value))) {
+        prefix = "";
+        suffix = "";
+    }
+    const strip = (value) => value.slice(prefix.length, suffix ? value.length - suffix.length : undefined);
+    const cores = distinct.map(strip);
+    const observed = classifyValues(cores);
+    const sorted = [...cores].sort((a, b) => a.localeCompare(b, "en"));
+    return {
+        location,
+        index,
+        tokenClass: observed.tokenClass,
+        ...(observed.width !== undefined ? { width: observed.width } : {}),
+        ...(observed.minLength !== undefined ? { minLength: observed.minLength, maxLength: observed.maxLength } : {}),
+        ...(observed.skeleton !== undefined ? { skeleton: observed.skeleton } : {}),
+        ...(prefix ? { prefix } : {}),
+        ...(suffix ? { suffix } : {}),
+        distinctValues: distinct.length,
+        sampleValues: sorted.slice(0, MAX_SLOT_SAMPLE_VALUES),
+        values: cores,
+        memberValues: rawValues.map(strip),
+    };
+}
+/** The template token a slot renders as. */
+export function slotToken(slot) {
+    switch (slot.tokenClass) {
+        case "digits-fixed":
+            return `<digits:${slot.width}>`;
+        case "pattern":
+            return `<pattern:${slot.skeleton}>`;
+        default:
+            return `<${slot.tokenClass}>`;
+    }
+}
+function affixText(slot, affix) {
+    return slot.tokenClass === "opaque-id" ? maskAffix(affix) : affix;
+}
+function suffixLiteral(group) {
+    if (group.suffixes.size <= 1)
+        return group.suffix ? `.${group.suffix}` : "";
+    const suffixes = [...group.suffixes.keys()].sort((a, b) => a.localeCompare(b, "en"));
+    return `.{${suffixes.join(",")}}`;
+}
+/**
+ * The name half of a set's template: its literal text, its slots, and the order they
+ * appear in. Directory slots are prepended by the assembly, which knows the context.
+ */
+export function describeGroup(group) {
+    const slots = [];
+    const segments = [];
+    const pushSlot = (slot) => {
+        if (slot.prefix)
+            segments.push({ kind: "literal", text: affixText(slot, slot.prefix) });
+        segments.push({ kind: "slot", slot: slots.length });
+        if (slot.suffix)
+            segments.push({ kind: "literal", text: affixText(slot, slot.suffix) });
+        slots.push(slot);
+    };
+    if (group.kind === "catch-all") {
+        pushSlot(describeSlot(group.items.map((item) => item.file.name), "name", 0));
+        return { slots, segments };
+    }
+    if (group.kind === "prefix") {
+        segments.push({ kind: "literal", text: group.prefixTokens.map((token) => token.value).join("") });
+        const rests = group.items.map((item) => item.tokens
+            .slice(group.prefixTokens.length)
+            .map((token) => token.value)
+            .join(""));
+        pushSlot(describeSlot(rests, "name", group.prefixTokens.length));
+        const literal = suffixLiteral(group);
+        if (literal)
+            segments.push({ kind: "literal", text: literal });
+        return { slots, segments };
+    }
+    for (let i = 0; i < group.tokenKinds.length; i++) {
+        const values = group.items.map((item) => item.tokens[i]?.value ?? "");
+        const distinct = new Set(values);
+        if (distinct.size === 1 && group.tokenKinds[i] !== "id") {
+            segments.push({ kind: "literal", text: [...distinct][0] });
+            continue;
+        }
+        pushSlot(describeSlot(values, "name", i));
+    }
+    const literal = suffixLiteral(group);
+    if (literal)
+        segments.push({ kind: "literal", text: literal });
+    return { slots, segments };
+}

package/dist/input-scan/tokens.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Tokenisation and the token-class lattice the detected-set pipeline mines over.
+ *
+ * A name splits on the boundaries that carry structure — the dotted suffix chain, and
+ * runs of letters, digits, and separators inside each dot-part — except where a run is
+ * one indivisible token. A machine-issued opaque identifier and a delimited date are
+ * each one token, because splitting them yields positions whose values never repeat,
+ * and a position whose values never repeat fragments the template space into one
+ * template per file.
+ *
+ * The lattice orders classes by specificity: a value set is described by the most
+ * specific class every one of its values satisfies.
+ */
+export type TokenKind = "alpha" | "digit" | "delim" | "id" | "date";
+export interface NameToken {
+    readonly kind: TokenKind;
+    readonly value: string;
+}
+export declare const ID_SYMBOL = "ID";
+export declare const DIGIT_SYMBOL = "#";
+export declare const DATE_SYMBOL = "@";
+export declare function basenameOf(path: string): string;
+export declare function dirnameOf(path: string): string;
+/**
+ * Split a basename into its stem and the terminal chain of recognised extensions.
+ *
+ * The stem may itself contain dots: only a trailing run of known extension tokens is
+ * the extension, so the categorical tokens of a dotted suffix chain stay minable.
+ */
+export declare function splitStem(fileName: string): {
+    stem: string;
+    suffix: string;
+};
+/**
+ * A machine-issued identifier: a UUID, a fixed-length hex digest, or a long
+ * class-mixed base64url token that is structurally random.
+ *
+ * Randomness shows either as case flipping inside letter runs or as digit
+ * fragmentation, and either signal alone suffices — a letter-heavy random token has
+ * few digit runs, and a digit-heavy one has few letters to flip. Separator density
+ * vetoes the second signal only: snake_case is how a person writes a name, and a
+ * person's name is not an identifier however many digits it carries.
+ */
+export declare function isOpaqueId(value: string): boolean;
+/** A calendar date written compactly or with a single separator: `20260824`, `2026-08-24`. */
+export declare function isDateToken(value: string): boolean;
+/**
+ * Tokenise one path segment — a directory name, or one dot-part of a stem.
+ *
+ * A maximal base64url-charset run is tested whole before it is split, which is what
+ * lets a long identifier collapse to one position where masking digit runs cannot. A
+ * date embedded in a longer run is lifted out whole for the same reason: split into
+ * year, month, and day it becomes three positions that vary together, and the template
+ * space grows by the product rather than by one.
+ */
+export declare function tokenizeSegment(segment: string): NameToken[];
+/** Tokenise a full basename. Dot-parts of the stem are joined by explicit delimiter tokens. */
+export declare function tokenizeName(fileName: string): {
+    tokens: NameToken[];
+    suffix: string;
+};
+/** The masked form of a token run: variable material becomes its class symbol. */
+export declare function skeletonOf(tokens: readonly NameToken[]): string;
+export declare function maskSegment(segment: string): string;
+export type SlotTokenClass = "constant" | "date" | "digits-fixed" | "digits" | "hex" | "opaque-id" | "word" | "pattern" | "any";
+export interface ValueClass {
+    readonly tokenClass: SlotTokenClass;
+    /** Digit width, when every value is a fixed-width number. */
+    readonly width?: number;
+    readonly minLength?: number;
+    readonly maxLength?: number;
+    /** The shared masked form, when the class is `pattern`. */
+    readonly skeleton?: string;
+}
+/** Place a set of observed values on the lattice, most specific class first. */
+export declare function classifyValues(values: readonly string[]): ValueClass;
+/**
+ * The longest prefix and suffix shared by every value.
+ *
+ * A greedy identifier run swallows the literal text fused to it; recovering the shared
+ * affixes puts that text back in the template where a reader can see it.
+ */
+export declare function commonAffixes(values: readonly string[]): {
+    prefix: string;
+    suffix: string;
+};
+/**
+ * Mask class-mixed runs inside a literal affix of an identifier position.
+ *
+ * The affix was recovered by comparing identifier values, so a run that looks like
+ * identifier material almost certainly is: it reached the template only because every
+ * member happened to share it.
+ */
+export declare function maskAffix(affix: string): string;