@lmzhen/dsh-evolution-core 0.26.26 → 0.27.1

package/lib/types/quality.js

@@ -0,0 +1,308 @@
+/**
+ * Quality scoring and near-duplicate detection for the curated skill library.
+ *
+ * Pure functions over data inputs so the scoring policy is unit-testable and
+ * the same math feeds the usage sidecar, the `skill_manage review` surface and
+ * the learning graph. Weights follow the Hermes/hermes-claw six-factor model;
+ * mutation maturity is a documented DSH approximation (single per-month patch
+ * trend ratio replaces the claw timestamp-trend formula, since DSH usage
+ * records only carry the last patched timestamp).
+ *
+ * Batch C (2026-09-27): the similarity half of this module is now TWO NAMED
+ * QUESTIONS plus a declared projection — {@link identity} ("is it the same
+ * text?") and {@link affinity} ("how much vocabulary do these two share?") asked
+ * over {@link projectionText}. Callers name the question and the projection
+ * instead of growing a fourth unnamed ruler (design §3C / §1 I2).
+ * @module @lmzhen/dsh-evolution-core
+ */
+import { createHash } from 'node:crypto';
+import { latestActivityAt } from "./usage.js";
+export const QUALITY_WEIGHTS = {
+    usageFrequency: 0.25,
+    stability: 0.20,
+    recency: 0.20,
+    references: 0.10,
+    mutationMaturity: 0.20,
+    richness: 0.05,
+};
+/** Score below which a skill is flagged for review. */
+export const LOW_QUALITY_THRESHOLD = 0.3;
+function clamp01(value) {
+    return Math.max(0, Math.min(1, value));
+}
+function daysBetween(from, now) {
+    // A2-9 (v18): an unparseable timestamp must not poison the score with NaN.
+    const t = Date.parse(from);
+    if (!Number.isFinite(t))
+        return 0;
+    return Math.max(0, (now.getTime() - t) / 86_400_000);
+}
+export function computeQualityScores(input) {
+    const now = input.now ?? new Date();
+    const scores = new Map();
+    for (const [name, record] of input.usage) {
+        const ageDays = Math.max(1, daysBetween(record.created_at, now));
+        const idleDays = daysBetween(latestActivityAt(record) ?? record.created_at, now);
+        const patchCount = record.patch_count;
+        // E-2 / G-1 (v18): a skill is "used" when it is LOADED. The in-tree
+        // producer is the platform `skill` tool's `view` bump (skill-usage
+        // READ_SKILL_TOOL_KIND); `use_count` has no in-tree producer and stays an
+        // EXTERNAL host signal. Summing both makes these two factors real today
+        // while still counting a host that writes `use` — no durable-format change
+        // and no silent refactor of the published field.
+        const loadCount = record.use_count + record.view_count;
+        const usageFrequency = clamp01(loadCount / ageDays);
+        const stability = loadCount === 0 ? 1 : clamp01(1 - patchCount / loadCount);
+        const recency = idleDays < 30 ? 1 : clamp01(1 - (idleDays - 30) / 150);
+        const references = clamp01((input.referenceCounts?.get(name) ?? 0) / 3);
+        const mutationMaturity = patchCount === 0 ? 0.3 : patchCount === 1 ? 0.4 : clamp01((patchCount - 1) / Math.max(1, ageDays / 30));
+        const richness = clamp01((input.supportDirs?.get(name) ?? 0) * 0.175);
+        const factors = { usageFrequency, stability, recency, references, mutationMaturity, richness };
+        const score = usageFrequency * QUALITY_WEIGHTS.usageFrequency
+            + stability * QUALITY_WEIGHTS.stability
+            + recency * QUALITY_WEIGHTS.recency
+            + references * QUALITY_WEIGHTS.references
+            + mutationMaturity * QUALITY_WEIGHTS.mutationMaturity
+            + richness * QUALITY_WEIGHTS.richness;
+        scores.set(name, { score, factors, warn: score < LOW_QUALITY_THRESHOLD });
+    }
+    return scores;
+}
+function normalize(content) {
+    return content.toLowerCase().replace(/\s+/g, ' ').trim();
+}
+/* ------------------------------------------------------------------------- *
+ * Similarity: two NAMED questions, and the projection a caller declares.
+ *
+ * Batch C (2026-09-27, design §3C) — the module used to carry three unnamed
+ * rulers (a normalized hash, a token Jaccard, a first-word name index) whose
+ * questions overlapped without saying so. The questions are now named, and a
+ * caller that wants a comparison must name the one it is asking:
+ *
+ *   1. {@link identity}  — "is this the SAME TEXT?" (normalized-content hash)
+ *   2. {@link affinity}  — "how much VOCABULARY do these two share?" (Jaccard
+ *                          over two {@link vocabulary} sets)
+ *
+ * {@link computePrefixClusters} stays what it always was — a name-side
+ * STRUCTURAL index (which names share a first word), not a similarity score.
+ * What a caller declares instead of inventing a fourth ruler is the PROJECTION
+ * ({@link projectionText}): `body` = the whole SKILL.md, `summary` = name +
+ * description. Adding a projection is one entry in that table, and the two
+ * callers stop disagreeing about what they compared (design §6.4).
+ * ------------------------------------------------------------------------- */
+/** Named similarity question 1 — "is it the same text?" — as a normalized
+ * content hash: case- and whitespace-insensitive, so two texts that differ only
+ * in formatting are the SAME text here. Invariant under which projection the
+ * caller read the text from.
+ *
+ * C-26 (v10 audit): NOT named `contentHash` — mutations.ts exports a
+ * contentHash of RAW bytes, this one hashes the NORMALIZED text for dedup
+ * grouping. Same helper as that rename, now a named question. */
+export function identity(text) {
+    return createHash('sha256').update(normalize(text)).digest('hex');
+}
+/**
+ * The declared projection: WHICH text a comparison weighs.
+ *
+ * `body` is the library-wide scan's projection (whole SKILL.md, read by the
+ * caller). `summary` is name + description in one string — the projection a
+ * listing already publishes (`SkillSummary` carries both fields, and a body is
+ * attached only when the caller asks for it) — so a summary comparison never
+ * reads a body OF ITS OWN: the create-time hint is fed by the listing pass the
+ * tool already pays for its frontmatter, and calls no read of its own.
+ */
+export function projectionText(projection, input) {
+    switch (projection) {
+        case 'body': return input.content ?? '';
+        case 'summary': return `${input.name ?? ''} ${input.description ?? ''}`;
+    }
+}
+/** The comparable vocabulary of one text: lowercase alphanumeric runs, plus
+ * whole CJK runs (a Chinese description tokenizes by punctuation, not by word —
+ * deliberately: the ruler is one home, and a real tokenizer is a different
+ * question than "do these two share vocabulary?"). */
+export function vocabulary(text) {
+    return new Set(normalize(text).split(/[^a-z0-9\u4e00-\u9fff]+/).filter(Boolean));
+}
+/** Named similarity question 2 — "how much vocabulary do these two share?" —
+ * Jaccard over two {@link vocabulary} sets, 0 (nothing in common, or either side
+ * empty) to 1 (identical vocabulary). Set-based on purpose: the bulk scan
+ * memoizes each name's set once and only the comparisons are budgeted. */
+export function affinity(a, b) {
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const token of a)
+        if (b.has(token))
+            intersection += 1;
+    return intersection / (a.size + b.size - intersection);
+}
+/** The `affinity` level at which two bodies are treated as the same skill
+ * (design §3C: the default is unchanged at 0.95 — it is now a NAME, because the
+ * docblock below has linked it since PLAN-R2 while the code carried a bare
+ * literal). Callers may still pass their own level. */
+export const DEDUP_SIMILARITY_THRESHOLD = 0.95;
+/** PLAN-R2 P2-8 (2026-09-16): default cap on two-name comparisons in
+ * {@link computeDedupGroups}. A 2000-skill library is ~2M pairs; the old
+ * unbounded two-two Jaccard ran seconds to tens of seconds per
+ * review/`dedup_group` probe. 250k comparisons bounds that to well under a
+ * second while staying far above every real library's pair count. Only
+ * pairwise comparisons are budgeted — materializing a name's token set
+ * (memoized, once per name) and the exact-hash pre-union phase are not. */
+export const DEDUP_MAX_PAIR_COMPARISONS = 250_000;
+/**
+ * Two-phase near-duplicate clustering: exact normalized-hash groups first,
+ * then token-Jaccard edges at {@link DEDUP_SIMILARITY_THRESHOLD} with a token
+ * ratio guard, union-find across the whole set.
+ *
+ * PLAN-R2 P2-8 (2026-09-16): each name's token set is materialized once
+ * (memoized map), each pair takes an O(1) size-ratio short-circuit before the
+ * intersection, and the pairwise loop is bounded by
+ * `maxPairComparisons` (default {@link DEDUP_MAX_PAIR_COMPARISONS}); hitting
+ * the budget stops the scan and `truncated: true` says so. Small libraries
+ * (below the budget) behave exactly as the unbounded scan did.
+ */
+export function computeDedupGroups(input) {
+    const threshold = input.threshold ?? DEDUP_SIMILARITY_THRESHOLD;
+    const maxPairComparisons = input.maxPairComparisons ?? DEDUP_MAX_PAIR_COMPARISONS;
+    const names = [...input.contents.keys()];
+    const hashes = new Map();
+    for (const name of names) {
+        // Question 1 over the declared projection: the body is what this scan weighs
+        // (it was handed bodies by its caller — see the tool path's exclusion notes).
+        const hash = identity(projectionText('body', { content: input.contents.get(name) ?? '' }));
+        const bucket = hashes.get(hash);
+        if (bucket)
+            bucket.push(name);
+        else
+            hashes.set(hash, [name]);
+    }
+    // Union-find over names; exact-hash peers are pre-united.
+    const parent = new Map();
+    const find = (x) => {
+        const root = parent.get(x) ?? x;
+        if (root !== x)
+            parent.set(x, find(root));
+        return parent.get(x) ?? x;
+    };
+    const union = (a, b) => {
+        const [ra, rb] = [find(a), find(b)];
+        if (ra !== rb)
+            parent.set(rb, ra);
+    };
+    for (const [, bucketNames] of hashes) {
+        const first = bucketNames[0];
+        if (first === undefined || bucketNames.length === 1)
+            continue;
+        for (let index = 1; index < bucketNames.length; index += 1) {
+            const peer = bucketNames[index];
+            if (peer)
+                union(first, peer);
+        }
+    }
+    const tokens = new Map();
+    const tokenSet = (name) => {
+        let set = tokens.get(name);
+        if (!set) {
+            set = vocabulary(projectionText('body', { content: input.contents.get(name) ?? '' }));
+            tokens.set(name, set);
+        }
+        return set;
+    };
+    // PLAN-R2 P2-8: the memoized tokenSet materializes each name's set at most
+    // once; only the two-name comparisons below count against the budget.
+    let compared = 0;
+    let truncated = false;
+    for (let index = 0; index < names.length && !truncated; index += 1) {
+        const a = names[index];
+        if (a === undefined)
+            continue;
+        for (let other = index + 1; other < names.length; other += 1) {
+            const b = names[other];
+            if (b === undefined)
+                continue;
+            if (compared >= maxPairComparisons) {
+                truncated = true;
+                break;
+            }
+            compared += 1;
+            const [ta, tb] = [tokenSet(a), tokenSet(b)];
+            if (Math.max(ta.size, tb.size) / Math.max(1, Math.min(ta.size, tb.size)) > 5)
+                continue;
+            if (affinity(ta, tb) >= threshold)
+                union(a, b);
+        }
+    }
+    const groups = new Map();
+    for (const name of names) {
+        const root = find(name);
+        const group = groups.get(root);
+        if (group)
+            group.push(name);
+        else
+            groups.set(root, [name]);
+    }
+    return { groups: [...groups.values()].filter(group => group.length > 1), truncated };
+}
+/**
+ * The create-time near-duplicate question (batch C, design §3C-2): WHICH
+ * existing skills is this authoring candidate about to near-copy?
+ *
+ * It is the SAME {@link affinity} question as the library-wide scan, asked over
+ * the `summary` projection — the one a listing already publishes
+ * (`SkillSummary` carries `name` and `description`; nothing here can see a
+ * body, which is the point: a create must not pay a SECOND whole-tree pass — the
+ * listing's own frontmatter read is what feeds this comparison). Deliberately hint-only: the caller writes regardless, and the
+ * matches are news, not a gate (G4 — the create-time foreground behavior is
+ * unchanged).
+ *
+ * The default level is far below the scan's {@link DEDUP_SIMILARITY_THRESHOLD}
+ * because the projection is thinner: a name plus one sentence shares far fewer
+ * tokens than two bodies, and a hint that fires on every new skill is noise.
+ * Exact same-name candidates are skipped — the library's own create refuses
+ * those, and this line must not repeat that refusal.
+ */
+export const SUMMARY_DUPLICATE_HINT_THRESHOLD = 0.5;
+export function nearDuplicateSummaries(input) {
+    const threshold = input.threshold ?? SUMMARY_DUPLICATE_HINT_THRESHOLD;
+    const limit = input.limit ?? 3;
+    const candidateTokens = vocabulary(projectionText('summary', input.candidate));
+    const matches = [];
+    for (const existing of input.existing) {
+        if (existing.name === input.candidate.name)
+            continue;
+        const score = affinity(candidateTokens, vocabulary(projectionText('summary', existing)));
+        if (score >= threshold)
+            matches.push({ name: existing.name, score });
+    }
+    return matches
+        .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name))
+        .slice(0, limit);
+}
+/**
+ * Prefix-cluster index over a name set (rc.67 merge heuristic, input side):
+ * the curator prompt asks the model to identify "prefix clusters — skills
+ * sharing a first word or domain keyword"; the deterministic index supplies
+ * stable ground truth instead of letting the model infer clusters from the
+ * raw list. Key = first alphanumeric run of the lowercased name; groups with
+ * at least two members, largest first then alphabetical. Orientation-only:
+ * nomination authority stays with the LLM and the candidate-pool gates.
+ */
+export function computePrefixClusters(names) {
+    const groups = new Map();
+    for (const name of names) {
+        const key = name.toLowerCase().split(/[^a-z0-9]+/)[0];
+        if (!key)
+            continue;
+        const bucket = groups.get(key);
+        if (bucket)
+            bucket.push(name);
+        else
+            groups.set(key, [name]);
+    }
+    return [...groups.entries()]
+        .filter(([, members]) => members.length >= 2)
+        .map(([key, members]) => ({ key, members }))
+        .sort((a, b) => b.members.length - a.members.length || a.key.localeCompare(b.key));
+}
+//# sourceMappingURL=quality.js.map

package/lib/types/reachability.js

@@ -0,0 +1,122 @@
+/**
+ * G3/3.4: the READ-ONLY reachability reconciliation.
+ *
+ * Three resource domains have grown past their owners' view: content blobs whose index entries were
+ * trimmed away (skill-history: "Trimming drops INDEX entries; orphan blobs stay"), report files no run
+ * record points at, and snapshot directories whose manifest holds almost nothing. The audit measured
+ * them once (233 blobs / 11.40 MiB / 75.4% unreachable; 52 orphan report .md; 4/5 empty snapshot trees,
+ * the newest holding 9 of 35 skills); this module answers the same three questions, from the tree in
+ * front of it, and NOTHING ELSE: it deletes nothing, repairs nothing, and adds no timer.
+ *
+ * Every path is a parameter. The reconciliation must not grow its own idea of where a domain lives -
+ * the callers already know (historyRoot / the reports directory / the snapshot root), and a second
+ * guess here would be a second owner of that fact.
+ */
+import { join } from 'node:path';
+import { isPresent, isUnknown, probeList, probeText } from "./probe.js";
+const HISTORY_BASELINE = '233 blobs / 11.40 MiB unreachable (75.4%)';
+const REPORTS_BASELINE = '52 orphan report .md';
+const BACKUPS_BASELINE = '4/5 empty snapshot trees; the newest held 9/35 skills';
+/** Every sha256 any skill index references, across the whole history root. */
+export async function referencedBlobs(io, historyRoot) {
+    const referenced = new Set();
+    const skillsDir = join(historyRoot, 'skills');
+    const namesProbe = await probeList(io, skillsDir);
+    // N36: an unreadable skills directory is not "no indexes"; the domain is unmeasurable.
+    if (isUnknown(namesProbe))
+        return undefined;
+    const names = isPresent(namesProbe) ? namesProbe.value : [];
+    for (const name of names) {
+        const probe = await probeText(io, join(skillsDir, name, 'index.json'));
+        // N36: a read FAILURE is not "no references" - it makes the domain unmeasurable, and the caller says so.
+        if (isUnknown(probe))
+            return undefined;
+        // N36: `probe === null` was never true - `Probe<T>` is a wrapper, not `T | null`. The absent case is the
+        // wrapper state, so an ABSENT index is skipped while an UNKNOWN one already returned above.
+        if (!isPresent(probe))
+            continue;
+        for (const match of probe.value.matchAll(/[a-f0-9]{64}/g))
+            referenced.add(match[0]);
+    }
+    return referenced;
+}
+/**
+ * The history domain: blobs on disk minus the blobs the indexes still reference.
+ * @param io - the tree reader.
+ * @param historyRoot - the history root (`.history`), passed in, never guessed.
+ * @returns the reading, with the audit baseline quoted for comparison.
+ */
+export async function historyReachability(io, historyRoot) {
+    const blobsDir = join(historyRoot, 'blobs');
+    const blobsProbe = await probeList(io, blobsDir);
+    const referenced = await referencedBlobs(io, historyRoot);
+    // N36: neither half may turn a read failure into a number. An unreadable blob directory or an
+    // unreadable index makes the domain unmeasured - "not read" is not "nothing unreachable".
+    if (isUnknown(blobsProbe) || referenced === undefined) {
+        return { domain: 'history.blobs', unit: 'bytes', total: 0, reachable: 0, unreachable: 0, baseline: HISTORY_BASELINE, unmeasured: 'a blob listing or an index read failed; this domain was not measured' };
+    }
+    const blobs = isPresent(blobsProbe) ? blobsProbe.value : [];
+    const reachable = blobs.filter(blob => referenced.has(blob)).length;
+    return {
+        domain: 'history.blobs',
+        unit: 'bytes',
+        total: blobs.length,
+        reachable,
+        unreachable: blobs.length - reachable,
+        baseline: HISTORY_BASELINE,
+    };
+}
+/**
+ * The reports domain: files no run record points at. A record WITHOUT a `resultRef` still owns the
+ * conventional path (`reports/maintain-<id>.json`), so that path counts as reachable - counting only
+ * explicit refs would report compliant files as orphans and the number could never match the baseline.
+ * @param io - the tree reader.
+ * @param reportsDir - the reports directory, passed in.
+ * @param runIds - every run id the run records carry.
+ * @param resultRefs - every explicit resultRef those records carry.
+ */
+export async function reportsReachability(io, reportsDir, runIds, resultRefs) {
+    const listed = await probeList(io, reportsDir);
+    if (isUnknown(listed)) {
+        return { domain: 'reports.files', unit: 'files', total: 0, reachable: 0, unreachable: 0, baseline: REPORTS_BASELINE, unmeasured: 'the reports directory could not be listed; this domain was not measured' };
+    }
+    const files = isPresent(listed) ? listed.value : [];
+    const reachableNames = new Set();
+    for (const ref of resultRefs)
+        reachableNames.add(ref.slice(ref.lastIndexOf('/') + 1));
+    for (const id of runIds)
+        reachableNames.add('maintain-' + id + '.json');
+    const reachable = files.filter(file => reachableNames.has(file)).length;
+    return {
+        domain: 'reports.files',
+        unit: 'files',
+        total: files.length,
+        reachable,
+        unreachable: files.length - reachable,
+        baseline: REPORTS_BASELINE,
+    };
+}
+/**
+ * The backups domain: snapshot trees and how much each holds. `skillsPerSnapshot` is measured by the
+ * caller (it already reads manifests); an empty tree is one whose manifest lists no skill.
+ * @param io - the tree reader.
+ * @param backupsRoot - the snapshot root, passed in.
+ * @param skillsPerSnapshot - the caller's manifest reading, one entry per snapshot directory.
+ */
+export async function backupsReachability(io, backupsRoot, skillsPerSnapshot) {
+    const listed = await probeList(io, backupsRoot);
+    if (isUnknown(listed)) {
+        return { domain: 'backups.trees', unit: 'files', total: 0, reachable: 0, unreachable: 0, baseline: BACKUPS_BASELINE, unmeasured: 'the snapshot root could not be listed; this domain was not measured' };
+    }
+    const dirs = isPresent(listed) ? listed.value : [];
+    const empty = skillsPerSnapshot.filter(count => count === 0).length;
+    return {
+        domain: 'backups.trees',
+        unit: 'files',
+        total: dirs.length,
+        reachable: dirs.length - empty,
+        unreachable: empty,
+        baseline: BACKUPS_BASELINE,
+    };
+}
+//# sourceMappingURL=reachability.js.map