@hyperspell/openclaw-hyperspell 0.19.0 → 0.21.0

package/dist/lib/filters.js

@@ -6,6 +6,10 @@
  * Filters match against memory METADATA keys by their bare name — the same
  * convention `buildScopeFilter` uses (`openclaw_scope`, `openclaw_user`).
  */
+/** Metadata tag on auto-trace session-end rows (see `sendTrace` in client.ts). */
+export const AGENT_END_SOURCE = "agent_end";
+/** Metadata tag on mood-weather roll records (see `recordMoodRoll` in hooks/mood-weather.ts). */
+export const MOOD_WEATHER_SOURCE = "mood_weather";
 /**
  * Memories produced by the auto-trace session-end hook are tagged in metadata
  * as `openclaw_source: "agent_end"` (see `sendTrace` in client.ts). Those should
@@ -34,22 +38,46 @@
  * "openclaw_agent_end" — wrong on BOTH counts (the tag lives in metadata under
  * `openclaw_source`, value `"agent_end"`), so it silently matched nothing.
  */
-export const EXCLUDE_SESSION_END_FILTER = {
-    openclaw_source: { $ne: "agent_end" },
-};
 /**
  * The exclude clause to apply for a given config — or `undefined` to skip the
- * filter. `agent_end` rows are written ONLY by the auto-trace hook, so when
- * auto-trace is OFF there are none to hide and we skip the filter to avoid its
- * ~1s/search latency cost (pure overhead otherwise — verified live against an
- * auto-trace-off agent with zero `agent_end` rows).
+ * filter. Each excluded tag is gated on the feature that writes it, for
+ * PERFORMANCE, not correctness (see the header comment): with the feature off
+ * there are no tagged rows to hide, so the ~1s predicate would be pure latency.
  *
- * When auto-trace is ON we apply `{$ne:"agent_end"}`, which post-#1921 drops the
- * traces while KEEPING untagged hot-buffer rows — so that path is correct now
- * (the old #40 hot-row-drop is fixed by the backend, not by gating).
+ *  - `agent_end` rows are written ONLY by the auto-trace hook → gate on
+ *    `autoTrace.enabled` (verified live against an auto-trace-off agent with
+ *    zero `agent_end` rows).
+ *  - `mood_weather` rolls are recorded ONLY when the emotional-context handler
+ *    is registered AND the dice are live → gate on both flags.
+ *
+ * Shape: a single excluded value keeps the proven plain-`$ne` form (byte-
+ * identical to the shipped filter, and post-#1921 it drops the tag while
+ * KEEPING untagged hot-buffer rows). Two values use `$nin`.
+ *
+ * ⚠️ The `$nin` two-value shape has NOT been re-verified live post-#1921 (the
+ * pre-#1921 truth table showed `$nin` diverging from `$ne`). Owner's post-merge
+ * step: run `node docs/filter-dialect-test.mjs` — the `$nin[agent_end,mood_weather]`
+ * row must show U=Y, A=N, M=N. Fallback plan if it fails: try the
+ * `$and[$ne,$ne]` probe row; if that also fails, gate back to single-value
+ * `$ne` for the two common one-feature-on configs, log a warning when both
+ * features are enabled, and file a backend dialect follow-up
+ * (docs/hyperspell-backend-followups.md style) for the both-on combo.
  */
 export function excludeFilterFor(cfg) {
-    return cfg.autoTrace.enabled ? EXCLUDE_SESSION_END_FILTER : undefined;
+    const excluded = [];
+    if (cfg.autoTrace.enabled)
+        excluded.push(AGENT_END_SOURCE);
+    // Mood rolls are recorded only when the emotional-context handler is
+    // registered AND the dice are live — same "no rows to hide → skip the
+    // ~1s predicate" gate as auto-trace.
+    if (cfg.emotionalContext && cfg.moodWeatherChance > 0) {
+        excluded.push(MOOD_WEATHER_SOURCE);
+    }
+    if (excluded.length === 0)
+        return undefined;
+    if (excluded.length === 1)
+        return { openclaw_source: { $ne: excluded[0] } };
+    return { openclaw_source: { $nin: excluded } };
 }
 /**
  * Combine a caller-supplied filter with the session-end exclude via `$and`.

package/dist/lib/loops-audit.js

@@ -0,0 +1,438 @@
+/**
+ * Pure logic for the unfinished-loops audit
+ * (docs/proposals/04-startup-orientation-loops-tuning.md) — extracted here so
+ * parsing/formatting/summary logic joins the hermetic unit suite while the
+ * live runner (scripts/audit-loops.ts) stays out of `npm test`, mirroring the
+ * lib/eval-matchers.ts + scripts/eval-retrieval.ts split.
+ *
+ * Nothing here touches the network or the production hook: the audit is
+ * read-only observation of what `gatherOrientation`'s loops search returns.
+ */
+/**
+ * Candidate reworded default queries from proposal 04 §3.3 (Candidate A).
+ * These are audit probes only — the shipped default in config.ts is untouched
+ * until a variant graduates via the rubric's paired-day gate.
+ */
+export const CANDIDATE_QUERIES = {
+    a1: "I'll get back to you, still need to, waiting on, didn't finish, let's pick this up later, promised to follow up",
+    a2: "unresolved follow-up: promised, waiting on an answer, blocked, next step, revisit, unfinished task, open question",
+};
+/**
+ * Caps from PR #107's `buildLoopsQuery` sketch
+ * (docs/plans/issue-73-static-loops-query.md) — reproduced here ONLY to
+ * simulate that candidate remedy side by side with the static query. This is
+ * deliberately not imported from production code: #107 is unimplemented by
+ * decision (audit-first), so the simulation owns its own copy of the shape.
+ */
+const LOOPS_TITLE_MAX = 80;
+const LOOPS_TOPICS_MAX = 300;
+/**
+ * Simulate #107's dynamic query: static intent base + recent conversation
+ * titles. Empty titles collapse to exactly the base — same graceful
+ * degradation the candidate specifies.
+ */
+export function deriveDynamicQuery(base, titles) {
+    const topics = [];
+    let used = 0;
+    for (const raw of titles) {
+        const title = raw.replace(/\s+/g, " ").trim().slice(0, LOOPS_TITLE_MAX);
+        if (title.length === 0)
+            continue;
+        if (used + title.length > LOOPS_TOPICS_MAX)
+            break;
+        topics.push(title);
+        used += title.length;
+    }
+    if (topics.length === 0)
+        return base;
+    return `${base} — recent topics: ${topics.join("; ")}`;
+}
+/**
+ * Recover titles from a formatted <hyperspell-recent-interactions> block
+ * (`- [3d ago] Title — snippet` / hot-buffer shape `- Title`). Parsing the
+ * rendered block instead of re-running a private fetch keeps the runner on
+ * gatherOrientation's real output; the trade-off (a title containing " — "
+ * gets truncated) is acceptable for a simulation input and documented in the
+ * rubric.
+ */
+export function titlesFromRecentBlock(recentBlock) {
+    if (!recentBlock)
+        return [];
+    const titles = [];
+    for (const line of recentBlock.split("\n")) {
+        const m = /^- (?:\[[^\]]*\] )?(.*)$/.exec(line);
+        if (!m)
+            continue;
+        const title = m[1].split(" — ")[0].trim();
+        if (title.length > 0)
+            titles.push(title);
+    }
+    return titles;
+}
+export const SIMULATE_NAMES = [
+    "a1",
+    "a2",
+    "dynamic",
+    "after30",
+    "after60",
+];
+/** Parse CLI args; throws with a usable message on anything malformed. */
+export function parseAuditArgs(argv) {
+    const args = { json: false, simulate: [] };
+    const take = (flag, i) => {
+        const v = argv[i];
+        if (v === undefined)
+            throw new Error(`${flag} requires a value`);
+        return v;
+    };
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        switch (a) {
+            case "-o": {
+                const fmt = take(a, ++i);
+                if (fmt !== "json")
+                    throw new Error(`unsupported output format "${fmt}" (only: json)`);
+                args.json = true;
+                break;
+            }
+            case "--query":
+                args.query = take(a, ++i);
+                break;
+            case "--limit": {
+                const n = Number(take(a, ++i));
+                if (!Number.isInteger(n) || n < 1)
+                    throw new Error("--limit must be a positive integer");
+                args.limit = n;
+                break;
+            }
+            case "--after":
+                args.after = take(a, ++i);
+                break;
+            case "--user":
+                args.user = take(a, ++i);
+                break;
+            case "--simulate": {
+                for (const name of take(a, ++i).split(",").map((s) => s.trim()).filter(Boolean)) {
+                    if (!SIMULATE_NAMES.includes(name)) {
+                        throw new Error(`unknown --simulate variant "${name}" (known: ${SIMULATE_NAMES.join(", ")})`);
+                    }
+                    args.simulate.push(name);
+                }
+                break;
+            }
+            case "--summary":
+                args.summary = { ledger: take(a, ++i), labels: take(a, ++i) };
+                break;
+            default:
+                throw new Error(`unknown argument "${a}"`);
+        }
+    }
+    return args;
+}
+/** `--after 30` = 30 days before `now`; anything else must parse as a date. */
+export function resolveAfter(value, now = new Date()) {
+    if (/^\d+$/.test(value)) {
+        const d = new Date(now);
+        d.setUTCDate(d.getUTCDate() - Number(value));
+        return d.toISOString();
+    }
+    const d = new Date(value);
+    if (Number.isNaN(d.getTime()))
+        throw new Error(`--after "${value}" is neither a day count nor a date`);
+    return d.toISOString();
+}
+/**
+ * The variant list for one invocation. The first entry is always the audited
+ * primary: named "baseline" only when it is exactly the production values, so
+ * ledger lines can't silently mix overridden runs into the baseline series.
+ */
+export function buildVariants(baseQuery, args, recentTitles, now = new Date()) {
+    const overridden = args.query !== undefined || args.after !== undefined || args.limit !== undefined;
+    const primary = {
+        name: overridden ? "custom" : "baseline",
+        query: baseQuery,
+        ...(args.after ? { after: resolveAfter(args.after, now) } : {}),
+    };
+    const variants = [primary];
+    for (const name of args.simulate) {
+        if (name === "a1" || name === "a2") {
+            variants.push({ name, query: CANDIDATE_QUERIES[name] });
+        }
+        else if (name === "dynamic") {
+            variants.push({ name, query: deriveDynamicQuery(baseQuery, recentTitles) });
+        }
+        else {
+            const days = name === "after30" ? "30" : "60";
+            variants.push({ name, query: baseQuery, after: resolveAfter(days, now) });
+        }
+    }
+    return variants;
+}
+const round4 = (n) => Number(n.toFixed(4));
+export function toAuditRow(r, rank, includeContent) {
+    // Mirror formatUnfinishedLoops: it renders highlights[0] as returned, not
+    // the top-scored highlight, and silently drops no-highlight results.
+    const top = r.highlights[0];
+    return {
+        rank,
+        resourceId: r.resourceId,
+        title: r.title === null ? null : r.title.slice(0, 120),
+        source: r.source,
+        score: r.score === null ? null : round4(r.score),
+        createdAt: r.createdAt,
+        highlightCount: r.highlights.length,
+        rendered: top !== undefined,
+        ...(includeContent && top ? { snippet: top.text.replace(/\s+/g, " ").slice(0, 160) } : {}),
+    };
+}
+/** Count rendered bullets in a formatted loops/recent block. */
+export function countBlockBullets(block) {
+    if (!block)
+        return 0;
+    return block.split("\n").filter((l) => l.startsWith("- ")).length;
+}
+export function buildRunRecord(input) {
+    const rows = input.results.map((r, i) => toAuditRow(r, i + 1, input.includeContent));
+    const renderedCount = rows.filter((r) => r.rendered).length;
+    const blockConsistent = input.gatherLoopsCount === null
+        ? null
+        : input.gatherLoopsCount === input.results.length &&
+            countBlockBullets(input.asInjectedBlock) === renderedCount;
+    return {
+        ts: input.ts,
+        variant: input.variant.name,
+        query: input.variant.query,
+        limit: input.limit,
+        after: input.variant.after ?? null,
+        userId: input.userId ?? null,
+        contentEnabled: input.includeContent,
+        resultCount: rows.length,
+        renderedCount,
+        wastedSlots: rows.length - renderedCount,
+        blockConsistent,
+        results: rows,
+        asInjected: input.includeContent ? input.asInjectedBlock : null,
+    };
+}
+/** Set-diff of surfaced resourceIds, rank order preserved within each list. */
+export function compareVariantSets(baseline, variant) {
+    const baseIds = baseline.results.map((r) => r.resourceId);
+    const varIds = variant.results.map((r) => r.resourceId);
+    const baseSet = new Set(baseIds);
+    const varSet = new Set(varIds);
+    return {
+        overlap: baseIds.filter((id) => varSet.has(id)),
+        onlyBaseline: baseIds.filter((id) => !varSet.has(id)),
+        onlyVariant: varIds.filter((id) => !baseSet.has(id)),
+    };
+}
+/** JSONL with blank lines and `//` comment lines allowed (fixture-file style). */
+export function parseJsonl(text) {
+    const out = [];
+    const lines = text.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i].trim();
+        if (line.length === 0 || line.startsWith("//"))
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            throw new Error(`invalid JSON on line ${i + 1}`);
+        }
+    }
+    return out;
+}
+export const LOOP_VERDICTS = [
+    "still-open",
+    "resolved",
+    "not-a-loop",
+];
+const labelKey = (ts, variant, resourceId) => `${ts}|${variant}|${resourceId}`;
+function emptyVerdictCounts() {
+    return { "still-open": 0, resolved: 0, "not-a-loop": 0 };
+}
+function dayHitRate(records, labels) {
+    let open = 0;
+    let labeled = 0;
+    for (const rec of records) {
+        for (const row of rec.results) {
+            if (!row.rendered)
+                continue;
+            const label = labels.get(labelKey(rec.ts, rec.variant, row.resourceId));
+            if (!label)
+                continue;
+            labeled++;
+            if (label.verdict === "still-open")
+                open++;
+        }
+    }
+    return labeled === 0 ? null : open / labeled;
+}
+/** Compute the rubric's metrics from ledger records plus hand labels. */
+export function summarizeAudit(records, labels) {
+    const labelIndex = new Map();
+    for (const l of labels)
+        labelIndex.set(labelKey(l.ts, l.variant, l.resourceId), l);
+    const byVariant = new Map();
+    for (const rec of records) {
+        const list = byVariant.get(rec.variant) ?? [];
+        list.push(rec);
+        byVariant.set(rec.variant, list);
+    }
+    const perDay = (recs) => {
+        const days = new Map();
+        for (const rec of recs) {
+            const day = rec.ts.slice(0, 10);
+            const list = days.get(day) ?? [];
+            list.push(rec);
+            days.set(day, list);
+        }
+        return days;
+    };
+    const baselineDays = perDay(byVariant.get("baseline") ?? []);
+    const summaries = [];
+    for (const [variant, recs] of byVariant) {
+        const verdicts = emptyVerdictCounts();
+        const ageDaysByVerdict = {
+            "still-open": [],
+            resolved: [],
+            "not-a-loop": [],
+        };
+        let bullets = 0;
+        let wastedSlots = 0;
+        let labeled = 0;
+        let snippetInsufficient = 0;
+        const seenInRun = new Map();
+        for (const rec of recs) {
+            bullets += rec.renderedCount;
+            wastedSlots += rec.wastedSlots;
+            for (const id of new Set(rec.results.map((r) => r.resourceId))) {
+                seenInRun.set(id, (seenInRun.get(id) ?? 0) + 1);
+            }
+            for (const row of rec.results) {
+                if (!row.rendered)
+                    continue;
+                const label = labelIndex.get(labelKey(rec.ts, rec.variant, row.resourceId));
+                if (!label)
+                    continue;
+                labeled++;
+                verdicts[label.verdict]++;
+                if (label.snippetInsufficient)
+                    snippetInsufficient++;
+                if (row.createdAt) {
+                    const age = (new Date(rec.ts).getTime() - new Date(row.createdAt).getTime()) / 86_400_000;
+                    if (Number.isFinite(age))
+                        ageDaysByVerdict[label.verdict].push(Math.max(0, Math.round(age)));
+                }
+            }
+        }
+        let repeatRate = null;
+        if (recs.length >= 2 && seenInRun.size > 0) {
+            const repeating = [...seenInRun.values()].filter((n) => n > recs.length / 2).length;
+            repeatRate = round4(repeating / seenInRun.size);
+        }
+        let pairedDays = null;
+        let pairedWins = null;
+        if (variant !== "baseline") {
+            pairedDays = 0;
+            pairedWins = 0;
+            for (const [day, dayRecs] of perDay(recs)) {
+                const baseRecs = baselineDays.get(day);
+                if (!baseRecs)
+                    continue;
+                const baseRate = dayHitRate(baseRecs, labelIndex);
+                const varRate = dayHitRate(dayRecs, labelIndex);
+                if (baseRate === null || varRate === null)
+                    continue;
+                pairedDays++;
+                if (varRate > baseRate)
+                    pairedWins++;
+            }
+        }
+        summaries.push({
+            variant,
+            runs: recs.length,
+            bullets,
+            wastedSlots,
+            labeled,
+            verdicts,
+            snippetInsufficient,
+            hitRate: labeled === 0 ? null : round4(verdicts["still-open"] / labeled),
+            repeatRate,
+            ageDaysByVerdict,
+            pairedDays,
+            pairedWins,
+        });
+    }
+    // Baseline first, then alphabetical — stable regardless of ledger order.
+    summaries.sort((a, b) => a.variant === "baseline" ? -1 : b.variant === "baseline" ? 1 : a.variant.localeCompare(b.variant));
+    return summaries;
+}
+function median(values) {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+export function formatRunHuman(rec) {
+    const lines = [
+        `=== loops audit — ${rec.variant} ===`,
+        `query: ${JSON.stringify(rec.query)}`,
+        `limit: ${rec.limit}   after: ${rec.after ?? "(none)"}   userId: ${rec.userId ?? "(config default)"}`,
+    ];
+    if (rec.results.length === 0)
+        lines.push("  (no results)");
+    for (const r of rec.results) {
+        const score = r.score === null ? "     —" : r.score.toFixed(4);
+        const when = r.createdAt ? r.createdAt.slice(0, 10) : "          ";
+        const state = r.rendered ? "rendered" : "DROPPED: no highlights (wasted slot)";
+        lines.push(`  ${r.rank}. ${score}  ${when}  ${r.title ?? `[${r.source}]`}  (${r.resourceId})  [${state}]`);
+        if (r.snippet)
+            lines.push(`       ${r.snippet}`);
+    }
+    lines.push(`rendered: ${rec.renderedCount}/${rec.resultCount}   wasted slots: ${rec.wastedSlots}`);
+    if (rec.blockConsistent !== null) {
+        lines.push(rec.blockConsistent
+            ? "as-injected block agrees with the diagnostic search"
+            : "WARNING: as-injected block and diagnostic search DIVERGED — corpus moved between calls; rerun");
+    }
+    if (rec.asInjected)
+        lines.push("", rec.asInjected);
+    else if (rec.blockConsistent !== null && !rec.contentEnabled) {
+        lines.push("(set HYPERSPELL_AUDIT_CONTENT=1 to include snippets + the as-injected block)");
+    }
+    return lines.join("\n");
+}
+export function formatComparisonHuman(variant, cmp) {
+    return [
+        `--- ${variant} vs baseline ---`,
+        `  overlap:       ${cmp.overlap.length === 0 ? "(none)" : cmp.overlap.join(", ")}`,
+        `  only baseline: ${cmp.onlyBaseline.length === 0 ? "(none)" : cmp.onlyBaseline.join(", ")}`,
+        `  only ${variant}: ${cmp.onlyVariant.length === 0 ? "(none)" : cmp.onlyVariant.join(", ")}`,
+    ].join("\n");
+}
+export function formatSummaryHuman(summaries) {
+    const lines = ["loops audit — summary (rubric: docs/loops-audit-rubric.md)", ""];
+    for (const s of summaries) {
+        lines.push(`${s.variant}: ${s.runs} run(s), ${s.bullets} bullet(s), ${s.wastedSlots} wasted slot(s)`);
+        if (s.labeled === 0) {
+            lines.push("  no labeled bullets yet");
+        }
+        else {
+            lines.push(`  hit rate: ${((s.hitRate ?? 0) * 100).toFixed(0)}%  (${s.verdicts["still-open"]} still-open / ${s.labeled} labeled)`, `  verdicts: still-open=${s.verdicts["still-open"]} resolved=${s.verdicts.resolved} not-a-loop=${s.verdicts["not-a-loop"]} snippet-insufficient=${s.snippetInsufficient}`);
+            for (const v of LOOP_VERDICTS) {
+                const m = median(s.ageDaysByVerdict[v]);
+                if (m !== null)
+                    lines.push(`  median age (${v}): ${m}d over ${s.ageDaysByVerdict[v].length} item(s)`);
+            }
+        }
+        if (s.repeatRate !== null)
+            lines.push(`  repeat rate: ${(s.repeatRate * 100).toFixed(0)}% of surfaced ids appear in >half the runs`);
+        if (s.pairedDays !== null)
+            lines.push(`  paired days vs baseline: ${s.pairedWins}/${s.pairedDays} win(s)`);
+        lines.push("");
+    }
+    return lines.join("\n").trimEnd();
+}