moflo 4.12.11 → 4.13.0

package/dist/src/cli/memory/learnings-tree.js

@@ -0,0 +1,187 @@
+/**
+ * The filesystem half of the dead-path pass (#1479).
+ *
+ * `memory/learnings-dead-paths.ts` decides what counts as a dead path and never
+ * touches a disk; this decides what "resolves" means against a real checkout.
+ * Kept out of the command so the split the audit is built on — pure judgement,
+ * injected I/O — survives having a second kind of I/O in it.
+ *
+ * Cross-platform (Rule #1): a cited path arrives in the forward-slash form
+ * entries are authored with on every platform and is re-joined with `path.join`,
+ * so the only string that reaches the filesystem carries the host separator.
+ *
+ * @module memory/learnings-tree
+ */
+import * as fs from 'fs';
+import * as pathModule from 'path';
+import { COMMON_WALK_SKIP_NAMES } from '../services/moflo-paths.js';
+/**
+ * Directories never offered as a workspace prefix.
+ *
+ * `COMMON_WALK_SKIP_NAMES` is exactly the right list and is shared rather than
+ * restated so a second copy cannot drift from it: `node_modules` is skipped for
+ * the same reason a `node_modules/` path is never scored — whether it resolves
+ * is a fact about the checkout, not the entry — and every build or vendor output
+ * in it (`dist`, `build`, `target`, `.next`, `vendor`, …) is skipped because a
+ * stale one would resolve a path whose source has been deleted, which is the one
+ * answer a dead-path pass must never give.
+ *
+ * Matched case-insensitively, as every other call site does: NTFS and APFS are
+ * case-insensitive by default, so `Dist/` is the same directory (Rule #1).
+ */
+const PREFIX_SCAN_SKIP = COMMON_WALK_SKIP_NAMES;
+/**
+ * How deep below the project root a prefix may reach.
+ *
+ * One level is not enough in practice. Measured against moflo's own store, a
+ * top-level-only retry left five of twelve nominations citing paths that plainly
+ * exist — `commands/index.ts` for `src/cli/commands/index.ts`, `fl/phases.md`
+ * for `.claude/skills/fl/phases.md` — because the source root a learning writes
+ * relative to is `src/cli`, not `src`. Two levels reaches those; a third starts
+ * resolving paths by coincidence rather than by layout, which costs real
+ * findings.
+ */
+const MAX_PREFIX_DEPTH = 2;
+/**
+ * The widest a directory may be and still be descended into.
+ *
+ * This is what makes depth 2 affordable AND useful, and it is a statement about
+ * layout rather than a budget. A directory holding one or two entries —
+ * `src/` over `cli/`, `packages/` over its packages — is a container, and the
+ * thing learnings write relative to is inside it. A directory holding thirty is
+ * already the source root, and its children are ordinary code directories that
+ * no one writes paths relative to.
+ *
+ * Without it, one wide scratch directory starves the whole prefix budget:
+ * measured on this repo, a `tmp/` full of test fixtures took every slot after
+ * the top level.
+ */
+const MAX_CHILDREN_TO_DESCEND = 12;
+/**
+ * Cap on workspace prefixes.
+ *
+ * Every unresolved path costs one `existsSync` per prefix. The list is built
+ * most-specific-first — declared workspaces, then depth 1, then depth 2 — so the
+ * cap truncates the least valuable end rather than an arbitrary one, and
+ * truncating only ever costs extra nominations, never a wrong archive.
+ */
+export const MAX_WORKSPACE_PREFIXES = 80;
+/** Immediate subdirectories of `dir`, sorted; unreadable reads as none. */
+function subdirectoryNames(dir) {
+    try {
+        return fs
+            .readdirSync(dir, { withFileTypes: true })
+            .filter((e) => (e.isDirectory() || e.isSymbolicLink()) && !PREFIX_SCAN_SKIP.has(e.name.toLowerCase()))
+            .map((e) => e.name)
+            .sort();
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Workspace directories an unresolved path is retried under.
+ *
+ * Learnings are authored from inside a workspace and routinely cite
+ * `src/routes/foo.ts` meaning `packages/api/src/routes/foo.ts`. Declared
+ * workspaces come first because `packages/api` is a more specific answer than
+ * the bare `packages` a directory walk offers; the walk then covers the flat
+ * repo, which has no manifest entry to read, and goes {@link MAX_PREFIX_DEPTH}
+ * deep because the directory a learning writes relative to is usually a source
+ * root nested inside a top-level one.
+ *
+ * Dot directories are included: `.claude/` and `.github/` hold files learnings
+ * cite constantly, and the only one worth skipping is named in
+ * {@link PREFIX_SCAN_SKIP}.
+ *
+ * Returned in forward-slash form: this is the wire format
+ * `memory/learnings-dead-paths.ts` composes with, not a host path.
+ */
+export function listWorkspacePrefixes(projectRoot) {
+    const prefixes = [];
+    const add = (value) => {
+        const clean = value.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+$/, '');
+        if (clean && !prefixes.includes(clean))
+            prefixes.push(clean);
+    };
+    try {
+        const manifest = JSON.parse(fs.readFileSync(pathModule.join(projectRoot, 'package.json'), 'utf-8'));
+        const declared = Array.isArray(manifest?.workspaces)
+            ? manifest.workspaces
+            : manifest?.workspaces?.packages;
+        for (const glob of Array.isArray(declared) ? declared : []) {
+            // `**` is left to the directory walk: expanding it means walking the whole
+            // tree, and the one level `*` covers is the shape every workspace uses.
+            if (typeof glob !== 'string' || glob.includes('**'))
+                continue;
+            const star = glob.indexOf('*');
+            if (star === -1) {
+                add(glob);
+                continue;
+            }
+            const base = glob.slice(0, star).replace(/\/+$/, '');
+            const baseDir = pathModule.join(projectRoot, ...base.split('/').filter(Boolean));
+            for (const child of subdirectoryNames(baseDir))
+                add(base ? pathModule.posix.join(base, child) : child);
+        }
+    }
+    catch {
+        /* No manifest, or not JSON. The directory walk below still applies. */
+    }
+    // Breadth-first, so every depth-1 prefix is in the list before any depth-2 one
+    // and the cap below never trades a shallower prefix for a deeper one.
+    let frontier = subdirectoryNames(projectRoot);
+    for (const name of frontier)
+        add(name);
+    for (let depth = 2; depth <= MAX_PREFIX_DEPTH && frontier.length > 0; depth++) {
+        const next = [];
+        for (const parent of frontier) {
+            // Every descent past the cap costs a `readdirSync` for a prefix the slice
+            // below is about to discard.
+            if (prefixes.length >= MAX_WORKSPACE_PREFIXES)
+                break;
+            const children = subdirectoryNames(pathModule.join(projectRoot, ...parent.split('/')));
+            if (children.length > MAX_CHILDREN_TO_DESCEND)
+                continue;
+            for (const child of children) {
+                const nested = pathModule.posix.join(parent, child);
+                next.push(nested);
+                add(nested);
+            }
+        }
+        frontier = next;
+    }
+    return prefixes.slice(0, MAX_WORKSPACE_PREFIXES);
+}
+/**
+ * Existence check for one repo-relative path cited by a learning.
+ *
+ * Rule #1: the path arrives in the forward-slash form entries are authored with
+ * on every platform, is split on that, and is re-joined with `path.join`, so the
+ * string reaching the filesystem carries the host separator and this file never
+ * writes one. A file OR a directory counts — a moved directory is the same
+ * finding as a moved file.
+ *
+ * Existence is the host's own answer, case-folding included, so a citation that
+ * differs from the real file only in case reads as alive on NTFS/APFS and dead
+ * on a case-sensitive filesystem. That is deliberate: matching the platform is
+ * the only defensible definition of "still in the tree", and the divergence can
+ * only change whether an entry is NOMINATED — never whether one is archived,
+ * which takes a model verdict either way.
+ */
+export function makeTreeResolver(projectRoot) {
+    return (relativePath) => {
+        const segments = relativePath.split('/').filter((s) => s.length > 0 && s !== '.');
+        // A traversal segment would resolve outside the project entirely, so it can
+        // say nothing about whether the repo still contains the cited file.
+        if (segments.length === 0 || segments.includes('..'))
+            return false;
+        try {
+            return fs.existsSync(pathModule.join(projectRoot, ...segments));
+        }
+        catch {
+            return false;
+        }
+    };
+}
+//# sourceMappingURL=learnings-tree.js.map

package/dist/src/cli/memory/memory-bridge.js

@@ -456,7 +456,35 @@ export async function bridgeConsolidate(_params) {
         return { success: false, error: e.message };
     }
 }
+/**
+ * Operations removed in #1465. Both wrote to the `episodes` store via a schema
+ * with no `namespace`, so neither could address a `memory_entries` row the
+ * caller had stored — yet both reported `success: true` with a count taken
+ * from the input array. Callers are redirected to the tools that work.
+ *
+ * A Map, not an object literal: `operation` is caller-controlled, and a plain
+ * object would resolve 'constructor'/'toString'/'__proto__' up the prototype
+ * chain to a truthy function.
+ */
+const BATCH_OPERATION_REDIRECTS = new Map([
+    [
+        'delete',
+        "moflodb_batch no longer supports 'delete' (#1465): it targeted the episodes store and could not address a namespaced memory entry, while reporting success. Use memory_delete with an explicit namespace.",
+    ],
+    [
+        'update',
+        "moflodb_batch no longer supports 'update' (#1465): it targeted the episodes store and could not address a namespaced memory entry, while reporting success. Use memory_store to overwrite an entry.",
+    ],
+]);
 export async function bridgeBatchOperation(params) {
+    // Rejected before the registry lookup so a removed operation reports the
+    // same error whether or not the bridge happens to be available.
+    const redirect = BATCH_OPERATION_REDIRECTS.get(params.operation);
+    if (redirect)
+        return { success: false, error: redirect };
+    if (params.operation !== 'insert') {
+        return { success: false, error: `Unknown operation: ${params.operation}` };
+    }
     const registry = await getRegistry();
     if (!registry)
         return null;
@@ -464,33 +492,15 @@ export async function bridgeBatchOperation(params) {
         const batch = registry.get('batchOperations');
         if (!batch)
             return { success: false, error: 'BatchOperations not available' };
-        let result;
-        switch (params.operation) {
-            case 'insert': {
-                const episodes = params.entries.map((e) => ({
-                    content: e.value || e.content || JSON.stringify(e),
-                    metadata: e.metadata || { key: e.key },
-                }));
-                result = await batch.insertEpisodes(episodes);
-                break;
-            }
-            case 'delete': {
-                const keys = params.entries.map((e) => e.key).filter(Boolean);
-                for (const key of keys)
-                    await batch.bulkDelete('episodes', { key });
-                result = { deleted: keys.length };
-                break;
-            }
-            case 'update': {
-                for (const entry of params.entries) {
-                    await batch.bulkUpdate('episodes', { content: entry.value || entry.content }, { key: entry.key });
-                }
-                result = { updated: params.entries.length };
-                break;
-            }
-            default: return { success: false, error: `Unknown operation: ${params.operation}` };
-        }
-        return { success: true, operation: params.operation, count: params.entries.length, result };
+        const episodes = params.entries.map((e) => ({
+            content: e.value || e.content || JSON.stringify(e),
+            metadata: e.metadata || { key: e.key },
+        }));
+        // `count` reports rows the store actually wrote, never `entries.length` —
+        // an input-derived count is how #1465's delete/update reported success
+        // while changing nothing.
+        const result = await batch.insertEpisodes(episodes);
+        return { success: true, operation: 'insert', count: result.inserted, result };
     }
     catch (e) {
         return { success: false, error: e.message };

package/dist/src/cli/memory/tool-call-markup.js

@@ -0,0 +1,218 @@
+/**
+ * Detect Claude Code tool-call markup captured into a memory value (#1467).
+ *
+ * A model emitting a `memory_store` call can have the harness' own parameter
+ * markup spill into the `value` string it is writing. The value then arrives at
+ * moflo already malformed, always as a fragment the text trails off into:
+ *
+ *     ...the actual lesson text.",
+ *     <parameter name="tags">["a","b","source:manual"]
+ *
+ * moflo does not cause this — the markup exists only in the model/harness layer —
+ * but it is the last component that can catch it. Left alone it is written to
+ * disk, embedded (so it degrades the vector for that entry and every search that
+ * would have matched it), and shared to the team artifact, at which point it is
+ * permanent and propagates to every machine that imports.
+ *
+ * ## Why the detector is anchored rather than a bare marker match
+ *
+ * The naive test — "does the value contain `</value>` or `<parameter name=`" —
+ * rejects the lesson that documents this very bug, and every guidance doc that
+ * quotes the markup. So a marker only counts when it is *structurally broken*
+ * AND *trailing*:
+ *
+ * | Rule | Fires on | Passes |
+ * |---|---|---|
+ * | A — trailing unmatched closer | value ends on a `</value>` / `</parameter>` that never had an opener | balanced `<value>x</value>` prose; a closer mentioned mid-text |
+ * | B — unterminated trailing opener | an opener with no closer, on its own line, within the last {@link TRAILING_WINDOW} chars | an opener quoted inline in a sentence; one explained in the 200+ chars that follow |
+ *
+ * A value that discusses the markup explains it, and the explanation is the
+ * anchor. A value that was *cut off by* the markup has nothing after it.
+ *
+ * ## Accepted blind spots
+ *
+ * The detector is deliberately tuned to miss rather than over-reject, because a
+ * false reject blocks a legitimate write in every consumer while a miss only
+ * leaves one entry as bad as it is today:
+ *
+ * - A captured fragment longer than {@link TRAILING_WINDOW}. Every fragment
+ *   observed in #1467 is under 60 characters; widening the window would start
+ *   rejecting docs that quote the markup.
+ * - A value that quotes `<value>` earlier and is *then* truncated by a stray
+ *   `</value>`: the quotation's opener absorbs the closer and both rules go
+ *   quiet. Matching by tag name is what makes the balanced-prose cases pass.
+ * - A value that begins with markup, having lost all of its own text.
+ *
+ * Rule A's one accepted false positive is the mirror image: a truncated XML
+ * snippet that happens to end on an unmatched `</value>`. It is refusable by
+ * design — the ticket asks for that shape — and {@link MARKUP_OVERRIDE_ENV} is
+ * the way through. Bulk writes (`storeEntries`, whose only caller is the
+ * pattern pre-trainer) are not checked at all, which is where machine-generated
+ * XML-bearing content actually arrives.
+ *
+ * Pure string logic — no fs, no db, no platform surface (Rule #1).
+ *
+ * @module memory/tool-call-markup
+ */
+/**
+ * How close to the end of the value an unterminated opener must sit to count as
+ * a captured fragment rather than a quotation. Observed fragments are short —
+ * `<parameter name="tags">["a","b","source:manual"]` is 47 characters — while
+ * prose that quotes an opener goes on to say something about it.
+ */
+export const TRAILING_WINDOW = 200;
+/** Escape hatch for a value that is genuinely shaped like the corruption. */
+export const MARKUP_OVERRIDE_ENV = 'MOFLO_ALLOW_TOOL_CALL_MARKUP';
+/** How much of the offending tail the error message quotes back. */
+const EXCERPT_LIMIT = 120;
+/**
+ * Matches the four token shapes the harness emits. `[^"]*` for the attribute
+ * keeps the pattern linear — no nested quantifier, so no backtracking blow-up
+ * on a long value. Built per scan rather than shared, so no `lastIndex` state
+ * outlives a call.
+ */
+function tokenPattern() {
+    return /<(\/?)(value|parameter)(\s+name="[^"]*")?\s*>/g;
+}
+/**
+ * Visit every token in `text` at or after `from`. Streaming rather than
+ * array-returning: Rule A has to see the whole value, and a 1 MB value of
+ * repeated `<value>` would otherwise build a token object per match.
+ */
+function scanTokens(text, from, visit) {
+    const re = tokenPattern();
+    re.lastIndex = from;
+    let match;
+    while ((match = re.exec(text)) !== null) {
+        const [full, slash, name, attr] = match;
+        // `<parameter>` without a name, or `<value name="x">`, is neither shape the
+        // harness emits; ignoring them keeps the detector to what it can justify.
+        if (name === 'parameter' && !slash && !attr)
+            continue;
+        if (name === 'value' && attr)
+            continue;
+        visit({
+            name: name,
+            kind: slash ? 'close' : 'open',
+            text: full,
+            index: match.index,
+        });
+    }
+}
+/**
+ * True when `index` starts a line that already had text before it.
+ *
+ * Both halves matter. The harness always puts the captured fragment on its own
+ * line, so prose that quotes an opener mid-sentence is excluded by the line
+ * break. And a value that *starts* at index 0 with markup never had text to be
+ * cut off from, so it is not the shape this detects — without the second half
+ * every value shorter than {@link TRAILING_WINDOW} that opens with markup would
+ * be rejected no matter what followed it.
+ */
+function startsOwnLineAfterText(text, index) {
+    for (let i = index - 1; i >= 0; i--) {
+        const ch = text[i];
+        if (ch === '\n' || ch === '\r')
+            return text.slice(0, i).trim().length > 0;
+        if (ch !== ' ' && ch !== '\t')
+            return false;
+    }
+    return false; // reached the start of the value with no text before the markup
+}
+/**
+ * Return the captured-markup fragment in `value`, or `null` when the value is
+ * clean. See the module header for what "captured" means and why a value that
+ * merely quotes the markup is not.
+ */
+export function detectToolCallMarkup(value) {
+    if (typeof value !== 'string' || value.length === 0)
+        return null;
+    if (!value.includes('<'))
+        return null;
+    const trimmed = value.trimEnd();
+    // Rule A — the value ends on a closing tag that never had an opener. This is
+    // the `</value>` shape: the harness closed the parameter and the closer
+    // landed inside the text it was closing. Two counters, no allocation, so the
+    // cost of a pathological value is time only.
+    const depth = { value: 0, parameter: 0 };
+    let trailingCloser = null;
+    scanTokens(trimmed, 0, (token) => {
+        if (token.kind === 'open') {
+            depth[token.name]++;
+            return;
+        }
+        if (depth[token.name] > 0) {
+            depth[token.name]--;
+            return;
+        }
+        trailingCloser = token.index + token.text.length === trimmed.length ? token : null;
+    });
+    if (trailingCloser) {
+        // The cast is for the narrowing only: TypeScript cannot see that the
+        // callback above assigns, so it holds `trailingCloser` at its initial
+        // `null` type. `scanTokens` is fully synchronous, so the assignment has
+        // certainly happened by here.
+        const hit = trailingCloser;
+        return { marker: hit.text, index: hit.index, reason: 'trailing-closer' };
+    }
+    // Rule B — an opener that is never closed, on its own line, near the end.
+    // This is the `<parameter name="tags">` shape: the value was cut off and the
+    // next parameter of the same call was appended to it.
+    //
+    // Scanned over the trailing window alone: nothing follows the end of the
+    // value, so an opener inside the window can only be closed inside it, and the
+    // cost is bounded no matter how large the value is.
+    //
+    // Not quite identical to a whole-value scan, in one direction only. A tag
+    // that straddles `windowStart` is invisible to this scan, so if its closer
+    // falls inside the window that closer reads as unmatched and can absorb a
+    // genuinely unterminated opener found later in the backward walk. That is a
+    // miss, never a false reject — the same direction every trade-off in this
+    // module leans.
+    const windowStart = Math.max(0, trimmed.length - TRAILING_WINDOW);
+    const windowTokens = [];
+    scanTokens(trimmed, windowStart, (token) => { windowTokens.push(token); });
+    const pendingClosers = { value: 0, parameter: 0 };
+    for (let i = windowTokens.length - 1; i >= 0; i--) {
+        const token = windowTokens[i];
+        if (token.kind === 'close') {
+            pendingClosers[token.name]++;
+            continue;
+        }
+        if (pendingClosers[token.name] > 0) {
+            pendingClosers[token.name]--;
+            continue;
+        }
+        // First unmatched opener found scanning backwards — the last one in the
+        // value, and the only one that can be the captured fragment: a fragment
+        // runs to the end, so anything after it is part of it. Stopping here rather
+        // than continuing to earlier openers is the miss-biased choice on purpose —
+        // if the text running to the end started mid-sentence, it is prose.
+        if (startsOwnLineAfterText(trimmed, token.index)) {
+            return { marker: token.text, index: token.index, reason: 'unterminated-opener' };
+        }
+        break;
+    }
+    return null;
+}
+/** True when the operator has explicitly opted this process out of the check. */
+export function markupCheckDisabled() {
+    return process.env[MARKUP_OVERRIDE_ENV] === '1';
+}
+/**
+ * The rejection message. It quotes the offending tail back so the caller can
+ * see exactly what leaked and re-send the write — the silent `{success: true}`
+ * this replaces is why 68 corrupted entries accumulated unnoticed.
+ */
+export function toolCallMarkupError(hit, value, namespace, key) {
+    const tail = value.trimEnd().slice(hit.index);
+    const excerpt = tail.length > EXCERPT_LIMIT ? `${tail.slice(0, EXCERPT_LIMIT)}…` : tail;
+    const shape = hit.reason === 'trailing-closer'
+        ? 'the value ends on a closing tag that was never opened'
+        : 'the value trails off into an unclosed opening tag';
+    return (`Refusing to store ${namespace}/${key}: the value contains captured tool-call markup — `
+        + `${shape}. Offending text at offset ${hit.index}: ${JSON.stringify(excerpt)}. `
+        + `Re-send the write with the markup removed. `
+        + `(Set ${MARKUP_OVERRIDE_ENV}=1 to store it anyway.)`);
+}
+//# sourceMappingURL=tool-call-markup.js.map

package/dist/src/cli/parser.js

@@ -59,10 +59,14 @@ export class CommandParser {
                 choices: ['text', 'json', 'table']
             },
             {
-                name: 'no-color',
-                description: 'Disable colored output',
+                // Declared POSITIVELY. The long-flag branch below turns `--no-<x>` into
+                // `<x> = false`, so an option named `no-color` can never produce a
+                // `noColor` key and every reader of one is dead (#1474). `--no-color`
+                // still works — that spelling IS the negation this parser performs.
+                name: 'color',
+                description: 'Colored output (--no-color to disable)',
                 type: 'boolean',
-                default: false
+                default: true
             },
             {
                 name: 'interactive',

package/dist/src/cli/services/cherry-pick-learnings.js

@@ -65,6 +65,14 @@ export function isDurableNamespace(namespace) {
  * between the two writers would silently mis-bind or drop rows (INSERT OR IGNORE
  * swallows the constraint violation). Single source of truth for both.
  */
+/**
+ * The 14 durable-row columns, in the order {@link DURABLE_INSERT_OR_IGNORE_SQL}
+ * binds them. Exported so every reader SELECTs exactly what the writer expects
+ * — the read half used to be retyped per call site, which is the same drift
+ * hazard the shared INSERT exists to prevent.
+ */
+export const DURABLE_ROW_COLUMNS = `id, key, namespace, content, type, embedding, embedding_model, ` +
+    `embedding_dimensions, tags, metadata, owner_id, created_at, updated_at, status`;
 export const DURABLE_INSERT_OR_IGNORE_SQL = `INSERT OR IGNORE INTO memory_entries ` +
     `(id, key, namespace, content, type, embedding, embedding_model, ` +
     ` embedding_dimensions, tags, metadata, owner_id, created_at, updated_at, status) ` +
@@ -151,9 +159,7 @@ export async function cherryPickLearningsFromLegacy(options = {}) {
     try {
         targetDb.run(MEMORY_SCHEMA_V3);
         const placeholders = namespaces.map(() => '?').join(',');
-        const selectSql = `SELECT id, key, namespace, content, type, embedding, embedding_model, ` +
-            `embedding_dimensions, tags, metadata, owner_id, created_at, updated_at, status ` +
-            `FROM memory_entries WHERE namespace IN (${placeholders})`;
+        const selectSql = `SELECT ${DURABLE_ROW_COLUMNS} FROM memory_entries WHERE namespace IN (${placeholders})`;
         // Hoisted prepare — avoids re-parsing the SQL for every INSERT. Matters
         // for legacy DBs with hundreds of learnings rows.
         insertStmt = targetDb.prepare(DURABLE_INSERT_OR_IGNORE_SQL);