@elvisw/dsh-magic-context 0.0.0-stage → 0.13.0

package/lib/core/machine-domain.js

@@ -0,0 +1,106 @@
+/**
+ * WP10 — the `machine` domain: what every project on THIS machine can share,
+ * with an operator-supplied exclusion list.
+ *
+ * It is a domain like the other two — same identity shape, same peer lane, same
+ * cap derivation — so nothing in recall, search or the watermark needs to know
+ * it exists. What is specific to it is only WHO the members are: this machine's
+ * projects, minus the ones the operator keeps out.
+ *
+ * Two decisions carry the design:
+ *
+ *  1. **Membership is added, never inferred away.** The identities this process
+ *     has seen are a PARTIAL view — it has served some projects, not all — so
+ *     "not seen" can never mean "not a member". Only an explicit exclusion
+ *     removes a member. Getting this wrong would delete a colleague's project
+ *     from the domain because nobody opened it this hour.
+ *  2. **Disabling is reversible and never destructive.** Turning the switch off
+ *     flips the domain's `enabled` flag and leaves the membership rows alone, so
+ *     switching it back on restores exactly what was there. The flag is what the
+ *     activation function reads, so a disabled domain is genuinely invisible
+ *     rather than merely unused.
+ */
+import { DEFAULT_DOMAIN_BUDGET_TOKENS, defaultDomainCap, domainIdentity, } from "./domains.js";
+/**
+ * The machine domain's BIRTH allowlist (2026-10-12, user verdict "A"): native
+ * machine facts are environment/configuration constraints, so the contract's
+ * conservative `["CONSTRAINTS"]` default forced every one of them to masquerade
+ * as CONSTRAINTS to publish at all (#1515 did, #1537 was refused). Scope is the
+ * v2 FIVE share categories (the only values `parseShareCategories` accepts);
+ * `ENVIRONMENT` exists in the 8-category decay table but is NOT a share
+ * category. Applies only to NEW domains — an existing row's hand-edited
+ * allowlist survives every sync, and the live row was widened by the same
+ * verdict.
+ */
+export const MACHINE_DEFAULT_SHARE_CATEGORIES = [
+    'CONSTRAINTS',
+    'CONFIG_VALUES',
+    'ARCHITECTURE',
+];
+/**
+ * Decide the machine domain's next state. Pure, so the membership rules are
+ * testable without a store.
+ */
+export function planMachineDomain(input) {
+    const enabled = input.config.enabled === true;
+    const excluded = new Set(input.config.excludeProjects ?? []);
+    const existing = [...new Set(input.existingMembers)];
+    if (!enabled) {
+        // While off, membership is not touched: the only thing that changes is the
+        // flag (applied by the caller), because off must be as reversible as on.
+        return { enabled: false, members: existing, add: [], remove: [], changed: false };
+    }
+    const target = [...new Set(input.identities.filter(identity => !excluded.has(identity)))].sort();
+    const targetSet = new Set(target);
+    const existingSet = new Set(existing);
+    const add = target.filter(identity => !existingSet.has(identity));
+    // Only explicit exclusions remove a member — never a partial view of identity.
+    const remove = existing.filter(identity => excluded.has(identity));
+    return {
+        enabled: true,
+        members: [...new Set([...target, ...existing.filter(identity => !excluded.has(identity))])].sort(),
+        add,
+        remove,
+        changed: add.length > 0 || remove.length > 0,
+    };
+}
+/**
+ * Apply the plan. Idempotent: a second call with the same inputs reports
+ * `changed: false` and writes nothing, so it is safe to run on every pass.
+ */
+export function syncMachineDomain(stores, input) {
+    const domainId = domainIdentity('machine');
+    const existing = stores.getDomain(domainId);
+    const enabled = input.config.enabled === true;
+    // Nothing configured and nothing remembered: not even a row to keep in sync.
+    if (existing === undefined && !enabled) {
+        return { enabled: false, members: [], add: [], remove: [], changed: false };
+    }
+    const existingMembers = stores.listDomainMembers(domainId).map(member => member.identity);
+    const plan = planMachineDomain({
+        identities: input.identities,
+        existingMembers,
+        config: input.config,
+    });
+    if (existing === undefined) {
+        // Born with the derived cap and the contract's default allowlist, exactly
+        // like an imported domain — the operator can still edit both afterwards.
+        stores.upsertDomain({
+            domainId,
+            kind: 'machine',
+            enabled: plan.enabled,
+            budgetTokens: DEFAULT_DOMAIN_BUDGET_TOKENS.machine,
+            cap: defaultDomainCap('machine'),
+            shareCategories: MACHINE_DEFAULT_SHARE_CATEGORIES,
+        }, input.now);
+    }
+    else if (existing.enabled !== plan.enabled) {
+        // Flip only the switch: a hand-edited cap or allowlist must survive a sync.
+        stores.setDomainEnabled(domainId, plan.enabled, input.now);
+    }
+    for (const identity of plan.add)
+        stores.addDomainMember(domainId, identity, input.now);
+    for (const identity of plan.remove)
+        stores.removeDomainMember(domainId, identity);
+    return plan;
+}

package/lib/core/maintenance.js

@@ -0,0 +1,380 @@
+import { memoryContentHash } from "./memory-write.js";
+import { LlmDeadlineError, withDeadline } from "./llm-deadline.js";
+/**
+ * Process-memory sample for the tick's `memory-sample` journal row.
+ *
+ * The host is long-lived, and the only leak class that ever bit upstream MC in
+ * this shape (0.49.1 #681) was "every completed pass kept the previous pass plus
+ * its full message history". Our static audit (2026-10-15) found no such
+ * retention, so a once-per-tick series is what keeps a future regression
+ * visible: rising RSS across days, with no profiler and no extra requests.
+ * Pure so the unit test can pin the rounding exactly.
+ */
+export function memorySample(usage) {
+    const mb = (bytes) => Math.round((bytes / (1024 * 1024)) * 10) / 10;
+    return {
+        rssMb: mb(usage.rss),
+        heapUsedMb: mb(usage.heapUsed),
+        heapTotalMb: mb(usage.heapTotal),
+        externalMb: mb(usage.external),
+    };
+}
+/** TTL rows archived per tick; the list is already capped by `listExpired`. */
+const EXPIRE_BATCH = 200;
+/** Unverified rows judged per tick — an LLM call per tick. Sized against the
+ * observed promotion inflow (peak ~165 rows/day on 2026-10-09): at the 30-min
+ * cadence the old 3/tick capped the sweep at 144 rows/day — the ceiling
+ * itself. 16 keeps ~6× headroom at a trivial prompt size (16 × 400-char
+ * previews). */
+const VERIFY_BATCH = 16;
+/** Per-row content budget in the verification prompt. */
+const CONTENT_PREVIEW = 400;
+/** Output cap. Reasoning is billed against it and a heavy thinking pass can
+ * consume the whole budget before emitting any text (measured: 8192 produced
+ * zero deltas on one tick and three verdicts on another). 32768 is the
+ * historian's ceiling — the same floor/ceiling reasoning applies here. */
+const VERIFY_MAX_TOKENS = 32768;
+/** Wall-clock budget for one verify round trip (upstream MC's dreamer applies a
+ * per-batch deadline for the same reason: a provider stream that never settles
+ * must not own the lane's only execution slot). Generous against observed calls
+ * (a few seconds), strict enough that a hung one lands in the next tick. */
+const VERIFY_TIMEOUT_MS = 120_000;
+/**
+ * Rows read from the unverified window per tick, before the skip filter.
+ *
+ * The window itself is a SQL LIMIT (see {@link listMemoriesSql} in
+ * memory-write.ts), so a row the model could not judge must not simply occupy a
+ * slot forever: `skip` verdicts are remembered in `metadata_json` and filtered
+ * AFTER the window read but BEFORE `VERIFY_BATCH` is taken. The scan window is
+ * 8× the batch so a realistic run of skips still leaves a full batch of live
+ * candidates; it stays one cheap indexed read either way.
+ */
+const VERIFY_SCAN = VERIFY_BATCH * 8;
+const VERIFY_INSTRUCTION = [
+    'You are auditing project memories for a coding agent. For each memory, judge whether it is still true and useful for this project:',
+    '- "verified": still accurate and worth keeping.',
+    '- "stale": outdated or partly wrong; provide the corrected "content" (same style, self-contained).',
+    '- "wrong": plain incorrect or harmful; provide a short "reason".',
+    '- "skip": cannot judge from the text alone.',
+    'Also score every memory with "importance" (1-100): how load-bearing is it for future sessions? Roughly: 80+ project rules/constraints, 60-79 architecture/config facts, 40-59 routine context, under 40 trivia.',
+    'Judge only from general software-engineering plausibility and internal consistency; you have no file access.',
+    'Reply with ONLY a JSON object: {"verdicts":[{"id":<number>,"verdict":"verified"|"stale"|"wrong"|"skip","importance"?:number,"content"?:string,"reason"?:string}]}',
+].join('\n');
+/** Salvage complete `{...}` objects from a truncated verdict array. A reasoning
+ * model that runs out of output mid-array still yields usable leading verdicts;
+ * discarding the whole call would just retry the same truncation next tick. */
+function salvageObjects(inner) {
+    const out = [];
+    let depth = 0;
+    let start = -1;
+    let inString = false;
+    let escaped = false;
+    for (let i = 0; i < inner.length; i += 1) {
+        const ch = inner[i];
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (ch === '\\')
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+            continue;
+        }
+        if (ch === '{') {
+            if (depth === 0)
+                start = i;
+            depth += 1;
+            continue;
+        }
+        if (ch === '}') {
+            depth -= 1;
+            if (depth === 0 && start >= 0) {
+                try {
+                    out.push(JSON.parse(inner.slice(start, i + 1)));
+                }
+                catch { /* skip malformed object */ }
+                start = -1;
+            }
+        }
+    }
+    return out;
+}
+/** Extract the first JSON object from a model reply (tolerates prose fences). */
+function parseVerdicts(text) {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start < 0 || end <= start)
+        return [];
+    const slice = text.slice(start, end + 1);
+    let parsed;
+    try {
+        parsed = JSON.parse(slice);
+    }
+    catch {
+        const arrayStart = slice.indexOf('[');
+        if (arrayStart < 0)
+            return [];
+        parsed = { verdicts: salvageObjects(slice.slice(arrayStart + 1)) };
+    }
+    const verdicts = parsed.verdicts;
+    if (!Array.isArray(verdicts))
+        return [];
+    const out = [];
+    for (const item of verdicts) {
+        const row = item;
+        if (typeof row.id !== 'number' || typeof row.verdict !== 'string')
+            continue;
+        out.push({
+            id: row.id,
+            verdict: row.verdict,
+            ...(typeof row.importance === 'number' && Number.isFinite(row.importance) ? { importance: row.importance } : {}),
+            ...(typeof row.content === 'string' && row.content.trim().length > 0 ? { content: row.content.trim() } : {}),
+            ...(typeof row.reason === 'string' && row.reason.trim().length > 0 ? { reason: row.reason.trim() } : {}),
+        });
+    }
+    return out;
+}
+/** One LLM round trip. Returns the raw text, or the reason it produced none —
+ * the reason is journaled (a bare "verify call failed" hid six live ticks
+ * worth of root cause). The provider's usage sample rides along when the
+ * stream reports one: it is the dream token anchor the patrol watches. */
+async function collectText(llm, deps, userPrompt) {
+    const budgetMs = deps.verifyTimeoutMs ?? VERIFY_TIMEOUT_MS;
+    try {
+        return await withDeadline(async (signal) => {
+            const byIndex = new Map();
+            let truncated = false;
+            let usage;
+            const chunks = llm.stream({
+                provider: deps.route.provider,
+                model: deps.route.model,
+                // The DSH llm seam takes content BLOCKS, not a bare string (the host
+                // calls content.some(...) — a string threw "content.some is not a
+                // function" and failed every tick silently).
+                messages: [{ role: 'user', content: [{ type: 'text', text: userPrompt }] }],
+                maxTokens: VERIFY_MAX_TOKENS,
+                sessionId: deps.sessionId,
+                purpose: 'compaction',
+                signal,
+            });
+            for await (const raw of chunks) {
+                const chunk = raw;
+                if (chunk.type === 'text-delta' && typeof chunk.index === 'number' && typeof chunk.text === 'string') {
+                    byIndex.set(chunk.index, (byIndex.get(chunk.index) ?? '') + chunk.text);
+                }
+                else if (chunk.type === 'usage') {
+                    // Same capture as the historian's collector: keep the numeric fields.
+                    const reported = raw.usage;
+                    const numeric = {};
+                    for (const [key, value] of Object.entries(reported ?? {})) {
+                        if (typeof value === 'number')
+                            numeric[key] = value;
+                    }
+                    usage = numeric;
+                }
+                else if (chunk.type === 'finish') {
+                    // Same terminal-reason semantics as the historian's collector:
+                    // error/aborted fail with the failure detail. max-tokens is NOT fatal
+                    // here: the verdict list is a flat JSON array, so whatever objects made
+                    // it out before the cap are still worth banking (MC's "a timed-out
+                    // batch banks what it verified"); rows without a verdict stay
+                    // unverified and are retried on a later tick.
+                    const reason = raw.reason;
+                    if (reason !== undefined && (reason.kind === 'error' || reason.kind === 'aborted')) {
+                        const detail = reason.failure?.message ?? reason.failure?.code;
+                        return { ok: false, why: `stream finished (${reason.kind})${detail === undefined ? '' : `: ${detail}`}` };
+                    }
+                    if (reason?.kind === 'max-tokens')
+                        truncated = true;
+                }
+            }
+            const text = [...byIndex.entries()].sort(([a], [b]) => a - b).map(([, value]) => value).join('');
+            if (text.length === 0) {
+                return { ok: false, why: truncated ? 'stream hit max-tokens with no text' : 'stream produced no text deltas' };
+            }
+            return { ok: true, text, truncated, ...(usage === undefined ? {} : { usage }) };
+        }, budgetMs);
+    }
+    catch (error) {
+        // A spent budget is reported like any other failed call (journaled by the
+        // caller, retried next tick) — the point is that it is reported at all.
+        return { ok: false, why: error instanceof LlmDeadlineError ? `deadline ${error.message}` : error instanceof Error ? error.message : String(error) };
+    }
+}
+function renderRow(row, nowMs) {
+    const ageDays = Math.max(0, Math.round((nowMs - row.createdAt) / 86_400_000));
+    const content = row.content.length > CONTENT_PREVIEW ? row.content.slice(0, CONTENT_PREVIEW) + '…' : row.content;
+    return `- id ${String(row.id)} [${row.category}] (age ${String(ageDays)}d): ${content.replace(/\s+/g, ' ')}`;
+}
+/** Run one maintenance tick. Never throws — every failure is journaled. */
+export async function runMaintenanceTick(deps) {
+    const now = (deps.now ?? Date.now)();
+    const writer = deps.writer;
+    const identity = deps.identity;
+    const tally = { expired: 0, verified: 0, updated: 0, archived: 0, verifyRan: false };
+    if (!writer || !identity)
+        return { ...tally, reason: 'writer-unavailable' };
+    if (writer.status.readOnly === true)
+        return { ...tally, reason: 'read-only' };
+    // 1) TTL expiry — archive is the contract action, so the mutation log and
+    //    authority guard stay authoritative even in bulk maintenance.
+    for (const id of writer.listExpired(identity, now, EXPIRE_BATCH)) {
+        const result = writer.archiveMemory({ identity, id, reason: 'ttl-expired' });
+        if (result.ok)
+            tally.expired += 1;
+    }
+    // 2) Verification — configured route or nothing (no session to inherit).
+    if (!deps.llm || !deps.route) {
+        const pool = writer.stats(identity).unverified;
+        deps.journal.info('maintenance', `maintenance: expired ${String(tally.expired)} (verify off: no llm route; pool ${String(pool)})`, {
+            expired: tally.expired,
+            poolUnverified: pool,
+            verify: 'off',
+        });
+        return { ...tally };
+    }
+    // The unverified filter runs in SQL (before the limit): filtering a fixed
+    // render window instead let the sweep stall once its first rows were
+    // verified, because verification deliberately leaves `updated_at` alone and
+    // those rows keep their window slots. Recurrence (seen_count ≥ 2) jumps the
+    // queue — see listMemoriesSql.
+    //
+    // Second stall guard (2026-11-11): a `skip` verdict ("cannot judge from the
+    // text alone") used to leave NO trace, so the same rows were asked again on
+    // every tick — and with ≥ VERIFY_BATCH such rows the sweep would circle
+    // forever while every tick paid for an LLM call. The verdict is now remembered
+    // in `metadata_json` (content hash), and remembered rows are stepped over
+    // here. The hash is compared against the CURRENT content, so any edit (a
+    // `stale` rewrite, a human edit, a re-promotion) puts the row back in the
+    // pool automatically.
+    const window = writer.listMemories(identity, { limit: VERIFY_SCAN, now, unverifiedOnly: true });
+    const skipHashes = window.length === 0
+        ? undefined
+        : writer.verifySkipHashes(identity, window.map(row => row.id));
+    const candidates = [];
+    let rememberedSkips = 0;
+    for (const row of window) {
+        if (skipHashes?.get(row.id) === memoryContentHash(row.content)) {
+            rememberedSkips += 1;
+            continue;
+        }
+        candidates.push(row);
+        if (candidates.length === VERIFY_BATCH)
+            break;
+    }
+    if (candidates.length === 0) {
+        deps.journal.info('maintenance', `maintenance: expired ${String(tally.expired)} (nothing to verify)`, {
+            expired: tally.expired,
+            poolUnverified: 0,
+            ...rememberedSkips === 0 ? {} : { rememberedSkips },
+            verify: 'idle',
+        });
+        return { ...tally };
+    }
+    const prompt = VERIFY_INSTRUCTION
+        + '\n\nProject identity: ' + identity + '\nMemories:\n'
+        + candidates.map(row => renderRow(row, now)).join('\n');
+    const call = await collectText(deps.llm, deps, prompt);
+    if (!call.ok) {
+        deps.journal.warn('maintenance', `maintenance: verify call failed (retried next tick): ${call.why}`, {
+            candidates: candidates.length,
+            why: call.why,
+            route: `${deps.route.provider}/${deps.route.model}`,
+            verify: 'failed',
+        });
+        return { ...tally, reason: 'llm-failed' };
+    }
+    const text = call.text;
+    const verdicts = parseVerdicts(text);
+    // Dream token anchor: the same {inputTokens,outputTokens} shape the
+    // historian banks, so mc-report parses both lanes identically.
+    const usage = call.usage !== undefined
+        ? { inputTokens: call.usage.inputTokens ?? 0, outputTokens: call.usage.outputTokens ?? 0 }
+        : undefined;
+    const verifiedIds = [];
+    const scored = [];
+    const lagMs = [];
+    const known = new Map(candidates.map(row => [row.id, row]));
+    let skipped = 0;
+    for (const verdict of verdicts) {
+        const row = known.get(verdict.id);
+        if (row === undefined)
+            continue;
+        // A score is independent of the verdict's direction — bank it whenever the
+        // model offered one; classifyMemories self-filters archived/foreign rows.
+        if (verdict.importance !== undefined)
+            scored.push({ id: verdict.id, importance: verdict.importance });
+        if (verdict.verdict === 'verified') {
+            verifiedIds.push(verdict.id);
+            lagMs.push(now - row.createdAt);
+        }
+        else if (verdict.verdict === 'stale' && verdict.content !== undefined) {
+            const result = writer.updateMemory({ identity, id: verdict.id, content: verdict.content });
+            if (result.ok) {
+                tally.updated += 1;
+                lagMs.push(now - row.createdAt);
+            }
+        }
+        else if (verdict.verdict === 'wrong') {
+            const result = writer.archiveMemory({ identity, id: verdict.id, reason: 'verify-wrong' });
+            if (result.ok) {
+                tally.archived += 1;
+                lagMs.push(now - row.createdAt);
+            }
+        }
+        else if (verdict.verdict === 'skip') {
+            skipped += 1;
+            // Remember it (metadata only, no status change: a skip stays
+            // inconclusive). Keyed by content hash so a later edit re-opens the row.
+            writer.recordVerifySkip({
+                identity,
+                id: verdict.id,
+                contentHash: memoryContentHash(row.content),
+                at: now,
+            });
+        }
+    }
+    if (verifiedIds.length > 0)
+        tally.verified += writer.markVerified(identity, verifiedIds, now);
+    // Classify scoring rides the same verdict round trip (MC's classify slice):
+    // metadata-only, applied AFTER the content verdicts so a stale rewrite is
+    // scored on its corrected text and archived rows self-filter through the
+    // status clause. Feed m[0]'s importance ranking (memory-selection order).
+    const scoredApplied = scored.length > 0 ? writer.classifyMemories(identity, scored, now) : 0;
+    tally.verifyRan = true;
+    // Telemetry anchors for the memory lane (mc-report aggregates these):
+    // verification lag and the unverified pool size are the staging trigger
+    // lines, and wrong-rate comes from archived/verdicts.
+    const lagHours = lagMs.map(ms => ms / 3_600_000);
+    const lagMaxH = lagHours.length > 0 ? Math.max(...lagHours) : undefined;
+    const lagAvgH = lagHours.length > 0 ? lagHours.reduce((sum, value) => sum + value, 0) / lagHours.length : undefined;
+    const round1 = (value) => Math.round(value * 10) / 10;
+    const pool = writer.stats(identity).unverified;
+    deps.journal.info('maintenance', `maintenance: expired ${String(tally.expired)} · verified ${String(tally.verified)}`
+        + ` · updated ${String(tally.updated)} · archived ${String(tally.archived)}`
+        + ` · scored ${String(scoredApplied)} · pool ${String(pool)}`
+        + (usage !== undefined ? ` · tokens ${String(usage.inputTokens)}/${String(usage.outputTokens)}` : '')
+        + (call.truncated ? ` (truncated: banked ${String(verdicts.length)}/${String(candidates.length)} verdicts)` : ''), {
+        expired: tally.expired,
+        verified: tally.verified,
+        updated: tally.updated,
+        archived: tally.archived,
+        scored: scoredApplied,
+        skipped,
+        candidates: candidates.length,
+        verdicts: verdicts.length,
+        poolUnverified: pool,
+        verify: 'ran',
+        ...rememberedSkips === 0 ? {} : { rememberedSkips },
+        ...(usage !== undefined ? { usage } : {}),
+        ...(lagMaxH !== undefined ? { lagMaxH: round1(lagMaxH) } : {}),
+        ...(lagAvgH !== undefined ? { lagAvgH: round1(lagAvgH) } : {}),
+        ...call.truncated ? { truncated: true } : {},
+        route: `${deps.route.provider}/${deps.route.model}`,
+    });
+    return { ...tally };
+}