@elvisw/dsh-magic-context 0.0.0-stage → 0.13.0

package/lib/dsh/live-journal.js

@@ -0,0 +1,81 @@
+/**
+ * The live journal wrapper — why it exists (measured on this adapter):
+ *
+ * `journal.*` leaves are declared `.volatile()` so the settings form can edit
+ * them, but a volatile-only commit NEVER remounts the plugin — and the journal
+ * used to be constructed once inside `apply()`, so every form edit saved
+ * cleanly and then did nothing until the next app start (the "dead option"
+ * audit, 2026-10-10). The wrapper moves the open/close and the knob reads to
+ * use time:
+ *
+ *  - `enabled`: false stops recording immediately; true (re)opens the journal
+ *    lazily on the next write.
+ *  - `level` / `mirror`: handed to `openJournal` as GETTERS, so an edit
+ *    applies to the very next entry (no reopen).
+ *  - `retentionDays`: the prune that used to run only at open re-runs the
+ *    first time a write observes the configured value changed (floored at 1
+ *    day so a stray manual-YAML zero cannot prune the whole table).
+ *
+ * One asymmetry is deliberate: when the journal is disabled at BOOT the
+ * adapter still wires `journal: undefined` (see index.ts) — the maintenance
+ * tick treats a missing journal as "do not run", and a boot-time opt-out must
+ * keep meaning exactly that. This wrapper only serves compositions that
+ * booted with the journal on.
+ */
+import { join } from 'node:path';
+import { openJournal } from "../core/journal.js";
+import { liveBoolean, liveNumber, liveString } from "./volatile.js";
+import { errText } from "../core/types.js";
+export function liveJournal(options) {
+    const { config, stores, storeDir, logger } = options;
+    const desiredRetention = () => Math.max(1, liveNumber(config.journal?.retentionDays, 30));
+    let inner;
+    let lastRetention;
+    /** (Re)open the journal lazily; re-prune when the retention value changed. */
+    const ensure = () => {
+        if (inner === undefined) {
+            inner = openJournal({
+                stores,
+                level: () => liveString(config.journal?.level, 'info'),
+                mirrorPath: () => liveBoolean(config.journal?.mirror, true)
+                    ? join(storeDir, 'activity.log')
+                    : undefined,
+                retentionDays: desiredRetention(),
+                logger,
+            });
+            lastRetention = desiredRetention();
+            return inner;
+        }
+        const retention = desiredRetention();
+        if (retention !== lastRetention) {
+            lastRetention = retention;
+            try {
+                const pruned = stores.pruneJournal(Date.now() - retention * 24 * 60 * 60_000);
+                if (pruned > 0) {
+                    logger?.info(`magic-context journal: pruned ${String(pruned)} entries past ${String(retention)}d retention (retention edited at runtime)`);
+                }
+            }
+            catch (error) {
+                logger?.warn(`magic-context journal: retention re-prune failed: ${errText(error)}`);
+            }
+        }
+        return inner;
+    };
+    const record = (level, kind, summary, data, sessionId) => {
+        if (liveBoolean(config.journal?.enabled, true) === false)
+            return;
+        ensure().record(level, kind, summary, data, sessionId);
+    };
+    return {
+        get enabled() { return liveBoolean(config.journal?.enabled, true) !== false; },
+        record,
+        info: (kind, summary, data, sessionId) => record('info', kind, summary, data, sessionId),
+        warn: (kind, summary, data, sessionId) => record('warn', kind, summary, data, sessionId),
+        error: (kind, summary, data, sessionId) => record('error', kind, summary, data, sessionId),
+        tail: (limit = 50, sessionId) => inner?.tail(limit, sessionId) ?? [],
+        byKind: since => inner?.byKind(since) ?? [],
+        get mirrorPath() {
+            return liveBoolean(config.journal?.mirror, true) ? join(storeDir, 'activity.log') : undefined;
+        },
+    };
+}

package/lib/dsh/maintenance-wiring.js

@@ -0,0 +1,229 @@
+import { memorySample, runMaintenanceTick } from "../core/maintenance.js";
+import { PROFILE_REVIEW_META_KEY, runProfileReview } from "../core/profile-review.js";
+import { quotaSkipSince, resolveMaintenanceRoute, ROUTE_QUOTA_HOLD_MS, RouteHealth } from "../core/route-health.js";
+/** WP11 default batch: how many vectors one maintenance tick may add. */
+export const VECTOR_BACKFILL_DEFAULT_LIMIT = 24;
+/**
+ * WP11 gate. Default TRUE since 2026-10-12: the memory write path never embeds,
+ * so with the lane off every NEW memory stayed vector-less forever and Chinese
+ * semantic recall (the only channel with real recall on this corpus — FTS with
+ * unicode61 hits ~18%) silently decayed as rows accumulated. The vectors land
+ * in the machine's own single-writer store, bounded by
+ * `vectorBackfillLimit` per tick and gated by the shared-space parity check
+ * (a closed gate aborts the whole batch). Set the key to `false` to keep the
+ * shared store free of writes.
+ */
+export function vectorBackfillEnabled(memory) {
+    return liveBoolean(memory?.vectorBackfill, true);
+}
+/** Live maintenance tickers keyed by adapter DB path — a same-DB remount replaces its predecessor (see wireMaintenance). */
+const maintenanceTickers = new Map();
+import { errText } from "../core/types.js";
+import { autoStageGlobalCandidates } from "../core/global-autostage.js";
+import { liveBoolean, liveNumber, liveString } from "./volatile.js";
+export function wireMaintenance(input) {
+    const { ctx, config, stores, journal, llm, writer, profileWriter, vectorBackfill, routeHealth, lastSessionRoute, projects, disposers, isDisposed } = input;
+    const { seenIdentities } = projects;
+    let maintenanceLast;
+    let profileReviewLast;
+    let vectorBackfillLast;
+    let maintenanceBusy = false;
+    let maintenanceBusySince;
+    const runMaintenance = async () => {
+        if (maintenanceBusy || isDisposed() || stores.closed || journal === undefined)
+            return;
+        if (liveBoolean(config.maintenance?.enabled, true) === false)
+            return;
+        maintenanceBusy = true;
+        maintenanceBusySince = Date.now();
+        try {
+            // Process memory, once per tick (2026-10-15). Upstream MC 0.49.1 (#681)
+            // leaked in exactly this shape — each completed pass kept the previous pass
+            // and its full message history; our static audit found no such retention,
+            // so this series is what would make a future regression visible across days
+            // (no profiler, no LLM call, one journal row per tick).
+            const memoryNow = memorySample(process.memoryUsage());
+            journal.info('memory-sample', `rss ${String(memoryNow.rssMb)} MB · heap ${String(memoryNow.heapUsedMb)}/${String(memoryNow.heapTotalMb)} MB`, memoryNow);
+            // The route every LLM step of this tick uses, resolved ONCE: the explicit
+            // pin wins (rollback lever), otherwise the main conversation's latest
+            // route. A quota observation skips the LLM steps ONLY when the tick is
+            // about to ride the main conversation's own exact route — the event
+            // carries a provider but no model, and quotas are route-scoped, so a
+            // provider-keyed skip would stall a live sibling model (2026-10-09:
+            // model-a exhausted while model-b verified normally).
+            // One info line, zero requests; the observation is volatile and owned by
+            // real traffic, so an externally reset quota re-opens the lane within one
+            // hold at the cost of one cheap probe.
+            const pinnedProvider = liveString(config.maintenance?.verifyProvider, '');
+            const pinnedModel = liveString(config.maintenance?.verifyModel, '');
+            const tickRoute = resolveMaintenanceRoute(pinnedProvider.length > 0 && pinnedModel.length > 0 ? { provider: pinnedProvider, model: pinnedModel } : undefined, lastSessionRoute());
+            const tickQuotaAge = quotaSkipSince(routeHealth, tickRoute, lastSessionRoute(), Date.now());
+            if (tickRoute !== undefined && tickQuotaAge !== undefined) {
+                journal.info('maintenance', `LLM steps skipped: ${tickRoute.provider}/${tickRoute.model} observed quota-exhausted ${String(Math.round(tickQuotaAge / 60_000))}m ago (hold ${String(Math.round(ROUTE_QUOTA_HOLD_MS / 60_000))}m)`, { provider: tickRoute.provider, model: tickRoute.model, quotaAgeMs: tickQuotaAge, route: `${tickRoute.provider}/${tickRoute.model}` });
+            }
+            const tickLlmReady = llm !== undefined && tickRoute !== undefined && tickQuotaAge === undefined;
+            const identities = new Set();
+            if (config.memory?.project !== undefined)
+                identities.add(config.memory.project);
+            for (const identity of seenIdentities)
+                identities.add(identity);
+            for (const identity of identities) {
+                const outcome = await runMaintenanceTick({
+                    writer,
+                    identity,
+                    sessionId: 'mc-maintenance',
+                    journal,
+                    ...(tickLlmReady && tickRoute !== undefined && llm !== undefined
+                        ? { llm: llm, route: tickRoute }
+                        : {}),
+                });
+                // A writer-level problem applies to every identity — stop the sweep.
+                if (outcome.reason === 'writer-unavailable' || outcome.reason === 'read-only')
+                    break;
+                maintenanceLast = {
+                    at: Date.now(),
+                    expired: outcome.expired,
+                    verified: outcome.verified,
+                    updated: outcome.updated,
+                    archived: outcome.archived,
+                };
+            }
+            // L1① v10: classify-then-route global auto-stage. The classifier needs
+            // the SAME LLM seam as the verify lane (route, model, session id), so it
+            // rides the tick's route. Fail-closed: no route (or the pin route on
+            // quota-hold) means NOTHING is staged — the memories stay at project
+            // level, never a guessed fallback domain.
+            if (liveBoolean(config.memory?.global?.machine?.autoStage, true)
+                && tickLlmReady && tickRoute !== undefined && llm !== undefined && writer !== undefined) {
+                const outcome = await autoStageGlobalCandidates({
+                    store: stores,
+                    corpusDbPath: writer.dbPath,
+                    now: Date.now(),
+                    llm: llm,
+                    route: tickRoute,
+                });
+                if (outcome.staged.length > 0) {
+                    journal.info('memory-promote', `auto-staged ${String(outcome.staged.length)} global candidate(s): ${outcome.staged
+                        .map(entry => `#${String(entry.candidateId)}(memory ${String(entry.memoryId)} → ${entry.domainId})`)
+                        .join(', ')}`, { phase: 'auto-stage', staged: outcome.staged, skipped: outcome.skipped });
+                }
+                if (outcome.reason !== undefined) {
+                    journal.info('memory-promote', `auto-stage paused: ${outcome.reason} (memories stay at project level)`, { phase: 'auto-stage', reason: outcome.reason });
+                }
+            }
+            // WP7: the `user` domain's review. It is GLOBAL — one candidate pool, one
+            // profile — so it runs once per tick rather than per identity, and it is
+            // deliberately outside the loop's `break` (the memory writer failing says
+            // nothing about the profile tables, which live in the same file but are
+            // reached through their own connection).
+            //
+            // The throttle is a stored timestamp because DSH has no daily scheduler to
+            // hang MC's `review-user-memories` cadence on. It is written when the pass
+            // reaches a DECIDED outcome — a run, or a nothing-to-do skip — which turns
+            // those skips into a once-per-day heartbeat row instead of per-tick noise
+            // (or per-tick silence). A FAILED call is never stamped, so it is retried
+            // on the next tick instead of being silenced for a day by its own failure.
+            if (liveBoolean(config.memory?.global?.user?.review, true) && profileWriter !== undefined) {
+                const lastReviewRaw = Number(stores.getMeta(PROFILE_REVIEW_META_KEY) ?? '');
+                const review = await runProfileReview({
+                    writer: profileWriter,
+                    sessionId: 'mc-maintenance',
+                    journal,
+                    ...(tickLlmReady && tickRoute !== undefined && llm !== undefined
+                        ? { llm: llm, route: tickRoute }
+                        : {}),
+                    ...(Number.isFinite(lastReviewRaw) && lastReviewRaw > 0 ? { lastReviewAt: lastReviewRaw } : {}),
+                });
+                if (review.attempted === true)
+                    stores.setMeta(PROFILE_REVIEW_META_KEY, String(Date.now()));
+                profileReviewLast = review;
+            }
+            // WP11: fill the vector lane. Also global, also once per tick, and on by
+            // default since 2026-10-12 (see {@link vectorBackfillEnabled} for the
+            // reasoning). `vectorBackfill` returns undefined when the lane is
+            // unusable (no reader/embedder/writer), which is reported as "not run"
+            // rather than silently looking like "nothing missing".
+            if (vectorBackfillEnabled(config.memory) && vectorBackfill !== undefined) {
+                const limit = Math.max(1, Math.floor(config.memory?.vectorBackfillLimit ?? VECTOR_BACKFILL_DEFAULT_LIMIT));
+                const outcome = await vectorBackfill(limit, journal);
+                if (outcome === undefined) {
+                    journal.info('vector-lane', 'backfill lane unavailable (no embedder or no writable store)', {
+                        outcome: 'backfill-unavailable',
+                    });
+                }
+                else {
+                    vectorBackfillLast = outcome;
+                }
+            }
+        }
+        catch (error) {
+            journal.error('maintenance', `maintenance tick failed: ${errText(error)}`);
+        }
+        finally {
+            maintenanceBusy = false;
+            maintenanceBusySince = undefined;
+        }
+    };
+    // One live ticker per adapter store, process-wide. A remount that arrives
+    // without its predecessor's disposal (the GUI-reconnect reload path) used to
+    // STACK a second 60s first-tick and a second interval on the same DB —
+    // measured at 6 ticks/75 minutes. The registry, keyed by `dbPath`, replaces
+    // the live ticker instead: a same-DB remount takes over, different-DB mounts
+    // (tests with separate data dirs) stay independent.
+    const predecessor = maintenanceTickers.get(stores.dbPath);
+    if (predecessor !== undefined)
+        predecessor();
+    let stopped = false;
+    let maintenanceTimer;
+    const stopTicker = () => {
+        if (stopped)
+            return;
+        stopped = true;
+        if (maintenanceTickers.get(stores.dbPath) === stopTicker)
+            maintenanceTickers.delete(stores.dbPath);
+        if (maintenanceTimer !== undefined)
+            clearInterval(maintenanceTimer);
+        clearTimeout(maintenanceFirst);
+    };
+    // The schedule is installed by its OWN timer, not from the first tick's
+    // `finally` (2026-11-11). The old shape — `runMaintenance().finally(() =>
+    // setInterval)` — meant that one tick which never resolved also never
+    // installed the schedule, so TTL expiry, verification, profile review and the
+    // vector lane all stopped for the life of the process, silently (the panel
+    // only ever showed "never ran"). The tick's LLM calls now carry wall-clock
+    // budgets (core/llm-deadline.ts), and the schedule itself no longer depends on
+    // any single tick returning. Cadence is unchanged: first tick at 60s, then one
+    // per `intervalMs`. A tick that still outlives its own interval is a bug, and
+    // is journaled below instead of hiding.
+    const intervalMs = Math.max(60_000, liveNumber(config.maintenance?.intervalMs, 1_800_000));
+    let stallWarned = false;
+    const installInterval = () => {
+        if (stopped || maintenanceTimer !== undefined)
+            return;
+        maintenanceTimer = setInterval(() => {
+            if (maintenanceBusy) {
+                if (!stallWarned && maintenanceBusySince !== undefined && Date.now() - maintenanceBusySince > intervalMs) {
+                    stallWarned = true;
+                    journal?.warn('maintenance', `maintenance tick has been running for over ${String(Math.round((Date.now() - maintenanceBusySince) / 60_000))}m — later ticks hold off until it returns`, { stalledMs: Date.now() - maintenanceBusySince });
+                }
+                return;
+            }
+            stallWarned = false;
+            void runMaintenance();
+        }, intervalMs);
+        maintenanceTimer.unref?.();
+    };
+    const maintenanceFirst = setTimeout(() => {
+        // Install first: the schedule must survive a tick that never settles.
+        installInterval();
+        void runMaintenance();
+    }, 60_000);
+    maintenanceFirst.unref?.();
+    maintenanceTickers.set(stores.dbPath, stopTicker);
+    disposers.push(stopTicker);
+    return {
+        maintenanceLast: () => maintenanceLast,
+        profileReviewLast: () => profileReviewLast,
+        vectorBackfillLast: () => vectorBackfillLast,
+    };
+}