@elvisw/dsh-magic-context 0.0.0-stage → 0.13.0

package/lib/core/tagger.js

@@ -0,0 +1,7 @@
+export function tagPrefix(tagNumber) {
+    return `§${tagNumber}§ `;
+}
+/** Allocate (idempotently) a tag for one target in one session. */
+export function ensureTag(stores, sessionId, target, now = Date.now()) {
+    return stores.allocateTag(sessionId, target, now);
+}

package/lib/core/types.js

@@ -0,0 +1,15 @@
+/**
+ * Neutral vocabulary shared by the DSH adapter's core logic.
+ *
+ * NOTHING in `src/core` may import from `@deepseek-ai/*`. The DSH contact
+ * surface lives entirely in `src/dsh/`; translation between the two
+ * vocabularies happens there. This is the upgrade-insulation contract:
+ * when DSH iterates, only `src/dsh/` changes; when Magic Context core is
+ * wired in (Phase 1+), it is consumed through its existing host seams and
+ * this package's stores, never through DSH types.
+ */
+/** The one error-to-text spelling: every catch that logs or journals a
+ * failure composes its message through this, never by re-testing the shape. */
+export function errText(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/lib/core/vector-backfill.js

@@ -0,0 +1,140 @@
+/**
+ * WP11 — filling the vector lane.
+ *
+ * Magic Context used to own `memory_embeddings` and refill it. That job left
+ * with it, and the consequence is measurable: on 2026-10-10 the live store had
+ * 1224 active memories, 820 embedded (67%), and the newest embedded id 210 rows
+ * behind the newest id — a lane degrading silently, since a missing vector only
+ * ever costs recall quality, never an error.
+ *
+ * This module writes the missing rows. Every rule below exists because the
+ * failure mode here is SILENT:
+ *
+ *  - **Parity is a precondition, not a per-row check.** A vector written into a
+ *    space this adapter cannot reproduce looks exactly like a good vector
+ *    afterwards, and it degrades every read that touches it. So the shared-space
+ *    gate runs first, and a closed gate aborts the WHOLE batch — never "skip the
+ *    bad ones and write the rest".
+ *  - **The planned space must be the proven space.** The plan re-reads the
+ *    dominant id from the store; if that disagrees with the space parity was
+ *    proven against, the batch is refused rather than written into whichever
+ *    space happens to be dominant now.
+ *  - **An empty plan does not call the embedder.** Not merely an optimisation:
+ *    it is the negative control that proves "no candidates" means "no writes"
+ *    rather than "the loop happened to have nothing to do".
+ *  - **A zero or short vector is never stored.** Both would break cosine
+ *    similarity for every query against that row, silently.
+ *  - **One row never aborts the batch.** A row that fails to embed or store is
+ *    counted and skipped; the rest still land, and the next tick retries the gap.
+ */
+import { errText } from "./types.js";
+/**
+ * A vector of all zeros is not "no information": cosine similarity against it is
+ * undefined (0/0), which some readers turn into NaN and others into a perfect
+ * match. Either way it is worse than a missing row, so it is never stored.
+ */
+export function isUsableVector(vector, dims) {
+    if (vector === undefined || vector.length !== dims)
+        return false;
+    // The whole vector must be scanned: an early `return true` on the first
+    // non-zero entry would never reach a NaN or an Infinity further along, which
+    // is precisely the value that ruins every cosine comparison.
+    let anyNonZero = false;
+    for (const value of vector) {
+        if (!Number.isFinite(value))
+            return false;
+        if (value !== 0)
+            anyNonZero = true;
+    }
+    return anyNonZero;
+}
+export async function runVectorBackfill(deps) {
+    const empty = { created: 0, present: 0, skipped: 0, failed: 0, total: 0 };
+    // (1) Parity first, for the whole batch. The one exception is the first-writer
+    // bootstrap: an empty store has no space to prove parity against, so the
+    // embedder's space becomes the target only when the caller supplies it.
+    if (deps.parityModelId === null && deps.bootstrapModelId === undefined) {
+        deps.journal.info('vector-lane', 'backfill skipped: shared-space parity is unproven', {
+            outcome: 'backfill-parity-unproven',
+            ...empty,
+        });
+        return { ...empty, reason: 'parity-unproven' };
+    }
+    const plan = deps.plan();
+    if (plan === undefined)
+        return { ...empty, reason: 'no-plan' };
+    if (plan.candidates.length === 0) {
+        return { ...empty, total: plan.total, reason: 'no-candidates' };
+    }
+    // (2) The plan's space is re-read from the store; it must be the one we proved,
+    // or the bootstrap space when there was nothing to prove yet.
+    const targetModelId = deps.parityModelId ?? deps.bootstrapModelId;
+    if (targetModelId === undefined || plan.modelId !== targetModelId) {
+        deps.journal.warn('vector-lane', 'backfill skipped: planned space is not the parity-proven space', {
+            outcome: 'backfill-parity-mismatch',
+            planned: plan.modelId,
+            ...deps.parityModelId === null ? { bootstrap: targetModelId } : { proven: targetModelId },
+        });
+        return { ...empty, total: plan.total, reason: 'parity-mismatch' };
+    }
+    const bootstrap = deps.parityModelId === null;
+    if (bootstrap) {
+        deps.journal.info('vector-lane', 'backfill: creating the first embedding space', {
+            outcome: 'backfill-bootstrap',
+            modelId: targetModelId,
+            total: plan.total,
+        });
+    }
+    const batch = plan.candidates.slice(0, Math.max(1, deps.limit));
+    const rows = deps.contents(batch).filter(row => row.content.trim().length > 0);
+    if (rows.length === 0)
+        return { ...empty, total: plan.total, reason: 'nothing-producible' };
+    let vectors;
+    try {
+        vectors = await deps.embed(rows.map(row => row.content));
+    }
+    catch (error) {
+        deps.journal.warn('vector-lane', `backfill embed failed: ${errText(error)}`, {
+            outcome: 'backfill-embed-failed',
+            attempted: rows.length,
+        });
+        return { ...empty, total: plan.total, reason: 'embed-failed' };
+    }
+    let created = 0;
+    let present = 0;
+    let skipped = 0;
+    let failed = 0;
+    for (const [index, row] of rows.entries()) {
+        const vector = vectors[index];
+        if (!isUsableVector(vector, deps.dims)) {
+            skipped += 1;
+            continue;
+        }
+        const stored = deps.store({ memoryId: row.id, modelId: plan.modelId, vector });
+        if (!stored.ok) {
+            failed += 1;
+            continue;
+        }
+        if (stored.created === true)
+            created += 1;
+        else
+            present += 1;
+    }
+    const outcome = {
+        created,
+        present,
+        skipped,
+        failed,
+        total: plan.total,
+    };
+    deps.journal.info('vector-lane', `backfill: +${String(created)} vector(s) in the ${bootstrap ? 'new' : 'dominant'} space`
+        + `${plan.total > batch.length ? ` (${String(plan.total - batch.length)} still missing)` : ''}`, {
+        outcome: 'backfill',
+        ...outcome,
+        modelId: plan.modelId,
+        batch: batch.length,
+        ...outcome.skipped > 0 ? { skipped: outcome.skipped } : {},
+        ...outcome.failed > 0 ? { failed: outcome.failed } : {},
+    });
+    return outcome;
+}

package/lib/core/vector-lane.js

@@ -0,0 +1,184 @@
+/**
+ * The vector lane: measuring the degradation the takeover caused, and planning
+ * (never performing) the backfill.
+ *
+ * Why measuring is the deliverable, not the fix
+ * ---------------------------------------------
+ * Magic Context used to own `memory_embeddings` and refill it. That job has no
+ * owner now, and nothing in the adapter writes embeddings, so the lane decays
+ * silently: recall still *works*, it just answers with fewer neighbours. On
+ * 2026-10-10 the damage was already measurable and unmeasured until now:
+ *
+ *   active 1224 · embedded 904 (73.9%) · missing 404
+ *   newest embedded memory id 1103, newest memory id 1326 — nothing written
+ *   since id 1103 has an embedding, and 194 rows BELOW that frontier lost one
+ *
+ * Two embedding spaces coexist (≈744 rows in one model, ≈96 in another as of
+ * 2026-10-10 — re-measure rather than trusting these). Writing into the wrong
+ * space, or into a third, is the specific failure this module exists to prevent:
+ * `memory_embeddings` is keyed `(memory_id, model_id)`, so a wrong-space row is
+ * not an error — it is simply invisible to the readers that use the other space,
+ * forever.
+ *
+ * So this module only reads: it reports coverage and hands back the id list a
+ * backfill WOULD take. The write itself lives in `vector-backfill.ts` (WP11),
+ * which consumes exactly this plan. The switch (`memory.vectorBackfill`)
+ * defaulted to off until 2026-10-12 and is on by default since (the write path
+ * never embeds, so an off-by-default lane let Chinese semantic recall decay
+ * with every new row); the phase-D blind spot that motivated the original
+ * wording is closed, the parity gate is not optional.
+ */
+import { DatabaseSync } from 'node:sqlite';
+function tableExists(db, name) {
+    return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) !== undefined;
+}
+function columnNames(db, table) {
+    return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(row => row.name));
+}
+/** Read the lane's state from the shared store. `undefined` when it cannot be read. */
+export function measureVectorLane(corpusDbPath) {
+    let db;
+    try {
+        db = new DatabaseSync(corpusDbPath, { readOnly: true });
+    }
+    catch {
+        return undefined;
+    }
+    try {
+        if (!tableExists(db, 'memories') || !tableExists(db, 'memory_embeddings'))
+            return undefined;
+        const memoryColumns = columnNames(db, 'memories');
+        const activeFilter = memoryColumns.has('status') ? "WHERE status = 'active'" : '';
+        const active = Number(db.prepare(`SELECT COUNT(*) AS c FROM memories ${activeFilter}`).get().c);
+        const newestMemoryId = Number(db.prepare(`SELECT COALESCE(MAX(id), 0) AS m FROM memories ${activeFilter}`).get().m);
+        // Space shares are measured over the ACTIVE corpus: the question they answer
+        // is "which space do live memories actually live in", and a reader only ever
+        // looks at active rows.
+        const activeJoin = memoryColumns.has('status') ? "WHERE m.status = 'active'" : '';
+        const spaces = db.prepare(`SELECT e.model_id AS modelId, COUNT(*) AS rows, COUNT(DISTINCT e.memory_id) AS memories
+         FROM memory_embeddings e JOIN memories m ON m.id = e.memory_id ${activeJoin}
+        GROUP BY e.model_id ORDER BY rows DESC, modelId`).all();
+        const totalRows = spaces.reduce((sum, space) => sum + Number(space.rows), 0);
+        const embeddedIds = db.prepare(`SELECT DISTINCT e.memory_id AS id FROM memory_embeddings e JOIN memories m ON m.id = e.memory_id ${memoryColumns.has('status') ? "WHERE m.status = 'active'" : ''}`).all();
+        const embedded = embeddedIds.length;
+        const dominantModelId = spaces[0]?.modelId;
+        const dominantIds = dominantModelId === undefined
+            ? new Set()
+            : new Set(db.prepare(`SELECT DISTINCT e.memory_id AS id FROM memory_embeddings e JOIN memories m ON m.id = e.memory_id
+            WHERE e.model_id = ? ${memoryColumns.has('status') ? "AND m.status = 'active'" : ''}`).all(dominantModelId).map(row => Number(row.id)));
+        const allEmbeddedIds = new Set(embeddedIds.map(row => Number(row.id)));
+        const activeIds = db.prepare(`SELECT id FROM memories ${activeFilter}`).all()
+            .map(row => Number(row.id));
+        // The frontier is the newest ACTIVE memory that has an embedding. Taking the
+        // raw MAX(memory_id) instead would let an archived row's leftover embedding
+        // look like progress and hide the backlog behind it.
+        const newestEmbeddedMemoryId = embeddedIds.reduce((max, row) => Math.max(max, Number(row.id)), 0);
+        let unembeddedAboveFrontier = 0;
+        let gapsBelowFrontier = 0;
+        let dominantCoverageCount = 0;
+        for (const id of activeIds) {
+            if (!allEmbeddedIds.has(id)) {
+                if (id > newestEmbeddedMemoryId)
+                    unembeddedAboveFrontier += 1;
+                else
+                    gapsBelowFrontier += 1;
+            }
+            if (dominantIds.has(id))
+                dominantCoverageCount += 1;
+        }
+        const multiSpaceMemories = Number(db.prepare('SELECT COUNT(*) AS c FROM (SELECT memory_id FROM memory_embeddings GROUP BY memory_id HAVING COUNT(DISTINCT model_id) > 1)').get().c);
+        return {
+            active,
+            embedded,
+            missing: active - embedded,
+            coverage: active === 0 ? 0 : embedded / active,
+            spaces: spaces.map(space => ({
+                modelId: space.modelId,
+                rows: Number(space.rows),
+                memories: Number(space.memories),
+                share: totalRows === 0 ? 0 : Number(space.rows) / totalRows,
+            })),
+            ...dominantModelId === undefined ? {} : { dominantModelId },
+            frontier: { newestEmbeddedMemoryId, newestMemoryId, unembeddedAboveFrontier },
+            gapsBelowFrontier,
+            multiSpaceMemories,
+            dominantCoverage: active === 0 ? 0 : dominantCoverageCount / active,
+        };
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * Which memories a backfill would take, and which space it may write into.
+ *
+ * A candidate is an ACTIVE memory with no embedding row under the DOMINANT
+ * model. That covers both "never embedded" and "embedded only in the other
+ * space" — the second case is how a space migration happens without creating a
+ * third space.
+ *
+ * `bootstrapModelId` is used only when the store has no space at all: it is the
+ * first-writer path for an empty corpus. It never overrides an existing
+ * dominant space.
+ *
+ * Writes nothing. The contract write is still ahead.
+ */
+export function planBackfill(corpusDbPath, options = {}) {
+    const report = measureVectorLane(corpusDbPath);
+    if (report === undefined)
+        return undefined;
+    const limit = options.limit ?? 64;
+    const db = new DatabaseSync(corpusDbPath, { readOnly: true });
+    try {
+        const bootstrap = options.bootstrapModelId?.trim() ?? '';
+        // A bootstrap id may only be used when no space exists at all. Once any
+        // embedding row is present, the dominant space is the only legal target;
+        // accepting a bootstrap id here would be the third-space bug this module
+        // exists to prevent.
+        const dominant = report.dominantModelId ?? (bootstrap === '' ? undefined : bootstrap);
+        if (dominant === undefined) {
+            return { modelId: '', candidates: [], total: 0, reason: 'no embedding space exists yet — the first writer decides the space' };
+        }
+        const memoryColumns = columnNames(db, 'memories');
+        const activeFilter = memoryColumns.has('status') ? "AND m.status = 'active'" : '';
+        // Newest first (2026-10-10). The rows a session is most likely to search for
+        // are the ones written most recently, so the lane closes the frontier gap
+        // before it grinds through the older churn holes. The order is a policy:
+        // tests/vector-lane.spec.ts pins it, and flipping it back is the rollback.
+        const missing = db.prepare(`SELECT m.id AS id FROM memories m
+        WHERE NOT EXISTS (SELECT 1 FROM memory_embeddings e WHERE e.memory_id = m.id AND e.model_id = ?)
+        ${activeFilter}
+        ORDER BY m.id DESC`).all(dominant);
+        const ids = missing.map(row => Number(row.id));
+        return {
+            modelId: dominant,
+            candidates: ids.slice(0, limit),
+            total: ids.length,
+            ...ids.length === 0
+                ? {
+                    reason: report.active === 0
+                        ? 'no active memories to embed'
+                        : 'every active memory already has a row in the dominant space',
+                }
+                : {},
+        };
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        db.close();
+    }
+}
+/** One line for a health report: the shape of the decay, not just a percentage. */
+export function describeVectorLane(report) {
+    const spaces = report.spaces
+        .map(space => `${space.modelId.replace('embedding-provider:', '')} ${String(space.rows)} 行 (${(space.share * 100).toFixed(0)}%)`)
+        .join(' / ');
+    return `active ${String(report.active)} · 已嵌入 ${String(report.embedded)} (${(report.coverage * 100).toFixed(1)}%) · `
+        + `缺 ${String(report.missing)}（前沿之上 ${String(report.frontier.unembeddedAboveFrontier)} / 前沿之下 ${String(report.gapsBelowFrontier)}）· `
+        + `空间：${spaces === '' ? '无' : spaces}`;
+}