@elvisw/dsh-magic-context 0.0.0-bootstrap.0

package/lib/core/embedding-local.js

@@ -0,0 +1,247 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { createHash } from 'node:crypto';
+import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, statSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline as streamPipeline } from 'node:stream/promises';
+/**
+ * Local MiniLM embedding provider — the adapter's twin of Magic Context's
+ * `memory/embedding-local.ts` (v0.45.0 defaults), running on **onnxruntime-web's
+ * WASM backend** instead of the native onnxruntime-node build.
+ *
+ * Why WASM: the native stack (~287 MB of platform binaries via
+ * `@huggingface/transformers`'s hard dependency) made the distributed package
+ * prohibitive; the WASM runtime is a vendored 27 MB binary with zero native
+ * install steps. Verified 2026-10-09: this path reproduces the native fp32
+ * pipeline byte-for-byte at cosine 1.0000000 over a Chinese/English probe set,
+ * so vectors stay inside Magic Context's shared space (see the parity gate in
+ * memory-lane.ts — verified empirically, never assumed).
+ *
+ * Why this exists at all: `ctx_search` scores meaning with
+ * `cosineSimilarity(query, stored)` (search.ts:722-727). The vectors in the
+ * shared `memory_embeddings` table were produced by Magic Context's own local
+ * pipeline, and our acceptance probe (2026-10-07) proved a locally-embedded
+ * fp32 vector matches a stored one with cosine 1.000000 — the same vector
+ * space, byte for byte. Reading that table and embedding queries here
+ * therefore needs no server.
+ *
+ * Discipline borrowed from MC's `embedding-identity.ts`: local vectors are
+ * fenced to the runtime stack that produced them. The fold below reproduces
+ * MC's identity input exactly (`provider/model/endpoint/apiKeyPresent/
+ * runtimeFingerprint`, fp32 omitted as the default) so `embeddingIdentityHash()`
+ * names the adapter's own vectors honestly; shared-space parity against a store
+ * is checked at runtime by the composition root's `resolveSharedSpace` closure
+ * (cosine against a stored vector), never assumed. The fingerprint names THIS
+ * stack (wasm), which differs from MC's native fold — that is honest naming,
+ * not a compatibility break: reads are gated by the empirical parity check.
+ *
+ * Everything here is lazy and fail-open: the model assets download on the first
+ * `embed()` call, the heavy modules load then too, and any failure resolves to
+ * `null`-shaped telemetry results — search degrades to its FTS lane rather
+ * than failing the tool call.
+ */
+export const EMBEDDING_MODEL = 'Xenova/all-MiniLM-L6-v2';
+/** MC default dtype (`DEFAULT_LOCAL_DTYPE`, embedding-local.ts) — must stay
+ *  fp32 or the vectors leave MC's space and every stored comparison degrades. */
+export const EMBEDDING_DTYPE = 'fp32';
+export const EMBEDDING_DIMS = 384;
+/** Sentence-transformers' convention for this model (tokenizer_config.json
+ *  `model_max_length`); the native pipeline truncated here too. */
+export const EMBEDDING_MAX_SEQ = 512;
+/** The runtime stack behind every vector we produce — onnxruntime-web's wasm
+ *  backend (vendored, see `vendor/onnx/`) plus the official tokenizers binding. */
+export const LOCAL_EMBEDDING_RUNTIME_FINGERPRINT = 'onnxruntime-web@1.31.0-dev.20260914-8d85527a0(wasm);@huggingface/tokenizers@0.2.0';
+/** Model assets, resolved once into `cacheDir` and reused forever after.
+ *  Both the upstream revision AND the sha256 of every byte are pinned: the
+ *  ONNX file is executed by the vendored WASM runtime, so a floating
+ *  `resolve/main` is both a silent-quality hazard (weights change → the
+ *  vector space drifts) and a supply-chain one (hf-mirror is a third party
+ *  serving executable input). Pinned at upstream sha 751bff37182d3f1213fa05d7196b954e230abad9
+ *  (2026-10-14, hashes verified against the mirror download); `minBytes`
+ *  stays as a cheap size sanity check, the hash is the authority. */
+export const EMBEDDING_ASSETS = [
+    { name: 'tokenizer.json', path: 'Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/tokenizer.json', minBytes: 100_000, sha256: 'da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0' },
+    { name: 'tokenizer_config.json', path: 'Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/tokenizer_config.json', minBytes: 32, sha256: '9261e7d79b44c8195c1cada2b453e55b00aeb81e907a6664974b4d7776172ab3' },
+    { name: 'model.onnx', path: 'Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/onnx/model.onnx', minBytes: 10_000_000, sha256: '759c3cd2b7fe7e93933ad23c4c9181b7396442a2ed746ec7c1d46192c469c46e' },
+];
+/** HuggingFace first, the public mirror as fallback — both plain GETs. */
+const MIRRORS = ['https://huggingface.co/', 'https://hf-mirror.com/'];
+/** MC `computeNormalizedHash` — the same md5 fold the memory rows use (§2.3). */
+export function normalizedEmbeddingHash(content) {
+    return createHash('md5').update(content.toLowerCase().replace(/\s+/g, ' ').trim()).digest('hex');
+}
+/** MC `getEmbeddingProviderIdentity` local branch (embedding-identity.ts:79-88). */
+export function embeddingIdentityHash() {
+    return `embedding-provider:${normalizedEmbeddingHash(JSON.stringify({
+        provider: 'local',
+        model: EMBEDDING_MODEL,
+        endpoint: '',
+        apiKeyPresent: false,
+        runtimeFingerprint: LOCAL_EMBEDDING_RUNTIME_FINGERPRINT,
+    }))}`;
+}
+/** Cosine similarity over equal-length vectors (MC cosine-similarity.ts). */
+export function cosineSimilarity(a, b) {
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    const n = Math.min(a.length, b.length);
+    for (let i = 0; i < n; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    const denom = Math.sqrt(na) * Math.sqrt(nb);
+    return denom === 0 ? 0 : dot / denom;
+}
+/**
+ * Mean-pool the model's `last_hidden_state` over the attention mask, then
+ * L2-normalize — the exact recipe of transformers.js' `pooling: 'mean',
+ * normalize: true`. The fp64 accumulator keeps the summation order-stable; the
+ * spike measured cosine 1.0000000 against the native pipeline with it.
+ * Exported for tests: this function IS the vector space.
+ */
+export function meanPoolNormalize(hidden, seq, dims, mask) {
+    const acc = new Float64Array(dims);
+    let maskSum = 0;
+    for (let s = 0; s < seq; s++) {
+        const m = mask[s] ?? 0;
+        maskSum += m;
+        for (let h = 0; h < dims; h++)
+            acc[h] = acc[h] + hidden[s * dims + h] * m;
+    }
+    if (maskSum === 0)
+        return new Float32Array(dims);
+    let norm = 0;
+    for (let h = 0; h < dims; h++) {
+        acc[h] = acc[h] / maskSum;
+        norm += acc[h] * acc[h];
+    }
+    norm = Math.sqrt(norm);
+    const out = new Float32Array(dims);
+    for (let h = 0; h < dims; h++)
+        out[h] = acc[h] / norm;
+    return out;
+}
+/** The vendored ORT node entry sits beside the built lib at `<pkg>/vendor/onnx/`;
+ *  `src/core/*` and `lib/core/*` are both two levels below the package root, so
+ *  one relative URL serves the source tree (vitest) and the built tree (DSH). */
+function ortVendorUrl() {
+    return new URL('../../vendor/onnx/ort.node.min.mjs', import.meta.url).href;
+}
+async function ensureAsset(cacheDir, asset, fetchImpl, logger) {
+    const dest = join(cacheDir, asset.name);
+    // Cached files were hash-verified when written by this code path; the size
+    // floor keeps the warm path free of any hashing cost.
+    if (existsSync(dest) && statSync(dest).size >= asset.minBytes)
+        return;
+    mkdirSync(cacheDir, { recursive: true });
+    for (const mirror of MIRRORS) {
+        try {
+            logger?.info?.(`magic-context embedding assets: fetching ${asset.name} from ${mirror}`);
+            const res = await fetchImpl(mirror + asset.path, { redirect: 'follow' });
+            if (!res.ok || !res.body)
+                throw new Error(`HTTP ${res.status}`);
+            const tmp = `${dest}.part`;
+            await streamPipeline(Readable.fromWeb(res.body), createWriteStream(tmp));
+            if (statSync(tmp).size < asset.minBytes)
+                throw new Error(`suspiciously small (${statSync(tmp).size} bytes)`);
+            const digest = createHash('sha256').update(readFileSync(tmp)).digest('hex');
+            if (digest !== asset.sha256)
+                throw new Error(`sha256 mismatch (got ${digest.slice(0, 16)}…, want ${asset.sha256.slice(0, 16)}…)`);
+            renameSync(tmp, dest);
+            logger?.info?.(`magic-context embedding assets: ${asset.name} cached (${statSync(dest).size} bytes)`);
+            return;
+        }
+        catch (error) {
+            logger?.warn?.(`magic-context embedding assets: ${asset.name} from ${mirror} failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    throw new Error(`all mirrors failed for ${asset.name}`);
+}
+/**
+ * Open the lazy local embedder. `embed()` fetches the model assets (first use),
+ * loads the vendored onnxruntime-web WASM backend and the official tokenizers
+ * binding once, then reuses both. Every failure mode resolves to `[]`-shaped
+ * results — telemetry and recall quality, never correctness.
+ */
+export function openLocalEmbedder(options) {
+    const logger = options.logger;
+    let runnerPromise = null;
+    const load = async () => {
+        if (runnerPromise === null) {
+            runnerPromise = (async () => {
+                // The vendored ORT loads first: a broken runtime fails fast and cheap,
+                // before any model asset traffic is even considered.
+                const ort = await import(__rewriteRelativeImportExtension(/* webpackIgnore: true */ options.ortUrl ?? ortVendorUrl()));
+                const fetchImpl = options.fetchImpl ?? globalThis.fetch;
+                for (const asset of options.assets ?? EMBEDDING_ASSETS)
+                    await ensureAsset(options.cacheDir, asset, fetchImpl, logger);
+                const { Tokenizer } = await import('@huggingface/tokenizers');
+                const tok = new Tokenizer(JSON.parse(readFileSync(join(options.cacheDir, 'tokenizer.json'), 'utf8')), JSON.parse(readFileSync(join(options.cacheDir, 'tokenizer_config.json'), 'utf8')));
+                const session = await ort.InferenceSession.create(join(options.cacheDir, 'model.onnx'), {
+                    executionProviders: ['wasm'],
+                });
+                logger?.info?.(`magic-context local embedder ready (${EMBEDDING_MODEL} ${EMBEDDING_DTYPE}, wasm backend)`);
+                return async (texts) => {
+                    const vectors = [];
+                    for (const text of texts) {
+                        const raw = tok.encode(text, { add_special_tokens: true, return_token_type_ids: true });
+                        const seq = Math.min(raw.ids.length, EMBEDDING_MAX_SEQ);
+                        const ids = BigInt64Array.from(raw.ids.slice(0, seq).map((v) => BigInt(v)));
+                        const mask = BigInt64Array.from(raw.attention_mask.slice(0, seq).map((v) => BigInt(v)));
+                        const types = BigInt64Array.from((raw.token_type_ids ?? raw.ids.map(() => 0)).slice(0, seq).map((v) => BigInt(v)));
+                        const out = await session.run({
+                            input_ids: new ort.Tensor('int64', ids, [1, seq]),
+                            attention_mask: new ort.Tensor('int64', mask, [1, seq]),
+                            token_type_ids: new ort.Tensor('int64', types, [1, seq]),
+                        });
+                        const hidden = out.last_hidden_state ?? Object.values(out)[0];
+                        const modelDims = hidden.dims.at(-1) ?? EMBEDDING_DIMS;
+                        if (modelDims !== EMBEDDING_DIMS) {
+                            logger?.warn?.(`magic-context embedder returned unexpected dims ${hidden.dims.join(',')}`);
+                        }
+                        const maskNums = raw.attention_mask.slice(0, seq);
+                        vectors.push(meanPoolNormalize(hidden.data, seq, modelDims, maskNums));
+                    }
+                    return vectors;
+                };
+            })().catch((error) => {
+                runnerPromise = null; // allow a later retry
+                throw error;
+            });
+        }
+        return runnerPromise;
+    };
+    return {
+        modelId: embeddingIdentityHash(),
+        dims: EMBEDDING_DIMS,
+        async embed(texts) {
+            if (texts.length === 0)
+                return [];
+            if (options.embedOverride !== undefined)
+                return options.embedOverride(texts);
+            try {
+                const run = await load();
+                return await run(texts);
+            }
+            catch (error) {
+                logger?.warn?.(`magic-context local embedder unavailable (search degrades to FTS): `
+                    + `${error instanceof Error ? error.message : String(error)}`);
+                return [];
+            }
+        },
+    };
+}
+/** Default model cache directory, derived from the adapter's own data dir
+ *  (the directory holding the adapter's SQLite store). */
+export function defaultEmbeddingCacheDir(adapterDbPath) {
+    return join(dirname(adapterDbPath), 'models');
+}

package/lib/core/global-autostage.js

@@ -0,0 +1,223 @@
+/**
+ * L1① v10 — classify-then-route machine/org auto-stage (2026-10-12, user-
+ * directed). The previous leg scanned the contract store and mechanically
+ * promoted every qualifying memory into the MACHINE domain — and that default
+ * was wrong in both directions: an organisation workflow ("project-a/project-b must
+ * deploy serially") got routed to machine because the SQL rule cannot read
+ * content, and a classification failure silently assumed machine. The user's
+ * correction: the LLM decides the domain, and when it cannot, the memory STAYS
+ * AT PROJECT LEVEL — never a guessed fallback domain (machine or org can both
+ * be "seen" by other projects, so guessing either one hides the misstep).
+ *
+ * Flow per maintenance tick:
+ *  1. scan recurred, allowlisted, non-global project memories (union of the
+ *     ENABLED global domains' shareCategories — machine and org together, so
+ *     neither audience is invisible to the classifier);
+ *  2. one LLM call per candidate: {"domain": "machine"|"org"|"project",
+ *     "reason": "<one line>"};
+ *  3. route: machine → machine pool, org → org pool (only when that domain is
+ *     enabled AND the candidate's category is on its current allowlist —
+ *     otherwise NOT staged); project / unparseable / no-LLM → NOT staged.
+ *
+ * The suggestion rides the candidate row (`suggested_domain` +
+ * `suggestion_reason`), so the review card can show what the classifier said
+ * and the human can overrule it (the card's "publish to another domain"
+ * option). Publishing stays human — this leg only fills the pools.
+ */
+import { DatabaseSync } from 'node:sqlite';
+const DEFAULT_LIMIT = 3;
+const DEFAULT_MAX_PENDING = 10;
+const CLASSIFY_MAX_TOKENS = 2048;
+const CLASSIFY_INSTRUCTION = [
+    'You are classifying a recurring project memory for possible global promotion. Judge which audience it belongs to:',
+    '- "machine" (the GLOBAL domain, visible to every project on this machine): local tooling, machine-wide conventions, environment facts.',
+    '- "org" (the WORK domain, visible only to a curated set of work projects): company/team infrastructure, cross-repo workflows, shared internal systems.',
+    '- "project": project-specific only; must NOT be promoted.',
+    'Reply with ONLY a JSON object: {"domain":"machine"|"org"|"project","reason":"<one line>"}',
+].join('\n');
+/** Extract the classification from a raw model reply; undefined on unparseable/absent. */
+function parseClassification(text) {
+    const start = text.indexOf('{');
+    const end = text.lastIndexOf('}');
+    if (start < 0 || end <= start)
+        return undefined;
+    let parsed;
+    try {
+        parsed = JSON.parse(text.slice(start, end + 1));
+    }
+    catch {
+        return undefined;
+    }
+    const record = parsed;
+    const domain = record.domain;
+    if (domain !== 'machine' && domain !== 'org' && domain !== 'project')
+        return undefined;
+    const reason = typeof record.reason === 'string' && record.reason.trim() !== '' ? record.reason.trim() : undefined;
+    return { domain, ...(reason === undefined ? {} : { reason }) };
+}
+async function classify(llm, route, row, sessionId) {
+    const prompt = CLASSIFY_INSTRUCTION
+        + '\n\nMemory content:\n' + row.content.slice(0, 600).replace(/\s+/g, ' ')
+        + '\nCategory: ' + row.category;
+    let text = '';
+    try {
+        const chunks = llm.stream({
+            provider: route.provider,
+            model: route.model,
+            messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
+            maxTokens: CLASSIFY_MAX_TOKENS,
+            sessionId,
+            purpose: 'compaction',
+        });
+        for await (const raw of chunks) {
+            const chunk = raw;
+            if (chunk.type === 'text-delta' && typeof chunk.index === 'number' && typeof chunk.text === 'string') {
+                text += chunk.text;
+            }
+        }
+    }
+    catch {
+        return undefined;
+    }
+    return parseClassification(text);
+}
+export async function autoStageGlobalCandidates(input) {
+    const enabled = input.store.listDomains()
+        .filter(domain => domain.enabled && (domain.kind === 'machine' || domain.kind === 'org'));
+    if (enabled.length === 0)
+        return { staged: [], skipped: [], reason: 'domain-missing' };
+    if (input.llm === undefined || input.route === undefined) {
+        // The user's fail-closed direction: without a classifier nothing moves —
+        // the memories stay at project level instead of being silently guessed
+        // into a domain.
+        return { staged: [], skipped: [], reason: 'llm-unavailable' };
+    }
+    const limit = input.limit ?? DEFAULT_LIMIT;
+    const maxPending = input.maxPending ?? DEFAULT_MAX_PENDING;
+    // The queue guard is domain-level: each pool pauses independently when IT is
+    // full (a full org pool must not starve the machine pool). When EVERY
+    // enabled target pool is full the whole pass pauses up front.
+    const queueGuard = (domainId) => input.store.listGlobalCandidates({ state: 'pending', domainId }).length >= maxPending;
+    if (enabled.every(domain => queueGuard(domain.domainId))) {
+        return { staged: [], skipped: [], reason: 'queue-full' };
+    }
+    let scan;
+    try {
+        const db = new DatabaseSync(input.corpusDbPath, { readOnly: true });
+        try {
+            const categories = JSON.stringify(enabled.flatMap(domain => [...domain.shareCategories]));
+            scan = db.prepare(`SELECT id, content, category FROM memories
+         WHERE status = 'active' AND scope = 'project' AND seen_count >= 2
+           AND category IN (SELECT value FROM json_each(?))
+           AND project_path NOT LIKE 'dir:global:%' AND project_path NOT LIKE 'git:global:%'
+         ORDER BY updated_at DESC LIMIT 50`).all(categories);
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch {
+        return { staged: [], skipped: [], reason: 'domain-missing' };
+    }
+    const staged = [];
+    const skipped = [];
+    const seenRefusals = new Set();
+    const sessionId = 'mc-maintenance';
+    // Exact "created this pass" detection: everything already in any pool is
+    // known before the loop; the upsert is idempotent per (domain, hash), so a
+    // re-scan returns the SAME row and must not count as new.
+    const known = new Set(input.store.listGlobalCandidates().map(candidate => candidate.id));
+    for (const row of scan) {
+        if (staged.length >= limit)
+            break;
+        const verdict = await classify(input.llm, input.route, row, sessionId);
+        if (verdict === undefined || verdict.domain === 'project') {
+            const reason = verdict === undefined ? 'unclassified' : 'project-scope';
+            if (!seenRefusals.has(`${String(row.id)}:${reason}`)) {
+                skipped.push({ memoryId: row.id, reason });
+                seenRefusals.add(`${String(row.id)}:${reason}`);
+            }
+            continue;
+        }
+        const target = enabled.find(domain => domain.kind === verdict.domain);
+        if (target === undefined) {
+            if (!seenRefusals.has(`${String(row.id)}:target-disabled`)) {
+                skipped.push({ memoryId: row.id, reason: 'target-disabled' });
+                seenRefusals.add(`${String(row.id)}:target-disabled`);
+            }
+            continue;
+        }
+        if (queueGuard(target.domainId)) {
+            if (!seenRefusals.has(`${String(row.id)}:queue-full`)) {
+                skipped.push({ memoryId: row.id, reason: 'queue-full' });
+                seenRefusals.add(`${String(row.id)}:queue-full`);
+            }
+            continue;
+        }
+        if (!target.shareCategories.includes(row.category)) {
+            if (!seenRefusals.has(`${String(row.id)}:outside-allowlist`)) {
+                skipped.push({ memoryId: row.id, reason: 'outside-allowlist' });
+                seenRefusals.add(`${String(row.id)}:outside-allowlist`);
+            }
+            continue;
+        }
+        const result = promoteMemoryToCandidate(input.store, { corpusDbPath: input.corpusDbPath, memoryId: row.id, domainId: target.domainId, now: input.now }, verdict.domain, verdict.reason);
+        if (result.ok) {
+            if (!known.has(result.candidate.id)) {
+                known.add(result.candidate.id);
+                staged.push({ candidateId: result.candidate.id, memoryId: row.id, domainId: target.domainId });
+            }
+            continue;
+        }
+        const key = `${String(row.id)}:${result.error}`;
+        if (!seenRefusals.has(key)) {
+            skipped.push({ memoryId: row.id, reason: result.error });
+            seenRefusals.add(key);
+        }
+    }
+    return { staged, skipped };
+}
+function promoteMemoryToCandidate(store, options, suggestedDomain, suggestionReason) {
+    const memory = readCorpusMemory(options.corpusDbPath, options.memoryId);
+    if (memory === undefined)
+        return { ok: false, error: 'no memory in shared store' };
+    if (memory.status !== 'active')
+        return { ok: false, error: `memory is ${memory.status}` };
+    const candidate = store.upsertGlobalCandidate({
+        domainId: options.domainId,
+        content: memory.content,
+        normalizedHash: memory.normalizedHash,
+        firstIdentity: memory.projectPath,
+        category: memory.category,
+        sourceMemoryId: memory.id,
+        suggestedDomain,
+        ...(suggestionReason === undefined ? {} : { suggestionReason }),
+    }, options.now);
+    store.recordCandidateObservation({ candidateId: candidate.id, identity: memory.projectPath, sessionId: `memory:${String(memory.id)}` }, options.now);
+    const refreshed = store.getGlobalCandidate(candidate.id) ?? candidate;
+    return { ok: true, candidate: refreshed };
+}
+function readCorpusMemory(corpusDbPath, memoryId) {
+    try {
+        const db = new DatabaseSync(corpusDbPath, { readOnly: true });
+        try {
+            const row = db.prepare('SELECT id, content, category, normalized_hash, project_path, status FROM memories WHERE id = ?').get(memoryId);
+            if (row === undefined)
+                return undefined;
+            return {
+                id: Number(row.id),
+                content: String(row.content),
+                category: String(row.category),
+                normalizedHash: String(row.normalized_hash),
+                projectPath: String(row.project_path),
+                status: String(row.status),
+            };
+        }
+        finally {
+            db.close();
+        }
+    }
+    catch {
+        return undefined;
+    }
+}