@polycode-projects/the-mechanical-code-talker 2.0.3 → 2.3.0

package/src/adapters/embed.mjs

@@ -1,169 +0,0 @@
-// embed.mjs — deterministic static-embedding lookup (model2vec potion-base-8M,
-// 29,528 WordPiece subwords × 256 fp32 dims). Pure table lookup + float
-// arithmetic, no ONNX runtime, no network after the one-time fetch — the same
-// text always embeds to the same vector.
-//
-// The safetensors reader and WordPiece tokenizer below are hand-rolled: both
-// formats are simple enough to parse directly, avoiding an ONNX/HF tokenizer
-// dependency for it.
-//
-// Weights are gitignored (vendor/, fetched by scripts/fetch-embeddings.mjs)
-// and never in the npm package; loadEmbedder() returns null when absent so
-// CI/tests never require the download.
-
-import { readFileSync, existsSync } from "node:fs";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const MODEL_FILE = "model.safetensors";
-const TOKENIZER_FILE = "tokenizer.json";
-const CONFIG_FILE = "config.json";
-
-/** Default artifact dir: $TMCT_EMBED_DIR, else <repo>/vendor/embeddings/potion-base-8M
- *  (gitignored via vendor/ — the location scripts/fetch-embeddings.mjs writes). */
-export function defaultEmbeddingsDir() {
-  if (process.env.TMCT_EMBED_DIR) return process.env.TMCT_EMBED_DIR;
-  const here = dirname(fileURLToPath(import.meta.url)); // packages/tmct/src
-  return join(here, "..", "..", "..", "..", "vendor", "embeddings", "potion-base-8M");
-}
-
-// ---- safetensors (hand-rolled: u64le header length + JSON header + raw tensors) -------
-
-/** Read the single 2-D F32 embedding tensor from a safetensors file →
- *  { matrix: Float32Array (row-major), rows, dim }. model2vec exports exactly one
- *  tensor named "embeddings"; any lone 2-D F32 tensor is accepted for test fixtures. */
-function readSafetensors(file) {
-  const buf = readFileSync(file);
-  const headerLen = Number(buf.readBigUInt64LE(0));
-  const header = JSON.parse(buf.subarray(8, 8 + headerLen).toString("utf8"));
-  const name = header.embeddings
-    ? "embeddings"
-    : Object.keys(header).find((k) => k !== "__metadata__" && header[k]?.shape?.length === 2);
-  const t = name && header[name];
-  if (!t) throw new Error(`no 2-D tensor in ${file}`);
-  if (t.dtype !== "F32") throw new Error(`unsupported dtype ${t.dtype} in ${file} (only F32)`);
-  const [rows, dim] = t.shape;
-  const [start, end] = t.data_offsets;
-  const bytes = buf.subarray(8 + headerLen + start, 8 + headerLen + end);
-  if (bytes.byteLength !== rows * dim * 4) throw new Error(`tensor size mismatch in ${file}`);
-  // Copy into a fresh ArrayBuffer: the slice's byteOffset inside the file buffer is not
-  // guaranteed 4-byte aligned, and Float32Array requires alignment.
-  const matrix = new Float32Array(rows * dim);
-  new Uint8Array(matrix.buffer).set(bytes);
-  return { matrix, rows, dim };
-}
-
-// ---- Bert-style WordPiece tokenizer (from tokenizer.json) -----------------------------
-
-const isPunct = (ch) => {
-  const c = ch.codePointAt(0);
-  // ASCII punctuation ranges (Bert treats these as standalone tokens) + general unicode P/S.
-  return (c >= 33 && c <= 47) || (c >= 58 && c <= 64) || (c >= 91 && c <= 96) || (c >= 123 && c <= 126) ||
-    /[\p{P}\p{S}]/u.test(ch);
-};
-const isCjk = (ch) => {
-  const c = ch.codePointAt(0);
-  return (c >= 0x4e00 && c <= 0x9fff) || (c >= 0x3400 && c <= 0x4dbf) ||
-    (c >= 0xf900 && c <= 0xfaff) || (c >= 0x20000 && c <= 0x2ffff);
-};
-
-function makeTokenizer(spec) {
-  if (spec?.model?.type !== "WordPiece") {
-    throw new Error(`unsupported tokenizer model "${spec?.model?.type}" (only WordPiece)`);
-  }
-  const vocab = spec.model.vocab; // token -> id
-  const contPrefix = spec.model.continuing_subword_prefix ?? "##";
-  const maxChars = spec.model.max_input_chars_per_word ?? 100;
-  const unkId = vocab[spec.model.unk_token] ?? null;
-  const lowercase = spec.normalizer?.lowercase !== false;
-
-  // BertNormalizer: clean control chars, pad CJK, lowercase (+ strip accents when lowercasing).
-  const normalize = (text) => {
-    let s = String(text).replace(/[\u0000\ufffd]/g, "").replace(/[\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ");
-    s = [...s].map((ch) => (isCjk(ch) ? ` ${ch} ` : ch)).join("");
-    if (lowercase) s = s.toLowerCase().normalize("NFD").replace(/\p{Mn}/gu, "");
-    return s;
-  };
-  // BertPreTokenizer: split on whitespace; every punctuation char is its own word.
-  const preTokenize = (s) => {
-    const words = [];
-    for (const chunk of s.split(/\s+/)) {
-      if (!chunk) continue;
-      let cur = "";
-      for (const ch of chunk) {
-        if (isPunct(ch)) {
-          if (cur) { words.push(cur); cur = ""; }
-          words.push(ch);
-        } else cur += ch;
-      }
-      if (cur) words.push(cur);
-    }
-    return words;
-  };
-  // Greedy longest-match WordPiece per word; a word with no valid segmentation → [UNK].
-  const wordPiece = (word) => {
-    if (word.length > maxChars) return unkId == null ? [] : [unkId];
-    const ids = [];
-    let start = 0;
-    while (start < word.length) {
-      let end = word.length;
-      let id = null;
-      while (end > start) {
-        const piece = (start > 0 ? contPrefix : "") + word.slice(start, end);
-        if (vocab[piece] !== undefined) { id = vocab[piece]; break; }
-        end--;
-      }
-      if (id == null) return unkId == null ? [] : [unkId];
-      ids.push(id);
-      start = end;
-    }
-    return ids;
-  };
-  // No [CLS]/[SEP] template — model2vec pools content subwords only.
-  return (text) => preTokenize(normalize(text)).flatMap(wordPiece);
-}
-
-// ---- embedder ---------------------------------------------------------------------------
-
-const LOADED = new Map(); // dir -> embedder | null (per-process cache; weights load once)
-
-/** Load tokenizer + embedding matrix from `dir` (default: defaultEmbeddingsDir()).
- *  Returns null — silently — when the artifacts are absent (the one-time
- *  `npm run refs:embeddings` fetch has not run): callers no-op, never fail.
- *  The returned embedder is { dim, dir, embed(text) → L2-normalised Float32Array }. */
-export function loadEmbedder({ dir = defaultEmbeddingsDir() } = {}) {
-  if (LOADED.has(dir)) return LOADED.get(dir);
-  const modelFile = join(dir, MODEL_FILE);
-  const tokFile = join(dir, TOKENIZER_FILE);
-  if (!existsSync(modelFile) || !existsSync(tokFile)) {
-    LOADED.set(dir, null);
-    return null;
-  }
-  const { matrix, rows, dim } = readSafetensors(modelFile);
-  const tokenize = makeTokenizer(JSON.parse(readFileSync(tokFile, "utf8")));
-  let cfg = {};
-  try { cfg = JSON.parse(readFileSync(join(dir, CONFIG_FILE), "utf8")); } catch { /* optional */ }
-  const doNormalize = cfg.normalize !== false;
-
-  const embed = (text) => {
-    const ids = tokenize(text);
-    const v = new Float32Array(dim);
-    if (!ids.length) return v; // zero vector: cosine 0 against everything
-    for (const id of ids) {
-      if (id < 0 || id >= rows) continue;
-      const off = id * dim;
-      for (let j = 0; j < dim; j++) v[j] += matrix[off + j];
-    }
-    for (let j = 0; j < dim; j++) v[j] /= ids.length; // mean pool
-    if (doNormalize) {
-      let norm = 0;
-      for (let j = 0; j < dim; j++) norm += v[j] * v[j];
-      norm = Math.sqrt(norm);
-      if (norm > 0) for (let j = 0; j < dim; j++) v[j] /= norm;
-    }
-    return v;
-  };
-  const embedder = { dim, dir, embed };
-  LOADED.set(dir, embedder);
-  return embedder;
-}

package/src/domain/router/guardrail.mjs

@@ -1,116 +0,0 @@
-// src/domain/router/guardrail.mjs — the guardrail. Validate an
-// EXTERNALLY-proposed `tool_use` against the registry's declared preconditions.
-//
-// Proves RESOLVABILITY (the tool is registered/declared, args are well-formed, every
-// `resolves(param, as)` precondition binds to a real graph entity) — NOT antecedent
-// correctness: a cross-turn mis-binding ("it" -> the wrong Commit) still resolves to a
-// real entity and passes. "This symbol denotes something real and the call is
-// well-formed", never "this is the right something".
-//
-// Pure over its inputs + ctx.resolve (the binding oracle). No network, no Date.now.
-
-import { capabilityByName, preconditionsOf, PRECOND } from "./registry.mjs";
-import { hallucinationsIn } from "./call-validator.mjs";
-
-/** The same read-only breadth-first enrichment as resolver.mjs's `dispatchEachCandidate`:
- *  dispatching the SAME tool once per tied candidate is safe for `readOnly` capabilities
- *  (dispatch performs no writes). Returns `[{candidate, result}, ...]`, or
- *  undefined when there is no dispatcher to run it with. */
-async function dispatchEachCandidate(pool, capName, arg, ctx) {
-  if (!ctx.dispatch) return undefined;
-  // Only readOnly capabilities may be dispatched here: the enrichment runs the
-  // SAME tool once per tied candidate, which is only safe when dispatch
-  // performs no writes. A registered world-mutating capability is planned
-  // over, never dispatched.
-  if (capabilityByName(capName)?.readOnly !== true) return undefined;
-  const results = [];
-  for (const c of pool) {
-    const res = await ctx.dispatch(capName, { [arg]: c.label });
-    results.push({ candidate: c.label, result: res });
-  }
-  return results;
-}
-
-/** Validate a proposed tool_use. Returns a glass-box verdict:
- *    { ok, tool, denied:[{reason,detail}], steps:[{pred,ok,...}], provenance, candidateResults? }
- *  ok=false with a default-deny/undeclared/unknown-arg/missing-arg denial is structural (no
- *  graph needed); an `unresolved` step is a binding rejection, and an ambiguous `resolves`
- *  term additionally carries `candidateResults` (the tool dispatched once per tied candidate)
- *  when `ctx.dispatch` is wired. `declaredNames=null` skips the declared-set check.
- *  `ctx.resolve(term)` is the resolveObject oracle; omit it for structural-only validation. */
-export async function guard(toolUse, declaredNames = null, ctx = {}) {
-  const name = toolUse?.name;
-  const input = toolUse && typeof toolUse.input === "object" && toolUse.input ? toolUse.input : {};
-  const denied = [];
-  const steps = [];
-
-  // Default-deny: unknown/unregistered tool is an automatic reject.
-  const declaredList = declaredNames ? [...declaredNames] : null;
-  const cap = capabilityByName(name);
-  if (!cap) {
-    denied.push({ reason: "default-deny", detail: `"${name ?? "(none)"}" is not a registered capability` });
-    return { ok: false, tool: name ?? null, denied, steps, provenance: "registry default-deny" };
-  }
-  const structural = hallucinationsIn({ name, input }, declaredList ?? [name]);
-  for (const p of structural) {
-    // no declared set supplied => "undeclared" isn't a real denial (we synthesised [name]).
-    if (!declaredList && p.reason === "undeclared") continue;
-    denied.push(p);
-  }
-
-  // Precondition check — the STRIPS safety gate, step by step.
-  let candidateResults;
-  for (const pre of preconditionsOf(name)) {
-    if (pre.pred === PRECOND.graphLoaded) {
-      steps.push({ step: "precondition", pred: pre.pred, ok: true });
-    } else if (pre.pred === PRECOND.anyPresent) {
-      const ok = pre.params.some((k) => input[k] !== undefined && input[k] !== null && String(input[k]).trim() !== "");
-      steps.push({ step: "precondition", pred: pre.pred, params: pre.params, ok });
-      if (!ok) denied.push({ reason: "missing-arg", detail: `${name} needs one of ${pre.params.join("|")}` });
-    } else if (pre.pred === PRECOND.resolves) {
-      const term = input[pre.param];
-      const present = term !== undefined && term !== null && String(term).trim() !== "";
-      if (!present) {
-        steps.push({ step: "precondition", pred: pre.pred, param: pre.param, value: null, ok: false });
-        continue;
-      }
-      // No oracle wired: assert the arg is PRESENT only, not that it binds.
-      if (!ctx.resolve) {
-        steps.push({ step: "precondition", pred: pre.pred, param: pre.param, value: term, ok: true, note: "structural-only (no resolver wired)" });
-        continue;
-      }
-      const r = ctx.resolve(String(term));
-      const resolvedOk = Boolean(r && r.match && !r.ambiguous);
-      steps.push({
-        step: "precondition", pred: pre.pred, param: pre.param, value: term,
-        ok: resolvedOk,
-        ...(r && r.match ? { boundTo: r.match.label, boundClass: r.match.class ?? null, tier: r.tier ?? null } : {}),
-        ...(r && r.ambiguous ? { ambiguous: true } : {}),
-      });
-      if (!resolvedOk) {
-        denied.push({
-          reason: "unresolved",
-          detail: r && r.ambiguous
-            ? `${name}.${pre.param}="${term}" is ambiguous (narrow it)`
-            : `${name}.${pre.param}="${term}" resolves to no graph entity`,
-        });
-        if (r && r.ambiguous) {
-          const pool = [r.match, ...(r.candidates || [])].slice(0, 4);
-          const dispatched = await dispatchEachCandidate(pool, name, pre.param, ctx);
-          if (dispatched) candidateResults = dispatched;
-        }
-      }
-    }
-  }
-
-  const ok = denied.length === 0;
-  return {
-    ok, tool: name, denied, steps, provenance: ok ? "resolvable (NOT proven antecedent-correct)" : "denied",
-    ...(candidateResults ? { candidateResults } : {}),
-  };
-}
-
-/** Convenience boolean: does a proposed tool_use PASS the guardrail? */
-export async function admits(toolUse, declaredNames = null, ctx = {}) {
-  return (await guard(toolUse, declaredNames, ctx)).ok;
-}

package/src/domain/vector.mjs

@@ -1,12 +0,0 @@
-// vector.mjs — pure vector arithmetic over embedding vectors. No model, no fs:
-// the loader that reads weights off disk lives in src/adapters/embed.mjs.
-
-/** Cosine similarity. Over L2-normalised vectors this is just the dot product, but the
- *  full form is kept so unnormalised test fixtures behave. 0 when either vector is zero. */
-export function cosine(a, b) {
-  let dot = 0, na = 0, nb = 0;
-  const n = Math.min(a.length, b.length);
-  for (let i = 0; i < n; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
-  if (na === 0 || nb === 0) return 0;
-  return dot / (Math.sqrt(na) * Math.sqrt(nb));
-}