@polycode-projects/the-mechanical-code-talker 0.2.0

package/src/telemetry.mjs

@@ -0,0 +1,90 @@
+// telemetry.mjs — OPT-IN, fire-and-forget query telemetry (PLAN_TMCT_TELEMETRY.md).
+//
+// The measurement contract is absolute: telemetry is OFF by default and the OFF path
+// must be BYTE-IDENTICAL — no file, no stdout/stderr change, ~zero cost. createTelemetry
+// returns NULL when disabled, so every call site is a single falsy check: `tel?.record(…)`.
+//
+// When enabled it appends ONE JSONL line per event to `<graphDir>/tmct-<id>.log`
+// (i.e. next to graph.json, in the repo's `.tmct/` artifact dir). Writes are
+// fire-and-forget with a swallowed catch: telemetry must NEVER throw, block, or write
+// to stdout/stderr (stdout belongs to the chat surface). A structural redact() keeps file
+// CONTENTS out of the log — it records only ids/paths/scores/sizes/counts.
+
+import { appendFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { uuidv7 } from "./uuid.mjs";
+
+/** Field names whose VALUES are (or embed) raw source and must never be logged. */
+const DROP_KEYS = new Set(["text", "content", "snippet"]);
+/** String fields longer than this are truncated — except query.raw (the user's own
+ *  question, which is the correlation key and is not file content). */
+const MAX_STR = 500;
+
+/** Enabled iff `TMCT_TELEMETRY==="1"` OR `[telemetry] enabled=true` in the toml.
+ *  The env wins BOTH directions: `TMCT_TELEMETRY==="0"` force-disables even when the
+ *  toml turns it on. Default OFF (anything but "1"/"0" in the env falls through to toml). */
+export function telemetryEnabled(env = process.env, toml = null) {
+  const e = env?.TMCT_TELEMETRY;
+  if (e === "1") return true;
+  if (e === "0") return false;
+  return toml?.telemetry?.enabled === true;
+}
+
+/** The invocation id that correlates a log ↔ a host process. A host (the bench rig,
+ *  a wrapping agent) may stamp `TMCT_INVOCATION_ID` so its trace/record and this log
+ *  share one id; absent, we mint a fresh time-sortable uuidv7. */
+export function invocationId(env = process.env) {
+  return (env?.TMCT_INVOCATION_ID && String(env.TMCT_INVOCATION_ID)) || uuidv7();
+}
+
+/** Structural redaction: drop any field named text/content/snippet at any depth, and
+ *  truncate string fields longer than MAX_STR — EXCEPT `query.raw`. Records only
+ *  ids/paths/scores/sizes/counts, never file contents. Pure; returns a fresh value. */
+export function redact(value, path = "") {
+  if (typeof value === "string") {
+    if (path === "query.raw") return value; // the correlation key — kept whole
+    return value.length > MAX_STR ? value.slice(0, MAX_STR) : value;
+  }
+  if (Array.isArray(value)) return value.map((v) => redact(v, path));
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (DROP_KEYS.has(k)) continue;
+      out[k] = redact(v, path ? `${path}.${k}` : k);
+    }
+    return out;
+  }
+  return value;
+}
+
+/**
+ * Build a telemetry sink for one surface, or NULL when telemetry is disabled (the
+ * common OFF path — the caller's `tel?.record(…)` then costs one falsy check).
+ *
+ * When enabled, returns { id, file, record(fields) }. `record` stamps the schema-v1
+ * envelope ({v, id, seq, ts, surface}) onto the redacted, sparse caller fields and
+ * appends ONE JSONL line, fire-and-forget with a swallowed catch. `seq` is strictly
+ * increasing per sink. Record schema v1 fields (all sparse/optional — only include
+ * what a surface actually has):
+ *   { v, id, seq, ts, surface, tool,
+ *     query:    { raw, family, ambiguous, candidates },
+ *     response: { node_ids, scores, count, truncated, tier, topup },
+ *     perf:     { ms_total, ms_load, cold_load, graph_modules, graph_edges },
+ *     quality:  {},
+ *     cost:     { returned_chars, returned_tokens_est, cache_channel } }
+ */
+export function createTelemetry({ env = process.env, config, toml = null, surface } = {}) {
+  if (!telemetryEnabled(env, toml)) return null;
+  const id = invocationId(env);
+  const file = join(dirname(config.graphFile), `tmct-${id}.log`);
+  let seq = 0;
+  const record = (fields = {}) => {
+    try {
+      const line = { v: 1, id, seq: seq++, ts: new Date().toISOString(), surface, ...redact(fields) };
+      // Fire-and-forget: never await, never let a write error surface. Telemetry must
+      // not block a query or corrupt stdout.
+      appendFile(file, JSON.stringify(line) + "\n").catch(() => {});
+    } catch { /* redaction/serialisation must never throw a caller's turn */ }
+  };
+  return { id, file, record };
+}

package/src/toml-config.mjs

@@ -0,0 +1,183 @@
+// tmct.toml loader (pure library — no cli wiring here).
+//
+// The runtime config still lives in config.mjs (`loadConfig`, the
+// TMCT_GRAPH_FILE knob) and stays byte-identical. This module is the *product*
+// config surface: an optional `tmct.toml` at a repo/estate root that a consumer
+// checks in to steer indexing and scoring. Absent file = shipped defaults (today's
+// behaviour, byte-for-byte).
+//
+// Three pure entry points, consumed later by the cli:
+//   loadTomlConfig(rootDir)          → raw parsed TOML, or null when absent
+//   normalizeConfig(raw,{configDir}) → canonical sparse shape (present keys only)
+//   mergeEffective({args,toml,defaults}) → {effective, sources} (arg>toml>default)
+
+import { readFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { parse } from "smol-toml";
+
+export const CONFIG_FILE = "tmct.toml";
+
+/** Read `<rootDir>/tmct.toml` and return the raw parsed table.
+ *  - Absent file → `null` (the "today" signal: shipped defaults, byte-identical).
+ *  - Present but unparseable → throws a clear error naming the file + parse cause
+ *    (a `secret_exclude` misparse is security-relevant — never swallow it). */
+export async function loadTomlConfig(rootDir) {
+  const file = join(rootDir, CONFIG_FILE);
+  let text;
+  try {
+    text = await readFile(file, "utf8");
+  } catch (err) {
+    if (err && err.code === "ENOENT") return null;
+    throw err;
+  }
+  try {
+    return parse(text);
+  } catch (err) {
+    throw new Error(`Invalid ${CONFIG_FILE} at ${file}: ${err && err.message ? err.message : err}`);
+  }
+}
+
+/** Resolve the `repositories` value → absolute paths.
+ *  - array  → each entry resolved relative to `configDir`
+ *  - string → path to a newline-delimited file (relative to `configDir`); each
+ *             non-blank, non-`#` line resolved relative to `configDir`. */
+async function resolveRepositories(value, configDir) {
+  let entries;
+  if (Array.isArray(value)) {
+    entries = value;
+  } else {
+    const file = resolve(configDir, String(value));
+    const text = await readFile(file, "utf8");
+    entries = text.split("\n");
+  }
+  const out = [];
+  for (const raw of entries) {
+    const line = String(raw).trim();
+    if (!line || line.startsWith("#")) continue;
+    out.push(resolve(configDir, line));
+  }
+  return out;
+}
+
+// Recognized keys that parse and normalize but are not yet honoured by any
+// consumer. Surfaced in `normalizeConfig().unwired` so the cli can warn once
+// rather than silently ignore them.
+const UNWIRED_KEYS = [
+  ["index", "include_text"],
+  ["index", "include_structure"],
+  ["index", "respect_gitignore"],
+  ["index", "markdown_sections"],
+  ["index", "vue"],
+];
+
+/** Map a raw parsed TOML table → a canonical, sparse shape. Only keys actually
+ *  present in `raw` appear (so a consumer can tell "unset" from "set to a value
+ *  that equals the default" — the tri-state that `mergeEffective` relies on).
+ *  `configDir` anchors every relative path. Async because `repositories` may name
+ *  a newline-delimited file that has to be read. */
+export async function normalizeConfig(raw, { configDir } = {}) {
+  const src = raw || {};
+  const dir = configDir || process.cwd();
+  const cfg = {};
+
+  if (src.repositories !== undefined) {
+    cfg.repositories = await resolveRepositories(src.repositories, dir);
+  }
+  if (src.out_root !== undefined) {
+    cfg.outRoot = resolve(dir, String(src.out_root));
+  }
+
+  const idx = src.index || {};
+  const index = {};
+  if (idx.languages !== undefined) index.languages = idx.languages;
+  if (idx.exclude !== undefined) index.exclude = idx.exclude;
+  if (idx.secret_exclude !== undefined) index.secretExclude = idx.secret_exclude;
+  if (idx.history_depth !== undefined) index.historyDepth = idx.history_depth;
+  if (idx.include_text !== undefined) index.includeText = idx.include_text;
+  if (idx.include_structure !== undefined) index.includeStructure = idx.include_structure;
+  if (idx.respect_gitignore !== undefined) index.respectGitignore = idx.respect_gitignore;
+  if (idx.markdown_sections !== undefined) index.markdownSections = idx.markdown_sections;
+  if (idx.vue !== undefined) index.vue = idx.vue;
+  if (Object.keys(index).length) cfg.index = index;
+
+  const t = src.tune || {};
+  const tune = {};
+  if (t.score_gap_k !== undefined) tune.scoreGapK = t.score_gap_k;
+  if (t.literal_mention !== undefined) tune.literalMention = t.literal_mention;
+  if (t.demote_non_prod !== undefined) tune.demoteNonProd = t.demote_non_prod;
+  if (t.call_adjacency !== undefined) tune.callAdjacency = t.call_adjacency;
+  if (t.impl_of_interface !== undefined) tune.implOfInterface = t.impl_of_interface;
+  if (t.beam_search !== undefined) tune.beamSearch = t.beam_search;
+  if (t.beam_width !== undefined) tune.beamWidth = t.beam_width;
+  if (t.embed_rank !== undefined) tune.embedRank = t.embed_rank;
+  if (t.prose_layers !== undefined) tune.proseLayers = t.prose_layers;
+  const exp = t.expansion || {};
+  const expansion = {};
+  if (exp.strategy !== undefined) expansion.strategy = exp.strategy;
+  if (exp.nodes !== undefined) expansion.nodes = exp.nodes;
+  if (exp.q !== undefined) expansion.q = exp.q;
+  if (exp.depth !== undefined) expansion.depth = exp.depth;
+  if (Object.keys(expansion).length) tune.expansion = expansion;
+  if (Object.keys(tune).length) cfg.tune = tune;
+
+  const tel = src.telemetry || {};
+  if (tel.enabled !== undefined) cfg.telemetry = { enabled: tel.enabled };
+
+  const unwired = [];
+  for (const [a, b] of UNWIRED_KEYS) {
+    if (src[a] && src[a][b] !== undefined) unwired.push(`${a}.${b}`);
+  }
+  cfg.unwired = unwired;
+
+  return cfg;
+}
+
+/** Flatten a nested object into a dotted-key map. Arrays and scalars are leaves
+ *  (never recursed into); `undefined`/`null` values are dropped so that "present
+ *  but false" survives while "absent" does not. */
+function flatten(obj, prefix = "", out = {}) {
+  for (const [k, v] of Object.entries(obj || {})) {
+    if (v === undefined || v === null) continue;
+    const key = prefix ? `${prefix}.${k}` : k;
+    if (Array.isArray(v) || typeof v !== "object") out[key] = v;
+    else flatten(v, key, out);
+  }
+  return out;
+}
+
+/** Merge config sources by per-key precedence: explicit `args` > `toml` >
+ *  `defaults`. Returns `{effective, sources}`, both keyed by the same sorted set
+ *  of dotted keys (deterministic output). `sources[key]` is one of
+ *  "arg" | "tmct.toml" | "default".
+ *
+ *  Tri-state note: presence is tested with `in`, not truthiness, so a `toml`
+ *  value of `false` (e.g. `literal_mention = false` disabling the shipped-ON
+ *  default) is honoured, while an explicit arg still wins over it. */
+export function mergeEffective({ args = {}, toml = null, defaults = {} } = {}) {
+  const dFlat = flatten(defaults);
+  // `unwired` is metadata, not a knob — keep it out of the effective config.
+  let tomlKnobs = null;
+  if (toml) {
+    const { unwired, ...rest } = toml;
+    tomlKnobs = rest;
+  }
+  const tFlat = tomlKnobs ? flatten(tomlKnobs) : {};
+  const aFlat = flatten(args);
+
+  const keys = [...new Set([...Object.keys(dFlat), ...Object.keys(tFlat), ...Object.keys(aFlat)])].sort();
+  const effective = {};
+  const sources = {};
+  for (const k of keys) {
+    if (k in aFlat) {
+      effective[k] = aFlat[k];
+      sources[k] = "arg";
+    } else if (k in tFlat) {
+      effective[k] = tFlat[k];
+      sources[k] = "tmct.toml";
+    } else {
+      effective[k] = dFlat[k];
+      sources[k] = "default";
+    }
+  }
+  return { effective, sources };
+}

package/src/uuid.mjs

@@ -0,0 +1,16 @@
+// uuid.mjs — RFC 9562 UUIDv7 (node:crypto ships v4 only). Lifted verbatim from
+// chat.mjs so the chat session id, the telemetry invocation id, and the bench
+// per-run stamp all share ONE time-sortable implementation (no second copy, no dep).
+
+import { randomBytes } from "node:crypto";
+
+/** UUID v7 (RFC 9562): 48-bit big-endian unix-ms timestamp, then version/variant
+ *  bits over crypto-random tail — time-sortable, unlike crypto.randomUUID()'s v4. */
+export function uuidv7(now = Date.now()) {
+  const b = randomBytes(16);
+  b.writeUIntBE(now, 0, 6); // 48-bit unix-ms timestamp, big-endian
+  b[6] = (b[6] & 0x0f) | 0x70; // version 7
+  b[8] = (b[8] & 0x3f) | 0x80; // variant 10xx
+  const h = b.toString("hex");
+  return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`;
+}