@polycode-projects/seonix 0.6.0 → 0.7.0

package/src/toml-config.mjs

@@ -0,0 +1,183 @@
+// seonix.toml loader (pure library — no cli wiring here).
+//
+// The MCP server's runtime config still lives in config.mjs (`loadConfig`, the
+// SEONIX_GRAPH_FILE knob) and stays byte-identical. This module is the *product*
+// config surface: an optional `seonix.toml` at a repo/estate root that a consumer
+// checks in to steer indexing and scoring. Absent file = shipped defaults (today's
+// behaviour, byte-for-byte).
+//
+// Three pure entry points, consumed later by the cli:
+//   loadTomlConfig(rootDir)          → raw parsed TOML, or null when absent
+//   normalizeConfig(raw,{configDir}) → canonical sparse shape (present keys only)
+//   mergeEffective({args,toml,defaults}) → {effective, sources} (arg>toml>default)
+
+import { readFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { parse } from "smol-toml";
+
+export const CONFIG_FILE = "seonix.toml";
+
+/** Read `<rootDir>/seonix.toml` and return the raw parsed table.
+ *  - Absent file → `null` (the "today" signal: shipped defaults, byte-identical).
+ *  - Present but unparseable → throws a clear error naming the file + parse cause
+ *    (a `secret_exclude` misparse is security-relevant — never swallow it). */
+export async function loadTomlConfig(rootDir) {
+  const file = join(rootDir, CONFIG_FILE);
+  let text;
+  try {
+    text = await readFile(file, "utf8");
+  } catch (err) {
+    if (err && err.code === "ENOENT") return null;
+    throw err;
+  }
+  try {
+    return parse(text);
+  } catch (err) {
+    throw new Error(`Invalid ${CONFIG_FILE} at ${file}: ${err && err.message ? err.message : err}`);
+  }
+}
+
+/** Resolve the `repositories` value → absolute paths.
+ *  - array  → each entry resolved relative to `configDir`
+ *  - string → path to a newline-delimited file (relative to `configDir`); each
+ *             non-blank, non-`#` line resolved relative to `configDir`. */
+async function resolveRepositories(value, configDir) {
+  let entries;
+  if (Array.isArray(value)) {
+    entries = value;
+  } else {
+    const file = resolve(configDir, String(value));
+    const text = await readFile(file, "utf8");
+    entries = text.split("\n");
+  }
+  const out = [];
+  for (const raw of entries) {
+    const line = String(raw).trim();
+    if (!line || line.startsWith("#")) continue;
+    out.push(resolve(configDir, line));
+  }
+  return out;
+}
+
+// Recognized keys that parse and normalize but are not yet honoured by any
+// consumer. Surfaced in `normalizeConfig().unwired` so the cli can warn once
+// rather than silently ignore them.
+const UNWIRED_KEYS = [
+  ["index", "include_text"],
+  ["index", "include_structure"],
+  ["index", "respect_gitignore"],
+  ["index", "markdown_sections"],
+  ["index", "vue"],
+];
+
+/** Map a raw parsed TOML table → a canonical, sparse shape. Only keys actually
+ *  present in `raw` appear (so a consumer can tell "unset" from "set to a value
+ *  that equals the default" — the tri-state that `mergeEffective` relies on).
+ *  `configDir` anchors every relative path. Async because `repositories` may name
+ *  a newline-delimited file that has to be read. */
+export async function normalizeConfig(raw, { configDir } = {}) {
+  const src = raw || {};
+  const dir = configDir || process.cwd();
+  const cfg = {};
+
+  if (src.repositories !== undefined) {
+    cfg.repositories = await resolveRepositories(src.repositories, dir);
+  }
+  if (src.out_root !== undefined) {
+    cfg.outRoot = resolve(dir, String(src.out_root));
+  }
+
+  const idx = src.index || {};
+  const index = {};
+  if (idx.languages !== undefined) index.languages = idx.languages;
+  if (idx.exclude !== undefined) index.exclude = idx.exclude;
+  if (idx.secret_exclude !== undefined) index.secretExclude = idx.secret_exclude;
+  if (idx.history_depth !== undefined) index.historyDepth = idx.history_depth;
+  if (idx.include_text !== undefined) index.includeText = idx.include_text;
+  if (idx.include_structure !== undefined) index.includeStructure = idx.include_structure;
+  if (idx.respect_gitignore !== undefined) index.respectGitignore = idx.respect_gitignore;
+  if (idx.markdown_sections !== undefined) index.markdownSections = idx.markdown_sections;
+  if (idx.vue !== undefined) index.vue = idx.vue;
+  if (Object.keys(index).length) cfg.index = index;
+
+  const t = src.tune || {};
+  const tune = {};
+  if (t.score_gap_k !== undefined) tune.scoreGapK = t.score_gap_k;
+  if (t.literal_mention !== undefined) tune.literalMention = t.literal_mention;
+  if (t.demote_non_prod !== undefined) tune.demoteNonProd = t.demote_non_prod;
+  if (t.call_adjacency !== undefined) tune.callAdjacency = t.call_adjacency;
+  if (t.impl_of_interface !== undefined) tune.implOfInterface = t.impl_of_interface;
+  if (t.beam_search !== undefined) tune.beamSearch = t.beam_search;
+  if (t.beam_width !== undefined) tune.beamWidth = t.beam_width;
+  if (t.embed_rank !== undefined) tune.embedRank = t.embed_rank;
+  if (t.prose_layers !== undefined) tune.proseLayers = t.prose_layers;
+  const exp = t.expansion || {};
+  const expansion = {};
+  if (exp.strategy !== undefined) expansion.strategy = exp.strategy;
+  if (exp.nodes !== undefined) expansion.nodes = exp.nodes;
+  if (exp.q !== undefined) expansion.q = exp.q;
+  if (exp.depth !== undefined) expansion.depth = exp.depth;
+  if (Object.keys(expansion).length) tune.expansion = expansion;
+  if (Object.keys(tune).length) cfg.tune = tune;
+
+  const tel = src.telemetry || {};
+  if (tel.enabled !== undefined) cfg.telemetry = { enabled: tel.enabled };
+
+  const unwired = [];
+  for (const [a, b] of UNWIRED_KEYS) {
+    if (src[a] && src[a][b] !== undefined) unwired.push(`${a}.${b}`);
+  }
+  cfg.unwired = unwired;
+
+  return cfg;
+}
+
+/** Flatten a nested object into a dotted-key map. Arrays and scalars are leaves
+ *  (never recursed into); `undefined`/`null` values are dropped so that "present
+ *  but false" survives while "absent" does not. */
+function flatten(obj, prefix = "", out = {}) {
+  for (const [k, v] of Object.entries(obj || {})) {
+    if (v === undefined || v === null) continue;
+    const key = prefix ? `${prefix}.${k}` : k;
+    if (Array.isArray(v) || typeof v !== "object") out[key] = v;
+    else flatten(v, key, out);
+  }
+  return out;
+}
+
+/** Merge config sources by per-key precedence: explicit `args` > `toml` >
+ *  `defaults`. Returns `{effective, sources}`, both keyed by the same sorted set
+ *  of dotted keys (deterministic output). `sources[key]` is one of
+ *  "arg" | "seonix.toml" | "default".
+ *
+ *  Tri-state note: presence is tested with `in`, not truthiness, so a `toml`
+ *  value of `false` (e.g. `literal_mention = false` disabling the shipped-ON
+ *  default) is honoured, while an explicit arg still wins over it. */
+export function mergeEffective({ args = {}, toml = null, defaults = {} } = {}) {
+  const dFlat = flatten(defaults);
+  // `unwired` is metadata, not a knob — keep it out of the effective config.
+  let tomlKnobs = null;
+  if (toml) {
+    const { unwired, ...rest } = toml;
+    tomlKnobs = rest;
+  }
+  const tFlat = tomlKnobs ? flatten(tomlKnobs) : {};
+  const aFlat = flatten(args);
+
+  const keys = [...new Set([...Object.keys(dFlat), ...Object.keys(tFlat), ...Object.keys(aFlat)])].sort();
+  const effective = {};
+  const sources = {};
+  for (const k of keys) {
+    if (k in aFlat) {
+      effective[k] = aFlat[k];
+      sources[k] = "arg";
+    } else if (k in tFlat) {
+      effective[k] = tFlat[k];
+      sources[k] = "seonix.toml";
+    } else {
+      effective[k] = dFlat[k];
+      sources[k] = "default";
+    }
+  }
+  return { effective, sources };
+}

package/src/uuid.mjs

@@ -0,0 +1,16 @@
+// uuid.mjs — RFC 9562 UUIDv7 (node:crypto ships v4 only). Lifted verbatim from
+// chat.mjs so the chat session id, the telemetry invocation id, and the bench
+// per-run stamp all share ONE time-sortable implementation (no second copy, no dep).
+
+import { randomBytes } from "node:crypto";
+
+/** UUID v7 (RFC 9562): 48-bit big-endian unix-ms timestamp, then version/variant
+ *  bits over crypto-random tail — time-sortable, unlike crypto.randomUUID()'s v4. */
+export function uuidv7(now = Date.now()) {
+  const b = randomBytes(16);
+  b.writeUIntBE(now, 0, 6); // 48-bit unix-ms timestamp, big-endian
+  b[6] = (b[6] & 0x0f) | 0x70; // version 7
+  b[8] = (b[8] & 0x3f) | 0x80; // variant 10xx
+  const h = b.toString("hex");
+  return `${h.slice(0, 8)}-${h.slice(8, 12)}-${h.slice(12, 16)}-${h.slice(16, 20)}-${h.slice(20)}`;
+}

package/src/viz.mjs

@@ -14,7 +14,7 @@
 // `buildSubgraph` and `buildViewerData` are pure and unit-tested; the CLI adds graph
 // loading, file I/O, and the zero-dependency node:http server.
 
-import { readFile, writeFile } from "node:fs/promises";
+import { readFile, stat, writeFile } from "node:fs/promises";
 import { execFile } from "node:child_process";
 import { createServer } from "node:http";
 import { createRequire } from "node:module";
@@ -28,6 +28,20 @@ import { buildTemporalGraph, buildBrowserData, gitCommitOrder, gitCommitParents,
 import { extractTimeline, renderTimelineHtml } from "./timeline.mjs";
 import { winkBrowserBundle } from "./nlp-bundle.mjs";
 
+// Portable single-file gate: above this serialized graph size, embedding the
+// full raw graph inline yields an unopenable multi-hundred-MB HTML, so the
+// portable branch writes the ask/graph payloads as sidecars instead. The env
+// override SEONIX_VIZ_INLINE_MAX (bytes) lets an operator raise/lower it and
+// doubles as the test hook; --force-inline forces embedding regardless.
+export const INLINE_ASKDATA_MAX_BYTES = 32 * 1024 * 1024;
+
+/** The gate threshold in bytes: env override when it parses as a number
+ *  (including 0, which forces the gate on), otherwise the 32 MiB default. */
+function inlineAskDataMaxBytes(env = process.env) {
+  const parsed = parseInt(env.SEONIX_VIZ_INLINE_MAX ?? "", 10);
+  return Number.isNaN(parsed) ? INLINE_ASKDATA_MAX_BYTES : parsed;
+}
+
 const here = dirname(fileURLToPath(import.meta.url));
 const execFileP = promisify(execFile);
 
@@ -673,7 +687,15 @@ async function loadGraph(values) {
   const config = loadConfig(values.graph ? { ...process.env, SEONIX_GRAPH_FILE: resolve(values.graph) } : process.env);
   const raw = await source.fetchEntities(config);
   const graph = parseEntities(raw);
-  return { config, graph, raw };
+  // Serialized on-disk size drives the portable-branch large-graph gate (the real
+  // failure mode). 0 on any stat failure so a missing/unstattable file never gates.
+  let graphBytes = 0;
+  try {
+    graphBytes = (await stat(config.graphFile)).size;
+  } catch {
+    graphBytes = 0;
+  }
+  return { config, graph, raw, graphBytes };
 }
 
 function resolveFocus(graph, focusArg) {
@@ -731,10 +753,13 @@ export async function startVizServer({ values, cytoscape, askEngine = "", nlpBun
     const tg = buildTemporalGraph(raw, order, { scope: values.scope, parentsBySha });
     return buildBrowserData(tg, { repoUrl, repoRef: values.ref, live: true, gitHead: await gitHeadOf(process.cwd()), nav });
   };
+  const limit = Math.max(0, parseInt(values.limit, 10) || 0);
   const buildTimelinePage = async () => {
     const { raw } = await loadGraph(values);
+    const totalCommits = (raw.individuals || []).filter((i) => i.class === "Commit").length;
     return renderTimelineHtml({
-      commits: extractTimeline(raw),
+      commits: extractTimeline(raw, { limit }),
+      totalCommits,
       repoUrl,
       repoRef: values.ref,
       generatedAt: raw.generated_at || "",
@@ -831,12 +856,20 @@ export async function runVizCli(argv) {
       "browser-data-out": { type: "string" },
       "timeline-out": { type: "string" },
       "graph-only": { type: "boolean", default: false },
+      // Commit-timeline cap: render only the newest N commits (0 = unlimited).
+      // Default 100 keeps an estate-scale timeline openable; the header notes the
+      // truncation ("newest N of M commits"). See src/timeline.mjs.
+      limit: { type: "string", default: "100" },
       scope: { type: "string", default: "product" },
       // Include the OPTIONAL wink-nlp lemma tier in the chat (see nlp-bundle.mjs).
       // Site mode (--data-out): written as a same-origin sibling the page lazy-loads.
       // Portable/serve mode: inlined (single-file) / served in-process. Off by default,
       // so the local single-file stays lean and lemma-off exactly as before.
       nlp: { type: "boolean", default: false },
+      // Portable branch only: embed the full graph inline even past the size gate
+      // (INLINE_ASKDATA_MAX_BYTES). Off by default — over-threshold graphs write
+      // sidecars instead of an unopenable multi-hundred-MB single file.
+      "force-inline": { type: "boolean", default: false },
     },
     allowPositionals: false,
   });
@@ -849,7 +882,8 @@ export async function runVizCli(argv) {
     await startVizServer({ values, cytoscape, askEngine, nlpBundle, log: (s) => process.stderr.write(s) });
     return; // keeps serving until Ctrl-C
   }
-  const { config, graph, raw } = await loadGraph(values);
+  const { config, graph, raw, graphBytes } = await loadGraph(values);
+  const limit = Math.max(0, parseInt(values.limit, 10) || 0); // 0 = unlimited timeline
   const focus = resolveFocus(graph, values.focus);
   if (!focus) {
     process.stderr.write(values.focus
@@ -888,16 +922,17 @@ export async function runVizCli(argv) {
     siteNav: values["site-nav"],
     nav: values["graph-only"] ? null : navFor(out),
   });
-  if (values["data-out"]) {
-    // site mode: viewer page + sibling data file (data updates don't touch the page).
-    // The chat panel's full-graph channel rides the same sibling-file convention,
-    // written next to the display data file rather than embedded in the page.
-    const dataOut = resolve(values["data-out"]);
+  // Split viewer: the page + its two sibling data files (display graph + full ask
+  // graph), the page pointing at them rather than embedding. Shared by --data-out
+  // (site build) and the portable large-graph gate below so the sidecar layout can
+  // never drift between them. `dataOut` is the display-graph sidecar path; the ask
+  // sidecar is always seonix-ask-data.json next to --out. Returns the resolved paths.
+  const writeSplitViewer = async (dataOut) => {
     const rel = relative(dirname(out), dataOut) || "seonix-graph-data.json";
     const askDataOut = resolve(dirname(out), "seonix-ask-data.json");
     const askRel = relative(dirname(out), askDataOut) || "seonix-ask-data.json";
-    // Site build: the wink lemma bundle rides the same sibling-asset convention as
-    // the data files — a same-origin ./seonix-nlp.js the page lazy-loads on first ask.
+    // The wink lemma bundle rides the same sibling-asset convention as the data
+    // files — a same-origin ./seonix-nlp.js the page lazy-loads on first ask.
     let nlpPath = null;
     if (nlpBundle) {
       const nlpOut = resolve(dirname(out), "seonix-nlp.js");
@@ -911,11 +946,34 @@ export async function runVizCli(argv) {
       askDataPath: askRel === "seonix-ask-data.json" ? null : askRel,
       nlpPath,
     }));
+    return { dataOut, askDataOut, nlpPath };
+  };
+  if (values["data-out"]) {
+    // site mode: viewer page + sibling data file (data updates don't touch the page).
+    // The chat panel's full-graph channel rides the same sibling-file convention,
+    // written next to the display data file rather than embedded in the page.
+    const { dataOut, askDataOut, nlpPath } = await writeSplitViewer(resolve(values["data-out"]));
     process.stderr.write(
       `seonix viz: ${subgraph.nodes.length} nodes, ${subgraph.edges.length} edges around "${focus.label}" ` +
         `(depth ${values.depth})${subgraph.truncated ? " [capped]" : ""} → ${out} + ${dataOut} + ${askDataOut}` +
         `${nlpPath ? ` + ${resolve(dirname(out), "seonix-nlp.js")} (wink lemma tier)` : ""}\n`,
     );
+  } else if (graphBytes > inlineAskDataMaxBytes() && !values["force-inline"]) {
+    // portable mode, but the graph is too big to embed: an estate-scale full-graph
+    // inlined as a single file produces an unopenable multi-hundred-MB HTML. Write
+    // the same sidecars as the site build (next to --out) and point the page at
+    // them. file:// pages can't fetch siblings, so this needs `viz --serve`;
+    // --force-inline overrides and embeds anyway.
+    const { dataOut, askDataOut } = await writeSplitViewer(resolve(dirname(out), "seonix-graph-data.json"));
+    const mb = Math.round(graphBytes / (1024 * 1024));
+    process.stderr.write(
+      `seonix viz: large graph (${mb} MB > 32 MB) — data written as sidecars next to ${out}; ` +
+        "file:// pages cannot fetch siblings, open via `seonix viz --serve` (or pass --force-inline to embed anyway)\n",
+    );
+    process.stderr.write(
+      `seonix viz: ${subgraph.nodes.length} nodes, ${subgraph.edges.length} edges around "${focus.label}" ` +
+        `(depth ${values.depth})${subgraph.truncated ? " [capped]" : ""} → ${out} + ${dataOut} + ${askDataOut}\n`,
+    );
   } else {
     // portable mode: one self-contained file, both channels embedded (+ inline wink
     // bundle when --nlp; otherwise lemma-off, no external fetch, exactly as before)
@@ -963,9 +1021,11 @@ export async function runVizCli(argv) {
         }
       }
       if (timelineOut) {
-        const commits = extractTimeline(raw);
+        const commits = extractTimeline(raw, { limit });
+        const totalCommits = (raw.individuals || []).filter((i) => i.class === "Commit").length;
         await writeFile(timelineOut, renderTimelineHtml({
           commits,
+          totalCommits,
           repoUrl,
           repoRef: values.ref,
           generatedAt: raw.generated_at || "",