@tiens.nguyen/gu-cli 1.0.686

package/rag-selftest.mjs

@@ -0,0 +1,215 @@
+/**
+ * Can THIS machine embed and retrieve? (`gu-cli rag --test`)
+ *
+ * Ships with the package so it runs anywhere gu is installed — Windows, macOS or Linux —
+ * without copying files about. There is nothing platform-specific in it: one Node module beats
+ * three per-platform archives that have to be kept in step.
+ *
+ * It asks the question that kept having a different answer in practice: can this machine reach
+ * an embedder, and can that embedder tell one part of a real project from another? Both halves
+ * have failed — the embed URL pointed at an Apple-only MLX port that was never running, and
+ * before that the API withheld RAG entirely unless S3 credentials existed. Neither was visible
+ * from any status output; both are visible here.
+ *
+ * The configuration comes from THIS machine (worker.env → the API's settings), never from
+ * defaults invented here: a check that supplies its own embedder passes happily on a machine
+ * where the agent cannot embed at all.
+ */
+import { mkdtemp, rm, readFile } from "node:fs/promises";
+import { tmpdir, homedir } from "node:os";
+import { join } from "node:path";
+
+/** One needle, many haystack files. The needle is what a semantic search must find. */
+export const NEEDLE_FILE = "invoices.js";
+export const NEEDLE_QUERY = "where are invoice totals computed";
+
+/**
+ * The needle deliberately never contains the query's words in a greppable form: the file says
+ * "computeInvoiceTotal" and "subtotal", the query asks "where are invoice totals computed".
+ * A search that finds this by substring is not doing what RAG is for.
+ */
+const NEEDLE = `
+// Billing rules for customer orders.
+function computeInvoiceTotal(items, taxRate) {
+  const subtotal = items.reduce((sum, i) => sum + i.price * i.qty, 0);
+  const shipping = subtotal > 100 ? 0 : 9.99;
+  return (subtotal + shipping) * (1 + taxRate);
+}
+
+function applyDiscountCode(code, amount) {
+  const table = { WELCOME10: 0.1, LOYAL20: 0.2 };
+  return amount * (1 - (table[code] ?? 0));
+}
+module.exports = { computeInvoiceTotal, applyDiscountCode };
+`.trim();
+
+/** Filler that is plausible code about something else entirely. */
+function haystackFile(n) {
+  const lines = [`// Service module ${n}: request handling for the platform's internal API.`];
+  for (let i = 1; i <= 40; i++) {
+    lines.push(
+      `// helper ${i}: validates the incoming payload, maps upstream errors and records timing`,
+      `function svc${n}Handler${i}(request, context) {`,
+      `  const parsed = validateSchema(request.body, schemas.v${i});`,
+      `  if (!parsed.ok) return respond(400, { error: parsed.reason });`,
+      `  return withRetries(() => upstream.call(parsed.value, { timeout: 5000 }), 3);`,
+      `}`,
+      ""
+    );
+  }
+  return lines.join("\n");
+}
+
+/**
+ * Write the fixture into `dir`. Returns what was written, so a caller can report the size it
+ * actually produced rather than a number someone wrote in a comment once.
+ */
+export async function writeRagFixture(dir, { files = 12 } = {}) {
+  const { mkdir, writeFile } = await import("node:fs/promises");
+  const { join } = await import("node:path");
+  await mkdir(dir, { recursive: true });
+
+  const written = [];
+  for (let n = 1; n <= files; n++) {
+    const name = `service${n}.js`;
+    const body = haystackFile(n);
+    await writeFile(join(dir, name), body);
+    written.push({ name, bytes: Buffer.byteLength(body) });
+  }
+  await writeFile(join(dir, NEEDLE_FILE), NEEDLE);
+  written.push({ name: NEEDLE_FILE, bytes: Buffer.byteLength(NEEDLE) });
+
+  return { dir, files: written, totalBytes: written.reduce((s, f) => s + f.bytes, 0) };
+}
+
+/**
+ * Split text the way the indexer does — roughly, and that is fine. This fixture exists to ask
+ * "can the embedder tell these documents apart", not to re-implement the chunker.
+ */
+export function chunk(text, size = 1200) {
+  const out = [];
+  for (let i = 0; i < text.length; i += size) out.push(text.slice(i, i + size));
+  return out;
+}
+
+/** Cosine similarity. The whole of retrieval, once the vectors exist. */
+export function cosine(a, b) {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < Math.min(a.length, b.length); i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
+}
+
+/** Read the API base + worker key exactly as the CLI does. */
+async function workerEnv() {
+  const txt = await readFile(join(homedir(), ".gonext", "worker.env"), "utf8").catch(() => "");
+  const get = (k) => (new RegExp(`^${k}=(.*)$`, "m").exec(txt)?.[1] ?? "").trim();
+  return { apiBase: get("GONEXT_API_BASE").replace(/\/+$/, ""), key: get("GONEXT_WORKER_KEY") };
+}
+
+/**
+ * Run the checks, reporting each through `onStep`.
+ * Returns { ok, checks } — the caller decides how to print and what exit code to use.
+ */
+export async function runRagSelfTest({ onStep = () => {} } = {}) {
+  const checks = [];
+  const step = async (title, fn) => {
+    try {
+      const detail = await fn();
+      checks.push({ title, ok: true, detail: detail ?? "" });
+      onStep({ title, ok: true, detail: detail ?? "" });
+    } catch (e) {
+      const detail = String(e?.message ?? e).split("\n")[0];
+      checks.push({ title, ok: false, detail });
+      onStep({ title, ok: false, detail });
+    }
+  };
+
+  let embedUrl = "";
+  let embedModel = "";
+
+  await step("read the embedder from this machine's settings", async () => {
+    const { apiBase, key } = await workerEnv();
+    if (!apiBase || !key) throw new Error("no ~/.gonext/worker.env — run `gu` first");
+    const res = await fetch(`${apiBase}/api/worker/settings`, {
+      headers: { "X-Worker-Key": key },
+      signal: AbortSignal.timeout(10_000),
+    });
+    if (!res.ok) throw new Error(`settings → HTTP ${res.status} (is the API running?)`);
+    const s = await res.json();
+    // The stored URL may or may not carry /v1 — the worker normalises it, so this must too.
+    // Without that the request goes to <host>/embeddings and 404s, which reads as "the
+    // embedder is down" when it is simply a different path.
+    const raw = String(s.ragEmbedUrl || "").replace(/\/+$/, "");
+    embedUrl = raw && !/\/v1$/i.test(raw) ? `${raw}/v1` : raw;
+    embedModel = String(s.ragEmbedModel || "");
+    if (!embedUrl) throw new Error("no RAG embed URL configured for this account");
+    return `${embedModel || "(no model name)"} @ ${embedUrl}`;
+  });
+
+  const embed = async (input) => {
+    const res = await fetch(`${embedUrl}/embeddings`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ model: embedModel, input }),
+      signal: AbortSignal.timeout(120_000),
+    });
+    if (!res.ok) throw new Error(`embeddings → HTTP ${res.status} ${(await res.text()).slice(0, 100)}`);
+    const body = await res.json();
+    const vectors = (body.data ?? []).map((d) => d.embedding);
+    if (!vectors.length || !vectors[0]?.length) throw new Error("no vectors came back");
+    return vectors;
+  };
+
+  if (embedUrl) {
+    await step("the embedder answers", async () => {
+      const t0 = Date.now();
+      const [v] = await embed(["a short piece of text"]);
+      return `${v.length}-dim in ${Date.now() - t0}ms`;
+    });
+  }
+
+  let dir = "";
+  let fixture = null;
+  await step("build a project big enough to be worth indexing", async () => {
+    dir = await mkdtemp(join(tmpdir(), "gu-rag-"));
+    fixture = await writeRagFixture(dir);
+    return `${fixture.files.length} files, ${Math.round(fixture.totalBytes / 1024)} KB`;
+  });
+
+  if (embedUrl && fixture) {
+    await step("the needle ranks first for a question that does not name it", async () => {
+      const docs = [];
+      for (const f of fixture.files) {
+        const text = await readFile(join(dir, f.name), "utf8");
+        for (const c of chunk(text)) docs.push({ file: f.name, text: c });
+      }
+      const vectors = [];
+      for (let i = 0; i < docs.length; i += 40) {
+        vectors.push(...(await embed(docs.slice(i, i + 40).map((d) => d.text))));
+      }
+      const [q] = await embed([NEEDLE_QUERY]);
+      const ranked = docs
+        .map((d, i) => ({ file: d.file, score: cosine(q, vectors[i]) }))
+        .sort((a, b) => b.score - a.score);
+      const top = ranked[0];
+      const runnerUp = ranked.find((r) => r.file !== top.file);
+      if (top.file !== NEEDLE_FILE) {
+        throw new Error(
+          `top hit was ${top.file} (${top.score.toFixed(3)}), expected ${NEEDLE_FILE} — ` +
+            "the embedder is reachable but is not separating this corpus"
+        );
+      }
+      const margin = (top.score - (runnerUp?.score ?? 0)).toFixed(3);
+      return `${docs.length} chunks · ${NEEDLE_FILE} first (${top.score.toFixed(3)}, +${margin} over the rest)`;
+    });
+  }
+
+  if (dir) await rm(dir, { recursive: true, force: true }).catch(() => {});
+  return { ok: checks.every((c) => c.ok), checks };
+}

package/s3-setup.mjs

@@ -0,0 +1,85 @@
+/**
+ * Pointing a machine at its own RAG S3 bucket from the TERMINAL (`/s3`).
+ *
+ * The web app has had this since RAG existed — Settings → RAG, with a "Test access" button. A
+ * Client Expert machine cannot reach any of it: PATCH /api/settings is behind a Firebase login
+ * that box deliberately does not have, so the one place these values could be set was the one
+ * place that machine cannot open. `/s3` is the same three fields and the same test, asked for
+ * over a worker key.
+ *
+ * PURE ON PURPOSE, like ollama-setup.mjs beside it: every decision here is a function from
+ * values to strings, so "what does this config look like / what did the test say" is a test
+ * rather than something you find out by pointing a REPL at a real bucket.
+ *
+ * WHAT IS NOT HERE, deliberately: parsing the S3 location into bucket + prefix. The API already
+ * does that (parseRagLocation) and it is the thing that decides whether a location is valid at
+ * all. A second copy here would be a second opinion — and the one the user sees would be the
+ * one that is not used to actually reach the bucket.
+ */
+
+/** An access key id, shown without showing it: "AKIA…MPLE". "" stays "". */
+export function maskKeyId(id) {
+  const s = String(id ?? "").trim();
+  if (!s) return "";
+  // Short enough that a mask would reveal most of it anyway → say nothing about the middle.
+  if (s.length <= 8) return "…";
+  return `${s.slice(0, 4)}…${s.slice(-4)}`;
+}
+
+/**
+ * The current config, as lines for the user to read before deciding to change it.
+ *
+ * The SECRET is reported as stored-or-not and never shown; `secretConfigured` is the boolean the
+ * API returns in place of it. An unset field says "not set" rather than printing an empty value,
+ * because a blank after a label reads as a rendering bug rather than as an answer.
+ */
+export function s3SummaryRows({
+  location = "",
+  region = "",
+  accessKeyId = "",
+  secretConfigured = false,
+} = {}) {
+  const val = (v) => (String(v ?? "").trim() ? String(v).trim() : "not set");
+  return [
+    `bucket   ${val(location)}`,
+    `region   ${val(region)}`,
+    `key id   ${accessKeyId ? maskKeyId(accessKeyId) : "not set"}`,
+    `secret   ${secretConfigured ? "stored" : "not set"}`,
+  ];
+}
+
+/** Is there enough here to be worth testing? → "" when yes, else what is missing. */
+export function whatIsMissing({ location = "", accessKeyId = "", secretConfigured = false } = {}) {
+  const missing = [];
+  if (!String(location ?? "").trim()) missing.push("a bucket");
+  if (!String(accessKeyId ?? "").trim()) missing.push("an access key id");
+  if (!secretConfigured) missing.push("a secret");
+  return missing.length ? missing.join(", ") : "";
+}
+
+/**
+ * The test result, as lines. → { rows, ok }
+ *
+ * READ AND WRITE ARE REPORTED SEPARATELY because they fail separately and for different
+ * reasons: a bucket policy that allows ListObjects and denies PutObject is a normal, common
+ * misconfiguration, and "S3 access failed" would send someone to check their keys when the keys
+ * are fine. The API probes both by actually doing them, so these are observations, not guesses.
+ *
+ * The error is printed VERBATIM. It is AWS's own sentence ("Access Denied", "The specified
+ * bucket does not exist"), and paraphrasing it would cost the one detail that identifies which
+ * of a dozen S3 misconfigurations this is.
+ */
+export function describeS3Test(result) {
+  if (!result) {
+    return { ok: false, rows: ["could not reach the API to run the test."] };
+  }
+  const rows = [];
+  const where = result.bucket
+    ? `${result.bucket}${result.prefix ? `/${result.prefix}` : ""}`
+    : "";
+  if (where) rows.push(`bucket   ${where}`);
+  rows.push(`read     ${result.canRead ? "ok" : "no"}`);
+  rows.push(`write    ${result.canWrite ? "ok" : "no"}`);
+  if (result.error) rows.push(`error    ${result.error}`);
+  return { ok: Boolean(result.ok), rows };
+}

package/terminal-copy.mjs

@@ -0,0 +1,248 @@
+/**
+ * Copying text out of the terminal (task #158).
+ *
+ * The problem this solves: gu renders its own view, so what is ON SCREEN is not the text
+ * anyone wants. An answer is hard-wrapped to the window width, indented into the gutter, and
+ * coloured — so a native drag-select yields hard newlines mid-sentence, three leading spaces
+ * per line, and escape codes. Pasting a code block that way does not run.
+ *
+ * The answer is to copy the SOURCE, not the screen: printed blocks register the text they came
+ * from, and a double-click resolves a screen row back to that. Cleaning rendered lines is the
+ * fallback for output that never registered anything.
+ *
+ * Everything here is pure. The IO — the clipboard write, the overlay paint — lives in the REPL
+ * where the terminal is.
+ */
+
+const CSI = /\x1b\[[0-9;?]*[A-Za-z]/g;
+const OSC = /\x1b\][^\x07\x1b]*(\x07|\x1b\\)/g;
+
+/** Strip every escape sequence, leaving the text a person sees. */
+export function stripAnsi(s) {
+  return String(s ?? "").replace(OSC, "").replace(CSI, "");
+}
+
+/**
+ * The gutter and margin markers are CHROME, not content.
+ *
+ * Every printed row is indented to one shared text column, and bullets hang their glyph in
+ * that margin (`markLine`). Copying those means every pasted line starts with three spaces and
+ * some start with "● " — which breaks indentation-sensitive languages outright.
+ */
+export function stripGutter(line, gutter = 3) {
+  const s = stripAnsi(line);
+  if (!s.trim()) return "";
+  // A hanging marker: glyph right-aligned in the gutter, then one space. Take the text column.
+  const marked = new RegExp(`^\\s{0,${Math.max(0, gutter - 2)}}[●⏺✓✗↻·>»]{1,2}\\s`).exec(s);
+  if (marked) return s.slice(marked[0].length);
+  // The ">>" prompt echo hangs the same way.
+  const prompt = /^\s*>>\s?/.exec(s);
+  if (prompt) return s.slice(prompt[0].length);
+  return s.startsWith(" ".repeat(gutter)) ? s.slice(gutter) : s.replace(/^\s+/, "");
+}
+
+/**
+ * Rendered rows → text worth pasting.
+ *
+ * A LAST RESORT. It cannot undo the hard wrap — `renderAnswer` split the paragraph into
+ * separate lines before they were ever recorded, and nothing in the rendered text says which
+ * newlines were the author's. That is exactly why printed blocks register their source; this
+ * runs only for output that did not.
+ */
+export function cleanRenderedLines(lines) {
+  return lines
+    .map((l) => stripGutter(l))
+    .join("\n")
+    .replace(/[ \t]+$/gm, "")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/**
+ * Was this the second click of a double-click?
+ *
+ * Terminals send no double-click event, so it is two presses close in time AND place. The cell
+ * must match: two clicks on different paragraphs are two single clicks, however fast.
+ */
+export function isDoubleClick(prev, next, maxMs = 400) {
+  if (!prev || !next) return false;
+  if (prev.row !== next.row) return false;
+  // A little horizontal slack — a hand moves a column between presses on a trackpad.
+  if (Math.abs((prev.col ?? 0) - (next.col ?? 0)) > 2) return false;
+  return next.at - prev.at <= maxMs && next.at >= prev.at;
+}
+
+/**
+ * How many clicks in a row landed on the same spot? → 1, 2 or 3.
+ *
+ * `prev` is the RUN so far ({ row, col, at, count }), not just the last click, because a triple
+ * is three clicks in sequence rather than "a double, then another double" — the second click is
+ * within 400ms of the first and the third within 400ms of the second, and comparing only pairs
+ * would either miss the third or count it as the start of a new double.
+ *
+ * Caps at 3: macOS has no fourth gesture here, and a fourth click restarting the run means a
+ * fast repeated click cycles word → line → word rather than doing nothing.
+ */
+export function clickRun(prev, next, maxMs = 400) {
+  if (!isDoubleClick(prev, next, maxMs)) return 1;
+  const count = (prev?.count ?? 1) + 1;
+  return count > 3 ? 1 : count;
+}
+
+/**
+ * The whole typed line, as a selection range over `text`. → { from, to } or null.
+ *
+ * WHAT "THE LINE" MEANS HERE. The input line WRAPS (#179), so the text a user sees can occupy
+ * several physical rows — and the gesture has to mean the whole LOGICAL line, all of `rl.line`.
+ * Selecting the clicked row would hand back a fragment of a long question, which is precisely
+ * the case someone reaches for this gesture to avoid.
+ *
+ * Trailing whitespace is left out: it is invisible, so including it makes the highlight run
+ * past the last character for no reason a user can see, and a copy then carries padding.
+ * Leading whitespace is KEPT — it is inside the text they typed, and deleting the selection
+ * should leave an empty line rather than a stub of spaces.
+ */
+export function lineRangeAt(text) {
+  const s = String(text ?? "");
+  const to = s.replace(/\s+$/, "").length;
+  return to > 0 ? { from: 0, to } : null;
+}
+
+/**
+ * What a keypress does to a live selection on the input line: "delete" | "replace" | "dismiss".
+ *
+ * Every key ends the selection — the only question is whether it takes the text with it. This
+ * is the decision most likely to be wrong by one case, which is why it is out here rather than
+ * inline in the readline hook: Enter, Tab, Escape and every arrow arrive as strings too, and
+ * treating any of them as typed text would silently eat the line.
+ *
+ * The printable test is on the STRING, not the key name: a control key's payload always lands
+ * in \x00-\x1f or \x7f (Enter is \r, Tab is \t, an arrow is a whole ESC [ D), while anything a
+ * person meant to type does not — including accented letters and emoji, which no name-based
+ * test gets right.
+ */
+export function selectionKeyAction(str, key) {
+  if (key?.name === "backspace" || key?.name === "delete") return "delete";
+  if (key?.ctrl || key?.meta) return "dismiss";  // Ctrl+U and friends do their own thing
+  return typeof str === "string" && /^[^\x00-\x1f\x7f]+$/.test(str) ? "replace" : "dismiss";
+}
+
+/** How the copy is reported. Lines matter as much as characters for a block of code. */
+export function copyBadge({ chars, lines, ok = true, why = "" }) {
+  if (!ok) return why ? `copy failed — ${why}` : "copy failed";
+  const n = Number(chars) || 0;
+  const l = Number(lines) || 0;
+  return l > 1 ? `${n} chars · ${l} lines copied` : `${n} chars copied`;
+}
+
+/**
+ * Where the badge goes on its row, or null when it does not fit.
+ *
+ * Right-aligned, and it must NEVER overwrite the row's own text — the feedback would then
+ * destroy what it is confirming. When the line reaches too far right the caller falls back to
+ * the bottom bar. Clipped, never wrapped: a wrapped overlay row breaks the cursor arithmetic
+ * that positions it (ESC[nA counts PHYSICAL rows — the #109 lesson).
+ */
+export function badgePlacement({ rowText, badge, width, gap = 2 }) {
+  const w = Math.max(1, Number(width) || 80);
+  const text = stripAnsi(rowText ?? "");
+  const b = String(badge ?? "");
+  if (!b) return null;
+  // 1-based column where the badge would start.
+  const col = w - b.length + 1;
+  if (col < 1) return null; // wider than the window
+  if (text.length + gap >= col) return null; // would land on the row's own text
+  return { col, text: b };
+}
+
+/**
+ * How to put text on the clipboard on this platform.
+ *
+ * `wl-copy` before `xclip`: Wayland is the default on current Ubuntu, and xclip on a Wayland
+ * session either fails or writes a clipboard nothing reads.
+ */
+export function clipboardCommands(platform = process.platform) {
+  if (platform === "darwin") return [{ cmd: "pbcopy", args: [] }];
+  if (platform === "win32") return [{ cmd: "clip.exe", args: [] }];
+  return [
+    { cmd: "wl-copy", args: [] },
+    { cmd: "xclip", args: ["-selection", "clipboard"] },
+    { cmd: "xsel", args: ["--clipboard", "--input"] },
+  ];
+}
+
+/**
+ * The OSC 52 sequence that asks the TERMINAL to set its clipboard.
+ *
+ * This is the one that works over SSH. `pbcopy` on a remote box writes the clipboard of the
+ * machine nobody is sitting at — which looks exactly like success. Not every terminal permits
+ * it (and there is no reply to check), so it is sent ALONGSIDE the local tool rather than
+ * instead of it.
+ */
+export function osc52(text) {
+  const b64 = Buffer.from(String(text ?? ""), "utf8").toString("base64");
+  return `\x1b]52;c;${b64}\x07`;
+}
+
+/**
+ * Is this text small enough for OSC 52?
+ *
+ * Terminals cap the sequence (commonly ~100KB, xterm's default is far lower). An oversized
+ * write is silently dropped by some and prints garbage into the session on others, so it is
+ * better not to send one at all.
+ */
+export function osc52Fits(text, limit = 74994) {
+  return Buffer.byteLength(String(text ?? ""), "utf8") <= limit;
+}
+
+/**
+ * Find the registered block containing an absolute transcript line.
+ *
+ * Newest first: a range can be re-registered when a block is expanded in place, and the most
+ * recent record is the true one.
+ */
+export function blockAt(blocks, abs) {
+  for (let i = blocks.length - 1; i >= 0; i--) {
+    const b = blocks[i];
+    if (abs >= b.from && abs <= b.to) return b;
+  }
+  return null;
+}
+
+/**
+ * The text a copy should place on the clipboard, given the block (if any) and the rendered
+ * lines that were double-clicked.
+ *
+ * Prefers the block's SOURCE — that is the whole point — and falls back to cleaning what is on
+ * screen for output that registered nothing.
+ */
+export function textToCopy({ block, renderedLines }) {
+  if (block && typeof block.text === "string" && block.text.trim()) {
+    return block.text.replace(/\s+$/, "");
+  }
+  return cleanRenderedLines(renderedLines ?? []);
+}
+
+/** What the badge reports about a piece of text. */
+export function measure(text) {
+  const s = String(text ?? "");
+  return { chars: s.length, lines: s ? s.split("\n").length : 0 };
+}
+
+/**
+ * Where a block just written landed, in ABSOLUTE transcript indices.
+ *
+ * Absolute because the buffer trims from the front, so a stored position has to survive that.
+ * Measured AFTER the write: the final transcript entry is the still-open line, so the last
+ * COMPLETED line is `length - 2`.
+ *
+ * `trailingBlanks` is the trap. `console.log(x + "\n")` emits TWO newlines — the string's and
+ * console.log's own — so a block printed that way is followed by a blank line, and the naive
+ * `length - 2` points at the blank rather than the block's last row. The range then covers the
+ * blank plus all but the final row: a double-click near the end of an answer silently copies
+ * the wrong lines, and nothing about the output looks wrong.
+ */
+export function registeredRange({ dropped, length, rowCount, trailingBlanks = 0 }) {
+  const endAbs = dropped + length - 2 - Math.max(0, trailingBlanks);
+  return { from: endAbs - rowCount + 1, to: endAbs };
+}