@opennous/cli 0.12.3 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @opennous/cli changelog
 
+## 0.13.0
+
+### Added: `backfill`, in one command
+
+`npx @opennous/cli@latest backfill` reads your history on your own Claude or ChatGPT plan. With no
+saved key it opens the browser sign-in and carries on. If neither Claude Code nor Codex is signed in,
+it says so before it asks you to sign in to anything (GET-1085, GET-1087).
+
+After a sync it only reads: nothing is pulled again and no new contacts appear.
+
 ## 0.12.3
 
 ### Fixed: a plain message when the server's certificate isn't trusted

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opennous/cli",
-  "version": "0.12.3",
+  "version": "0.14.0",
   "description": "Nous CLI — set up Nous, connect it to your coding agent, and read the Context API from your terminal",
   "type": "module",
   "bin": {

package/src/backfill.js

@@ -0,0 +1,157 @@
+// `nous backfill` (GET-1085): read your history on your own Claude or ChatGPT plan. Our server runs
+// the production read and hands this loop one model call at a time; each is answered by the claude
+// or codex CLI you are already signed in to.
+//
+// EMAIL TEXT IS UNTRUSTED. A call's prompt holds other people's words, so the model answering it
+// gets no tools at all: an email must not be able to read your disk. For Codex the read-only
+// sandbox is not enough (it can still read files), so its shell tool is turned off.
+
+import { spawn, spawnSync } from "node:child_process";
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+/** What to type to carry on. `npx` because most people never installed `nous` (GET-1087). */
+export const RERUN = "npx @opennous/cli@latest backfill";
+export const NO_AGENT = "Sign in to Claude Code (`claude`) or Codex (`codex login`) first: this runs on your own plan, so it needs one of them. Then run this again.";
+
+export const POLL_MS = 2000;
+export const WORKERS = 3;
+export const LIMIT_WAIT_MS = 30 * 60_000;
+const CALL_TIMEOUT_MS = 5 * 60_000;
+const RETRIES = 2;
+const PROGRESS_MS = 30_000;
+
+/** The command for one call. Pure. Model and effort come from the server. */
+export function commandFor(agent, { model, effort }, { system, messages }) {
+  const user = messages.map((m) => (m.role === "assistant" ? `ASSISTANT: ${m.content}` : m.content)).join("\n\n");
+  if (agent === "claude") {
+    return {
+      cmd: "claude",
+      args: ["-p", "--model", model, "--effort", effort, "--output-format", "json", "--tools", "",
+        "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}', "--setting-sources", "",
+        "--disable-slash-commands", "--no-session-persistence",
+        "--system-prompt", system || "Follow the instructions in the user message exactly."],
+      input: user,
+    };
+  }
+  return {
+    cmd: "codex",
+    args: ["exec", "--json", "-m", model, "--skip-git-repo-check", "--sandbox", "read-only", "--ephemeral",
+      "--ignore-user-config", "-c", "mcp_servers={}", "-c", "features.shell_tool=false",
+      "-c", `model_reasoning_effort="${effort}"`],
+    input: `${system ? `<instructions>\n${system}\n</instructions>\n\n` : ""}${user}\n\nReply only with the answer.`,
+  };
+}
+
+/** The answer and token use from a CLI's output. Pure. Throws when there is no answer. */
+export function parseAnswer(agent, out) {
+  if (agent === "claude") {
+    const j = JSON.parse(out);
+    if (j.is_error) throw new Error(String(j.result));
+    return { text: j.result, usage: { input_tokens: j.usage?.input_tokens ?? 0, output_tokens: j.usage?.output_tokens ?? 0 } };
+  }
+  let text = "", usage = null;
+  for (const line of out.split("\n")) {
+    let e;
+    try { e = JSON.parse(line); } catch { continue; }
+    if (e.item?.type === "agent_message") text = e.item.text;
+    if (e.usage) usage = e.usage;
+  }
+  if (!text) throw new Error("codex gave no answer");
+  return { text, usage: { input_tokens: usage?.input_tokens ?? 0, output_tokens: usage?.output_tokens ?? 0 } };
+}
+
+/** Whether an error means the plan's limit, which passes with time, rather than a bad call. Pure. */
+export function isPlanLimit(message) {
+  return /usage limit|rate.?limit|hit your .*limit|limit reached|too many requests|\b429\b|quota/i.test(String(message));
+}
+
+export function detectAgents(run = spawnSync) {
+  const found = [];
+  if (run("claude", ["auth", "status"], { encoding: "utf8" }).status === 0) found.push("claude");
+  if (run("codex", ["login", "status"], { encoding: "utf8" }).status === 0) found.push("codex");
+  return found;
+}
+
+/** Without the API key variables, so the call runs on the plan, not a key billed by the token. */
+function planEnv(env = process.env) {
+  const clean = { ...env, DISABLE_AUTOUPDATER: "1" };
+  for (const k of ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "CODEX_API_KEY"]) delete clean[k];
+  return clean;
+}
+
+function runCommand({ cmd, args, input }, cwd) {
+  return new Promise((resolve, reject) => {
+    const p = spawn(cmd, args, { cwd, env: planEnv(), stdio: ["pipe", "pipe", "pipe"] });
+    let out = "", err = "";
+    p.stdout.on("data", (d) => { out += d; });
+    p.stderr.on("data", (d) => { err += d; });
+    const timer = setTimeout(() => { p.kill(); reject(new Error("timed out")); }, CALL_TIMEOUT_MS);
+    p.on("error", (e) => { clearTimeout(timer); reject(e); });
+    p.on("close", (code) => {
+      clearTimeout(timer);
+      if (code === 0) resolve(out); else reject(new Error((err || out).slice(-400) || `exit ${code}`));
+    });
+    p.stdin.end(input);
+  });
+}
+
+/** Nothing to read in this window. Pure. */
+export const emptyEstimate = (counts) => !Object.values(counts ?? {}).some((n) => n > 0);
+
+/** One line on how far the job is, or null when there is nothing to say yet. Pure. */
+export function progressLine(reading, answered) {
+  if (reading?.finished) return "Wrapping up: last summaries for your accounts.";
+  const stop = `Stop any time with Ctrl-C; run \`${RERUN}\` again to carry on.`;
+  if (reading?.total) return `Read ${reading.done} of ${reading.total} ${reading.kind === "meetings" ? "meetings" : "email threads"}. ${stop}`;
+  if (answered) return `${answered} reads done on your plan. ${stop}`;
+  return `Getting ready to read. This can take a few minutes. ${stop}`;
+}
+
+/**
+ * The loop: lease calls, answer them, until the job ends. Injected: `api` ({ calls, answer, resume }),
+ * `run` (one command → stdout), `sleep`, `log`. Stopping loses nothing: every read that finished
+ * is saved, and unanswered calls come back to the next run.
+ */
+export async function runBackfill({ api, agent, jobId, run, sleep, log, now = Date.now, workers = WORKERS, limitWaitMs = LIMIT_WAIT_MS, pollMs = POLL_MS, progressMs = PROGRESS_MS }) {
+  let waitUntil = 0, answered = 0, lastLine = null, lastAt = -Infinity;
+  const say = (line) => {
+    if (!line || (line === lastLine && now() - lastAt < progressMs) || now() - lastAt < Math.min(progressMs, 1000)) return;
+    lastLine = line; lastAt = now(); log(line);
+  };
+  const answerOne = async (call, models) => {
+    for (let attempt = 0; ; attempt++) {
+      if (now() < waitUntil) await sleep(waitUntil - now());
+      try {
+        const { text, usage } = parseAnswer(agent, await run(commandFor(agent, models[agent], call)));
+        await api.answer(call.id, { answer: text, model: models[agent].model, usage });
+        answered++;
+        return;
+      } catch (err) {
+        if (isPlanLimit(err.message)) {
+          if (now() >= waitUntil) log(`Your ${agent} plan hit its limit. Waiting ${Math.round(limitWaitMs / 60000)} minutes, then carrying on. You can stop now and run this again later instead.`);
+          waitUntil = Math.max(waitUntil, now() + limitWaitMs);
+          continue;
+        }
+        if (attempt >= RETRIES) { await api.answer(call.id, { error: err.message }); return; }
+      }
+    }
+  };
+  for (;;) {
+    const page = await api.calls(jobId, workers);
+    if (page.status === "done") return { status: "done", answered, read: page.reading?.read ?? 0, failed: page.reading?.failed ?? 0 };
+    if (page.status === "failed") throw new Error(page.error || "The backfill failed on our side.");
+    // The server gave up waiting for us (a long sleep, say). We're back, so pick it up.
+    if (page.status === "paused_own_model") { await api.resume?.(); await sleep(pollMs); continue; }
+    say(progressLine(page.reading, answered));
+    if (!page.calls.length) { await sleep(pollMs); continue; }
+    await Promise.all(page.calls.map((c) => answerOne(c, page.models)));
+  }
+}
+
+/** The real pieces for runBackfill. */
+export function liveRun() {
+  const cwd = mkdtempSync(join(tmpdir(), "nous-backfill-"));
+  return (command) => runCommand(command, cwd);
+}

package/src/index.js

@@ -107,9 +107,12 @@ function writeConfig(cfg) {
   writeFileSync(CONFIG_PATH, JSON.stringify(cfg, null, 2));
 }
 
-function apiClient() {
+/** A key is there to use: from the environment, or saved by a sign-in. */
+const hasKey = () => !!(process.env.NOUS_API_KEY || readConfig().apiKey);
+
+function apiClient(key) {
   const cfg = readConfig();
-  const apiKey = process.env.NOUS_API_KEY || cfg.apiKey;
+  const apiKey = key || process.env.NOUS_API_KEY || cfg.apiKey;
   const apiUrl = process.env.NOUS_API_URL || cfg.apiUrl || "https://api.opennous.cloud";
 
   if (!apiKey) {
@@ -137,7 +140,7 @@ function apiClient() {
     }
     if (!res.ok) {
       const err = await res.json().catch(() => ({ message: res.statusText }));
-      console.error(`Error: ${err.error || err.message}`);
+      console.error(`Error: ${err.message || err.error}`);
       process.exit(1);
     }
     if (res.status === 204) return {};
@@ -285,14 +288,18 @@ async function browserLogin(url, opts = {}) {
     } catch { continue; }
 
     if (j.status === "approved" && j.api_key) {
-      const cfg = readConfig();
-      cfg.apiKey = j.api_key;
-      cfg.apiUrl = apiUrl;
-      writeConfig(cfg);
+      // `save: false` is one command's key: the sign-in on this machine stays what it was.
+      if (opts.save !== false) {
+        const cfg = readConfig();
+        cfg.apiKey = j.api_key;
+        cfg.apiUrl = apiUrl;
+        writeConfig(cfg);
+      }
       const who = await fetchWhoami(apiUrl, j.api_key);
       console.log(`\n\n  ${green("✓")} ${bold("Signed in")}${who ? ` as ${identityLine(who)}` : ""}`);
-      console.log(`  ${dim("Key saved to ~/.nous/config.json")}\n`);
-      return true;
+      if (opts.save !== false) console.log(`  ${dim("Key saved to ~/.nous/config.json")}`);
+      console.log("");
+      return j.api_key;
     }
     if (j.status === "denied")  { console.log("\nAuthorization was denied."); process.exit(1); }
     if (j.status === "expired") { console.log("\nThis sign-in expired. Run it again."); process.exit(1); }
@@ -538,6 +545,81 @@ program
   });
 
 // nous logout — top-level alias of `auth logout`. Removes the saved API key.
+program
+  .command("backfill")
+  .description("Read your email and meeting history on your own Claude or ChatGPT plan")
+  .option("--agent <agent>", "claude or codex (default: whichever you're signed in to)")
+  .option("--days <days>", "how many days back to read, 1 to 730", "180")
+  .option("--workspace <id>", "the workspace to read; the app's command fills this in")
+  .option("--yes", "start without asking")
+  .option("--url <url>", "API base URL (default: https://api.opennous.cloud)")
+  .action(async (opts) => {
+    const { detectAgents, runBackfill, liveRun, emptyEstimate, RERUN, NO_AGENT } = await import("./backfill.js");
+    const days = Number(opts.days);
+    if (!Number.isInteger(days) || days < 1 || days > 730) {
+      console.error("--days must be a whole number from 1 to 730.");
+      process.exit(1);
+    }
+    const agents = detectAgents();
+    const agent = opts.agent ?? agents[0];
+    if (!agent || !agents.includes(agent)) {
+      console.error(opts.agent ? `You're not signed in to ${opts.agent}.` : NO_AGENT);
+      process.exit(1);
+    }
+    // Asked after the check above, so nobody signs in to be told they can't run this.
+    if (!hasKey()) await browserLogin(opts.url);
+    // A key saved here may be for another workspace, so say which one this is, and never read the
+    // wrong one: the app's command names its workspace, and a key for any other is not used.
+    const cfg = readConfig();
+    const whoamiUrl = opts.url || process.env.NOUS_API_URL || cfg.apiUrl || "https://api.opennous.cloud";
+    let key = process.env.NOUS_API_KEY || cfg.apiKey;
+    let who = await fetchWhoami(whoamiUrl, key);
+    if (opts.workspace && who?.workspace?.id !== opts.workspace) {
+      console.log(`This machine is signed in to ${who?.workspace?.name ?? "another workspace"}. Sign in to the workspace this backfill is for.`);
+      key = await browserLogin(opts.url, { save: false });
+      who = await fetchWhoami(whoamiUrl, key);
+      if (who?.workspace?.id !== opts.workspace) {
+        console.error(`That sign-in is for ${who?.workspace?.name ?? "another workspace"}. Open the right workspace in the app, then run this again.`);
+        process.exit(1);
+      }
+    }
+    const workspace = who?.workspace?.name ?? "your workspace";
+    const api = apiClient(key);
+    const est = await api.post("/v2/backfill/own/estimate", { days });
+    if (emptyEstimate(est.counts)) {
+      console.log(`\nNothing to read in ${workspace} for the last ${days} days. If your sync is still running, try again when it's done.`);
+      return;
+    }
+    console.log(`\nWorkspace: ${workspace}`);
+    console.log(`Last ${days} days: about ${est.counts.email_thread} email threads, ${est.counts.meeting} meetings, ${est.counts.call} calls.`);
+    console.log(`They're filtered first, then read on your ${agent} plan. None of it uses your OpenNous credits.`);
+    console.log("You can stop at any time and run this again: it carries on where it stopped.\n");
+    if (!opts.yes) {
+      const { createInterface } = await import("node:readline/promises");
+      const rl = createInterface({ input: process.stdin, output: process.stdout });
+      const ok = /^y/i.test(await rl.question("Start? (y/N) "));
+      rl.close();
+      if (!ok) return;
+    }
+    const job = await api.post("/v2/backfill/own", { days });
+    console.log(job.resumed ? "Picking up where you left off. Stop any time with Ctrl-C." : `Started. Stop any time with Ctrl-C; run \`${RERUN}\` again to carry on.`);
+    process.once("SIGINT", () => {
+      console.log(`\nStopped. Everything read so far is saved. Run \`${RERUN}\` to carry on.`);
+      process.exit(130);
+    });
+    const result = await runBackfill({
+      agent, jobId: job.job_id, run: liveRun(), log: console.log,
+      sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+      api: {
+        calls: (id, limit) => api.get(`/v2/backfill/own/${id}/calls`, { limit }),
+        answer: (id, body) => api.post(`/v2/backfill/own/calls/${id}`, body),
+        resume: () => api.post("/v2/backfill/own", { days }),
+      },
+    });
+    console.log(`Done. This run: ${result.read} conversations read, ${result.answered} model calls on your ${agent} plan.`);
+    if (result.failed) console.log(`${result.failed} conversations couldn't be read. Run \`${RERUN}\` again to retry them.`);
+  });
+
 program
   .command("logout")
   .description("Sign out — remove the saved API key")
@@ -550,4 +632,5 @@ program
   });
 
 // ---------------------------------------------------------------------------
-program.name("nous").version("0.12.0").parse();
+const { version } = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
+program.name("nous").version(version).parse();