@cairnvibe/indexer 0.2.0 → 0.2.1

package/dist/concurrency.js

@@ -25,17 +25,6 @@ async function mapWithConcurrency(items, limit, fn) {
     await Promise.all(Array.from({ length: workerCount }, () => worker()));
     return results;
 }
-/**
- * Retries a rate-limited (429) or transient-server-error (5xx) call with
- * backoff — found live and necessary, not theoretical: raising describeAll's
- * concurrency (see DEFAULT_DESCRIBE_CONCURRENCY) surfaced a real 429 from
- * Groq on a 40-page build within seconds ("Rate limit reached... tokens per
- * minute"), which without this would have thrown straight out of
- * mapWithConcurrency's Promise.all and aborted the *entire* build,
- * discarding every other page's already-completed work too. Honors the
- * provider's Retry-After header when present (both the Anthropic and Groq
- * SDKs expose one on a 429), falls back to exponential backoff otherwise.
- */
 async function withRetry(fn, opts) {
     const maxAttempts = opts?.maxAttempts ?? 4;
     const baseDelayMs = opts?.baseDelayMs ?? 1000;
@@ -49,7 +38,16 @@ async function withRetry(fn, opts) {
             if (!isRetryable(err) || attempt === maxAttempts)
                 throw err;
             const delayMs = retryAfterMs(err) ?? baseDelayMs * 2 ** (attempt - 1);
-            console.error(`[cairn] retryable error (attempt ${attempt}/${maxAttempts}), waiting ${Math.round(delayMs)}ms:`, errorMessage(err));
+            const message = errorMessage(err);
+            // Default behavior (plain `cairn build`, unchanged): log every attempt directly.
+            // A caller that wants something quieter/nicer (`cairn setup`'s spinner) passes
+            // its own onRetry instead of getting this raw per-attempt line.
+            if (opts?.onRetry) {
+                opts.onRetry({ attempt, maxAttempts, delayMs: Math.round(delayMs), message });
+            }
+            else {
+                console.error(`[cairn] retryable error (attempt ${attempt}/${maxAttempts}), waiting ${Math.round(delayMs)}ms:`, message);
+            }
             await new Promise((resolve) => setTimeout(resolve, delayMs));
         }
     }

package/dist/l3-describe.js

@@ -18,7 +18,7 @@ const GLOBAL_ROUTE_LABEL = "(present on every page — layout/framework elements
 // rather than maximal: still bounded by whatever the provider's real rate
 // limit is, this just stops leaving 3 of 4 rotated keys idle.
 const DEFAULT_DESCRIBE_CONCURRENCY = 6;
-async function describeAll(rootDir, facts, client, concurrency = DEFAULT_DESCRIBE_CONCURRENCY) {
+async function describeAll(rootDir, facts, client, concurrency = DEFAULT_DESCRIBE_CONCURRENCY, onRetry) {
     const absRoot = node_path_1.default.resolve(rootDir);
     const cacheDir = node_path_1.default.join(absRoot, CACHE_DIR);
     node_fs_1.default.mkdirSync(cacheDir, { recursive: true });
@@ -51,7 +51,7 @@ async function describeAll(rootDir, facts, client, concurrency = DEFAULT_DESCRIB
                 file: page.file,
                 source,
                 elements: page.elements.map(toDescribeElementInput),
-            }));
+            }), { onRetry });
         }
         catch (err) {
             // One page permanently failing (retries exhausted, or a
@@ -94,7 +94,7 @@ async function describeAll(rootDir, facts, client, concurrency = DEFAULT_DESCRIB
                     file: files.join(", "),
                     source,
                     elements: facts.frameworkElements.map(toDescribeElementInput),
-                }));
+                }), { onRetry });
             }
             catch (err) {
                 console.error(`[cairn] describing framework elements failed after retries — degrading:`, err);

package/dist/prompt.js

@@ -12,6 +12,7 @@ exports.closePrompts = closePrompts;
 exports.ask = ask;
 exports.askYesNo = askYesNo;
 exports.askOptional = askOptional;
+exports.selectFromList = selectFromList;
 const node_readline_1 = __importDefault(require("node:readline"));
 let sharedInterface = null;
 function rl() {
@@ -40,3 +41,25 @@ async function askOptional(question) {
     const answer = await ask(question);
     return answer.length > 0 ? answer : null;
 }
+/**
+ * A numbered menu instead of free text — the actual fix for "I had to
+ * type the exact provider name." Prints the choices, accepts a number,
+ * and (for anyone who prefers typing) also accepts the label/value text
+ * itself, case-insensitively. Empty answer takes `defaultIndex`.
+ */
+async function selectFromList(question, options, defaultIndex = 0) {
+    console.log(question);
+    options.forEach((o, i) => {
+        const marker = i === defaultIndex ? " (default)" : "";
+        console.log(`  ${i + 1}. ${o.label}${marker}`);
+    });
+    const answer = await ask(`Choose [1-${options.length}]: `);
+    if (!answer)
+        return options[defaultIndex].value;
+    const asNumber = Number(answer);
+    if (Number.isInteger(asNumber) && asNumber >= 1 && asNumber <= options.length) {
+        return options[asNumber - 1].value;
+    }
+    const byText = options.find((o) => o.value.toLowerCase() === answer.toLowerCase() || o.label.toLowerCase().includes(answer.toLowerCase()));
+    return byText ? byText.value : options[defaultIndex].value;
+}

package/dist/setup.js

@@ -1,7 +1,8 @@
 "use strict";
 // `cairn setup` — the one-command onboarding path: install what's
-// needed, ask only for what's actually optional (skippable), scaffold
-// the backend, wire the widget into the real layout file, build the
+// needed, ask only for what's actually optional (skippable, and picked
+// from a real numbered menu rather than typed free text), scaffold the
+// backend, wire the widget into the real layout file, build the
 // manifest once now, and leave a `prebuild` hook so it rebuilds itself
 // on every future `npm run build` without another manual step.
 //
@@ -9,8 +10,8 @@
 // `init` stays the safe, deterministic, non-interactive primitive
 // (never touches an existing file, never installs anything, never
 // prompts); `setup` is the opinionated wizard built from those same
-// primitives plus the two things `init` intentionally doesn't do:
-// install dependencies and edit an existing layout file.
+// primitives plus the things `init` intentionally doesn't do: install
+// dependencies, edit an existing layout file, and actually build.
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -28,7 +29,14 @@ const l3_describe_1 = require("./l3-describe");
 const llm_1 = require("./llm");
 const manifest_1 = require("./manifest");
 const core_1 = require("@cairnvibe/core");
+const ui_1 = require("./ui");
 const PACKAGES = ["@cairnvibe/core", "@cairnvibe/sdk", "@cairnvibe/indexer"];
+// Lower than cairn build's own default (6) — a first-time setup is exactly
+// the scenario most likely to be running on a free-tier key with a tight
+// per-minute token budget; found live, not theoretical (a real `cairn
+// setup` run against Groq's on-demand tier hit a 429-retry cascade at the
+// default concurrency on a small handful of pages).
+const SETUP_BUILD_CONCURRENCY = 3;
 function readPackageJson(absDir) {
     const p = node_path_1.default.join(absDir, "package.json");
     if (!node_fs_1.default.existsSync(p))
@@ -46,13 +54,98 @@ function alreadyInstalled(pkg) {
     const deps = { ...pkg.dependencies, ...pkg.devDependencies };
     return PACKAGES.every((p) => !!deps[p]);
 }
+/** One build attempt — spinner-driven, quiet on individual retries (they
+ * update the same line instead of scrolling the terminal), and honest
+ * about failure instead of throwing a raw stack trace at the user. */
+async function attemptBuild(dir, provider, key) {
+    if (provider === "anthropic")
+        process.env.ANTHROPIC_API_KEY = key;
+    if (provider === "groq")
+        process.env.GROQ_API_KEYS = key;
+    const spinner = new ui_1.Spinner(`Building the manifest (${provider}) ...`);
+    spinner.start();
+    try {
+        const client = provider === "anthropic" ? new llm_1.AnthropicDescribeClient() : new llm_1.GroqDescribeClient();
+        const facts = (0, l1_scan_1.scanL1)(dir);
+        const l2 = (0, l2_reachability_1.computeL2)(dir, facts);
+        const l3 = await (0, l3_describe_1.describeAll)(dir, facts, client, SETUP_BUILD_CONCURRENCY, (info) => {
+            spinner.update(`Building the manifest (${provider}) ... rate-limited, retrying in ${Math.round(info.delayMs / 1000)}s (attempt ${info.attempt}/${info.maxAttempts})`);
+        });
+        const manifest = core_1.ManifestSchema.parse((0, manifest_1.assembleManifest)(dir, facts, l2, l3));
+        node_fs_1.default.writeFileSync(node_path_1.default.join(node_path_1.default.resolve(dir), "ui-manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
+        spinner.stop((0, ui_1.green)(`✓ wrote ui-manifest.json (${manifest.pages.length} page(s))`));
+        return { ok: true, pageCount: manifest.pages.length };
+    }
+    catch (err) {
+        spinner.stop((0, ui_1.red)("✗ build failed"));
+        return { ok: false, error: err };
+    }
+}
+/** Runs after a failed build: explains what actually went wrong in plain
+ * English, then offers real next actions instead of just dying. Loops
+ * until the user picks something that resolves (a successful retry) or
+ * explicitly chooses to skip. */
+async function recoverFromBuildFailure(dir, provider, key, err) {
+    const classified = (0, ui_1.classifyError)(err);
+    console.log("");
+    console.log((0, ui_1.yellow)(`Here's what happened: ${classified.summary}`));
+    const options = classified.kind === "rate_limit"
+        ? [
+            { label: "Try again in a bit (same provider)", value: "retry" },
+            { label: "Switch to the other provider and try that instead", value: "switch" },
+            { label: "Skip for now — I'll run `npx cairn build .` later", value: "skip" },
+        ]
+        : classified.kind === "auth"
+            ? [
+                { label: "Paste the key again (I probably mistyped it)", value: "rekey" },
+                { label: "Switch to the other provider instead", value: "switch" },
+                { label: "Skip for now — I'll run `npx cairn build .` later", value: "skip" },
+            ]
+            : [
+                { label: "Try again", value: "retry" },
+                { label: "Switch to the other provider instead", value: "switch" },
+                { label: "Skip for now — I'll run `npx cairn build .` later", value: "skip" },
+            ];
+    for (;;) {
+        const choice = await (0, prompt_1.selectFromList)("What do you want to do?", options, 0);
+        if (choice === "skip")
+            return null;
+        let nextProvider = provider;
+        let nextKey = key;
+        if (choice === "switch") {
+            nextProvider = provider === "anthropic" ? "groq" : "anthropic";
+            const pasted = await (0, prompt_1.askOptional)(nextProvider === "anthropic" ? "Paste your ANTHROPIC_API_KEY: " : "Paste your GROQ_API_KEYS: ");
+            if (!pasted) {
+                console.log((0, ui_1.dim)("No key given — back to the menu."));
+                continue;
+            }
+            nextKey = pasted;
+        }
+        else if (choice === "rekey") {
+            const pasted = await (0, prompt_1.askOptional)(provider === "anthropic" ? "Paste your ANTHROPIC_API_KEY: " : "Paste your GROQ_API_KEYS: ");
+            if (!pasted) {
+                console.log((0, ui_1.dim)("No key given — back to the menu."));
+                continue;
+            }
+            nextKey = pasted;
+        }
+        // choice === "retry" falls through with the same provider/key.
+        const result = await attemptBuild(dir, nextProvider, nextKey);
+        if (result.ok)
+            return { provider: nextProvider, key: nextKey };
+        console.log("");
+        console.log((0, ui_1.yellow)(`Still failing: ${(0, ui_1.classifyError)(result.error).summary}`));
+        // loop back to the menu rather than recursing — keeps this one flat retry
+        // loop instead of a call stack that grows with every attempt
+    }
+}
 async function runSetup(dir) {
     const absDir = node_path_1.default.resolve(dir);
-    console.log(`cairn setup: looking at ${absDir}\n`);
+    console.log(`${(0, ui_1.bold)("cairn setup")} — looking at ${absDir}\n`);
     // 1. Scaffold what init already safely can — framework detection, the
     // backend route, .env.example. Never overwrites anything that exists.
     const init = (0, init_1.runInit)(dir);
-    console.log(`Detected: ${init.framework}`);
+    console.log(`Detected: ${(0, ui_1.bold)(init.framework)}`);
     for (const f of init.filesWritten)
         console.log(`  wrote   ${node_path_1.default.relative(absDir, f) || f}`);
     for (const f of init.filesSkipped)
@@ -71,31 +164,43 @@ async function runSetup(dir) {
     // cleanly if already present (e.g. re-running setup after a partial run).
     const pkg = readPackageJson(absDir);
     if (!alreadyInstalled(pkg)) {
-        console.log(`Installing ${PACKAGES.join(", ")} ...`);
+        const spinner = new ui_1.Spinner(`Installing ${PACKAGES.join(", ")} ...`);
+        spinner.start();
         try {
-            (0, node_child_process_1.execSync)(`npm install ${PACKAGES.join(" ")}`, { cwd: absDir, stdio: "inherit" });
+            (0, node_child_process_1.execSync)(`npm install ${PACKAGES.join(" ")}`, { cwd: absDir, stdio: "pipe" });
+            spinner.stop((0, ui_1.green)(`✓ installed ${PACKAGES.join(", ")}`));
         }
         catch {
-            console.error("\nnpm install failed — install these yourself and re-run `cairn setup`:");
-            console.error(`  npm install ${PACKAGES.join(" ")}`);
+            spinner.stop((0, ui_1.red)("✗ npm install failed"));
+            console.error(`Install these yourself and re-run \`cairn setup\`:\n  npm install ${PACKAGES.join(" ")}`);
             return;
         }
     }
     else {
-        console.log("Dependencies already installed — skipping.\n");
+        console.log((0, ui_1.dim)("Dependencies already installed — skipping."));
     }
-    // 3. Ask only what's actually needed, everything skippable.
-    console.log("\nA couple of quick questions — press enter to skip anything you'll add later.\n");
+    // 3. Ask only what's actually needed, everything skippable, picked from a
+    // real menu rather than typed free text.
+    console.log(`\n${(0, ui_1.bold)("A couple of quick questions")} — press enter to skip anything you'll add later.\n`);
     let provider = null;
     let providerKey = null;
-    const wantsLLM = await (0, prompt_1.askYesNo)("Set up an LLM provider now? (needed for the agent to actually answer anything)", true);
-    if (wantsLLM) {
-        const choice = (await (0, prompt_1.ask)("Anthropic or Groq? [anthropic] ")).toLowerCase();
-        provider = choice.startsWith("g") ? "groq" : "anthropic";
+    const llmChoice = await (0, prompt_1.selectFromList)("Set up an LLM provider now? (needed for the agent to actually answer anything)", [
+        { label: "Anthropic (Claude)", value: "anthropic" },
+        { label: "Groq", value: "groq" },
+        { label: "Skip — I'll add one to .env later", value: "skip" },
+    ], 0);
+    if (llmChoice !== "skip") {
+        provider = llmChoice;
         providerKey = await (0, prompt_1.askOptional)(provider === "anthropic" ? "Paste your ANTHROPIC_API_KEY: " : "Paste your GROQ_API_KEYS: ");
     }
-    const wantsVoice = await (0, prompt_1.askYesNo)("Set up voice (Deepgram — speech in/out) now?", false);
-    const deepgramKey = wantsVoice ? await (0, prompt_1.askOptional)("Paste your DEEPGRAM_API_KEY: ") : null;
+    // Honest about what's actually implemented here — Deepgram is the only
+    // voice provider this SDK wires up today, so this is "on or off," not a
+    // real multi-provider menu dressed up as one.
+    const voiceChoice = await (0, prompt_1.selectFromList)("Set up voice now?", [
+        { label: "Deepgram (speech in + out)", value: "deepgram" },
+        { label: "Skip — no voice for now", value: "skip" },
+    ], 1);
+    const deepgramKey = voiceChoice === "deepgram" ? await (0, prompt_1.askOptional)("Paste your DEEPGRAM_API_KEY: ") : null;
     (0, prompt_1.closePrompts)();
     // 4. Write a real .env (not just .env.example) with whatever was actually given.
     const envLines = [];
@@ -120,37 +225,29 @@ async function runSetup(dir) {
     const framework = init.framework;
     const inject = (0, inject_widget_1.injectWidget)(dir, framework);
     if (inject.injected) {
-        console.log(`wired <Copilot/> into ${node_path_1.default.relative(absDir, inject.filePath)}`);
+        console.log((0, ui_1.green)(`✓ wired <Copilot/> into ${node_path_1.default.relative(absDir, inject.filePath)}`));
     }
     else {
         console.log(`\n<Copilot/> not auto-wired (${inject.reason}). Add it yourself:`);
         console.log('  import { Copilot } from "@cairnvibe/sdk";');
         console.log("  <Copilot registeredActions={[]} onDo={(action, target) => { /* run it */ }} />");
     }
-    // 6. Build the manifest once now, if we actually have a usable key —
-    // no point trying (and failing loudly) with nothing to call.
-    const haveUsableKey = (provider === "anthropic" && providerKey) || (provider === "groq" && providerKey);
-    if (haveUsableKey) {
-        console.log(`\nBuilding the manifest (${provider}) ...`);
-        try {
-            // Reuse the same env-var-driven construction `cairn build` uses, rather
-            // than duplicating each client's options shape here — the .env written
-            // above already has this exact value, this just makes it live for the
-            // rest of this process too.
-            if (provider === "anthropic")
-                process.env.ANTHROPIC_API_KEY = providerKey;
-            if (provider === "groq")
-                process.env.GROQ_API_KEYS = providerKey;
-            const client = provider === "anthropic" ? new llm_1.AnthropicDescribeClient() : new llm_1.GroqDescribeClient();
-            const facts = (0, l1_scan_1.scanL1)(dir);
-            const l2 = (0, l2_reachability_1.computeL2)(dir, facts);
-            const l3 = await (0, l3_describe_1.describeAll)(dir, facts, client);
-            const manifest = core_1.ManifestSchema.parse((0, manifest_1.assembleManifest)(dir, facts, l2, l3));
-            node_fs_1.default.writeFileSync(node_path_1.default.join(absDir, "ui-manifest.json"), JSON.stringify(manifest, null, 2) + "\n");
-            console.log(`wrote ui-manifest.json (${manifest.pages.length} page(s))`);
-        }
-        catch (err) {
-            console.error(`manifest build failed (${err.message}) — run \`npx cairn build .\` yourself once your key is confirmed working.`);
+    // 6. Build the manifest once now, if we actually have a usable key — no
+    // point trying (and failing loudly) with nothing to call. On failure,
+    // don't just print a stack trace and give up: classify what went wrong
+    // and offer real next steps (retry / switch provider / skip).
+    if (provider && providerKey) {
+        console.log("");
+        let result = await attemptBuild(dir, provider, providerKey);
+        if (!result.ok) {
+            const recovered = await recoverFromBuildFailure(dir, provider, providerKey, result.error);
+            if (recovered) {
+                provider = recovered.provider;
+                providerKey = recovered.key;
+            }
+            // else: user chose to skip — fall through with the original provider/key
+            // still recorded for the prebuild script below; ui-manifest.json is
+            // simply not written yet.
         }
     }
     else {
@@ -167,9 +264,9 @@ async function runSetup(dir) {
             ? `${fresh.scripts.prebuild} && cairn build . --provider ${providerFlag} --if-configured`
             : `cairn build . --provider ${providerFlag} --if-configured`;
         node_fs_1.default.writeFileSync(pkgPath, JSON.stringify(fresh, null, 2) + "\n");
-        console.log('added a "prebuild" script — the manifest regenerates automatically on every `npm run build`.');
-        console.log("(--if-configured means a build with no key set yet skips this step instead of failing the whole build —");
-        console.log(" set the same key as an environment variable on whatever platform you deploy to.)");
+        console.log('\nadded a "prebuild" script — the manifest regenerates automatically on every `npm run build`.');
+        console.log((0, ui_1.dim)("(--if-configured means a build with no key set yet skips this step instead of failing the whole build —"));
+        console.log((0, ui_1.dim)(" set the same key as an environment variable on whatever platform you deploy to.)"));
     }
-    console.log("\nDone. `npm run dev` and ask it something.");
+    console.log(`\n${(0, ui_1.bold)("Done.")} \`npm run dev\` and ask it something.`);
 }

package/dist/ui.js

@@ -0,0 +1,79 @@
+"use strict";
+// A small terminal UI toolkit for `cairn setup` — zero new dependencies,
+// plain ANSI codes (this is a handful of escape sequences, not a reason
+// to pull in a whole chalk/ora dependency tree). Every piece degrades
+// gracefully when stdout isn't a real TTY (CI logs, piped output): the
+// spinner just prints its text once instead of animating, and colors
+// still work fine (most CI systems render ANSI color codes correctly
+// even without an interactive terminal).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Spinner = exports.cyan = exports.red = exports.yellow = exports.green = exports.bold = exports.dim = void 0;
+exports.classifyError = classifyError;
+const ESC = "\x1b[";
+const color = (code, s) => `${ESC}${code}m${s}${ESC}0m`;
+const dim = (s) => color("2", s);
+exports.dim = dim;
+const bold = (s) => color("1", s);
+exports.bold = bold;
+const green = (s) => color("32", s);
+exports.green = green;
+const yellow = (s) => color("33", s);
+exports.yellow = yellow;
+const red = (s) => color("31", s);
+exports.red = red;
+const cyan = (s) => color("36", s);
+exports.cyan = cyan;
+const FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+class Spinner {
+    text;
+    frame = 0;
+    timer = null;
+    animated;
+    constructor(text) {
+        this.text = text;
+        this.animated = !!process.stdout.isTTY;
+    }
+    start() {
+        if (!this.animated) {
+            console.log((0, exports.dim)(this.text));
+            return;
+        }
+        this.timer = setInterval(() => {
+            process.stdout.write(`\r${ESC}K${(0, exports.cyan)(FRAMES[(this.frame = (this.frame + 1) % FRAMES.length)])} ${this.text}`);
+        }, 80);
+    }
+    /** Update the line in place without starting a new one. */
+    update(text) {
+        this.text = text;
+        if (!this.animated)
+            console.log((0, exports.dim)(text));
+    }
+    /** Stop animating and print a final, non-spinning result line. */
+    stop(finalLine) {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        if (this.animated)
+            process.stdout.write(`\r${ESC}K`);
+        console.log(finalLine);
+    }
+}
+exports.Spinner = Spinner;
+/** Turns a raw thrown error into a plain-English category + summary —
+ * the thing `cairn setup` actually reasons about when deciding what to
+ * offer next (switch provider vs. just retry vs. nothing to fix). */
+function classifyError(err) {
+    const status = err?.status;
+    const message = err instanceof Error ? err.message : String(err);
+    if (status === 429 || /rate.?limit/i.test(message)) {
+        return { kind: "rate_limit", summary: "the provider is rate-limiting requests — this key has hit its per-minute quota" };
+    }
+    if (status === 401 || status === 403 || /invalid.*key|unauthorized|authentication/i.test(message)) {
+        return { kind: "auth", summary: "that API key was rejected — check it's correct and active" };
+    }
+    if ((typeof status === "number" && status >= 500) || /ECONNRESET|ETIMEDOUT|network/i.test(message)) {
+        return { kind: "network", summary: "the provider's servers had a problem — often transient" };
+    }
+    return { kind: "unknown", summary: message };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cairnvibe/indexer",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Cairn's analyzer and installer (the `cairn` CLI) — scans Next.js source or crawls any running app, and scaffolds the backend either way.",
   "license": "MIT",
   "publishConfig": { "access": "public" },