@markbrutx/promptbook-cli 0.1.0

package/src/commands/annotations.ts

@@ -0,0 +1,107 @@
+import { join } from "node:path";
+import type { Annotation } from "@markbrutx/promptbook-core";
+import {
+  ANNOTATION_QUEUE_DIR,
+  ANNOTATION_QUEUE_FILE,
+  parseInbox,
+  serializeInbox,
+} from "@markbrutx/promptbook-core";
+import type { ParsedArgs } from "../args.js";
+import { requirePromptsDir } from "../config.js";
+import type { IO } from "../io.js";
+import { formatContext, plural } from "../style.js";
+
+function queueFile(promptsDir: string): string {
+  return join(promptsDir, ANNOTATION_QUEUE_DIR, ANNOTATION_QUEUE_FILE);
+}
+
+/** Read the queue via the injected fs; a missing file reads as empty. */
+async function readQueue(io: IO, file: string): Promise<Annotation[]> {
+  try {
+    return parseInbox(await io.fs.readFile(file));
+  } catch {
+    return [];
+  }
+}
+
+function describeTarget(annotation: Annotation): string {
+  const { target } = annotation;
+  if (target.prompt !== undefined) {
+    const ctx = formatContext(target.context ?? {});
+    return ctx === "" ? target.prompt : `${target.prompt} @ ${ctx}`;
+  }
+  return target.fragmentId ?? "(unknown)";
+}
+
+function renderAnnotation(annotation: Annotation): string {
+  return [
+    `• ${annotation.id}  [${describeTarget(annotation)}]`,
+    `  anchor: ${annotation.anchor.fragmentId} — “${annotation.anchor.anchorText}”`,
+    `  ${annotation.comment}`,
+  ].join("\n");
+}
+
+async function listAnnotations(args: ParsedArgs, io: IO, file: string): Promise<number> {
+  const open = (await readQueue(io, file)).filter((a) => a.status === "open");
+  if (args.json) {
+    io.stdout(`${JSON.stringify(open, null, 2)}\n`);
+    return 0;
+  }
+  if (open.length === 0) {
+    io.stdout("No open annotations.\n");
+    return 0;
+  }
+  io.stdout(`${open.map(renderAnnotation).join("\n\n")}\n`);
+  return 0;
+}
+
+async function resolveAnnotation(args: ParsedArgs, io: IO, file: string): Promise<number> {
+  const id = args.operands[1];
+  if (id === undefined || id === "") {
+    io.stderr('error: "annotations resolve" needs an <id>.\n');
+    return 1;
+  }
+  const all = await readQueue(io, file);
+  const next = all.filter((a) => a.id !== id);
+  if (next.length === all.length) {
+    io.stderr(`error: no annotation with id "${id}".\n`);
+    return 1;
+  }
+  await io.writeFile(file, serializeInbox(next));
+  io.stdout(`Resolved ${id}.\n`);
+  return 0;
+}
+
+async function clearAnnotations(io: IO, file: string): Promise<number> {
+  const open = (await readQueue(io, file)).filter((a) => a.status === "open");
+  await io.writeFile(file, "");
+  io.stdout(`Cleared ${plural(open.length, "annotation")}.\n`);
+  return 0;
+}
+
+/**
+ * `annotations`: the terminal-agnostic side of the viewer's feedback bridge.
+ * `list` prints open annotations (or `--json`), `resolve <id>` removes one, and
+ * `clear` empties the queue. All operate on `<dir>/.annotations/inbox.jsonl` —
+ * the same file the viewer writes — so any agent can drain it from any terminal.
+ */
+export async function cmdAnnotations(args: ParsedArgs, io: IO): Promise<number> {
+  const promptsDir = await requirePromptsDir(io, args.dir);
+  if (promptsDir === null) {
+    return 1;
+  }
+  const file = queueFile(promptsDir);
+  const action = args.operands[0] ?? "list";
+
+  switch (action) {
+    case "list":
+      return listAnnotations(args, io, file);
+    case "resolve":
+      return resolveAnnotation(args, io, file);
+    case "clear":
+      return clearAnnotations(io, file);
+    default:
+      io.stderr(`error: unknown annotations action "${action}". Use list, resolve <id> or clear.\n`);
+      return 1;
+  }
+}

package/src/commands/bundle.ts

@@ -0,0 +1,71 @@
+import { isAbsolute, relative, resolve as resolvePath, sep } from "node:path";
+import type { PromptBook } from "@markbrutx/promptbook-core";
+import { loadPrompts, serializeBook, serializeBookJson } from "@markbrutx/promptbook-core";
+import type { ParsedArgs } from "../args.js";
+import { requirePromptsDir } from "../config.js";
+import { emitWarnings, type IO } from "../io.js";
+
+/** Rewrite an absolute source path to a portable, forward-slash path relative to `dir`. */
+function relativizeSource(sourceFile: string, dir: string): string {
+  if (!isAbsolute(sourceFile)) {
+    return sourceFile;
+  }
+  return relative(dir, sourceFile).split(sep).join("/");
+}
+
+/** Copy a keyed map, relativizing each value's `sourceFile`. */
+function relativizeMap<T extends { sourceFile: string }>(map: Map<string, T>, dir: string): Map<string, T> {
+  return new Map(
+    [...map].map(([key, value]) => [key, { ...value, sourceFile: relativizeSource(value.sourceFile, dir) }]),
+  );
+}
+
+/**
+ * Copy the book with every `sourceFile` made relative to the prompts folder, so
+ * the generated module carries portable paths instead of one machine's absolute
+ * layout. Resolution and lint ignore `sourceFile`, so this never affects output.
+ */
+function portableBook(book: PromptBook, dir: string): PromptBook {
+  return {
+    fragments: relativizeMap(book.fragments, dir),
+    compositions: relativizeMap(book.compositions, dir),
+    codePrompts: relativizeMap(book.codePrompts, dir),
+    warnings: book.warnings,
+  };
+}
+
+/**
+ * `bundle [<dir>]`: load a prompts folder and emit it as a single importable
+ * module exporting `book: PromptBook` (so a runtime can import the book instead
+ * of reading the disk). Writes to stdout, or to a file with `-o`. `--json`
+ * emits a structured dump instead of the TypeScript module.
+ *
+ * The folder comes from the positional `<dir>`, else `--dir`, else config /
+ * `./prompts` (the standard resolution).
+ */
+export async function cmdBundle(args: ParsedArgs, io: IO): Promise<number> {
+  const promptsDir = await requirePromptsDir(io, args.operands[0] ?? args.dir);
+  if (promptsDir === null) {
+    return 1;
+  }
+
+  const book = portableBook(await loadPrompts(promptsDir, io.fs), promptsDir);
+  emitWarnings(io, book.warnings);
+
+  const output = args.json ? serializeBookJson(book) : serializeBook(book, { typed: !args.plain });
+
+  if (args.out === undefined) {
+    io.stdout(output);
+    return 0;
+  }
+
+  const outPath = resolvePath(io.cwd(), args.out);
+  try {
+    await io.writeFile(outPath, output);
+  } catch (error) {
+    io.stderr(`error: cannot write "${outPath}": ${(error as Error).message}\n`);
+    return 1;
+  }
+  io.stderr(`wrote ${outPath}\n`);
+  return 0;
+}

package/src/commands/eval.ts

@@ -0,0 +1,137 @@
+import type {
+  EvalReport,
+  Fixture,
+  LintFinding,
+  LintRule,
+  ModelAdapter,
+  PromptBook,
+} from "@markbrutx/promptbook-core";
+import {
+  defaultRules,
+  evaluate,
+  lint,
+  loadFixtures,
+  loadPrompts,
+  resolveBook,
+} from "@markbrutx/promptbook-core";
+import { openRouterAdapter } from "@markbrutx/promptbook-openrouter";
+import type { ParsedArgs } from "../args.js";
+import { type EvalConfig, evalConfigFrom, loadConfig, requirePromptsDir } from "../config.js";
+import { type AdapterOptions, colorEnabled, type IO } from "../io.js";
+import { renderEvalReport } from "../render-eval.js";
+
+/** Convert a `name|glob` pattern (only `*` is special) into an anchored regex. */
+function globToRegExp(pattern: string): RegExp {
+  const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+  return new RegExp(`^${escaped}$`);
+}
+
+/** Build the model adapter: the injected one for tests, else OpenRouter. */
+function buildAdapter(io: IO, options: AdapterOptions): ModelAdapter {
+  if (io.makeAdapter) {
+    return io.makeAdapter(options);
+  }
+  if (options.model === undefined) {
+    throw new Error('eval requires a model: pass --model <id> or set "eval.model" in promptbook.json.');
+  }
+  const adapterOptions = { model: options.model, apiKey: options.apiKey, baseUrl: options.baseUrl };
+  return openRouterAdapter(adapterOptions);
+}
+
+/** Lint every resolved variant; return findings with error severity. */
+function lintGate(book: PromptBook, fixtures: Fixture[], rules: LintRule[]): LintFinding[] {
+  const errors: LintFinding[] = [];
+  for (const fixture of fixtures) {
+    const result = resolveBook(book, fixture.prompt, fixture.context ?? {});
+    const report = lint({ book, result }, rules);
+    for (const finding of report.findings) {
+      if (finding.severity === "error") {
+        errors.push({ ...finding, message: `${fixture.name}: ${finding.message}` });
+      }
+    }
+  }
+  return errors;
+}
+
+/**
+ * `eval [<name|glob>]`: load fixtures from `<dir>/fixtures`, assemble each
+ * prompt, sample it through a model adapter and report pass-rate. A fixture
+ * passes when its `passRate >= --threshold` (default 1). Exit is non-zero when
+ * any fixture fails the gate or on any error (no key, unknown prompt/fixture,
+ * missing folder). `--lint` runs a static gate over every variant first; with
+ * any lint error it aborts before spending tokens. Adapter is injectable via
+ * `io.makeAdapter` so tests never touch the network.
+ */
+export async function cmdEval(args: ParsedArgs, io: IO): Promise<number> {
+  const config = await loadConfig(io);
+  const promptsDir = await requirePromptsDir(io, args.dir, config);
+  if (promptsDir === null) {
+    return 1;
+  }
+
+  let fixtures: Fixture[];
+  try {
+    fixtures = await loadFixtures(promptsDir, io.fs);
+  } catch (error) {
+    io.stderr(`error: ${(error as Error).message}\n`);
+    return 1;
+  }
+
+  const filter = args.operands[0];
+  if (filter !== undefined) {
+    const re = globToRegExp(filter);
+    fixtures = fixtures.filter((f) => re.test(f.name));
+    if (fixtures.length === 0) {
+      io.stderr(`error: no fixtures match "${filter}".\n`);
+      return 1;
+    }
+  }
+  if (fixtures.length === 0) {
+    io.stderr(`error: no fixtures found in ${promptsDir}/fixtures.\n`);
+    return 1;
+  }
+
+  const book = await loadPrompts(promptsDir, io.fs);
+  const threshold = args.threshold ?? 1;
+
+  let report: EvalReport;
+  try {
+    if (args.lint) {
+      const errors = lintGate(book, fixtures, defaultRules());
+      if (errors.length > 0) {
+        io.stderr("error: lint gate failed before sampling:\n");
+        for (const finding of errors) {
+          io.stderr(`  ${finding.ruleId}: ${finding.message}\n`);
+        }
+        return 1;
+      }
+    }
+
+    const evalConfig: EvalConfig = evalConfigFrom(config);
+    const adapter = buildAdapter(io, {
+      model: args.model ?? evalConfig.model,
+      apiKey: io.env.OPENROUTER_API_KEY,
+      baseUrl: evalConfig.baseUrl,
+    });
+
+    const evalInput = {
+      book,
+      fixtures,
+      adapter,
+      passThreshold: threshold,
+      ...(args.samples !== undefined ? { samples: args.samples } : {}),
+    };
+    report = await evaluate(evalInput);
+  } catch (error) {
+    io.stderr(`error: ${(error as Error).message}\n`);
+    return 1;
+  }
+
+  if (args.json) {
+    io.stdout(`${JSON.stringify(report, null, 2)}\n`);
+  } else {
+    io.stdout(renderEvalReport(report, threshold, colorEnabled(io)));
+  }
+
+  return report.failed > 0 ? 1 : 0;
+}

package/src/commands/lint.ts

@@ -0,0 +1,71 @@
+import type { DefaultRulesOptions, LintReport } from "@markbrutx/promptbook-core";
+import { defaultRules, lint, loadPrompts, resolveBook } from "@markbrutx/promptbook-core";
+import type { ParsedArgs } from "../args.js";
+import { buildContext, lintConfigFrom, loadConfig, requirePromptsDir } from "../config.js";
+import { colorEnabled, emitWarnings, type IO } from "../io.js";
+import { renderLintReport } from "../render-lint.js";
+
+/**
+ * `lint [<prompt>]`: run static checks over the prompts folder. With a prompt
+ * it resolves under the given context and runs both book- and resolved-scope
+ * rules; without one it runs book-scope rules over the whole book. The report
+ * (human-readable or `--json`) goes to stdout; warnings go to stderr. Exit is
+ * non-zero when any error is found, or any warning under `--strict`.
+ */
+export async function cmdLint(args: ParsedArgs, io: IO): Promise<number> {
+  const config = await loadConfig(io);
+  const promptsDir = await requirePromptsDir(io, args.dir, config);
+  if (promptsDir === null) {
+    return 1;
+  }
+
+  const lintConfig = lintConfigFrom(config);
+  const ruleOptions: DefaultRulesOptions = {};
+  const maxTokens = args.maxTokens ?? lintConfig.maxTokens;
+  if (maxTokens !== undefined) {
+    ruleOptions.maxTokens = maxTokens;
+  }
+  if (lintConfig.bannedTokens !== undefined) {
+    ruleOptions.bannedTokens = lintConfig.bannedTokens;
+  }
+  const rules = defaultRules(ruleOptions);
+
+  const book = await loadPrompts(promptsDir, io.fs);
+
+  const prompt = args.operands[0];
+  let report: LintReport;
+  let label: string;
+  let warnings: string[] = book.warnings;
+  try {
+    if (prompt !== undefined) {
+      const context = await buildContext(io, args.ctx, args.contextFile);
+      const result = resolveBook(book, prompt, context);
+      // trace.warnings already includes the load-time book.warnings.
+      warnings = result.trace.warnings;
+      report = lint({ book, result }, rules);
+      label = prompt;
+    } else {
+      report = lint({ book }, rules);
+      label = "book";
+    }
+  } catch (error) {
+    io.stderr(`error: ${(error as Error).message}\n`);
+    return 1;
+  }
+
+  emitWarnings(io, warnings);
+
+  if (args.json) {
+    io.stdout(`${JSON.stringify(report, null, 2)}\n`);
+  } else {
+    io.stdout(renderLintReport(report, label, colorEnabled(io)));
+  }
+
+  if (report.errorCount > 0) {
+    return 1;
+  }
+  if (args.strict && report.warningCount > 0) {
+    return 1;
+  }
+  return 0;
+}

package/src/commands/ls.ts

@@ -0,0 +1,90 @@
+import { relative } from "node:path";
+import { loadPrompts } from "@markbrutx/promptbook-core";
+import type { ParsedArgs } from "../args.js";
+import { requirePromptsDir } from "../config.js";
+import { emitWarnings, type IO } from "../io.js";
+
+/**
+ * `ls`: list compositions (name, base length, rule count), code-prompts
+ * (name, description, sample count) and fragments (id, kind, tags, source) —
+ * the unified menu of every prompt in the book. `--fragments`/`--compositions`
+ * narrow the output (code-prompts ride with compositions, both being prompts);
+ * `--json` emits a structured list instead of the human-readable tree.
+ */
+export async function cmdLs(args: ParsedArgs, io: IO): Promise<number> {
+  const promptsDir = await requirePromptsDir(io, args.dir);
+  if (promptsDir === null) {
+    return 1;
+  }
+
+  const book = await loadPrompts(promptsDir, io.fs);
+  emitWarnings(io, book.warnings);
+
+  const onlyOneSection = args.fragments || args.compositions;
+  const showCompositions = args.compositions || !onlyOneSection;
+  const showFragments = args.fragments || !onlyOneSection;
+
+  const compositions = [...book.compositions.values()].sort((a, b) => a.name.localeCompare(b.name));
+  const codePrompts = [...book.codePrompts.values()].sort((a, b) => a.name.localeCompare(b.name));
+  const fragments = [...book.fragments.values()].sort((a, b) => a.id.localeCompare(b.id));
+
+  if (args.json) {
+    const out: Record<string, unknown> = {};
+    if (showCompositions) {
+      out.compositions = compositions.map((c) => ({
+        name: c.name,
+        base: c.base.length,
+        rules: c.rules.length,
+      }));
+      out.codePrompts = codePrompts.map((c) => ({
+        name: c.name,
+        description: c.description ?? null,
+        samples: c.samples.length,
+      }));
+    }
+    if (showFragments) {
+      out.fragments = fragments.map((f) => ({
+        id: f.id,
+        kind: f.kind ?? null,
+        tags: f.tags ?? [],
+        sourceFile: f.sourceFile,
+      }));
+    }
+    io.stdout(`${JSON.stringify(out, null, 2)}\n`);
+    return 0;
+  }
+
+  const lines: string[] = [];
+  if (showCompositions) {
+    lines.push("compositions:");
+    if (compositions.length === 0) {
+      lines.push("  (none)");
+    }
+    for (const c of compositions) {
+      lines.push(`  ${c.name}  base=${c.base.length}  rules=${c.rules.length}`);
+    }
+    if (codePrompts.length > 0) {
+      lines.push("");
+      lines.push("code-prompts:");
+      for (const c of codePrompts) {
+        lines.push(`  ${c.name}  kind=code  samples=${c.samples.length}`);
+      }
+    }
+  }
+  if (showFragments) {
+    if (lines.length > 0) {
+      lines.push("");
+    }
+    lines.push("fragments:");
+    if (fragments.length === 0) {
+      lines.push("  (none)");
+    }
+    for (const f of fragments) {
+      const tags = f.tags && f.tags.length > 0 ? f.tags.join(",") : "-";
+      const source = relative(promptsDir, f.sourceFile) || f.sourceFile;
+      lines.push(`  ${f.id}  kind=${f.kind ?? "-"}  tags=${tags}  ${source}`);
+    }
+  }
+  io.stdout(`${lines.join("\n")}\n`);
+  return 0;
+}

package/src/commands/resolve.ts

@@ -0,0 +1,47 @@
+import type { ResolveResult } from "@markbrutx/promptbook-core";
+import { resolve } from "@markbrutx/promptbook-core";
+import type { ParsedArgs } from "../args.js";
+import { buildContext, requirePromptsDir } from "../config.js";
+import { colorEnabled, emitWarnings, type IO } from "../io.js";
+import { renderExplain } from "../render-explain.js";
+
+/**
+ * `resolve <prompt>`: assemble the prompt and print its text to stdout. With
+ * `--json`, print `{ text, trace }` instead; with `--explain`, additionally
+ * print the trace to stderr. Warnings always go to stderr so nothing is lost.
+ */
+export async function cmdResolve(args: ParsedArgs, io: IO): Promise<number> {
+  const prompt = args.operands[0];
+  if (prompt === undefined) {
+    io.stderr('error: resolve requires a <prompt> name. Run "promptbook --help".\n');
+    return 1;
+  }
+
+  const promptsDir = await requirePromptsDir(io, args.dir);
+  if (promptsDir === null) {
+    return 1;
+  }
+
+  let result: ResolveResult;
+  try {
+    const context = await buildContext(io, args.ctx, args.contextFile);
+    result = await resolve({ promptsDir, prompt, context, fs: io.fs });
+  } catch (error) {
+    io.stderr(`error: ${(error as Error).message}\n`);
+    return 1;
+  }
+
+  const { text, trace } = result;
+  emitWarnings(io, trace.warnings);
+
+  if (args.json) {
+    io.stdout(`${JSON.stringify({ text, trace }, null, 2)}\n`);
+    return 0;
+  }
+
+  if (args.explain) {
+    io.stderr(renderExplain(trace, colorEnabled(io)));
+  }
+  io.stdout(`${text}\n`);
+  return 0;
+}

package/src/commands/view.ts

@@ -0,0 +1,82 @@
+import type { ParsedArgs } from "../args.js";
+import { requirePromptsDir } from "../config.js";
+import type { IO } from "../io.js";
+
+/** Options forwarded to `@markbrutx/promptbook-viewer`'s `startViewer`. */
+export interface ViewerStartOptions {
+  promptsDir: string;
+  port?: number;
+  open: boolean;
+}
+
+/** The subset of the viewer's running handle the CLI uses. */
+export interface ViewerHandle {
+  url: string;
+  close(): Promise<void>;
+}
+
+/**
+ * Injectable seam for `view`: how to start the viewer and how to keep the
+ * process alive. Defaults dynamically import the optional `@markbrutx/promptbook-viewer`
+ * package and block until a termination signal; tests inject fakes so no
+ * server or signal handling is needed.
+ */
+export interface ViewDeps {
+  start(options: ViewerStartOptions): Promise<ViewerHandle>;
+  hold(handle: ViewerHandle): Promise<number>;
+}
+
+/** Thrown by the default `start` when the optional viewer package is absent. */
+class ViewerNotInstalledError extends Error {}
+
+interface ViewerModule {
+  startViewer(options: ViewerStartOptions): Promise<ViewerHandle>;
+}
+
+const defaultViewDeps: ViewDeps = {
+  async start(options) {
+    let mod: ViewerModule;
+    try {
+      mod = (await import("@markbrutx/promptbook-viewer")) as unknown as ViewerModule;
+    } catch {
+      throw new ViewerNotInstalledError();
+    }
+    return mod.startViewer(options);
+  },
+  hold(handle) {
+    return new Promise<number>((resolve) => {
+      const shutdown = (): void => {
+        void handle.close().then(() => resolve(0));
+      };
+      process.once("SIGINT", shutdown);
+      process.once("SIGTERM", shutdown);
+    });
+  },
+};
+
+/**
+ * `view`: start the local web viewer over the prompts folder, print its URL,
+ * and keep running until interrupted. Render-only; the viewer makes no model
+ * calls. `--no-open` skips launching the browser; `--port` pins the port.
+ */
+export async function cmdView(args: ParsedArgs, io: IO, deps: ViewDeps = defaultViewDeps): Promise<number> {
+  const promptsDir = await requirePromptsDir(io, args.dir);
+  if (promptsDir === null) {
+    return 1;
+  }
+
+  let handle: ViewerHandle;
+  try {
+    handle = await deps.start({ promptsDir, port: args.port, open: !args.noOpen });
+  } catch (error) {
+    if (error instanceof ViewerNotInstalledError) {
+      io.stderr('error: the viewer is not installed. Add it with "npm i -D @markbrutx/promptbook-viewer".\n');
+      return 1;
+    }
+    io.stderr(`error: ${(error as Error).message}\n`);
+    return 1;
+  }
+
+  io.stdout(`promptbook viewer running at ${handle.url}\n`);
+  return deps.hold(handle);
+}