libretto 0.6.33 → 0.6.35

package/dist/runtime/recovery/recovery.js

@@ -14,7 +14,10 @@ async function attemptWithRecovery(page, fn, logger, model) {
       throw error;
     }
     if (!model) {
-      throw error;
+      throw new Error(
+        "Recovery was not attempted because no model is configured. Pass a LanguageModel as the fourth argument to attemptWithRecovery, or call the action directly without recovery.",
+        { cause: error }
+      );
     }
     log.info("Action failed, attempting popup recovery", {
       error: error instanceof Error ? error.message : String(error)

package/docs/releasing.md

@@ -49,7 +49,7 @@ The workflow needs `contents: write` to create the GitHub release and tag, and `
 
 After trusted publishing is working, remove any old npm publish token from the repo secrets. npm recommends restricting token-based publishing after the migration.
 
-GitHub release notes are generated by `packages/libretto/scripts/generate-changelog.ts`. The script finds the previous release, compares that tag to the release commit, and passes only the merged PRs in that range to the release notes agent.
+GitHub release notes are generated by `packages/libretto/scripts/generate-changelog.ts`. The script finds the previous release, compares that tag to the release commit, and passes only the merged PRs in that range to the release notes agent. The release workflow uploads the generated notes and a changelog audit JSON artifact containing the release range, eligible PRs, inspected PR diffs, tool calls, and final model output.
 
 ## Prepare a release PR
 
@@ -108,6 +108,8 @@ The GitHub Releases page is the changelog for this repo.
 
 When the workflow runs `pnpm generate-changelog vX.Y.Z`, the script finds the previous GitHub release, compares that tag to the release commit, and extracts PR numbers from the compare commits. The release notes agent can inspect only those PRs with `gh pr view` and `gh pr diff`.
 
+The generator logs the release range and changelog-eligible PRs to stderr. When `CHANGELOG_AUDIT_FILE` is set, it also writes a JSON audit file with the eligible PR list, inspected diff PRs, tool-call summaries, and raw/final release notes. Generation fails if the agent does not inspect every eligible PR diff, so omissions are easier to diagnose before a GitHub release is created.
+
 To keep release notes readable, use clear PR titles and descriptions before merging. If a PR should not appear in the changelog, add the `skip-changelog` label.
 
 ## Notes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "libretto",
-  "version": "0.6.33",
+  "version": "0.6.35",
   "description": "AI-powered browser automation library and CLI built on Playwright",
   "license": "MIT",
   "homepage": "https://libretto.sh",
@@ -70,7 +70,7 @@
     "playwright": "^1.58.2",
     "tsx": "^4.21.0",
     "zod": "^4.3.6",
-    "affordance": "^0.2.1"
+    "affordance": "^0.2.2"
   },
   "scripts": {
     "sync:mirrors": "node ../dev-tools/scripts/sync-mirrors.mjs",
@@ -78,7 +78,6 @@
     "sync-skills": "pnpm run sync:mirrors",
     "check:skills": "pnpm run check:mirrors",
     "build": "tsup --config tsup.config.ts",
-    "lint": "lintcn lint --tsconfig tsconfig.json",
     "type-check": "tsc --noEmit",
     "test": "turbo run test:vitest --filter=libretto --log-order=grouped",
     "test:vitest": "vitest run",

package/scripts/generate-changelog.ts

@@ -1,4 +1,6 @@
 import { execFileSync } from "node:child_process";
+import { dirname } from "node:path";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { Agent, type AgentTool, type AgentEvent } from "@mariozechner/pi-agent-core";
 import { getModel } from "@mariozechner/pi-ai";
 import { Type, type Static } from "@sinclair/typebox";
@@ -60,6 +62,34 @@ interface ReleaseContext {
   pullRequests: PullRequestDetails[];
 }
 
+interface ToolCallAudit {
+  toolCallId: string;
+  toolName: string;
+  args: unknown;
+  isError?: boolean;
+  resultTextBytes?: number;
+  resultTextPreview?: string;
+}
+
+interface ChangelogAudit {
+  tag: string;
+  generatedAt: string;
+  previousTag: string;
+  currentRef: string;
+  eligiblePullRequests: Array<{
+    number: number;
+    title: string;
+    labels: string[];
+    files: string[];
+    url: string;
+  }>;
+  inspectedDiffPullRequests: number[];
+  missingDiffPullRequests: number[];
+  toolCalls: ToolCallAudit[];
+  rawFinalText: string;
+  finalText: string;
+}
+
 function runGh(args: string[]): string {
   return execFileSync("gh", args, {
     encoding: "utf8",
@@ -190,6 +220,7 @@ function buildReleaseContext(currentTag: string): ReleaseContext {
 
 const releaseContext = buildReleaseContext(tag);
 const allowedPullRequestNumbers = new Set(releaseContext.pullRequests.map((pr) => String(pr.number)));
+const auditFilePath = process.env.CHANGELOG_AUDIT_FILE;
 const pullRequestSummary = releaseContext.pullRequests
   .map((pr) => {
     const labels = pr.labels.map((label) => label.name).join(", ") || "none";
@@ -204,6 +235,13 @@ const pullRequestSummary = releaseContext.pullRequests
   })
   .join("\n\n");
 
+console.error(`Changelog release range: ${releaseContext.previousTag}...${releaseContext.currentRef}`);
+console.error("Changelog-eligible PRs:");
+for (const pr of releaseContext.pullRequests) {
+  const labels = pr.labels.map((label) => label.name).join(", ") || "none";
+  console.error(`- #${pr.number} ${pr.title} [${labels}]`);
+}
+
 const GhToolParamsSchema = Type.Object({
   args: Type.String({ description: "Arguments to pass to gh (without the leading 'gh')" }),
 });
@@ -242,6 +280,12 @@ const ghTool: AgentTool = {
           `PR #${number ?? "(missing)"} is outside the ${releaseContext.previousTag}...${releaseContext.currentRef} release range.`,
         );
       }
+
+      // Diff inspection enforcement counts successful 'pr diff NUMBER' calls,
+      // so reject flags like --name-only that would return less than the full diff.
+      if (action === "diff" && parts.length > 3) {
+        throw new Error("'pr diff' accepts only a PR number. Run 'pr diff NUMBER' with no extra arguments.");
+      }
     }
     try {
       const output = runGh(parts);
@@ -277,6 +321,9 @@ const agent = new Agent({
       "- Write concise, user-facing release notes in markdown.",
       "- Group changes into sections like Features, Fixes, and Improvements. Only include sections that have entries.",
       "- Focus on what changed from the user's perspective, not internal implementation details.",
+      "- For feature and improvement entries, include minimal code blocks when they make the change easier to use.",
+      "- Each relevant feature or improvement can have its own snippet.",
+      "- Keep each code block to at most 10 lines, and use at most one comment line inside the block if helpful.",
       "- Do NOT include PR numbers or links.",
       "- Skip PRs labeled 'skip-changelog'.",
       "- Your response must contain ONLY the raw markdown release notes. No preamble like 'Here are the release notes'. No commentary or explanation. No '---' separators. The very first character of your response must be '#'. Example format:",
@@ -291,8 +338,123 @@ const agent = new Agent({
 });
 
 let finalText = "";
+let rawFinalText = "";
+const toolCalls = new Map<string, ToolCallAudit>();
+
+function getObjectProperty(value: unknown, key: string): unknown {
+  if (typeof value !== "object" || value === null || !(key in value)) {
+    return undefined;
+  }
+
+  return (value as Record<string, unknown>)[key];
+}
+
+function getTextContent(value: unknown): string {
+  const content = getObjectProperty(value, "content");
+  if (!Array.isArray(content)) {
+    return "";
+  }
+
+  return content
+    .map((block) => {
+      if (typeof block !== "object" || block === null) {
+        return "";
+      }
+      const type = getObjectProperty(block, "type");
+      const text = getObjectProperty(block, "text");
+      return type === "text" && typeof text === "string" ? text : "";
+    })
+    .filter((text) => text.length > 0)
+    .join("\n");
+}
+
+function getInspectedDiffPullRequests(): number[] {
+  const inspected = new Set<number>();
+  for (const toolCall of toolCalls.values()) {
+    if (toolCall.isError !== false) {
+      // Only count tool calls that completed successfully.
+      continue;
+    }
+
+    const args = getObjectProperty(toolCall.args, "args");
+    if (typeof args !== "string") {
+      continue;
+    }
+
+    const parts = args.trim().split(/\s+/);
+    if (parts[0] !== "pr" || parts[1] !== "diff") {
+      continue;
+    }
+
+    const number = Number(parts[2]);
+    if (Number.isInteger(number)) {
+      inspected.add(number);
+    }
+  }
+
+  return [...inspected].sort((a, b) => a - b);
+}
+
+function buildAudit(): ChangelogAudit {
+  const inspectedDiffPullRequests = getInspectedDiffPullRequests();
+  const inspected = new Set(inspectedDiffPullRequests);
+  const missingDiffPullRequests = releaseContext.pullRequests
+    .map((pr) => pr.number)
+    .filter((number) => !inspected.has(number));
+
+  return {
+    tag,
+    generatedAt: new Date().toISOString(),
+    previousTag: releaseContext.previousTag,
+    currentRef: releaseContext.currentRef,
+    eligiblePullRequests: releaseContext.pullRequests.map((pr) => ({
+      number: pr.number,
+      title: pr.title,
+      labels: pr.labels.map((label) => label.name),
+      files: pr.files.map((file) => file.path),
+      url: pr.url,
+    })),
+    inspectedDiffPullRequests,
+    missingDiffPullRequests,
+    toolCalls: [...toolCalls.values()],
+    rawFinalText,
+    finalText,
+  };
+}
+
+function writeAuditFile(): void {
+  if (!auditFilePath) {
+    return;
+  }
+
+  mkdirSync(dirname(auditFilePath), { recursive: true });
+  writeFileSync(auditFilePath, `${JSON.stringify(buildAudit(), null, 2)}\n`);
+}
 
 agent.subscribe((event: AgentEvent) => {
+  if (event.type === "tool_execution_start") {
+    toolCalls.set(event.toolCallId, {
+      toolCallId: event.toolCallId,
+      toolName: event.toolName,
+      args: event.args,
+    });
+  }
+
+  if (event.type === "tool_execution_end") {
+    const text = getTextContent(event.result);
+    const existing = toolCalls.get(event.toolCallId) ?? {
+      toolCallId: event.toolCallId,
+      toolName: event.toolName,
+      args: undefined,
+    };
+    toolCalls.set(event.toolCallId, {
+      ...existing,
+      isError: event.isError,
+      resultTextBytes: Buffer.byteLength(text, "utf8"),
+      resultTextPreview: text.slice(0, 2000),
+    });
+  }
+
   if (event.type === "agent_end") {
     const messages = event.messages;
     for (let i = messages.length - 1; i >= 0; i--) {
@@ -300,7 +462,8 @@ agent.subscribe((event: AgentEvent) => {
       if (msg.role === "assistant" && Array.isArray(msg.content)) {
         for (const block of msg.content) {
           if (typeof block === "object" && "type" in block && block.type === "text" && "text" in block) {
-            finalText = block.text as string;
+            rawFinalText = block.text as string;
+            finalText = rawFinalText;
             return;
           }
         }
@@ -309,10 +472,18 @@ agent.subscribe((event: AgentEvent) => {
   }
 });
 
-await agent.prompt("Generate the release notes now.");
+try {
+  await agent.prompt("Generate the release notes now.");
+} catch (error) {
+  console.error("Changelog generation failed: agent prompt threw an error.");
+  console.error(error);
+  writeAuditFile();
+  process.exit(1);
+}
 
 if (!finalText) {
   console.error("Changelog generation failed: no text output from agent.");
+  writeAuditFile();
   process.exit(1);
 }
 
@@ -324,7 +495,30 @@ if (headingIndex >= 0) {
   // Already starts with a heading, keep as-is.
 } else {
   console.error("Changelog generation failed: output does not contain markdown headings.");
+  writeAuditFile();
   process.exit(1);
 }
 
+const inspectedDiffPullRequests = getInspectedDiffPullRequests();
+const missingDiffPullRequests = releaseContext.pullRequests
+  .map((pr) => pr.number)
+  .filter((number) => !inspectedDiffPullRequests.includes(number));
+
+console.error(`Inspected PR diffs: ${inspectedDiffPullRequests.map((number) => `#${number}`).join(", ") || "none"}`);
+
+if (missingDiffPullRequests.length > 0) {
+  console.error(
+    `Changelog generation failed: the agent did not inspect diffs for PRs ${missingDiffPullRequests
+      .map((number) => `#${number}`)
+      .join(", ")}.`,
+  );
+  writeAuditFile();
+  process.exit(1);
+}
+
+writeAuditFile();
+if (auditFilePath) {
+  console.error(`Wrote changelog audit: ${auditFilePath}`);
+}
+
 process.stdout.write(finalText);

package/skills/libretto/SKILL.md

@@ -4,7 +4,7 @@ description: "Browser automation CLI for building, maintaining, and running brow
 license: MIT
 metadata:
   author: saffron-health
-  version: "0.6.33"
+  version: "0.6.35"
 ---
 
 ## How Libretto Works

package/skills/libretto-readonly/SKILL.md

@@ -4,7 +4,7 @@ description: "Read-only Libretto workflow for diagnosing live browser state with
 license: MIT
 metadata:
   author: saffron-health
-  version: "0.6.33"
+  version: "0.6.35"
 ---
 
 ## How Libretto Read-Only Works

package/src/cli/core/resolve-model.ts

@@ -105,7 +105,7 @@ async function getProviderModel(
       if (!apiKey) {
         throw new Error(missingProviderCredentialsMessage(provider));
       }
-      // @lintc-ignore Human-approved: we don't want to import unless the user is using that subagent.
+      // oxlint-disable-next-line libretto/no-await-import -- Human-approved: we don't want to import unless the user is using that subagent.
       const { createGoogleGenerativeAI } = await import("@ai-sdk/google");
       const google = createGoogleGenerativeAI({ apiKey });
       return google(modelId);
@@ -115,7 +115,7 @@ async function getProviderModel(
       if (!project) {
         throw new Error(missingProviderCredentialsMessage(provider));
       }
-      // @lintc-ignore Human-approved: we don't want to import unless the user is using that subagent.
+      // oxlint-disable-next-line libretto/no-await-import -- Human-approved: we don't want to import unless the user is using that subagent.
       const { createVertex } = await import("@ai-sdk/google-vertex");
       const vertex = createVertex({
         project,
@@ -128,7 +128,7 @@ async function getProviderModel(
       if (!apiKey) {
         throw new Error(missingProviderCredentialsMessage(provider));
       }
-      // @lintc-ignore Human-approved: we don't want to import unless the user is using that subagent.
+      // oxlint-disable-next-line libretto/no-await-import -- Human-approved: we don't want to import unless the user is using that subagent.
       const { createAnthropic } = await import("@ai-sdk/anthropic");
       const anthropic = createAnthropic({ apiKey });
       return anthropic(modelId);
@@ -138,7 +138,7 @@ async function getProviderModel(
       if (!apiKey) {
         throw new Error(missingProviderCredentialsMessage(provider));
       }
-      // @lintc-ignore Human-approved: we don't want to import unless the user is using that subagent.
+      // oxlint-disable-next-line libretto/no-await-import -- Human-approved: we don't want to import unless the user is using that subagent.
       const { createOpenAI } = await import("@ai-sdk/openai");
       const openai = createOpenAI({ apiKey });
       return openai(modelId);
@@ -148,7 +148,7 @@ async function getProviderModel(
       if (!apiKey) {
         throw new Error(missingProviderCredentialsMessage(provider));
       }
-      // @lintc-ignore Human-approved: we don't want to import unless the user is using that subagent.
+      // oxlint-disable-next-line libretto/no-await-import -- Human-approved: we don't want to import unless the user is using that subagent.
       const { createOpenAI } = await import("@ai-sdk/openai");
       const openrouter = createOpenAI({
         apiKey,

package/src/cli/core/workflow-runtime.ts

@@ -37,7 +37,7 @@ export async function loadDefaultWorkflow(
 ): Promise<ExportedLibrettoWorkflow> {
   let loadedModule: Record<string, unknown>;
   try {
-    // @lintc-ignore Human-approved: user workflow files must be loaded dynamically from the CLI argument.
+    // oxlint-disable-next-line libretto/no-await-import -- Human-approved: user workflow files must be loaded dynamically from the CLI argument.
     loadedModule = (await import(pathToFileURL(absolutePath).href)) as Record<
       string,
       unknown

package/src/runtime/recovery/recovery.ts

@@ -34,7 +34,10 @@ export async function attemptWithRecovery<T>(
     }
 
     if (!model) {
-      throw error;
+      throw new Error(
+        "Recovery was not attempted because no model is configured. Pass a LanguageModel as the fourth argument to attemptWithRecovery, or call the action directly without recovery.",
+        { cause: error },
+      );
     }
 
     log.info("Action failed, attempting popup recovery", {