@letta-ai/letta-code 0.31.5 → 0.31.7

package/scripts/builtin-skills-watch/finalize-result.ts

@@ -0,0 +1,44 @@
+#!/usr/bin/env bun
+/** Validates and encodes one agent-authored review result for the runner. */
+
+import { readAnalysis, readReviewResult } from "./update-tracker.ts";
+
+interface Args {
+  analysisFile: string | null;
+  resultFile: string | null;
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = { analysisFile: null, resultFile: null };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--analysis-file") {
+      args.analysisFile = argv[++index] ?? null;
+    } else if (arg === "--result-file") {
+      args.resultFile = argv[++index] ?? null;
+    } else if (arg === "--help" || arg === "-h") {
+      console.log(
+        "Usage: bun scripts/builtin-skills-watch/finalize-result.ts --analysis-file FILE --result-file FILE",
+      );
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (!args.analysisFile || !args.resultFile) {
+    throw new Error("--analysis-file and --result-file are required");
+  }
+  return args;
+}
+
+function main(): void {
+  const args = parseArgs(process.argv.slice(2));
+  const analysis = readAnalysis(args.analysisFile as string);
+  const result = readReviewResult(args.resultFile as string, analysis);
+  const encoded = Buffer.from(JSON.stringify(result), "utf8").toString(
+    "base64",
+  );
+  console.log(`SKILL_WATCH_RESULT ${encoded}`);
+}
+
+if (import.meta.main) main();

package/scripts/builtin-skills-watch/reconcile-results.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from "bun:test";
+import {
+  type CandidatePullRequest,
+  selectCanonicalPullRequest,
+} from "./reconcile-results.ts";
+
+describe("pending watcher PR reconciliation", () => {
+  test("prefers a merged PR and closes every open duplicate", () => {
+    const merged = pullRequest(4067, "MERGED");
+    const firstDuplicate = pullRequest(4077, "OPEN");
+    const secondDuplicate = pullRequest(4099, "OPEN");
+
+    expect(
+      selectCanonicalPullRequest([firstDuplicate, secondDuplicate, merged]),
+    ).toEqual({
+      canonical: merged,
+      duplicateOpen: [firstDuplicate, secondDuplicate],
+    });
+  });
+
+  test("uses the oldest open PR when no candidate was merged", () => {
+    const older = pullRequest(4078, "OPEN");
+    const newer = pullRequest(4098, "OPEN");
+
+    expect(selectCanonicalPullRequest([newer, older])).toEqual({
+      canonical: older,
+      duplicateOpen: [newer],
+    });
+  });
+
+  test("ignores closed unmerged PRs and rejects multiple merged PRs", () => {
+    expect(selectCanonicalPullRequest([pullRequest(1, "CLOSED")])).toEqual({
+      canonical: null,
+      duplicateOpen: [],
+    });
+    expect(() =>
+      selectCanonicalPullRequest([
+        pullRequest(1, "MERGED"),
+        pullRequest(2, "MERGED"),
+      ]),
+    ).toThrow("Multiple merged watcher PRs");
+  });
+});
+
+function pullRequest(number: number, state: string): CandidatePullRequest {
+  return {
+    number,
+    author: { login: "amelia-letta" },
+    baseRefName: "main",
+    body: "Builtin-skill-watch: creating-skills@aaaaaaaaaaaa-0000000000000001",
+    files: [{ path: "src/skills/builtin/creating-skills/SKILL.md" }],
+    headRefOid: number.toString(16).padStart(40, "0"),
+    isDraft: state === "OPEN",
+    mergedAt: state === "MERGED" ? "2026-08-27T00:00:00Z" : null,
+    state,
+    url: `https://github.com/letta-ai/letta-code/pull/${number}`,
+  };
+}

package/scripts/builtin-skills-watch/reconcile-results.ts

@@ -0,0 +1,311 @@
+#!/usr/bin/env bun
+/** Reconciles pending tracker candidates against PR side effects before retries. */
+
+import {
+  type BuiltinSkillWatchAnalysis,
+  buildAnalysis,
+  DEFAULT_TARGET_REPO,
+  listBuiltinSkillsAtCommit,
+} from "./analysis.ts";
+import type { ReviewEvidence } from "./evidence.ts";
+import { createIssueComment, editIssueBody, ghJson, runGh } from "./github.ts";
+import {
+  parseTrackerState,
+  recordOutcome,
+  renderTrackerBody,
+} from "./tracker.ts";
+import {
+  getOpenTrackerIssue,
+  type PullRequestView,
+  verifyReconciledPullRequest,
+} from "./update-tracker.ts";
+
+interface Args {
+  repo: string;
+  trackerIssue: number | null;
+  expectedGithubLogin: string | null;
+  dryRun: boolean;
+}
+
+export interface CandidatePullRequest extends PullRequestView {
+  mergedAt: string | null;
+  number: number;
+}
+
+export interface PullRequestReconciliation {
+  canonical: CandidatePullRequest | null;
+  duplicateOpen: CandidatePullRequest[];
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    repo: DEFAULT_TARGET_REPO,
+    trackerIssue: null,
+    expectedGithubLogin: null,
+    dryRun: false,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--repo") args.repo = argv[++index] ?? args.repo;
+    else if (arg === "--tracker-issue") {
+      args.trackerIssue = Number(argv[++index]);
+    } else if (arg === "--expected-github-login") {
+      args.expectedGithubLogin = argv[++index] ?? null;
+    } else if (arg === "--dry-run") args.dryRun = true;
+    else if (arg === "--help" || arg === "-h") {
+      console.log(
+        "Usage: bun scripts/builtin-skills-watch/reconcile-results.ts --tracker-issue ISSUE --expected-github-login LOGIN [--repo OWNER/REPO] [--dry-run]",
+      );
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (!args.trackerIssue || Number.isNaN(args.trackerIssue)) {
+    throw new Error("--tracker-issue is required");
+  }
+  if (!args.expectedGithubLogin) {
+    throw new Error("--expected-github-login is required");
+  }
+  return args;
+}
+
+function main(): void {
+  const args = parseArgs(process.argv.slice(2));
+  assertAuthenticatedLogin(args.expectedGithubLogin as string);
+  const tracker = getOpenTrackerIssue(args.repo, args.trackerIssue as number);
+  let state = parseTrackerState(tracker.body);
+  const inventory = listBuiltinSkillsAtCommit("HEAD");
+  let reconciled = 0;
+
+  for (const skill of Object.keys(state.pending).sort()) {
+    const pending = state.pending[skill];
+    if (!pending) continue;
+    const analysis = buildAnalysis({
+      skill,
+      currentSha: pending.current_sha,
+      auditAt: pending.audit_at,
+      previousAudit: previousAudit(state, skill),
+    });
+    analysis.workflow_run_url = workflowRunUrl(pending.workflow_run_id);
+    const pullRequests = findCandidatePullRequests(
+      args.repo,
+      analysis.candidate_id,
+    );
+    const selected = selectCanonicalPullRequest(pullRequests);
+    if (!selected.canonical) continue;
+
+    verifyReconciledPullRequest(
+      args.repo,
+      selected.canonical.url,
+      analysis,
+      args.expectedGithubLogin as string,
+    );
+    for (const duplicate of selected.duplicateOpen) {
+      verifyReconciledPullRequest(
+        args.repo,
+        duplicate.url,
+        analysis,
+        args.expectedGithubLogin as string,
+      );
+    }
+
+    if (args.dryRun) {
+      console.log(
+        `${analysis.skill}: would reconcile ${selected.canonical.url}${renderDuplicatePlan(selected.duplicateOpen)}`,
+      );
+      continue;
+    }
+
+    for (const duplicate of selected.duplicateOpen) {
+      closeDuplicatePullRequest(args.repo, duplicate, selected.canonical);
+    }
+    const evidence = reconciliationEvidence(analysis, selected.canonical);
+    const evidenceUrl = createIssueComment(
+      args.repo,
+      args.trackerIssue as number,
+      renderEvidenceComment(analysis, selected.canonical, evidence),
+    );
+    state = recordOutcome(state, {
+      analysis,
+      outcome: "pr_created",
+      notes: reconciliationNotes(selected.canonical),
+      prUrl: selected.canonical.url,
+      evidence,
+      evidenceUrl,
+    });
+    reconciled += 1;
+  }
+
+  if (args.dryRun) return;
+  const latest = getOpenTrackerIssue(args.repo, args.trackerIssue as number);
+  if (latest.body !== tracker.body) {
+    throw new Error(
+      "Tracker body changed while PR side effects were reconciled",
+    );
+  }
+  if (reconciled > 0) {
+    editIssueBody(
+      args.repo,
+      args.trackerIssue as number,
+      renderTrackerBody(state, inventory),
+    );
+  }
+  console.log(`Reconciled ${reconciled} pending built-in skill candidates`);
+}
+
+export function selectCanonicalPullRequest(
+  pullRequests: CandidatePullRequest[],
+): PullRequestReconciliation {
+  const merged = pullRequests
+    .filter((pullRequest) => pullRequest.state === "MERGED")
+    .sort(byPullRequestNumber);
+  if (merged.length > 1) {
+    throw new Error(
+      `Multiple merged watcher PRs found for one candidate: ${merged.map((pullRequest) => `#${pullRequest.number}`).join(", ")}`,
+    );
+  }
+  const open = pullRequests
+    .filter((pullRequest) => pullRequest.state === "OPEN")
+    .sort(byPullRequestNumber);
+  const canonical = merged[0] ?? open[0] ?? null;
+  return {
+    canonical,
+    duplicateOpen: canonical
+      ? open.filter((pullRequest) => pullRequest.number !== canonical.number)
+      : [],
+  };
+}
+
+function findCandidatePullRequests(
+  repo: string,
+  candidateId: string,
+): CandidatePullRequest[] {
+  const marker = `Builtin-skill-watch: ${candidateId}`;
+  return ghJson<CandidatePullRequest[]>([
+    "pr",
+    "list",
+    "--repo",
+    repo,
+    "--state",
+    "all",
+    "--search",
+    `${marker} in:body`,
+    "--limit",
+    "20",
+    "--json",
+    "number,state,isDraft,mergedAt,author,baseRefName,headRefOid,body,files,url",
+  ]).filter((pullRequest) => hasExactMarker(pullRequest.body, marker));
+}
+
+function assertAuthenticatedLogin(expectedGithubLogin: string): void {
+  const authenticated = ghJson<{ login: string }>(["api", "user"]).login;
+  if (authenticated !== expectedGithubLogin) {
+    throw new Error(
+      `Authenticated GitHub login ${authenticated} does not match ${expectedGithubLogin}`,
+    );
+  }
+}
+
+function closeDuplicatePullRequest(
+  repo: string,
+  duplicate: CandidatePullRequest,
+  canonical: CandidatePullRequest,
+): void {
+  runGh([
+    "pr",
+    "close",
+    String(duplicate.number),
+    "--repo",
+    repo,
+    "--comment",
+    `Closing as a duplicate of ${canonical.url}, which has the same exact built-in skill watcher candidate.`,
+  ]);
+}
+
+function reconciliationEvidence(
+  analysis: BuiltinSkillWatchAnalysis,
+  pullRequest: CandidatePullRequest,
+): ReviewEvidence {
+  const status = pullRequest.state === "MERGED" ? "merged" : "open draft";
+  return {
+    schema_version: 1,
+    candidate_id: analysis.candidate_id,
+    skill: analysis.skill,
+    sources: [
+      {
+        locator: pullRequest.url,
+        revision: pullRequest.headRefOid,
+        content_digest: null,
+        retrieved_at: new Date().toISOString(),
+        excerpt: `The ${status} watcher PR has the exact candidate marker and a validated skill-only diff.`,
+        claims: [
+          "the watcher already created and validated a PR for this exact candidate",
+        ],
+      },
+    ],
+    probes: [],
+  };
+}
+
+function renderEvidenceComment(
+  analysis: BuiltinSkillWatchAnalysis,
+  pullRequest: CandidatePullRequest,
+  evidence: ReviewEvidence,
+): string {
+  return [
+    `## Reconciled built-in skill audit: ${analysis.skill}`,
+    "",
+    `Candidate: \`${analysis.candidate_id}\``,
+    `PR: ${pullRequest.url}`,
+    `PR state: \`${pullRequest.state.toLowerCase()}\``,
+    "",
+    "```json",
+    JSON.stringify(evidence, null, 2),
+    "```",
+  ].join("\n");
+}
+
+function reconciliationNotes(pullRequest: CandidatePullRequest): string {
+  return pullRequest.state === "MERGED"
+    ? `reconciled merged watcher PR #${pullRequest.number}`
+    : `reconciled open draft watcher PR #${pullRequest.number}`;
+}
+
+function renderDuplicatePlan(duplicates: CandidatePullRequest[]): string {
+  return duplicates.length === 0
+    ? ""
+    : ` and close duplicate ${duplicates.map((pullRequest) => `#${pullRequest.number}`).join(", ")}`;
+}
+
+function previousAudit(
+  state: ReturnType<typeof parseTrackerState>,
+  skill: string,
+): BuiltinSkillWatchAnalysis["previous_audit"] {
+  const audit = state.skills[skill];
+  return audit
+    ? {
+        candidate_id: audit.candidate_id,
+        audited_sha: audit.audited_sha,
+        skill_digest: audit.skill_digest,
+        audited_at: audit.audited_at,
+      }
+    : null;
+}
+
+function workflowRunUrl(runId: string): string {
+  return `https://github.com/letta-ai/letta-code/actions/runs/${runId}`;
+}
+
+function hasExactMarker(body: string, marker: string): boolean {
+  return body.split(/\r?\n/).some((line) => line.trim() === marker);
+}
+
+function byPullRequestNumber(
+  left: CandidatePullRequest,
+  right: CandidatePullRequest,
+): number {
+  return left.number - right.number;
+}
+
+if (import.meta.main) main();

package/scripts/builtin-skills-watch/result-artifacts.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, test } from "bun:test";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  discoverReviewArtifacts,
+  formatFailureReceipt,
+} from "./result-artifacts.ts";
+
+const CANDIDATE = "creating-skills@aaaaaaaaaaaa-abcdef0123456789";
+
+describe("review artifact discovery", () => {
+  test("finds both nested and root-level result artifacts", () => {
+    const directory = mkdtempSync(join(tmpdir(), "skill-artifacts-"));
+    try {
+      const nested = join(directory, "builtin-skill-result-creating-skills");
+      mkdirSync(nested);
+      writeFileSync(
+        join(nested, "result.json"),
+        JSON.stringify({ candidate_id: CANDIDATE }),
+      );
+
+      const discovered = discoverReviewArtifacts(directory);
+      expect(discovered.results.get(CANDIDATE)).toBe(
+        join(nested, "result.json"),
+      );
+
+      rmSync(nested, { recursive: true });
+      writeFileSync(
+        join(directory, "result.json"),
+        JSON.stringify({ candidate_id: CANDIDATE }),
+      );
+      expect(discoverReviewArtifacts(directory).results.get(CANDIDATE)).toBe(
+        join(directory, "result.json"),
+      );
+    } finally {
+      rmSync(directory, { recursive: true, force: true });
+    }
+  });
+
+  test("preserves a runner-authored failure receipt", () => {
+    const directory = mkdtempSync(join(tmpdir(), "skill-artifacts-"));
+    try {
+      writeFileSync(
+        join(directory, "failure.json"),
+        JSON.stringify({
+          schema_version: 1,
+          candidate_id: CANDIDATE,
+          skill: "creating-skills",
+          kind: "action_failed",
+          message: "Letta Code Action failed before returning a result",
+          conversation_id: "conv-123",
+        }),
+      );
+
+      const receipt =
+        discoverReviewArtifacts(directory).failures.get(CANDIDATE);
+      expect(receipt).toBeDefined();
+      expect(formatFailureReceipt(receipt!)).toContain("conversation conv-123");
+    } finally {
+      rmSync(directory, { recursive: true, force: true });
+    }
+  });
+
+  test("rejects duplicate results for one candidate", () => {
+    const directory = mkdtempSync(join(tmpdir(), "skill-artifacts-"));
+    try {
+      mkdirSync(join(directory, "first"));
+      mkdirSync(join(directory, "second"));
+      for (const subdirectory of ["first", "second"]) {
+        writeFileSync(
+          join(directory, subdirectory, "result.json"),
+          JSON.stringify({ candidate_id: CANDIDATE }),
+        );
+      }
+      expect(() => discoverReviewArtifacts(directory)).toThrow(
+        `duplicate review result for ${CANDIDATE}`,
+      );
+    } finally {
+      rmSync(directory, { recursive: true, force: true });
+    }
+  });
+});

package/scripts/builtin-skills-watch/result-artifacts.ts

@@ -0,0 +1,160 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { basename, join } from "node:path";
+
+export interface ReviewFailureReceipt {
+  schema_version: 1;
+  candidate_id: string;
+  skill: string;
+  kind:
+    | "action_failed"
+    | "execution_file_missing"
+    | "result_marker_missing"
+    | "result_decode_failed";
+  message: string;
+  conversation_id: string | null;
+}
+
+export interface ReviewArtifacts {
+  results: Map<string, string>;
+  failures: Map<string, ReviewFailureReceipt>;
+}
+
+export function discoverReviewArtifacts(root: string): ReviewArtifacts {
+  const artifacts: ReviewArtifacts = {
+    results: new Map(),
+    failures: new Map(),
+  };
+  for (const path of walkFiles(root)) {
+    const name = basename(path);
+    if (name === "result.json") {
+      const candidateId = readCandidateId(path, "review result");
+      addUnique(artifacts.results, candidateId, path, "review result");
+    } else if (name === "failure.json") {
+      const receipt = parseFailureReceipt(
+        JSON.parse(readFileSync(path, "utf8")) as unknown,
+      );
+      addUnique(
+        artifacts.failures,
+        receipt.candidate_id,
+        receipt,
+        "failure receipt",
+      );
+    }
+  }
+  return artifacts;
+}
+
+export function formatFailureReceipt(receipt: ReviewFailureReceipt): string {
+  const conversation = receipt.conversation_id
+    ? ` (conversation ${receipt.conversation_id})`
+    : "";
+  return `${receipt.kind}: ${receipt.message}${conversation}`;
+}
+
+function walkFiles(root: string): string[] {
+  const files: string[] = [];
+  let entries: ReturnType<typeof readdirSync>;
+  try {
+    entries = readdirSync(root, { withFileTypes: true });
+  } catch (error) {
+    if (isMissingPath(error)) return files;
+    throw error;
+  }
+  for (const entry of entries) {
+    const path = join(root, entry.name);
+    if (entry.isDirectory()) files.push(...walkFiles(path));
+    else if (entry.isFile()) files.push(path);
+  }
+  return files.sort();
+}
+
+function readCandidateId(path: string, description: string): string {
+  const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
+  if (!isRecord(value) || !isCandidateId(value.candidate_id)) {
+    throw new Error(`${description} ${path} has no valid candidate_id`);
+  }
+  return value.candidate_id;
+}
+
+function parseFailureReceipt(value: unknown): ReviewFailureReceipt {
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, [
+      "schema_version",
+      "candidate_id",
+      "skill",
+      "kind",
+      "message",
+      "conversation_id",
+    ]) ||
+    value.schema_version !== 1 ||
+    !isCandidateId(value.candidate_id) ||
+    !isSkillName(value.skill) ||
+    !isFailureKind(value.kind) ||
+    typeof value.message !== "string" ||
+    value.message.length === 0 ||
+    value.message.length > 500 ||
+    (value.conversation_id !== null && !isConversationId(value.conversation_id))
+  ) {
+    throw new Error("Review failure receipt is invalid");
+  }
+  return value as unknown as ReviewFailureReceipt;
+}
+
+function addUnique<T>(
+  values: Map<string, T>,
+  candidateId: string,
+  value: T,
+  description: string,
+): void {
+  if (values.has(candidateId)) {
+    throw new Error(`Found duplicate ${description} for ${candidateId}`);
+  }
+  values.set(candidateId, value);
+}
+
+function isFailureKind(value: unknown): value is ReviewFailureReceipt["kind"] {
+  return (
+    value === "action_failed" ||
+    value === "execution_file_missing" ||
+    value === "result_marker_missing" ||
+    value === "result_decode_failed"
+  );
+}
+
+function isCandidateId(value: unknown): value is string {
+  return (
+    typeof value === "string" &&
+    /^[a-z0-9]+(?:-[a-z0-9]+)*@[a-f0-9]{12}-[a-f0-9]{16}$/.test(value)
+  );
+}
+
+function isSkillName(value: unknown): value is string {
+  return typeof value === "string" && /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value);
+}
+
+function isConversationId(value: unknown): value is string {
+  return typeof value === "string" && /^conv-[a-zA-Z0-9-]+$/.test(value);
+}
+
+function isMissingPath(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    "code" in error &&
+    (error as NodeJS.ErrnoException).code === "ENOENT"
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function hasExactKeys(
+  value: Record<string, unknown>,
+  expected: string[],
+): boolean {
+  return (
+    JSON.stringify(Object.keys(value).sort()) ===
+    JSON.stringify([...expected].sort())
+  );
+}

package/scripts/builtin-skills-watch/update-tracker.test.ts

@@ -6,6 +6,7 @@ import {
   parseReviewResult,
   type TrackerIssueView,
   validatePullRequestView,
+  validateReconciledPullRequestView,
   validateTrackerIssueView,
 } from "./update-tracker.ts";
 
@@ -79,6 +80,21 @@ describe("watcher PR validation", () => {
       ),
     ).toThrow("outside the selected skill scope");
   });
+
+  test("accepts a merged exact-candidate PR only during reconciliation", () => {
+    const merged = validPullRequest({
+      isDraft: false,
+      mergedAt: "2026-08-27T00:00:00Z",
+      state: "MERGED",
+    });
+
+    expect(() =>
+      validateReconciledPullRequestView(merged, URL, analysis(), LOGIN),
+    ).not.toThrow();
+    expect(() =>
+      validatePullRequestView(merged, URL, analysis(), LOGIN),
+    ).toThrow("open and draft");
+  });
 });
 
 describe("watcher analysis validation", () => {
@@ -135,7 +151,7 @@ describe("watcher result validation", () => {
       evidence: evidence(current),
     };
     expect(() => parseReviewResult({ ...base, secret: "no" }, current)).toThrow(
-      "does not match",
+      "unknown or missing fields",
     );
     expect(() =>
       parseReviewResult(