@letta-ai/letta-code 0.31.6 → 0.31.7

package/scripts/builtin-skills-watch/evidence.ts

@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
 const MAX_SOURCES = 5;
 const MAX_PROBES = 5;
 const MAX_CLAIMS_PER_SOURCE = 5;
-const MAX_SERIALIZED_BYTES = 650;
+const MAX_SERIALIZED_BYTES = 16 * 1024;
 
 export interface EvidenceSource {
   locator: string;
@@ -51,29 +51,24 @@ export function parseReviewEvidence(value: unknown): ReviewEvidence {
   if (
     !Array.isArray(value.sources) ||
     value.sources.length === 0 ||
-    value.sources.length > MAX_SOURCES ||
-    !value.sources.every(isEvidenceSource)
+    value.sources.length > MAX_SOURCES
   ) {
-    throw new TypeError("review evidence sources are invalid");
+    throw new TypeError("review evidence must include between 1 and 5 sources");
   }
-  if (
-    !Array.isArray(value.probes) ||
-    value.probes.length > MAX_PROBES ||
-    !value.probes.every(isEvidenceProbe)
-  ) {
-    throw new TypeError("review evidence probes are invalid");
+  if (!Array.isArray(value.probes) || value.probes.length > MAX_PROBES) {
+    throw new TypeError("review evidence must include at most 5 probes");
   }
   const evidence: ReviewEvidence = {
     schema_version: 1,
     candidate_id: value.candidate_id,
     skill: value.skill,
-    sources: value.sources,
-    probes: value.probes,
+    sources: value.sources.map(parseEvidenceSource),
+    probes: value.probes.map(parseEvidenceProbe),
   };
   if (
     Buffer.byteLength(JSON.stringify(evidence), "utf8") > MAX_SERIALIZED_BYTES
   ) {
-    throw new TypeError("review evidence exceeds 650 bytes");
+    throw new TypeError("review evidence exceeds 16384 bytes");
   }
   return evidence;
 }
@@ -82,44 +77,77 @@ export function digestReviewEvidence(evidence: ReviewEvidence): string {
   return createHash("sha256").update(JSON.stringify(evidence)).digest("hex");
 }
 
-function isEvidenceSource(value: unknown): value is EvidenceSource {
-  return (
-    isRecord(value) &&
-    hasExactKeys(value, [
+function parseEvidenceSource(value: unknown, index: number): EvidenceSource {
+  const description = `review evidence source ${index + 1}`;
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, [
       "locator",
       "revision",
       "content_digest",
       "retrieved_at",
       "excerpt",
       "claims",
-    ]) &&
-    isBoundedString(value.locator, 500) &&
-    (value.revision === null || isBoundedString(value.revision, 300)) &&
-    (value.content_digest === null || isDigest(value.content_digest)) &&
-    (value.revision !== null || value.content_digest !== null) &&
-    isIsoTimestamp(value.retrieved_at) &&
-    isBoundedString(value.excerpt, 300) &&
-    Array.isArray(value.claims) &&
-    value.claims.length > 0 &&
-    value.claims.length <= MAX_CLAIMS_PER_SOURCE &&
-    value.claims.every((claim) => isBoundedString(claim, 500))
-  );
+    ])
+  ) {
+    throw new TypeError(`${description} has unknown or missing fields`);
+  }
+  if (!isBoundedString(value.locator, 500)) {
+    throw new TypeError(`${description} locator is invalid`);
+  }
+  if (value.revision !== null && !isBoundedString(value.revision, 300)) {
+    throw new TypeError(`${description} revision is invalid`);
+  }
+  if (value.content_digest !== null && !isDigest(value.content_digest)) {
+    throw new TypeError(`${description} content_digest is invalid`);
+  }
+  if (value.revision === null && value.content_digest === null) {
+    throw new TypeError(`${description} requires a revision or content_digest`);
+  }
+  if (!isIsoTimestamp(value.retrieved_at)) {
+    throw new TypeError(`${description} retrieved_at is not an ISO timestamp`);
+  }
+  if (!isBoundedString(value.excerpt, 300)) {
+    throw new TypeError(`${description} excerpt is invalid`);
+  }
+  if (
+    !Array.isArray(value.claims) ||
+    value.claims.length === 0 ||
+    value.claims.length > MAX_CLAIMS_PER_SOURCE ||
+    !value.claims.every((claim) => isBoundedString(claim, 500))
+  ) {
+    throw new TypeError(`${description} claims are invalid`);
+  }
+  return value as unknown as EvidenceSource;
 }
 
-function isEvidenceProbe(value: unknown): value is EvidenceProbe {
-  return (
-    isRecord(value) &&
-    hasExactKeys(value, ["command", "result_digest", "summary"]) &&
-    isBoundedString(value.command, 500) &&
-    isDigest(value.result_digest) &&
-    isBoundedString(value.summary, 500)
-  );
+function parseEvidenceProbe(value: unknown, index: number): EvidenceProbe {
+  const description = `review evidence probe ${index + 1}`;
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, ["command", "result_digest", "summary"])
+  ) {
+    throw new TypeError(`${description} has unknown or missing fields`);
+  }
+  if (!isBoundedString(value.command, 500)) {
+    throw new TypeError(`${description} command is invalid`);
+  }
+  if (!isDigest(value.result_digest)) {
+    throw new TypeError(`${description} result_digest is invalid`);
+  }
+  if (!isBoundedString(value.summary, 500)) {
+    throw new TypeError(`${description} summary is invalid`);
+  }
+  return value as unknown as EvidenceProbe;
 }
 
 function isIsoTimestamp(value: unknown): value is string {
   if (typeof value !== "string") return false;
   const parsed = new Date(value);
-  return !Number.isNaN(parsed.getTime()) && parsed.toISOString() === value;
+  return (
+    !Number.isNaN(parsed.getTime()) &&
+    /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/.test(value)
+  );
 }
 
 function isDigest(value: unknown): value is string {

package/scripts/builtin-skills-watch/finalize-result.test.ts

@@ -0,0 +1,125 @@
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { BuiltinSkillWatchAnalysis } from "./analysis.ts";
+
+describe("review result finalization", () => {
+  test("validates with the runner parser before encoding the result", () => {
+    const directory = mkdtempSync(join(tmpdir(), "skill-result-finalizer-"));
+    try {
+      const analysis = fakeAnalysis();
+      const analysisFile = join(directory, "analysis.json");
+      const resultFile = join(directory, "result.json");
+      writeFileSync(analysisFile, JSON.stringify(analysis));
+      writeFileSync(resultFile, JSON.stringify(fakeResult(analysis)));
+
+      const finalized = Bun.spawnSync([
+        "bun",
+        "scripts/builtin-skills-watch/finalize-result.ts",
+        "--analysis-file",
+        analysisFile,
+        "--result-file",
+        resultFile,
+      ]);
+
+      expect(finalized.exitCode).toBe(0);
+      const line = finalized.stdout.toString().trim();
+      expect(line).toStartWith("SKILL_WATCH_RESULT ");
+      expect(
+        JSON.parse(
+          Buffer.from(
+            line.slice("SKILL_WATCH_RESULT ".length),
+            "base64",
+          ).toString("utf8"),
+        ),
+      ).toEqual(fakeResult(analysis));
+    } finally {
+      rmSync(directory, { recursive: true, force: true });
+    }
+  });
+
+  test("rejects a result for another candidate", () => {
+    const directory = mkdtempSync(join(tmpdir(), "skill-result-finalizer-"));
+    try {
+      const analysis = fakeAnalysis();
+      const analysisFile = join(directory, "analysis.json");
+      const resultFile = join(directory, "result.json");
+      writeFileSync(analysisFile, JSON.stringify(analysis));
+      writeFileSync(
+        resultFile,
+        JSON.stringify({
+          ...fakeResult(analysis),
+          candidate_id: "creating-skills@bbbbbbbbbbbb-abcdef0123456789",
+        }),
+      );
+
+      const finalized = Bun.spawnSync([
+        "bun",
+        "scripts/builtin-skills-watch/finalize-result.ts",
+        "--analysis-file",
+        analysisFile,
+        "--result-file",
+        resultFile,
+      ]);
+
+      expect(finalized.exitCode).not.toBe(0);
+      expect(finalized.stderr.toString()).toContain(
+        "does not match the pending candidate",
+      );
+    } finally {
+      rmSync(directory, { recursive: true, force: true });
+    }
+  });
+});
+
+function fakeAnalysis(): BuiltinSkillWatchAnalysis {
+  const skill = "creating-skills";
+  return {
+    schema_version: 1,
+    candidate_id: `${skill}@${"a".repeat(12)}-abcdef0123456789`,
+    skill,
+    skill_path: `src/skills/builtin/${skill}`,
+    skill_files: [`src/skills/builtin/${skill}/SKILL.md`],
+    skill_digest: "b".repeat(64),
+    current_sha: "a".repeat(40),
+    audit_at: "2026-08-26T00:00:00.000Z",
+    previous_audit: null,
+    repository_changes: {
+      previous_sha: null,
+      changed_files: [],
+      commits: [],
+      history_available: false,
+      truncated: false,
+    },
+    skill_inventory: [skill],
+    workflow_run_url: "https://github.com/letta-ai/letta-code/actions/runs/1",
+  };
+}
+
+function fakeResult(analysis: BuiltinSkillWatchAnalysis) {
+  return {
+    schema_version: 1,
+    candidate_id: analysis.candidate_id,
+    skill: analysis.skill,
+    outcome: "no_drift",
+    notes: "current source and skill agree",
+    pr_url: null,
+    evidence: {
+      schema_version: 1,
+      candidate_id: analysis.candidate_id,
+      skill: analysis.skill,
+      sources: [
+        {
+          locator: analysis.skill_path,
+          revision: analysis.current_sha,
+          content_digest: analysis.skill_digest,
+          retrieved_at: analysis.audit_at,
+          excerpt: "the current skill matches its owning source",
+          claims: ["checked current source"],
+        },
+      ],
+      probes: [],
+    },
+  } as const;
+}

package/scripts/builtin-skills-watch/finalize-result.ts

@@ -0,0 +1,44 @@
+#!/usr/bin/env bun
+/** Validates and encodes one agent-authored review result for the runner. */
+
+import { readAnalysis, readReviewResult } from "./update-tracker.ts";
+
+interface Args {
+  analysisFile: string | null;
+  resultFile: string | null;
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = { analysisFile: null, resultFile: null };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--analysis-file") {
+      args.analysisFile = argv[++index] ?? null;
+    } else if (arg === "--result-file") {
+      args.resultFile = argv[++index] ?? null;
+    } else if (arg === "--help" || arg === "-h") {
+      console.log(
+        "Usage: bun scripts/builtin-skills-watch/finalize-result.ts --analysis-file FILE --result-file FILE",
+      );
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (!args.analysisFile || !args.resultFile) {
+    throw new Error("--analysis-file and --result-file are required");
+  }
+  return args;
+}
+
+function main(): void {
+  const args = parseArgs(process.argv.slice(2));
+  const analysis = readAnalysis(args.analysisFile as string);
+  const result = readReviewResult(args.resultFile as string, analysis);
+  const encoded = Buffer.from(JSON.stringify(result), "utf8").toString(
+    "base64",
+  );
+  console.log(`SKILL_WATCH_RESULT ${encoded}`);
+}
+
+if (import.meta.main) main();

package/scripts/builtin-skills-watch/reconcile-results.test.ts

@@ -0,0 +1,58 @@
+import { describe, expect, test } from "bun:test";
+import {
+  type CandidatePullRequest,
+  selectCanonicalPullRequest,
+} from "./reconcile-results.ts";
+
+describe("pending watcher PR reconciliation", () => {
+  test("prefers a merged PR and closes every open duplicate", () => {
+    const merged = pullRequest(4067, "MERGED");
+    const firstDuplicate = pullRequest(4077, "OPEN");
+    const secondDuplicate = pullRequest(4099, "OPEN");
+
+    expect(
+      selectCanonicalPullRequest([firstDuplicate, secondDuplicate, merged]),
+    ).toEqual({
+      canonical: merged,
+      duplicateOpen: [firstDuplicate, secondDuplicate],
+    });
+  });
+
+  test("uses the oldest open PR when no candidate was merged", () => {
+    const older = pullRequest(4078, "OPEN");
+    const newer = pullRequest(4098, "OPEN");
+
+    expect(selectCanonicalPullRequest([newer, older])).toEqual({
+      canonical: older,
+      duplicateOpen: [newer],
+    });
+  });
+
+  test("ignores closed unmerged PRs and rejects multiple merged PRs", () => {
+    expect(selectCanonicalPullRequest([pullRequest(1, "CLOSED")])).toEqual({
+      canonical: null,
+      duplicateOpen: [],
+    });
+    expect(() =>
+      selectCanonicalPullRequest([
+        pullRequest(1, "MERGED"),
+        pullRequest(2, "MERGED"),
+      ]),
+    ).toThrow("Multiple merged watcher PRs");
+  });
+});
+
+function pullRequest(number: number, state: string): CandidatePullRequest {
+  return {
+    number,
+    author: { login: "amelia-letta" },
+    baseRefName: "main",
+    body: "Builtin-skill-watch: creating-skills@aaaaaaaaaaaa-0000000000000001",
+    files: [{ path: "src/skills/builtin/creating-skills/SKILL.md" }],
+    headRefOid: number.toString(16).padStart(40, "0"),
+    isDraft: state === "OPEN",
+    mergedAt: state === "MERGED" ? "2026-08-27T00:00:00Z" : null,
+    state,
+    url: `https://github.com/letta-ai/letta-code/pull/${number}`,
+  };
+}

package/scripts/builtin-skills-watch/reconcile-results.ts

@@ -0,0 +1,311 @@
+#!/usr/bin/env bun
+/** Reconciles pending tracker candidates against PR side effects before retries. */
+
+import {
+  type BuiltinSkillWatchAnalysis,
+  buildAnalysis,
+  DEFAULT_TARGET_REPO,
+  listBuiltinSkillsAtCommit,
+} from "./analysis.ts";
+import type { ReviewEvidence } from "./evidence.ts";
+import { createIssueComment, editIssueBody, ghJson, runGh } from "./github.ts";
+import {
+  parseTrackerState,
+  recordOutcome,
+  renderTrackerBody,
+} from "./tracker.ts";
+import {
+  getOpenTrackerIssue,
+  type PullRequestView,
+  verifyReconciledPullRequest,
+} from "./update-tracker.ts";
+
+interface Args {
+  repo: string;
+  trackerIssue: number | null;
+  expectedGithubLogin: string | null;
+  dryRun: boolean;
+}
+
+export interface CandidatePullRequest extends PullRequestView {
+  mergedAt: string | null;
+  number: number;
+}
+
+export interface PullRequestReconciliation {
+  canonical: CandidatePullRequest | null;
+  duplicateOpen: CandidatePullRequest[];
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    repo: DEFAULT_TARGET_REPO,
+    trackerIssue: null,
+    expectedGithubLogin: null,
+    dryRun: false,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--repo") args.repo = argv[++index] ?? args.repo;
+    else if (arg === "--tracker-issue") {
+      args.trackerIssue = Number(argv[++index]);
+    } else if (arg === "--expected-github-login") {
+      args.expectedGithubLogin = argv[++index] ?? null;
+    } else if (arg === "--dry-run") args.dryRun = true;
+    else if (arg === "--help" || arg === "-h") {
+      console.log(
+        "Usage: bun scripts/builtin-skills-watch/reconcile-results.ts --tracker-issue ISSUE --expected-github-login LOGIN [--repo OWNER/REPO] [--dry-run]",
+      );
+      process.exit(0);
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  if (!args.trackerIssue || Number.isNaN(args.trackerIssue)) {
+    throw new Error("--tracker-issue is required");
+  }
+  if (!args.expectedGithubLogin) {
+    throw new Error("--expected-github-login is required");
+  }
+  return args;
+}
+
+function main(): void {
+  const args = parseArgs(process.argv.slice(2));
+  assertAuthenticatedLogin(args.expectedGithubLogin as string);
+  const tracker = getOpenTrackerIssue(args.repo, args.trackerIssue as number);
+  let state = parseTrackerState(tracker.body);
+  const inventory = listBuiltinSkillsAtCommit("HEAD");
+  let reconciled = 0;
+
+  for (const skill of Object.keys(state.pending).sort()) {
+    const pending = state.pending[skill];
+    if (!pending) continue;
+    const analysis = buildAnalysis({
+      skill,
+      currentSha: pending.current_sha,
+      auditAt: pending.audit_at,
+      previousAudit: previousAudit(state, skill),
+    });
+    analysis.workflow_run_url = workflowRunUrl(pending.workflow_run_id);
+    const pullRequests = findCandidatePullRequests(
+      args.repo,
+      analysis.candidate_id,
+    );
+    const selected = selectCanonicalPullRequest(pullRequests);
+    if (!selected.canonical) continue;
+
+    verifyReconciledPullRequest(
+      args.repo,
+      selected.canonical.url,
+      analysis,
+      args.expectedGithubLogin as string,
+    );
+    for (const duplicate of selected.duplicateOpen) {
+      verifyReconciledPullRequest(
+        args.repo,
+        duplicate.url,
+        analysis,
+        args.expectedGithubLogin as string,
+      );
+    }
+
+    if (args.dryRun) {
+      console.log(
+        `${analysis.skill}: would reconcile ${selected.canonical.url}${renderDuplicatePlan(selected.duplicateOpen)}`,
+      );
+      continue;
+    }
+
+    for (const duplicate of selected.duplicateOpen) {
+      closeDuplicatePullRequest(args.repo, duplicate, selected.canonical);
+    }
+    const evidence = reconciliationEvidence(analysis, selected.canonical);
+    const evidenceUrl = createIssueComment(
+      args.repo,
+      args.trackerIssue as number,
+      renderEvidenceComment(analysis, selected.canonical, evidence),
+    );
+    state = recordOutcome(state, {
+      analysis,
+      outcome: "pr_created",
+      notes: reconciliationNotes(selected.canonical),
+      prUrl: selected.canonical.url,
+      evidence,
+      evidenceUrl,
+    });
+    reconciled += 1;
+  }
+
+  if (args.dryRun) return;
+  const latest = getOpenTrackerIssue(args.repo, args.trackerIssue as number);
+  if (latest.body !== tracker.body) {
+    throw new Error(
+      "Tracker body changed while PR side effects were reconciled",
+    );
+  }
+  if (reconciled > 0) {
+    editIssueBody(
+      args.repo,
+      args.trackerIssue as number,
+      renderTrackerBody(state, inventory),
+    );
+  }
+  console.log(`Reconciled ${reconciled} pending built-in skill candidates`);
+}
+
+export function selectCanonicalPullRequest(
+  pullRequests: CandidatePullRequest[],
+): PullRequestReconciliation {
+  const merged = pullRequests
+    .filter((pullRequest) => pullRequest.state === "MERGED")
+    .sort(byPullRequestNumber);
+  if (merged.length > 1) {
+    throw new Error(
+      `Multiple merged watcher PRs found for one candidate: ${merged.map((pullRequest) => `#${pullRequest.number}`).join(", ")}`,
+    );
+  }
+  const open = pullRequests
+    .filter((pullRequest) => pullRequest.state === "OPEN")
+    .sort(byPullRequestNumber);
+  const canonical = merged[0] ?? open[0] ?? null;
+  return {
+    canonical,
+    duplicateOpen: canonical
+      ? open.filter((pullRequest) => pullRequest.number !== canonical.number)
+      : [],
+  };
+}
+
+function findCandidatePullRequests(
+  repo: string,
+  candidateId: string,
+): CandidatePullRequest[] {
+  const marker = `Builtin-skill-watch: ${candidateId}`;
+  return ghJson<CandidatePullRequest[]>([
+    "pr",
+    "list",
+    "--repo",
+    repo,
+    "--state",
+    "all",
+    "--search",
+    `${marker} in:body`,
+    "--limit",
+    "20",
+    "--json",
+    "number,state,isDraft,mergedAt,author,baseRefName,headRefOid,body,files,url",
+  ]).filter((pullRequest) => hasExactMarker(pullRequest.body, marker));
+}
+
+function assertAuthenticatedLogin(expectedGithubLogin: string): void {
+  const authenticated = ghJson<{ login: string }>(["api", "user"]).login;
+  if (authenticated !== expectedGithubLogin) {
+    throw new Error(
+      `Authenticated GitHub login ${authenticated} does not match ${expectedGithubLogin}`,
+    );
+  }
+}
+
+function closeDuplicatePullRequest(
+  repo: string,
+  duplicate: CandidatePullRequest,
+  canonical: CandidatePullRequest,
+): void {
+  runGh([
+    "pr",
+    "close",
+    String(duplicate.number),
+    "--repo",
+    repo,
+    "--comment",
+    `Closing as a duplicate of ${canonical.url}, which has the same exact built-in skill watcher candidate.`,
+  ]);
+}
+
+function reconciliationEvidence(
+  analysis: BuiltinSkillWatchAnalysis,
+  pullRequest: CandidatePullRequest,
+): ReviewEvidence {
+  const status = pullRequest.state === "MERGED" ? "merged" : "open draft";
+  return {
+    schema_version: 1,
+    candidate_id: analysis.candidate_id,
+    skill: analysis.skill,
+    sources: [
+      {
+        locator: pullRequest.url,
+        revision: pullRequest.headRefOid,
+        content_digest: null,
+        retrieved_at: new Date().toISOString(),
+        excerpt: `The ${status} watcher PR has the exact candidate marker and a validated skill-only diff.`,
+        claims: [
+          "the watcher already created and validated a PR for this exact candidate",
+        ],
+      },
+    ],
+    probes: [],
+  };
+}
+
+function renderEvidenceComment(
+  analysis: BuiltinSkillWatchAnalysis,
+  pullRequest: CandidatePullRequest,
+  evidence: ReviewEvidence,
+): string {
+  return [
+    `## Reconciled built-in skill audit: ${analysis.skill}`,
+    "",
+    `Candidate: \`${analysis.candidate_id}\``,
+    `PR: ${pullRequest.url}`,
+    `PR state: \`${pullRequest.state.toLowerCase()}\``,
+    "",
+    "```json",
+    JSON.stringify(evidence, null, 2),
+    "```",
+  ].join("\n");
+}
+
+function reconciliationNotes(pullRequest: CandidatePullRequest): string {
+  return pullRequest.state === "MERGED"
+    ? `reconciled merged watcher PR #${pullRequest.number}`
+    : `reconciled open draft watcher PR #${pullRequest.number}`;
+}
+
+function renderDuplicatePlan(duplicates: CandidatePullRequest[]): string {
+  return duplicates.length === 0
+    ? ""
+    : ` and close duplicate ${duplicates.map((pullRequest) => `#${pullRequest.number}`).join(", ")}`;
+}
+
+function previousAudit(
+  state: ReturnType<typeof parseTrackerState>,
+  skill: string,
+): BuiltinSkillWatchAnalysis["previous_audit"] {
+  const audit = state.skills[skill];
+  return audit
+    ? {
+        candidate_id: audit.candidate_id,
+        audited_sha: audit.audited_sha,
+        skill_digest: audit.skill_digest,
+        audited_at: audit.audited_at,
+      }
+    : null;
+}
+
+function workflowRunUrl(runId: string): string {
+  return `https://github.com/letta-ai/letta-code/actions/runs/${runId}`;
+}
+
+function hasExactMarker(body: string, marker: string): boolean {
+  return body.split(/\r?\n/).some((line) => line.trim() === marker);
+}
+
+function byPullRequestNumber(
+  left: CandidatePullRequest,
+  right: CandidatePullRequest,
+): number {
+  return left.number - right.number;
+}
+
+if (import.meta.main) main();