@letta-ai/letta-code 0.30.32 → 0.31.1

package/scripts/builtin-skills-watch/aggregate-results.ts

@@ -0,0 +1,326 @@
+#!/usr/bin/env bun
+/** Validates a daily skill-review batch and updates the tracker once. */
+
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  type BuiltinSkillWatchAnalysis,
+  buildAnalysis,
+  DEFAULT_TARGET_REPO,
+} from "./analysis.ts";
+import { createIssueComment, editIssueBody } from "./github.ts";
+import {
+  hasTerminalCandidate,
+  parseTrackerState,
+  recordOutcome,
+  renderTrackerBody,
+} from "./tracker.ts";
+import {
+  assertAnalysisIdentity,
+  getOpenTrackerIssue,
+  type ReviewResult,
+  readAnalysis,
+  readReviewResult,
+  verifyPullRequest,
+} from "./update-tracker.ts";
+
+interface Args {
+  repo: string;
+  trackerIssue: number | null;
+  manifestFile: string | null;
+  analysisDir: string | null;
+  resultsDir: string | null;
+  expectedGithubLogin: string | null;
+  dryRun: boolean;
+}
+
+interface Manifest {
+  schema_version: 1;
+  tracker_issue: number;
+  inventory: string[];
+  candidates: Array<{ skill: string; candidate_id: string }>;
+}
+
+export interface ValidatedOutcome {
+  analysis: BuiltinSkillWatchAnalysis;
+  result: ReviewResult | null;
+}
+
+export interface EvidenceCommentBatch {
+  candidateIds: string[];
+  body: string;
+}
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    repo: DEFAULT_TARGET_REPO,
+    trackerIssue: null,
+    manifestFile: null,
+    analysisDir: null,
+    resultsDir: null,
+    expectedGithubLogin: null,
+    dryRun: false,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--repo") args.repo = argv[++index] ?? args.repo;
+    else if (arg === "--tracker-issue") {
+      args.trackerIssue = Number(argv[++index]);
+    } else if (arg === "--manifest-file") {
+      args.manifestFile = argv[++index] ?? null;
+    } else if (arg === "--analysis-dir") {
+      args.analysisDir = argv[++index] ?? null;
+    } else if (arg === "--results-dir") {
+      args.resultsDir = argv[++index] ?? null;
+    } else if (arg === "--expected-github-login") {
+      args.expectedGithubLogin = argv[++index] ?? null;
+    } else if (arg === "--dry-run") args.dryRun = true;
+    else throw new Error(`Unknown argument: ${arg}`);
+  }
+  if (!args.trackerIssue || Number.isNaN(args.trackerIssue)) {
+    throw new Error("--tracker-issue is required");
+  }
+  if (
+    !args.manifestFile ||
+    !args.analysisDir ||
+    !args.resultsDir ||
+    !args.expectedGithubLogin
+  ) {
+    throw new Error(
+      "--manifest-file, --analysis-dir, --results-dir, and --expected-github-login are required",
+    );
+  }
+  return args;
+}
+
+function main(): void {
+  const args = parseArgs(process.argv.slice(2));
+  const manifest = readManifest(args.manifestFile as string);
+  if (manifest.tracker_issue !== args.trackerIssue) {
+    throw new Error("Manifest tracker issue does not match the workflow input");
+  }
+  const original = getOpenTrackerIssue(args.repo, args.trackerIssue as number);
+  let state = parseTrackerState(original.body);
+  let failed = false;
+  const outcomes: ValidatedOutcome[] = [];
+
+  for (const candidate of manifest.candidates) {
+    const analysisPath = join(
+      args.analysisDir as string,
+      `${candidate.skill}.json`,
+    );
+    const received = readAnalysis(analysisPath);
+    if (
+      received.candidate_id !== candidate.candidate_id ||
+      received.skill !== candidate.skill
+    ) {
+      throw new Error(`Manifest entry for ${candidate.skill} is invalid`);
+    }
+    if (hasTerminalCandidate(state, received.candidate_id)) continue;
+    const pending = state.pending[received.skill];
+    if (!pending || pending.candidate_id !== received.candidate_id) {
+      throw new Error(`Candidate ${received.candidate_id} is not pending`);
+    }
+    const analysis = buildAnalysis({
+      skill: received.skill,
+      currentSha: pending.current_sha,
+      auditAt: pending.audit_at,
+      previousAudit: previousAudit(state, received.skill),
+    });
+    analysis.workflow_run_url = workflowRunUrl(pending.workflow_run_id);
+    assertAnalysisIdentity(received, analysis);
+
+    const resultPath = join(
+      args.resultsDir as string,
+      `builtin-skill-result-${received.skill}`,
+      "result.json",
+    );
+    try {
+      if (!existsSync(resultPath)) {
+        throw new Error("Amelia result artifact is missing");
+      }
+      const result = readReviewResult(resultPath, analysis);
+      if (result.pr_url) {
+        verifyPullRequest(
+          args.repo,
+          result.pr_url,
+          analysis,
+          args.expectedGithubLogin as string,
+        );
+      }
+      outcomes.push({ analysis, result });
+    } catch (error) {
+      failed = true;
+      console.error(`${received.skill}: ${errorMessage(error)}`);
+      outcomes.push({ analysis, result: null });
+    }
+  }
+
+  assertTrackerUnchanged(args, original.body);
+  const evidenceUrls = args.dryRun
+    ? dryRunEvidenceUrls(outcomes, args.trackerIssue as number)
+    : postEvidenceComments(args.repo, args.trackerIssue as number, outcomes);
+  for (const outcome of outcomes) {
+    if (outcome.result) {
+      state = recordOutcome(state, {
+        analysis: outcome.analysis,
+        outcome: outcome.result.outcome,
+        notes: outcome.result.notes,
+        prUrl: outcome.result.pr_url,
+        evidence: outcome.result.evidence,
+        evidenceUrl: evidenceUrls.get(outcome.analysis.candidate_id),
+      });
+    } else {
+      state = recordOutcome(state, {
+        analysis: outcome.analysis,
+        outcome: "error",
+        notes: "Amelia review failed before a valid result; retry this skill",
+      });
+    }
+  }
+
+  const nextBody = renderTrackerBody(state, manifest.inventory);
+  if (args.dryRun) console.log(nextBody);
+  else {
+    assertTrackerUnchanged(args, original.body);
+    editIssueBody(args.repo, args.trackerIssue as number, nextBody);
+  }
+  if (failed) process.exit(1);
+}
+
+function previousAudit(
+  state: ReturnType<typeof parseTrackerState>,
+  skill: string,
+): BuiltinSkillWatchAnalysis["previous_audit"] {
+  const audit = state.skills[skill];
+  return audit
+    ? {
+        candidate_id: audit.candidate_id,
+        audited_sha: audit.audited_sha,
+        skill_digest: audit.skill_digest,
+        audited_at: audit.audited_at,
+      }
+    : null;
+}
+
+function workflowRunUrl(runId: string): string {
+  return `https://github.com/letta-ai/letta-code/actions/runs/${runId}`;
+}
+
+function assertTrackerUnchanged(args: Args, expectedBody: string): void {
+  const latest = getOpenTrackerIssue(args.repo, args.trackerIssue as number);
+  if (latest.body !== expectedBody) {
+    throw new Error("Tracker body changed while results were being validated");
+  }
+}
+
+function postEvidenceComments(
+  repo: string,
+  issueNumber: number,
+  outcomes: ValidatedOutcome[],
+): Map<string, string> {
+  const urls = new Map<string, string>();
+  for (const batch of buildEvidenceCommentBatches(outcomes)) {
+    const url = createIssueComment(repo, issueNumber, batch.body);
+    for (const candidateId of batch.candidateIds) urls.set(candidateId, url);
+  }
+  return urls;
+}
+
+export function buildEvidenceCommentBatches(
+  outcomes: ValidatedOutcome[],
+): EvidenceCommentBatch[] {
+  const batches: EvidenceCommentBatch[] = [];
+  let blocks: Array<{ candidateId: string; text: string }> = [];
+  let bytes = 0;
+  const flush = (): void => {
+    if (blocks.length === 0) return;
+    const body = [
+      "## Built-in skill audit evidence",
+      "",
+      ...blocks.map((block) => block.text),
+    ].join("\n");
+    batches.push({
+      candidateIds: blocks.map((block) => block.candidateId),
+      body,
+    });
+    blocks = [];
+    bytes = 0;
+  };
+  for (const outcome of terminalOutcomes(outcomes)) {
+    const block = renderEvidenceBlock(outcome.analysis, outcome.result);
+    const blockBytes = Buffer.byteLength(block, "utf8");
+    if (bytes + blockBytes > 60_000) flush();
+    blocks.push({ candidateId: outcome.analysis.candidate_id, text: block });
+    bytes += blockBytes;
+  }
+  flush();
+  return batches;
+}
+
+function dryRunEvidenceUrls(
+  outcomes: ValidatedOutcome[],
+  issueNumber: number,
+): Map<string, string> {
+  return new Map(
+    terminalOutcomes(outcomes).map((outcome) => [
+      outcome.analysis.candidate_id,
+      `https://github.com/letta-ai/letta-code/issues/${issueNumber}#issuecomment-1`,
+    ]),
+  );
+}
+
+function terminalOutcomes(
+  outcomes: ValidatedOutcome[],
+): Array<ValidatedOutcome & { result: ReviewResult }> {
+  return outcomes.filter(
+    (outcome): outcome is ValidatedOutcome & { result: ReviewResult } =>
+      outcome.result !== null,
+  );
+}
+
+function renderEvidenceBlock(
+  analysis: BuiltinSkillWatchAnalysis,
+  result: ReviewResult,
+): string {
+  return [
+    `<details><summary>${analysis.skill}: ${result.outcome}</summary>`,
+    "",
+    `Candidate: \`${analysis.candidate_id}\``,
+    `Outcome: \`${result.outcome}\``,
+    `Notes: ${result.notes}`,
+    `PR: ${result.pr_url ?? "-"}`,
+    "",
+    "```json",
+    JSON.stringify(result.evidence, null, 2),
+    "```",
+    "",
+    "</details>",
+    "",
+  ].join("\n");
+}
+
+function readManifest(path: string): Manifest {
+  const value = JSON.parse(readFileSync(path, "utf8")) as Manifest;
+  if (
+    value.schema_version !== 1 ||
+    !Number.isInteger(value.tracker_issue) ||
+    !Array.isArray(value.inventory) ||
+    !Array.isArray(value.candidates) ||
+    value.candidates.some(
+      (candidate) =>
+        typeof candidate.skill !== "string" ||
+        typeof candidate.candidate_id !== "string" ||
+        !value.inventory.includes(candidate.skill),
+    )
+  ) {
+    throw new Error("Built-in skill watch manifest is invalid");
+  }
+  return value;
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+if (import.meta.main) main();

package/scripts/builtin-skills-watch/analysis.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, test } from "bun:test";
+import {
+  buildAnalysis,
+  collectSkillFilesAtCommit,
+  listBuiltinSkillsAtCommit,
+} from "./analysis.ts";
+
+const HEAD = gitHead();
+const AUDIT_AT = "2026-08-26T00:00:00.000Z";
+
+describe("bundled skill inventory", () => {
+  test("lists bundled skill directories from an exact commit", () => {
+    const skills = listBuiltinSkillsAtCommit(HEAD);
+
+    expect(skills).toContain("creating-skills");
+    expect(skills).toContain("syncing-memory-filesystem");
+    expect(skills).toEqual([...skills].sort());
+    expect(new Set(skills).size).toBe(skills.length);
+  });
+
+  test("includes every tracked file in the selected skill digest", () => {
+    const files = collectSkillFilesAtCommit(HEAD, "creating-skills");
+
+    expect(files).toContain("src/skills/builtin/creating-skills/SKILL.md");
+    expect(files.some((path) => path.includes("/scripts/"))).toBe(true);
+  });
+});
+
+describe("buildAnalysis", () => {
+  test("builds a reproducible exact-commit candidate", () => {
+    const first = buildAnalysis({
+      skill: "syncing-memory-filesystem",
+      currentSha: HEAD,
+      auditAt: AUDIT_AT,
+    });
+    const second = buildAnalysis({
+      skill: "syncing-memory-filesystem",
+      currentSha: HEAD,
+      auditAt: AUDIT_AT,
+    });
+
+    expect(second).toEqual(first);
+    expect(first.candidate_id).toStartWith(
+      `syncing-memory-filesystem@${HEAD.slice(0, 12)}-`,
+    );
+    expect(first.skill_digest).toMatch(/^[a-f0-9]{64}$/);
+    expect(first.skill_files).toContain(
+      "src/skills/builtin/syncing-memory-filesystem/SKILL.md",
+    );
+    expect(first.skill_inventory).toContain("syncing-memory-filesystem");
+  });
+
+  test("rejects an unknown skill", () => {
+    expect(() =>
+      buildAnalysis({
+        skill: "not-a-bundled-skill",
+        currentSha: HEAD,
+        auditAt: AUDIT_AT,
+      }),
+    ).toThrow("does not exist");
+  });
+});
+
+function gitHead(): string {
+  const result = Bun.spawnSync(["git", "rev-parse", "HEAD"], {
+    stdout: "pipe",
+  });
+  if (result.exitCode !== 0) throw new Error("Could not resolve HEAD");
+  return result.stdout.toString().trim();
+}

package/scripts/builtin-skills-watch/analysis.ts

@@ -0,0 +1,228 @@
+import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+
+export const DEFAULT_TARGET_REPO = "letta-ai/letta-code";
+export const BUILTIN_SKILLS_DIR = "src/skills/builtin";
+
+export interface PriorSkillAudit {
+  candidate_id: string;
+  audited_sha: string;
+  skill_digest: string;
+  audited_at: string;
+}
+
+export interface RepositoryChanges {
+  previous_sha: string | null;
+  changed_files: string[];
+  commits: string[];
+  history_available: boolean;
+  truncated: boolean;
+}
+
+export interface BuiltinSkillWatchAnalysis {
+  schema_version: 1;
+  candidate_id: string;
+  skill: string;
+  skill_path: string;
+  skill_files: string[];
+  skill_digest: string;
+  current_sha: string;
+  audit_at: string;
+  previous_audit: PriorSkillAudit | null;
+  repository_changes: RepositoryChanges;
+  skill_inventory: string[];
+  workflow_run_url: string;
+}
+
+export interface BuildAnalysisOptions {
+  skill: string;
+  currentSha: string;
+  auditAt: string;
+  previousAudit?: PriorSkillAudit | null;
+}
+
+export function listBuiltinSkillsAtCommit(commit: string): string[] {
+  const files = runGit([
+    "ls-tree",
+    "-r",
+    "--name-only",
+    commit,
+    "--",
+    BUILTIN_SKILLS_DIR,
+  ]);
+  return files
+    .split("\n")
+    .map(
+      (path) => path.match(/^src\/skills\/builtin\/([^/]+)\/SKILL\.md$/)?.[1],
+    )
+    .filter((skill): skill is string => skill !== undefined)
+    .sort();
+}
+
+export function collectSkillFilesAtCommit(
+  commit: string,
+  skill: string,
+): string[] {
+  assertSkillName(skill);
+  const prefix = `${BUILTIN_SKILLS_DIR}/${skill}`;
+  return runGit(["ls-tree", "-r", "--name-only", commit, "--", prefix])
+    .split("\n")
+    .filter(Boolean)
+    .sort();
+}
+
+export function buildAnalysis(
+  options: BuildAnalysisOptions,
+): BuiltinSkillWatchAnalysis {
+  const currentSha = resolveCommit(options.currentSha);
+  assertAuditAt(options.auditAt);
+  const inventory = listBuiltinSkillsAtCommit(currentSha);
+  if (!inventory.includes(options.skill)) {
+    throw new Error(
+      `Bundled skill ${options.skill} does not exist at ${currentSha}`,
+    );
+  }
+
+  const skillFiles = collectSkillFilesAtCommit(currentSha, options.skill);
+  const skillDigest = digestFiles(currentSha, skillFiles);
+  const runUrl = workflowRunUrl();
+  const candidateHash = createHash("sha256")
+    .update(options.skill)
+    .update("\0")
+    .update(currentSha)
+    .update("\0")
+    .update(skillDigest)
+    .update("\0")
+    .update(options.auditAt)
+    .digest("hex")
+    .slice(0, 16);
+  const candidateId = `${options.skill}@${currentSha.slice(0, 12)}-${candidateHash}`;
+
+  return {
+    schema_version: 1,
+    candidate_id: candidateId,
+    skill: options.skill,
+    skill_path: `${BUILTIN_SKILLS_DIR}/${options.skill}`,
+    skill_files: skillFiles,
+    skill_digest: skillDigest,
+    current_sha: currentSha,
+    audit_at: options.auditAt,
+    previous_audit: options.previousAudit ?? null,
+    repository_changes: collectRepositoryChanges(
+      options.previousAudit?.audited_sha ?? null,
+      currentSha,
+    ),
+    skill_inventory: inventory,
+    workflow_run_url: runUrl,
+  };
+}
+
+function digestFiles(commit: string, files: string[]): string {
+  const hash = createHash("sha256");
+  for (const path of files) {
+    hash.update(path);
+    hash.update("\0");
+    hash.update(runGitBuffer(["show", `${commit}:${path}`]));
+    hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+function collectRepositoryChanges(
+  previousSha: string | null,
+  currentSha: string,
+): RepositoryChanges {
+  if (!previousSha || previousSha === currentSha) {
+    return {
+      previous_sha: previousSha,
+      changed_files: [],
+      commits: [],
+      history_available: previousSha === currentSha,
+      truncated: false,
+    };
+  }
+
+  const ancestor = spawnSync(
+    "git",
+    ["merge-base", "--is-ancestor", previousSha, currentSha],
+    { encoding: "utf8" },
+  );
+  if (ancestor.status !== 0) {
+    return {
+      previous_sha: previousSha,
+      changed_files: [],
+      commits: [],
+      history_available: false,
+      truncated: false,
+    };
+  }
+
+  const allFiles = runGit([
+    "diff",
+    "--name-only",
+    `${previousSha}..${currentSha}`,
+  ])
+    .split("\n")
+    .filter(Boolean);
+  const allCommits = runGit([
+    "log",
+    "--format=%H %s",
+    `${previousSha}..${currentSha}`,
+  ])
+    .split("\n")
+    .filter(Boolean);
+  return {
+    previous_sha: previousSha,
+    changed_files: allFiles.slice(0, 500),
+    commits: allCommits.slice(0, 100),
+    history_available: true,
+    truncated: allFiles.length > 500 || allCommits.length > 100,
+  };
+}
+
+function runGit(args: string[], trim = true): string {
+  const result = spawnSync("git", args, {
+    encoding: "utf8",
+    maxBuffer: 50 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    throw new Error(`git ${args.join(" ")} failed:\n${result.stderr}`);
+  }
+  return trim ? result.stdout.trim() : result.stdout;
+}
+
+function runGitBuffer(args: string[]): Buffer {
+  const result = spawnSync("git", args, {
+    maxBuffer: 50 * 1024 * 1024,
+  });
+  if (result.status !== 0) {
+    throw new Error(`git ${args.join(" ")} failed:\n${result.stderr}`);
+  }
+  return result.stdout;
+}
+
+function resolveCommit(commit: string): string {
+  return runGit(["rev-parse", `${commit}^{commit}`]);
+}
+
+function assertSkillName(skill: string): void {
+  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(skill)) {
+    throw new Error(`Invalid bundled skill name: ${skill}`);
+  }
+}
+
+function assertAuditAt(auditAt: string): void {
+  const parsed = new Date(auditAt);
+  if (Number.isNaN(parsed.getTime()) || parsed.toISOString() !== auditAt) {
+    throw new Error(`Invalid audit timestamp: ${auditAt}`);
+  }
+}
+
+function workflowRunUrl(): string {
+  const server = process.env.GITHUB_SERVER_URL;
+  const repository = process.env.GITHUB_REPOSITORY;
+  const runId = process.env.GITHUB_RUN_ID;
+  return server && repository && runId
+    ? `${server}/${repository}/actions/runs/${runId}`
+    : "";
+}