@letta-ai/letta-code 0.30.32 → 0.31.0

package/scripts/builtin-skills-watch/evidence.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, test } from "bun:test";
+import { digestReviewEvidence, parseReviewEvidence } from "./evidence.ts";
+
+describe("review evidence", () => {
+  test("accepts bounded source and probe evidence", () => {
+    const evidence = parseReviewEvidence({
+      schema_version: 1,
+      candidate_id: "creating-skills@abc-candidate",
+      skill: "creating-skills",
+      sources: [
+        {
+          locator: "src/skills/builtin/creating-skills/SKILL.md",
+          revision: "a".repeat(40),
+          content_digest: "b".repeat(64),
+          retrieved_at: "2026-08-26T00:00:00.000Z",
+          excerpt: "name must match the skill directory",
+          claims: ["frontmatter rules match the validator"],
+        },
+      ],
+      probes: [
+        {
+          command: "bun test src/skills/builtin/creating-skills",
+          result_digest: "c".repeat(64),
+          summary: "validator tests passed",
+        },
+      ],
+    });
+
+    expect(digestReviewEvidence(evidence)).toMatch(/^[a-f0-9]{64}$/);
+  });
+
+  test("requires at least one source", () => {
+    expect(() =>
+      parseReviewEvidence({
+        schema_version: 1,
+        candidate_id: "candidate",
+        skill: "creating-skills",
+        sources: [],
+        probes: [],
+      }),
+    ).toThrow("sources are invalid");
+  });
+
+  test("rejects malformed digests and timestamps", () => {
+    expect(() =>
+      parseReviewEvidence({
+        schema_version: 1,
+        candidate_id: "candidate",
+        skill: "creating-skills",
+        sources: [
+          {
+            locator: "https://docs.letta.com/",
+            revision: null,
+            content_digest: "not-a-digest",
+            retrieved_at: "yesterday",
+            excerpt: "current docs text",
+            claims: ["checked docs"],
+          },
+        ],
+        probes: [],
+      }),
+    ).toThrow("sources are invalid");
+  });
+
+  test("rejects evidence too large for the tracker", () => {
+    expect(() =>
+      parseReviewEvidence({
+        schema_version: 1,
+        candidate_id: "candidate",
+        skill: "creating-skills",
+        sources: [
+          {
+            locator: "x".repeat(500),
+            revision: "y".repeat(300),
+            content_digest: null,
+            retrieved_at: "2026-08-26T00:00:00.000Z",
+            excerpt: "q".repeat(300),
+            claims: ["z".repeat(500)],
+          },
+        ],
+        probes: [],
+      }),
+    ).toThrow("exceeds 650 bytes");
+  });
+
+  test("requires a revision or digest and rejects unknown fields", () => {
+    const source = {
+      locator: "https://docs.letta.com/",
+      revision: null,
+      content_digest: null,
+      retrieved_at: "2026-08-26T00:00:00.000Z",
+      excerpt: "current docs text",
+      claims: ["checked docs"],
+    };
+    expect(() =>
+      parseReviewEvidence({
+        schema_version: 1,
+        candidate_id: "candidate",
+        skill: "creating-skills",
+        sources: [source],
+        probes: [],
+      }),
+    ).toThrow("sources are invalid");
+    expect(() =>
+      parseReviewEvidence({
+        schema_version: 1,
+        candidate_id: "candidate",
+        skill: "creating-skills",
+        sources: [{ ...source, content_digest: "a".repeat(64), secret: "no" }],
+        probes: [],
+      }),
+    ).toThrow("sources are invalid");
+  });
+});

package/scripts/builtin-skills-watch/evidence.ts

@@ -0,0 +1,145 @@
+import { createHash } from "node:crypto";
+
+const MAX_SOURCES = 5;
+const MAX_PROBES = 5;
+const MAX_CLAIMS_PER_SOURCE = 5;
+const MAX_SERIALIZED_BYTES = 650;
+
+export interface EvidenceSource {
+  locator: string;
+  revision: string | null;
+  content_digest: string | null;
+  retrieved_at: string;
+  excerpt: string;
+  claims: string[];
+}
+
+export interface EvidenceProbe {
+  command: string;
+  result_digest: string;
+  summary: string;
+}
+
+export interface ReviewEvidence {
+  schema_version: 1;
+  candidate_id: string;
+  skill: string;
+  sources: EvidenceSource[];
+  probes: EvidenceProbe[];
+}
+
+export function parseReviewEvidence(value: unknown): ReviewEvidence {
+  if (
+    !isRecord(value) ||
+    !hasExactKeys(value, [
+      "schema_version",
+      "candidate_id",
+      "skill",
+      "sources",
+      "probes",
+    ]) ||
+    value.schema_version !== 1
+  ) {
+    throw new TypeError("review evidence must use schema version 1");
+  }
+  if (!isBoundedString(value.candidate_id, 300)) {
+    throw new TypeError("review evidence candidate_id is invalid");
+  }
+  if (!isBoundedString(value.skill, 100)) {
+    throw new TypeError("review evidence skill is invalid");
+  }
+  if (
+    !Array.isArray(value.sources) ||
+    value.sources.length === 0 ||
+    value.sources.length > MAX_SOURCES ||
+    !value.sources.every(isEvidenceSource)
+  ) {
+    throw new TypeError("review evidence sources are invalid");
+  }
+  if (
+    !Array.isArray(value.probes) ||
+    value.probes.length > MAX_PROBES ||
+    !value.probes.every(isEvidenceProbe)
+  ) {
+    throw new TypeError("review evidence probes are invalid");
+  }
+  const evidence: ReviewEvidence = {
+    schema_version: 1,
+    candidate_id: value.candidate_id,
+    skill: value.skill,
+    sources: value.sources,
+    probes: value.probes,
+  };
+  if (
+    Buffer.byteLength(JSON.stringify(evidence), "utf8") > MAX_SERIALIZED_BYTES
+  ) {
+    throw new TypeError("review evidence exceeds 650 bytes");
+  }
+  return evidence;
+}
+
+export function digestReviewEvidence(evidence: ReviewEvidence): string {
+  return createHash("sha256").update(JSON.stringify(evidence)).digest("hex");
+}
+
+function isEvidenceSource(value: unknown): value is EvidenceSource {
+  return (
+    isRecord(value) &&
+    hasExactKeys(value, [
+      "locator",
+      "revision",
+      "content_digest",
+      "retrieved_at",
+      "excerpt",
+      "claims",
+    ]) &&
+    isBoundedString(value.locator, 500) &&
+    (value.revision === null || isBoundedString(value.revision, 300)) &&
+    (value.content_digest === null || isDigest(value.content_digest)) &&
+    (value.revision !== null || value.content_digest !== null) &&
+    isIsoTimestamp(value.retrieved_at) &&
+    isBoundedString(value.excerpt, 300) &&
+    Array.isArray(value.claims) &&
+    value.claims.length > 0 &&
+    value.claims.length <= MAX_CLAIMS_PER_SOURCE &&
+    value.claims.every((claim) => isBoundedString(claim, 500))
+  );
+}
+
+function isEvidenceProbe(value: unknown): value is EvidenceProbe {
+  return (
+    isRecord(value) &&
+    hasExactKeys(value, ["command", "result_digest", "summary"]) &&
+    isBoundedString(value.command, 500) &&
+    isDigest(value.result_digest) &&
+    isBoundedString(value.summary, 500)
+  );
+}
+
+function isIsoTimestamp(value: unknown): value is string {
+  if (typeof value !== "string") return false;
+  const parsed = new Date(value);
+  return !Number.isNaN(parsed.getTime()) && parsed.toISOString() === value;
+}
+
+function isDigest(value: unknown): value is string {
+  return typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+}
+
+function isBoundedString(value: unknown, maxLength: number): value is string {
+  return (
+    typeof value === "string" && value.length > 0 && value.length <= maxLength
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function hasExactKeys(
+  value: Record<string, unknown>,
+  expected: string[],
+): boolean {
+  const keys = Object.keys(value).sort();
+  return JSON.stringify(keys) === JSON.stringify([...expected].sort());
+}

package/scripts/builtin-skills-watch/github.ts

@@ -0,0 +1,161 @@
+import { spawnSync } from "node:child_process";
+import { rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+export function runGh(args: string[], input?: string): string {
+  const result = spawnSync("gh", args, {
+    encoding: "utf8",
+    input,
+    maxBuffer: 50 * 1024 * 1024,
+    stdio: input ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"],
+  });
+  if (result.status !== 0) {
+    throw new Error(`gh ${args.join(" ")} failed:\n${result.stderr}`);
+  }
+  return result.stdout;
+}
+
+export function ghJson<T>(args: string[], input?: string): T {
+  return JSON.parse(runGh(args, input)) as T;
+}
+
+export function ensureLabels(repo: string, labels: string[]): void {
+  for (const label of labels) {
+    const result = spawnSync("gh", ["label", "create", label, "--repo", repo], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    if (result.status !== 0 && !result.stderr.includes("already exists")) {
+      throw new Error(`gh label create ${label} failed:\n${result.stderr}`);
+    }
+  }
+}
+
+export function createIssueWithBody(
+  repo: string,
+  title: string,
+  body: string,
+  labels: string[] = [],
+): string {
+  const bodyFile = writeTempMarkdown(body, "builtin-skills-watch-issue");
+  try {
+    const args = [
+      "issue",
+      "create",
+      "--repo",
+      repo,
+      "--title",
+      title,
+      "--body-file",
+      bodyFile,
+    ];
+    for (const label of labels) args.push("--label", label);
+    return runGh(args).trim();
+  } finally {
+    rmSync(bodyFile, { force: true });
+  }
+}
+
+export function editIssueBody(
+  repo: string,
+  issueNumber: number,
+  body: string,
+): void {
+  const bodyFile = writeTempMarkdown(body, "builtin-skills-watch-tracker");
+  try {
+    runGh([
+      "issue",
+      "edit",
+      String(issueNumber),
+      "--repo",
+      repo,
+      "--body-file",
+      bodyFile,
+    ]);
+  } finally {
+    rmSync(bodyFile, { force: true });
+  }
+}
+
+export function createIssueComment(
+  repo: string,
+  issueNumber: number,
+  body: string,
+): string {
+  const bodyFile = writeTempMarkdown(body, "builtin-skills-watch-evidence");
+  try {
+    return runGh([
+      "issue",
+      "comment",
+      String(issueNumber),
+      "--repo",
+      repo,
+      "--body-file",
+      bodyFile,
+    ]).trim();
+  } finally {
+    rmSync(bodyFile, { force: true });
+  }
+}
+
+export function getIssueBody(repo: string, issueNumber: number): string {
+  const issue = ghJson<{ body: string | null }>([
+    "issue",
+    "view",
+    String(issueNumber),
+    "--repo",
+    repo,
+    "--json",
+    "body",
+  ]);
+  return issue.body ?? "";
+}
+
+export function findIssuesByExactTitle(
+  repo: string,
+  title: string,
+  label: string,
+): Array<{
+  number: number;
+  title: string;
+  state: string;
+  author: { login: string };
+  labels: Array<{ name: string }>;
+}> {
+  const issues = ghJson<
+    Array<{
+      number: number;
+      title: string;
+      state: string;
+      author: { login: string };
+      labels: Array<{ name: string }>;
+    }>
+  >([
+    "issue",
+    "list",
+    "--repo",
+    repo,
+    "--state",
+    "all",
+    "--label",
+    label,
+    "--search",
+    `${title} in:title`,
+    "--limit",
+    "20",
+    "--json",
+    "number,title,state,author,labels",
+  ]);
+  return issues.filter(
+    (issue) =>
+      issue.title === title &&
+      issue.labels.some((candidate) => candidate.name === label),
+  );
+}
+
+function writeTempMarkdown(body: string, prefix: string): string {
+  const path = join(tmpdir(), `${prefix}-${Date.now()}.md`);
+  writeFileSync(path, body);
+  return path;
+}

package/scripts/builtin-skills-watch/tracker.test.ts

@@ -0,0 +1,249 @@
+import { describe, expect, test } from "bun:test";
+import type { BuiltinSkillWatchAnalysis } from "./analysis.ts";
+import type { ReviewEvidence } from "./evidence.ts";
+import {
+  emptyTrackerState,
+  hasTerminalCandidate,
+  parseTrackerState,
+  recordOutcome,
+  renderTrackerBody,
+  startCandidate,
+} from "./tracker.ts";
+
+const INVENTORY = ["creating-skills", "syncing-memory-filesystem"];
+const EVIDENCE_URL =
+  "https://github.com/letta-ai/letta-code/issues/1#issuecomment-1";
+
+describe("built-in skills tracker", () => {
+  test("round trips hidden state and shows never-audited skills", () => {
+    const state = emptyTrackerState();
+    const body = renderTrackerBody(state, INVENTORY);
+
+    expect(parseTrackerState(body)).toEqual(state);
+    expect(body).toContain("| creating-skills | never | - | - |");
+  });
+
+  test("terminal outcomes advance only the selected skill", () => {
+    const analysis = fakeAnalysis("creating-skills", 1);
+    const next = recordOutcome(startCandidate(emptyTrackerState(), analysis), {
+      analysis,
+      outcome: "no_drift",
+      notes: "current",
+      processedAt: "2026-08-26T01:00:00.000Z",
+      evidence: fakeEvidence(analysis),
+      evidenceUrl: EVIDENCE_URL,
+    });
+
+    expect(next.skills["creating-skills"]).toMatchObject({
+      candidate_id: analysis.candidate_id,
+      audited_sha: "a".repeat(40),
+      outcome: "no_drift",
+    });
+    expect(next.skills["syncing-memory-filesystem"]).toBeUndefined();
+    expect(hasTerminalCandidate(next, analysis.candidate_id)).toBe(true);
+  });
+
+  test("errors remain retryable and do not advance the skill audit", () => {
+    const analysis = fakeAnalysis("creating-skills", 2);
+    const next = recordOutcome(startCandidate(emptyTrackerState(), analysis), {
+      analysis,
+      outcome: "error",
+      notes: "action failed",
+      processedAt: "2026-08-26T01:00:00.000Z",
+    });
+
+    expect(next.skills["creating-skills"]).toBeUndefined();
+    expect(next.history[0]?.outcome).toBe("error");
+    expect(hasTerminalCandidate(next, analysis.candidate_id)).toBe(false);
+    expect(next.pending["creating-skills"]?.candidate_id).toBe(
+      analysis.candidate_id,
+    );
+    expect(startCandidate(next, analysis)).toEqual(next);
+    expect(parseTrackerState(renderTrackerBody(next, INVENTORY))).toEqual(next);
+  });
+
+  test("a terminal retry replaces the same candidate history entry", () => {
+    const analysis = fakeAnalysis("creating-skills", 3);
+    const failed = recordOutcome(
+      startCandidate(emptyTrackerState(), analysis),
+      {
+        analysis,
+        outcome: "error",
+        notes: "failed",
+        processedAt: "2026-08-26T01:00:00.000Z",
+      },
+    );
+    const recovered = recordOutcome(failed, {
+      analysis,
+      outcome: "pr_created",
+      notes: "fixed",
+      prUrl: "https://github.com/letta-ai/letta-code/pull/1",
+      processedAt: "2026-08-26T02:00:00.000Z",
+      evidence: fakeEvidence(analysis),
+      evidenceUrl: EVIDENCE_URL,
+    });
+
+    expect(recovered.history).toHaveLength(1);
+    expect(recovered.history[0]?.outcome).toBe("pr_created");
+    expect(recovered.history[0]?.pr_url).toBe(
+      "https://github.com/letta-ai/letta-code/pull/1",
+    );
+  });
+
+  test("keeps history bounded", () => {
+    let state = emptyTrackerState();
+    for (let index = 0; index < 60; index += 1) {
+      const analysis = fakeAnalysis("creating-skills", index);
+      state = startCandidate(state, analysis);
+      state = recordOutcome(state, {
+        analysis,
+        outcome: "no_drift",
+        notes: "current",
+        processedAt: `2026-08-${String((index % 26) + 1).padStart(2, "0")}T00:00:00.000Z`,
+        evidence: fakeEvidence(analysis),
+        evidenceUrl: EVIDENCE_URL,
+      });
+    }
+
+    expect(state.history).toHaveLength(10);
+    expect(state.history[0]?.candidate_id).toBe(
+      fakeAnalysis("creating-skills", 59).candidate_id,
+    );
+  });
+
+  test("rejects malformed hidden state", () => {
+    expect(() =>
+      parseTrackerState("<!-- builtin-skills-agent-watch-state\n{}\n-->"),
+    ).toThrow("hidden state is invalid");
+  });
+
+  test("does not let a later error replace a terminal outcome", () => {
+    const analysis = fakeAnalysis("creating-skills", 4);
+    const terminal = recordOutcome(
+      startCandidate(emptyTrackerState(), analysis),
+      {
+        analysis,
+        outcome: "no_drift",
+        notes: "current",
+        evidence: fakeEvidence(analysis),
+        evidenceUrl: EVIDENCE_URL,
+        processedAt: "2026-08-26T01:00:00.000Z",
+      },
+    );
+    const afterError = recordOutcome(terminal, {
+      analysis,
+      outcome: "error",
+      notes: "late failure",
+      processedAt: "2026-08-26T02:00:00.000Z",
+    });
+
+    expect(afterError).toEqual(terminal);
+    expect(afterError.history[0]?.outcome).toBe("no_drift");
+  });
+
+  test("tracks and completes pending candidates independently", () => {
+    const first = fakeAnalysis("creating-skills", 5);
+    const second = fakeAnalysis("syncing-memory-filesystem", 6);
+    const pending = startCandidate(
+      startCandidate(emptyTrackerState(), first),
+      second,
+    );
+
+    expect(Object.keys(pending.pending).sort()).toEqual(INVENTORY);
+    const completed = recordOutcome(pending, {
+      analysis: first,
+      outcome: "no_drift",
+      notes: "current",
+      evidence: fakeEvidence(first),
+      evidenceUrl: EVIDENCE_URL,
+    });
+    expect(completed.pending["creating-skills"]).toBeUndefined();
+    expect(completed.pending["syncing-memory-filesystem"]?.candidate_id).toBe(
+      second.candidate_id,
+    );
+  });
+
+  test("keeps a full daily skill batch below the issue body limit", () => {
+    let state = emptyTrackerState();
+    const inventory: string[] = [];
+    const skillCount = 50;
+    for (let index = 0; index < skillCount; index += 1) {
+      const skill = `skill-${index}`;
+      inventory.push(skill);
+      const analysis = fakeAnalysis(skill, index);
+      state = startCandidate(state, analysis);
+    }
+    expect(Object.keys(state.pending)).toHaveLength(skillCount);
+    for (let index = 0; index < skillCount; index += 1) {
+      const skill = `skill-${index}`;
+      const analysis = fakeAnalysis(skill, index);
+      state = recordOutcome(state, {
+        analysis,
+        outcome: "no_drift",
+        notes: "n".repeat(120),
+        evidence: fakeEvidence(analysis),
+        evidenceUrl: EVIDENCE_URL,
+        processedAt: `2026-08-26T${String(Math.floor(index / 60)).padStart(2, "0")}:${String(index % 60).padStart(2, "0")}:00.000Z`,
+      });
+    }
+
+    expect(
+      Buffer.byteLength(renderTrackerBody(state, inventory), "utf8"),
+    ).toBeLessThan(60_000);
+    for (let index = 0; index < skillCount; index += 1) {
+      state = startCandidate(
+        state,
+        fakeAnalysis(`skill-${index}`, index + skillCount),
+      );
+    }
+    expect(
+      Buffer.byteLength(renderTrackerBody(state, inventory), "utf8"),
+    ).toBeLessThan(60_000);
+  });
+});
+
+function fakeAnalysis(
+  skill: string,
+  candidateNumber: number,
+): BuiltinSkillWatchAnalysis {
+  const candidateId = `${skill}@aaaaaaaaaaaa-${candidateNumber.toString(16).padStart(16, "0")}`;
+  return {
+    schema_version: 1,
+    candidate_id: candidateId,
+    skill,
+    skill_path: `src/skills/builtin/${skill}`,
+    skill_files: [`src/skills/builtin/${skill}/SKILL.md`],
+    skill_digest: "b".repeat(64),
+    current_sha: "a".repeat(40),
+    audit_at: "2026-08-26T00:00:00.000Z",
+    previous_audit: null,
+    repository_changes: {
+      previous_sha: null,
+      changed_files: [],
+      commits: [],
+      history_available: false,
+      truncated: false,
+    },
+    skill_inventory: INVENTORY,
+    workflow_run_url: "https://github.com/letta-ai/letta-code/actions/runs/1",
+  };
+}
+
+function fakeEvidence(analysis: BuiltinSkillWatchAnalysis): ReviewEvidence {
+  return {
+    schema_version: 1,
+    candidate_id: analysis.candidate_id,
+    skill: analysis.skill,
+    sources: [
+      {
+        locator: analysis.skill_path,
+        revision: analysis.current_sha,
+        content_digest: analysis.skill_digest,
+        retrieved_at: analysis.audit_at,
+        excerpt: "the selected skill matches its current owning source",
+        claims: ["skill files reviewed against current source"],
+      },
+    ],
+    probes: [],
+  };
+}