@letta-ai/letta-code 0.30.26 → 0.30.28

package/scripts/claude-watch/fixtures/historical-replays.json

@@ -0,0 +1,52 @@
+[
+  {
+    "id": "read-tab-prefix",
+    "previous_version": "2.1.227",
+    "current_version": "2.1.228",
+    "previous_tools": ["Read"],
+    "current_tools": ["Read"],
+    "probe_name": "read-lines-9-10-tab-prefix",
+    "previous_result": " 9→line9\n10→line10",
+    "current_result": "9\tline9\n10\tline10",
+    "expected_verdict": "tool contract review needed",
+    "local_surfaces": ["src/tools/impl/read.ts", "src/tools/read.test.ts"]
+  },
+  {
+    "id": "task-metadata-delete-contract",
+    "previous_version": "2.1.19",
+    "current_version": "2.1.20",
+    "previous_tools": ["TaskCreate", "TaskGet", "TaskList", "TaskUpdate"],
+    "current_tools": ["TaskCreate", "TaskGet", "TaskList", "TaskUpdate"],
+    "probe_name": "task-metadata-delete-contract",
+    "previous_result": "metadata values must be strings; deleted task remains retrievable",
+    "current_result": "metadata accepts arbitrary values; null deletes a key; deleted task is absent",
+    "expected_verdict": "tool contract review needed",
+    "local_surfaces": [
+      "src/tools/schemas/TaskUpdate.json",
+      "src/tools/impl/task-update.ts",
+      "src/tools/impl/tasks/store.ts"
+    ]
+  },
+  {
+    "id": "task-tool-transition",
+    "previous_version": "2.1.82",
+    "current_version": "2.1.83",
+    "previous_tools": ["LS", "MultiEdit", "Read", "TaskOutput", "TodoWrite"],
+    "current_tools": [
+      "Read",
+      "TaskCreate",
+      "TaskGet",
+      "TaskList",
+      "TaskUpdate"
+    ],
+    "probe_name": "task-metadata-delete-contract",
+    "previous_result": "legacy task surface",
+    "current_result": "task family advertised; TaskOutput deprecated in favor of Read output path",
+    "expected_verdict": "tool surface review needed",
+    "local_surfaces": [
+      "src/agent/subagents/builtin/fork.md",
+      "src/agent/subagents/builtin/general-purpose.md",
+      "src/tools/descriptions/TaskOutput.md"
+    ]
+  }
+]

package/scripts/claude-watch/github.ts

@@ -0,0 +1,137 @@
+import { spawnSync } from "node:child_process";
+import { rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+export function runGh(args: string[], input?: string): string {
+  const result = spawnSync("gh", args, {
+    encoding: "utf8",
+    input,
+    maxBuffer: 50 * 1024 * 1024,
+    stdio: input ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"],
+  });
+  if (result.status !== 0) {
+    throw new Error(`gh ${args.join(" ")} failed:\n${result.stderr}`);
+  }
+  return result.stdout;
+}
+
+export function ghJson<T>(args: string[]): T {
+  return JSON.parse(runGh(args)) as T;
+}
+
+export function ensureLabel(repo: string, label: string): void {
+  const result = spawnSync(
+    "gh",
+    [
+      "label",
+      "create",
+      label,
+      "--repo",
+      repo,
+      "--description",
+      "Automated Claude Code upstream monitoring",
+      "--color",
+      "7E57C2",
+    ],
+    { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] },
+  );
+  if (result.status !== 0 && !result.stderr.includes("already exists")) {
+    throw new Error(`gh label create ${label} failed:\n${result.stderr}`);
+  }
+}
+
+export function findIssueByExactTitle(
+  repo: string,
+  title: string,
+): { number: number; url: string; body: string } | null {
+  const issues = ghJson<Array<{ number: number; title: string; url: string }>>([
+    "issue",
+    "list",
+    "--repo",
+    repo,
+    "--state",
+    "all",
+    "--search",
+    `${title} in:title`,
+    "--limit",
+    "20",
+    "--json",
+    "number,title,url",
+  ]);
+  const issue = issues.find((candidate) => candidate.title === title);
+  if (!issue) return null;
+  return { ...issue, body: getIssueBody(repo, issue.number) };
+}
+
+export function createIssue(
+  repo: string,
+  title: string,
+  body: string,
+  label: string,
+): { number: number; url: string; body: string } {
+  const bodyFile = writeTemp(body, "claude-watch-issue");
+  try {
+    const url = runGh([
+      "issue",
+      "create",
+      "--repo",
+      repo,
+      "--title",
+      title,
+      "--body-file",
+      bodyFile,
+      "--label",
+      label,
+    ]).trim();
+    const issue = ghJson<{ number: number; url: string; body: string | null }>([
+      "issue",
+      "view",
+      url,
+      "--repo",
+      repo,
+      "--json",
+      "number,url,body",
+    ]);
+    return { ...issue, body: issue.body ?? "" };
+  } finally {
+    rmSync(bodyFile, { force: true });
+  }
+}
+
+export function getIssueBody(repo: string, issue: number): string {
+  return (
+    ghJson<{ body: string | null }>([
+      "issue",
+      "view",
+      String(issue),
+      "--repo",
+      repo,
+      "--json",
+      "body",
+    ]).body ?? ""
+  );
+}
+
+export function editIssueBody(repo: string, issue: number, body: string): void {
+  const bodyFile = writeTemp(body, "claude-watch-tracker");
+  try {
+    runGh([
+      "issue",
+      "edit",
+      String(issue),
+      "--repo",
+      repo,
+      "--body-file",
+      bodyFile,
+    ]);
+  } finally {
+    rmSync(bodyFile, { force: true });
+  }
+}
+
+function writeTemp(body: string, prefix: string): string {
+  const path = join(tmpdir(), `${prefix}-${process.pid}-${Date.now()}.md`);
+  writeFileSync(path, body);
+  return path;
+}

package/scripts/claude-watch/release-analysis.test.ts

@@ -0,0 +1,235 @@
+import { describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  analyzeClaudeCandidate,
+  buildClaudeCandidateId,
+} from "./release-analysis.ts";
+import type {
+  ClaudeDocsSnapshot,
+  ClaudeProbeObservation,
+  ClaudeReleaseCandidate,
+  ClaudeRuntimeSnapshot,
+  ClaudeWatchStateSnapshot,
+  ClaudeWatchVerdict,
+} from "./types.ts";
+
+interface ReplayFixture {
+  id: string;
+  previous_version: string;
+  current_version: string;
+  previous_tools: string[];
+  current_tools: string[];
+  probe_name: string;
+  previous_result: string;
+  current_result: string;
+  expected_verdict: ClaudeWatchVerdict;
+  local_surfaces: string[];
+}
+
+const fixtures = JSON.parse(
+  readFileSync(
+    join(import.meta.dir, "fixtures", "historical-replays.json"),
+    "utf8",
+  ),
+) as ReplayFixture[];
+
+function docs(digest = "docs-stable"): ClaudeDocsSnapshot {
+  return {
+    index_url: "https://code.claude.com/docs/llms.txt",
+    index_hash: "index",
+    digest,
+    full_scan: true,
+    scanned_at: "2026-08-01T00:00:00Z",
+    pages: {},
+  };
+}
+
+function probe(name: string, result: string): ClaudeProbeObservation {
+  return {
+    name,
+    status: "passed",
+    attempts: 1,
+    assertions: { observed_result: result },
+    tool_calls: [],
+    tool_results: [result],
+    filesystem_changes: [],
+    error: null,
+  };
+}
+
+function runtime(
+  version: string,
+  tools: string[],
+  observation: ClaudeProbeObservation,
+): ClaudeRuntimeSnapshot {
+  return {
+    version,
+    version_output: `${version} (Claude Code)`,
+    help_text: "--permission-mode <mode>",
+    help_hash: "help",
+    doctor: { exit_code: 0, summary: "No installation issues found." },
+    auto_mode_defaults: {},
+    init: {
+      tools,
+      model: "claude-test",
+      capabilities: null,
+      stable_fields: {},
+    },
+    event_inventory: ["assistant", "system/init", "user"],
+    probes: [observation],
+    digest: `${version}-runtime`,
+  };
+}
+
+function candidate(version: string): ClaudeReleaseCandidate {
+  return {
+    version,
+    published_at: "2026-08-01T00:00:00Z",
+    integrity: `sha512-${version}`,
+    tarball_url: `https://registry.npmjs.org/${version}.tgz`,
+    release_url: `https://github.com/anthropics/claude-code/releases/tag/v${version}`,
+    release_notes_md: "Observable Claude Code harness behavior changed.",
+    release_published_at: "2026-08-01T00:01:00Z",
+    dist_tags: { latest: version, stable: version, next: null },
+  };
+}
+
+function previousState(
+  version: string,
+  previousRuntime: ClaudeRuntimeSnapshot,
+): ClaudeWatchStateSnapshot {
+  return {
+    schema_version: 1,
+    candidate_id: buildClaudeCandidateId(
+      version,
+      docs().digest,
+      previousRuntime.digest,
+    ),
+    package_version: version,
+    npm_integrity: `sha512-${version}`,
+    npm_published_at: "2026-08-01T00:00:00Z",
+    release_url: `https://github.com/anthropics/claude-code/releases/tag/v${version}`,
+    release_notes_md: "previous",
+    release_published_at: "2026-08-01T00:00:00Z",
+    dist_tags: { latest: version, stable: version, next: null },
+    docs: docs(),
+    runtime: previousRuntime,
+    fetched_at: "2026-08-01T00:00:00Z",
+    workflow_run_url: "https://example.test/actions/1",
+    state_commit_parent: null,
+  };
+}
+
+describe("historical Claude parity routing fixtures", () => {
+  for (const fixture of fixtures) {
+    test(`${fixture.id} routes captured evidence to semantic review`, () => {
+      const oldRuntime = runtime(
+        fixture.previous_version,
+        fixture.previous_tools,
+        probe(fixture.probe_name, fixture.previous_result),
+      );
+      const currentRuntime = runtime(
+        fixture.current_version,
+        fixture.current_tools,
+        probe(fixture.probe_name, fixture.current_result),
+      );
+      const analysis = analyzeClaudeCandidate({
+        candidate: candidate(fixture.current_version),
+        previous: previousState(fixture.previous_version, oldRuntime),
+        currentDocs: docs(),
+        currentRuntime,
+        workflowRunUrl: "https://example.test/actions/2",
+        stateBaseSha: "state-parent",
+      });
+
+      expect(analysis.verdict).toBe(fixture.expected_verdict);
+      expect(fixture.local_surfaces.length).toBeGreaterThan(0);
+    });
+  }
+});
+
+describe("Claude verdict routing", () => {
+  test("bootstrap requires a current-vs-local audit", () => {
+    const currentRuntime = runtime(
+      "2.1.237",
+      ["Read"],
+      probe("read-lines-9-10-tab-prefix", "9\tline9\n10\tline10"),
+    );
+    const analysis = analyzeClaudeCandidate({
+      candidate: candidate("2.1.237"),
+      previous: null,
+      currentDocs: docs(),
+      currentRuntime,
+      workflowRunUrl: "https://example.test/actions/bootstrap",
+      stateBaseSha: null,
+    });
+
+    expect(analysis.verdict).toBe("manual review required");
+    expect(analysis.runtime_snapshot?.init?.tools).toEqual(["Read"]);
+  });
+
+  test("inconclusive behavioral probes require manual review", () => {
+    const old = runtime(
+      "2.1.236",
+      ["Read"],
+      probe("read-lines-9-10-tab-prefix", "old"),
+    );
+    const inconclusive = runtime("2.1.237", ["Read"], {
+      ...probe("read-lines-9-10-tab-prefix", ""),
+      status: "inconclusive",
+      error: "model did not call Read",
+    });
+    const analysis = analyzeClaudeCandidate({
+      candidate: candidate("2.1.237"),
+      previous: previousState("2.1.236", old),
+      currentDocs: docs(),
+      currentRuntime: inconclusive,
+      workflowRunUrl: "https://example.test/actions/3",
+      stateBaseSha: "parent",
+    });
+    expect(analysis.verdict).toBe("manual review required");
+    expect(analysis.verdict_reasons[0]).toContain("inconclusive");
+  });
+
+  test("an unwatched docs-only digest change still receives semantic review", () => {
+    const stableRuntime = runtime(
+      "2.1.237",
+      ["Read"],
+      probe("read-lines-9-10-tab-prefix", "stable"),
+    );
+    const previous = previousState("2.1.237", stableRuntime);
+    previous.docs = {
+      ...docs("old-docs"),
+      pages: {
+        "https://code.claude.com/docs/en/billing.md": {
+          url: "https://code.claude.com/docs/en/billing.md",
+          hash: "old",
+          watched: false,
+          source_path: null,
+        },
+      },
+    };
+    const currentDocs = {
+      ...docs("new-docs"),
+      pages: {
+        "https://code.claude.com/docs/en/billing.md": {
+          url: "https://code.claude.com/docs/en/billing.md",
+          hash: "new",
+          watched: false,
+          source_path: null,
+        },
+      },
+    };
+    const analysis = analyzeClaudeCandidate({
+      candidate: candidate("2.1.237"),
+      previous,
+      currentDocs,
+      currentRuntime: stableRuntime,
+      workflowRunUrl: "https://example.test/actions/4",
+      stateBaseSha: "parent",
+    });
+    expect(analysis.verdict).toBe("harness behavior review needed");
+    expect(analysis.verdict_reasons[0]).toContain("unwatched");
+  });
+});