@letta-ai/letta-code 0.31.6 → 0.31.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/mcp-client.js +2 -2
- package/dist/mcp-client.js.map +1 -1
- package/dist/types/backend/backend.d.ts +7 -0
- package/dist/types/backend/backend.d.ts.map +1 -1
- package/dist/types/utils/secrets-store.d.ts +4 -0
- package/dist/types/utils/secrets-store.d.ts.map +1 -1
- package/letta.js +18 -14
- package/package.json +1 -1
- package/scripts/agent-watch/verify-pr-identity.test.ts +97 -0
- package/scripts/agent-watch/verify-pr-identity.ts +119 -0
- package/scripts/builtin-skills-watch/aggregate-results.ts +13 -8
- package/scripts/builtin-skills-watch/evidence.test.ts +61 -17
- package/scripts/builtin-skills-watch/evidence.ts +66 -38
- package/scripts/builtin-skills-watch/finalize-result.test.ts +125 -0
- package/scripts/builtin-skills-watch/finalize-result.ts +44 -0
- package/scripts/builtin-skills-watch/reconcile-results.test.ts +58 -0
- package/scripts/builtin-skills-watch/reconcile-results.ts +311 -0
- package/scripts/builtin-skills-watch/result-artifacts.test.ts +83 -0
- package/scripts/builtin-skills-watch/result-artifacts.ts +160 -0
- package/scripts/builtin-skills-watch/update-tracker.test.ts +17 -1
- package/scripts/builtin-skills-watch/update-tracker.ts +89 -15
- package/scripts/claude-watch/update-tracker.test.ts +30 -0
- package/scripts/claude-watch/update-tracker.ts +19 -5
- package/scripts/codex-watch/update-tracker.test.ts +28 -0
- package/scripts/codex-watch/update-tracker.ts +59 -7
- package/skills/letta-guide/SKILL.md +6 -2
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import {
|
|
6
|
+
discoverReviewArtifacts,
|
|
7
|
+
formatFailureReceipt,
|
|
8
|
+
} from "./result-artifacts.ts";
|
|
9
|
+
|
|
10
|
+
const CANDIDATE = "creating-skills@aaaaaaaaaaaa-abcdef0123456789";
|
|
11
|
+
|
|
12
|
+
describe("review artifact discovery", () => {
|
|
13
|
+
test("finds both nested and root-level result artifacts", () => {
|
|
14
|
+
const directory = mkdtempSync(join(tmpdir(), "skill-artifacts-"));
|
|
15
|
+
try {
|
|
16
|
+
const nested = join(directory, "builtin-skill-result-creating-skills");
|
|
17
|
+
mkdirSync(nested);
|
|
18
|
+
writeFileSync(
|
|
19
|
+
join(nested, "result.json"),
|
|
20
|
+
JSON.stringify({ candidate_id: CANDIDATE }),
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
const discovered = discoverReviewArtifacts(directory);
|
|
24
|
+
expect(discovered.results.get(CANDIDATE)).toBe(
|
|
25
|
+
join(nested, "result.json"),
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
rmSync(nested, { recursive: true });
|
|
29
|
+
writeFileSync(
|
|
30
|
+
join(directory, "result.json"),
|
|
31
|
+
JSON.stringify({ candidate_id: CANDIDATE }),
|
|
32
|
+
);
|
|
33
|
+
expect(discoverReviewArtifacts(directory).results.get(CANDIDATE)).toBe(
|
|
34
|
+
join(directory, "result.json"),
|
|
35
|
+
);
|
|
36
|
+
} finally {
|
|
37
|
+
rmSync(directory, { recursive: true, force: true });
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("preserves a runner-authored failure receipt", () => {
|
|
42
|
+
const directory = mkdtempSync(join(tmpdir(), "skill-artifacts-"));
|
|
43
|
+
try {
|
|
44
|
+
writeFileSync(
|
|
45
|
+
join(directory, "failure.json"),
|
|
46
|
+
JSON.stringify({
|
|
47
|
+
schema_version: 1,
|
|
48
|
+
candidate_id: CANDIDATE,
|
|
49
|
+
skill: "creating-skills",
|
|
50
|
+
kind: "action_failed",
|
|
51
|
+
message: "Letta Code Action failed before returning a result",
|
|
52
|
+
conversation_id: "conv-123",
|
|
53
|
+
}),
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
const receipt =
|
|
57
|
+
discoverReviewArtifacts(directory).failures.get(CANDIDATE);
|
|
58
|
+
expect(receipt).toBeDefined();
|
|
59
|
+
expect(formatFailureReceipt(receipt!)).toContain("conversation conv-123");
|
|
60
|
+
} finally {
|
|
61
|
+
rmSync(directory, { recursive: true, force: true });
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("rejects duplicate results for one candidate", () => {
|
|
66
|
+
const directory = mkdtempSync(join(tmpdir(), "skill-artifacts-"));
|
|
67
|
+
try {
|
|
68
|
+
mkdirSync(join(directory, "first"));
|
|
69
|
+
mkdirSync(join(directory, "second"));
|
|
70
|
+
for (const subdirectory of ["first", "second"]) {
|
|
71
|
+
writeFileSync(
|
|
72
|
+
join(directory, subdirectory, "result.json"),
|
|
73
|
+
JSON.stringify({ candidate_id: CANDIDATE }),
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
expect(() => discoverReviewArtifacts(directory)).toThrow(
|
|
77
|
+
`duplicate review result for ${CANDIDATE}`,
|
|
78
|
+
);
|
|
79
|
+
} finally {
|
|
80
|
+
rmSync(directory, { recursive: true, force: true });
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { basename, join } from "node:path";
|
|
3
|
+
|
|
4
|
+
export interface ReviewFailureReceipt {
|
|
5
|
+
schema_version: 1;
|
|
6
|
+
candidate_id: string;
|
|
7
|
+
skill: string;
|
|
8
|
+
kind:
|
|
9
|
+
| "action_failed"
|
|
10
|
+
| "execution_file_missing"
|
|
11
|
+
| "result_marker_missing"
|
|
12
|
+
| "result_decode_failed";
|
|
13
|
+
message: string;
|
|
14
|
+
conversation_id: string | null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface ReviewArtifacts {
|
|
18
|
+
results: Map<string, string>;
|
|
19
|
+
failures: Map<string, ReviewFailureReceipt>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function discoverReviewArtifacts(root: string): ReviewArtifacts {
|
|
23
|
+
const artifacts: ReviewArtifacts = {
|
|
24
|
+
results: new Map(),
|
|
25
|
+
failures: new Map(),
|
|
26
|
+
};
|
|
27
|
+
for (const path of walkFiles(root)) {
|
|
28
|
+
const name = basename(path);
|
|
29
|
+
if (name === "result.json") {
|
|
30
|
+
const candidateId = readCandidateId(path, "review result");
|
|
31
|
+
addUnique(artifacts.results, candidateId, path, "review result");
|
|
32
|
+
} else if (name === "failure.json") {
|
|
33
|
+
const receipt = parseFailureReceipt(
|
|
34
|
+
JSON.parse(readFileSync(path, "utf8")) as unknown,
|
|
35
|
+
);
|
|
36
|
+
addUnique(
|
|
37
|
+
artifacts.failures,
|
|
38
|
+
receipt.candidate_id,
|
|
39
|
+
receipt,
|
|
40
|
+
"failure receipt",
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return artifacts;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function formatFailureReceipt(receipt: ReviewFailureReceipt): string {
|
|
48
|
+
const conversation = receipt.conversation_id
|
|
49
|
+
? ` (conversation ${receipt.conversation_id})`
|
|
50
|
+
: "";
|
|
51
|
+
return `${receipt.kind}: ${receipt.message}${conversation}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function walkFiles(root: string): string[] {
|
|
55
|
+
const files: string[] = [];
|
|
56
|
+
let entries: ReturnType<typeof readdirSync>;
|
|
57
|
+
try {
|
|
58
|
+
entries = readdirSync(root, { withFileTypes: true });
|
|
59
|
+
} catch (error) {
|
|
60
|
+
if (isMissingPath(error)) return files;
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
for (const entry of entries) {
|
|
64
|
+
const path = join(root, entry.name);
|
|
65
|
+
if (entry.isDirectory()) files.push(...walkFiles(path));
|
|
66
|
+
else if (entry.isFile()) files.push(path);
|
|
67
|
+
}
|
|
68
|
+
return files.sort();
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function readCandidateId(path: string, description: string): string {
|
|
72
|
+
const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
73
|
+
if (!isRecord(value) || !isCandidateId(value.candidate_id)) {
|
|
74
|
+
throw new Error(`${description} ${path} has no valid candidate_id`);
|
|
75
|
+
}
|
|
76
|
+
return value.candidate_id;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function parseFailureReceipt(value: unknown): ReviewFailureReceipt {
|
|
80
|
+
if (
|
|
81
|
+
!isRecord(value) ||
|
|
82
|
+
!hasExactKeys(value, [
|
|
83
|
+
"schema_version",
|
|
84
|
+
"candidate_id",
|
|
85
|
+
"skill",
|
|
86
|
+
"kind",
|
|
87
|
+
"message",
|
|
88
|
+
"conversation_id",
|
|
89
|
+
]) ||
|
|
90
|
+
value.schema_version !== 1 ||
|
|
91
|
+
!isCandidateId(value.candidate_id) ||
|
|
92
|
+
!isSkillName(value.skill) ||
|
|
93
|
+
!isFailureKind(value.kind) ||
|
|
94
|
+
typeof value.message !== "string" ||
|
|
95
|
+
value.message.length === 0 ||
|
|
96
|
+
value.message.length > 500 ||
|
|
97
|
+
(value.conversation_id !== null && !isConversationId(value.conversation_id))
|
|
98
|
+
) {
|
|
99
|
+
throw new Error("Review failure receipt is invalid");
|
|
100
|
+
}
|
|
101
|
+
return value as unknown as ReviewFailureReceipt;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function addUnique<T>(
|
|
105
|
+
values: Map<string, T>,
|
|
106
|
+
candidateId: string,
|
|
107
|
+
value: T,
|
|
108
|
+
description: string,
|
|
109
|
+
): void {
|
|
110
|
+
if (values.has(candidateId)) {
|
|
111
|
+
throw new Error(`Found duplicate ${description} for ${candidateId}`);
|
|
112
|
+
}
|
|
113
|
+
values.set(candidateId, value);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function isFailureKind(value: unknown): value is ReviewFailureReceipt["kind"] {
|
|
117
|
+
return (
|
|
118
|
+
value === "action_failed" ||
|
|
119
|
+
value === "execution_file_missing" ||
|
|
120
|
+
value === "result_marker_missing" ||
|
|
121
|
+
value === "result_decode_failed"
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function isCandidateId(value: unknown): value is string {
|
|
126
|
+
return (
|
|
127
|
+
typeof value === "string" &&
|
|
128
|
+
/^[a-z0-9]+(?:-[a-z0-9]+)*@[a-f0-9]{12}-[a-f0-9]{16}$/.test(value)
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function isSkillName(value: unknown): value is string {
|
|
133
|
+
return typeof value === "string" && /^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function isConversationId(value: unknown): value is string {
|
|
137
|
+
return typeof value === "string" && /^conv-[a-zA-Z0-9-]+$/.test(value);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function isMissingPath(error: unknown): boolean {
|
|
141
|
+
return (
|
|
142
|
+
error instanceof Error &&
|
|
143
|
+
"code" in error &&
|
|
144
|
+
(error as NodeJS.ErrnoException).code === "ENOENT"
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
149
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function hasExactKeys(
|
|
153
|
+
value: Record<string, unknown>,
|
|
154
|
+
expected: string[],
|
|
155
|
+
): boolean {
|
|
156
|
+
return (
|
|
157
|
+
JSON.stringify(Object.keys(value).sort()) ===
|
|
158
|
+
JSON.stringify([...expected].sort())
|
|
159
|
+
);
|
|
160
|
+
}
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
parseReviewResult,
|
|
7
7
|
type TrackerIssueView,
|
|
8
8
|
validatePullRequestView,
|
|
9
|
+
validateReconciledPullRequestView,
|
|
9
10
|
validateTrackerIssueView,
|
|
10
11
|
} from "./update-tracker.ts";
|
|
11
12
|
|
|
@@ -79,6 +80,21 @@ describe("watcher PR validation", () => {
|
|
|
79
80
|
),
|
|
80
81
|
).toThrow("outside the selected skill scope");
|
|
81
82
|
});
|
|
83
|
+
|
|
84
|
+
test("accepts a merged exact-candidate PR only during reconciliation", () => {
|
|
85
|
+
const merged = validPullRequest({
|
|
86
|
+
isDraft: false,
|
|
87
|
+
mergedAt: "2026-08-27T00:00:00Z",
|
|
88
|
+
state: "MERGED",
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
expect(() =>
|
|
92
|
+
validateReconciledPullRequestView(merged, URL, analysis(), LOGIN),
|
|
93
|
+
).not.toThrow();
|
|
94
|
+
expect(() =>
|
|
95
|
+
validatePullRequestView(merged, URL, analysis(), LOGIN),
|
|
96
|
+
).toThrow("open and draft");
|
|
97
|
+
});
|
|
82
98
|
});
|
|
83
99
|
|
|
84
100
|
describe("watcher analysis validation", () => {
|
|
@@ -135,7 +151,7 @@ describe("watcher result validation", () => {
|
|
|
135
151
|
evidence: evidence(current),
|
|
136
152
|
};
|
|
137
153
|
expect(() => parseReviewResult({ ...base, secret: "no" }, current)).toThrow(
|
|
138
|
-
"
|
|
154
|
+
"unknown or missing fields",
|
|
139
155
|
);
|
|
140
156
|
expect(() =>
|
|
141
157
|
parseReviewResult(
|
|
@@ -37,6 +37,7 @@ export interface PullRequestView {
|
|
|
37
37
|
files: Array<{ path: string }>;
|
|
38
38
|
headRefOid: string;
|
|
39
39
|
isDraft: boolean;
|
|
40
|
+
mergedAt?: string | null;
|
|
40
41
|
state: string;
|
|
41
42
|
url: string;
|
|
42
43
|
}
|
|
@@ -337,19 +338,35 @@ export function parseReviewResult(
|
|
|
337
338
|
"notes",
|
|
338
339
|
"pr_url",
|
|
339
340
|
"evidence",
|
|
340
|
-
])
|
|
341
|
-
|
|
341
|
+
])
|
|
342
|
+
) {
|
|
343
|
+
throw new Error("Review result has unknown or missing fields");
|
|
344
|
+
}
|
|
345
|
+
if (value.schema_version !== 1) {
|
|
346
|
+
throw new Error("Review result must use schema version 1");
|
|
347
|
+
}
|
|
348
|
+
if (
|
|
342
349
|
value.candidate_id !== analysis.candidate_id ||
|
|
343
|
-
value.skill !== analysis.skill
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
350
|
+
value.skill !== analysis.skill
|
|
351
|
+
) {
|
|
352
|
+
throw new Error("Review result does not match the pending candidate");
|
|
353
|
+
}
|
|
354
|
+
if (
|
|
355
|
+
value.outcome !== "no_drift" &&
|
|
356
|
+
value.outcome !== "pr_created" &&
|
|
357
|
+
value.outcome !== "needs_human_review"
|
|
358
|
+
) {
|
|
359
|
+
throw new Error("Review result outcome is invalid");
|
|
360
|
+
}
|
|
361
|
+
if (
|
|
347
362
|
typeof value.notes !== "string" ||
|
|
348
363
|
value.notes.length === 0 ||
|
|
349
|
-
value.notes.length > 120
|
|
350
|
-
(value.pr_url !== null && typeof value.pr_url !== "string")
|
|
364
|
+
value.notes.length > 120
|
|
351
365
|
) {
|
|
352
|
-
throw new Error("Review result
|
|
366
|
+
throw new Error("Review result notes must contain 1 to 120 characters");
|
|
367
|
+
}
|
|
368
|
+
if (value.pr_url !== null && typeof value.pr_url !== "string") {
|
|
369
|
+
throw new Error("Review result pr_url must be a string or null");
|
|
353
370
|
}
|
|
354
371
|
if (
|
|
355
372
|
(value.outcome === "pr_created") !==
|
|
@@ -381,28 +398,61 @@ export function verifyPullRequest(
|
|
|
381
398
|
analysis: BuiltinSkillWatchAnalysis,
|
|
382
399
|
expectedGithubLogin: string,
|
|
383
400
|
): void {
|
|
401
|
+
verifyPullRequestIdentity(expectedGithubLogin);
|
|
402
|
+
const pullRequest = getPullRequest(repo, prUrl);
|
|
403
|
+
validatePullRequestView(pullRequest, prUrl, analysis, expectedGithubLogin);
|
|
404
|
+
verifyPullRequestAncestry(repo, pullRequest, analysis);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
export function verifyReconciledPullRequest(
|
|
408
|
+
repo: string,
|
|
409
|
+
prUrl: string,
|
|
410
|
+
analysis: BuiltinSkillWatchAnalysis,
|
|
411
|
+
expectedGithubLogin: string,
|
|
412
|
+
): void {
|
|
413
|
+
verifyPullRequestIdentity(expectedGithubLogin);
|
|
414
|
+
const pullRequest = getPullRequest(repo, prUrl);
|
|
415
|
+
validateReconciledPullRequestView(
|
|
416
|
+
pullRequest,
|
|
417
|
+
prUrl,
|
|
418
|
+
analysis,
|
|
419
|
+
expectedGithubLogin,
|
|
420
|
+
);
|
|
421
|
+
verifyPullRequestAncestry(repo, pullRequest, analysis);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function verifyPullRequestIdentity(expectedGithubLogin: string): void {
|
|
384
425
|
const authenticatedLogin = ghJson<{ login: string }>(["api", "user"]).login;
|
|
385
426
|
if (authenticatedLogin !== expectedGithubLogin) {
|
|
386
427
|
throw new Error(
|
|
387
428
|
`Authenticated GitHub login ${authenticatedLogin} does not match ${expectedGithubLogin}`,
|
|
388
429
|
);
|
|
389
430
|
}
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
function getPullRequest(repo: string, prUrl: string): PullRequestView {
|
|
390
434
|
const match = prUrl.match(
|
|
391
435
|
/^https:\/\/github\.com\/([^/]+\/[^/]+)\/pull\/(\d+)\/?$/,
|
|
392
436
|
);
|
|
393
437
|
if (!match || match[1] !== repo) {
|
|
394
438
|
throw new Error(`PR URL must belong to https://github.com/${repo}`);
|
|
395
439
|
}
|
|
396
|
-
|
|
440
|
+
return ghJson<PullRequestView>([
|
|
397
441
|
"pr",
|
|
398
442
|
"view",
|
|
399
443
|
match[2] as string,
|
|
400
444
|
"--repo",
|
|
401
445
|
repo,
|
|
402
446
|
"--json",
|
|
403
|
-
"author,baseRefName,body,files,headRefOid,isDraft,state,url",
|
|
447
|
+
"author,baseRefName,body,files,headRefOid,isDraft,mergedAt,state,url",
|
|
404
448
|
]);
|
|
405
|
-
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function verifyPullRequestAncestry(
|
|
452
|
+
repo: string,
|
|
453
|
+
pullRequest: PullRequestView,
|
|
454
|
+
analysis: BuiltinSkillWatchAnalysis,
|
|
455
|
+
): void {
|
|
406
456
|
const comparison = ghJson<{
|
|
407
457
|
status: string;
|
|
408
458
|
merge_base_commit: { sha: string };
|
|
@@ -423,6 +473,33 @@ export function validatePullRequestView(
|
|
|
423
473
|
prUrl: string,
|
|
424
474
|
analysis: BuiltinSkillWatchAnalysis,
|
|
425
475
|
expectedGithubLogin: string,
|
|
476
|
+
): void {
|
|
477
|
+
validatePullRequestScope(pullRequest, prUrl, analysis, expectedGithubLogin);
|
|
478
|
+
if (pullRequest.state !== "OPEN" || !pullRequest.isDraft) {
|
|
479
|
+
throw new Error("Watcher PR must be open and draft");
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
export function validateReconciledPullRequestView(
|
|
484
|
+
pullRequest: PullRequestView,
|
|
485
|
+
prUrl: string,
|
|
486
|
+
analysis: BuiltinSkillWatchAnalysis,
|
|
487
|
+
expectedGithubLogin: string,
|
|
488
|
+
): void {
|
|
489
|
+
validatePullRequestScope(pullRequest, prUrl, analysis, expectedGithubLogin);
|
|
490
|
+
const isOpenDraft = pullRequest.state === "OPEN" && pullRequest.isDraft;
|
|
491
|
+
const isMerged =
|
|
492
|
+
pullRequest.state === "MERGED" && typeof pullRequest.mergedAt === "string";
|
|
493
|
+
if (!isOpenDraft && !isMerged) {
|
|
494
|
+
throw new Error("Reconciled watcher PR must be an open draft or merged");
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
function validatePullRequestScope(
|
|
499
|
+
pullRequest: PullRequestView,
|
|
500
|
+
prUrl: string,
|
|
501
|
+
analysis: BuiltinSkillWatchAnalysis,
|
|
502
|
+
expectedGithubLogin: string,
|
|
426
503
|
): void {
|
|
427
504
|
if (pullRequest.url !== prUrl.replace(/\/$/, "")) {
|
|
428
505
|
throw new Error(`PR URL mismatch: ${pullRequest.url}`);
|
|
@@ -432,9 +509,6 @@ export function validatePullRequestView(
|
|
|
432
509
|
`PR author ${pullRequest.author.login} does not match ${expectedGithubLogin}`,
|
|
433
510
|
);
|
|
434
511
|
}
|
|
435
|
-
if (pullRequest.state !== "OPEN" || !pullRequest.isDraft) {
|
|
436
|
-
throw new Error("Watcher PR must be open and draft");
|
|
437
|
-
}
|
|
438
512
|
if (pullRequest.baseRefName !== "main") {
|
|
439
513
|
throw new Error("Watcher PR must target main");
|
|
440
514
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { parseArgs } from "./update-tracker.ts";
|
|
3
|
+
|
|
4
|
+
describe("parseArgs", () => {
|
|
5
|
+
const required = [
|
|
6
|
+
"--tracker-issue",
|
|
7
|
+
"123",
|
|
8
|
+
"--analysis-file",
|
|
9
|
+
"/tmp/analysis.json",
|
|
10
|
+
"--state-commit-sha",
|
|
11
|
+
"abc123",
|
|
12
|
+
"--outcome",
|
|
13
|
+
"pr_created",
|
|
14
|
+
"--pr-url",
|
|
15
|
+
"https://github.com/letta-ai/letta-code/pull/456",
|
|
16
|
+
];
|
|
17
|
+
|
|
18
|
+
test("requires the expected GitHub login for a PR", () => {
|
|
19
|
+
expect(() => parseArgs(required)).toThrow(
|
|
20
|
+
"--expected-github-login is required for pr_created",
|
|
21
|
+
);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
test("accepts the expected GitHub login for a PR", () => {
|
|
25
|
+
expect(
|
|
26
|
+
parseArgs([...required, "--expected-github-login", "amelia-letta"])
|
|
27
|
+
.expectedGithubLogin,
|
|
28
|
+
).toBe("amelia-letta");
|
|
29
|
+
});
|
|
30
|
+
});
|
|
@@ -27,6 +27,7 @@ interface Args {
|
|
|
27
27
|
outcome: ClaudeWatchOutcome | null;
|
|
28
28
|
notes: string;
|
|
29
29
|
prUrl: string | null;
|
|
30
|
+
expectedGithubLogin: string | null;
|
|
30
31
|
assertTerminal: boolean;
|
|
31
32
|
dryRun: boolean;
|
|
32
33
|
}
|
|
@@ -41,6 +42,7 @@ export function parseArgs(argv: string[]): Args {
|
|
|
41
42
|
outcome: null,
|
|
42
43
|
notes: "",
|
|
43
44
|
prUrl: null,
|
|
45
|
+
expectedGithubLogin: null,
|
|
44
46
|
assertTerminal: false,
|
|
45
47
|
dryRun: false,
|
|
46
48
|
};
|
|
@@ -59,6 +61,8 @@ export function parseArgs(argv: string[]): Args {
|
|
|
59
61
|
args.outcome = parseOutcome(argv[++index]);
|
|
60
62
|
else if (argument === "--notes") args.notes = argv[++index] ?? "";
|
|
61
63
|
else if (argument === "--pr-url") args.prUrl = argv[++index] ?? null;
|
|
64
|
+
else if (argument === "--expected-github-login")
|
|
65
|
+
args.expectedGithubLogin = argv[++index] ?? null;
|
|
62
66
|
else if (argument === "--assert-terminal") args.assertTerminal = true;
|
|
63
67
|
else if (argument === "--dry-run") args.dryRun = true;
|
|
64
68
|
else throw new Error(`Unknown argument: ${argument}`);
|
|
@@ -75,8 +79,12 @@ export function parseArgs(argv: string[]): Args {
|
|
|
75
79
|
if (!args.outcome) throw new Error("--outcome is required");
|
|
76
80
|
if (isTerminalOutcome(args.outcome) && !args.stateCommitSha)
|
|
77
81
|
throw new Error("terminal outcomes require --state-commit-sha");
|
|
78
|
-
if (args.outcome === "pr_created"
|
|
79
|
-
throw new Error("--pr-url is required for pr_created");
|
|
82
|
+
if (args.outcome === "pr_created") {
|
|
83
|
+
if (!args.prUrl) throw new Error("--pr-url is required for pr_created");
|
|
84
|
+
if (!args.expectedGithubLogin) {
|
|
85
|
+
throw new Error("--expected-github-login is required for pr_created");
|
|
86
|
+
}
|
|
87
|
+
}
|
|
80
88
|
}
|
|
81
89
|
return args;
|
|
82
90
|
}
|
|
@@ -117,15 +125,16 @@ function verifyParityPr(
|
|
|
117
125
|
repo: string,
|
|
118
126
|
prUrl: string,
|
|
119
127
|
candidateId: string,
|
|
128
|
+
expectedGithubLogin: string,
|
|
120
129
|
): void {
|
|
121
130
|
const pr = ghJson<{
|
|
122
131
|
isDraft: boolean;
|
|
123
132
|
author: { login: string };
|
|
124
133
|
body: string | null;
|
|
125
134
|
}>(["pr", "view", prUrl, "--repo", repo, "--json", "isDraft,author,body"]);
|
|
126
|
-
if (!pr.isDraft || pr.author.login !==
|
|
135
|
+
if (!pr.isDraft || pr.author.login !== expectedGithubLogin) {
|
|
127
136
|
throw new Error(
|
|
128
|
-
`Parity PR must be a draft authored by
|
|
137
|
+
`Parity PR must be a draft authored by ${expectedGithubLogin} (got draft=${pr.isDraft}, author=${pr.author.login})`,
|
|
129
138
|
);
|
|
130
139
|
}
|
|
131
140
|
if (!pr.body?.includes(`Claude-watch: ${candidateId}`)) {
|
|
@@ -164,7 +173,12 @@ export function main(argv = process.argv.slice(2)): void {
|
|
|
164
173
|
verifyStateCandidate(analysis.candidate_id, args.stateCommitSha as string);
|
|
165
174
|
}
|
|
166
175
|
if (args.outcome === "pr_created") {
|
|
167
|
-
verifyParityPr(
|
|
176
|
+
verifyParityPr(
|
|
177
|
+
args.repo,
|
|
178
|
+
args.prUrl as string,
|
|
179
|
+
analysis.candidate_id,
|
|
180
|
+
args.expectedGithubLogin as string,
|
|
181
|
+
);
|
|
168
182
|
}
|
|
169
183
|
const next = recordAnalysis(state, {
|
|
170
184
|
analysis,
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { parseArgs } from "./update-tracker.ts";
|
|
3
|
+
|
|
4
|
+
describe("parseArgs", () => {
|
|
5
|
+
const required = [
|
|
6
|
+
"--tracker-issue",
|
|
7
|
+
"123",
|
|
8
|
+
"--analysis-file",
|
|
9
|
+
"/tmp/analysis.json",
|
|
10
|
+
"--outcome",
|
|
11
|
+
"pr_created",
|
|
12
|
+
"--pr-url",
|
|
13
|
+
"https://github.com/letta-ai/letta-code/pull/456",
|
|
14
|
+
];
|
|
15
|
+
|
|
16
|
+
test("requires the expected GitHub login for a PR", () => {
|
|
17
|
+
expect(() => parseArgs(required)).toThrow(
|
|
18
|
+
"--expected-github-login is required for pr_created",
|
|
19
|
+
);
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
test("accepts the expected GitHub login for a PR", () => {
|
|
23
|
+
expect(
|
|
24
|
+
parseArgs([...required, "--expected-github-login", "amelia-letta"])
|
|
25
|
+
.expectedGithubLogin,
|
|
26
|
+
).toBe("amelia-letta");
|
|
27
|
+
});
|
|
28
|
+
});
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { readFileSync } from "node:fs";
|
|
10
|
-
import { editIssueBody, getIssueBody } from "./github.ts";
|
|
10
|
+
import { editIssueBody, getIssueBody, ghJson } from "./github.ts";
|
|
11
11
|
import {
|
|
12
12
|
type CodexWatchAnalysis,
|
|
13
13
|
DEFAULT_TARGET_REPO,
|
|
@@ -26,10 +26,11 @@ interface Args {
|
|
|
26
26
|
outcome: TrackerOutcome | null;
|
|
27
27
|
notes: string;
|
|
28
28
|
prUrl: string | null;
|
|
29
|
+
expectedGithubLogin: string | null;
|
|
29
30
|
dryRun: boolean;
|
|
30
31
|
}
|
|
31
32
|
|
|
32
|
-
function parseArgs(argv: string[]): Args {
|
|
33
|
+
export function parseArgs(argv: string[]): Args {
|
|
33
34
|
const args: Args = {
|
|
34
35
|
repo: DEFAULT_TARGET_REPO,
|
|
35
36
|
trackerIssue: null,
|
|
@@ -37,6 +38,7 @@ function parseArgs(argv: string[]): Args {
|
|
|
37
38
|
outcome: null,
|
|
38
39
|
notes: "",
|
|
39
40
|
prUrl: null,
|
|
41
|
+
expectedGithubLogin: null,
|
|
40
42
|
dryRun: false,
|
|
41
43
|
};
|
|
42
44
|
|
|
@@ -49,10 +51,12 @@ function parseArgs(argv: string[]): Args {
|
|
|
49
51
|
else if (a === "--outcome") args.outcome = parseOutcome(argv[++i]);
|
|
50
52
|
else if (a === "--notes") args.notes = argv[++i] ?? "";
|
|
51
53
|
else if (a === "--pr-url") args.prUrl = argv[++i] ?? null;
|
|
52
|
-
else if (a === "--
|
|
54
|
+
else if (a === "--expected-github-login") {
|
|
55
|
+
args.expectedGithubLogin = argv[++i] ?? null;
|
|
56
|
+
} else if (a === "--dry-run") args.dryRun = true;
|
|
53
57
|
else if (a === "--help" || a === "-h") {
|
|
54
58
|
console.log(
|
|
55
|
-
"Usage: bun scripts/codex-watch/update-tracker.ts --tracker-issue ISSUE --analysis-file FILE --outcome OUTCOME [--notes TEXT] [--pr-url URL] [--repo OWNER/REPO] [--dry-run]",
|
|
59
|
+
"Usage: bun scripts/codex-watch/update-tracker.ts --tracker-issue ISSUE --analysis-file FILE --outcome OUTCOME [--notes TEXT] [--pr-url URL --expected-github-login LOGIN] [--repo OWNER/REPO] [--dry-run]",
|
|
56
60
|
);
|
|
57
61
|
process.exit(0);
|
|
58
62
|
} else {
|
|
@@ -65,8 +69,13 @@ function parseArgs(argv: string[]): Args {
|
|
|
65
69
|
}
|
|
66
70
|
if (!args.analysisFile) throw new Error("--analysis-file is required");
|
|
67
71
|
if (!args.outcome) throw new Error("--outcome is required");
|
|
68
|
-
if (args.outcome === "pr_created"
|
|
69
|
-
|
|
72
|
+
if (args.outcome === "pr_created") {
|
|
73
|
+
if (!args.prUrl) {
|
|
74
|
+
throw new Error("--pr-url is required when --outcome pr_created");
|
|
75
|
+
}
|
|
76
|
+
if (!args.expectedGithubLogin) {
|
|
77
|
+
throw new Error("--expected-github-login is required for pr_created");
|
|
78
|
+
}
|
|
70
79
|
}
|
|
71
80
|
|
|
72
81
|
return args;
|
|
@@ -94,6 +103,14 @@ function main() {
|
|
|
94
103
|
const analysis = readAnalysis(args.analysisFile as string);
|
|
95
104
|
const body = getIssueBody(args.repo, args.trackerIssue as number);
|
|
96
105
|
const state = parseTrackerState(body);
|
|
106
|
+
if (args.outcome === "pr_created") {
|
|
107
|
+
verifyParityPr(
|
|
108
|
+
args.repo,
|
|
109
|
+
args.prUrl as string,
|
|
110
|
+
args.expectedGithubLogin as string,
|
|
111
|
+
analysis.current_tag,
|
|
112
|
+
);
|
|
113
|
+
}
|
|
97
114
|
const next = recordAnalysis(state, {
|
|
98
115
|
analysis,
|
|
99
116
|
outcome: args.outcome as TrackerOutcome,
|
|
@@ -113,6 +130,41 @@ function main() {
|
|
|
113
130
|
);
|
|
114
131
|
}
|
|
115
132
|
|
|
133
|
+
function verifyParityPr(
|
|
134
|
+
repo: string,
|
|
135
|
+
prUrl: string,
|
|
136
|
+
expectedGithubLogin: string,
|
|
137
|
+
currentTag: string,
|
|
138
|
+
): void {
|
|
139
|
+
const pullRequest = ghJson<{
|
|
140
|
+
author: { login: string };
|
|
141
|
+
body: string | null;
|
|
142
|
+
isDraft: boolean;
|
|
143
|
+
state: string;
|
|
144
|
+
}>([
|
|
145
|
+
"pr",
|
|
146
|
+
"view",
|
|
147
|
+
prUrl,
|
|
148
|
+
"--repo",
|
|
149
|
+
repo,
|
|
150
|
+
"--json",
|
|
151
|
+
"author,body,isDraft,state",
|
|
152
|
+
]);
|
|
153
|
+
if (
|
|
154
|
+
pullRequest.author.login !== expectedGithubLogin ||
|
|
155
|
+
!pullRequest.isDraft ||
|
|
156
|
+
pullRequest.state !== "OPEN"
|
|
157
|
+
) {
|
|
158
|
+
throw new Error(
|
|
159
|
+
`Codex PR must be an open draft authored by ${expectedGithubLogin} (got author=${pullRequest.author.login}, draft=${pullRequest.isDraft}, state=${pullRequest.state})`,
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
const marker = `Codex-watch: openai/codex ${currentTag}`;
|
|
163
|
+
if (!pullRequest.body?.includes(marker)) {
|
|
164
|
+
throw new Error(`Codex PR body is missing marker: ${marker}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
116
168
|
function defaultNotes(outcome: TrackerOutcome): string {
|
|
117
169
|
switch (outcome) {
|
|
118
170
|
case "recorded_noop":
|
|
@@ -128,4 +180,4 @@ function defaultNotes(outcome: TrackerOutcome): string {
|
|
|
128
180
|
}
|
|
129
181
|
}
|
|
130
182
|
|
|
131
|
-
main();
|
|
183
|
+
if (import.meta.main) main();
|