@zzzen/pyright-internal 1.2.0-dev.20260913 → 1.2.0-dev.20260920
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyzer/checker.js +2 -3
- package/dist/analyzer/checker.js.map +1 -1
- package/dist/analyzer/program.js +1 -0
- package/dist/analyzer/program.js.map +1 -1
- package/dist/analyzer/typeCacheUtils.d.ts +1 -0
- package/dist/analyzer/typeCacheUtils.js +28 -1
- package/dist/analyzer/typeCacheUtils.js.map +1 -1
- package/dist/analyzer/typeEvaluator.d.ts +1 -1
- package/dist/analyzer/typeEvaluator.js +7 -10
- package/dist/analyzer/typeEvaluator.js.map +1 -1
- package/dist/analyzer/typeEvaluatorTypes.d.ts +1 -0
- package/dist/analyzer/typedDicts.js +3 -7
- package/dist/analyzer/typedDicts.js.map +1 -1
- package/dist/analyzer/types.js +7 -0
- package/dist/analyzer/types.js.map +1 -1
- package/dist/common/configOptions.d.ts +1 -0
- package/dist/common/configOptions.js +16 -0
- package/dist/common/configOptions.js.map +1 -1
- package/dist/common/envVarUtils.js +3 -3
- package/dist/common/envVarUtils.js.map +1 -1
- package/dist/tests/envVarUtils.test.js +16 -0
- package/dist/tests/envVarUtils.test.js.map +1 -1
- package/dist/tests/evaluationTrace.integration.test.d.ts +1 -0
- package/dist/tests/evaluationTrace.integration.test.js +266 -0
- package/dist/tests/evaluationTrace.integration.test.js.map +1 -0
- package/dist/tests/evaluationTrace.test.d.ts +1 -0
- package/dist/tests/evaluationTrace.test.js +422 -0
- package/dist/tests/evaluationTrace.test.js.map +1 -0
- package/dist/tests/evaluationTrace.viewer.test.d.ts +1 -0
- package/dist/tests/evaluationTrace.viewer.test.js +529 -0
- package/dist/tests/evaluationTrace.viewer.test.js.map +1 -0
- package/dist/tests/mypyPrimerAnalysis.test.js +1193 -47
- package/dist/tests/mypyPrimerAnalysis.test.js.map +1 -1
- package/dist/tests/mypyPrimerMcp.test.d.ts +1 -0
- package/dist/tests/mypyPrimerMcp.test.js +186 -0
- package/dist/tests/mypyPrimerMcp.test.js.map +1 -0
- package/dist/tests/typeCacheUtils.test.js +15 -0
- package/dist/tests/typeCacheUtils.test.js.map +1 -1
- package/dist/tests/typeEvaluator7.test.js +4 -0
- package/dist/tests/typeEvaluator7.test.js.map +1 -1
- package/dist/trace/capture.d.ts +31 -0
- package/dist/trace/capture.js +272 -0
- package/dist/trace/capture.js.map +1 -0
- package/dist/trace/cli.d.ts +1 -0
- package/dist/trace/cli.js +196 -0
- package/dist/trace/cli.js.map +1 -0
- package/dist/trace/nativeAdapter.d.ts +16 -0
- package/dist/trace/nativeAdapter.js +422 -0
- package/dist/trace/nativeAdapter.js.map +1 -0
- package/dist/trace/nativeBridge.d.ts +46 -0
- package/dist/trace/nativeBridge.js +416 -0
- package/dist/trace/nativeBridge.js.map +1 -0
- package/dist/trace/reader.d.ts +83 -0
- package/dist/trace/reader.js +296 -0
- package/dist/trace/reader.js.map +1 -0
- package/dist/trace/recorder.d.ts +59 -0
- package/dist/trace/recorder.js +303 -0
- package/dist/trace/recorder.js.map +1 -0
- package/dist/trace/runtimeLoader.d.ts +23 -0
- package/dist/trace/runtimeLoader.js +172 -0
- package/dist/trace/runtimeLoader.js.map +1 -0
- package/dist/trace/viewerAssets.d.ts +2 -0
- package/dist/trace/viewerAssets.js +76 -0
- package/dist/trace/viewerAssets.js.map +1 -0
- package/dist/trace/viewerClient.d.ts +1 -0
- package/dist/trace/viewerClient.js +485 -0
- package/dist/trace/viewerClient.js.map +1 -0
- package/dist/trace/viewerModel.d.ts +308 -0
- package/dist/trace/viewerModel.js +524 -0
- package/dist/trace/viewerModel.js.map +1 -0
- package/dist/trace/viewerServer.d.ts +7 -0
- package/dist/trace/viewerServer.js +185 -0
- package/dist/trace/viewerServer.js.map +1 -0
- package/package.json +4 -2
|
@@ -8,6 +8,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
8
8
|
const fs_1 = require("fs");
|
|
9
9
|
const os_1 = require("os");
|
|
10
10
|
const path_1 = require("path");
|
|
11
|
+
const vm_1 = require("vm");
|
|
11
12
|
const yaml_1 = require("yaml");
|
|
12
13
|
const mypyPrimerAnalysis_1 = require("../../../../build/ci/mypyPrimerAnalysis");
|
|
13
14
|
const repository = 'microsoft/pyright';
|
|
@@ -69,6 +70,16 @@ const expectedDetailOnlyProject = {
|
|
|
69
70
|
],
|
|
70
71
|
};
|
|
71
72
|
const expectedDetailOnlySummary = {
|
|
73
|
+
regressionSignals: {
|
|
74
|
+
example: [
|
|
75
|
+
{
|
|
76
|
+
kind: 'gradual-detail',
|
|
77
|
+
count: 1,
|
|
78
|
+
examplesAdded: [expectedDetailOnlyProject.detailsAdded[0].text],
|
|
79
|
+
examplesRemoved: expectedDetailOnlyProject.detailsRemoved.map((entry) => entry.text),
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
},
|
|
72
83
|
projects: [
|
|
73
84
|
{
|
|
74
85
|
name: 'example',
|
|
@@ -102,13 +113,16 @@ const expectedDetailOnlySummary = {
|
|
|
102
113
|
},
|
|
103
114
|
},
|
|
104
115
|
};
|
|
105
|
-
const
|
|
116
|
+
const expectedReportProvenance = [
|
|
106
117
|
`<!-- pyright-primer-analysis:${headSha}:42:1 -->`,
|
|
107
118
|
'## mypy_primer analysis',
|
|
108
119
|
'',
|
|
109
120
|
'Advisory AI analysis of [primer run 42, attempt 1](https://github.com/microsoft/pyright/actions/runs/42)',
|
|
110
121
|
`for commit \`${headSha}\`. This is not an approval or proof of correctness.`,
|
|
111
122
|
'',
|
|
123
|
+
];
|
|
124
|
+
const expectedReportHeading = [
|
|
125
|
+
...expectedReportProvenance,
|
|
112
126
|
'Added/removed counts are diagnostic headers; message rewrites can appear on both sides.',
|
|
113
127
|
'Detail lines are counted separately, without assuming a location or association with nearby headers.',
|
|
114
128
|
'',
|
|
@@ -117,11 +131,21 @@ const expectedReportHeading = [
|
|
|
117
131
|
];
|
|
118
132
|
const expectedReportRow = '| example | 1 | 1 | 1 / 1 | Needs human review | low | The argument error changed its wording. |';
|
|
119
133
|
const expectedReportDetails = [
|
|
120
|
-
'### example',
|
|
134
|
+
'### example: The argument error changed its wording.',
|
|
135
|
+
'',
|
|
136
|
+
'**AI assessment:** Needs human review (low confidence). **PR attribution:** Not established.',
|
|
137
|
+
'',
|
|
138
|
+
'**Before / baseline evidence:** An argument error was reported.',
|
|
139
|
+
'',
|
|
140
|
+
'**After / PR evidence:** An argument error remains.',
|
|
141
|
+
'',
|
|
142
|
+
'**Why this matters:** No changed checking has been established.',
|
|
143
|
+
'',
|
|
144
|
+
'**Causal analysis:**',
|
|
121
145
|
'',
|
|
122
146
|
'The error remains at the same location. Its cause has not been established.',
|
|
123
147
|
'',
|
|
124
|
-
'**
|
|
148
|
+
'**Uncertainty / next check:** Exact dependency revision unavailable.',
|
|
125
149
|
'',
|
|
126
150
|
'',
|
|
127
151
|
];
|
|
@@ -130,36 +154,55 @@ const expectedReportFooter = [
|
|
|
130
154
|
'[this analysis run](https://github.com/microsoft/pyright/actions/runs/100).',
|
|
131
155
|
'The original raw primer comment is unchanged. Treat uncertainty and possible regressions as requests for human review.',
|
|
132
156
|
];
|
|
157
|
+
const incompleteOverview = '**Regression analysis incomplete.** No concrete regression has been established; unresolved investigations follow.';
|
|
158
|
+
const riskOverview = '**Regression review required.** Potential regressions or recorded warning signals are listed first; causation must be established separately.';
|
|
133
159
|
const expectedReport = {
|
|
134
160
|
body: [
|
|
135
|
-
...
|
|
136
|
-
|
|
161
|
+
...expectedReportProvenance,
|
|
162
|
+
incompleteOverview,
|
|
137
163
|
'',
|
|
138
|
-
'
|
|
139
|
-
'<summary>Evidence and limitations by project</summary>',
|
|
164
|
+
'## Unresolved investigations',
|
|
140
165
|
'',
|
|
141
166
|
...expectedReportDetails,
|
|
142
|
-
'</details>',
|
|
143
|
-
'',
|
|
144
167
|
...expectedReportFooter,
|
|
145
168
|
].join('\n'),
|
|
146
169
|
fullReport: [...expectedReportHeading, expectedReportRow, '', ...expectedReportDetails].join('\n'),
|
|
147
170
|
};
|
|
148
|
-
function
|
|
171
|
+
function expectedLargeReport(projectCount, explanation) {
|
|
172
|
+
const escapedNames = Array.from({ length: projectCount }, (_, index) => `project\\_${index}`);
|
|
173
|
+
const rows = escapedNames.map((name) => expectedReportRow.replace('example', name));
|
|
174
|
+
const details = escapedNames.flatMap((name) => expectedReportDetails.map((line) => line
|
|
175
|
+
.replace('### example:', `### ${name}:`)
|
|
176
|
+
.replace('The error remains at the same location. Its cause has not been established.', explanation)));
|
|
177
|
+
return {
|
|
178
|
+
body: [
|
|
179
|
+
...expectedReportProvenance,
|
|
180
|
+
incompleteOverview,
|
|
181
|
+
'',
|
|
182
|
+
'Full causal analyses are available in the report artifact.',
|
|
183
|
+
'',
|
|
184
|
+
...escapedNames.map((name) => `- **${name}: Needs human review.** The argument error changed its wording. PR attribution: Not established.`),
|
|
185
|
+
'',
|
|
186
|
+
...expectedReportFooter,
|
|
187
|
+
].join('\n'),
|
|
188
|
+
fullReport: [...expectedReportHeading, ...rows, '', ...details].join('\n'),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
function fixture(contents = sample, headRepository = repository) {
|
|
149
192
|
const source = {
|
|
150
193
|
runId: 42,
|
|
151
194
|
runAttempt: 1,
|
|
152
195
|
headSha,
|
|
153
|
-
headRepository
|
|
196
|
+
headRepository,
|
|
154
197
|
headBranch: 'feature',
|
|
155
|
-
pullRequests: [7],
|
|
198
|
+
pullRequests: headRepository === repository ? [7] : [],
|
|
156
199
|
};
|
|
157
200
|
const manifest = { repository, source, prNumber: 7, ...(0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(contents, 0)) };
|
|
158
201
|
const pr = {
|
|
159
202
|
number: 7,
|
|
160
203
|
state: 'open',
|
|
161
204
|
base: { repo: { full_name: repository } },
|
|
162
|
-
head: { sha: headSha, ref: 'feature', repo: { full_name:
|
|
205
|
+
head: { sha: headSha, ref: 'feature', repo: { full_name: headRepository } },
|
|
163
206
|
};
|
|
164
207
|
const run = {
|
|
165
208
|
id: 42,
|
|
@@ -169,9 +212,9 @@ function fixture(contents = sample) {
|
|
|
169
212
|
conclusion: 'success',
|
|
170
213
|
repository: { full_name: repository },
|
|
171
214
|
head_sha: headSha,
|
|
172
|
-
head_repository: { full_name:
|
|
215
|
+
head_repository: { full_name: headRepository },
|
|
173
216
|
head_branch: 'feature',
|
|
174
|
-
pull_requests:
|
|
217
|
+
pull_requests: source.pullRequests.map((number) => ({ number })),
|
|
175
218
|
run_started_at: '2026-09-10T00:00:00Z',
|
|
176
219
|
};
|
|
177
220
|
const artifacts = [
|
|
@@ -191,6 +234,9 @@ function fixture(contents = sample) {
|
|
|
191
234
|
return { data: pr };
|
|
192
235
|
}
|
|
193
236
|
if (route === 'GET /repos/{owner}/{repo}/commits/{commit_sha}/pulls') {
|
|
237
|
+
return { data: [] };
|
|
238
|
+
}
|
|
239
|
+
if (route === 'GET /repos/{owner}/{repo}/pulls') {
|
|
194
240
|
return { data: associated };
|
|
195
241
|
}
|
|
196
242
|
if (route === 'GET /repos/{owner}/{repo}/issues/{issue_number}/comments') {
|
|
@@ -207,14 +253,23 @@ function fixture(contents = sample) {
|
|
|
207
253
|
name: 'example',
|
|
208
254
|
assessment: 'needs-review',
|
|
209
255
|
confidence: 'low',
|
|
256
|
+
attribution: 'unclear',
|
|
210
257
|
summary: 'The argument error changed its wording.',
|
|
258
|
+
before: 'An argument error was reported.',
|
|
259
|
+
after: 'An argument error remains.',
|
|
260
|
+
impact: 'No changed checking has been established.',
|
|
211
261
|
explanation: 'The error remains at the same location. Its cause has not been established.',
|
|
212
262
|
unresolved: 'Exact dependency revision unavailable.',
|
|
213
263
|
evidence: [],
|
|
214
264
|
},
|
|
215
265
|
],
|
|
216
266
|
};
|
|
217
|
-
const output = () => ({
|
|
267
|
+
const output = (perProject = false) => ({
|
|
268
|
+
items: (perProject ? report.projects.map((project) => ({ projects: [project] })) : [report]).map((report) => ({
|
|
269
|
+
type: 'publish_primer_analysis',
|
|
270
|
+
report: JSON.stringify(report),
|
|
271
|
+
})),
|
|
272
|
+
});
|
|
218
273
|
return { source, manifest, pr, run, artifacts, comments, associated, request, report, output };
|
|
219
274
|
}
|
|
220
275
|
describe('mypy_primer analysis', () => {
|
|
@@ -384,6 +439,57 @@ describe('mypy_primer analysis', () => {
|
|
|
384
439
|
f.pr.head.sha = 'b'.repeat(40);
|
|
385
440
|
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).resolves.toBeUndefined();
|
|
386
441
|
});
|
|
442
|
+
test.each([repository, 'contributor/pyright'])('previews a verified closed PR from %s', async (headRepository) => {
|
|
443
|
+
writeShards();
|
|
444
|
+
const f = fixture(sample, headRepository);
|
|
445
|
+
f.pr.state = 'closed';
|
|
446
|
+
f.source.pullRequests = [];
|
|
447
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).resolves.toBeUndefined();
|
|
448
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory, true)).resolves.toStrictEqual({
|
|
449
|
+
...f.manifest,
|
|
450
|
+
preview: true,
|
|
451
|
+
});
|
|
452
|
+
expect(f.request.mock.calls.filter(([route]) => route === 'GET /repos/{owner}/{repo}/pulls')).toStrictEqual([
|
|
453
|
+
[
|
|
454
|
+
'GET /repos/{owner}/{repo}/pulls',
|
|
455
|
+
{
|
|
456
|
+
owner: 'microsoft',
|
|
457
|
+
repo: 'pyright',
|
|
458
|
+
state: 'all',
|
|
459
|
+
head: `${headRepository.split('/')[0]}:feature`,
|
|
460
|
+
per_page: 100,
|
|
461
|
+
page: 1,
|
|
462
|
+
},
|
|
463
|
+
],
|
|
464
|
+
]);
|
|
465
|
+
});
|
|
466
|
+
test('previews reject ambiguous closed-PR associations and superseded heads', async () => {
|
|
467
|
+
writeShards();
|
|
468
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
469
|
+
f.pr.state = 'closed';
|
|
470
|
+
f.associated.push({ ...f.pr, number: 8 });
|
|
471
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory, true)).rejects.toThrow('found 2');
|
|
472
|
+
f.pr.head.sha = 'b'.repeat(40);
|
|
473
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory, true)).resolves.toBeUndefined();
|
|
474
|
+
});
|
|
475
|
+
test.each(['repository', 'branch', 'number', 'shard'])('previews retain %s validation', async (mismatch) => {
|
|
476
|
+
writeShards();
|
|
477
|
+
const f = fixture();
|
|
478
|
+
f.pr.state = 'closed';
|
|
479
|
+
if (mismatch === 'repository') {
|
|
480
|
+
f.pr.head.repo.full_name = 'someone/else';
|
|
481
|
+
}
|
|
482
|
+
else if (mismatch === 'branch') {
|
|
483
|
+
f.pr.head.ref = 'different';
|
|
484
|
+
}
|
|
485
|
+
else if (mismatch === 'number') {
|
|
486
|
+
f.source.pullRequests = [99];
|
|
487
|
+
}
|
|
488
|
+
else {
|
|
489
|
+
(0, fs_1.rmSync)((0, path_1.join)(directory, 'mypy_primer_diffs_7'), { recursive: true });
|
|
490
|
+
}
|
|
491
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory, true)).rejects.toThrow();
|
|
492
|
+
});
|
|
387
493
|
test('prepares and renders a detail-only project without treating zero headers as no changes', async () => {
|
|
388
494
|
writeShards(detailOnlySample);
|
|
389
495
|
const f = fixture(detailOnlySample);
|
|
@@ -398,6 +504,7 @@ describe('mypy_primer analysis', () => {
|
|
|
398
504
|
throw new Error('Preparation unexpectedly skipped the detail-only project');
|
|
399
505
|
}
|
|
400
506
|
f.report.projects[0] = {
|
|
507
|
+
...f.report.projects[0],
|
|
401
508
|
name: 'example',
|
|
402
509
|
assessment: 'needs-review',
|
|
403
510
|
confidence: 'low',
|
|
@@ -407,26 +514,36 @@ describe('mypy_primer analysis', () => {
|
|
|
407
514
|
evidence: [],
|
|
408
515
|
};
|
|
409
516
|
const expectedDetails = [
|
|
410
|
-
'### example',
|
|
517
|
+
'### example: Only diagnostic details changed.',
|
|
518
|
+
'',
|
|
519
|
+
'**Regression warning signals require review, regardless of the AI assessment:**',
|
|
520
|
+
'- Changed diagnostic details introduce Any/Unknown.',
|
|
521
|
+
'',
|
|
522
|
+
'**AI assessment:** Needs human review (low confidence). **PR attribution:** Not established.',
|
|
523
|
+
'',
|
|
524
|
+
'**Before / baseline evidence:** An argument error was reported.',
|
|
525
|
+
'',
|
|
526
|
+
'**After / PR evidence:** An argument error remains.',
|
|
527
|
+
'',
|
|
528
|
+
'**Why this matters:** No changed checking has been established.',
|
|
529
|
+
'',
|
|
530
|
+
'**Causal analysis:**',
|
|
411
531
|
'',
|
|
412
532
|
'The return type changed from Match\\[str\\] \\| None to Unknown.',
|
|
413
533
|
'',
|
|
414
|
-
'**
|
|
534
|
+
'**Uncertainty / next check:** Diagnostic headers and locations are absent from the concise diff.',
|
|
415
535
|
'',
|
|
416
536
|
'',
|
|
417
537
|
];
|
|
418
538
|
const expectedRow = '| example | 0 | 0 | 2 / 2 | Needs human review | low | Only diagnostic details changed. |';
|
|
419
539
|
const expected = {
|
|
420
540
|
body: [
|
|
421
|
-
...
|
|
422
|
-
|
|
541
|
+
...expectedReportProvenance,
|
|
542
|
+
riskOverview,
|
|
423
543
|
'',
|
|
424
|
-
'
|
|
425
|
-
'<summary>Evidence and limitations by project</summary>',
|
|
544
|
+
'## Potential regressions',
|
|
426
545
|
'',
|
|
427
546
|
...expectedDetails,
|
|
428
|
-
'</details>',
|
|
429
|
-
'',
|
|
430
547
|
...expectedReportFooter,
|
|
431
548
|
].join('\n'),
|
|
432
549
|
fullReport: [...expectedReportHeading, expectedRow, '', ...expectedDetails].join('\n'),
|
|
@@ -446,15 +563,96 @@ describe('mypy_primer analysis', () => {
|
|
|
446
563
|
(0, fs_1.writeFileSync)((0, path_1.join)(directory, 'mypy_primer_diffs_pr_number', 'pr_number.txt'), '7; echo unsafe');
|
|
447
564
|
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).rejects.toThrow('Invalid PR number');
|
|
448
565
|
});
|
|
449
|
-
test('
|
|
566
|
+
test('associates a fork by its head branch when GitHub omits commit and run PR associations', async () => {
|
|
450
567
|
writeShards();
|
|
451
|
-
const f = fixture();
|
|
452
|
-
f.source.
|
|
453
|
-
|
|
454
|
-
f.
|
|
455
|
-
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository,
|
|
568
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
569
|
+
f.source.headBranch = f.run.head_branch = f.pr.head.ref = 'fix/feature';
|
|
570
|
+
const source = await (0, mypyPrimerAnalysis_1.loadSource)(f.request, repository, f.source);
|
|
571
|
+
expect(source).toStrictEqual(f.source);
|
|
572
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, source, directory)).resolves.toStrictEqual(f.manifest);
|
|
573
|
+
expect(f.request).toHaveBeenCalledWith('GET /repos/{owner}/{repo}/pulls', {
|
|
574
|
+
owner: 'microsoft',
|
|
575
|
+
repo: 'pyright',
|
|
576
|
+
state: 'open',
|
|
577
|
+
head: 'contributor:fix/feature',
|
|
578
|
+
per_page: 100,
|
|
579
|
+
page: 1,
|
|
580
|
+
});
|
|
581
|
+
expect(f.request.mock.calls.some(([route]) => route.includes('/commits/'))).toBe(false);
|
|
582
|
+
});
|
|
583
|
+
test('rejects missing or ambiguous fork associations', async () => {
|
|
584
|
+
writeShards();
|
|
585
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
586
|
+
f.associated.length = 0;
|
|
587
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).rejects.toThrow('found 0');
|
|
588
|
+
f.associated.push(f.pr);
|
|
456
589
|
f.associated.push({ ...f.pr, number: 8 });
|
|
457
|
-
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).rejects.toThrow('
|
|
590
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).rejects.toThrow('found 2');
|
|
591
|
+
});
|
|
592
|
+
test('filters fork candidates by state, head SHA, branch, head repository, and base repository', async () => {
|
|
593
|
+
writeShards();
|
|
594
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
595
|
+
const other = { ...f.pr, number: 8 };
|
|
596
|
+
f.associated.push({ ...other, state: 'closed' }, { ...other, head: { ...other.head, sha: 'b'.repeat(40) } }, { ...other, head: { ...other.head, ref: 'different' } }, { ...other, head: { ...other.head, repo: { full_name: 'someone/pyright' } } }, { ...other, head: { ...other.head, repo: { full_name: 'contributor/other' } } }, { ...other, base: { repo: { full_name: 'someone/else' } } });
|
|
597
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).resolves.toStrictEqual(f.manifest);
|
|
598
|
+
});
|
|
599
|
+
test('does not trust a fork PR number artifact without a matching branch association', async () => {
|
|
600
|
+
writeShards();
|
|
601
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
602
|
+
f.associated.splice(0, 1, { ...f.pr, number: 8 });
|
|
603
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).rejects.toThrow('not associated');
|
|
604
|
+
});
|
|
605
|
+
test('paginates fork branch associations before requiring a unique match', async () => {
|
|
606
|
+
writeShards();
|
|
607
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
608
|
+
const original = f.request.getMockImplementation();
|
|
609
|
+
f.request.mockImplementation(async (route, parameters) => {
|
|
610
|
+
if (route === 'GET /repos/{owner}/{repo}/pulls' && parameters.page === 1) {
|
|
611
|
+
return { data: Array.from({ length: 100 }, () => ({ ...f.pr, state: 'closed' })) };
|
|
612
|
+
}
|
|
613
|
+
return original(route, parameters);
|
|
614
|
+
});
|
|
615
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).resolves.toStrictEqual(f.manifest);
|
|
616
|
+
expect(f.request.mock.calls
|
|
617
|
+
.filter(([route]) => route === 'GET /repos/{owner}/{repo}/pulls')
|
|
618
|
+
.map(([, parameters]) => parameters.page)).toStrictEqual([1, 2]);
|
|
619
|
+
});
|
|
620
|
+
test.each(['closed', 'superseded'])('skips a %s fork PR without looking up associations', async (state) => {
|
|
621
|
+
writeShards();
|
|
622
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
623
|
+
if (state === 'closed') {
|
|
624
|
+
f.pr.state = 'closed';
|
|
625
|
+
}
|
|
626
|
+
else {
|
|
627
|
+
f.pr.head.sha = 'b'.repeat(40);
|
|
628
|
+
}
|
|
629
|
+
await expect((0, mypyPrimerAnalysis_1.prepareAnalysis)(f.request, repository, f.source, directory)).resolves.toBeUndefined();
|
|
630
|
+
expect(f.request.mock.calls.map(([route]) => route)).toStrictEqual([
|
|
631
|
+
'GET /repos/{owner}/{repo}/pulls/{pull_number}',
|
|
632
|
+
]);
|
|
633
|
+
});
|
|
634
|
+
test('revalidates fork branch associations before publishing', async () => {
|
|
635
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
636
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, f.output(), 100, (0, path_1.join)(directory, 'report.md'), false)).resolves.toBe('Published: advisory analysis of every changed project');
|
|
637
|
+
expect(f.request.mock.calls.filter(([route]) => route === 'GET /repos/{owner}/{repo}/pulls')).toHaveLength(2);
|
|
638
|
+
expect(f.request.mock.calls.filter(([route]) => /^(POST|PATCH) /.test(route))).toStrictEqual([
|
|
639
|
+
[
|
|
640
|
+
'POST /repos/{owner}/{repo}/issues/{issue_number}/comments',
|
|
641
|
+
{ owner: 'microsoft', repo: 'pyright', issue_number: 7, body: expectedReport.body },
|
|
642
|
+
],
|
|
643
|
+
]);
|
|
644
|
+
});
|
|
645
|
+
test('refuses to publish when a fork branch association becomes ambiguous', async () => {
|
|
646
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
647
|
+
const original = f.request.getMockImplementation();
|
|
648
|
+
f.request.mockImplementation(async (route, parameters) => {
|
|
649
|
+
if (route === 'GET /repos/{owner}/{repo}/issues/{issue_number}/comments') {
|
|
650
|
+
f.associated.push({ ...f.pr, number: 8 });
|
|
651
|
+
}
|
|
652
|
+
return original(route, parameters);
|
|
653
|
+
});
|
|
654
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, f.output(), 100, (0, path_1.join)(directory, 'report.md'), false)).rejects.toThrow('found 2');
|
|
655
|
+
expect(f.request.mock.calls.some(([route]) => /^(POST|PATCH) /.test(route))).toBe(false);
|
|
458
656
|
});
|
|
459
657
|
test('rejects duplicate project sections and unexpected shard files', async () => {
|
|
460
658
|
writeShards(sample + '\n' + sample);
|
|
@@ -468,6 +666,386 @@ describe('mypy_primer analysis', () => {
|
|
|
468
666
|
const result = (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100);
|
|
469
667
|
expect(result).toStrictEqual(expectedReport);
|
|
470
668
|
});
|
|
669
|
+
test('extracts type-erasure and other assertion signals without inferring baseline reveals', () => {
|
|
670
|
+
const erased = [
|
|
671
|
+
'.../projects/pandas-stubs/test.py:1:1 - error: "assert_type" mismatch: expected "Index[Any]" but received "Any" (reportAssertTypeFailure)',
|
|
672
|
+
'.../projects/pandas-stubs/test.py:2:1 - error: "assert_type" mismatch: expected "Rotation[tuple[Any, ...]]" but received "Unknown" (reportAssertTypeFailure)',
|
|
673
|
+
];
|
|
674
|
+
const assertion = '.../projects/pandas-stubs/test.py:3:1 - error: "assert_type" mismatch: expected "int" but received "Literal[1]" (reportAssertTypeFailure)';
|
|
675
|
+
const diff = [
|
|
676
|
+
'pandas-stubs (https://github.com/pandas-dev/pandas-stubs)',
|
|
677
|
+
...[...erased, assertion].map((line) => `+ ${line}`),
|
|
678
|
+
'- 0 errors, 0 warnings, 0 informations',
|
|
679
|
+
'+ 3 errors, 0 warnings, 0 informations',
|
|
680
|
+
].join('\n');
|
|
681
|
+
expect((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(diff, 0)).regressionSignals).toStrictEqual({
|
|
682
|
+
'pandas-stubs': [
|
|
683
|
+
{ kind: 'type-erasure', count: 2, examplesAdded: erased, examplesRemoved: [] },
|
|
684
|
+
{ kind: 'assertion-failure', count: 1, examplesAdded: [assertion], examplesRemoved: [] },
|
|
685
|
+
],
|
|
686
|
+
});
|
|
687
|
+
});
|
|
688
|
+
test('flags removed checks but not replacements or unrelated lint removals', () => {
|
|
689
|
+
const removed = '.../projects/example/test.py:2:1 - error: Missing member (reportAttributeAccessIssue)';
|
|
690
|
+
const diff = [
|
|
691
|
+
'example (https://github.com/example/project)',
|
|
692
|
+
'- .../projects/example/test.py:1:1 - error: Old argument error (reportArgumentType)',
|
|
693
|
+
'+ .../projects/example/test.py:1:1 - error: Reworded argument error (reportArgumentType)',
|
|
694
|
+
`- ${removed}`,
|
|
695
|
+
'+ .../projects/example/test.py:2:1 - error: Different argument error (reportArgumentType)',
|
|
696
|
+
'- .../projects/example/test.py:3:1 - warning: Unused import (reportUnusedImport)',
|
|
697
|
+
'- 2 errors, 1 warning, 0 informations',
|
|
698
|
+
'+ 2 errors, 0 warnings, 0 informations',
|
|
699
|
+
].join('\n');
|
|
700
|
+
expect((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(diff, 0)).regressionSignals).toStrictEqual({
|
|
701
|
+
example: [{ kind: 'removed-check', count: 1, examplesAdded: [], examplesRemoved: [removed] }],
|
|
702
|
+
});
|
|
703
|
+
expect((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(sample, 0)).regressionSignals).toBeUndefined();
|
|
704
|
+
});
|
|
705
|
+
test('detects new Unknown returns despite pre-existing Unknown parameters', () => {
|
|
706
|
+
const before = ' Return type mismatch: override returns type "(value: Unknown) -> int"';
|
|
707
|
+
const after = ' Return type mismatch: override returns type "(value: Unknown) -> Unknown"';
|
|
708
|
+
const diff = ['pip (https://github.com/pypa/pip)', `-${before}`, `+${after}`].join('\n');
|
|
709
|
+
expect((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(diff, 0)).regressionSignals).toStrictEqual({
|
|
710
|
+
pip: [{ kind: 'gradual-detail', count: 1, examplesAdded: [after], examplesRemoved: [before] }],
|
|
711
|
+
});
|
|
712
|
+
});
|
|
713
|
+
test('does not flag a gradual type merely moving from a header into a detail', () => {
|
|
714
|
+
const diff = [
|
|
715
|
+
'example (https://github.com/example/project)',
|
|
716
|
+
'- .../projects/example/test.py:1:1 - error: Argument of type Unknown is incompatible (reportArgumentType)',
|
|
717
|
+
'+ .../projects/example/test.py:1:1 - error: Argument is incompatible (reportArgumentType)',
|
|
718
|
+
'+ Type Unknown is incompatible (reportArgumentType)',
|
|
719
|
+
].join('\n');
|
|
720
|
+
expect((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(diff, 0)).regressionSignals).toBeUndefined();
|
|
721
|
+
});
|
|
722
|
+
test('bounds warning examples without undercounting the observed signal', () => {
|
|
723
|
+
const lines = Array.from({ length: 5 }, (_, index) => `.../projects/example/test.py:${index + 1}:1 - error: "assert_type" mismatch: expected "${'X'.repeat(800)}" but received "Any" (reportAssertTypeFailure)`);
|
|
724
|
+
const diff = [
|
|
725
|
+
'example (https://github.com/example/project)',
|
|
726
|
+
...lines.map((line) => `+ ${line}`),
|
|
727
|
+
'- 0 errors, 0 warnings, 0 informations',
|
|
728
|
+
'+ 5 errors, 0 warnings, 0 informations',
|
|
729
|
+
].join('\n');
|
|
730
|
+
expect((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(diff, 0)).regressionSignals).toStrictEqual({
|
|
731
|
+
example: [
|
|
732
|
+
{
|
|
733
|
+
kind: 'type-erasure',
|
|
734
|
+
count: 5,
|
|
735
|
+
examplesAdded: lines.slice(0, 3).map((line) => line.slice(0, 600)),
|
|
736
|
+
examplesRemoved: [],
|
|
737
|
+
},
|
|
738
|
+
],
|
|
739
|
+
});
|
|
740
|
+
});
|
|
741
|
+
test.each([
|
|
742
|
+
['__proto__', '\\_\\_proto\\_\\_'],
|
|
743
|
+
['constructor', 'constructor'],
|
|
744
|
+
['toString', 'toString'],
|
|
745
|
+
])('treats project name %s as data rather than an inherited key', (name, escapedName) => {
|
|
746
|
+
const f = fixture(sample.replace(/example/g, name));
|
|
747
|
+
f.report.projects[0].name = name;
|
|
748
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)({ ...f.manifest, regressionSignals: {} }, f.report, 100)).toStrictEqual({
|
|
749
|
+
body: expectedReport.body.replace(/example/g, escapedName),
|
|
750
|
+
fullReport: expectedReport.fullReport.replace(/example/g, escapedName),
|
|
751
|
+
});
|
|
752
|
+
const line = `.../projects/${name}/test.py:1:1 - error: "assert_type" mismatch: expected "int" but received "Any" (reportAssertTypeFailure)`;
|
|
753
|
+
const diff = [
|
|
754
|
+
`${name} (https://github.com/example/project)`,
|
|
755
|
+
`+ ${line}`,
|
|
756
|
+
'- 0 errors, 0 warnings, 0 informations',
|
|
757
|
+
'+ 1 error, 0 warnings, 0 informations',
|
|
758
|
+
].join('\n');
|
|
759
|
+
// Compare entries so Jest does not treat a constructor data key as a class identity.
|
|
760
|
+
expect(Object.entries(JSON.parse(JSON.stringify((0, mypyPrimerAnalysis_1.summarizeDiffs)((0, mypyPrimerAnalysis_1.parseDiff)(diff, 0)).regressionSignals)))).toStrictEqual([[name, [{ kind: 'type-erasure', count: 1, examplesAdded: [line], examplesRemoved: [] }]]]);
|
|
761
|
+
});
|
|
762
|
+
test('keeps type-erasure risks first even when the model blames stubs', () => {
|
|
763
|
+
const diff = [
|
|
764
|
+
sample,
|
|
765
|
+
'pandas-stubs (https://github.com/pandas-dev/pandas-stubs)',
|
|
766
|
+
'+ .../projects/pandas-stubs/test.py:1:1 - error: "assert_type" mismatch: expected "Index[Any]" but received "Any" (reportAssertTypeFailure)',
|
|
767
|
+
'- 0 errors, 0 warnings, 0 informations',
|
|
768
|
+
'+ 1 error, 0 warnings, 0 informations',
|
|
769
|
+
].join('\n');
|
|
770
|
+
const f = fixture(diff);
|
|
771
|
+
const pandas = {
|
|
772
|
+
...f.report.projects[0],
|
|
773
|
+
name: 'pandas-stubs',
|
|
774
|
+
assessment: 'exposed-typing-issue',
|
|
775
|
+
confidence: 'high',
|
|
776
|
+
summary: 'The model claims that the stub expectation is wrong.',
|
|
777
|
+
before: 'The assertion expects Index[Any]; the baseline reveal is not recorded.',
|
|
778
|
+
after: 'The new assertion receives Any.',
|
|
779
|
+
impact: 'The expected outer container type is absent from the reported result.',
|
|
780
|
+
explanation: 'Conformance intent alone does not establish that the lost type information is harmless.',
|
|
781
|
+
evidence: [
|
|
782
|
+
{
|
|
783
|
+
url: 'https://typing.python.org/en/latest/spec/overload.html',
|
|
784
|
+
detail: 'Specification context, not proof of a bad stub.',
|
|
785
|
+
},
|
|
786
|
+
],
|
|
787
|
+
};
|
|
788
|
+
const pandasDetails = [
|
|
789
|
+
'### pandas-stubs: The model claims that the stub expectation is wrong.',
|
|
790
|
+
'',
|
|
791
|
+
'**Regression warning signals require review, regardless of the AI assessment:**',
|
|
792
|
+
'- New assertions receive bare Any/Unknown instead of their expected type.',
|
|
793
|
+
'',
|
|
794
|
+
'**AI assessment:** Exposed typing issue (high confidence). **PR attribution:** Not established.',
|
|
795
|
+
'',
|
|
796
|
+
'**Before / baseline evidence:** The assertion expects Index\\[Any\\]; the baseline reveal is not recorded.',
|
|
797
|
+
'',
|
|
798
|
+
'**After / PR evidence:** The new assertion receives Any.',
|
|
799
|
+
'',
|
|
800
|
+
'**Why this matters:** The expected outer container type is absent from the reported result.',
|
|
801
|
+
'',
|
|
802
|
+
'**Causal analysis:**',
|
|
803
|
+
'',
|
|
804
|
+
'Conformance intent alone does not establish that the lost type information is harmless.',
|
|
805
|
+
'',
|
|
806
|
+
'**Uncertainty / next check:** Exact dependency revision unavailable.',
|
|
807
|
+
'',
|
|
808
|
+
'- Specification context, not proof of a bad stub.: <https://typing.python.org/en/latest/spec/overload.html>',
|
|
809
|
+
'',
|
|
810
|
+
];
|
|
811
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, { projects: [...f.report.projects, pandas] }, 100)).toStrictEqual({
|
|
812
|
+
body: [
|
|
813
|
+
...expectedReportProvenance,
|
|
814
|
+
riskOverview,
|
|
815
|
+
'',
|
|
816
|
+
'## Potential regressions',
|
|
817
|
+
'',
|
|
818
|
+
...pandasDetails,
|
|
819
|
+
'## Unresolved investigations',
|
|
820
|
+
'',
|
|
821
|
+
...expectedReportDetails,
|
|
822
|
+
...expectedReportFooter,
|
|
823
|
+
].join('\n'),
|
|
824
|
+
fullReport: [
|
|
825
|
+
...expectedReportHeading,
|
|
826
|
+
expectedReportRow,
|
|
827
|
+
'| pandas-stubs | 1 | 0 | 0 / 0 | Exposed typing issue | high | The model claims that the stub expectation is wrong. |',
|
|
828
|
+
'',
|
|
829
|
+
...pandasDetails,
|
|
830
|
+
...expectedReportDetails,
|
|
831
|
+
].join('\n'),
|
|
832
|
+
});
|
|
833
|
+
});
|
|
834
|
+
test.each([
|
|
835
|
+
`https://github.com/microsoft/pyright/blob/main/file.ts#L1`,
|
|
836
|
+
`https://github.com/microsoft/pyright/blob/${'b'.repeat(40)}/file.ts#L1`,
|
|
837
|
+
`https://github.com/microsoft/pyright/blob/${headSha}/file.ts`,
|
|
838
|
+
`https://github.com/other/repo/blob/${headSha}/file.ts#L1`,
|
|
839
|
+
'https://typing.python.org/en/latest/spec/overload.html#step-5',
|
|
840
|
+
])('rejects likely PR attribution without exact head and line evidence: %s', (url) => {
|
|
841
|
+
const f = fixture();
|
|
842
|
+
const report = {
|
|
843
|
+
projects: [
|
|
844
|
+
{
|
|
845
|
+
...f.report.projects[0],
|
|
846
|
+
attribution: 'likely-pr',
|
|
847
|
+
evidence: [{ url, detail: 'Insufficient attribution evidence.' }],
|
|
848
|
+
},
|
|
849
|
+
],
|
|
850
|
+
};
|
|
851
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toThrow(new Error('PR attribution requires a line-pinned citation at the analyzed head for example'));
|
|
852
|
+
});
|
|
853
|
+
test.each([
|
|
854
|
+
{ headRepository: repository, evidenceRepository: repository },
|
|
855
|
+
{ headRepository: 'contributor/pyright', evidenceRepository: repository },
|
|
856
|
+
{ headRepository: 'contributor/pyright', evidenceRepository: 'contributor/pyright' },
|
|
857
|
+
])('accepts $evidenceRepository evidence for a PR from $headRepository', ({ headRepository, evidenceRepository }) => {
|
|
858
|
+
const f = fixture(sample, headRepository);
|
|
859
|
+
const url = `https://github.com/${evidenceRepository}/blob/${headSha}/file.ts#L1-L5`;
|
|
860
|
+
const report = {
|
|
861
|
+
projects: [
|
|
862
|
+
{
|
|
863
|
+
...f.report.projects[0],
|
|
864
|
+
attribution: 'likely-pr',
|
|
865
|
+
evidence: [{ url, detail: 'Changed branch.' }],
|
|
866
|
+
},
|
|
867
|
+
],
|
|
868
|
+
};
|
|
869
|
+
const details = [...expectedReportDetails.slice(0, -1), `- Changed branch.: <${url}>`, '']
|
|
870
|
+
.join('\n')
|
|
871
|
+
.replace('PR attribution:** Not established.', 'PR attribution:** Likely caused by the PR.');
|
|
872
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toStrictEqual({
|
|
873
|
+
body: expectedReport.body.replace(expectedReportDetails.join('\n'), details),
|
|
874
|
+
fullReport: expectedReport.fullReport.replace(expectedReportDetails.join('\n'), details),
|
|
875
|
+
});
|
|
876
|
+
});
|
|
877
|
+
test.each([
|
|
878
|
+
`https://github.com/contributor/pyright/blob/${'b'.repeat(40)}/file.ts#L1`,
|
|
879
|
+
`https://github.com/contributor/pyright/blob/${headSha}/file.ts`,
|
|
880
|
+
`https://github.com/contributor/pyright-extra/blob/${headSha}/file.ts#L1`,
|
|
881
|
+
`https://github.com/other/pyright/blob/${headSha}/file.ts#L1`,
|
|
882
|
+
])('rejects mismatched attribution evidence for a fork PR: %s', (url) => {
|
|
883
|
+
const f = fixture(sample, 'contributor/pyright');
|
|
884
|
+
const report = {
|
|
885
|
+
projects: [
|
|
886
|
+
{
|
|
887
|
+
...f.report.projects[0],
|
|
888
|
+
attribution: 'likely-pr',
|
|
889
|
+
evidence: [{ url, detail: 'Insufficient attribution evidence.' }],
|
|
890
|
+
},
|
|
891
|
+
],
|
|
892
|
+
};
|
|
893
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toThrow(new Error('PR attribution requires a line-pinned citation at the analyzed head for example'));
|
|
894
|
+
});
|
|
895
|
+
test.each(['before', 'after', 'impact', 'attribution'])('requires the %s investigation field', (field) => {
|
|
896
|
+
const f = fixture();
|
|
897
|
+
const report = { projects: [{ ...f.report.projects[0], [field]: undefined }] };
|
|
898
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toThrow(new Error(`Expected nonempty text of at most ${field === 'attribution' ? 2000 : 1200} characters`));
|
|
899
|
+
});
|
|
900
|
+
test('rejects unsupported PR attribution classifications', () => {
|
|
901
|
+
const f = fixture();
|
|
902
|
+
f.report.projects[0].attribution = 'definitely-caused-by-pr';
|
|
903
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100)).toThrow(new Error('Invalid PR attribution for example'));
|
|
904
|
+
});
|
|
905
|
+
test('uses a short non-blocking notice only for canonical SymPy-only changes', () => {
|
|
906
|
+
const f = fixture(sample
|
|
907
|
+
.replace(/example/g, 'sympy')
|
|
908
|
+
.replace('https://github.com/sympy/project', 'https://github.com/sympy/sympy'));
|
|
909
|
+
f.report.projects[0].name = 'sympy';
|
|
910
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100)).toStrictEqual({
|
|
911
|
+
body: [
|
|
912
|
+
...expectedReportProvenance,
|
|
913
|
+
'**Only SymPy changed.** These differences are treated as non-blocking primer noise; no other project changed.',
|
|
914
|
+
'',
|
|
915
|
+
'Recorded changes: 1 added / 1 removed diagnostic headers; 1 added / 1 removed detail lines.',
|
|
916
|
+
'',
|
|
917
|
+
...expectedReportFooter,
|
|
918
|
+
].join('\n'),
|
|
919
|
+
fullReport: expectedReport.fullReport.replace(/example/g, 'sympy'),
|
|
920
|
+
});
|
|
921
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, { projects: [] }, 100)).toThrow('The analysis must cover every changed project exactly once');
|
|
922
|
+
f.report.projects[0].explanation = '';
|
|
923
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100)).toThrow('Expected nonempty text');
|
|
924
|
+
});
|
|
925
|
+
test('keeps normal analysis for noncanonical SymPy and mixed-project changes', () => {
|
|
926
|
+
const f = fixture();
|
|
927
|
+
const sympy = { ...f.report.projects[0], name: 'sympy' };
|
|
928
|
+
const manifest = { ...f.manifest, projects: [{ ...f.manifest.projects[0], name: 'sympy' }] };
|
|
929
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(manifest, { projects: [sympy] }, 100)).toStrictEqual({
|
|
930
|
+
body: expectedReport.body.replace(/example/g, 'sympy'),
|
|
931
|
+
fullReport: expectedReport.fullReport.replace(/example/g, 'sympy'),
|
|
932
|
+
});
|
|
933
|
+
manifest.projects[0].url = 'https://github.com/sympy/sympy';
|
|
934
|
+
manifest.projects.push(f.manifest.projects[0]);
|
|
935
|
+
const rows = [expectedReportRow.replace('example', 'sympy'), expectedReportRow];
|
|
936
|
+
const details = [
|
|
937
|
+
...expectedReportDetails.map((line) => line.replace('example', 'sympy')),
|
|
938
|
+
...expectedReportDetails,
|
|
939
|
+
];
|
|
940
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(manifest, { projects: [sympy, ...f.report.projects] }, 100)).toStrictEqual({
|
|
941
|
+
body: [
|
|
942
|
+
...expectedReportProvenance,
|
|
943
|
+
incompleteOverview,
|
|
944
|
+
'',
|
|
945
|
+
'## Unresolved investigations',
|
|
946
|
+
'',
|
|
947
|
+
...details,
|
|
948
|
+
...expectedReportFooter,
|
|
949
|
+
].join('\n'),
|
|
950
|
+
fullReport: [...expectedReportHeading, ...rows, '', ...details].join('\n'),
|
|
951
|
+
});
|
|
952
|
+
});
|
|
953
|
+
test('labels preview reports and refuses to send them through the publisher', async () => {
|
|
954
|
+
const f = fixture();
|
|
955
|
+
const manifest = { ...f.manifest, preview: true };
|
|
956
|
+
const asPreview = (content) => content
|
|
957
|
+
.replace('<!-- pyright-primer-analysis:', '<!-- pyright-primer-analysis-preview:')
|
|
958
|
+
.replace('## mypy_primer analysis\n\n', '## mypy_primer analysis preview\n\n**Preview only. No PR comment was posted.**\n\n');
|
|
959
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(manifest, f.report, 100)).toStrictEqual({
|
|
960
|
+
body: asPreview(expectedReport.body),
|
|
961
|
+
fullReport: asPreview(expectedReport.fullReport),
|
|
962
|
+
});
|
|
963
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, manifest, f.output(), 100, (0, path_1.join)(directory, 'report.md'), false)).rejects.toThrow('Preview analyses cannot publish PR comments');
|
|
964
|
+
expect(f.request.mock.calls).toStrictEqual([]);
|
|
965
|
+
});
|
|
966
|
+
test.each([1799, 1800, 1801, 2143])('previews a %i-character explanation without losing the full report', (length) => {
|
|
967
|
+
const f = fixture();
|
|
968
|
+
const original = f.report.projects[0].explanation;
|
|
969
|
+
const explanation = 'x'.repeat(length);
|
|
970
|
+
f.report.projects[0].explanation = explanation;
|
|
971
|
+
const notice = length > 1800 ? '\n\n**Explanation truncated; see the full report artifact linked below.**' : '';
|
|
972
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100)).toStrictEqual({
|
|
973
|
+
body: expectedReport.body.replace(original, 'x'.repeat(Math.min(length, 1800)) + notice),
|
|
974
|
+
fullReport: expectedReport.fullReport.replace(original, explanation),
|
|
975
|
+
});
|
|
976
|
+
});
|
|
977
|
+
test.each([
|
|
978
|
+
['@', '@'],
|
|
979
|
+
['|', '\\|'],
|
|
980
|
+
['\\', '\\\\'],
|
|
981
|
+
['\u{1f642}', '\u{1f642}'],
|
|
982
|
+
])('truncates before escaping and preserves the last Unicode code point: %s', (character, escaped) => {
|
|
983
|
+
const f = fixture();
|
|
984
|
+
const original = f.report.projects[0].explanation;
|
|
985
|
+
f.report.projects[0].explanation = '@'.repeat(1799) + character + ' Omitted qualification.';
|
|
986
|
+
const preview = '@'.repeat(1799) + escaped;
|
|
987
|
+
expect((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100)).toStrictEqual({
|
|
988
|
+
body: expectedReport.body.replace(original, preview + '\n\n**Explanation truncated; see the full report artifact linked below.**'),
|
|
989
|
+
fullReport: expectedReport.fullReport.replace(original, preview + ' Omitted qualification.'),
|
|
990
|
+
});
|
|
991
|
+
});
|
|
992
|
+
test.each([undefined, null, 42, '', ' \n '])('rejects an invalid explanation: %s', (explanation) => {
|
|
993
|
+
const f = fixture();
|
|
994
|
+
const report = { projects: [{ ...f.report.projects[0], explanation }] };
|
|
995
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toThrow('Expected nonempty text');
|
|
996
|
+
});
|
|
997
|
+
test('keeps other field limits and the overall input bound', async () => {
|
|
998
|
+
const f = fixture();
|
|
999
|
+
for (const [field, value] of [
|
|
1000
|
+
['summary', 'x'.repeat(241)],
|
|
1001
|
+
['unresolved', 'x'.repeat(801)],
|
|
1002
|
+
['explanation', 'x'.repeat(1000001)],
|
|
1003
|
+
['before', 'x'.repeat(1201)],
|
|
1004
|
+
['after', 'x'.repeat(1201)],
|
|
1005
|
+
['impact', 'x'.repeat(1201)],
|
|
1006
|
+
]) {
|
|
1007
|
+
const report = { projects: [{ ...f.report.projects[0], [field]: value }] };
|
|
1008
|
+
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toThrow('Expected nonempty text');
|
|
1009
|
+
}
|
|
1010
|
+
f.report.projects[0].explanation = 'x'.repeat(1000000);
|
|
1011
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, f.output(), 100, (0, path_1.join)(directory, 'report.md'), false)).rejects.toThrow('at most 1000000 characters');
|
|
1012
|
+
expect(f.request).not.toHaveBeenCalled();
|
|
1013
|
+
});
|
|
1014
|
+
test('publishes an oversized explanation preview and saves its complete text', async () => {
|
|
1015
|
+
const f = fixture();
|
|
1016
|
+
const original = f.report.projects[0].explanation;
|
|
1017
|
+
const preview = 'x'.repeat(1800);
|
|
1018
|
+
const explanation = preview + ' Qualification that must remain in the full artifact.';
|
|
1019
|
+
f.report.projects[0].explanation = explanation;
|
|
1020
|
+
const path = (0, path_1.join)(directory, 'report.md');
|
|
1021
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, f.output(), 100, path, false)).resolves.toBe('Published: advisory analysis of every changed project');
|
|
1022
|
+
expect((0, fs_1.readFileSync)(path, 'utf8')).toBe(expectedReport.fullReport.replace(original, explanation));
|
|
1023
|
+
const calls = f.request.mock.calls.filter(([route]) => /^(POST|PATCH) /.test(route));
|
|
1024
|
+
expect(calls).toStrictEqual([
|
|
1025
|
+
[
|
|
1026
|
+
'POST /repos/{owner}/{repo}/issues/{issue_number}/comments',
|
|
1027
|
+
{
|
|
1028
|
+
owner: 'microsoft',
|
|
1029
|
+
repo: 'pyright',
|
|
1030
|
+
issue_number: 7,
|
|
1031
|
+
body: expectedReport.body.replace(original, preview + '\n\n**Explanation truncated; see the full report artifact linked below.**'),
|
|
1032
|
+
},
|
|
1033
|
+
],
|
|
1034
|
+
]);
|
|
1035
|
+
});
|
|
1036
|
+
test('long explanations still allow compact fallback without losing projects or artifact text', () => {
|
|
1037
|
+
const f = fixture();
|
|
1038
|
+
const names = Array.from({ length: 40 }, (_, index) => `project_${index}`);
|
|
1039
|
+
const manifest = {
|
|
1040
|
+
...f.manifest,
|
|
1041
|
+
projects: names.map((name) => ({ ...f.manifest.projects[0], name })),
|
|
1042
|
+
};
|
|
1043
|
+
const explanation = 'x'.repeat(2143);
|
|
1044
|
+
const report = { projects: names.map((name) => ({ ...f.report.projects[0], name, explanation })) };
|
|
1045
|
+
const result = (0, mypyPrimerAnalysis_1.renderReport)(manifest, report, 100);
|
|
1046
|
+
expect(result.body.length).toBeLessThanOrEqual(60000);
|
|
1047
|
+
expect(result).toStrictEqual(expectedLargeReport(names.length, explanation));
|
|
1048
|
+
});
|
|
471
1049
|
test('large reports retain every project in the comment and full artifact', () => {
|
|
472
1050
|
const f = fixture();
|
|
473
1051
|
const names = Array.from({ length: 40 }, (_, index) => `project_${index}`);
|
|
@@ -481,20 +1059,7 @@ describe('mypy_primer analysis', () => {
|
|
|
481
1059
|
const result = (0, mypyPrimerAnalysis_1.renderReport)(manifest, report, 100);
|
|
482
1060
|
expect(result.body.length).toBeLessThan(60000);
|
|
483
1061
|
expect(result.fullReport.length).toBeGreaterThan(60000);
|
|
484
|
-
|
|
485
|
-
const expectedDetails = names.flatMap((_, index) => [
|
|
486
|
-
`### project\\_${index}`,
|
|
487
|
-
'',
|
|
488
|
-
'x'.repeat(1800),
|
|
489
|
-
'',
|
|
490
|
-
'**Unresolved / coverage limits:** Exact dependency revision unavailable.',
|
|
491
|
-
'',
|
|
492
|
-
'',
|
|
493
|
-
]);
|
|
494
|
-
expect(result).toStrictEqual({
|
|
495
|
-
body: [...expectedReportHeading, ...expectedRows, '', ...expectedReportFooter].join('\n'),
|
|
496
|
-
fullReport: [...expectedReportHeading, ...expectedRows, '', ...expectedDetails].join('\n'),
|
|
497
|
-
});
|
|
1062
|
+
expect(result).toStrictEqual(expectedLargeReport(names.length, 'x'.repeat(1800)));
|
|
498
1063
|
});
|
|
499
1064
|
test('rejects missing projects, unsupported classifications, and uncited certainty', () => {
|
|
500
1065
|
const f = fixture();
|
|
@@ -512,8 +1077,8 @@ describe('mypy_primer analysis', () => {
|
|
|
512
1077
|
f.report.projects[0].summary = '<script> @someone | injected row';
|
|
513
1078
|
const escapedSummary = '\\<script\\> @someone \\| injected row';
|
|
514
1079
|
expect((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100)).toStrictEqual({
|
|
515
|
-
body: expectedReport.body.replace(
|
|
516
|
-
fullReport: expectedReport.fullReport.replace(
|
|
1080
|
+
body: expectedReport.body.replace(/The argument error changed its wording\./g, escapedSummary),
|
|
1081
|
+
fullReport: expectedReport.fullReport.replace(/The argument error changed its wording\./g, escapedSummary),
|
|
517
1082
|
});
|
|
518
1083
|
const report = {
|
|
519
1084
|
projects: [
|
|
@@ -522,6 +1087,246 @@ describe('mypy_primer analysis', () => {
|
|
|
522
1087
|
};
|
|
523
1088
|
expect(() => (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, report, 100)).toThrow('Unsupported evidence URL');
|
|
524
1089
|
});
|
|
1090
|
+
test.each([
|
|
1091
|
+
{ name: 'missing items', output: {} },
|
|
1092
|
+
{ name: 'empty output', output: { items: [] } },
|
|
1093
|
+
{
|
|
1094
|
+
name: 'no-op claiming the report is prepared',
|
|
1095
|
+
output: { items: [{ type: 'noop', message: 'The required report has been prepared for publication.' }] },
|
|
1096
|
+
},
|
|
1097
|
+
{ name: 'incomplete-only output', output: { items: [{ type: 'report_incomplete', details: 'No evidence' }] } },
|
|
1098
|
+
{ name: 'missing payload', output: { items: [{ type: 'publish_primer_analysis' }] } },
|
|
1099
|
+
{ name: 'malformed payload', output: { items: [{ type: 'publish_primer_analysis', report: '{' }] } },
|
|
1100
|
+
{
|
|
1101
|
+
name: 'omitted project',
|
|
1102
|
+
output: { items: [{ type: 'publish_primer_analysis', report: '{"projects":[]}' }] },
|
|
1103
|
+
},
|
|
1104
|
+
])('report completion rejects $name before GitHub access', async ({ output }) => {
|
|
1105
|
+
const f = fixture();
|
|
1106
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, output, 100)).toThrow();
|
|
1107
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, output, 100, (0, path_1.join)(directory, 'report.md'), false)).rejects.toThrow();
|
|
1108
|
+
expect(f.request).not.toHaveBeenCalled();
|
|
1109
|
+
});
|
|
1110
|
+
test('report completion rejects duplicate and oversized submissions', () => {
|
|
1111
|
+
const f = fixture();
|
|
1112
|
+
const output = f.output();
|
|
1113
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, { items: [...output.items, ...output.items] }, 100)).toThrow('Expected 1 to 1 primer analysis submissions; found 2');
|
|
1114
|
+
output.items[0].report = ' '.repeat(1000001);
|
|
1115
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, output, 100)).toThrow('Expected nonempty text of at most 1000000 characters');
|
|
1116
|
+
});
|
|
1117
|
+
test('assembles per-project submissions larger than the single-call transport limit', () => {
|
|
1118
|
+
const f = fixture();
|
|
1119
|
+
const names = Array.from({ length: 11 }, (_, index) => `project_${index}`);
|
|
1120
|
+
const manifest = {
|
|
1121
|
+
...f.manifest,
|
|
1122
|
+
projects: names.map((name) => ({ ...f.manifest.projects[0], name })),
|
|
1123
|
+
};
|
|
1124
|
+
const report = {
|
|
1125
|
+
projects: names.map((name) => ({
|
|
1126
|
+
...f.report.projects[0],
|
|
1127
|
+
name,
|
|
1128
|
+
assessment: 'possible-regression',
|
|
1129
|
+
attribution: 'likely-pr',
|
|
1130
|
+
explanation: 'x'.repeat(2143),
|
|
1131
|
+
evidence: [
|
|
1132
|
+
{
|
|
1133
|
+
url: `https://github.com/${repository}/blob/${headSha}/example.py#L1`,
|
|
1134
|
+
detail: 'The analyzed implementation.',
|
|
1135
|
+
},
|
|
1136
|
+
],
|
|
1137
|
+
})),
|
|
1138
|
+
};
|
|
1139
|
+
const output = {
|
|
1140
|
+
items: report.projects.map((project) => ({
|
|
1141
|
+
type: 'publish_primer_analysis',
|
|
1142
|
+
report: JSON.stringify({ projects: [project] }),
|
|
1143
|
+
})),
|
|
1144
|
+
};
|
|
1145
|
+
expect(Buffer.byteLength(JSON.stringify(report), 'utf8')).toBeGreaterThan(10240);
|
|
1146
|
+
for (const item of output.items) {
|
|
1147
|
+
expect(Buffer.byteLength(item.report, 'utf8')).toBeLessThanOrEqual(10240);
|
|
1148
|
+
}
|
|
1149
|
+
const path = (0, path_1.join)(directory, 'agent_output.json');
|
|
1150
|
+
(0, fs_1.writeFileSync)(path, JSON.stringify(output));
|
|
1151
|
+
expect((0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(manifest, path, 100)).toStrictEqual((0, mypyPrimerAnalysis_1.renderReport)(manifest, report, 100));
|
|
1152
|
+
});
|
|
1153
|
+
test('retains compatibility with a single report covering multiple projects', () => {
|
|
1154
|
+
const f = fixture([sample, sample.replace(/example/g, 'second')].join('\n'));
|
|
1155
|
+
f.report.projects.push({ ...f.report.projects[0], name: 'second' });
|
|
1156
|
+
expect((0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, f.output(), 100)).toStrictEqual((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100));
|
|
1157
|
+
});
|
|
1158
|
+
test('publishes per-project submissions only after every project is present', async () => {
|
|
1159
|
+
const f = fixture([sample, sample.replace(/example/g, 'second')].join('\n'));
|
|
1160
|
+
f.report.projects.push({ ...f.report.projects[0], name: 'second' });
|
|
1161
|
+
const output = f.output(true);
|
|
1162
|
+
const path = (0, path_1.join)(directory, 'report.md');
|
|
1163
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, { items: output.items.slice(0, 1) }, 100, path, false)).rejects.toThrow('every changed project exactly once');
|
|
1164
|
+
expect(f.request).not.toHaveBeenCalled();
|
|
1165
|
+
expect(await (0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, output, 100, path, false)).toBe('Published: advisory analysis of every changed project');
|
|
1166
|
+
const expected = (0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100);
|
|
1167
|
+
expect((0, fs_1.readFileSync)(path, 'utf8')).toBe(expected.fullReport);
|
|
1168
|
+
expect(f.request).toHaveBeenLastCalledWith('POST /repos/{owner}/{repo}/issues/{issue_number}/comments', {
|
|
1169
|
+
owner: 'microsoft',
|
|
1170
|
+
repo: 'pyright',
|
|
1171
|
+
issue_number: 7,
|
|
1172
|
+
body: expected.body,
|
|
1173
|
+
});
|
|
1174
|
+
});
|
|
1175
|
+
test('rejects invalid per-project payloads and evidence before GitHub access', async () => {
|
|
1176
|
+
const f = fixture([sample, sample.replace(/example/g, 'second')].join('\n'));
|
|
1177
|
+
f.report.projects.push({ ...f.report.projects[0], name: 'second' });
|
|
1178
|
+
const [first, second] = f.report.projects;
|
|
1179
|
+
const invalidReports = [
|
|
1180
|
+
[JSON.stringify({ projects: [first] }), 'every changed project exactly once'],
|
|
1181
|
+
[JSON.stringify({ projects: [{ ...second, name: 'unexpected' }] }), 'every changed project exactly once'],
|
|
1182
|
+
[JSON.stringify({ projects: [] }), 'exactly one project'],
|
|
1183
|
+
[JSON.stringify(f.report), 'exactly one project'],
|
|
1184
|
+
['{}', 'Expected a JSON array'],
|
|
1185
|
+
['{', 'JSON'],
|
|
1186
|
+
[
|
|
1187
|
+
JSON.stringify({ projects: [{ ...second, assessment: 'possible-regression' }] }),
|
|
1188
|
+
'Expected cited evidence',
|
|
1189
|
+
],
|
|
1190
|
+
[
|
|
1191
|
+
JSON.stringify({
|
|
1192
|
+
projects: [
|
|
1193
|
+
{
|
|
1194
|
+
...second,
|
|
1195
|
+
attribution: 'likely-pr',
|
|
1196
|
+
evidence: [
|
|
1197
|
+
{
|
|
1198
|
+
url: `https://github.com/${repository}/blob/main/example.py#L1`,
|
|
1199
|
+
detail: 'Not the analyzed head.',
|
|
1200
|
+
},
|
|
1201
|
+
],
|
|
1202
|
+
},
|
|
1203
|
+
],
|
|
1204
|
+
}),
|
|
1205
|
+
'PR attribution requires a line-pinned citation',
|
|
1206
|
+
],
|
|
1207
|
+
];
|
|
1208
|
+
for (const [report, error] of invalidReports) {
|
|
1209
|
+
const output = f.output(true);
|
|
1210
|
+
output.items[1].report = report;
|
|
1211
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, output, 100)).toThrow(error);
|
|
1212
|
+
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, output, 100, (0, path_1.join)(directory, 'report.md'), false)).rejects.toThrow(error);
|
|
1213
|
+
}
|
|
1214
|
+
expect(f.request).not.toHaveBeenCalled();
|
|
1215
|
+
});
|
|
1216
|
+
test.each([10000, 10001])('enforces the combined bound for 100 submissions of %s characters', (length) => {
|
|
1217
|
+
const f = fixture();
|
|
1218
|
+
const names = Array.from({ length: 100 }, (_, index) => `project_${index}`);
|
|
1219
|
+
f.manifest.projects = names.map((name) => ({ ...f.manifest.projects[0], name }));
|
|
1220
|
+
f.report.projects = names.map((name) => {
|
|
1221
|
+
const project = { ...f.report.projects[0], name, explanation: '' };
|
|
1222
|
+
project.explanation = 'x'.repeat(length - JSON.stringify({ projects: [project] }).length);
|
|
1223
|
+
return project;
|
|
1224
|
+
});
|
|
1225
|
+
const output = f.output(true);
|
|
1226
|
+
expect(output.items.map((item) => Buffer.byteLength(item.report, 'utf8'))).toStrictEqual(Array.from({ length: 100 }, () => length));
|
|
1227
|
+
expect(length).toBeLessThanOrEqual(10240);
|
|
1228
|
+
if (length === 10000) {
|
|
1229
|
+
expect((0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, output, 100)).toStrictEqual((0, mypyPrimerAnalysis_1.renderReport)(f.manifest, f.report, 100));
|
|
1230
|
+
}
|
|
1231
|
+
else {
|
|
1232
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, output, 100)).toThrow('The combined primer analysis report exceeds 1000000 characters');
|
|
1233
|
+
}
|
|
1234
|
+
});
|
|
1235
|
+
test('report completion reuses evidence and project validation', () => {
|
|
1236
|
+
const f = fixture();
|
|
1237
|
+
f.report.projects[0].assessment = 'expected-improvement';
|
|
1238
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, f.output(), 100)).toThrow('evidence');
|
|
1239
|
+
f.report.projects[0].assessment = 'needs-review';
|
|
1240
|
+
f.report.projects.push(f.report.projects[0]);
|
|
1241
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReport)(f.manifest, f.output(), 100)).toThrow();
|
|
1242
|
+
});
|
|
1243
|
+
test('report completion accepts an honest needs-review report without evidence', () => {
|
|
1244
|
+
const f = fixture();
|
|
1245
|
+
const path = (0, path_1.join)(directory, 'agent_output.json');
|
|
1246
|
+
(0, fs_1.writeFileSync)(path, JSON.stringify(f.output()));
|
|
1247
|
+
expect((0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(f.manifest, path, 100)).toStrictEqual(expectedReport);
|
|
1248
|
+
expect(f.request).not.toHaveBeenCalled();
|
|
1249
|
+
});
|
|
1250
|
+
test('report completion requires a bounded regular JSON output file', () => {
|
|
1251
|
+
const f = fixture();
|
|
1252
|
+
const path = (0, path_1.join)(directory, 'agent_output.json');
|
|
1253
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(f.manifest, path, 100)).toThrow('ENOENT');
|
|
1254
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(f.manifest, directory, 100)).toThrow('bounded regular input file');
|
|
1255
|
+
(0, fs_1.writeFileSync)(path, '');
|
|
1256
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(f.manifest, path, 100)).toThrow();
|
|
1257
|
+
(0, fs_1.writeFileSync)(path, ' '.repeat(8 * 1024 * 1024 + 1));
|
|
1258
|
+
expect(() => (0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(f.manifest, path, 100)).toThrow('bounded regular input file');
|
|
1259
|
+
});
|
|
1260
|
+
test('report completion preserves typing specification citations in serialized reports', () => {
|
|
1261
|
+
const f = fixture();
|
|
1262
|
+
const url = 'https://typing.python.org/en/latest/spec/typeddict.html';
|
|
1263
|
+
const report = {
|
|
1264
|
+
projects: [
|
|
1265
|
+
{
|
|
1266
|
+
...f.report.projects[0],
|
|
1267
|
+
evidence: [{ url, detail: 'TypedDict specification' }],
|
|
1268
|
+
},
|
|
1269
|
+
],
|
|
1270
|
+
};
|
|
1271
|
+
const output = { items: [{ type: 'publish_primer_analysis', report: JSON.stringify(report) }] };
|
|
1272
|
+
const path = (0, path_1.join)(directory, 'agent_output.json');
|
|
1273
|
+
(0, fs_1.writeFileSync)(path, JSON.stringify(output));
|
|
1274
|
+
const originalDetails = expectedReportDetails.join('\n');
|
|
1275
|
+
const details = [
|
|
1276
|
+
...expectedReportDetails.slice(0, -1),
|
|
1277
|
+
'- TypedDict specification: <https://typing.python.org/en/latest/spec/typeddict.html>',
|
|
1278
|
+
'',
|
|
1279
|
+
].join('\n');
|
|
1280
|
+
expect((0, mypyPrimerAnalysis_1.validateSubmittedReportFile)(f.manifest, path, 100)).toStrictEqual({
|
|
1281
|
+
body: expectedReport.body.replace(originalDetails, details),
|
|
1282
|
+
fullReport: expectedReport.fullReport.replace(originalDetails, details),
|
|
1283
|
+
});
|
|
1284
|
+
});
|
|
1285
|
+
test('compiled completion step rejects no-op output and accepts a submitted report', () => {
|
|
1286
|
+
const workflow = (0, yaml_1.parseDocument)((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.lock.yml'), 'utf8'));
|
|
1287
|
+
const steps = workflow.getIn(['jobs', 'agent', 'steps'], true);
|
|
1288
|
+
if (!(0, yaml_1.isSeq)(steps)) {
|
|
1289
|
+
throw new Error('Missing agent steps');
|
|
1290
|
+
}
|
|
1291
|
+
const step = steps.items.find((item) => (0, yaml_1.isMap)(item) && item.get('id') === 'require_primer_report');
|
|
1292
|
+
if (!(0, yaml_1.isMap)(step)) {
|
|
1293
|
+
throw new Error('Missing report completion step');
|
|
1294
|
+
}
|
|
1295
|
+
const script = step.getIn(['with', 'script']);
|
|
1296
|
+
if (typeof script !== 'string') {
|
|
1297
|
+
throw new Error('Missing report completion script');
|
|
1298
|
+
}
|
|
1299
|
+
const f = fixture();
|
|
1300
|
+
const actionsDir = (0, path_1.join)(directory, 'gh-aw', 'actions');
|
|
1301
|
+
const path = (0, path_1.join)(directory, 'agent_output.json');
|
|
1302
|
+
(0, fs_1.mkdirSync)(actionsDir, { recursive: true });
|
|
1303
|
+
(0, fs_1.writeFileSync)((0, path_1.join)(actionsDir, 'primer-manifest.json'), JSON.stringify(f.manifest));
|
|
1304
|
+
const notice = jest.fn();
|
|
1305
|
+
const run = () => (0, vm_1.runInNewContext)(script, {
|
|
1306
|
+
require: (name) => {
|
|
1307
|
+
if (name === 'fs') {
|
|
1308
|
+
return { readFileSync: fs_1.readFileSync };
|
|
1309
|
+
}
|
|
1310
|
+
if (name === 'path') {
|
|
1311
|
+
return { join: path_1.join };
|
|
1312
|
+
}
|
|
1313
|
+
if (name === (0, path_1.join)(actionsDir, 'mypyPrimerAnalysis.ts')) {
|
|
1314
|
+
return { validateSubmittedReportFile: mypyPrimerAnalysis_1.validateSubmittedReportFile };
|
|
1315
|
+
}
|
|
1316
|
+
throw new Error(`Unexpected module: ${name}`);
|
|
1317
|
+
},
|
|
1318
|
+
process: { env: { RUNNER_TEMP: directory, PRIMER_AGENT_OUTPUT: path } },
|
|
1319
|
+
context: { runId: 100 },
|
|
1320
|
+
core: { notice },
|
|
1321
|
+
});
|
|
1322
|
+
(0, fs_1.writeFileSync)(path, JSON.stringify({ items: [{ type: 'noop', message: 'Report prepared' }], errors: [] }));
|
|
1323
|
+
expect(run).toThrow('Submit publish_primer_analysis, not noop or report_incomplete');
|
|
1324
|
+
expect(notice).not.toHaveBeenCalled();
|
|
1325
|
+
(0, fs_1.writeFileSync)(path, JSON.stringify(f.output()));
|
|
1326
|
+
run();
|
|
1327
|
+
expect(notice).toHaveBeenCalledWith('A complete primer report was submitted; threat detection and publisher checks still apply');
|
|
1328
|
+
expect(f.request).not.toHaveBeenCalled();
|
|
1329
|
+
});
|
|
525
1330
|
test('staged reports never mutate GitHub', async () => {
|
|
526
1331
|
const f = fixture();
|
|
527
1332
|
const path = (0, path_1.join)(directory, 'report.md');
|
|
@@ -595,6 +1400,203 @@ describe('mypy_primer analysis', () => {
|
|
|
595
1400
|
await expect((0, mypyPrimerAnalysis_1.publishAnalysis)(f.request, f.manifest, f.output(), 100, (0, path_1.join)(directory, 'report.md'), false)).resolves.toBe('Skipped: a newer primer analysis has already been posted for this commit');
|
|
596
1401
|
expect(f.request.mock.calls.some(([route]) => /^(POST|PATCH) /.test(route))).toBe(false);
|
|
597
1402
|
});
|
|
1403
|
+
test.each(['workflow_run', 'workflow_dispatch'])('compiled collector accepts %s source context', async (eventName) => {
|
|
1404
|
+
const workflow = (0, yaml_1.parseDocument)((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.lock.yml'), 'utf8'));
|
|
1405
|
+
const steps = workflow.getIn(['jobs', 'collect', 'steps'], true);
|
|
1406
|
+
if (!(0, yaml_1.isSeq)(steps)) {
|
|
1407
|
+
throw new Error('Missing collection steps');
|
|
1408
|
+
}
|
|
1409
|
+
const step = steps.items.find((item) => (0, yaml_1.isMap)(item) && item.get('id') === 'source');
|
|
1410
|
+
if (!(0, yaml_1.isMap)(step)) {
|
|
1411
|
+
throw new Error('Missing source validation step');
|
|
1412
|
+
}
|
|
1413
|
+
const script = step.getIn(['with', 'script']);
|
|
1414
|
+
if (typeof script !== 'string') {
|
|
1415
|
+
throw new Error('Missing source validation script');
|
|
1416
|
+
}
|
|
1417
|
+
const f = fixture();
|
|
1418
|
+
if (eventName === 'workflow_run') {
|
|
1419
|
+
(0, fs_1.mkdirSync)((0, path_1.join)(directory, 'primer-context'));
|
|
1420
|
+
(0, fs_1.writeFileSync)((0, path_1.join)(directory, 'primer-context', 'primer-analysis-context.json'), JSON.stringify(f.source));
|
|
1421
|
+
}
|
|
1422
|
+
const result = await (0, vm_1.runInNewContext)(`(async () => { ${script} })()`, {
|
|
1423
|
+
require: (name) => {
|
|
1424
|
+
if (name === 'fs') {
|
|
1425
|
+
return { readFileSync: fs_1.readFileSync, writeFileSync: fs_1.writeFileSync };
|
|
1426
|
+
}
|
|
1427
|
+
if (name === 'path') {
|
|
1428
|
+
return { join: path_1.join };
|
|
1429
|
+
}
|
|
1430
|
+
if (name === './build/ci/mypyPrimerAnalysis.ts') {
|
|
1431
|
+
return { loadSource: mypyPrimerAnalysis_1.loadSource };
|
|
1432
|
+
}
|
|
1433
|
+
throw new Error(`Unexpected module: ${name}`);
|
|
1434
|
+
},
|
|
1435
|
+
process: {
|
|
1436
|
+
env: {
|
|
1437
|
+
RUNNER_TEMP: directory,
|
|
1438
|
+
PRIMER_EVENT_NAME: eventName,
|
|
1439
|
+
PRIMER_RUN_ID: eventName === 'workflow_dispatch' ? '42' : 'ignored',
|
|
1440
|
+
PRIMER_RUN_ATTEMPT: eventName === 'workflow_dispatch' ? '1' : 'ignored',
|
|
1441
|
+
},
|
|
1442
|
+
},
|
|
1443
|
+
github: { request: f.request },
|
|
1444
|
+
context: { repo: { owner: 'microsoft', repo: 'pyright' } },
|
|
1445
|
+
});
|
|
1446
|
+
expect(result).toBe(42);
|
|
1447
|
+
expect(JSON.parse((0, fs_1.readFileSync)((0, path_1.join)(directory, 'primer-source.json'), 'utf8'))).toStrictEqual(f.source);
|
|
1448
|
+
});
|
|
1449
|
+
test('compiled manual preview is read-only and isolated from publication', async () => {
|
|
1450
|
+
const workflow = (0, yaml_1.parseDocument)((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.lock.yml'), 'utf8'));
|
|
1451
|
+
const inputs = workflow.getIn(['on', 'workflow_dispatch', 'inputs'], true);
|
|
1452
|
+
const preview = workflow.getIn(['jobs', 'preview'], true);
|
|
1453
|
+
if (!(0, yaml_1.isMap)(inputs) || !(0, yaml_1.isMap)(preview)) {
|
|
1454
|
+
throw new Error('Missing manual preview configuration');
|
|
1455
|
+
}
|
|
1456
|
+
expect(inputs.toJSON()).toStrictEqual({
|
|
1457
|
+
aw_context: {
|
|
1458
|
+
default: '',
|
|
1459
|
+
description: 'Agent caller context (used internally by Agentic Workflows).',
|
|
1460
|
+
required: false,
|
|
1461
|
+
type: 'string',
|
|
1462
|
+
},
|
|
1463
|
+
'primer-run-id': {
|
|
1464
|
+
description: 'Successful Run mypy_primer on PR run ID to preview, including merged PRs.',
|
|
1465
|
+
required: true,
|
|
1466
|
+
type: 'string',
|
|
1467
|
+
},
|
|
1468
|
+
'primer-run-attempt': {
|
|
1469
|
+
description: 'Current attempt of the recorded primer run.',
|
|
1470
|
+
required: true,
|
|
1471
|
+
default: '1',
|
|
1472
|
+
type: 'string',
|
|
1473
|
+
},
|
|
1474
|
+
});
|
|
1475
|
+
expect(String(workflow.getIn(['jobs', 'publish_primer_analysis', 'if'])).trim()).toBe("(!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'publish_primer_analysis') && (github.event_name != 'workflow_dispatch')");
|
|
1476
|
+
expect(String(preview.get('if')).trim()).toBe("${{ !cancelled() && github.event_name == 'workflow_dispatch' && needs.agent.result == 'success' && needs.detection.result == 'success' && needs.detection.outputs.detection_conclusion == 'success' }}");
|
|
1477
|
+
const permissions = preview.get('permissions');
|
|
1478
|
+
const steps = preview.get('steps');
|
|
1479
|
+
if (!(0, yaml_1.isMap)(permissions) || !(0, yaml_1.isSeq)(steps)) {
|
|
1480
|
+
throw new Error('Missing preview permissions or steps');
|
|
1481
|
+
}
|
|
1482
|
+
expect(permissions.toJSON()).toStrictEqual({ contents: 'read', actions: 'read' });
|
|
1483
|
+
const renderStep = steps.items.find((item) => (0, yaml_1.isMap)(item) && item.get('id') === 'render_preview');
|
|
1484
|
+
if (!(0, yaml_1.isMap)(renderStep)) {
|
|
1485
|
+
throw new Error('Missing preview rendering step');
|
|
1486
|
+
}
|
|
1487
|
+
const script = renderStep.getIn(['with', 'script']);
|
|
1488
|
+
if (typeof script !== 'string') {
|
|
1489
|
+
throw new Error('Missing preview rendering script');
|
|
1490
|
+
}
|
|
1491
|
+
const f = fixture();
|
|
1492
|
+
const manifestPath = (0, path_1.join)(directory, 'primer-input', 'manifest.json');
|
|
1493
|
+
const outputPath = (0, path_1.join)(directory, 'primer-preview-output', 'agent_output.json');
|
|
1494
|
+
(0, fs_1.mkdirSync)((0, path_1.join)(directory, 'primer-input'));
|
|
1495
|
+
(0, fs_1.mkdirSync)((0, path_1.join)(directory, 'primer-preview-output'));
|
|
1496
|
+
(0, fs_1.writeFileSync)(outputPath, JSON.stringify(f.output()));
|
|
1497
|
+
const summary = { addRaw: jest.fn().mockReturnThis(), write: jest.fn().mockResolvedValue(undefined) };
|
|
1498
|
+
const run = () => (0, vm_1.runInNewContext)(`(async () => { ${script} })()`, {
|
|
1499
|
+
require: (name) => {
|
|
1500
|
+
if (name === 'fs') {
|
|
1501
|
+
return { mkdirSync: fs_1.mkdirSync, readFileSync: fs_1.readFileSync, writeFileSync: fs_1.writeFileSync };
|
|
1502
|
+
}
|
|
1503
|
+
if (name === 'path') {
|
|
1504
|
+
return { join: path_1.join };
|
|
1505
|
+
}
|
|
1506
|
+
if (name === './build/ci/mypyPrimerAnalysis.ts') {
|
|
1507
|
+
return { validateSubmittedReportFile: mypyPrimerAnalysis_1.validateSubmittedReportFile };
|
|
1508
|
+
}
|
|
1509
|
+
throw new Error(`Unexpected module: ${name}`);
|
|
1510
|
+
},
|
|
1511
|
+
process: { env: { RUNNER_TEMP: directory } },
|
|
1512
|
+
context: { runId: 100 },
|
|
1513
|
+
core: { summary },
|
|
1514
|
+
});
|
|
1515
|
+
(0, fs_1.writeFileSync)(manifestPath, JSON.stringify(f.manifest));
|
|
1516
|
+
await expect(run()).rejects.toThrow('Expected a trusted manual-preview manifest');
|
|
1517
|
+
expect(summary.addRaw.mock.calls).toStrictEqual([]);
|
|
1518
|
+
(0, fs_1.writeFileSync)(manifestPath, JSON.stringify({ ...f.manifest, preview: true }));
|
|
1519
|
+
await run();
|
|
1520
|
+
const asPreview = (content) => content
|
|
1521
|
+
.replace('<!-- pyright-primer-analysis:', '<!-- pyright-primer-analysis-preview:')
|
|
1522
|
+
.replace('## mypy_primer analysis\n\n', '## mypy_primer analysis preview\n\n**Preview only. No PR comment was posted.**\n\n');
|
|
1523
|
+
expect((0, fs_1.readFileSync)((0, path_1.join)(directory, 'primer-preview', 'comment.md'), 'utf8')).toBe(asPreview(expectedReport.body));
|
|
1524
|
+
expect((0, fs_1.readFileSync)((0, path_1.join)(directory, 'primer-preview', 'full-report.md'), 'utf8')).toBe(asPreview(expectedReport.fullReport));
|
|
1525
|
+
expect(summary.addRaw.mock.calls).toStrictEqual([[asPreview(expectedReport.body)]]);
|
|
1526
|
+
expect(summary.write.mock.calls).toStrictEqual([[]]);
|
|
1527
|
+
expect(f.request.mock.calls).toStrictEqual([]);
|
|
1528
|
+
});
|
|
1529
|
+
test('compiled ingestion permits typing citations without expanding network access', () => {
|
|
1530
|
+
const source = (0, yaml_1.parseDocument)((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.md'), 'utf8').split(/^---\r?$/m)[1]);
|
|
1531
|
+
const networkDomains = source.getIn(['network', 'allowed'], true);
|
|
1532
|
+
const outputDomains = source.getIn(['safe-outputs', 'allowed-domains'], true);
|
|
1533
|
+
if (!(0, yaml_1.isSeq)(networkDomains) || !(0, yaml_1.isSeq)(outputDomains)) {
|
|
1534
|
+
throw new Error('Missing network or safe-output domain configuration');
|
|
1535
|
+
}
|
|
1536
|
+
expect(networkDomains.toJSON()).toStrictEqual(['defaults', 'github']);
|
|
1537
|
+
expect(outputDomains.toJSON()).toStrictEqual(['typing.python.org']);
|
|
1538
|
+
const workflow = (0, yaml_1.parseDocument)((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.lock.yml'), 'utf8'));
|
|
1539
|
+
const steps = workflow.getIn(['jobs', 'agent', 'steps'], true);
|
|
1540
|
+
if (!(0, yaml_1.isSeq)(steps)) {
|
|
1541
|
+
throw new Error('Missing agent steps');
|
|
1542
|
+
}
|
|
1543
|
+
const ingestion = steps.items.find((item) => (0, yaml_1.isMap)(item) && item.get('id') === 'collect_output');
|
|
1544
|
+
if (!(0, yaml_1.isMap)(ingestion)) {
|
|
1545
|
+
throw new Error('Missing safe-output ingestion step');
|
|
1546
|
+
}
|
|
1547
|
+
expect(ingestion.getIn(['env', 'GH_AW_ALLOWED_DOMAINS'])).toBe([
|
|
1548
|
+
'*.githubusercontent.com',
|
|
1549
|
+
'api.snapcraft.io',
|
|
1550
|
+
'archive.ubuntu.com',
|
|
1551
|
+
'azure.archive.ubuntu.com',
|
|
1552
|
+
'codeload.github.com',
|
|
1553
|
+
'crl.geotrust.com',
|
|
1554
|
+
'crl.globalsign.com',
|
|
1555
|
+
'crl.identrust.com',
|
|
1556
|
+
'crl.sectigo.com',
|
|
1557
|
+
'crl.thawte.com',
|
|
1558
|
+
'crl.usertrust.com',
|
|
1559
|
+
'crl.verisign.com',
|
|
1560
|
+
'crl3.digicert.com',
|
|
1561
|
+
'crl4.digicert.com',
|
|
1562
|
+
'crls.ssl.com',
|
|
1563
|
+
'docs.github.com',
|
|
1564
|
+
'github-cloud.githubusercontent.com',
|
|
1565
|
+
'github-cloud.s3.amazonaws.com',
|
|
1566
|
+
'github.blog',
|
|
1567
|
+
'github.com',
|
|
1568
|
+
'github.githubassets.com',
|
|
1569
|
+
'json-schema.org',
|
|
1570
|
+
'json.schemastore.org',
|
|
1571
|
+
'keyserver.ubuntu.com',
|
|
1572
|
+
'lfs.github.com',
|
|
1573
|
+
'localhost', // gh-aw includes this implicitly when safe-output domains are configured.
|
|
1574
|
+
'objects.githubusercontent.com',
|
|
1575
|
+
'ocsp.digicert.com',
|
|
1576
|
+
'ocsp.geotrust.com',
|
|
1577
|
+
'ocsp.globalsign.com',
|
|
1578
|
+
'ocsp.identrust.com',
|
|
1579
|
+
'ocsp.sectigo.com',
|
|
1580
|
+
'ocsp.ssl.com',
|
|
1581
|
+
'ocsp.thawte.com',
|
|
1582
|
+
'ocsp.usertrust.com',
|
|
1583
|
+
'ocsp.verisign.com',
|
|
1584
|
+
'packagecloud.io',
|
|
1585
|
+
'packages.cloud.google.com',
|
|
1586
|
+
'packages.microsoft.com',
|
|
1587
|
+
'patch-diff.githubusercontent.com',
|
|
1588
|
+
'patchdiff.githubusercontent.com',
|
|
1589
|
+
'ppa.launchpad.net',
|
|
1590
|
+
'raw.githubusercontent.com',
|
|
1591
|
+
's.symcb.com',
|
|
1592
|
+
's.symcd.com',
|
|
1593
|
+
'security.ubuntu.com',
|
|
1594
|
+
'ts-crl.ws.symantec.com',
|
|
1595
|
+
'ts-ocsp.ws.symantec.com',
|
|
1596
|
+
'typing.python.org',
|
|
1597
|
+
'www.googleapis.com',
|
|
1598
|
+
].join(','));
|
|
1599
|
+
});
|
|
598
1600
|
test('compiled workflow runtime matches the compiler resolution lock', () => {
|
|
599
1601
|
const content = (0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.lock.yml'), 'utf8');
|
|
600
1602
|
const metadata = (0, yaml_1.parseDocument)(content.split('\n')[0].replace('# gh-aw-metadata: ', ''));
|
|
@@ -629,5 +1631,149 @@ describe('mypy_primer analysis', () => {
|
|
|
629
1631
|
expect(String(publishingSteps)).toContain('getStagedMode(activation,');
|
|
630
1632
|
expect(String(publishingSteps)).toContain('primer-activation');
|
|
631
1633
|
});
|
|
1634
|
+
test('compiled workflow exposes GitHub and safe outputs through MCP without CLI-only instructions', () => {
|
|
1635
|
+
const source = (0, yaml_1.parseDocument)((0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.md'), 'utf8').split(/^---\r?$/m)[1]);
|
|
1636
|
+
expect(source.getIn(['tools', 'cli-proxy'])).toBe(false);
|
|
1637
|
+
expect(source.getIn(['tools', 'bash'])).toBe(false);
|
|
1638
|
+
expect(source.getIn(['tools', 'edit'])).toBe(false);
|
|
1639
|
+
expect(source.getIn(['tools', 'github', 'read-only'])).toBe(true);
|
|
1640
|
+
expect(source.get('max-ai-credits')).toBe(200);
|
|
1641
|
+
expect(source.getIn(['engine', 'model'])).toBe('gpt-5.6-sol');
|
|
1642
|
+
expect(source.getIn(['safe-outputs', 'jobs', 'publish-primer-analysis', 'max'])).toBe(100);
|
|
1643
|
+
expect(source.getIn(['safe-outputs', 'noop'])).toBe(false);
|
|
1644
|
+
expect(source.getIn(['sandbox', 'agent', 'token-steering'])).toBe(false);
|
|
1645
|
+
expect(source.getIn(['safe-outputs', 'threat-detection', 'engine', 'model'])).toBe('detection');
|
|
1646
|
+
expect(source.getIn(['safe-outputs', 'threat-detection', 'engine', 'version'])).toBe('1.0.80');
|
|
1647
|
+
expect(source.getIn(['engine', 'version'])).toBe('1.0.80');
|
|
1648
|
+
expect(source.getIn(['engine', 'harness'])).toBe('mypyPrimerCopilotHarness.cjs');
|
|
1649
|
+
const content = (0, fs_1.readFileSync)((0, path_1.join)(__dirname, '../../../../.github/workflows/mypy-primer-analysis.lock.yml'), 'utf8');
|
|
1650
|
+
const metadata = (0, yaml_1.parseDocument)(content.split('\n')[0].replace('# gh-aw-metadata: ', ''));
|
|
1651
|
+
expect(metadata.get('compiler_version')).toBe('v0.88.7');
|
|
1652
|
+
expect(metadata.getIn(['engine_versions', 'copilot'])).toBe('1.0.80');
|
|
1653
|
+
expect(content.includes('mcp_cli_tools')).toBe(false);
|
|
1654
|
+
expect(content.includes('GH_AW_MCP_CLI_SERVERS')).toBe(false);
|
|
1655
|
+
const workflow = (0, yaml_1.parseDocument)(content);
|
|
1656
|
+
const agentSteps = workflow.getIn(['jobs', 'agent', 'steps'], true);
|
|
1657
|
+
if (!(0, yaml_1.isSeq)(agentSteps)) {
|
|
1658
|
+
throw new Error('Missing agent steps');
|
|
1659
|
+
}
|
|
1660
|
+
const getStep = (id) => {
|
|
1661
|
+
const step = agentSteps.items.find((item) => (0, yaml_1.isMap)(item) && item.get('id') === id);
|
|
1662
|
+
if (!(0, yaml_1.isMap)(step)) {
|
|
1663
|
+
throw new Error(`Missing agent step: ${id}`);
|
|
1664
|
+
}
|
|
1665
|
+
return step;
|
|
1666
|
+
};
|
|
1667
|
+
const normalizeScript = (script) => {
|
|
1668
|
+
if (typeof script !== 'string') {
|
|
1669
|
+
throw new Error('Missing workflow script');
|
|
1670
|
+
}
|
|
1671
|
+
return script.replace(/\r\n/g, '\n').trim();
|
|
1672
|
+
};
|
|
1673
|
+
const gateway = getStep('start-mcp-gateway').get('run');
|
|
1674
|
+
expect(gateway).toContain('export GH_AW_ENGINE="copilot"');
|
|
1675
|
+
expect(gateway).toContain('"github": {');
|
|
1676
|
+
expect(gateway).toContain('"safeoutputs": {');
|
|
1677
|
+
expect(gateway).toContain('"GITHUB_READ_ONLY": "1"');
|
|
1678
|
+
const install = agentSteps.items.find((item) => (0, yaml_1.isMap)(item) && String(item.get('run')).includes('install_copilot_cli.sh'));
|
|
1679
|
+
if (!(0, yaml_1.isMap)(install)) {
|
|
1680
|
+
throw new Error('Missing Copilot installation step');
|
|
1681
|
+
}
|
|
1682
|
+
expect(install.get('run')).toContain('install_copilot_cli.sh" 1.0.80');
|
|
1683
|
+
const preflight = getStep('stage_mcp_preflight');
|
|
1684
|
+
expect(preflight.get('continue-on-error')).toBeUndefined();
|
|
1685
|
+
expect(preflight.get('if')).toBeUndefined();
|
|
1686
|
+
expect(normalizeScript(preflight.getIn(['with', 'script']))).toBe([
|
|
1687
|
+
"const fs = require('fs');",
|
|
1688
|
+
"const path = require('path');",
|
|
1689
|
+
"for (const file of ['mypyPrimerMcp.ts', 'mypyPrimerCopilotHarness.cjs', 'mypyPrimerAnalysis.ts']) {",
|
|
1690
|
+
' fs.copyFileSync(',
|
|
1691
|
+
" path.join('/tmp/gh-aw/primer-input', file),",
|
|
1692
|
+
" path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions', file)",
|
|
1693
|
+
' );',
|
|
1694
|
+
'}',
|
|
1695
|
+
'fs.copyFileSync(',
|
|
1696
|
+
" '/tmp/gh-aw/primer-input/manifest.json',",
|
|
1697
|
+
" path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions', 'primer-manifest.json')",
|
|
1698
|
+
');',
|
|
1699
|
+
].join('\n'));
|
|
1700
|
+
expect(agentSteps.items.indexOf(preflight)).toBeLessThan(agentSteps.items.indexOf(getStep('agentic_execution')));
|
|
1701
|
+
expect(getStep('agentic_execution').get('if')).toBeUndefined();
|
|
1702
|
+
const collectSteps = workflow.getIn(['jobs', 'collect', 'steps'], true);
|
|
1703
|
+
if (!(0, yaml_1.isSeq)(collectSteps)) {
|
|
1704
|
+
throw new Error('Missing collection steps');
|
|
1705
|
+
}
|
|
1706
|
+
const prepare = collectSteps.items.find((item) => (0, yaml_1.isMap)(item) && item.get('id') === 'prepare');
|
|
1707
|
+
if (!(0, yaml_1.isMap)(prepare)) {
|
|
1708
|
+
throw new Error('Missing preparation step');
|
|
1709
|
+
}
|
|
1710
|
+
expect(prepare.getIn(['env', 'PRIMER_PREVIEW'])).toBe("${{ github.event_name == 'workflow_dispatch' }}");
|
|
1711
|
+
expect(normalizeScript(prepare.getIn(['with', 'script']))).toBe([
|
|
1712
|
+
"const fs = require('fs');",
|
|
1713
|
+
"const path = require('path');",
|
|
1714
|
+
"const { prepareAnalysis } = require('./build/ci/mypyPrimerAnalysis.ts');",
|
|
1715
|
+
"const source = JSON.parse(fs.readFileSync(path.join(process.env.RUNNER_TEMP, 'primer-source.json'), 'utf8'));",
|
|
1716
|
+
"const folder = path.join(process.env.RUNNER_TEMP, 'primer-input');",
|
|
1717
|
+
"const manifest = await prepareAnalysis(github.request.bind(github), `${context.repo.owner}/${context.repo.repo}`, source, path.join(folder, 'raw'), process.env.PRIMER_PREVIEW === 'true');",
|
|
1718
|
+
'if (!manifest) {',
|
|
1719
|
+
" core.notice('Skipping analysis: the PR is closed or its head has changed');",
|
|
1720
|
+
" core.setOutput('has_changes', 'false');",
|
|
1721
|
+
' return;',
|
|
1722
|
+
'}',
|
|
1723
|
+
"fs.writeFileSync(path.join(folder, 'manifest.json'), JSON.stringify(manifest, null, 2));",
|
|
1724
|
+
"fs.copyFileSync('./build/ci/mypyPrimerMcp.ts', path.join(folder, 'mypyPrimerMcp.ts'));",
|
|
1725
|
+
"fs.copyFileSync('./build/ci/mypyPrimerCopilotHarness.cjs', path.join(folder, 'mypyPrimerCopilotHarness.cjs'));",
|
|
1726
|
+
"fs.copyFileSync('./build/ci/mypyPrimerAnalysis.ts', path.join(folder, 'mypyPrimerAnalysis.ts'));",
|
|
1727
|
+
"core.setOutput('has_changes', String(manifest.projects.length > 0));",
|
|
1728
|
+
'core.notice(`${manifest.projects.length} changed projects; all eight shards accounted for`);',
|
|
1729
|
+
].join('\n'));
|
|
1730
|
+
const reportCheck = getStep('require_primer_report');
|
|
1731
|
+
expect(reportCheck.get('continue-on-error')).toBeUndefined();
|
|
1732
|
+
expect(reportCheck.get('if')).toBe("${{ !cancelled() && steps.agentic_execution.outcome == 'success' }}");
|
|
1733
|
+
expect(reportCheck.getIn(['env', 'PRIMER_AGENT_OUTPUT'])).toBe('/tmp/gh-aw/agent_output.json');
|
|
1734
|
+
expect(agentSteps.items.indexOf(getStep('agentic_execution'))).toBeLessThan(agentSteps.items.indexOf(getStep('collect_output')));
|
|
1735
|
+
expect(agentSteps.items.indexOf(getStep('collect_output'))).toBeLessThan(agentSteps.items.indexOf(reportCheck));
|
|
1736
|
+
const upload = agentSteps.items.find((item) => (0, yaml_1.isMap)(item) && item.getIn(['with', 'name']) === 'agent-output-fallback');
|
|
1737
|
+
if (!(0, yaml_1.isMap)(upload)) {
|
|
1738
|
+
throw new Error('Missing output artifact upload');
|
|
1739
|
+
}
|
|
1740
|
+
expect(upload.get('if')).toBe('always()');
|
|
1741
|
+
expect(agentSteps.items.indexOf(reportCheck)).toBeLessThan(agentSteps.items.indexOf(upload));
|
|
1742
|
+
const configStep = agentSteps.items.find((item) => (0, yaml_1.isMap)(item) && item.getIn(['env', 'GH_AW_SAFE_OUTPUTS_CONFIG']) !== undefined);
|
|
1743
|
+
if (!(0, yaml_1.isMap)(configStep)) {
|
|
1744
|
+
throw new Error('Missing safe outputs configuration');
|
|
1745
|
+
}
|
|
1746
|
+
const safeOutputConfig = (0, yaml_1.parseDocument)(String(configStep.getIn(['env', 'GH_AW_SAFE_OUTPUTS_CONFIG'])));
|
|
1747
|
+
expect(safeOutputConfig.has('noop')).toBe(false);
|
|
1748
|
+
expect(safeOutputConfig.has('publish-primer-analysis')).toBe(true);
|
|
1749
|
+
expect(safeOutputConfig.getIn(['publish-primer-analysis', 'max'])).toBe(100);
|
|
1750
|
+
const execution = getStep('agentic_execution').get('run');
|
|
1751
|
+
expect(getStep('agentic_execution').getIn(['env', 'COPILOT_MODEL'])).toBe('gpt-5.6-sol');
|
|
1752
|
+
const awfConfig = /^printf '%s\\n' '(.+)' > "\$\{RUNNER_TEMP\}\/gh-aw\/awf-config\.json"$/m.exec(String(execution));
|
|
1753
|
+
if (!awfConfig) {
|
|
1754
|
+
throw new Error('Missing generated firewall configuration');
|
|
1755
|
+
}
|
|
1756
|
+
const proxy = (0, yaml_1.parseDocument)(awfConfig[1]);
|
|
1757
|
+
expect(proxy.getIn(['apiProxy', 'enableTokenSteering'])).toBe(false);
|
|
1758
|
+
expect(proxy.getIn(['apiProxy', 'maxAiCredits'])).toBe(200);
|
|
1759
|
+
expect(proxy.getIn(['apiProxy', 'maxRuns'])).toBe(30);
|
|
1760
|
+
expect(execution).toContain('/gh-aw/actions/mypyPrimerCopilotHarness.cjs"');
|
|
1761
|
+
expect(execution).toContain('export GH_AW_MCP_CONFIG="$HOME/.copilot/mcp-config.json"');
|
|
1762
|
+
expect(execution).toContain('--allow-tool github');
|
|
1763
|
+
expect(execution).toContain('--allow-tool safeoutputs');
|
|
1764
|
+
expect(execution).toContain('--deny-tool shell');
|
|
1765
|
+
expect(execution).toContain('--deny-tool write');
|
|
1766
|
+
expect(execution).not.toContain('--allow-tool shell');
|
|
1767
|
+
expect(execution).not.toContain('--allow-tool write');
|
|
1768
|
+
const detectionSteps = workflow.getIn(['jobs', 'detection', 'steps'], true);
|
|
1769
|
+
if (!(0, yaml_1.isSeq)(detectionSteps)) {
|
|
1770
|
+
throw new Error('Missing detection steps');
|
|
1771
|
+
}
|
|
1772
|
+
const detection = detectionSteps.items.find((item) => (0, yaml_1.isMap)(item) && item.getIn(['env', 'COPILOT_MODEL']) !== undefined);
|
|
1773
|
+
if (!(0, yaml_1.isMap)(detection)) {
|
|
1774
|
+
throw new Error('Missing threat detection model configuration');
|
|
1775
|
+
}
|
|
1776
|
+
expect(detection.getIn(['env', 'COPILOT_MODEL'])).toBe('detection');
|
|
1777
|
+
});
|
|
632
1778
|
});
|
|
633
1779
|
//# sourceMappingURL=mypyPrimerAnalysis.test.js.map
|