@tea-agent/loop-agent 0.30.0 → 0.31.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/executors/dag-pi-executor.js +72 -4
- package/dist/executors/pi-sdk-executor.js +61 -0
- package/dist/executors/shell-executor.js +136 -79
- package/dist/worker/observability/dag-execution-trajectory.js +591 -0
- package/dist/worker/observability/read-model.js +258 -31
- package/dist/worker/observe/dag-node-execution-output.js +180 -0
- package/dist/worker/observe/routes.js +53 -6
- package/dist/worker/observe/static/dag-edge-routing.js +368 -0
- package/dist/worker/observe/static/dag-history-labels.js +95 -0
- package/dist/worker/observe/static/dag-layout.d.ts +12 -7
- package/dist/worker/observe/static/dag-layout.js +101 -21
- package/dist/worker/observe/static/favicon.svg +37 -0
- package/dist/worker/observe/static/format.js +31 -1
- package/dist/worker/observe/static/index.html +1 -1
- package/dist/worker/observe/static/state.js +102 -0
- package/dist/worker/observe/static/styles.css +267 -7
- package/dist/worker/observe/static/views/dag-graph.js +414 -154
- package/dist/worker/observe/static/views/dag-inspector.js +478 -27
- package/dist/worker/observe/static/views/dag-trajectory.js +313 -0
- package/dist/worker/observe/static/views/dag.js +20 -3
- package/dist/workflows/dag/backend-test-pytest-collection.js +162 -7
- package/dist/workflows/dag/backend-test-result-contract.js +105 -67
- package/dist/workflows/dag/backend-test-scenario-param.js +92 -30
- package/dist/workflows/dag/backend-test-writer-completeness.js +55 -0
- package/dist/workflows/dag/init-hybrid.js +46 -49
- package/dist/workflows/dag/rerun-task.js +86 -0
- package/docs/architecture/README.md +4 -0
- package/docs/architecture/dag-execution.md +1 -1
- package/docs/architecture/worker-and-feature.md +1 -1
- package/docs/governance/README.md +3 -0
- package/docs/operations/README.md +1 -0
- package/docs/templates/backend-test-dag.json +40 -60
- package/docs/templates/frontend-test-dag.json +1 -1
- package/harness.json +3 -3
- package/package.json +3 -2
- package/scripts/kb-bootstrap-init-skeleton.sh +1 -1
|
@@ -17,6 +17,8 @@ export const BACKEND_TEST_CLASSIFICATION_CATEGORIES = [
|
|
|
17
17
|
export const backendTestExecutionStatusSchema = z.enum([
|
|
18
18
|
"completed",
|
|
19
19
|
"collection-error",
|
|
20
|
+
"setup-error",
|
|
21
|
+
"teardown-error",
|
|
20
22
|
"command-error",
|
|
21
23
|
"report-error",
|
|
22
24
|
]);
|
|
@@ -38,6 +40,8 @@ export const backendTestOutcomeSchema = z.enum([
|
|
|
38
40
|
"passed",
|
|
39
41
|
"completed-with-failures",
|
|
40
42
|
"collection-error",
|
|
43
|
+
"setup-error",
|
|
44
|
+
"teardown-error",
|
|
41
45
|
"command-error",
|
|
42
46
|
"report-error",
|
|
43
47
|
]);
|
|
@@ -55,6 +59,10 @@ const failureSummarySchema = z
|
|
|
55
59
|
name: z.string().min(1),
|
|
56
60
|
message: z.string().min(1),
|
|
57
61
|
kind: z.enum(["failure", "error"]).default("failure"),
|
|
62
|
+
nodeId: z.string().min(1).optional(),
|
|
63
|
+
caseId: z.string().min(1).optional(),
|
|
64
|
+
tpId: z.string().min(1).optional(),
|
|
65
|
+
phase: z.enum(["collection", "setup", "call", "teardown", "unknown"]).optional(),
|
|
58
66
|
})
|
|
59
67
|
.strict();
|
|
60
68
|
export const backendTestResultContractSchema = z
|
|
@@ -70,6 +78,12 @@ export const backendTestResultContractSchema = z
|
|
|
70
78
|
failed: z.number().int().min(0),
|
|
71
79
|
error: z.number().int().min(0),
|
|
72
80
|
skipped: z.number().int().min(0),
|
|
81
|
+
collectedItemCount: z.number().int().min(0),
|
|
82
|
+
setupStartedCount: z.number().int().min(0),
|
|
83
|
+
businessTestBodyExecutedCount: z.number().int().min(0),
|
|
84
|
+
setupErrorCount: z.number().int().min(0),
|
|
85
|
+
testFailureCount: z.number().int().min(0),
|
|
86
|
+
teardownErrorCount: z.number().int().min(0),
|
|
73
87
|
durationMs: z.number().nonnegative().optional(),
|
|
74
88
|
junit: z
|
|
75
89
|
.object({
|
|
@@ -403,6 +417,39 @@ function parseDurationLabelMs(raw) {
|
|
|
403
417
|
// pytest-html commonly emits bare milliseconds without a unit suffix.
|
|
404
418
|
return Math.round(value);
|
|
405
419
|
}
|
|
420
|
+
function canonicalPytestItemId(testId) {
|
|
421
|
+
return testId.replace(/::(?:setup|teardown)$/i, "");
|
|
422
|
+
}
|
|
423
|
+
function pytestHtmlPhase(testId, log) {
|
|
424
|
+
if (/::setup$/i.test(testId) || /ERROR at setup|Captured setup/i.test(log))
|
|
425
|
+
return "setup";
|
|
426
|
+
if (/::teardown$/i.test(testId) || /ERROR at teardown|Captured teardown/i.test(log))
|
|
427
|
+
return "teardown";
|
|
428
|
+
if (/collecting|ERROR collecting/i.test(log))
|
|
429
|
+
return "collection";
|
|
430
|
+
return "call";
|
|
431
|
+
}
|
|
432
|
+
function pytestFailureIdentity(nodeId) {
|
|
433
|
+
const caseToken = nodeId.match(/BE[-_][A-Z0-9_-]+[-_]\d{2,3}/i)?.[0];
|
|
434
|
+
const caseId = caseToken
|
|
435
|
+
? caseToken.replace(/^BE_/i, "BE-").replaceAll("_", "-").toUpperCase()
|
|
436
|
+
: undefined;
|
|
437
|
+
const tpId = nodeId.match(/\[(TP-[A-Z0-9-]+)\]/i)?.[1]?.toUpperCase();
|
|
438
|
+
return { ...(caseId ? { caseId } : {}), ...(tpId ? { tpId } : {}) };
|
|
439
|
+
}
|
|
440
|
+
function parsedExecutionCounts(parsed) {
|
|
441
|
+
const setupErrorCount = parsed.cases.filter((item) => item.status === "error" && (item.phase === "setup" || /setup|fixture ['"][^'"]+['"] not found/i.test(`${item.name} ${item.message ?? ""} ${item.details ?? ""}`))).length;
|
|
442
|
+
const teardownErrorCount = parsed.cases.filter((item) => item.status === "error" && (item.phase === "teardown" || /teardown/i.test(`${item.name} ${item.message ?? ""} ${item.details ?? ""}`))).length;
|
|
443
|
+
const businessTestBodyExecutedCount = parsed.cases.filter((item) => item.status === "passed" || item.status === "failure" || (item.status === "error" && item.phase === "call" && !/fixture ['"][^'"]+['"] not found/i.test(`${item.message ?? ""} ${item.details ?? ""}`))).length;
|
|
444
|
+
return {
|
|
445
|
+
collectedItemCount: parsed.tests,
|
|
446
|
+
setupStartedCount: Math.max(0, parsed.tests - parsed.cases.filter((item) => item.phase === "collection").length),
|
|
447
|
+
businessTestBodyExecutedCount,
|
|
448
|
+
setupErrorCount,
|
|
449
|
+
testFailureCount: parsed.failed,
|
|
450
|
+
teardownErrorCount,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
406
453
|
/**
|
|
407
454
|
* Extract the pytest-html 4.x JSON island from a self-contained HTML report.
|
|
408
455
|
*
|
|
@@ -451,93 +498,46 @@ export function parsePytestHtmlReport(html) {
|
|
|
451
498
|
if (!record)
|
|
452
499
|
continue;
|
|
453
500
|
const rawResult = (record.result ?? "").toLowerCase();
|
|
454
|
-
const
|
|
455
|
-
const
|
|
501
|
+
const rawTestId = record.testId ?? nodeId;
|
|
502
|
+
const capturedLog = record.log ?? "";
|
|
503
|
+
const phase = pytestHtmlPhase(rawTestId, capturedLog);
|
|
504
|
+
const canonicalNodeId = canonicalPytestItemId(nodeId.includes("::") ? nodeId : rawTestId);
|
|
505
|
+
const { classname, name, filePath } = splitPytestNodeId(canonicalNodeId);
|
|
506
|
+
const identity = pytestFailureIdentity(canonicalNodeId);
|
|
456
507
|
const filePathFields = filePath ? { filePath } : {};
|
|
457
508
|
const durationMs = parseDurationLabelMs(record.duration);
|
|
458
|
-
const capturedLog = record.log ?? "";
|
|
459
509
|
const decodedCapturedLog = decodeHtmlEntitiesDeep(capturedLog);
|
|
460
|
-
// pytest-html collapses captured stdout/stderr into a single `log` field
|
|
461
|
-
// annotated with section markers. Preserve the whole log as stdout so the
|
|
462
|
-
// HTTP_REQUEST/HTTP_RESPONSE lines stay reachable for the per-case card.
|
|
463
510
|
const stdout = splitPytestHtmlLogSections(capturedLog).stdout || undefined;
|
|
464
511
|
const stderr = splitPytestHtmlLogSections(capturedLog).stderr || undefined;
|
|
512
|
+
const baseCase = { classname, name, nodeId: canonicalNodeId, phase, ...filePathFields, durationMs };
|
|
465
513
|
if (rawResult === "passed") {
|
|
466
514
|
passed += 1;
|
|
467
|
-
cases.push({
|
|
468
|
-
classname,
|
|
469
|
-
name,
|
|
470
|
-
...filePathFields,
|
|
471
|
-
...filePathFields,
|
|
472
|
-
durationMs,
|
|
473
|
-
status: "passed",
|
|
474
|
-
...(stdout ? { stdout } : {}),
|
|
475
|
-
...(stderr ? { stderr } : {}),
|
|
476
|
-
});
|
|
515
|
+
cases.push({ ...baseCase, status: "passed", ...(stdout ? { stdout } : {}), ...(stderr ? { stderr } : {}) });
|
|
477
516
|
}
|
|
478
517
|
else if (rawResult === "failed") {
|
|
479
518
|
failed += 1;
|
|
480
519
|
const message = extractPytestHtmlFailureMessage(capturedLog) || "failure";
|
|
481
520
|
const summary = truncate(message);
|
|
482
|
-
failures.push({ classname, name, message: summary, kind: "failure" });
|
|
483
|
-
cases.push({
|
|
484
|
-
classname,
|
|
485
|
-
name,
|
|
486
|
-
...filePathFields,
|
|
487
|
-
durationMs,
|
|
488
|
-
status: "failure",
|
|
489
|
-
message: summary,
|
|
490
|
-
details: decodedCapturedLog || summary,
|
|
491
|
-
...(stdout ? { stdout } : {}),
|
|
492
|
-
...(stderr ? { stderr } : {}),
|
|
493
|
-
});
|
|
521
|
+
failures.push({ classname, name, message: summary, kind: "failure", nodeId: canonicalNodeId, phase, ...identity });
|
|
522
|
+
cases.push({ ...baseCase, status: "failure", message: summary, details: decodedCapturedLog || summary, ...(stdout ? { stdout } : {}), ...(stderr ? { stderr } : {}) });
|
|
494
523
|
}
|
|
495
524
|
else if (rawResult === "error") {
|
|
496
525
|
errors += 1;
|
|
497
526
|
const message = extractPytestHtmlFailureMessage(capturedLog) || "error";
|
|
498
527
|
const summary = truncate(message);
|
|
499
|
-
failures.push({ classname, name, message: summary, kind: "error" });
|
|
500
|
-
cases.push({
|
|
501
|
-
classname,
|
|
502
|
-
name,
|
|
503
|
-
...filePathFields,
|
|
504
|
-
durationMs,
|
|
505
|
-
status: "error",
|
|
506
|
-
message: summary,
|
|
507
|
-
details: decodedCapturedLog || summary,
|
|
508
|
-
...(stdout ? { stdout } : {}),
|
|
509
|
-
...(stderr ? { stderr } : {}),
|
|
510
|
-
});
|
|
528
|
+
failures.push({ classname, name, message: summary, kind: "error", nodeId: canonicalNodeId, phase, ...identity });
|
|
529
|
+
cases.push({ ...baseCase, status: "error", message: summary, details: decodedCapturedLog || summary, ...(stdout ? { stdout } : {}), ...(stderr ? { stderr } : {}) });
|
|
511
530
|
}
|
|
512
531
|
else if (rawResult === "skipped" || rawResult === "xfailed") {
|
|
513
532
|
skipped += 1;
|
|
514
|
-
cases.push({
|
|
515
|
-
classname,
|
|
516
|
-
name,
|
|
517
|
-
...filePathFields,
|
|
518
|
-
durationMs,
|
|
519
|
-
status: "skipped",
|
|
520
|
-
...(stdout ? { stdout } : {}),
|
|
521
|
-
...(stderr ? { stderr } : {}),
|
|
522
|
-
});
|
|
533
|
+
cases.push({ ...baseCase, status: "skipped", ...(stdout ? { stdout } : {}), ...(stderr ? { stderr } : {}) });
|
|
523
534
|
}
|
|
524
535
|
else {
|
|
525
|
-
// Unknown outcome label: treat as error to stay fail-safe.
|
|
526
536
|
errors += 1;
|
|
527
537
|
const message = `unexpected pytest-html result label: ${record.result ?? "(empty)"}`;
|
|
528
538
|
const summary = truncate(message);
|
|
529
|
-
failures.push({ classname, name, message: summary, kind: "error" });
|
|
530
|
-
cases.push({
|
|
531
|
-
classname,
|
|
532
|
-
name,
|
|
533
|
-
...filePathFields,
|
|
534
|
-
durationMs,
|
|
535
|
-
status: "error",
|
|
536
|
-
message: summary,
|
|
537
|
-
details: decodedCapturedLog || summary,
|
|
538
|
-
...(stdout ? { stdout } : {}),
|
|
539
|
-
...(stderr ? { stderr } : {}),
|
|
540
|
-
});
|
|
539
|
+
failures.push({ classname, name, message: summary, kind: "error", nodeId: canonicalNodeId, phase, ...identity });
|
|
540
|
+
cases.push({ ...baseCase, status: "error", message: summary, details: decodedCapturedLog || summary, ...(stdout ? { stdout } : {}), ...(stderr ? { stderr } : {}) });
|
|
541
541
|
}
|
|
542
542
|
}
|
|
543
543
|
const tests = cases.length;
|
|
@@ -666,6 +666,12 @@ export function deriveBackendTestResult(input) {
|
|
|
666
666
|
failed: 0,
|
|
667
667
|
error: 0,
|
|
668
668
|
skipped: 0,
|
|
669
|
+
collectedItemCount: 0,
|
|
670
|
+
setupStartedCount: 0,
|
|
671
|
+
businessTestBodyExecutedCount: 0,
|
|
672
|
+
setupErrorCount: 0,
|
|
673
|
+
testFailureCount: 0,
|
|
674
|
+
teardownErrorCount: 0,
|
|
669
675
|
junit: emptyJunitMeta,
|
|
670
676
|
commandSummary: input.commandSummary,
|
|
671
677
|
failures: [],
|
|
@@ -682,13 +688,11 @@ export function deriveBackendTestResult(input) {
|
|
|
682
688
|
}
|
|
683
689
|
const sha256 = createHash("sha256").update(input.junitXml).digest("hex");
|
|
684
690
|
const exit = input.pytestExitCode;
|
|
691
|
+
const executionCounts = parsedExecutionCounts(parsed);
|
|
685
692
|
let executionStatus = "completed";
|
|
686
693
|
let collectionStatus = "ok";
|
|
687
694
|
let outcome = "passed";
|
|
688
|
-
// Collection-heavy signals: pytest exit 2 is common for collection errors;
|
|
689
|
-
// also when error cases exist with zero/low completed tests.
|
|
690
695
|
const looksLikeCollection = exit === 2 ||
|
|
691
|
-
(parsed.errors > 0 && parsed.passed + parsed.failed === 0) ||
|
|
692
696
|
parsed.failures.some((f) => f.kind === "error" &&
|
|
693
697
|
/collect|import|syntax/i.test(`${f.name} ${f.message}`));
|
|
694
698
|
if (looksLikeCollection && (parsed.errors > 0 || exit >= 2)) {
|
|
@@ -696,6 +700,16 @@ export function deriveBackendTestResult(input) {
|
|
|
696
700
|
collectionStatus = "error";
|
|
697
701
|
outcome = "collection-error";
|
|
698
702
|
}
|
|
703
|
+
else if (executionCounts.setupErrorCount > 0) {
|
|
704
|
+
executionStatus = "setup-error";
|
|
705
|
+
collectionStatus = "ok";
|
|
706
|
+
outcome = "setup-error";
|
|
707
|
+
}
|
|
708
|
+
else if (executionCounts.teardownErrorCount > 0) {
|
|
709
|
+
executionStatus = "teardown-error";
|
|
710
|
+
collectionStatus = "ok";
|
|
711
|
+
outcome = "teardown-error";
|
|
712
|
+
}
|
|
699
713
|
else if (exit >= 2 && parsed.failed === 0 && parsed.errors === 0) {
|
|
700
714
|
executionStatus = "command-error";
|
|
701
715
|
collectionStatus = "unknown";
|
|
@@ -726,6 +740,7 @@ export function deriveBackendTestResult(input) {
|
|
|
726
740
|
failed: parsed.failed,
|
|
727
741
|
error: parsed.errors,
|
|
728
742
|
skipped: parsed.skipped,
|
|
743
|
+
...executionCounts,
|
|
729
744
|
durationMs: parsed.durationMs,
|
|
730
745
|
junit: {
|
|
731
746
|
relativePath: input.junitRelativePath,
|
|
@@ -737,6 +752,10 @@ export function deriveBackendTestResult(input) {
|
|
|
737
752
|
name: f.name || "unknown",
|
|
738
753
|
message: truncate(f.message || "failure"),
|
|
739
754
|
kind: f.kind,
|
|
755
|
+
...(f.nodeId ? { nodeId: f.nodeId } : {}),
|
|
756
|
+
...(f.caseId ? { caseId: f.caseId } : {}),
|
|
757
|
+
...(f.tpId ? { tpId: f.tpId } : {}),
|
|
758
|
+
...(f.phase ? { phase: f.phase } : {}),
|
|
740
759
|
})),
|
|
741
760
|
outcome,
|
|
742
761
|
};
|
|
@@ -759,15 +778,19 @@ export function classifyCategoryHints(result) {
|
|
|
759
778
|
// Single observation cannot prove flakiness.
|
|
760
779
|
forbidden.add("FlakyTest");
|
|
761
780
|
if (result.executionStatus === "collection-error" ||
|
|
781
|
+
result.executionStatus === "setup-error" ||
|
|
782
|
+
result.executionStatus === "teardown-error" ||
|
|
762
783
|
result.executionStatus === "command-error" ||
|
|
763
784
|
result.executionStatus === "report-error" ||
|
|
764
785
|
result.outcome === "collection-error" ||
|
|
786
|
+
result.outcome === "setup-error" ||
|
|
787
|
+
result.outcome === "teardown-error" ||
|
|
765
788
|
result.outcome === "command-error" ||
|
|
766
789
|
result.outcome === "report-error") {
|
|
767
790
|
forbidden.add("ProductBug");
|
|
768
791
|
suggested.add("EnvFailure");
|
|
769
792
|
suggested.add("Unknown");
|
|
770
|
-
if (
|
|
793
|
+
if (["collection-error", "setup-error", "teardown-error"].includes(result.executionStatus)) {
|
|
771
794
|
suggested.add("TestBug");
|
|
772
795
|
}
|
|
773
796
|
return {
|
|
@@ -865,11 +888,11 @@ export async function materializeBackendTestResultFromPytestHtml(input) {
|
|
|
865
888
|
}
|
|
866
889
|
const sha256 = createHash("sha256").update(html).digest("hex");
|
|
867
890
|
const exit = input.pytestExitCode;
|
|
891
|
+
const executionCounts = parsedExecutionCounts(parsed);
|
|
868
892
|
let executionStatus = "completed";
|
|
869
893
|
let collectionStatus = "ok";
|
|
870
894
|
let outcome = "passed";
|
|
871
895
|
const looksLikeCollection = exit === 2 ||
|
|
872
|
-
(parsed.errors > 0 && parsed.passed + parsed.failed === 0) ||
|
|
873
896
|
parsed.failures.some((f) => f.kind === "error" &&
|
|
874
897
|
/collect|import|syntax/i.test(`${f.name} ${f.message}`));
|
|
875
898
|
if (looksLikeCollection && (parsed.errors > 0 || exit >= 2)) {
|
|
@@ -877,6 +900,16 @@ export async function materializeBackendTestResultFromPytestHtml(input) {
|
|
|
877
900
|
collectionStatus = "error";
|
|
878
901
|
outcome = "collection-error";
|
|
879
902
|
}
|
|
903
|
+
else if (executionCounts.setupErrorCount > 0) {
|
|
904
|
+
executionStatus = "setup-error";
|
|
905
|
+
collectionStatus = "ok";
|
|
906
|
+
outcome = "setup-error";
|
|
907
|
+
}
|
|
908
|
+
else if (executionCounts.teardownErrorCount > 0) {
|
|
909
|
+
executionStatus = "teardown-error";
|
|
910
|
+
collectionStatus = "ok";
|
|
911
|
+
outcome = "teardown-error";
|
|
912
|
+
}
|
|
880
913
|
else if (exit >= 2 && parsed.failed === 0 && parsed.errors === 0) {
|
|
881
914
|
executionStatus = "command-error";
|
|
882
915
|
collectionStatus = "unknown";
|
|
@@ -910,6 +943,7 @@ export async function materializeBackendTestResultFromPytestHtml(input) {
|
|
|
910
943
|
failed: parsed.failed,
|
|
911
944
|
error: parsed.errors,
|
|
912
945
|
skipped: parsed.skipped,
|
|
946
|
+
...executionCounts,
|
|
913
947
|
durationMs: parsed.durationMs,
|
|
914
948
|
junit: {
|
|
915
949
|
relativePath: htmlRelativePath,
|
|
@@ -921,6 +955,10 @@ export async function materializeBackendTestResultFromPytestHtml(input) {
|
|
|
921
955
|
name: f.name || "unknown",
|
|
922
956
|
message: truncate(f.message || "failure"),
|
|
923
957
|
kind: f.kind,
|
|
958
|
+
...(f.nodeId ? { nodeId: f.nodeId } : {}),
|
|
959
|
+
...(f.caseId ? { caseId: f.caseId } : {}),
|
|
960
|
+
...(f.tpId ? { tpId: f.tpId } : {}),
|
|
961
|
+
...(f.phase ? { phase: f.phase } : {}),
|
|
924
962
|
})),
|
|
925
963
|
outcome,
|
|
926
964
|
});
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { spawnSync } from "node:child_process";
|
|
2
3
|
import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { z } from "zod";
|
|
@@ -8,6 +9,7 @@ const factsSchema = z
|
|
|
8
9
|
.object({
|
|
9
10
|
schemaId: z.literal(BACKEND_TEST_SCENARIO_PARAM_FACTS_SCHEMA_ID),
|
|
10
11
|
phase: z.enum(["initial", "final"]),
|
|
12
|
+
overallStatus: z.enum(["PASS", "PARTIAL", "FAIL", "UNAVAILABLE"]),
|
|
11
13
|
repairEligible: z.boolean(),
|
|
12
14
|
repairAttempt: z.number().int().min(0).max(1),
|
|
13
15
|
strictScenarioParamGate: z.boolean(),
|
|
@@ -115,23 +117,28 @@ async function listMarkdownCases(workspaceRoot) {
|
|
|
115
117
|
}
|
|
116
118
|
export function inferScenarioParamIntent(input) {
|
|
117
119
|
const { tpId, caseBody } = input;
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
+
const preciseLine = /场景意图\s*[::]\s*(TP-[A-Z0-9-]+)\s*[|;,,]\s*operation\s*=\s*([^|;,,\n]+)\s*[|;,,]\s*target\s*=\s*([A-Za-z0-9_.-]+)\s*[|;,,]\s*intent\s*=\s*([a-z0-9+._:-]+)(?:\s*[|;,,]\s*bound\s*=\s*(\d+))?(?:\s*[|;,,]\s*example\s*=\s*([^|\n]+))?/i.exec(caseBody);
|
|
121
|
+
const legacyLine = /场景意图\s*[::]\s*([a-z0-9+._:-]+)(?:\s*[|;,,]\s*field\s*=\s*([A-Za-z0-9_.]+))?(?:\s*[|;,,]\s*bound\s*=\s*(\d+))?(?:\s*[|;,,]\s*example\s*=\s*([^|\n]+))?/i.exec(caseBody);
|
|
122
|
+
const fallbackLine = new RegExp(`${tpId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}[^\\n]{0,160}?intent\\s*[=:]\\s*([a-z0-9+._:-]+)`, "i").exec(caseBody);
|
|
123
|
+
const machineLine = preciseLine ?? legacyLine ?? fallbackLine;
|
|
120
124
|
if (machineLine) {
|
|
121
|
-
const rawIntent = (machineLine[1] ?? "unknown").toLowerCase();
|
|
122
|
-
const
|
|
123
|
-
const
|
|
124
|
-
const
|
|
125
|
-
|
|
125
|
+
const rawIntent = (preciseLine ? preciseLine[4] : machineLine[1] ?? "unknown").toLowerCase();
|
|
126
|
+
const target = preciseLine?.[3];
|
|
127
|
+
const field = target && target !== "request" ? target.split(".").at(-1) : legacyLine?.[2];
|
|
128
|
+
const boundRaw = preciseLine?.[5] ?? legacyLine?.[3];
|
|
129
|
+
const example = (preciseLine?.[6] ?? legacyLine?.[4])?.trim();
|
|
130
|
+
const bound = boundRaw ? Number(boundRaw) : undefined;
|
|
131
|
+
const normalizedIntent = rawIntent === "nominal-operation" ? "nominal" : rawIntent;
|
|
132
|
+
if (normalizedIntent.startsWith("custom-literal:")) {
|
|
126
133
|
return {
|
|
127
|
-
intent:
|
|
134
|
+
intent: normalizedIntent,
|
|
128
135
|
field,
|
|
129
136
|
bound,
|
|
130
137
|
example,
|
|
131
138
|
};
|
|
132
139
|
}
|
|
133
140
|
return {
|
|
134
|
-
intent: (CLOSED_SET.has(
|
|
141
|
+
intent: (CLOSED_SET.has(normalizedIntent) ? normalizedIntent : "unknown"),
|
|
135
142
|
field,
|
|
136
143
|
bound,
|
|
137
144
|
example,
|
|
@@ -240,6 +247,9 @@ export function observeParamFeatures(block, field) {
|
|
|
240
247
|
// Prefer dict-like payload extraction.
|
|
241
248
|
const dictMatch = /\{\s*([\s\S]*?)\s*\}/.exec(block) ??
|
|
242
249
|
/(?:payload|body|data|json)\s*=\s*(\{[\s\S]*?\})/.exec(block);
|
|
250
|
+
if (dictMatch && !field) {
|
|
251
|
+
return { kind: "dict", text: (dictMatch[0] ?? dictMatch[1] ?? "{}").replace(/\s+/g, " ").slice(0, 160) };
|
|
252
|
+
}
|
|
243
253
|
if (dictMatch && field) {
|
|
244
254
|
const dict = dictMatch[0] ?? dictMatch[1] ?? "";
|
|
245
255
|
const fieldPattern = new RegExp(`["']${field.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}["']\\s*:`);
|
|
@@ -531,19 +541,29 @@ export async function assessBackendScenarioParamConsistency(input) {
|
|
|
531
541
|
});
|
|
532
542
|
}
|
|
533
543
|
const repairableCount = entries.filter((entry) => entry.repairability === "repairable" && entry.status === "MISMATCH").length;
|
|
544
|
+
const matchCount = entries.filter((entry) => entry.status === "MATCH").length;
|
|
545
|
+
const mismatchCount = entries.filter((entry) => entry.status === "MISMATCH").length;
|
|
546
|
+
const undeterminedCount = entries.filter((entry) => entry.status === "UNDETERMINED").length;
|
|
547
|
+
const overallStatus = mismatchCount > 0
|
|
548
|
+
? "FAIL"
|
|
549
|
+
: matchCount === 0 && undeterminedCount > 0
|
|
550
|
+
? "UNAVAILABLE"
|
|
551
|
+
: undeterminedCount > 0
|
|
552
|
+
? "PARTIAL"
|
|
553
|
+
: "PASS";
|
|
534
554
|
const facts = {
|
|
535
555
|
schemaId: BACKEND_TEST_SCENARIO_PARAM_FACTS_SCHEMA_ID,
|
|
536
556
|
phase: input.phase,
|
|
557
|
+
overallStatus,
|
|
537
558
|
repairEligible: input.phase === "initial" && repairableCount > 0,
|
|
538
559
|
repairAttempt: input.repairAttempt ?? (input.phase === "final" ? 1 : 0),
|
|
539
560
|
strictScenarioParamGate: input.strictScenarioParamGate === true,
|
|
540
561
|
entries,
|
|
541
562
|
assetHashes,
|
|
542
563
|
summary: {
|
|
543
|
-
matchCount
|
|
544
|
-
mismatchCount
|
|
545
|
-
|
|
546
|
-
undeterminedCount: entries.filter((entry) => entry.status === "UNDETERMINED").length,
|
|
564
|
+
matchCount,
|
|
565
|
+
mismatchCount,
|
|
566
|
+
undeterminedCount,
|
|
547
567
|
repairableCount,
|
|
548
568
|
blockedCount: entries.filter((entry) => entry.repairability === "blocked")
|
|
549
569
|
.length,
|
|
@@ -554,7 +574,7 @@ export async function assessBackendScenarioParamConsistency(input) {
|
|
|
554
574
|
"",
|
|
555
575
|
`## Status`,
|
|
556
576
|
"",
|
|
557
|
-
facts.
|
|
577
|
+
facts.overallStatus,
|
|
558
578
|
"",
|
|
559
579
|
`## Summary`,
|
|
560
580
|
"",
|
|
@@ -615,10 +635,12 @@ function replaceFieldInDictLiteral(dictLiteral, field, mode, valueLiteral) {
|
|
|
615
635
|
if (fieldPattern.test(dictLiteral)) {
|
|
616
636
|
return dictLiteral.replace(fieldPattern, `$1$2${valueLiteral}`);
|
|
617
637
|
}
|
|
618
|
-
// Insert before closing brace.
|
|
619
|
-
|
|
620
|
-
const
|
|
621
|
-
|
|
638
|
+
// Insert before closing brace. Normalize an existing trailing comma first so
|
|
639
|
+
// adding a missing field can never produce `,,` and corrupt Python syntax.
|
|
640
|
+
const withoutClosingBrace = dictLiteral.replace(/\s*\}$/, "");
|
|
641
|
+
const trimmed = withoutClosingBrace.replace(/,\s*$/, "").trimEnd();
|
|
642
|
+
const hasEntries = trimmed.trim() !== "{";
|
|
643
|
+
return `${trimmed}${hasEntries ? ", " : ""}${JSON.stringify(field)}: ${valueLiteral}}`;
|
|
622
644
|
}
|
|
623
645
|
export function deterministicRewriteScenarioParam(input) {
|
|
624
646
|
const { entry } = input;
|
|
@@ -693,6 +715,32 @@ export function deterministicRewriteScenarioParam(input) {
|
|
|
693
715
|
}
|
|
694
716
|
return { ok: true, source: nextSource, detail: "rewritten" };
|
|
695
717
|
}
|
|
718
|
+
function validatePythonSyntaxWithoutExecution(source) {
|
|
719
|
+
const result = spawnSync("python", ["-c", "import ast,sys; ast.parse(sys.stdin.read())"], {
|
|
720
|
+
input: source,
|
|
721
|
+
encoding: "utf8",
|
|
722
|
+
env: {
|
|
723
|
+
...process.env,
|
|
724
|
+
PYTHONDONTWRITEBYTECODE: "1",
|
|
725
|
+
PYTHONUTF8: "1",
|
|
726
|
+
},
|
|
727
|
+
timeout: 15_000,
|
|
728
|
+
});
|
|
729
|
+
if (result.error) {
|
|
730
|
+
return {
|
|
731
|
+
ok: false,
|
|
732
|
+
detail: `python syntax validation unavailable: ${result.error.message}`,
|
|
733
|
+
};
|
|
734
|
+
}
|
|
735
|
+
if (result.status !== 0) {
|
|
736
|
+
const detail = String(result.stderr || result.stdout || "invalid Python syntax")
|
|
737
|
+
.trim()
|
|
738
|
+
.replace(/\s+/g, " ")
|
|
739
|
+
.slice(0, 500);
|
|
740
|
+
return { ok: false, detail: `python syntax validation failed: ${detail}` };
|
|
741
|
+
}
|
|
742
|
+
return { ok: true, detail: "python ast.parse PASS" };
|
|
743
|
+
}
|
|
696
744
|
export async function applyDeterministicScenarioParamRepairs(input) {
|
|
697
745
|
const repairable = input.facts.entries.filter((entry) => entry.status === "MISMATCH" && entry.repairability === "repairable");
|
|
698
746
|
const byScript = new Map();
|
|
@@ -722,7 +770,7 @@ export async function applyDeterministicScenarioParamRepairs(input) {
|
|
|
722
770
|
continue;
|
|
723
771
|
}
|
|
724
772
|
let source = await readFile(absolute, "utf8");
|
|
725
|
-
|
|
773
|
+
const staged = [];
|
|
726
774
|
for (const entry of entries) {
|
|
727
775
|
const result = deterministicRewriteScenarioParam({ source, entry });
|
|
728
776
|
if (!result.ok) {
|
|
@@ -735,17 +783,31 @@ export async function applyDeterministicScenarioParamRepairs(input) {
|
|
|
735
783
|
continue;
|
|
736
784
|
}
|
|
737
785
|
source = result.source;
|
|
738
|
-
|
|
739
|
-
repaired.push(entry.tpId);
|
|
740
|
-
auditEntries.push({
|
|
741
|
-
tpId: entry.tpId,
|
|
742
|
-
result: "rewritten",
|
|
743
|
-
detail: result.detail,
|
|
744
|
-
});
|
|
786
|
+
staged.push({ entry, detail: result.detail });
|
|
745
787
|
}
|
|
746
|
-
if (
|
|
788
|
+
if (staged.length > 0) {
|
|
789
|
+
const syntax = validatePythonSyntaxWithoutExecution(source);
|
|
790
|
+
if (!syntax.ok) {
|
|
791
|
+
for (const item of staged) {
|
|
792
|
+
skipped.push({ tpId: item.entry.tpId, detail: syntax.detail });
|
|
793
|
+
auditEntries.push({
|
|
794
|
+
tpId: item.entry.tpId,
|
|
795
|
+
result: "rolled-back-invalid-python",
|
|
796
|
+
detail: syntax.detail,
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
747
801
|
await writeFile(absolute, source, "utf8");
|
|
748
802
|
changedFiles.push(scriptPath);
|
|
803
|
+
for (const item of staged) {
|
|
804
|
+
repaired.push(item.entry.tpId);
|
|
805
|
+
auditEntries.push({
|
|
806
|
+
tpId: item.entry.tpId,
|
|
807
|
+
result: "rewritten",
|
|
808
|
+
detail: `${item.detail}; ${syntax.detail}`,
|
|
809
|
+
});
|
|
810
|
+
}
|
|
749
811
|
}
|
|
750
812
|
}
|
|
751
813
|
const afterHashes = [];
|
|
@@ -777,15 +839,15 @@ export async function writeScenarioParamRepairAudit(input) {
|
|
|
777
839
|
return filePath;
|
|
778
840
|
}
|
|
779
841
|
export function renderBackendTestFailureAnalysis(input) {
|
|
780
|
-
const overviewNarrative = input.failed === 0
|
|
842
|
+
const overviewNarrative = input.failed + input.errors === 0
|
|
781
843
|
? "本轮无失败用例。报告仍保留场景-参数一致性与执行摘要,供审计。"
|
|
782
|
-
: `本轮共 ${input.failed}
|
|
844
|
+
: `本轮共 ${input.failed} 个失败用例、${input.errors} 个错误用例。分类优先参考场景-参数一致性 final facts,再结合断言与环境证据。`;
|
|
783
845
|
return [
|
|
784
846
|
"# 测试失败用例分析报告",
|
|
785
847
|
"",
|
|
786
848
|
`> 报告生成时间:${input.generatedAt}`,
|
|
787
849
|
`> 测试报告来源:${input.htmlReportPath}`,
|
|
788
|
-
`> 测试总计:${input.total} 个用例(${input.passed} Passed, ${input.failed} Failed)`,
|
|
850
|
+
`> 测试总计:${input.total} 个用例(${input.passed} Passed, ${input.failed} Failed, ${input.errors} Error)`,
|
|
789
851
|
`> 测试耗时:${input.durationLabel}`,
|
|
790
852
|
`> 测试环境:${input.environmentSummary.split("\n")[0] ?? "unavailable"}`,
|
|
791
853
|
`> 接口:${input.primaryOperation ?? "—"}`,
|
|
@@ -271,6 +271,39 @@ function pythonParseable(source) {
|
|
|
271
271
|
// legal multi-line definitions such as `def f(\n x,\n):` — removed.
|
|
272
272
|
return true;
|
|
273
273
|
}
|
|
274
|
+
/**
|
|
275
|
+
* Forbidden generated shared helper/factory namespaces for the self-contained
|
|
276
|
+
* pytest module contract. A self-contained testcase/test_*.py child must not
|
|
277
|
+
* import from generated shared namespaces such as `testcase_helpers`,
|
|
278
|
+
* `testcase.helpers`, or `testcase.factories`; those belong to the pytest plan
|
|
279
|
+
* node's shared resources, not to a per-module pytest script. Rejecting such
|
|
280
|
+
* imports early (before pytest collection) gives an actionable, recoverable
|
|
281
|
+
* incomplete-write-set signal carrying the exact module target path.
|
|
282
|
+
*/
|
|
283
|
+
export const FORBIDDEN_GENERATED_IMPORT_NAMESPACES = [
|
|
284
|
+
"testcase_helpers",
|
|
285
|
+
"testcase.helpers",
|
|
286
|
+
"testcase.factories",
|
|
287
|
+
];
|
|
288
|
+
const FORBIDDEN_GENERATED_IMPORT_PATTERN = /^\s*(?:from\s+(testcase_helpers|testcase\.helpers|testcase\.factories)\b|import\s+(testcase_helpers|testcase\.helpers|testcase\.factories)\b)/gm;
|
|
289
|
+
/**
|
|
290
|
+
* Scan Python source for imports from forbidden generated shared namespaces.
|
|
291
|
+
* Anchored at line start with a word boundary so `testcase.helpers_other` or a
|
|
292
|
+
* substring inside a docstring sentence is not falsely flagged. Returns the
|
|
293
|
+
* list of offending namespace module names (deduplicated, order-preserving).
|
|
294
|
+
*/
|
|
295
|
+
export function findForbiddenGeneratedImports(source) {
|
|
296
|
+
const hits = [];
|
|
297
|
+
const seen = new Set();
|
|
298
|
+
for (const match of source.matchAll(FORBIDDEN_GENERATED_IMPORT_PATTERN)) {
|
|
299
|
+
const ns = (match[1] ?? match[2]);
|
|
300
|
+
if (!seen.has(ns)) {
|
|
301
|
+
seen.add(ns);
|
|
302
|
+
hits.push(ns);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
return hits;
|
|
306
|
+
}
|
|
274
307
|
/**
|
|
275
308
|
* Looser Python validity check for shared helpers/factories emitted by the
|
|
276
309
|
* pytest plan node. Unlike pythonParseable, it does NOT require balanced
|
|
@@ -417,6 +450,17 @@ export async function assessBackendTestShardChildCompleteness(input) {
|
|
|
417
450
|
recoverable: true,
|
|
418
451
|
});
|
|
419
452
|
}
|
|
453
|
+
const forbidden = findForbiddenGeneratedImports(body);
|
|
454
|
+
if (forbidden.length > 0) {
|
|
455
|
+
if (!brokenPaths.includes(target))
|
|
456
|
+
brokenPaths.push(target);
|
|
457
|
+
issues.push({
|
|
458
|
+
code: "T5",
|
|
459
|
+
path: target,
|
|
460
|
+
detail: `forbidden import from ${forbidden.join(", ")} in ${target}; self-contained pytest module must not import generated shared helper/factory namespaces`,
|
|
461
|
+
recoverable: true,
|
|
462
|
+
});
|
|
463
|
+
}
|
|
420
464
|
}
|
|
421
465
|
}
|
|
422
466
|
const recoverableTargets = orderedUnique([...missingPaths, ...brokenPaths]);
|
|
@@ -584,6 +628,17 @@ export async function assessBackendTestPytestWriterCompleteness(workspaceRoot) {
|
|
|
584
628
|
});
|
|
585
629
|
continue;
|
|
586
630
|
}
|
|
631
|
+
const forbidden = findForbiddenGeneratedImports(source);
|
|
632
|
+
if (forbidden.length > 0) {
|
|
633
|
+
brokenPaths.push(expected);
|
|
634
|
+
issues.push({
|
|
635
|
+
code: "T5",
|
|
636
|
+
path: expected,
|
|
637
|
+
detail: `forbidden import from ${forbidden.join(", ")} in ${expected}; self-contained pytest module must not import generated shared helper/factory namespaces`,
|
|
638
|
+
recoverable: true,
|
|
639
|
+
});
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
587
642
|
const markdown = await readFile(path.join(workspaceRoot, moduleRel), "utf8");
|
|
588
643
|
const caseIds = orderedUnique([...markdown.matchAll(/\bBE-[A-Z0-9_-]+-\d{2,3}\b/g)].map((item) => item[0]));
|
|
589
644
|
const primaryHints = orderedUnique([
|