@ryuenn3123/agentic-senior-core 4.3.15 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +21 -0
- package/.agents/rules/agentic-senior-core.md +126 -0
- package/.claude-plugin/marketplace.json +17 -0
- package/.claude-plugin/plugin.json +17 -0
- package/.clinerules/agentic-senior-core.md +94 -0
- package/.codex-plugin/plugin.json +24 -0
- package/.cursor/rules/agentic-senior-core.mdc +99 -0
- package/.devin-plugin/plugin.json +13 -0
- package/.github/copilot-instructions.md +94 -0
- package/.github/plugin/marketplace.json +20 -0
- package/.github/plugin/plugin.json +16 -0
- package/.kiro/steering/agentic-senior-core.md +94 -0
- package/.openclaw/skills/asc/SKILL.md +23 -0
- package/.openclaw/skills/asc-audit/SKILL.md +28 -0
- package/.openclaw/skills/asc-review/SKILL.md +54 -0
- package/.opencode/plugins/agentic-senior-core.mjs +31 -0
- package/.windsurf/rules/agentic-senior-core.md +94 -0
- package/AGENTS.md +126 -179
- package/CONTRIBUTING.md +93 -63
- package/README.md +160 -111
- package/__init__.py +61 -0
- package/bin/agentic-senior-core.js +41 -43
- package/commands/asc-audit.md +1 -0
- package/commands/asc-audit.toml +2 -0
- package/commands/asc-help.md +9 -0
- package/commands/asc-help.toml +2 -0
- package/commands/asc-refactor.md +1 -0
- package/commands/asc-refactor.toml +2 -0
- package/commands/asc-review.md +1 -0
- package/commands/asc-review.toml +2 -0
- package/gemini-extension.json +10 -0
- package/hooks/copilot-hooks.json +21 -0
- package/hooks/hooks.json +31 -0
- package/hooks/session-start.js +36 -0
- package/hooks/subagent-start.js +38 -0
- package/lib/cli/commands/adapter.mjs +109 -0
- package/lib/cli/commands/clean.mjs +63 -0
- package/lib/cli/commands/status.mjs +94 -0
- package/package.json +43 -67
- package/plugin.yaml +16 -0
- package/scripts/uninstall.js +38 -0
- package/skills/asc/SKILL.md +16 -0
- package/skills/asc-audit/SKILL.md +28 -0
- package/skills/asc-refactor/SKILL.md +35 -0
- package/skills/asc-review/SKILL.md +54 -0
- package/.agent-context/policies/llm-judge-threshold.json +0 -29
- package/.agent-context/prompts/bootstrap-design.md +0 -63
- package/.agent-context/prompts/compact-natural-mode.md +0 -100
- package/.agent-context/prompts/init-project.md +0 -60
- package/.agent-context/prompts/refactor.md +0 -38
- package/.agent-context/prompts/review-code.md +0 -31
- package/.agent-context/review-checklists/architecture-review.md +0 -57
- package/.agent-context/review-checklists/pr-checklist.md +0 -136
- package/.agent-context/rules/api-docs.md +0 -24
- package/.agent-context/rules/api-versioning.md +0 -16
- package/.agent-context/rules/architecture.md +0 -33
- package/.agent-context/rules/background-jobs.md +0 -17
- package/.agent-context/rules/config-and-flags.md +0 -16
- package/.agent-context/rules/database-design.md +0 -18
- package/.agent-context/rules/docker-runtime.md +0 -17
- package/.agent-context/rules/error-handling.md +0 -16
- package/.agent-context/rules/event-driven.md +0 -16
- package/.agent-context/rules/frontend-architecture.md +0 -54
- package/.agent-context/rules/microservices.md +0 -16
- package/.agent-context/rules/migrations.md +0 -16
- package/.agent-context/rules/observability.md +0 -16
- package/.agent-context/rules/performance.md +0 -16
- package/.agent-context/rules/realtime.md +0 -16
- package/.agent-context/rules/resilience.md +0 -18
- package/.agent-context/rules/security.md +0 -18
- package/.agent-context/rules/testing.md +0 -16
- package/.agent-context/state/README.md +0 -25
- package/.agent-context/state/architecture-map.md +0 -40
- package/.agent-context/state/benchmark-comparison-schema.json +0 -181
- package/.agent-context/state/benchmark-reproducibility.json +0 -87
- package/.agent-context/state/benchmark-thresholds.json +0 -10
- package/.agent-context/state/benchmark-writer-judge-config.json +0 -58
- package/.agent-context/state/dependency-map.md +0 -41
- package/.agent-context/state/memory-adapter-contract.json +0 -52
- package/.agent-context/state/memory-schema-v1.json +0 -88
- package/GEMINI.md +0 -1
- package/benchmarks/README.md +0 -60
- package/benchmarks/compact-natural-mode/fixtures.mjs +0 -359
- package/benchmarks/compact-natural-mode/scorer.mjs +0 -331
- package/benchmarks/runtime-token-saver/fixtures.mjs +0 -836
- package/lib/cli/adaptive-context/catalog.mjs +0 -390
- package/lib/cli/adaptive-context/file-signals.mjs +0 -100
- package/lib/cli/adaptive-context/implications.mjs +0 -44
- package/lib/cli/adaptive-context.mjs +0 -365
- package/lib/cli/audits/typography-palette-anti-repeat/color-utils.mjs +0 -156
- package/lib/cli/audits/typography-palette-anti-repeat/file-scanner.mjs +0 -103
- package/lib/cli/audits/typography-palette-anti-repeat/typography-utils.mjs +0 -70
- package/lib/cli/audits/typography-palette-anti-repeat-audit.mjs +0 -239
- package/lib/cli/backup.mjs +0 -210
- package/lib/cli/commands/audit-design-anti-repeat.mjs +0 -39
- package/lib/cli/commands/context.mjs +0 -140
- package/lib/cli/commands/init/project-context.mjs +0 -60
- package/lib/cli/commands/init/runtime-environment.mjs +0 -59
- package/lib/cli/commands/init/setup-decisions.mjs +0 -83
- package/lib/cli/commands/init.mjs +0 -654
- package/lib/cli/commands/launch.mjs +0 -76
- package/lib/cli/commands/optimize.mjs +0 -284
- package/lib/cli/commands/rollback.mjs +0 -64
- package/lib/cli/commands/upgrade/token-optimization-state.mjs +0 -51
- package/lib/cli/commands/upgrade.mjs +0 -451
- package/lib/cli/compiler.mjs +0 -570
- package/lib/cli/constants.mjs +0 -136
- package/lib/cli/detector/constants.mjs +0 -135
- package/lib/cli/detector/design-evidence/collector.mjs +0 -256
- package/lib/cli/detector/design-evidence/constants.mjs +0 -39
- package/lib/cli/detector/design-evidence/file-traversal.mjs +0 -83
- package/lib/cli/detector/design-evidence/structured-attribute-evidence.mjs +0 -117
- package/lib/cli/detector/design-evidence/summary.mjs +0 -109
- package/lib/cli/detector/design-evidence/utility-helpers.mjs +0 -122
- package/lib/cli/detector/design-evidence.mjs +0 -25
- package/lib/cli/detector/stack-detection.mjs +0 -243
- package/lib/cli/detector/ui-signals.mjs +0 -150
- package/lib/cli/detector/workspace-scan.mjs +0 -177
- package/lib/cli/detector.mjs +0 -23
- package/lib/cli/init-detection-flow.mjs +0 -48
- package/lib/cli/init-options.mjs +0 -237
- package/lib/cli/init-selection.mjs +0 -29
- package/lib/cli/memory-continuity.mjs +0 -537
- package/lib/cli/preflight.mjs +0 -76
- package/lib/cli/project-scaffolder/constants.mjs +0 -69
- package/lib/cli/project-scaffolder/discovery.mjs +0 -272
- package/lib/cli/project-scaffolder/prompt-builders.mjs +0 -171
- package/lib/cli/project-scaffolder/storage.mjs +0 -144
- package/lib/cli/project-scaffolder/ui-scope-detection.mjs +0 -36
- package/lib/cli/project-scaffolder.mjs +0 -29
- package/lib/cli/rollback.mjs +0 -66
- package/lib/cli/token-optimization.mjs +0 -486
- package/lib/cli/utils/filesystem.mjs +0 -81
- package/lib/cli/utils/managed-surface.mjs +0 -280
- package/lib/cli/utils/prompting.mjs +0 -44
- package/lib/cli/utils.mjs +0 -479
- package/mcp.json +0 -134
- package/scripts/adaptive-context/fixtures.mjs +0 -183
- package/scripts/adaptive-context-benchmark.mjs +0 -9
- package/scripts/ascx-runtime-token-saver-benchmark.mjs +0 -9
- package/scripts/audit-cache-layer-contract.mjs +0 -263
- package/scripts/audit-caching-scope-hygiene.mjs +0 -268
- package/scripts/audit-file-size.mjs +0 -219
- package/scripts/audit-reflection-citations.mjs +0 -163
- package/scripts/audit-release-bundle.mjs +0 -170
- package/scripts/audit-rule-id-uniqueness.mjs +0 -313
- package/scripts/audit-typography-palette-anti-repeat.mjs +0 -120
- package/scripts/benchmark-evidence-bundle.mjs +0 -645
- package/scripts/benchmark-gate.mjs +0 -121
- package/scripts/benchmark-intelligence.mjs +0 -33
- package/scripts/benchmark-writer-judge-matrix.mjs +0 -379
- package/scripts/build-release-benchmark-bundle.mjs +0 -202
- package/scripts/bump-version.mjs +0 -103
- package/scripts/clean-local-artifacts.mjs +0 -78
- package/scripts/compact-natural-mode-benchmark.mjs +0 -9
- package/scripts/context-triggered-audit.mjs +0 -391
- package/scripts/detection-benchmark.mjs +0 -127
- package/scripts/docs-quality-drift-report.mjs +0 -358
- package/scripts/documentation-boundary-audit.mjs +0 -305
- package/scripts/explain-on-demand-audit.mjs +0 -427
- package/scripts/forbidden-content-check.mjs +0 -123
- package/scripts/frontend-usability-audit.mjs +0 -243
- package/scripts/generate-sbom.mjs +0 -61
- package/scripts/governance-weekly-report.mjs +0 -466
- package/scripts/llm-judge/checklist-loader.mjs +0 -45
- package/scripts/llm-judge/constants.mjs +0 -66
- package/scripts/llm-judge/diff-collection.mjs +0 -74
- package/scripts/llm-judge/prompting.mjs +0 -78
- package/scripts/llm-judge/providers.mjs +0 -111
- package/scripts/llm-judge/verdict.mjs +0 -134
- package/scripts/llm-judge.mjs +0 -200
- package/scripts/memory-continuity-benchmark.mjs +0 -322
- package/scripts/quality-trend-report.mjs +0 -289
- package/scripts/release-gate/audit-checks.mjs +0 -441
- package/scripts/release-gate/constants.mjs +0 -53
- package/scripts/release-gate/runtime.mjs +0 -63
- package/scripts/release-gate/static-checks.mjs +0 -182
- package/scripts/release-gate.mjs +0 -42
- package/scripts/rules-guardian-audit.mjs +0 -580
- package/scripts/single-source-lazy-loading-audit.mjs +0 -449
- package/scripts/sync-thin-adapters.mjs +0 -72
- package/scripts/token-optimization-benchmark.mjs +0 -252
- package/scripts/ui-design-judge/constants.mjs +0 -13
- package/scripts/ui-design-judge/design-execution-summary.mjs +0 -81
- package/scripts/ui-design-judge/git-input.mjs +0 -134
- package/scripts/ui-design-judge/prompting.mjs +0 -52
- package/scripts/ui-design-judge/providers.mjs +0 -102
- package/scripts/ui-design-judge/reporting.mjs +0 -183
- package/scripts/ui-design-judge/rubric-calibration.mjs +0 -214
- package/scripts/ui-design-judge/rubric-goldset.json +0 -188
- package/scripts/ui-design-judge.mjs +0 -319
- package/scripts/ui-rubric-calibration.mjs +0 -35
- package/scripts/validate/config.mjs +0 -519
- package/scripts/validate/coverage-checks.mjs +0 -446
- package/scripts/validate/file-structure.mjs +0 -162
- package/scripts/validate/markdown-content.mjs +0 -109
- package/scripts/validate/project-metadata.mjs +0 -166
- package/scripts/validate.mjs +0 -338
|
@@ -1,645 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
// @file-size-exception: Pre-existing benchmark bundler (sequential per-section assembly); planned for split in Phase 1.
|
|
4
|
-
/**
|
|
5
|
-
* benchmark-evidence-bundle.mjs
|
|
6
|
-
*
|
|
7
|
-
* Benchmark evidence bundle with reproducibility, trend history,
|
|
8
|
-
* security signals, and reliability early warnings.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
12
|
-
import fs from 'node:fs/promises';
|
|
13
|
-
import { spawnSync } from 'node:child_process';
|
|
14
|
-
import { dirname, join, relative, resolve } from 'node:path';
|
|
15
|
-
import { fileURLToPath } from 'node:url';
|
|
16
|
-
|
|
17
|
-
const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
|
|
18
|
-
const SCRIPT_DIR = dirname(SCRIPT_FILE_PATH);
|
|
19
|
-
const REPOSITORY_ROOT = resolve(SCRIPT_DIR, '..');
|
|
20
|
-
const ARGUMENT_FLAGS = new Set(process.argv.slice(2));
|
|
21
|
-
const isStdoutOnlyMode = ARGUMENT_FLAGS.has('--stdout-only');
|
|
22
|
-
|
|
23
|
-
const PACKAGE_JSON_PATH = join(REPOSITORY_ROOT, 'package.json');
|
|
24
|
-
const REPRO_PROFILE_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-reproducibility.json');
|
|
25
|
-
const BENCHMARK_THRESHOLD_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-thresholds.json');
|
|
26
|
-
const MEMORY_SCHEMA_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-schema-v1.json');
|
|
27
|
-
const MEMORY_ADAPTER_CONTRACT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-adapter-contract.json');
|
|
28
|
-
const OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-evidence-bundle.json');
|
|
29
|
-
const HISTORY_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-history.json');
|
|
30
|
-
const TREND_JSON_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.json');
|
|
31
|
-
const TREND_CSV_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.csv');
|
|
32
|
-
|
|
33
|
-
const MAX_HISTORY_ENTRIES = 90;
|
|
34
|
-
const RELIABILITY_THRESHOLDS = {
|
|
35
|
-
minimumConfidenceGap: 0.1,
|
|
36
|
-
maximumLowConfidenceRate: 0.2,
|
|
37
|
-
maximumIncorrectDetectionRate: 0.1,
|
|
38
|
-
};
|
|
39
|
-
|
|
40
|
-
function readJsonOrNull(filePath) {
|
|
41
|
-
if (!existsSync(filePath)) {
|
|
42
|
-
return null;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
try {
|
|
46
|
-
return JSON.parse(readFileSync(filePath, 'utf8'));
|
|
47
|
-
} catch {
|
|
48
|
-
return null;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
function toRelativePath(filePath) {
|
|
53
|
-
return relative(REPOSITORY_ROOT, filePath).replace(/\\/g, '/');
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
function toFiniteNumber(rawValue, fallbackValue = null) {
|
|
57
|
-
const parsedValue = Number(rawValue);
|
|
58
|
-
if (!Number.isFinite(parsedValue)) {
|
|
59
|
-
return fallbackValue;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
return parsedValue;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function parseJsonPayload(rawPayload) {
|
|
66
|
-
const payloadText = String(rawPayload || '').trim();
|
|
67
|
-
if (!payloadText) {
|
|
68
|
-
return { parsed: null, error: 'Payload is empty' };
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
try {
|
|
72
|
-
return { parsed: JSON.parse(payloadText), error: null };
|
|
73
|
-
} catch {
|
|
74
|
-
const firstCurlyBracketIndex = payloadText.indexOf('{');
|
|
75
|
-
const lastCurlyBracketIndex = payloadText.lastIndexOf('}');
|
|
76
|
-
|
|
77
|
-
if (firstCurlyBracketIndex !== -1 && lastCurlyBracketIndex > firstCurlyBracketIndex) {
|
|
78
|
-
const candidatePayload = payloadText.slice(firstCurlyBracketIndex, lastCurlyBracketIndex + 1);
|
|
79
|
-
try {
|
|
80
|
-
return { parsed: JSON.parse(candidatePayload), error: null };
|
|
81
|
-
} catch (secondError) {
|
|
82
|
-
return {
|
|
83
|
-
parsed: null,
|
|
84
|
-
error: secondError instanceof Error ? secondError.message : String(secondError),
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
return { parsed: null, error: 'No JSON object found in payload' };
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function runJsonScript(scriptRelativePath) {
|
|
94
|
-
const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
|
|
95
|
-
const executionResult = spawnSync('node', [absoluteScriptPath], {
|
|
96
|
-
cwd: REPOSITORY_ROOT,
|
|
97
|
-
encoding: 'utf8',
|
|
98
|
-
maxBuffer: 1024 * 1024 * 10,
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
const stdoutContent = (executionResult.stdout || '').trim();
|
|
102
|
-
const stderrContent = (executionResult.stderr || '').trim();
|
|
103
|
-
const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
|
|
104
|
-
|
|
105
|
-
if (!stdoutContent) {
|
|
106
|
-
return {
|
|
107
|
-
scriptPath: scriptRelativePath,
|
|
108
|
-
exitCode,
|
|
109
|
-
parsedReport: null,
|
|
110
|
-
parseError: 'Script produced no stdout JSON payload',
|
|
111
|
-
stderr: stderrContent,
|
|
112
|
-
};
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
try {
|
|
116
|
-
return {
|
|
117
|
-
scriptPath: scriptRelativePath,
|
|
118
|
-
exitCode,
|
|
119
|
-
parsedReport: JSON.parse(stdoutContent),
|
|
120
|
-
parseError: null,
|
|
121
|
-
stderr: stderrContent,
|
|
122
|
-
};
|
|
123
|
-
} catch (jsonParseError) {
|
|
124
|
-
const parseErrorMessage = jsonParseError instanceof Error ? jsonParseError.message : String(jsonParseError);
|
|
125
|
-
return {
|
|
126
|
-
scriptPath: scriptRelativePath,
|
|
127
|
-
exitCode,
|
|
128
|
-
parsedReport: null,
|
|
129
|
-
parseError: parseErrorMessage,
|
|
130
|
-
stderr: stderrContent,
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
function runNodeScript(scriptRelativePath, argumentsList = []) {
|
|
136
|
-
const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
|
|
137
|
-
const executionResult = spawnSync('node', [absoluteScriptPath, ...argumentsList], {
|
|
138
|
-
cwd: REPOSITORY_ROOT,
|
|
139
|
-
encoding: 'utf8',
|
|
140
|
-
maxBuffer: 1024 * 1024 * 10,
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
return {
|
|
144
|
-
scriptPath: scriptRelativePath,
|
|
145
|
-
exitCode: typeof executionResult.status === 'number' ? executionResult.status : 1,
|
|
146
|
-
stdout: (executionResult.stdout || '').trim(),
|
|
147
|
-
stderr: (executionResult.stderr || '').trim(),
|
|
148
|
-
};
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function runNpmAuditIndicator() {
|
|
152
|
-
const executionResult = spawnSync('npm', ['audit', '--json', '--omit=dev'], {
|
|
153
|
-
cwd: REPOSITORY_ROOT,
|
|
154
|
-
encoding: 'utf8',
|
|
155
|
-
maxBuffer: 1024 * 1024 * 10,
|
|
156
|
-
});
|
|
157
|
-
|
|
158
|
-
const combinedOutput = [executionResult.stdout, executionResult.stderr].filter(Boolean).join('\n').trim();
|
|
159
|
-
const { parsed: parsedAuditReport, error: parseError } = parseJsonPayload(combinedOutput);
|
|
160
|
-
const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
|
|
161
|
-
|
|
162
|
-
if (!parsedAuditReport || parseError) {
|
|
163
|
-
return {
|
|
164
|
-
isAvailable: false,
|
|
165
|
-
exitCode,
|
|
166
|
-
severityCounts: null,
|
|
167
|
-
hasKnownVulnerabilities: null,
|
|
168
|
-
error: parseError || 'Unable to parse npm audit output',
|
|
169
|
-
};
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
const vulnerabilityMetadata = parsedAuditReport.metadata?.vulnerabilities || null;
|
|
173
|
-
const severityCounts = vulnerabilityMetadata
|
|
174
|
-
? {
|
|
175
|
-
info: toFiniteNumber(vulnerabilityMetadata.info, 0),
|
|
176
|
-
low: toFiniteNumber(vulnerabilityMetadata.low, 0),
|
|
177
|
-
moderate: toFiniteNumber(vulnerabilityMetadata.moderate, 0),
|
|
178
|
-
high: toFiniteNumber(vulnerabilityMetadata.high, 0),
|
|
179
|
-
critical: toFiniteNumber(vulnerabilityMetadata.critical, 0),
|
|
180
|
-
total: toFiniteNumber(vulnerabilityMetadata.total, 0),
|
|
181
|
-
}
|
|
182
|
-
: null;
|
|
183
|
-
|
|
184
|
-
if (!severityCounts) {
|
|
185
|
-
return {
|
|
186
|
-
isAvailable: false,
|
|
187
|
-
exitCode,
|
|
188
|
-
severityCounts: null,
|
|
189
|
-
hasKnownVulnerabilities: null,
|
|
190
|
-
error: parsedAuditReport.error?.summary || 'npm audit report does not include vulnerability metadata',
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
return {
|
|
195
|
-
isAvailable: true,
|
|
196
|
-
exitCode,
|
|
197
|
-
severityCounts,
|
|
198
|
-
hasKnownVulnerabilities: severityCounts.total > 0,
|
|
199
|
-
error: null,
|
|
200
|
-
};
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
function summarizeExecution(scriptExecutionResult) {
|
|
204
|
-
return {
|
|
205
|
-
scriptPath: scriptExecutionResult.scriptPath,
|
|
206
|
-
exitCode: scriptExecutionResult.exitCode,
|
|
207
|
-
parseError: scriptExecutionResult.parseError,
|
|
208
|
-
stderr: scriptExecutionResult.stderr || null,
|
|
209
|
-
reportName: scriptExecutionResult.parsedReport?.reportName || scriptExecutionResult.parsedReport?.gateName || null,
|
|
210
|
-
passed: typeof scriptExecutionResult.parsedReport?.passed === 'boolean'
|
|
211
|
-
? scriptExecutionResult.parsedReport.passed
|
|
212
|
-
: null,
|
|
213
|
-
};
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
function appendUniqueTextValues(baseValues, additionalValues) {
|
|
217
|
-
const mergedValues = [...baseValues];
|
|
218
|
-
for (const additionalValue of additionalValues) {
|
|
219
|
-
if (!mergedValues.includes(additionalValue)) {
|
|
220
|
-
mergedValues.push(additionalValue);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
return mergedValues;
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
function buildRubricSummary(thresholdConfiguration, intelligenceReport, memoryContinuityReport) {
|
|
228
|
-
return {
|
|
229
|
-
benchmarkThresholds: {
|
|
230
|
-
minimumTop1Accuracy: thresholdConfiguration?.minimumTop1Accuracy ?? null,
|
|
231
|
-
maximumManualCorrectionRate: thresholdConfiguration?.maximumManualCorrectionRate ?? null,
|
|
232
|
-
maximumTop1AccuracyDrop: thresholdConfiguration?.maximumTop1AccuracyDrop ?? null,
|
|
233
|
-
maximumManualCorrectionIncrease: thresholdConfiguration?.maximumManualCorrectionIncrease ?? null,
|
|
234
|
-
},
|
|
235
|
-
intelligenceSlaDays: intelligenceReport?.reviewSlaDays ?? null,
|
|
236
|
-
staticExternalWatchlistRetired: intelligenceReport?.staticExternalWatchlistRetired === true,
|
|
237
|
-
reliabilityThresholds: RELIABILITY_THRESHOLDS,
|
|
238
|
-
continuityThresholds: memoryContinuityReport?.thresholds || null,
|
|
239
|
-
};
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
function buildReliabilitySignals(detectionBenchmarkReport) {
|
|
243
|
-
const fixtureResults = Array.isArray(detectionBenchmarkReport?.fixtures) ? detectionBenchmarkReport.fixtures : [];
|
|
244
|
-
const fixtureCount = fixtureResults.length;
|
|
245
|
-
|
|
246
|
-
const incorrectFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.isCorrect === false);
|
|
247
|
-
const lowConfidenceFixtures = fixtureResults.filter((fixtureResult) => {
|
|
248
|
-
const confidenceGap = toFiniteNumber(fixtureResult?.confidenceGap, 0);
|
|
249
|
-
return confidenceGap < RELIABILITY_THRESHOLDS.minimumConfidenceGap;
|
|
250
|
-
});
|
|
251
|
-
const manualCorrectionFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.needsManualCorrection === true);
|
|
252
|
-
|
|
253
|
-
const incorrectDetectionRate = fixtureCount === 0
|
|
254
|
-
? 0
|
|
255
|
-
: Number((incorrectFixtures.length / fixtureCount).toFixed(4));
|
|
256
|
-
const lowConfidenceRate = fixtureCount === 0
|
|
257
|
-
? 0
|
|
258
|
-
: Number((lowConfidenceFixtures.length / fixtureCount).toFixed(4));
|
|
259
|
-
const manualCorrectionRate = fixtureCount === 0
|
|
260
|
-
? 0
|
|
261
|
-
: Number((manualCorrectionFixtures.length / fixtureCount).toFixed(4));
|
|
262
|
-
|
|
263
|
-
const reliabilityChecks = [
|
|
264
|
-
{
|
|
265
|
-
checkName: 'incorrect-detection-rate',
|
|
266
|
-
passed: incorrectDetectionRate <= RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate,
|
|
267
|
-
details: `incorrectRate=${incorrectDetectionRate} max=${RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate}`,
|
|
268
|
-
},
|
|
269
|
-
{
|
|
270
|
-
checkName: 'low-confidence-rate',
|
|
271
|
-
passed: lowConfidenceRate <= RELIABILITY_THRESHOLDS.maximumLowConfidenceRate,
|
|
272
|
-
details: `lowConfidenceRate=${lowConfidenceRate} max=${RELIABILITY_THRESHOLDS.maximumLowConfidenceRate}`,
|
|
273
|
-
},
|
|
274
|
-
{
|
|
275
|
-
checkName: 'manual-correction-early-warning',
|
|
276
|
-
passed: manualCorrectionRate <= 0.12,
|
|
277
|
-
details: `manualCorrectionRate=${manualCorrectionRate} warningThreshold=0.12`,
|
|
278
|
-
},
|
|
279
|
-
];
|
|
280
|
-
|
|
281
|
-
const failureCount = reliabilityChecks.filter((reliabilityCheck) => !reliabilityCheck.passed).length;
|
|
282
|
-
const riskLevel = failureCount === 0
|
|
283
|
-
? (incorrectFixtures.length > 0 || lowConfidenceFixtures.length > 0 ? 'monitor' : 'stable')
|
|
284
|
-
: (failureCount >= 2 ? 'high' : 'elevated');
|
|
285
|
-
|
|
286
|
-
return {
|
|
287
|
-
passed: failureCount === 0,
|
|
288
|
-
failureCount,
|
|
289
|
-
riskLevel,
|
|
290
|
-
thresholds: RELIABILITY_THRESHOLDS,
|
|
291
|
-
metrics: {
|
|
292
|
-
fixtureCount,
|
|
293
|
-
incorrectFixtureCount: incorrectFixtures.length,
|
|
294
|
-
lowConfidenceFixtureCount: lowConfidenceFixtures.length,
|
|
295
|
-
manualCorrectionFixtureCount: manualCorrectionFixtures.length,
|
|
296
|
-
incorrectDetectionRate,
|
|
297
|
-
lowConfidenceRate,
|
|
298
|
-
manualCorrectionRate,
|
|
299
|
-
},
|
|
300
|
-
checks: reliabilityChecks,
|
|
301
|
-
flaggedFixtures: fixtureResults
|
|
302
|
-
.filter((fixtureResult) => fixtureResult?.isCorrect === false || fixtureResult?.needsManualCorrection === true)
|
|
303
|
-
.map((fixtureResult) => ({
|
|
304
|
-
fixtureName: fixtureResult.fixtureName,
|
|
305
|
-
confidenceGap: fixtureResult.confidenceGap,
|
|
306
|
-
detectedStack: fixtureResult.detectedStack,
|
|
307
|
-
expectedStack: fixtureResult.expectedStack,
|
|
308
|
-
isCorrect: fixtureResult.isCorrect,
|
|
309
|
-
needsManualCorrection: fixtureResult.needsManualCorrection,
|
|
310
|
-
})),
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
function buildBugIndicators(reliabilitySignals) {
|
|
315
|
-
return {
|
|
316
|
-
incorrectFixtureCount: reliabilitySignals.metrics.incorrectFixtureCount,
|
|
317
|
-
incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
|
|
318
|
-
manualCorrectionFixtureCount: reliabilitySignals.metrics.manualCorrectionFixtureCount,
|
|
319
|
-
manualCorrectionRate: reliabilitySignals.metrics.manualCorrectionRate,
|
|
320
|
-
lowConfidenceFixtureCount: reliabilitySignals.metrics.lowConfidenceFixtureCount,
|
|
321
|
-
lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
|
|
322
|
-
flaggedFixtures: reliabilitySignals.flaggedFixtures,
|
|
323
|
-
};
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
function buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator) {
|
|
327
|
-
const forbiddenContentPassed = forbiddenContentExecution.exitCode === 0;
|
|
328
|
-
|
|
329
|
-
return {
|
|
330
|
-
forbiddenContent: {
|
|
331
|
-
checkName: 'forbidden-content-scan',
|
|
332
|
-
passed: forbiddenContentPassed,
|
|
333
|
-
exitCode: forbiddenContentExecution.exitCode,
|
|
334
|
-
details: forbiddenContentPassed
|
|
335
|
-
? 'No forbidden content detected'
|
|
336
|
-
: 'Forbidden content scan found one or more violations',
|
|
337
|
-
},
|
|
338
|
-
vulnerabilityScan: {
|
|
339
|
-
checkName: 'npm-audit-indicator',
|
|
340
|
-
isAvailable: npmAuditIndicator.isAvailable,
|
|
341
|
-
hasKnownVulnerabilities: npmAuditIndicator.hasKnownVulnerabilities,
|
|
342
|
-
severityCounts: npmAuditIndicator.severityCounts,
|
|
343
|
-
exitCode: npmAuditIndicator.exitCode,
|
|
344
|
-
error: npmAuditIndicator.error,
|
|
345
|
-
},
|
|
346
|
-
};
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
function readReleaseVersion() {
|
|
350
|
-
const packageJson = readJsonOrNull(PACKAGE_JSON_PATH);
|
|
351
|
-
return typeof packageJson?.version === 'string' && packageJson.version.trim().length > 0
|
|
352
|
-
? packageJson.version.trim()
|
|
353
|
-
: 'unknown';
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
function loadBenchmarkHistory() {
|
|
357
|
-
const historyPayload = readJsonOrNull(HISTORY_OUTPUT_PATH);
|
|
358
|
-
|
|
359
|
-
if (Array.isArray(historyPayload?.history)) {
|
|
360
|
-
return historyPayload.history;
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
if (Array.isArray(historyPayload)) {
|
|
364
|
-
return historyPayload;
|
|
365
|
-
}
|
|
366
|
-
|
|
367
|
-
return [];
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
function mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot) {
|
|
371
|
-
const mergedHistoryEntries = [...previousHistoryEntries, currentSnapshot];
|
|
372
|
-
if (mergedHistoryEntries.length <= MAX_HISTORY_ENTRIES) {
|
|
373
|
-
return mergedHistoryEntries;
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
return mergedHistoryEntries.slice(mergedHistoryEntries.length - MAX_HISTORY_ENTRIES);
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
function buildHistorySnapshot({
|
|
380
|
-
generatedAt,
|
|
381
|
-
releaseVersion,
|
|
382
|
-
detectionBenchmarkReport,
|
|
383
|
-
benchmarkGateReport,
|
|
384
|
-
benchmarkIntelligenceReport,
|
|
385
|
-
reliabilitySignals,
|
|
386
|
-
securityIndicators,
|
|
387
|
-
}) {
|
|
388
|
-
return {
|
|
389
|
-
generatedAt,
|
|
390
|
-
releaseVersion,
|
|
391
|
-
fixtureCount: toFiniteNumber(detectionBenchmarkReport?.fixtureCount, 0),
|
|
392
|
-
top1Accuracy: toFiniteNumber(detectionBenchmarkReport?.top1Accuracy, 0),
|
|
393
|
-
manualCorrectionRate: toFiniteNumber(detectionBenchmarkReport?.manualCorrectionRate, 0),
|
|
394
|
-
benchmarkGatePassed: benchmarkGateReport?.passed === true,
|
|
395
|
-
intelligencePassed: benchmarkIntelligenceReport?.passed === true,
|
|
396
|
-
staticExternalWatchlistRetired: benchmarkIntelligenceReport?.staticExternalWatchlistRetired === true,
|
|
397
|
-
reliabilityPassed: reliabilitySignals.passed,
|
|
398
|
-
reliabilityRiskLevel: reliabilitySignals.riskLevel,
|
|
399
|
-
incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
|
|
400
|
-
lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
|
|
401
|
-
vulnerabilityTotal: securityIndicators.vulnerabilityScan.severityCounts?.total ?? null,
|
|
402
|
-
criticalVulnerabilityCount: securityIndicators.vulnerabilityScan.severityCounts?.critical ?? null,
|
|
403
|
-
forbiddenContentPassed: securityIndicators.forbiddenContent.passed,
|
|
404
|
-
};
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
function buildReleaseDelta(historyEntries, currentSnapshot) {
|
|
408
|
-
const previousSnapshots = historyEntries.slice(0, -1);
|
|
409
|
-
if (previousSnapshots.length === 0) {
|
|
410
|
-
return null;
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
const previousReleaseSnapshot = [...previousSnapshots].reverse().find(
|
|
414
|
-
(historyEntry) => historyEntry.releaseVersion !== currentSnapshot.releaseVersion
|
|
415
|
-
) || previousSnapshots[previousSnapshots.length - 1];
|
|
416
|
-
|
|
417
|
-
const top1AccuracyDelta = Number((currentSnapshot.top1Accuracy - previousReleaseSnapshot.top1Accuracy).toFixed(4));
|
|
418
|
-
const manualCorrectionDelta = Number((currentSnapshot.manualCorrectionRate - previousReleaseSnapshot.manualCorrectionRate).toFixed(4));
|
|
419
|
-
const vulnerabilityDelta =
|
|
420
|
-
(toFiniteNumber(currentSnapshot.vulnerabilityTotal, 0) - toFiniteNumber(previousReleaseSnapshot.vulnerabilityTotal, 0));
|
|
421
|
-
|
|
422
|
-
return {
|
|
423
|
-
currentReleaseVersion: currentSnapshot.releaseVersion,
|
|
424
|
-
previousReleaseVersion: previousReleaseSnapshot.releaseVersion,
|
|
425
|
-
comparedSnapshot: {
|
|
426
|
-
currentGeneratedAt: currentSnapshot.generatedAt,
|
|
427
|
-
previousGeneratedAt: previousReleaseSnapshot.generatedAt,
|
|
428
|
-
},
|
|
429
|
-
top1AccuracyDelta,
|
|
430
|
-
manualCorrectionRateDelta: manualCorrectionDelta,
|
|
431
|
-
vulnerabilityTotalDelta: vulnerabilityDelta,
|
|
432
|
-
summary: [
|
|
433
|
-
`top1Accuracy: ${top1AccuracyDelta >= 0 ? '+' : ''}${top1AccuracyDelta}`,
|
|
434
|
-
`manualCorrectionRate: ${manualCorrectionDelta >= 0 ? '+' : ''}${manualCorrectionDelta}`,
|
|
435
|
-
`vulnerabilityTotal: ${vulnerabilityDelta >= 0 ? '+' : ''}${vulnerabilityDelta}`,
|
|
436
|
-
],
|
|
437
|
-
};
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
function buildTrendTable(historyEntries) {
|
|
441
|
-
return historyEntries.map((historyEntry, index) => ({
|
|
442
|
-
snapshotIndex: index + 1,
|
|
443
|
-
generatedAt: historyEntry.generatedAt,
|
|
444
|
-
releaseVersion: historyEntry.releaseVersion,
|
|
445
|
-
top1Accuracy: historyEntry.top1Accuracy,
|
|
446
|
-
manualCorrectionRate: historyEntry.manualCorrectionRate,
|
|
447
|
-
incorrectDetectionRate: historyEntry.incorrectDetectionRate,
|
|
448
|
-
lowConfidenceRate: historyEntry.lowConfidenceRate,
|
|
449
|
-
staticExternalWatchlistRetired: historyEntry.staticExternalWatchlistRetired === true,
|
|
450
|
-
vulnerabilityTotal: historyEntry.vulnerabilityTotal,
|
|
451
|
-
criticalVulnerabilityCount: historyEntry.criticalVulnerabilityCount,
|
|
452
|
-
benchmarkGatePassed: historyEntry.benchmarkGatePassed,
|
|
453
|
-
intelligencePassed: historyEntry.intelligencePassed,
|
|
454
|
-
reliabilityPassed: historyEntry.reliabilityPassed,
|
|
455
|
-
reliabilityRiskLevel: historyEntry.reliabilityRiskLevel,
|
|
456
|
-
}));
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
function buildChartSeries(historyEntries) {
|
|
460
|
-
return {
|
|
461
|
-
generatedAt: historyEntries.map((historyEntry) => historyEntry.generatedAt),
|
|
462
|
-
top1Accuracy: historyEntries.map((historyEntry) => historyEntry.top1Accuracy),
|
|
463
|
-
manualCorrectionRate: historyEntries.map((historyEntry) => historyEntry.manualCorrectionRate),
|
|
464
|
-
incorrectDetectionRate: historyEntries.map((historyEntry) => historyEntry.incorrectDetectionRate),
|
|
465
|
-
lowConfidenceRate: historyEntries.map((historyEntry) => historyEntry.lowConfidenceRate),
|
|
466
|
-
staticExternalWatchlistRetired: historyEntries.map((historyEntry) => historyEntry.staticExternalWatchlistRetired === true),
|
|
467
|
-
vulnerabilityTotal: historyEntries.map((historyEntry) => historyEntry.vulnerabilityTotal),
|
|
468
|
-
};
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
function convertTrendTableToCsv(trendTable) {
|
|
472
|
-
if (trendTable.length === 0) {
|
|
473
|
-
return '';
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
const headers = Object.keys(trendTable[0]);
|
|
477
|
-
const csvRows = [headers.join(',')];
|
|
478
|
-
|
|
479
|
-
for (const trendRow of trendTable) {
|
|
480
|
-
const rowValues = headers.map((header) => {
|
|
481
|
-
const rawValue = trendRow[header];
|
|
482
|
-
if (rawValue === null || rawValue === undefined) {
|
|
483
|
-
return '';
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
const normalizedValue = String(rawValue).replace(/"/g, '""');
|
|
487
|
-
return `"${normalizedValue}"`;
|
|
488
|
-
});
|
|
489
|
-
|
|
490
|
-
csvRows.push(rowValues.join(','));
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
return `${csvRows.join('\n')}\n`;
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
async function runBenchmarkEvidenceBundle() {
|
|
497
|
-
const reproducibilityProfile = readJsonOrNull(REPRO_PROFILE_PATH);
|
|
498
|
-
const thresholdConfiguration = readJsonOrNull(BENCHMARK_THRESHOLD_PATH);
|
|
499
|
-
const memorySchemaConfiguration = readJsonOrNull(MEMORY_SCHEMA_PATH);
|
|
500
|
-
const memoryAdapterContractConfiguration = readJsonOrNull(MEMORY_ADAPTER_CONTRACT_PATH);
|
|
501
|
-
const releaseVersion = readReleaseVersion();
|
|
502
|
-
|
|
503
|
-
const detectionBenchmarkExecution = runJsonScript('scripts/detection-benchmark.mjs');
|
|
504
|
-
const benchmarkGateExecution = runJsonScript('scripts/benchmark-gate.mjs');
|
|
505
|
-
const benchmarkIntelligenceExecution = runJsonScript('scripts/benchmark-intelligence.mjs');
|
|
506
|
-
const memoryContinuityExecution = runJsonScript('scripts/memory-continuity-benchmark.mjs');
|
|
507
|
-
const forbiddenContentExecution = runNodeScript('scripts/forbidden-content-check.mjs');
|
|
508
|
-
const npmAuditIndicator = runNpmAuditIndicator();
|
|
509
|
-
|
|
510
|
-
const executionSummaries = [
|
|
511
|
-
summarizeExecution(detectionBenchmarkExecution),
|
|
512
|
-
summarizeExecution(benchmarkGateExecution),
|
|
513
|
-
summarizeExecution(benchmarkIntelligenceExecution),
|
|
514
|
-
summarizeExecution(memoryContinuityExecution),
|
|
515
|
-
];
|
|
516
|
-
|
|
517
|
-
const executionFailureCount = executionSummaries.filter((executionSummary) => {
|
|
518
|
-
if (executionSummary.parseError) {
|
|
519
|
-
return true;
|
|
520
|
-
}
|
|
521
|
-
|
|
522
|
-
if (typeof executionSummary.passed === 'boolean') {
|
|
523
|
-
return executionSummary.passed === false;
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
return executionSummary.exitCode !== 0;
|
|
527
|
-
}).length;
|
|
528
|
-
|
|
529
|
-
const reliabilitySignals = buildReliabilitySignals(detectionBenchmarkExecution.parsedReport);
|
|
530
|
-
const reliabilityFailureCount = reliabilitySignals.failureCount > 0 ? 1 : 0;
|
|
531
|
-
const failureCount = executionFailureCount + reliabilityFailureCount;
|
|
532
|
-
const securityIndicators = buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator);
|
|
533
|
-
const bugIndicators = buildBugIndicators(reliabilitySignals);
|
|
534
|
-
|
|
535
|
-
const generatedAt = new Date().toISOString();
|
|
536
|
-
const currentSnapshot = buildHistorySnapshot({
|
|
537
|
-
generatedAt,
|
|
538
|
-
releaseVersion,
|
|
539
|
-
detectionBenchmarkReport: detectionBenchmarkExecution.parsedReport,
|
|
540
|
-
benchmarkGateReport: benchmarkGateExecution.parsedReport,
|
|
541
|
-
benchmarkIntelligenceReport: benchmarkIntelligenceExecution.parsedReport,
|
|
542
|
-
reliabilitySignals,
|
|
543
|
-
securityIndicators,
|
|
544
|
-
});
|
|
545
|
-
const previousHistoryEntries = loadBenchmarkHistory();
|
|
546
|
-
const historyEntries = mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot);
|
|
547
|
-
const releaseDelta = buildReleaseDelta(historyEntries, currentSnapshot);
|
|
548
|
-
const trendTable = buildTrendTable(historyEntries);
|
|
549
|
-
const chartSeries = buildChartSeries(historyEntries);
|
|
550
|
-
|
|
551
|
-
const trendReport = {
|
|
552
|
-
generatedAt,
|
|
553
|
-
reportName: 'benchmark-trend-report',
|
|
554
|
-
releaseVersion,
|
|
555
|
-
historyCount: historyEntries.length,
|
|
556
|
-
releaseDelta,
|
|
557
|
-
trendTable,
|
|
558
|
-
chartSeries,
|
|
559
|
-
artifacts: {
|
|
560
|
-
historyPath: toRelativePath(HISTORY_OUTPUT_PATH),
|
|
561
|
-
jsonPath: toRelativePath(TREND_JSON_OUTPUT_PATH),
|
|
562
|
-
csvPath: toRelativePath(TREND_CSV_OUTPUT_PATH),
|
|
563
|
-
writeMode: isStdoutOnlyMode ? 'stdout-only' : 'stdout-and-file',
|
|
564
|
-
},
|
|
565
|
-
};
|
|
566
|
-
|
|
567
|
-
const historyPayload = {
|
|
568
|
-
generatedAt,
|
|
569
|
-
reportName: 'benchmark-history',
|
|
570
|
-
maxEntries: MAX_HISTORY_ENTRIES,
|
|
571
|
-
history: historyEntries,
|
|
572
|
-
};
|
|
573
|
-
|
|
574
|
-
const trendCsvPayload = convertTrendTableToCsv(trendTable);
|
|
575
|
-
const baseRerunInstructions = Array.isArray(reproducibilityProfile?.rerunInstructions)
|
|
576
|
-
? reproducibilityProfile.rerunInstructions
|
|
577
|
-
: [];
|
|
578
|
-
const baseCommandExamples = Array.isArray(reproducibilityProfile?.commandExamples)
|
|
579
|
-
? reproducibilityProfile.commandExamples
|
|
580
|
-
: [];
|
|
581
|
-
const rerunInstructions = appendUniqueTextValues(baseRerunInstructions, [
|
|
582
|
-
'Run npm run benchmark:continuity to validate cross-agent memory hydration, privacy redaction, and token-savings behavior.',
|
|
583
|
-
]);
|
|
584
|
-
const commandExamples = appendUniqueTextValues(baseCommandExamples, [
|
|
585
|
-
'npm run benchmark:continuity',
|
|
586
|
-
'node ./scripts/memory-continuity-benchmark.mjs --stdout-only',
|
|
587
|
-
]);
|
|
588
|
-
|
|
589
|
-
const evidenceBundleReport = {
|
|
590
|
-
generatedAt,
|
|
591
|
-
reportName: 'benchmark-evidence-bundle',
|
|
592
|
-
phase: 'v2.5.2',
|
|
593
|
-
releaseVersion,
|
|
594
|
-
passed: failureCount === 0,
|
|
595
|
-
failureCount,
|
|
596
|
-
methodology: {
|
|
597
|
-
deterministicRuntime: reproducibilityProfile?.deterministicRuntime || null,
|
|
598
|
-
scenarioCount: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios.length : 0,
|
|
599
|
-
commandCount: commandExamples.length,
|
|
600
|
-
},
|
|
601
|
-
rerunInstructions,
|
|
602
|
-
commandExamples,
|
|
603
|
-
rawInputs: {
|
|
604
|
-
scenarios: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios : [],
|
|
605
|
-
benchmarkThresholds: thresholdConfiguration,
|
|
606
|
-
benchmarkWatchlist: {
|
|
607
|
-
retired: true,
|
|
608
|
-
reason: 'Static external benchmark watchlists were removed to avoid stale or biasing research inputs.',
|
|
609
|
-
},
|
|
610
|
-
memorySchema: memorySchemaConfiguration,
|
|
611
|
-
memoryAdapterContract: memoryAdapterContractConfiguration,
|
|
612
|
-
},
|
|
613
|
-
rubric: buildRubricSummary(
|
|
614
|
-
thresholdConfiguration,
|
|
615
|
-
benchmarkIntelligenceExecution.parsedReport,
|
|
616
|
-
memoryContinuityExecution.parsedReport
|
|
617
|
-
),
|
|
618
|
-
bugIndicators,
|
|
619
|
-
reliabilitySignals,
|
|
620
|
-
securityIndicators,
|
|
621
|
-
releaseDelta,
|
|
622
|
-
history: historyEntries,
|
|
623
|
-
trendReport,
|
|
624
|
-
outputs: {
|
|
625
|
-
detectionBenchmark: detectionBenchmarkExecution.parsedReport,
|
|
626
|
-
benchmarkGate: benchmarkGateExecution.parsedReport,
|
|
627
|
-
benchmarkIntelligence: benchmarkIntelligenceExecution.parsedReport,
|
|
628
|
-
memoryContinuityBenchmark: memoryContinuityExecution.parsedReport,
|
|
629
|
-
},
|
|
630
|
-
executions: executionSummaries,
|
|
631
|
-
};
|
|
632
|
-
|
|
633
|
-
if (!isStdoutOnlyMode) {
|
|
634
|
-
await fs.mkdir(join(REPOSITORY_ROOT, '.agent-context', 'state'), { recursive: true });
|
|
635
|
-
await fs.writeFile(OUTPUT_PATH, JSON.stringify(evidenceBundleReport, null, 2) + '\n', 'utf8');
|
|
636
|
-
await fs.writeFile(HISTORY_OUTPUT_PATH, JSON.stringify(historyPayload, null, 2) + '\n', 'utf8');
|
|
637
|
-
await fs.writeFile(TREND_JSON_OUTPUT_PATH, JSON.stringify(trendReport, null, 2) + '\n', 'utf8');
|
|
638
|
-
await fs.writeFile(TREND_CSV_OUTPUT_PATH, trendCsvPayload, 'utf8');
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
console.log(JSON.stringify(evidenceBundleReport, null, 2));
|
|
642
|
-
process.exit(evidenceBundleReport.passed ? 0 : 1);
|
|
643
|
-
}
|
|
644
|
-
|
|
645
|
-
runBenchmarkEvidenceBundle();
|