@ryuenn3123/agentic-senior-core 4.4.0 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/.agents/plugins/marketplace.json +21 -0
  2. package/.agents/rules/agentic-senior-core.md +126 -0
  3. package/.claude-plugin/marketplace.json +17 -0
  4. package/.claude-plugin/plugin.json +17 -0
  5. package/.clinerules/agentic-senior-core.md +94 -0
  6. package/.codex-plugin/plugin.json +24 -0
  7. package/.cursor/rules/agentic-senior-core.mdc +99 -0
  8. package/.devin-plugin/plugin.json +13 -0
  9. package/.github/copilot-instructions.md +94 -0
  10. package/.github/plugin/marketplace.json +20 -0
  11. package/.github/plugin/plugin.json +16 -0
  12. package/.kiro/steering/agentic-senior-core.md +94 -0
  13. package/.openclaw/skills/asc/SKILL.md +23 -0
  14. package/.openclaw/skills/asc-audit/SKILL.md +28 -0
  15. package/.openclaw/skills/asc-review/SKILL.md +54 -0
  16. package/.opencode/plugins/agentic-senior-core.mjs +31 -0
  17. package/.windsurf/rules/agentic-senior-core.md +94 -0
  18. package/AGENTS.md +126 -127
  19. package/CONTRIBUTING.md +93 -63
  20. package/README.md +160 -111
  21. package/__init__.py +61 -0
  22. package/bin/agentic-senior-core.js +42 -41
  23. package/commands/asc-audit.md +1 -0
  24. package/commands/asc-audit.toml +2 -0
  25. package/commands/asc-help.md +9 -0
  26. package/commands/asc-help.toml +2 -0
  27. package/commands/asc-refactor.md +1 -0
  28. package/commands/asc-refactor.toml +2 -0
  29. package/commands/asc-review.md +1 -0
  30. package/commands/asc-review.toml +2 -0
  31. package/gemini-extension.json +10 -0
  32. package/hooks/copilot-hooks.json +21 -0
  33. package/hooks/hooks.json +31 -0
  34. package/hooks/session-start.js +36 -0
  35. package/hooks/subagent-start.js +38 -0
  36. package/lib/cli/commands/adapter.mjs +109 -0
  37. package/lib/cli/commands/clean.mjs +63 -0
  38. package/lib/cli/commands/status.mjs +94 -0
  39. package/package.json +43 -60
  40. package/plugin.yaml +16 -0
  41. package/scripts/uninstall.js +38 -0
  42. package/skills/asc/SKILL.md +16 -0
  43. package/skills/asc-audit/SKILL.md +28 -0
  44. package/skills/asc-refactor/SKILL.md +35 -0
  45. package/skills/asc-review/SKILL.md +54 -0
  46. package/.agent-context/policies/llm-judge-threshold.json +0 -29
  47. package/.agent-context/prompts/bootstrap-design.md +0 -63
  48. package/.agent-context/prompts/compact-natural-mode.md +0 -100
  49. package/.agent-context/prompts/init-project.md +0 -60
  50. package/.agent-context/prompts/refactor.md +0 -38
  51. package/.agent-context/prompts/review-code.md +0 -31
  52. package/.agent-context/review-checklists/architecture-review.md +0 -57
  53. package/.agent-context/review-checklists/pr-checklist.md +0 -134
  54. package/.agent-context/rules/api-docs.md +0 -24
  55. package/.agent-context/rules/api-versioning.md +0 -16
  56. package/.agent-context/rules/architecture.md +0 -38
  57. package/.agent-context/rules/background-jobs.md +0 -17
  58. package/.agent-context/rules/config-and-flags.md +0 -16
  59. package/.agent-context/rules/database-design.md +0 -18
  60. package/.agent-context/rules/docker-runtime.md +0 -17
  61. package/.agent-context/rules/error-handling.md +0 -16
  62. package/.agent-context/rules/event-driven.md +0 -16
  63. package/.agent-context/rules/frontend-architecture.md +0 -54
  64. package/.agent-context/rules/microservices.md +0 -16
  65. package/.agent-context/rules/migrations.md +0 -16
  66. package/.agent-context/rules/observability.md +0 -16
  67. package/.agent-context/rules/performance.md +0 -16
  68. package/.agent-context/rules/realtime.md +0 -16
  69. package/.agent-context/rules/resilience.md +0 -18
  70. package/.agent-context/rules/security.md +0 -18
  71. package/.agent-context/rules/testing.md +0 -16
  72. package/.agent-context/state/README.md +0 -25
  73. package/.agent-context/state/architecture-map.md +0 -40
  74. package/.agent-context/state/benchmark-comparison-schema.json +0 -181
  75. package/.agent-context/state/benchmark-reproducibility.json +0 -87
  76. package/.agent-context/state/benchmark-thresholds.json +0 -10
  77. package/.agent-context/state/benchmark-writer-judge-config.json +0 -58
  78. package/.agent-context/state/dependency-map.md +0 -41
  79. package/.agent-context/state/memory-adapter-contract.json +0 -52
  80. package/.agent-context/state/memory-schema-v1.json +0 -88
  81. package/GEMINI.md +0 -1
  82. package/benchmarks/README.md +0 -60
  83. package/benchmarks/compact-natural-mode/fixtures.mjs +0 -359
  84. package/benchmarks/compact-natural-mode/scorer.mjs +0 -331
  85. package/benchmarks/runtime-token-saver/fixtures.mjs +0 -836
  86. package/lib/cli/adaptive-context/catalog.mjs +0 -394
  87. package/lib/cli/adaptive-context/file-signals.mjs +0 -100
  88. package/lib/cli/adaptive-context/implications.mjs +0 -44
  89. package/lib/cli/adaptive-context.mjs +0 -151
  90. package/lib/cli/audits/typography-palette-anti-repeat/color-utils.mjs +0 -156
  91. package/lib/cli/audits/typography-palette-anti-repeat/file-scanner.mjs +0 -103
  92. package/lib/cli/audits/typography-palette-anti-repeat/typography-utils.mjs +0 -70
  93. package/lib/cli/backup.mjs +0 -210
  94. package/lib/cli/commands/context.mjs +0 -140
  95. package/lib/cli/commands/init/project-context.mjs +0 -60
  96. package/lib/cli/commands/init/runtime-environment.mjs +0 -59
  97. package/lib/cli/commands/init/setup-decisions.mjs +0 -83
  98. package/lib/cli/commands/init.mjs +0 -654
  99. package/lib/cli/commands/launch.mjs +0 -76
  100. package/lib/cli/commands/optimize.mjs +0 -284
  101. package/lib/cli/commands/rollback.mjs +0 -64
  102. package/lib/cli/commands/upgrade/token-optimization-state.mjs +0 -51
  103. package/lib/cli/commands/upgrade.mjs +0 -451
  104. package/lib/cli/compiler.mjs +0 -187
  105. package/lib/cli/constants.mjs +0 -136
  106. package/lib/cli/detector/constants.mjs +0 -135
  107. package/lib/cli/detector/design-evidence/collector.mjs +0 -256
  108. package/lib/cli/detector/design-evidence/constants.mjs +0 -39
  109. package/lib/cli/detector/design-evidence/file-traversal.mjs +0 -83
  110. package/lib/cli/detector/design-evidence/structured-attribute-evidence.mjs +0 -117
  111. package/lib/cli/detector/design-evidence/summary.mjs +0 -109
  112. package/lib/cli/detector/design-evidence/utility-helpers.mjs +0 -122
  113. package/lib/cli/detector/design-evidence.mjs +0 -25
  114. package/lib/cli/detector/stack-detection.mjs +0 -243
  115. package/lib/cli/detector/ui-signals.mjs +0 -150
  116. package/lib/cli/detector/workspace-scan.mjs +0 -177
  117. package/lib/cli/detector.mjs +0 -23
  118. package/lib/cli/init-detection-flow.mjs +0 -48
  119. package/lib/cli/init-options.mjs +0 -237
  120. package/lib/cli/init-selection.mjs +0 -29
  121. package/lib/cli/memory-continuity.mjs +0 -537
  122. package/lib/cli/preflight.mjs +0 -76
  123. package/lib/cli/project-scaffolder/constants.mjs +0 -69
  124. package/lib/cli/project-scaffolder/discovery.mjs +0 -272
  125. package/lib/cli/project-scaffolder/prompt-builders.mjs +0 -171
  126. package/lib/cli/project-scaffolder/storage.mjs +0 -144
  127. package/lib/cli/project-scaffolder/ui-scope-detection.mjs +0 -36
  128. package/lib/cli/project-scaffolder.mjs +0 -29
  129. package/lib/cli/rollback.mjs +0 -66
  130. package/lib/cli/token-optimization.mjs +0 -401
  131. package/lib/cli/utils/filesystem.mjs +0 -81
  132. package/lib/cli/utils/managed-surface.mjs +0 -280
  133. package/lib/cli/utils/prompting.mjs +0 -44
  134. package/lib/cli/utils.mjs +0 -479
  135. package/mcp.json +0 -134
  136. package/scripts/adaptive-context/fixtures.mjs +0 -183
  137. package/scripts/adaptive-context-benchmark.mjs +0 -9
  138. package/scripts/ascx-runtime-token-saver-benchmark.mjs +0 -9
  139. package/scripts/benchmark-evidence-bundle.mjs +0 -645
  140. package/scripts/benchmark-gate.mjs +0 -121
  141. package/scripts/benchmark-intelligence.mjs +0 -33
  142. package/scripts/benchmark-writer-judge-matrix.mjs +0 -379
  143. package/scripts/build-release-benchmark-bundle.mjs +0 -202
  144. package/scripts/bump-version.mjs +0 -103
  145. package/scripts/clean-local-artifacts.mjs +0 -78
  146. package/scripts/compact-natural-mode-benchmark.mjs +0 -9
  147. package/scripts/context-triggered-audit.mjs +0 -391
  148. package/scripts/detection-benchmark.mjs +0 -127
  149. package/scripts/docs-quality-drift-report.mjs +0 -358
  150. package/scripts/documentation-boundary-audit.mjs +0 -305
  151. package/scripts/explain-on-demand-audit.mjs +0 -427
  152. package/scripts/forbidden-content-check.mjs +0 -123
  153. package/scripts/frontend-usability-audit.mjs +0 -243
  154. package/scripts/generate-sbom.mjs +0 -61
  155. package/scripts/governance-weekly-report.mjs +0 -466
  156. package/scripts/llm-judge/checklist-loader.mjs +0 -45
  157. package/scripts/llm-judge/constants.mjs +0 -66
  158. package/scripts/llm-judge/diff-collection.mjs +0 -74
  159. package/scripts/llm-judge/prompting.mjs +0 -78
  160. package/scripts/llm-judge/providers.mjs +0 -111
  161. package/scripts/llm-judge/verdict.mjs +0 -134
  162. package/scripts/llm-judge.mjs +0 -200
  163. package/scripts/memory-continuity-benchmark.mjs +0 -322
  164. package/scripts/quality-trend-report.mjs +0 -289
  165. package/scripts/release-gate/audit-checks.mjs +0 -441
  166. package/scripts/release-gate/constants.mjs +0 -53
  167. package/scripts/release-gate/runtime.mjs +0 -63
  168. package/scripts/release-gate/static-checks.mjs +0 -182
  169. package/scripts/release-gate.mjs +0 -42
  170. package/scripts/rules-guardian-audit.mjs +0 -580
  171. package/scripts/single-source-lazy-loading-audit.mjs +0 -449
  172. package/scripts/sync-thin-adapters.mjs +0 -72
  173. package/scripts/token-optimization-benchmark.mjs +0 -252
  174. package/scripts/ui-design-judge/constants.mjs +0 -13
  175. package/scripts/ui-design-judge/design-execution-summary.mjs +0 -81
  176. package/scripts/ui-design-judge/git-input.mjs +0 -134
  177. package/scripts/ui-design-judge/prompting.mjs +0 -52
  178. package/scripts/ui-design-judge/providers.mjs +0 -102
  179. package/scripts/ui-design-judge/reporting.mjs +0 -183
  180. package/scripts/ui-design-judge/rubric-calibration.mjs +0 -214
  181. package/scripts/ui-design-judge/rubric-goldset.json +0 -188
  182. package/scripts/ui-design-judge.mjs +0 -319
  183. package/scripts/ui-rubric-calibration.mjs +0 -35
  184. package/scripts/validate/audits/cache-layer-contract.mjs +0 -230
  185. package/scripts/validate/audits/caching-scope-hygiene.mjs +0 -235
  186. package/scripts/validate/audits/file-size.mjs +0 -158
  187. package/scripts/validate/audits/reflection-citations.mjs +0 -130
  188. package/scripts/validate/audits/release-bundle.mjs +0 -135
  189. package/scripts/validate/audits/rule-id-uniqueness.mjs +0 -278
  190. package/scripts/validate/config.mjs +0 -501
  191. package/scripts/validate/coverage-checks.mjs +0 -446
  192. package/scripts/validate/file-structure.mjs +0 -158
  193. package/scripts/validate/markdown-content.mjs +0 -109
  194. package/scripts/validate/project-metadata.mjs +0 -166
  195. package/scripts/validate/utils.mjs +0 -52
  196. package/scripts/validate.mjs +0 -267
@@ -1,645 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- // @file-size-exception: Pre-existing benchmark bundler (sequential per-section assembly); planned for split in Phase 1.
4
- /**
5
- * benchmark-evidence-bundle.mjs
6
- *
7
- * Benchmark evidence bundle with reproducibility, trend history,
8
- * security signals, and reliability early warnings.
9
- */
10
-
11
- import { existsSync, readFileSync } from 'node:fs';
12
- import fs from 'node:fs/promises';
13
- import { spawnSync } from 'node:child_process';
14
- import { dirname, join, relative, resolve } from 'node:path';
15
- import { fileURLToPath } from 'node:url';
16
-
17
- const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
18
- const SCRIPT_DIR = dirname(SCRIPT_FILE_PATH);
19
- const REPOSITORY_ROOT = resolve(SCRIPT_DIR, '..');
20
- const ARGUMENT_FLAGS = new Set(process.argv.slice(2));
21
- const isStdoutOnlyMode = ARGUMENT_FLAGS.has('--stdout-only');
22
-
23
- const PACKAGE_JSON_PATH = join(REPOSITORY_ROOT, 'package.json');
24
- const REPRO_PROFILE_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-reproducibility.json');
25
- const BENCHMARK_THRESHOLD_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-thresholds.json');
26
- const MEMORY_SCHEMA_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-schema-v1.json');
27
- const MEMORY_ADAPTER_CONTRACT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-adapter-contract.json');
28
- const OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-evidence-bundle.json');
29
- const HISTORY_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-history.json');
30
- const TREND_JSON_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.json');
31
- const TREND_CSV_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.csv');
32
-
33
- const MAX_HISTORY_ENTRIES = 90;
34
- const RELIABILITY_THRESHOLDS = {
35
- minimumConfidenceGap: 0.1,
36
- maximumLowConfidenceRate: 0.2,
37
- maximumIncorrectDetectionRate: 0.1,
38
- };
39
-
40
- function readJsonOrNull(filePath) {
41
- if (!existsSync(filePath)) {
42
- return null;
43
- }
44
-
45
- try {
46
- return JSON.parse(readFileSync(filePath, 'utf8'));
47
- } catch {
48
- return null;
49
- }
50
- }
51
-
52
- function toRelativePath(filePath) {
53
- return relative(REPOSITORY_ROOT, filePath).replace(/\\/g, '/');
54
- }
55
-
56
- function toFiniteNumber(rawValue, fallbackValue = null) {
57
- const parsedValue = Number(rawValue);
58
- if (!Number.isFinite(parsedValue)) {
59
- return fallbackValue;
60
- }
61
-
62
- return parsedValue;
63
- }
64
-
65
- function parseJsonPayload(rawPayload) {
66
- const payloadText = String(rawPayload || '').trim();
67
- if (!payloadText) {
68
- return { parsed: null, error: 'Payload is empty' };
69
- }
70
-
71
- try {
72
- return { parsed: JSON.parse(payloadText), error: null };
73
- } catch {
74
- const firstCurlyBracketIndex = payloadText.indexOf('{');
75
- const lastCurlyBracketIndex = payloadText.lastIndexOf('}');
76
-
77
- if (firstCurlyBracketIndex !== -1 && lastCurlyBracketIndex > firstCurlyBracketIndex) {
78
- const candidatePayload = payloadText.slice(firstCurlyBracketIndex, lastCurlyBracketIndex + 1);
79
- try {
80
- return { parsed: JSON.parse(candidatePayload), error: null };
81
- } catch (secondError) {
82
- return {
83
- parsed: null,
84
- error: secondError instanceof Error ? secondError.message : String(secondError),
85
- };
86
- }
87
- }
88
-
89
- return { parsed: null, error: 'No JSON object found in payload' };
90
- }
91
- }
92
-
93
- function runJsonScript(scriptRelativePath) {
94
- const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
95
- const executionResult = spawnSync('node', [absoluteScriptPath], {
96
- cwd: REPOSITORY_ROOT,
97
- encoding: 'utf8',
98
- maxBuffer: 1024 * 1024 * 10,
99
- });
100
-
101
- const stdoutContent = (executionResult.stdout || '').trim();
102
- const stderrContent = (executionResult.stderr || '').trim();
103
- const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
104
-
105
- if (!stdoutContent) {
106
- return {
107
- scriptPath: scriptRelativePath,
108
- exitCode,
109
- parsedReport: null,
110
- parseError: 'Script produced no stdout JSON payload',
111
- stderr: stderrContent,
112
- };
113
- }
114
-
115
- try {
116
- return {
117
- scriptPath: scriptRelativePath,
118
- exitCode,
119
- parsedReport: JSON.parse(stdoutContent),
120
- parseError: null,
121
- stderr: stderrContent,
122
- };
123
- } catch (jsonParseError) {
124
- const parseErrorMessage = jsonParseError instanceof Error ? jsonParseError.message : String(jsonParseError);
125
- return {
126
- scriptPath: scriptRelativePath,
127
- exitCode,
128
- parsedReport: null,
129
- parseError: parseErrorMessage,
130
- stderr: stderrContent,
131
- };
132
- }
133
- }
134
-
135
- function runNodeScript(scriptRelativePath, argumentsList = []) {
136
- const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
137
- const executionResult = spawnSync('node', [absoluteScriptPath, ...argumentsList], {
138
- cwd: REPOSITORY_ROOT,
139
- encoding: 'utf8',
140
- maxBuffer: 1024 * 1024 * 10,
141
- });
142
-
143
- return {
144
- scriptPath: scriptRelativePath,
145
- exitCode: typeof executionResult.status === 'number' ? executionResult.status : 1,
146
- stdout: (executionResult.stdout || '').trim(),
147
- stderr: (executionResult.stderr || '').trim(),
148
- };
149
- }
150
-
151
- function runNpmAuditIndicator() {
152
- const executionResult = spawnSync('npm', ['audit', '--json', '--omit=dev'], {
153
- cwd: REPOSITORY_ROOT,
154
- encoding: 'utf8',
155
- maxBuffer: 1024 * 1024 * 10,
156
- });
157
-
158
- const combinedOutput = [executionResult.stdout, executionResult.stderr].filter(Boolean).join('\n').trim();
159
- const { parsed: parsedAuditReport, error: parseError } = parseJsonPayload(combinedOutput);
160
- const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
161
-
162
- if (!parsedAuditReport || parseError) {
163
- return {
164
- isAvailable: false,
165
- exitCode,
166
- severityCounts: null,
167
- hasKnownVulnerabilities: null,
168
- error: parseError || 'Unable to parse npm audit output',
169
- };
170
- }
171
-
172
- const vulnerabilityMetadata = parsedAuditReport.metadata?.vulnerabilities || null;
173
- const severityCounts = vulnerabilityMetadata
174
- ? {
175
- info: toFiniteNumber(vulnerabilityMetadata.info, 0),
176
- low: toFiniteNumber(vulnerabilityMetadata.low, 0),
177
- moderate: toFiniteNumber(vulnerabilityMetadata.moderate, 0),
178
- high: toFiniteNumber(vulnerabilityMetadata.high, 0),
179
- critical: toFiniteNumber(vulnerabilityMetadata.critical, 0),
180
- total: toFiniteNumber(vulnerabilityMetadata.total, 0),
181
- }
182
- : null;
183
-
184
- if (!severityCounts) {
185
- return {
186
- isAvailable: false,
187
- exitCode,
188
- severityCounts: null,
189
- hasKnownVulnerabilities: null,
190
- error: parsedAuditReport.error?.summary || 'npm audit report does not include vulnerability metadata',
191
- };
192
- }
193
-
194
- return {
195
- isAvailable: true,
196
- exitCode,
197
- severityCounts,
198
- hasKnownVulnerabilities: severityCounts.total > 0,
199
- error: null,
200
- };
201
- }
202
-
203
- function summarizeExecution(scriptExecutionResult) {
204
- return {
205
- scriptPath: scriptExecutionResult.scriptPath,
206
- exitCode: scriptExecutionResult.exitCode,
207
- parseError: scriptExecutionResult.parseError,
208
- stderr: scriptExecutionResult.stderr || null,
209
- reportName: scriptExecutionResult.parsedReport?.reportName || scriptExecutionResult.parsedReport?.gateName || null,
210
- passed: typeof scriptExecutionResult.parsedReport?.passed === 'boolean'
211
- ? scriptExecutionResult.parsedReport.passed
212
- : null,
213
- };
214
- }
215
-
216
- function appendUniqueTextValues(baseValues, additionalValues) {
217
- const mergedValues = [...baseValues];
218
- for (const additionalValue of additionalValues) {
219
- if (!mergedValues.includes(additionalValue)) {
220
- mergedValues.push(additionalValue);
221
- }
222
- }
223
-
224
- return mergedValues;
225
- }
226
-
227
- function buildRubricSummary(thresholdConfiguration, intelligenceReport, memoryContinuityReport) {
228
- return {
229
- benchmarkThresholds: {
230
- minimumTop1Accuracy: thresholdConfiguration?.minimumTop1Accuracy ?? null,
231
- maximumManualCorrectionRate: thresholdConfiguration?.maximumManualCorrectionRate ?? null,
232
- maximumTop1AccuracyDrop: thresholdConfiguration?.maximumTop1AccuracyDrop ?? null,
233
- maximumManualCorrectionIncrease: thresholdConfiguration?.maximumManualCorrectionIncrease ?? null,
234
- },
235
- intelligenceSlaDays: intelligenceReport?.reviewSlaDays ?? null,
236
- staticExternalWatchlistRetired: intelligenceReport?.staticExternalWatchlistRetired === true,
237
- reliabilityThresholds: RELIABILITY_THRESHOLDS,
238
- continuityThresholds: memoryContinuityReport?.thresholds || null,
239
- };
240
- }
241
-
242
- function buildReliabilitySignals(detectionBenchmarkReport) {
243
- const fixtureResults = Array.isArray(detectionBenchmarkReport?.fixtures) ? detectionBenchmarkReport.fixtures : [];
244
- const fixtureCount = fixtureResults.length;
245
-
246
- const incorrectFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.isCorrect === false);
247
- const lowConfidenceFixtures = fixtureResults.filter((fixtureResult) => {
248
- const confidenceGap = toFiniteNumber(fixtureResult?.confidenceGap, 0);
249
- return confidenceGap < RELIABILITY_THRESHOLDS.minimumConfidenceGap;
250
- });
251
- const manualCorrectionFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.needsManualCorrection === true);
252
-
253
- const incorrectDetectionRate = fixtureCount === 0
254
- ? 0
255
- : Number((incorrectFixtures.length / fixtureCount).toFixed(4));
256
- const lowConfidenceRate = fixtureCount === 0
257
- ? 0
258
- : Number((lowConfidenceFixtures.length / fixtureCount).toFixed(4));
259
- const manualCorrectionRate = fixtureCount === 0
260
- ? 0
261
- : Number((manualCorrectionFixtures.length / fixtureCount).toFixed(4));
262
-
263
- const reliabilityChecks = [
264
- {
265
- checkName: 'incorrect-detection-rate',
266
- passed: incorrectDetectionRate <= RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate,
267
- details: `incorrectRate=${incorrectDetectionRate} max=${RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate}`,
268
- },
269
- {
270
- checkName: 'low-confidence-rate',
271
- passed: lowConfidenceRate <= RELIABILITY_THRESHOLDS.maximumLowConfidenceRate,
272
- details: `lowConfidenceRate=${lowConfidenceRate} max=${RELIABILITY_THRESHOLDS.maximumLowConfidenceRate}`,
273
- },
274
- {
275
- checkName: 'manual-correction-early-warning',
276
- passed: manualCorrectionRate <= 0.12,
277
- details: `manualCorrectionRate=${manualCorrectionRate} warningThreshold=0.12`,
278
- },
279
- ];
280
-
281
- const failureCount = reliabilityChecks.filter((reliabilityCheck) => !reliabilityCheck.passed).length;
282
- const riskLevel = failureCount === 0
283
- ? (incorrectFixtures.length > 0 || lowConfidenceFixtures.length > 0 ? 'monitor' : 'stable')
284
- : (failureCount >= 2 ? 'high' : 'elevated');
285
-
286
- return {
287
- passed: failureCount === 0,
288
- failureCount,
289
- riskLevel,
290
- thresholds: RELIABILITY_THRESHOLDS,
291
- metrics: {
292
- fixtureCount,
293
- incorrectFixtureCount: incorrectFixtures.length,
294
- lowConfidenceFixtureCount: lowConfidenceFixtures.length,
295
- manualCorrectionFixtureCount: manualCorrectionFixtures.length,
296
- incorrectDetectionRate,
297
- lowConfidenceRate,
298
- manualCorrectionRate,
299
- },
300
- checks: reliabilityChecks,
301
- flaggedFixtures: fixtureResults
302
- .filter((fixtureResult) => fixtureResult?.isCorrect === false || fixtureResult?.needsManualCorrection === true)
303
- .map((fixtureResult) => ({
304
- fixtureName: fixtureResult.fixtureName,
305
- confidenceGap: fixtureResult.confidenceGap,
306
- detectedStack: fixtureResult.detectedStack,
307
- expectedStack: fixtureResult.expectedStack,
308
- isCorrect: fixtureResult.isCorrect,
309
- needsManualCorrection: fixtureResult.needsManualCorrection,
310
- })),
311
- };
312
- }
313
-
314
- function buildBugIndicators(reliabilitySignals) {
315
- return {
316
- incorrectFixtureCount: reliabilitySignals.metrics.incorrectFixtureCount,
317
- incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
318
- manualCorrectionFixtureCount: reliabilitySignals.metrics.manualCorrectionFixtureCount,
319
- manualCorrectionRate: reliabilitySignals.metrics.manualCorrectionRate,
320
- lowConfidenceFixtureCount: reliabilitySignals.metrics.lowConfidenceFixtureCount,
321
- lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
322
- flaggedFixtures: reliabilitySignals.flaggedFixtures,
323
- };
324
- }
325
-
326
- function buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator) {
327
- const forbiddenContentPassed = forbiddenContentExecution.exitCode === 0;
328
-
329
- return {
330
- forbiddenContent: {
331
- checkName: 'forbidden-content-scan',
332
- passed: forbiddenContentPassed,
333
- exitCode: forbiddenContentExecution.exitCode,
334
- details: forbiddenContentPassed
335
- ? 'No forbidden content detected'
336
- : 'Forbidden content scan found one or more violations',
337
- },
338
- vulnerabilityScan: {
339
- checkName: 'npm-audit-indicator',
340
- isAvailable: npmAuditIndicator.isAvailable,
341
- hasKnownVulnerabilities: npmAuditIndicator.hasKnownVulnerabilities,
342
- severityCounts: npmAuditIndicator.severityCounts,
343
- exitCode: npmAuditIndicator.exitCode,
344
- error: npmAuditIndicator.error,
345
- },
346
- };
347
- }
348
-
349
- function readReleaseVersion() {
350
- const packageJson = readJsonOrNull(PACKAGE_JSON_PATH);
351
- return typeof packageJson?.version === 'string' && packageJson.version.trim().length > 0
352
- ? packageJson.version.trim()
353
- : 'unknown';
354
- }
355
-
356
- function loadBenchmarkHistory() {
357
- const historyPayload = readJsonOrNull(HISTORY_OUTPUT_PATH);
358
-
359
- if (Array.isArray(historyPayload?.history)) {
360
- return historyPayload.history;
361
- }
362
-
363
- if (Array.isArray(historyPayload)) {
364
- return historyPayload;
365
- }
366
-
367
- return [];
368
- }
369
-
370
- function mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot) {
371
- const mergedHistoryEntries = [...previousHistoryEntries, currentSnapshot];
372
- if (mergedHistoryEntries.length <= MAX_HISTORY_ENTRIES) {
373
- return mergedHistoryEntries;
374
- }
375
-
376
- return mergedHistoryEntries.slice(mergedHistoryEntries.length - MAX_HISTORY_ENTRIES);
377
- }
378
-
379
- function buildHistorySnapshot({
380
- generatedAt,
381
- releaseVersion,
382
- detectionBenchmarkReport,
383
- benchmarkGateReport,
384
- benchmarkIntelligenceReport,
385
- reliabilitySignals,
386
- securityIndicators,
387
- }) {
388
- return {
389
- generatedAt,
390
- releaseVersion,
391
- fixtureCount: toFiniteNumber(detectionBenchmarkReport?.fixtureCount, 0),
392
- top1Accuracy: toFiniteNumber(detectionBenchmarkReport?.top1Accuracy, 0),
393
- manualCorrectionRate: toFiniteNumber(detectionBenchmarkReport?.manualCorrectionRate, 0),
394
- benchmarkGatePassed: benchmarkGateReport?.passed === true,
395
- intelligencePassed: benchmarkIntelligenceReport?.passed === true,
396
- staticExternalWatchlistRetired: benchmarkIntelligenceReport?.staticExternalWatchlistRetired === true,
397
- reliabilityPassed: reliabilitySignals.passed,
398
- reliabilityRiskLevel: reliabilitySignals.riskLevel,
399
- incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
400
- lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
401
- vulnerabilityTotal: securityIndicators.vulnerabilityScan.severityCounts?.total ?? null,
402
- criticalVulnerabilityCount: securityIndicators.vulnerabilityScan.severityCounts?.critical ?? null,
403
- forbiddenContentPassed: securityIndicators.forbiddenContent.passed,
404
- };
405
- }
406
-
407
- function buildReleaseDelta(historyEntries, currentSnapshot) {
408
- const previousSnapshots = historyEntries.slice(0, -1);
409
- if (previousSnapshots.length === 0) {
410
- return null;
411
- }
412
-
413
- const previousReleaseSnapshot = [...previousSnapshots].reverse().find(
414
- (historyEntry) => historyEntry.releaseVersion !== currentSnapshot.releaseVersion
415
- ) || previousSnapshots[previousSnapshots.length - 1];
416
-
417
- const top1AccuracyDelta = Number((currentSnapshot.top1Accuracy - previousReleaseSnapshot.top1Accuracy).toFixed(4));
418
- const manualCorrectionDelta = Number((currentSnapshot.manualCorrectionRate - previousReleaseSnapshot.manualCorrectionRate).toFixed(4));
419
- const vulnerabilityDelta =
420
- (toFiniteNumber(currentSnapshot.vulnerabilityTotal, 0) - toFiniteNumber(previousReleaseSnapshot.vulnerabilityTotal, 0));
421
-
422
- return {
423
- currentReleaseVersion: currentSnapshot.releaseVersion,
424
- previousReleaseVersion: previousReleaseSnapshot.releaseVersion,
425
- comparedSnapshot: {
426
- currentGeneratedAt: currentSnapshot.generatedAt,
427
- previousGeneratedAt: previousReleaseSnapshot.generatedAt,
428
- },
429
- top1AccuracyDelta,
430
- manualCorrectionRateDelta: manualCorrectionDelta,
431
- vulnerabilityTotalDelta: vulnerabilityDelta,
432
- summary: [
433
- `top1Accuracy: ${top1AccuracyDelta >= 0 ? '+' : ''}${top1AccuracyDelta}`,
434
- `manualCorrectionRate: ${manualCorrectionDelta >= 0 ? '+' : ''}${manualCorrectionDelta}`,
435
- `vulnerabilityTotal: ${vulnerabilityDelta >= 0 ? '+' : ''}${vulnerabilityDelta}`,
436
- ],
437
- };
438
- }
439
-
440
- function buildTrendTable(historyEntries) {
441
- return historyEntries.map((historyEntry, index) => ({
442
- snapshotIndex: index + 1,
443
- generatedAt: historyEntry.generatedAt,
444
- releaseVersion: historyEntry.releaseVersion,
445
- top1Accuracy: historyEntry.top1Accuracy,
446
- manualCorrectionRate: historyEntry.manualCorrectionRate,
447
- incorrectDetectionRate: historyEntry.incorrectDetectionRate,
448
- lowConfidenceRate: historyEntry.lowConfidenceRate,
449
- staticExternalWatchlistRetired: historyEntry.staticExternalWatchlistRetired === true,
450
- vulnerabilityTotal: historyEntry.vulnerabilityTotal,
451
- criticalVulnerabilityCount: historyEntry.criticalVulnerabilityCount,
452
- benchmarkGatePassed: historyEntry.benchmarkGatePassed,
453
- intelligencePassed: historyEntry.intelligencePassed,
454
- reliabilityPassed: historyEntry.reliabilityPassed,
455
- reliabilityRiskLevel: historyEntry.reliabilityRiskLevel,
456
- }));
457
- }
458
-
459
- function buildChartSeries(historyEntries) {
460
- return {
461
- generatedAt: historyEntries.map((historyEntry) => historyEntry.generatedAt),
462
- top1Accuracy: historyEntries.map((historyEntry) => historyEntry.top1Accuracy),
463
- manualCorrectionRate: historyEntries.map((historyEntry) => historyEntry.manualCorrectionRate),
464
- incorrectDetectionRate: historyEntries.map((historyEntry) => historyEntry.incorrectDetectionRate),
465
- lowConfidenceRate: historyEntries.map((historyEntry) => historyEntry.lowConfidenceRate),
466
- staticExternalWatchlistRetired: historyEntries.map((historyEntry) => historyEntry.staticExternalWatchlistRetired === true),
467
- vulnerabilityTotal: historyEntries.map((historyEntry) => historyEntry.vulnerabilityTotal),
468
- };
469
- }
470
-
471
- function convertTrendTableToCsv(trendTable) {
472
- if (trendTable.length === 0) {
473
- return '';
474
- }
475
-
476
- const headers = Object.keys(trendTable[0]);
477
- const csvRows = [headers.join(',')];
478
-
479
- for (const trendRow of trendTable) {
480
- const rowValues = headers.map((header) => {
481
- const rawValue = trendRow[header];
482
- if (rawValue === null || rawValue === undefined) {
483
- return '';
484
- }
485
-
486
- const normalizedValue = String(rawValue).replace(/"/g, '""');
487
- return `"${normalizedValue}"`;
488
- });
489
-
490
- csvRows.push(rowValues.join(','));
491
- }
492
-
493
- return `${csvRows.join('\n')}\n`;
494
- }
495
-
496
- async function runBenchmarkEvidenceBundle() {
497
- const reproducibilityProfile = readJsonOrNull(REPRO_PROFILE_PATH);
498
- const thresholdConfiguration = readJsonOrNull(BENCHMARK_THRESHOLD_PATH);
499
- const memorySchemaConfiguration = readJsonOrNull(MEMORY_SCHEMA_PATH);
500
- const memoryAdapterContractConfiguration = readJsonOrNull(MEMORY_ADAPTER_CONTRACT_PATH);
501
- const releaseVersion = readReleaseVersion();
502
-
503
- const detectionBenchmarkExecution = runJsonScript('scripts/detection-benchmark.mjs');
504
- const benchmarkGateExecution = runJsonScript('scripts/benchmark-gate.mjs');
505
- const benchmarkIntelligenceExecution = runJsonScript('scripts/benchmark-intelligence.mjs');
506
- const memoryContinuityExecution = runJsonScript('scripts/memory-continuity-benchmark.mjs');
507
- const forbiddenContentExecution = runNodeScript('scripts/forbidden-content-check.mjs');
508
- const npmAuditIndicator = runNpmAuditIndicator();
509
-
510
- const executionSummaries = [
511
- summarizeExecution(detectionBenchmarkExecution),
512
- summarizeExecution(benchmarkGateExecution),
513
- summarizeExecution(benchmarkIntelligenceExecution),
514
- summarizeExecution(memoryContinuityExecution),
515
- ];
516
-
517
- const executionFailureCount = executionSummaries.filter((executionSummary) => {
518
- if (executionSummary.parseError) {
519
- return true;
520
- }
521
-
522
- if (typeof executionSummary.passed === 'boolean') {
523
- return executionSummary.passed === false;
524
- }
525
-
526
- return executionSummary.exitCode !== 0;
527
- }).length;
528
-
529
- const reliabilitySignals = buildReliabilitySignals(detectionBenchmarkExecution.parsedReport);
530
- const reliabilityFailureCount = reliabilitySignals.failureCount > 0 ? 1 : 0;
531
- const failureCount = executionFailureCount + reliabilityFailureCount;
532
- const securityIndicators = buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator);
533
- const bugIndicators = buildBugIndicators(reliabilitySignals);
534
-
535
- const generatedAt = new Date().toISOString();
536
- const currentSnapshot = buildHistorySnapshot({
537
- generatedAt,
538
- releaseVersion,
539
- detectionBenchmarkReport: detectionBenchmarkExecution.parsedReport,
540
- benchmarkGateReport: benchmarkGateExecution.parsedReport,
541
- benchmarkIntelligenceReport: benchmarkIntelligenceExecution.parsedReport,
542
- reliabilitySignals,
543
- securityIndicators,
544
- });
545
- const previousHistoryEntries = loadBenchmarkHistory();
546
- const historyEntries = mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot);
547
- const releaseDelta = buildReleaseDelta(historyEntries, currentSnapshot);
548
- const trendTable = buildTrendTable(historyEntries);
549
- const chartSeries = buildChartSeries(historyEntries);
550
-
551
- const trendReport = {
552
- generatedAt,
553
- reportName: 'benchmark-trend-report',
554
- releaseVersion,
555
- historyCount: historyEntries.length,
556
- releaseDelta,
557
- trendTable,
558
- chartSeries,
559
- artifacts: {
560
- historyPath: toRelativePath(HISTORY_OUTPUT_PATH),
561
- jsonPath: toRelativePath(TREND_JSON_OUTPUT_PATH),
562
- csvPath: toRelativePath(TREND_CSV_OUTPUT_PATH),
563
- writeMode: isStdoutOnlyMode ? 'stdout-only' : 'stdout-and-file',
564
- },
565
- };
566
-
567
- const historyPayload = {
568
- generatedAt,
569
- reportName: 'benchmark-history',
570
- maxEntries: MAX_HISTORY_ENTRIES,
571
- history: historyEntries,
572
- };
573
-
574
- const trendCsvPayload = convertTrendTableToCsv(trendTable);
575
- const baseRerunInstructions = Array.isArray(reproducibilityProfile?.rerunInstructions)
576
- ? reproducibilityProfile.rerunInstructions
577
- : [];
578
- const baseCommandExamples = Array.isArray(reproducibilityProfile?.commandExamples)
579
- ? reproducibilityProfile.commandExamples
580
- : [];
581
- const rerunInstructions = appendUniqueTextValues(baseRerunInstructions, [
582
- 'Run npm run benchmark:continuity to validate cross-agent memory hydration, privacy redaction, and token-savings behavior.',
583
- ]);
584
- const commandExamples = appendUniqueTextValues(baseCommandExamples, [
585
- 'npm run benchmark:continuity',
586
- 'node ./scripts/memory-continuity-benchmark.mjs --stdout-only',
587
- ]);
588
-
589
- const evidenceBundleReport = {
590
- generatedAt,
591
- reportName: 'benchmark-evidence-bundle',
592
- phase: 'v2.5.2',
593
- releaseVersion,
594
- passed: failureCount === 0,
595
- failureCount,
596
- methodology: {
597
- deterministicRuntime: reproducibilityProfile?.deterministicRuntime || null,
598
- scenarioCount: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios.length : 0,
599
- commandCount: commandExamples.length,
600
- },
601
- rerunInstructions,
602
- commandExamples,
603
- rawInputs: {
604
- scenarios: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios : [],
605
- benchmarkThresholds: thresholdConfiguration,
606
- benchmarkWatchlist: {
607
- retired: true,
608
- reason: 'Static external benchmark watchlists were removed to avoid stale or biasing research inputs.',
609
- },
610
- memorySchema: memorySchemaConfiguration,
611
- memoryAdapterContract: memoryAdapterContractConfiguration,
612
- },
613
- rubric: buildRubricSummary(
614
- thresholdConfiguration,
615
- benchmarkIntelligenceExecution.parsedReport,
616
- memoryContinuityExecution.parsedReport
617
- ),
618
- bugIndicators,
619
- reliabilitySignals,
620
- securityIndicators,
621
- releaseDelta,
622
- history: historyEntries,
623
- trendReport,
624
- outputs: {
625
- detectionBenchmark: detectionBenchmarkExecution.parsedReport,
626
- benchmarkGate: benchmarkGateExecution.parsedReport,
627
- benchmarkIntelligence: benchmarkIntelligenceExecution.parsedReport,
628
- memoryContinuityBenchmark: memoryContinuityExecution.parsedReport,
629
- },
630
- executions: executionSummaries,
631
- };
632
-
633
- if (!isStdoutOnlyMode) {
634
- await fs.mkdir(join(REPOSITORY_ROOT, '.agent-context', 'state'), { recursive: true });
635
- await fs.writeFile(OUTPUT_PATH, JSON.stringify(evidenceBundleReport, null, 2) + '\n', 'utf8');
636
- await fs.writeFile(HISTORY_OUTPUT_PATH, JSON.stringify(historyPayload, null, 2) + '\n', 'utf8');
637
- await fs.writeFile(TREND_JSON_OUTPUT_PATH, JSON.stringify(trendReport, null, 2) + '\n', 'utf8');
638
- await fs.writeFile(TREND_CSV_OUTPUT_PATH, trendCsvPayload, 'utf8');
639
- }
640
-
641
- console.log(JSON.stringify(evidenceBundleReport, null, 2));
642
- process.exit(evidenceBundleReport.passed ? 0 : 1);
643
- }
644
-
645
- runBenchmarkEvidenceBundle();