@ryuenn3123/agentic-senior-core 4.4.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/.agents/plugins/agentic-senior-core/plugin.json +3 -0
  2. package/.agents/plugins/agentic-senior-core/rules/agentic-senior-core.md +131 -0
  3. package/.agents/plugins/agentic-senior-core/skills/asc/SKILL.md +16 -0
  4. package/.agents/plugins/agentic-senior-core/skills/asc-audit/SKILL.md +28 -0
  5. package/.agents/plugins/agentic-senior-core/skills/asc-refactor/SKILL.md +35 -0
  6. package/.agents/plugins/agentic-senior-core/skills/asc-review/SKILL.md +54 -0
  7. package/.agents/rules/agentic-senior-core.md +131 -0
  8. package/.claude-plugin/marketplace.json +17 -0
  9. package/.claude-plugin/plugin.json +17 -0
  10. package/.clinerules/agentic-senior-core.md +94 -0
  11. package/.codex-plugin/plugin.json +24 -0
  12. package/.cursor/rules/agentic-senior-core.mdc +99 -0
  13. package/.devin-plugin/plugin.json +13 -0
  14. package/.github/copilot-instructions.md +94 -0
  15. package/.github/plugin/marketplace.json +20 -0
  16. package/.github/plugin/plugin.json +16 -0
  17. package/.kiro/steering/agentic-senior-core.md +94 -0
  18. package/.openclaw/skills/asc/SKILL.md +23 -0
  19. package/.openclaw/skills/asc-audit/SKILL.md +28 -0
  20. package/.openclaw/skills/asc-review/SKILL.md +54 -0
  21. package/.opencode/plugins/agentic-senior-core.mjs +31 -0
  22. package/.windsurf/rules/agentic-senior-core.md +94 -0
  23. package/AGENTS.md +126 -127
  24. package/CONTRIBUTING.md +93 -63
  25. package/README.md +190 -110
  26. package/__init__.py +61 -0
  27. package/bin/agentic-senior-core.js +42 -41
  28. package/commands/asc-audit.md +1 -0
  29. package/commands/asc-audit.toml +2 -0
  30. package/commands/asc-help.md +9 -0
  31. package/commands/asc-help.toml +2 -0
  32. package/commands/asc-refactor.md +1 -0
  33. package/commands/asc-refactor.toml +2 -0
  34. package/commands/asc-review.md +1 -0
  35. package/commands/asc-review.toml +2 -0
  36. package/gemini-extension.json +10 -0
  37. package/hooks/copilot-hooks.json +21 -0
  38. package/hooks/hooks.json +31 -0
  39. package/hooks/session-start.js +36 -0
  40. package/hooks/subagent-start.js +38 -0
  41. package/lib/cli/commands/adapter.mjs +109 -0
  42. package/lib/cli/commands/clean.mjs +63 -0
  43. package/lib/cli/commands/status.mjs +94 -0
  44. package/package.json +43 -60
  45. package/plugin.yaml +16 -0
  46. package/scripts/uninstall.js +38 -0
  47. package/skills/asc/SKILL.md +16 -0
  48. package/skills/asc-audit/SKILL.md +28 -0
  49. package/skills/asc-refactor/SKILL.md +35 -0
  50. package/skills/asc-review/SKILL.md +54 -0
  51. package/.agent-context/policies/llm-judge-threshold.json +0 -29
  52. package/.agent-context/prompts/bootstrap-design.md +0 -63
  53. package/.agent-context/prompts/compact-natural-mode.md +0 -100
  54. package/.agent-context/prompts/init-project.md +0 -60
  55. package/.agent-context/prompts/refactor.md +0 -38
  56. package/.agent-context/prompts/review-code.md +0 -31
  57. package/.agent-context/review-checklists/architecture-review.md +0 -57
  58. package/.agent-context/review-checklists/pr-checklist.md +0 -134
  59. package/.agent-context/rules/api-docs.md +0 -24
  60. package/.agent-context/rules/api-versioning.md +0 -16
  61. package/.agent-context/rules/architecture.md +0 -38
  62. package/.agent-context/rules/background-jobs.md +0 -17
  63. package/.agent-context/rules/config-and-flags.md +0 -16
  64. package/.agent-context/rules/database-design.md +0 -18
  65. package/.agent-context/rules/docker-runtime.md +0 -17
  66. package/.agent-context/rules/error-handling.md +0 -16
  67. package/.agent-context/rules/event-driven.md +0 -16
  68. package/.agent-context/rules/frontend-architecture.md +0 -54
  69. package/.agent-context/rules/microservices.md +0 -16
  70. package/.agent-context/rules/migrations.md +0 -16
  71. package/.agent-context/rules/observability.md +0 -16
  72. package/.agent-context/rules/performance.md +0 -16
  73. package/.agent-context/rules/realtime.md +0 -16
  74. package/.agent-context/rules/resilience.md +0 -18
  75. package/.agent-context/rules/security.md +0 -18
  76. package/.agent-context/rules/testing.md +0 -16
  77. package/.agent-context/state/README.md +0 -25
  78. package/.agent-context/state/architecture-map.md +0 -40
  79. package/.agent-context/state/benchmark-comparison-schema.json +0 -181
  80. package/.agent-context/state/benchmark-reproducibility.json +0 -87
  81. package/.agent-context/state/benchmark-thresholds.json +0 -10
  82. package/.agent-context/state/benchmark-writer-judge-config.json +0 -58
  83. package/.agent-context/state/dependency-map.md +0 -41
  84. package/.agent-context/state/memory-adapter-contract.json +0 -52
  85. package/.agent-context/state/memory-schema-v1.json +0 -88
  86. package/GEMINI.md +0 -1
  87. package/benchmarks/README.md +0 -60
  88. package/benchmarks/compact-natural-mode/fixtures.mjs +0 -359
  89. package/benchmarks/compact-natural-mode/scorer.mjs +0 -331
  90. package/benchmarks/runtime-token-saver/fixtures.mjs +0 -836
  91. package/lib/cli/adaptive-context/catalog.mjs +0 -394
  92. package/lib/cli/adaptive-context/file-signals.mjs +0 -100
  93. package/lib/cli/adaptive-context/implications.mjs +0 -44
  94. package/lib/cli/adaptive-context.mjs +0 -151
  95. package/lib/cli/audits/typography-palette-anti-repeat/color-utils.mjs +0 -156
  96. package/lib/cli/audits/typography-palette-anti-repeat/file-scanner.mjs +0 -103
  97. package/lib/cli/audits/typography-palette-anti-repeat/typography-utils.mjs +0 -70
  98. package/lib/cli/backup.mjs +0 -210
  99. package/lib/cli/commands/context.mjs +0 -140
  100. package/lib/cli/commands/init/project-context.mjs +0 -60
  101. package/lib/cli/commands/init/runtime-environment.mjs +0 -59
  102. package/lib/cli/commands/init/setup-decisions.mjs +0 -83
  103. package/lib/cli/commands/init.mjs +0 -654
  104. package/lib/cli/commands/launch.mjs +0 -76
  105. package/lib/cli/commands/optimize.mjs +0 -284
  106. package/lib/cli/commands/rollback.mjs +0 -64
  107. package/lib/cli/commands/upgrade/token-optimization-state.mjs +0 -51
  108. package/lib/cli/commands/upgrade.mjs +0 -451
  109. package/lib/cli/compiler.mjs +0 -187
  110. package/lib/cli/constants.mjs +0 -136
  111. package/lib/cli/detector/constants.mjs +0 -135
  112. package/lib/cli/detector/design-evidence/collector.mjs +0 -256
  113. package/lib/cli/detector/design-evidence/constants.mjs +0 -39
  114. package/lib/cli/detector/design-evidence/file-traversal.mjs +0 -83
  115. package/lib/cli/detector/design-evidence/structured-attribute-evidence.mjs +0 -117
  116. package/lib/cli/detector/design-evidence/summary.mjs +0 -109
  117. package/lib/cli/detector/design-evidence/utility-helpers.mjs +0 -122
  118. package/lib/cli/detector/design-evidence.mjs +0 -25
  119. package/lib/cli/detector/stack-detection.mjs +0 -243
  120. package/lib/cli/detector/ui-signals.mjs +0 -150
  121. package/lib/cli/detector/workspace-scan.mjs +0 -177
  122. package/lib/cli/detector.mjs +0 -23
  123. package/lib/cli/init-detection-flow.mjs +0 -48
  124. package/lib/cli/init-options.mjs +0 -237
  125. package/lib/cli/init-selection.mjs +0 -29
  126. package/lib/cli/memory-continuity.mjs +0 -537
  127. package/lib/cli/preflight.mjs +0 -76
  128. package/lib/cli/project-scaffolder/constants.mjs +0 -69
  129. package/lib/cli/project-scaffolder/discovery.mjs +0 -272
  130. package/lib/cli/project-scaffolder/prompt-builders.mjs +0 -171
  131. package/lib/cli/project-scaffolder/storage.mjs +0 -144
  132. package/lib/cli/project-scaffolder/ui-scope-detection.mjs +0 -36
  133. package/lib/cli/project-scaffolder.mjs +0 -29
  134. package/lib/cli/rollback.mjs +0 -66
  135. package/lib/cli/token-optimization.mjs +0 -401
  136. package/lib/cli/utils/filesystem.mjs +0 -81
  137. package/lib/cli/utils/managed-surface.mjs +0 -280
  138. package/lib/cli/utils/prompting.mjs +0 -44
  139. package/lib/cli/utils.mjs +0 -479
  140. package/mcp.json +0 -134
  141. package/scripts/adaptive-context/fixtures.mjs +0 -183
  142. package/scripts/adaptive-context-benchmark.mjs +0 -9
  143. package/scripts/ascx-runtime-token-saver-benchmark.mjs +0 -9
  144. package/scripts/benchmark-evidence-bundle.mjs +0 -645
  145. package/scripts/benchmark-gate.mjs +0 -121
  146. package/scripts/benchmark-intelligence.mjs +0 -33
  147. package/scripts/benchmark-writer-judge-matrix.mjs +0 -379
  148. package/scripts/build-release-benchmark-bundle.mjs +0 -202
  149. package/scripts/bump-version.mjs +0 -103
  150. package/scripts/clean-local-artifacts.mjs +0 -78
  151. package/scripts/compact-natural-mode-benchmark.mjs +0 -9
  152. package/scripts/context-triggered-audit.mjs +0 -391
  153. package/scripts/detection-benchmark.mjs +0 -127
  154. package/scripts/docs-quality-drift-report.mjs +0 -358
  155. package/scripts/documentation-boundary-audit.mjs +0 -305
  156. package/scripts/explain-on-demand-audit.mjs +0 -427
  157. package/scripts/forbidden-content-check.mjs +0 -123
  158. package/scripts/frontend-usability-audit.mjs +0 -243
  159. package/scripts/generate-sbom.mjs +0 -61
  160. package/scripts/governance-weekly-report.mjs +0 -466
  161. package/scripts/llm-judge/checklist-loader.mjs +0 -45
  162. package/scripts/llm-judge/constants.mjs +0 -66
  163. package/scripts/llm-judge/diff-collection.mjs +0 -74
  164. package/scripts/llm-judge/prompting.mjs +0 -78
  165. package/scripts/llm-judge/providers.mjs +0 -111
  166. package/scripts/llm-judge/verdict.mjs +0 -134
  167. package/scripts/llm-judge.mjs +0 -200
  168. package/scripts/memory-continuity-benchmark.mjs +0 -322
  169. package/scripts/quality-trend-report.mjs +0 -289
  170. package/scripts/release-gate/audit-checks.mjs +0 -441
  171. package/scripts/release-gate/constants.mjs +0 -53
  172. package/scripts/release-gate/runtime.mjs +0 -63
  173. package/scripts/release-gate/static-checks.mjs +0 -182
  174. package/scripts/release-gate.mjs +0 -42
  175. package/scripts/rules-guardian-audit.mjs +0 -580
  176. package/scripts/single-source-lazy-loading-audit.mjs +0 -449
  177. package/scripts/sync-thin-adapters.mjs +0 -72
  178. package/scripts/token-optimization-benchmark.mjs +0 -252
  179. package/scripts/ui-design-judge/constants.mjs +0 -13
  180. package/scripts/ui-design-judge/design-execution-summary.mjs +0 -81
  181. package/scripts/ui-design-judge/git-input.mjs +0 -134
  182. package/scripts/ui-design-judge/prompting.mjs +0 -52
  183. package/scripts/ui-design-judge/providers.mjs +0 -102
  184. package/scripts/ui-design-judge/reporting.mjs +0 -183
  185. package/scripts/ui-design-judge/rubric-calibration.mjs +0 -214
  186. package/scripts/ui-design-judge/rubric-goldset.json +0 -188
  187. package/scripts/ui-design-judge.mjs +0 -319
  188. package/scripts/ui-rubric-calibration.mjs +0 -35
  189. package/scripts/validate/audits/cache-layer-contract.mjs +0 -230
  190. package/scripts/validate/audits/caching-scope-hygiene.mjs +0 -235
  191. package/scripts/validate/audits/file-size.mjs +0 -158
  192. package/scripts/validate/audits/reflection-citations.mjs +0 -130
  193. package/scripts/validate/audits/release-bundle.mjs +0 -135
  194. package/scripts/validate/audits/rule-id-uniqueness.mjs +0 -278
  195. package/scripts/validate/config.mjs +0 -501
  196. package/scripts/validate/coverage-checks.mjs +0 -446
  197. package/scripts/validate/file-structure.mjs +0 -158
  198. package/scripts/validate/markdown-content.mjs +0 -109
  199. package/scripts/validate/project-metadata.mjs +0 -166
  200. package/scripts/validate/utils.mjs +0 -52
  201. package/scripts/validate.mjs +0 -267
@@ -1,645 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- // @file-size-exception: Pre-existing benchmark bundler (sequential per-section assembly); planned for split in Phase 1.
4
- /**
5
- * benchmark-evidence-bundle.mjs
6
- *
7
- * Benchmark evidence bundle with reproducibility, trend history,
8
- * security signals, and reliability early warnings.
9
- */
10
-
11
- import { existsSync, readFileSync } from 'node:fs';
12
- import fs from 'node:fs/promises';
13
- import { spawnSync } from 'node:child_process';
14
- import { dirname, join, relative, resolve } from 'node:path';
15
- import { fileURLToPath } from 'node:url';
16
-
17
- const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
18
- const SCRIPT_DIR = dirname(SCRIPT_FILE_PATH);
19
- const REPOSITORY_ROOT = resolve(SCRIPT_DIR, '..');
20
- const ARGUMENT_FLAGS = new Set(process.argv.slice(2));
21
- const isStdoutOnlyMode = ARGUMENT_FLAGS.has('--stdout-only');
22
-
23
- const PACKAGE_JSON_PATH = join(REPOSITORY_ROOT, 'package.json');
24
- const REPRO_PROFILE_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-reproducibility.json');
25
- const BENCHMARK_THRESHOLD_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-thresholds.json');
26
- const MEMORY_SCHEMA_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-schema-v1.json');
27
- const MEMORY_ADAPTER_CONTRACT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-adapter-contract.json');
28
- const OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-evidence-bundle.json');
29
- const HISTORY_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-history.json');
30
- const TREND_JSON_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.json');
31
- const TREND_CSV_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.csv');
32
-
33
- const MAX_HISTORY_ENTRIES = 90;
34
- const RELIABILITY_THRESHOLDS = {
35
- minimumConfidenceGap: 0.1,
36
- maximumLowConfidenceRate: 0.2,
37
- maximumIncorrectDetectionRate: 0.1,
38
- };
39
-
40
- function readJsonOrNull(filePath) {
41
- if (!existsSync(filePath)) {
42
- return null;
43
- }
44
-
45
- try {
46
- return JSON.parse(readFileSync(filePath, 'utf8'));
47
- } catch {
48
- return null;
49
- }
50
- }
51
-
52
- function toRelativePath(filePath) {
53
- return relative(REPOSITORY_ROOT, filePath).replace(/\\/g, '/');
54
- }
55
-
56
- function toFiniteNumber(rawValue, fallbackValue = null) {
57
- const parsedValue = Number(rawValue);
58
- if (!Number.isFinite(parsedValue)) {
59
- return fallbackValue;
60
- }
61
-
62
- return parsedValue;
63
- }
64
-
65
- function parseJsonPayload(rawPayload) {
66
- const payloadText = String(rawPayload || '').trim();
67
- if (!payloadText) {
68
- return { parsed: null, error: 'Payload is empty' };
69
- }
70
-
71
- try {
72
- return { parsed: JSON.parse(payloadText), error: null };
73
- } catch {
74
- const firstCurlyBracketIndex = payloadText.indexOf('{');
75
- const lastCurlyBracketIndex = payloadText.lastIndexOf('}');
76
-
77
- if (firstCurlyBracketIndex !== -1 && lastCurlyBracketIndex > firstCurlyBracketIndex) {
78
- const candidatePayload = payloadText.slice(firstCurlyBracketIndex, lastCurlyBracketIndex + 1);
79
- try {
80
- return { parsed: JSON.parse(candidatePayload), error: null };
81
- } catch (secondError) {
82
- return {
83
- parsed: null,
84
- error: secondError instanceof Error ? secondError.message : String(secondError),
85
- };
86
- }
87
- }
88
-
89
- return { parsed: null, error: 'No JSON object found in payload' };
90
- }
91
- }
92
-
93
- function runJsonScript(scriptRelativePath) {
94
- const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
95
- const executionResult = spawnSync('node', [absoluteScriptPath], {
96
- cwd: REPOSITORY_ROOT,
97
- encoding: 'utf8',
98
- maxBuffer: 1024 * 1024 * 10,
99
- });
100
-
101
- const stdoutContent = (executionResult.stdout || '').trim();
102
- const stderrContent = (executionResult.stderr || '').trim();
103
- const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
104
-
105
- if (!stdoutContent) {
106
- return {
107
- scriptPath: scriptRelativePath,
108
- exitCode,
109
- parsedReport: null,
110
- parseError: 'Script produced no stdout JSON payload',
111
- stderr: stderrContent,
112
- };
113
- }
114
-
115
- try {
116
- return {
117
- scriptPath: scriptRelativePath,
118
- exitCode,
119
- parsedReport: JSON.parse(stdoutContent),
120
- parseError: null,
121
- stderr: stderrContent,
122
- };
123
- } catch (jsonParseError) {
124
- const parseErrorMessage = jsonParseError instanceof Error ? jsonParseError.message : String(jsonParseError);
125
- return {
126
- scriptPath: scriptRelativePath,
127
- exitCode,
128
- parsedReport: null,
129
- parseError: parseErrorMessage,
130
- stderr: stderrContent,
131
- };
132
- }
133
- }
134
-
135
- function runNodeScript(scriptRelativePath, argumentsList = []) {
136
- const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
137
- const executionResult = spawnSync('node', [absoluteScriptPath, ...argumentsList], {
138
- cwd: REPOSITORY_ROOT,
139
- encoding: 'utf8',
140
- maxBuffer: 1024 * 1024 * 10,
141
- });
142
-
143
- return {
144
- scriptPath: scriptRelativePath,
145
- exitCode: typeof executionResult.status === 'number' ? executionResult.status : 1,
146
- stdout: (executionResult.stdout || '').trim(),
147
- stderr: (executionResult.stderr || '').trim(),
148
- };
149
- }
150
-
151
- function runNpmAuditIndicator() {
152
- const executionResult = spawnSync('npm', ['audit', '--json', '--omit=dev'], {
153
- cwd: REPOSITORY_ROOT,
154
- encoding: 'utf8',
155
- maxBuffer: 1024 * 1024 * 10,
156
- });
157
-
158
- const combinedOutput = [executionResult.stdout, executionResult.stderr].filter(Boolean).join('\n').trim();
159
- const { parsed: parsedAuditReport, error: parseError } = parseJsonPayload(combinedOutput);
160
- const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
161
-
162
- if (!parsedAuditReport || parseError) {
163
- return {
164
- isAvailable: false,
165
- exitCode,
166
- severityCounts: null,
167
- hasKnownVulnerabilities: null,
168
- error: parseError || 'Unable to parse npm audit output',
169
- };
170
- }
171
-
172
- const vulnerabilityMetadata = parsedAuditReport.metadata?.vulnerabilities || null;
173
- const severityCounts = vulnerabilityMetadata
174
- ? {
175
- info: toFiniteNumber(vulnerabilityMetadata.info, 0),
176
- low: toFiniteNumber(vulnerabilityMetadata.low, 0),
177
- moderate: toFiniteNumber(vulnerabilityMetadata.moderate, 0),
178
- high: toFiniteNumber(vulnerabilityMetadata.high, 0),
179
- critical: toFiniteNumber(vulnerabilityMetadata.critical, 0),
180
- total: toFiniteNumber(vulnerabilityMetadata.total, 0),
181
- }
182
- : null;
183
-
184
- if (!severityCounts) {
185
- return {
186
- isAvailable: false,
187
- exitCode,
188
- severityCounts: null,
189
- hasKnownVulnerabilities: null,
190
- error: parsedAuditReport.error?.summary || 'npm audit report does not include vulnerability metadata',
191
- };
192
- }
193
-
194
- return {
195
- isAvailable: true,
196
- exitCode,
197
- severityCounts,
198
- hasKnownVulnerabilities: severityCounts.total > 0,
199
- error: null,
200
- };
201
- }
202
-
203
- function summarizeExecution(scriptExecutionResult) {
204
- return {
205
- scriptPath: scriptExecutionResult.scriptPath,
206
- exitCode: scriptExecutionResult.exitCode,
207
- parseError: scriptExecutionResult.parseError,
208
- stderr: scriptExecutionResult.stderr || null,
209
- reportName: scriptExecutionResult.parsedReport?.reportName || scriptExecutionResult.parsedReport?.gateName || null,
210
- passed: typeof scriptExecutionResult.parsedReport?.passed === 'boolean'
211
- ? scriptExecutionResult.parsedReport.passed
212
- : null,
213
- };
214
- }
215
-
216
- function appendUniqueTextValues(baseValues, additionalValues) {
217
- const mergedValues = [...baseValues];
218
- for (const additionalValue of additionalValues) {
219
- if (!mergedValues.includes(additionalValue)) {
220
- mergedValues.push(additionalValue);
221
- }
222
- }
223
-
224
- return mergedValues;
225
- }
226
-
227
- function buildRubricSummary(thresholdConfiguration, intelligenceReport, memoryContinuityReport) {
228
- return {
229
- benchmarkThresholds: {
230
- minimumTop1Accuracy: thresholdConfiguration?.minimumTop1Accuracy ?? null,
231
- maximumManualCorrectionRate: thresholdConfiguration?.maximumManualCorrectionRate ?? null,
232
- maximumTop1AccuracyDrop: thresholdConfiguration?.maximumTop1AccuracyDrop ?? null,
233
- maximumManualCorrectionIncrease: thresholdConfiguration?.maximumManualCorrectionIncrease ?? null,
234
- },
235
- intelligenceSlaDays: intelligenceReport?.reviewSlaDays ?? null,
236
- staticExternalWatchlistRetired: intelligenceReport?.staticExternalWatchlistRetired === true,
237
- reliabilityThresholds: RELIABILITY_THRESHOLDS,
238
- continuityThresholds: memoryContinuityReport?.thresholds || null,
239
- };
240
- }
241
-
242
- function buildReliabilitySignals(detectionBenchmarkReport) {
243
- const fixtureResults = Array.isArray(detectionBenchmarkReport?.fixtures) ? detectionBenchmarkReport.fixtures : [];
244
- const fixtureCount = fixtureResults.length;
245
-
246
- const incorrectFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.isCorrect === false);
247
- const lowConfidenceFixtures = fixtureResults.filter((fixtureResult) => {
248
- const confidenceGap = toFiniteNumber(fixtureResult?.confidenceGap, 0);
249
- return confidenceGap < RELIABILITY_THRESHOLDS.minimumConfidenceGap;
250
- });
251
- const manualCorrectionFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.needsManualCorrection === true);
252
-
253
- const incorrectDetectionRate = fixtureCount === 0
254
- ? 0
255
- : Number((incorrectFixtures.length / fixtureCount).toFixed(4));
256
- const lowConfidenceRate = fixtureCount === 0
257
- ? 0
258
- : Number((lowConfidenceFixtures.length / fixtureCount).toFixed(4));
259
- const manualCorrectionRate = fixtureCount === 0
260
- ? 0
261
- : Number((manualCorrectionFixtures.length / fixtureCount).toFixed(4));
262
-
263
- const reliabilityChecks = [
264
- {
265
- checkName: 'incorrect-detection-rate',
266
- passed: incorrectDetectionRate <= RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate,
267
- details: `incorrectRate=${incorrectDetectionRate} max=${RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate}`,
268
- },
269
- {
270
- checkName: 'low-confidence-rate',
271
- passed: lowConfidenceRate <= RELIABILITY_THRESHOLDS.maximumLowConfidenceRate,
272
- details: `lowConfidenceRate=${lowConfidenceRate} max=${RELIABILITY_THRESHOLDS.maximumLowConfidenceRate}`,
273
- },
274
- {
275
- checkName: 'manual-correction-early-warning',
276
- passed: manualCorrectionRate <= 0.12,
277
- details: `manualCorrectionRate=${manualCorrectionRate} warningThreshold=0.12`,
278
- },
279
- ];
280
-
281
- const failureCount = reliabilityChecks.filter((reliabilityCheck) => !reliabilityCheck.passed).length;
282
- const riskLevel = failureCount === 0
283
- ? (incorrectFixtures.length > 0 || lowConfidenceFixtures.length > 0 ? 'monitor' : 'stable')
284
- : (failureCount >= 2 ? 'high' : 'elevated');
285
-
286
- return {
287
- passed: failureCount === 0,
288
- failureCount,
289
- riskLevel,
290
- thresholds: RELIABILITY_THRESHOLDS,
291
- metrics: {
292
- fixtureCount,
293
- incorrectFixtureCount: incorrectFixtures.length,
294
- lowConfidenceFixtureCount: lowConfidenceFixtures.length,
295
- manualCorrectionFixtureCount: manualCorrectionFixtures.length,
296
- incorrectDetectionRate,
297
- lowConfidenceRate,
298
- manualCorrectionRate,
299
- },
300
- checks: reliabilityChecks,
301
- flaggedFixtures: fixtureResults
302
- .filter((fixtureResult) => fixtureResult?.isCorrect === false || fixtureResult?.needsManualCorrection === true)
303
- .map((fixtureResult) => ({
304
- fixtureName: fixtureResult.fixtureName,
305
- confidenceGap: fixtureResult.confidenceGap,
306
- detectedStack: fixtureResult.detectedStack,
307
- expectedStack: fixtureResult.expectedStack,
308
- isCorrect: fixtureResult.isCorrect,
309
- needsManualCorrection: fixtureResult.needsManualCorrection,
310
- })),
311
- };
312
- }
313
-
314
- function buildBugIndicators(reliabilitySignals) {
315
- return {
316
- incorrectFixtureCount: reliabilitySignals.metrics.incorrectFixtureCount,
317
- incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
318
- manualCorrectionFixtureCount: reliabilitySignals.metrics.manualCorrectionFixtureCount,
319
- manualCorrectionRate: reliabilitySignals.metrics.manualCorrectionRate,
320
- lowConfidenceFixtureCount: reliabilitySignals.metrics.lowConfidenceFixtureCount,
321
- lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
322
- flaggedFixtures: reliabilitySignals.flaggedFixtures,
323
- };
324
- }
325
-
326
- function buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator) {
327
- const forbiddenContentPassed = forbiddenContentExecution.exitCode === 0;
328
-
329
- return {
330
- forbiddenContent: {
331
- checkName: 'forbidden-content-scan',
332
- passed: forbiddenContentPassed,
333
- exitCode: forbiddenContentExecution.exitCode,
334
- details: forbiddenContentPassed
335
- ? 'No forbidden content detected'
336
- : 'Forbidden content scan found one or more violations',
337
- },
338
- vulnerabilityScan: {
339
- checkName: 'npm-audit-indicator',
340
- isAvailable: npmAuditIndicator.isAvailable,
341
- hasKnownVulnerabilities: npmAuditIndicator.hasKnownVulnerabilities,
342
- severityCounts: npmAuditIndicator.severityCounts,
343
- exitCode: npmAuditIndicator.exitCode,
344
- error: npmAuditIndicator.error,
345
- },
346
- };
347
- }
348
-
349
- function readReleaseVersion() {
350
- const packageJson = readJsonOrNull(PACKAGE_JSON_PATH);
351
- return typeof packageJson?.version === 'string' && packageJson.version.trim().length > 0
352
- ? packageJson.version.trim()
353
- : 'unknown';
354
- }
355
-
356
- function loadBenchmarkHistory() {
357
- const historyPayload = readJsonOrNull(HISTORY_OUTPUT_PATH);
358
-
359
- if (Array.isArray(historyPayload?.history)) {
360
- return historyPayload.history;
361
- }
362
-
363
- if (Array.isArray(historyPayload)) {
364
- return historyPayload;
365
- }
366
-
367
- return [];
368
- }
369
-
370
- function mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot) {
371
- const mergedHistoryEntries = [...previousHistoryEntries, currentSnapshot];
372
- if (mergedHistoryEntries.length <= MAX_HISTORY_ENTRIES) {
373
- return mergedHistoryEntries;
374
- }
375
-
376
- return mergedHistoryEntries.slice(mergedHistoryEntries.length - MAX_HISTORY_ENTRIES);
377
- }
378
-
379
- function buildHistorySnapshot({
380
- generatedAt,
381
- releaseVersion,
382
- detectionBenchmarkReport,
383
- benchmarkGateReport,
384
- benchmarkIntelligenceReport,
385
- reliabilitySignals,
386
- securityIndicators,
387
- }) {
388
- return {
389
- generatedAt,
390
- releaseVersion,
391
- fixtureCount: toFiniteNumber(detectionBenchmarkReport?.fixtureCount, 0),
392
- top1Accuracy: toFiniteNumber(detectionBenchmarkReport?.top1Accuracy, 0),
393
- manualCorrectionRate: toFiniteNumber(detectionBenchmarkReport?.manualCorrectionRate, 0),
394
- benchmarkGatePassed: benchmarkGateReport?.passed === true,
395
- intelligencePassed: benchmarkIntelligenceReport?.passed === true,
396
- staticExternalWatchlistRetired: benchmarkIntelligenceReport?.staticExternalWatchlistRetired === true,
397
- reliabilityPassed: reliabilitySignals.passed,
398
- reliabilityRiskLevel: reliabilitySignals.riskLevel,
399
- incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
400
- lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
401
- vulnerabilityTotal: securityIndicators.vulnerabilityScan.severityCounts?.total ?? null,
402
- criticalVulnerabilityCount: securityIndicators.vulnerabilityScan.severityCounts?.critical ?? null,
403
- forbiddenContentPassed: securityIndicators.forbiddenContent.passed,
404
- };
405
- }
406
-
407
- function buildReleaseDelta(historyEntries, currentSnapshot) {
408
- const previousSnapshots = historyEntries.slice(0, -1);
409
- if (previousSnapshots.length === 0) {
410
- return null;
411
- }
412
-
413
- const previousReleaseSnapshot = [...previousSnapshots].reverse().find(
414
- (historyEntry) => historyEntry.releaseVersion !== currentSnapshot.releaseVersion
415
- ) || previousSnapshots[previousSnapshots.length - 1];
416
-
417
- const top1AccuracyDelta = Number((currentSnapshot.top1Accuracy - previousReleaseSnapshot.top1Accuracy).toFixed(4));
418
- const manualCorrectionDelta = Number((currentSnapshot.manualCorrectionRate - previousReleaseSnapshot.manualCorrectionRate).toFixed(4));
419
- const vulnerabilityDelta =
420
- (toFiniteNumber(currentSnapshot.vulnerabilityTotal, 0) - toFiniteNumber(previousReleaseSnapshot.vulnerabilityTotal, 0));
421
-
422
- return {
423
- currentReleaseVersion: currentSnapshot.releaseVersion,
424
- previousReleaseVersion: previousReleaseSnapshot.releaseVersion,
425
- comparedSnapshot: {
426
- currentGeneratedAt: currentSnapshot.generatedAt,
427
- previousGeneratedAt: previousReleaseSnapshot.generatedAt,
428
- },
429
- top1AccuracyDelta,
430
- manualCorrectionRateDelta: manualCorrectionDelta,
431
- vulnerabilityTotalDelta: vulnerabilityDelta,
432
- summary: [
433
- `top1Accuracy: ${top1AccuracyDelta >= 0 ? '+' : ''}${top1AccuracyDelta}`,
434
- `manualCorrectionRate: ${manualCorrectionDelta >= 0 ? '+' : ''}${manualCorrectionDelta}`,
435
- `vulnerabilityTotal: ${vulnerabilityDelta >= 0 ? '+' : ''}${vulnerabilityDelta}`,
436
- ],
437
- };
438
- }
439
-
440
- function buildTrendTable(historyEntries) {
441
- return historyEntries.map((historyEntry, index) => ({
442
- snapshotIndex: index + 1,
443
- generatedAt: historyEntry.generatedAt,
444
- releaseVersion: historyEntry.releaseVersion,
445
- top1Accuracy: historyEntry.top1Accuracy,
446
- manualCorrectionRate: historyEntry.manualCorrectionRate,
447
- incorrectDetectionRate: historyEntry.incorrectDetectionRate,
448
- lowConfidenceRate: historyEntry.lowConfidenceRate,
449
- staticExternalWatchlistRetired: historyEntry.staticExternalWatchlistRetired === true,
450
- vulnerabilityTotal: historyEntry.vulnerabilityTotal,
451
- criticalVulnerabilityCount: historyEntry.criticalVulnerabilityCount,
452
- benchmarkGatePassed: historyEntry.benchmarkGatePassed,
453
- intelligencePassed: historyEntry.intelligencePassed,
454
- reliabilityPassed: historyEntry.reliabilityPassed,
455
- reliabilityRiskLevel: historyEntry.reliabilityRiskLevel,
456
- }));
457
- }
458
-
459
- function buildChartSeries(historyEntries) {
460
- return {
461
- generatedAt: historyEntries.map((historyEntry) => historyEntry.generatedAt),
462
- top1Accuracy: historyEntries.map((historyEntry) => historyEntry.top1Accuracy),
463
- manualCorrectionRate: historyEntries.map((historyEntry) => historyEntry.manualCorrectionRate),
464
- incorrectDetectionRate: historyEntries.map((historyEntry) => historyEntry.incorrectDetectionRate),
465
- lowConfidenceRate: historyEntries.map((historyEntry) => historyEntry.lowConfidenceRate),
466
- staticExternalWatchlistRetired: historyEntries.map((historyEntry) => historyEntry.staticExternalWatchlistRetired === true),
467
- vulnerabilityTotal: historyEntries.map((historyEntry) => historyEntry.vulnerabilityTotal),
468
- };
469
- }
470
-
471
- function convertTrendTableToCsv(trendTable) {
472
- if (trendTable.length === 0) {
473
- return '';
474
- }
475
-
476
- const headers = Object.keys(trendTable[0]);
477
- const csvRows = [headers.join(',')];
478
-
479
- for (const trendRow of trendTable) {
480
- const rowValues = headers.map((header) => {
481
- const rawValue = trendRow[header];
482
- if (rawValue === null || rawValue === undefined) {
483
- return '';
484
- }
485
-
486
- const normalizedValue = String(rawValue).replace(/"/g, '""');
487
- return `"${normalizedValue}"`;
488
- });
489
-
490
- csvRows.push(rowValues.join(','));
491
- }
492
-
493
- return `${csvRows.join('\n')}\n`;
494
- }
495
-
496
- async function runBenchmarkEvidenceBundle() {
497
- const reproducibilityProfile = readJsonOrNull(REPRO_PROFILE_PATH);
498
- const thresholdConfiguration = readJsonOrNull(BENCHMARK_THRESHOLD_PATH);
499
- const memorySchemaConfiguration = readJsonOrNull(MEMORY_SCHEMA_PATH);
500
- const memoryAdapterContractConfiguration = readJsonOrNull(MEMORY_ADAPTER_CONTRACT_PATH);
501
- const releaseVersion = readReleaseVersion();
502
-
503
- const detectionBenchmarkExecution = runJsonScript('scripts/detection-benchmark.mjs');
504
- const benchmarkGateExecution = runJsonScript('scripts/benchmark-gate.mjs');
505
- const benchmarkIntelligenceExecution = runJsonScript('scripts/benchmark-intelligence.mjs');
506
- const memoryContinuityExecution = runJsonScript('scripts/memory-continuity-benchmark.mjs');
507
- const forbiddenContentExecution = runNodeScript('scripts/forbidden-content-check.mjs');
508
- const npmAuditIndicator = runNpmAuditIndicator();
509
-
510
- const executionSummaries = [
511
- summarizeExecution(detectionBenchmarkExecution),
512
- summarizeExecution(benchmarkGateExecution),
513
- summarizeExecution(benchmarkIntelligenceExecution),
514
- summarizeExecution(memoryContinuityExecution),
515
- ];
516
-
517
- const executionFailureCount = executionSummaries.filter((executionSummary) => {
518
- if (executionSummary.parseError) {
519
- return true;
520
- }
521
-
522
- if (typeof executionSummary.passed === 'boolean') {
523
- return executionSummary.passed === false;
524
- }
525
-
526
- return executionSummary.exitCode !== 0;
527
- }).length;
528
-
529
- const reliabilitySignals = buildReliabilitySignals(detectionBenchmarkExecution.parsedReport);
530
- const reliabilityFailureCount = reliabilitySignals.failureCount > 0 ? 1 : 0;
531
- const failureCount = executionFailureCount + reliabilityFailureCount;
532
- const securityIndicators = buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator);
533
- const bugIndicators = buildBugIndicators(reliabilitySignals);
534
-
535
- const generatedAt = new Date().toISOString();
536
- const currentSnapshot = buildHistorySnapshot({
537
- generatedAt,
538
- releaseVersion,
539
- detectionBenchmarkReport: detectionBenchmarkExecution.parsedReport,
540
- benchmarkGateReport: benchmarkGateExecution.parsedReport,
541
- benchmarkIntelligenceReport: benchmarkIntelligenceExecution.parsedReport,
542
- reliabilitySignals,
543
- securityIndicators,
544
- });
545
- const previousHistoryEntries = loadBenchmarkHistory();
546
- const historyEntries = mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot);
547
- const releaseDelta = buildReleaseDelta(historyEntries, currentSnapshot);
548
- const trendTable = buildTrendTable(historyEntries);
549
- const chartSeries = buildChartSeries(historyEntries);
550
-
551
- const trendReport = {
552
- generatedAt,
553
- reportName: 'benchmark-trend-report',
554
- releaseVersion,
555
- historyCount: historyEntries.length,
556
- releaseDelta,
557
- trendTable,
558
- chartSeries,
559
- artifacts: {
560
- historyPath: toRelativePath(HISTORY_OUTPUT_PATH),
561
- jsonPath: toRelativePath(TREND_JSON_OUTPUT_PATH),
562
- csvPath: toRelativePath(TREND_CSV_OUTPUT_PATH),
563
- writeMode: isStdoutOnlyMode ? 'stdout-only' : 'stdout-and-file',
564
- },
565
- };
566
-
567
- const historyPayload = {
568
- generatedAt,
569
- reportName: 'benchmark-history',
570
- maxEntries: MAX_HISTORY_ENTRIES,
571
- history: historyEntries,
572
- };
573
-
574
- const trendCsvPayload = convertTrendTableToCsv(trendTable);
575
- const baseRerunInstructions = Array.isArray(reproducibilityProfile?.rerunInstructions)
576
- ? reproducibilityProfile.rerunInstructions
577
- : [];
578
- const baseCommandExamples = Array.isArray(reproducibilityProfile?.commandExamples)
579
- ? reproducibilityProfile.commandExamples
580
- : [];
581
- const rerunInstructions = appendUniqueTextValues(baseRerunInstructions, [
582
- 'Run npm run benchmark:continuity to validate cross-agent memory hydration, privacy redaction, and token-savings behavior.',
583
- ]);
584
- const commandExamples = appendUniqueTextValues(baseCommandExamples, [
585
- 'npm run benchmark:continuity',
586
- 'node ./scripts/memory-continuity-benchmark.mjs --stdout-only',
587
- ]);
588
-
589
- const evidenceBundleReport = {
590
- generatedAt,
591
- reportName: 'benchmark-evidence-bundle',
592
- phase: 'v2.5.2',
593
- releaseVersion,
594
- passed: failureCount === 0,
595
- failureCount,
596
- methodology: {
597
- deterministicRuntime: reproducibilityProfile?.deterministicRuntime || null,
598
- scenarioCount: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios.length : 0,
599
- commandCount: commandExamples.length,
600
- },
601
- rerunInstructions,
602
- commandExamples,
603
- rawInputs: {
604
- scenarios: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios : [],
605
- benchmarkThresholds: thresholdConfiguration,
606
- benchmarkWatchlist: {
607
- retired: true,
608
- reason: 'Static external benchmark watchlists were removed to avoid stale or biasing research inputs.',
609
- },
610
- memorySchema: memorySchemaConfiguration,
611
- memoryAdapterContract: memoryAdapterContractConfiguration,
612
- },
613
- rubric: buildRubricSummary(
614
- thresholdConfiguration,
615
- benchmarkIntelligenceExecution.parsedReport,
616
- memoryContinuityExecution.parsedReport
617
- ),
618
- bugIndicators,
619
- reliabilitySignals,
620
- securityIndicators,
621
- releaseDelta,
622
- history: historyEntries,
623
- trendReport,
624
- outputs: {
625
- detectionBenchmark: detectionBenchmarkExecution.parsedReport,
626
- benchmarkGate: benchmarkGateExecution.parsedReport,
627
- benchmarkIntelligence: benchmarkIntelligenceExecution.parsedReport,
628
- memoryContinuityBenchmark: memoryContinuityExecution.parsedReport,
629
- },
630
- executions: executionSummaries,
631
- };
632
-
633
- if (!isStdoutOnlyMode) {
634
- await fs.mkdir(join(REPOSITORY_ROOT, '.agent-context', 'state'), { recursive: true });
635
- await fs.writeFile(OUTPUT_PATH, JSON.stringify(evidenceBundleReport, null, 2) + '\n', 'utf8');
636
- await fs.writeFile(HISTORY_OUTPUT_PATH, JSON.stringify(historyPayload, null, 2) + '\n', 'utf8');
637
- await fs.writeFile(TREND_JSON_OUTPUT_PATH, JSON.stringify(trendReport, null, 2) + '\n', 'utf8');
638
- await fs.writeFile(TREND_CSV_OUTPUT_PATH, trendCsvPayload, 'utf8');
639
- }
640
-
641
- console.log(JSON.stringify(evidenceBundleReport, null, 2));
642
- process.exit(evidenceBundleReport.passed ? 0 : 1);
643
- }
644
-
645
- runBenchmarkEvidenceBundle();