@ryuenn3123/agentic-senior-core 4.3.15 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/.agents/plugins/marketplace.json +21 -0
  2. package/.agents/rules/agentic-senior-core.md +126 -0
  3. package/.claude-plugin/marketplace.json +17 -0
  4. package/.claude-plugin/plugin.json +17 -0
  5. package/.clinerules/agentic-senior-core.md +94 -0
  6. package/.codex-plugin/plugin.json +24 -0
  7. package/.cursor/rules/agentic-senior-core.mdc +99 -0
  8. package/.devin-plugin/plugin.json +13 -0
  9. package/.github/copilot-instructions.md +94 -0
  10. package/.github/plugin/marketplace.json +20 -0
  11. package/.github/plugin/plugin.json +16 -0
  12. package/.kiro/steering/agentic-senior-core.md +94 -0
  13. package/.openclaw/skills/asc/SKILL.md +23 -0
  14. package/.openclaw/skills/asc-audit/SKILL.md +28 -0
  15. package/.openclaw/skills/asc-review/SKILL.md +54 -0
  16. package/.opencode/plugins/agentic-senior-core.mjs +31 -0
  17. package/.windsurf/rules/agentic-senior-core.md +94 -0
  18. package/AGENTS.md +126 -179
  19. package/CONTRIBUTING.md +93 -63
  20. package/README.md +160 -111
  21. package/__init__.py +61 -0
  22. package/bin/agentic-senior-core.js +41 -43
  23. package/commands/asc-audit.md +1 -0
  24. package/commands/asc-audit.toml +2 -0
  25. package/commands/asc-help.md +9 -0
  26. package/commands/asc-help.toml +2 -0
  27. package/commands/asc-refactor.md +1 -0
  28. package/commands/asc-refactor.toml +2 -0
  29. package/commands/asc-review.md +1 -0
  30. package/commands/asc-review.toml +2 -0
  31. package/gemini-extension.json +10 -0
  32. package/hooks/copilot-hooks.json +21 -0
  33. package/hooks/hooks.json +31 -0
  34. package/hooks/session-start.js +36 -0
  35. package/hooks/subagent-start.js +38 -0
  36. package/lib/cli/commands/adapter.mjs +109 -0
  37. package/lib/cli/commands/clean.mjs +63 -0
  38. package/lib/cli/commands/status.mjs +94 -0
  39. package/package.json +43 -67
  40. package/plugin.yaml +16 -0
  41. package/scripts/uninstall.js +38 -0
  42. package/skills/asc/SKILL.md +16 -0
  43. package/skills/asc-audit/SKILL.md +28 -0
  44. package/skills/asc-refactor/SKILL.md +35 -0
  45. package/skills/asc-review/SKILL.md +54 -0
  46. package/.agent-context/policies/llm-judge-threshold.json +0 -29
  47. package/.agent-context/prompts/bootstrap-design.md +0 -63
  48. package/.agent-context/prompts/compact-natural-mode.md +0 -100
  49. package/.agent-context/prompts/init-project.md +0 -60
  50. package/.agent-context/prompts/refactor.md +0 -38
  51. package/.agent-context/prompts/review-code.md +0 -31
  52. package/.agent-context/review-checklists/architecture-review.md +0 -57
  53. package/.agent-context/review-checklists/pr-checklist.md +0 -136
  54. package/.agent-context/rules/api-docs.md +0 -24
  55. package/.agent-context/rules/api-versioning.md +0 -16
  56. package/.agent-context/rules/architecture.md +0 -33
  57. package/.agent-context/rules/background-jobs.md +0 -17
  58. package/.agent-context/rules/config-and-flags.md +0 -16
  59. package/.agent-context/rules/database-design.md +0 -18
  60. package/.agent-context/rules/docker-runtime.md +0 -17
  61. package/.agent-context/rules/error-handling.md +0 -16
  62. package/.agent-context/rules/event-driven.md +0 -16
  63. package/.agent-context/rules/frontend-architecture.md +0 -54
  64. package/.agent-context/rules/microservices.md +0 -16
  65. package/.agent-context/rules/migrations.md +0 -16
  66. package/.agent-context/rules/observability.md +0 -16
  67. package/.agent-context/rules/performance.md +0 -16
  68. package/.agent-context/rules/realtime.md +0 -16
  69. package/.agent-context/rules/resilience.md +0 -18
  70. package/.agent-context/rules/security.md +0 -18
  71. package/.agent-context/rules/testing.md +0 -16
  72. package/.agent-context/state/README.md +0 -25
  73. package/.agent-context/state/architecture-map.md +0 -40
  74. package/.agent-context/state/benchmark-comparison-schema.json +0 -181
  75. package/.agent-context/state/benchmark-reproducibility.json +0 -87
  76. package/.agent-context/state/benchmark-thresholds.json +0 -10
  77. package/.agent-context/state/benchmark-writer-judge-config.json +0 -58
  78. package/.agent-context/state/dependency-map.md +0 -41
  79. package/.agent-context/state/memory-adapter-contract.json +0 -52
  80. package/.agent-context/state/memory-schema-v1.json +0 -88
  81. package/GEMINI.md +0 -1
  82. package/benchmarks/README.md +0 -60
  83. package/benchmarks/compact-natural-mode/fixtures.mjs +0 -359
  84. package/benchmarks/compact-natural-mode/scorer.mjs +0 -331
  85. package/benchmarks/runtime-token-saver/fixtures.mjs +0 -836
  86. package/lib/cli/adaptive-context/catalog.mjs +0 -390
  87. package/lib/cli/adaptive-context/file-signals.mjs +0 -100
  88. package/lib/cli/adaptive-context/implications.mjs +0 -44
  89. package/lib/cli/adaptive-context.mjs +0 -365
  90. package/lib/cli/audits/typography-palette-anti-repeat/color-utils.mjs +0 -156
  91. package/lib/cli/audits/typography-palette-anti-repeat/file-scanner.mjs +0 -103
  92. package/lib/cli/audits/typography-palette-anti-repeat/typography-utils.mjs +0 -70
  93. package/lib/cli/audits/typography-palette-anti-repeat-audit.mjs +0 -239
  94. package/lib/cli/backup.mjs +0 -210
  95. package/lib/cli/commands/audit-design-anti-repeat.mjs +0 -39
  96. package/lib/cli/commands/context.mjs +0 -140
  97. package/lib/cli/commands/init/project-context.mjs +0 -60
  98. package/lib/cli/commands/init/runtime-environment.mjs +0 -59
  99. package/lib/cli/commands/init/setup-decisions.mjs +0 -83
  100. package/lib/cli/commands/init.mjs +0 -654
  101. package/lib/cli/commands/launch.mjs +0 -76
  102. package/lib/cli/commands/optimize.mjs +0 -284
  103. package/lib/cli/commands/rollback.mjs +0 -64
  104. package/lib/cli/commands/upgrade/token-optimization-state.mjs +0 -51
  105. package/lib/cli/commands/upgrade.mjs +0 -451
  106. package/lib/cli/compiler.mjs +0 -570
  107. package/lib/cli/constants.mjs +0 -136
  108. package/lib/cli/detector/constants.mjs +0 -135
  109. package/lib/cli/detector/design-evidence/collector.mjs +0 -256
  110. package/lib/cli/detector/design-evidence/constants.mjs +0 -39
  111. package/lib/cli/detector/design-evidence/file-traversal.mjs +0 -83
  112. package/lib/cli/detector/design-evidence/structured-attribute-evidence.mjs +0 -117
  113. package/lib/cli/detector/design-evidence/summary.mjs +0 -109
  114. package/lib/cli/detector/design-evidence/utility-helpers.mjs +0 -122
  115. package/lib/cli/detector/design-evidence.mjs +0 -25
  116. package/lib/cli/detector/stack-detection.mjs +0 -243
  117. package/lib/cli/detector/ui-signals.mjs +0 -150
  118. package/lib/cli/detector/workspace-scan.mjs +0 -177
  119. package/lib/cli/detector.mjs +0 -23
  120. package/lib/cli/init-detection-flow.mjs +0 -48
  121. package/lib/cli/init-options.mjs +0 -237
  122. package/lib/cli/init-selection.mjs +0 -29
  123. package/lib/cli/memory-continuity.mjs +0 -537
  124. package/lib/cli/preflight.mjs +0 -76
  125. package/lib/cli/project-scaffolder/constants.mjs +0 -69
  126. package/lib/cli/project-scaffolder/discovery.mjs +0 -272
  127. package/lib/cli/project-scaffolder/prompt-builders.mjs +0 -171
  128. package/lib/cli/project-scaffolder/storage.mjs +0 -144
  129. package/lib/cli/project-scaffolder/ui-scope-detection.mjs +0 -36
  130. package/lib/cli/project-scaffolder.mjs +0 -29
  131. package/lib/cli/rollback.mjs +0 -66
  132. package/lib/cli/token-optimization.mjs +0 -486
  133. package/lib/cli/utils/filesystem.mjs +0 -81
  134. package/lib/cli/utils/managed-surface.mjs +0 -280
  135. package/lib/cli/utils/prompting.mjs +0 -44
  136. package/lib/cli/utils.mjs +0 -479
  137. package/mcp.json +0 -134
  138. package/scripts/adaptive-context/fixtures.mjs +0 -183
  139. package/scripts/adaptive-context-benchmark.mjs +0 -9
  140. package/scripts/ascx-runtime-token-saver-benchmark.mjs +0 -9
  141. package/scripts/audit-cache-layer-contract.mjs +0 -263
  142. package/scripts/audit-caching-scope-hygiene.mjs +0 -268
  143. package/scripts/audit-file-size.mjs +0 -219
  144. package/scripts/audit-reflection-citations.mjs +0 -163
  145. package/scripts/audit-release-bundle.mjs +0 -170
  146. package/scripts/audit-rule-id-uniqueness.mjs +0 -313
  147. package/scripts/audit-typography-palette-anti-repeat.mjs +0 -120
  148. package/scripts/benchmark-evidence-bundle.mjs +0 -645
  149. package/scripts/benchmark-gate.mjs +0 -121
  150. package/scripts/benchmark-intelligence.mjs +0 -33
  151. package/scripts/benchmark-writer-judge-matrix.mjs +0 -379
  152. package/scripts/build-release-benchmark-bundle.mjs +0 -202
  153. package/scripts/bump-version.mjs +0 -103
  154. package/scripts/clean-local-artifacts.mjs +0 -78
  155. package/scripts/compact-natural-mode-benchmark.mjs +0 -9
  156. package/scripts/context-triggered-audit.mjs +0 -391
  157. package/scripts/detection-benchmark.mjs +0 -127
  158. package/scripts/docs-quality-drift-report.mjs +0 -358
  159. package/scripts/documentation-boundary-audit.mjs +0 -305
  160. package/scripts/explain-on-demand-audit.mjs +0 -427
  161. package/scripts/forbidden-content-check.mjs +0 -123
  162. package/scripts/frontend-usability-audit.mjs +0 -243
  163. package/scripts/generate-sbom.mjs +0 -61
  164. package/scripts/governance-weekly-report.mjs +0 -466
  165. package/scripts/llm-judge/checklist-loader.mjs +0 -45
  166. package/scripts/llm-judge/constants.mjs +0 -66
  167. package/scripts/llm-judge/diff-collection.mjs +0 -74
  168. package/scripts/llm-judge/prompting.mjs +0 -78
  169. package/scripts/llm-judge/providers.mjs +0 -111
  170. package/scripts/llm-judge/verdict.mjs +0 -134
  171. package/scripts/llm-judge.mjs +0 -200
  172. package/scripts/memory-continuity-benchmark.mjs +0 -322
  173. package/scripts/quality-trend-report.mjs +0 -289
  174. package/scripts/release-gate/audit-checks.mjs +0 -441
  175. package/scripts/release-gate/constants.mjs +0 -53
  176. package/scripts/release-gate/runtime.mjs +0 -63
  177. package/scripts/release-gate/static-checks.mjs +0 -182
  178. package/scripts/release-gate.mjs +0 -42
  179. package/scripts/rules-guardian-audit.mjs +0 -580
  180. package/scripts/single-source-lazy-loading-audit.mjs +0 -449
  181. package/scripts/sync-thin-adapters.mjs +0 -72
  182. package/scripts/token-optimization-benchmark.mjs +0 -252
  183. package/scripts/ui-design-judge/constants.mjs +0 -13
  184. package/scripts/ui-design-judge/design-execution-summary.mjs +0 -81
  185. package/scripts/ui-design-judge/git-input.mjs +0 -134
  186. package/scripts/ui-design-judge/prompting.mjs +0 -52
  187. package/scripts/ui-design-judge/providers.mjs +0 -102
  188. package/scripts/ui-design-judge/reporting.mjs +0 -183
  189. package/scripts/ui-design-judge/rubric-calibration.mjs +0 -214
  190. package/scripts/ui-design-judge/rubric-goldset.json +0 -188
  191. package/scripts/ui-design-judge.mjs +0 -319
  192. package/scripts/ui-rubric-calibration.mjs +0 -35
  193. package/scripts/validate/config.mjs +0 -519
  194. package/scripts/validate/coverage-checks.mjs +0 -446
  195. package/scripts/validate/file-structure.mjs +0 -162
  196. package/scripts/validate/markdown-content.mjs +0 -109
  197. package/scripts/validate/project-metadata.mjs +0 -166
  198. package/scripts/validate.mjs +0 -338
@@ -1,645 +0,0 @@
1
- #!/usr/bin/env node
2
-
3
- // @file-size-exception: Pre-existing benchmark bundler (sequential per-section assembly); planned for split in Phase 1.
4
- /**
5
- * benchmark-evidence-bundle.mjs
6
- *
7
- * Benchmark evidence bundle with reproducibility, trend history,
8
- * security signals, and reliability early warnings.
9
- */
10
-
11
- import { existsSync, readFileSync } from 'node:fs';
12
- import fs from 'node:fs/promises';
13
- import { spawnSync } from 'node:child_process';
14
- import { dirname, join, relative, resolve } from 'node:path';
15
- import { fileURLToPath } from 'node:url';
16
-
17
- const SCRIPT_FILE_PATH = fileURLToPath(import.meta.url);
18
- const SCRIPT_DIR = dirname(SCRIPT_FILE_PATH);
19
- const REPOSITORY_ROOT = resolve(SCRIPT_DIR, '..');
20
- const ARGUMENT_FLAGS = new Set(process.argv.slice(2));
21
- const isStdoutOnlyMode = ARGUMENT_FLAGS.has('--stdout-only');
22
-
23
- const PACKAGE_JSON_PATH = join(REPOSITORY_ROOT, 'package.json');
24
- const REPRO_PROFILE_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-reproducibility.json');
25
- const BENCHMARK_THRESHOLD_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-thresholds.json');
26
- const MEMORY_SCHEMA_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-schema-v1.json');
27
- const MEMORY_ADAPTER_CONTRACT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'memory-adapter-contract.json');
28
- const OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-evidence-bundle.json');
29
- const HISTORY_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-history.json');
30
- const TREND_JSON_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.json');
31
- const TREND_CSV_OUTPUT_PATH = join(REPOSITORY_ROOT, '.agent-context', 'state', 'benchmark-trend-report.csv');
32
-
33
- const MAX_HISTORY_ENTRIES = 90;
34
- const RELIABILITY_THRESHOLDS = {
35
- minimumConfidenceGap: 0.1,
36
- maximumLowConfidenceRate: 0.2,
37
- maximumIncorrectDetectionRate: 0.1,
38
- };
39
-
40
- function readJsonOrNull(filePath) {
41
- if (!existsSync(filePath)) {
42
- return null;
43
- }
44
-
45
- try {
46
- return JSON.parse(readFileSync(filePath, 'utf8'));
47
- } catch {
48
- return null;
49
- }
50
- }
51
-
52
- function toRelativePath(filePath) {
53
- return relative(REPOSITORY_ROOT, filePath).replace(/\\/g, '/');
54
- }
55
-
56
- function toFiniteNumber(rawValue, fallbackValue = null) {
57
- const parsedValue = Number(rawValue);
58
- if (!Number.isFinite(parsedValue)) {
59
- return fallbackValue;
60
- }
61
-
62
- return parsedValue;
63
- }
64
-
65
- function parseJsonPayload(rawPayload) {
66
- const payloadText = String(rawPayload || '').trim();
67
- if (!payloadText) {
68
- return { parsed: null, error: 'Payload is empty' };
69
- }
70
-
71
- try {
72
- return { parsed: JSON.parse(payloadText), error: null };
73
- } catch {
74
- const firstCurlyBracketIndex = payloadText.indexOf('{');
75
- const lastCurlyBracketIndex = payloadText.lastIndexOf('}');
76
-
77
- if (firstCurlyBracketIndex !== -1 && lastCurlyBracketIndex > firstCurlyBracketIndex) {
78
- const candidatePayload = payloadText.slice(firstCurlyBracketIndex, lastCurlyBracketIndex + 1);
79
- try {
80
- return { parsed: JSON.parse(candidatePayload), error: null };
81
- } catch (secondError) {
82
- return {
83
- parsed: null,
84
- error: secondError instanceof Error ? secondError.message : String(secondError),
85
- };
86
- }
87
- }
88
-
89
- return { parsed: null, error: 'No JSON object found in payload' };
90
- }
91
- }
92
-
93
- function runJsonScript(scriptRelativePath) {
94
- const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
95
- const executionResult = spawnSync('node', [absoluteScriptPath], {
96
- cwd: REPOSITORY_ROOT,
97
- encoding: 'utf8',
98
- maxBuffer: 1024 * 1024 * 10,
99
- });
100
-
101
- const stdoutContent = (executionResult.stdout || '').trim();
102
- const stderrContent = (executionResult.stderr || '').trim();
103
- const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
104
-
105
- if (!stdoutContent) {
106
- return {
107
- scriptPath: scriptRelativePath,
108
- exitCode,
109
- parsedReport: null,
110
- parseError: 'Script produced no stdout JSON payload',
111
- stderr: stderrContent,
112
- };
113
- }
114
-
115
- try {
116
- return {
117
- scriptPath: scriptRelativePath,
118
- exitCode,
119
- parsedReport: JSON.parse(stdoutContent),
120
- parseError: null,
121
- stderr: stderrContent,
122
- };
123
- } catch (jsonParseError) {
124
- const parseErrorMessage = jsonParseError instanceof Error ? jsonParseError.message : String(jsonParseError);
125
- return {
126
- scriptPath: scriptRelativePath,
127
- exitCode,
128
- parsedReport: null,
129
- parseError: parseErrorMessage,
130
- stderr: stderrContent,
131
- };
132
- }
133
- }
134
-
135
- function runNodeScript(scriptRelativePath, argumentsList = []) {
136
- const absoluteScriptPath = join(REPOSITORY_ROOT, scriptRelativePath);
137
- const executionResult = spawnSync('node', [absoluteScriptPath, ...argumentsList], {
138
- cwd: REPOSITORY_ROOT,
139
- encoding: 'utf8',
140
- maxBuffer: 1024 * 1024 * 10,
141
- });
142
-
143
- return {
144
- scriptPath: scriptRelativePath,
145
- exitCode: typeof executionResult.status === 'number' ? executionResult.status : 1,
146
- stdout: (executionResult.stdout || '').trim(),
147
- stderr: (executionResult.stderr || '').trim(),
148
- };
149
- }
150
-
151
- function runNpmAuditIndicator() {
152
- const executionResult = spawnSync('npm', ['audit', '--json', '--omit=dev'], {
153
- cwd: REPOSITORY_ROOT,
154
- encoding: 'utf8',
155
- maxBuffer: 1024 * 1024 * 10,
156
- });
157
-
158
- const combinedOutput = [executionResult.stdout, executionResult.stderr].filter(Boolean).join('\n').trim();
159
- const { parsed: parsedAuditReport, error: parseError } = parseJsonPayload(combinedOutput);
160
- const exitCode = typeof executionResult.status === 'number' ? executionResult.status : 1;
161
-
162
- if (!parsedAuditReport || parseError) {
163
- return {
164
- isAvailable: false,
165
- exitCode,
166
- severityCounts: null,
167
- hasKnownVulnerabilities: null,
168
- error: parseError || 'Unable to parse npm audit output',
169
- };
170
- }
171
-
172
- const vulnerabilityMetadata = parsedAuditReport.metadata?.vulnerabilities || null;
173
- const severityCounts = vulnerabilityMetadata
174
- ? {
175
- info: toFiniteNumber(vulnerabilityMetadata.info, 0),
176
- low: toFiniteNumber(vulnerabilityMetadata.low, 0),
177
- moderate: toFiniteNumber(vulnerabilityMetadata.moderate, 0),
178
- high: toFiniteNumber(vulnerabilityMetadata.high, 0),
179
- critical: toFiniteNumber(vulnerabilityMetadata.critical, 0),
180
- total: toFiniteNumber(vulnerabilityMetadata.total, 0),
181
- }
182
- : null;
183
-
184
- if (!severityCounts) {
185
- return {
186
- isAvailable: false,
187
- exitCode,
188
- severityCounts: null,
189
- hasKnownVulnerabilities: null,
190
- error: parsedAuditReport.error?.summary || 'npm audit report does not include vulnerability metadata',
191
- };
192
- }
193
-
194
- return {
195
- isAvailable: true,
196
- exitCode,
197
- severityCounts,
198
- hasKnownVulnerabilities: severityCounts.total > 0,
199
- error: null,
200
- };
201
- }
202
-
203
- function summarizeExecution(scriptExecutionResult) {
204
- return {
205
- scriptPath: scriptExecutionResult.scriptPath,
206
- exitCode: scriptExecutionResult.exitCode,
207
- parseError: scriptExecutionResult.parseError,
208
- stderr: scriptExecutionResult.stderr || null,
209
- reportName: scriptExecutionResult.parsedReport?.reportName || scriptExecutionResult.parsedReport?.gateName || null,
210
- passed: typeof scriptExecutionResult.parsedReport?.passed === 'boolean'
211
- ? scriptExecutionResult.parsedReport.passed
212
- : null,
213
- };
214
- }
215
-
216
- function appendUniqueTextValues(baseValues, additionalValues) {
217
- const mergedValues = [...baseValues];
218
- for (const additionalValue of additionalValues) {
219
- if (!mergedValues.includes(additionalValue)) {
220
- mergedValues.push(additionalValue);
221
- }
222
- }
223
-
224
- return mergedValues;
225
- }
226
-
227
- function buildRubricSummary(thresholdConfiguration, intelligenceReport, memoryContinuityReport) {
228
- return {
229
- benchmarkThresholds: {
230
- minimumTop1Accuracy: thresholdConfiguration?.minimumTop1Accuracy ?? null,
231
- maximumManualCorrectionRate: thresholdConfiguration?.maximumManualCorrectionRate ?? null,
232
- maximumTop1AccuracyDrop: thresholdConfiguration?.maximumTop1AccuracyDrop ?? null,
233
- maximumManualCorrectionIncrease: thresholdConfiguration?.maximumManualCorrectionIncrease ?? null,
234
- },
235
- intelligenceSlaDays: intelligenceReport?.reviewSlaDays ?? null,
236
- staticExternalWatchlistRetired: intelligenceReport?.staticExternalWatchlistRetired === true,
237
- reliabilityThresholds: RELIABILITY_THRESHOLDS,
238
- continuityThresholds: memoryContinuityReport?.thresholds || null,
239
- };
240
- }
241
-
242
- function buildReliabilitySignals(detectionBenchmarkReport) {
243
- const fixtureResults = Array.isArray(detectionBenchmarkReport?.fixtures) ? detectionBenchmarkReport.fixtures : [];
244
- const fixtureCount = fixtureResults.length;
245
-
246
- const incorrectFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.isCorrect === false);
247
- const lowConfidenceFixtures = fixtureResults.filter((fixtureResult) => {
248
- const confidenceGap = toFiniteNumber(fixtureResult?.confidenceGap, 0);
249
- return confidenceGap < RELIABILITY_THRESHOLDS.minimumConfidenceGap;
250
- });
251
- const manualCorrectionFixtures = fixtureResults.filter((fixtureResult) => fixtureResult?.needsManualCorrection === true);
252
-
253
- const incorrectDetectionRate = fixtureCount === 0
254
- ? 0
255
- : Number((incorrectFixtures.length / fixtureCount).toFixed(4));
256
- const lowConfidenceRate = fixtureCount === 0
257
- ? 0
258
- : Number((lowConfidenceFixtures.length / fixtureCount).toFixed(4));
259
- const manualCorrectionRate = fixtureCount === 0
260
- ? 0
261
- : Number((manualCorrectionFixtures.length / fixtureCount).toFixed(4));
262
-
263
- const reliabilityChecks = [
264
- {
265
- checkName: 'incorrect-detection-rate',
266
- passed: incorrectDetectionRate <= RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate,
267
- details: `incorrectRate=${incorrectDetectionRate} max=${RELIABILITY_THRESHOLDS.maximumIncorrectDetectionRate}`,
268
- },
269
- {
270
- checkName: 'low-confidence-rate',
271
- passed: lowConfidenceRate <= RELIABILITY_THRESHOLDS.maximumLowConfidenceRate,
272
- details: `lowConfidenceRate=${lowConfidenceRate} max=${RELIABILITY_THRESHOLDS.maximumLowConfidenceRate}`,
273
- },
274
- {
275
- checkName: 'manual-correction-early-warning',
276
- passed: manualCorrectionRate <= 0.12,
277
- details: `manualCorrectionRate=${manualCorrectionRate} warningThreshold=0.12`,
278
- },
279
- ];
280
-
281
- const failureCount = reliabilityChecks.filter((reliabilityCheck) => !reliabilityCheck.passed).length;
282
- const riskLevel = failureCount === 0
283
- ? (incorrectFixtures.length > 0 || lowConfidenceFixtures.length > 0 ? 'monitor' : 'stable')
284
- : (failureCount >= 2 ? 'high' : 'elevated');
285
-
286
- return {
287
- passed: failureCount === 0,
288
- failureCount,
289
- riskLevel,
290
- thresholds: RELIABILITY_THRESHOLDS,
291
- metrics: {
292
- fixtureCount,
293
- incorrectFixtureCount: incorrectFixtures.length,
294
- lowConfidenceFixtureCount: lowConfidenceFixtures.length,
295
- manualCorrectionFixtureCount: manualCorrectionFixtures.length,
296
- incorrectDetectionRate,
297
- lowConfidenceRate,
298
- manualCorrectionRate,
299
- },
300
- checks: reliabilityChecks,
301
- flaggedFixtures: fixtureResults
302
- .filter((fixtureResult) => fixtureResult?.isCorrect === false || fixtureResult?.needsManualCorrection === true)
303
- .map((fixtureResult) => ({
304
- fixtureName: fixtureResult.fixtureName,
305
- confidenceGap: fixtureResult.confidenceGap,
306
- detectedStack: fixtureResult.detectedStack,
307
- expectedStack: fixtureResult.expectedStack,
308
- isCorrect: fixtureResult.isCorrect,
309
- needsManualCorrection: fixtureResult.needsManualCorrection,
310
- })),
311
- };
312
- }
313
-
314
- function buildBugIndicators(reliabilitySignals) {
315
- return {
316
- incorrectFixtureCount: reliabilitySignals.metrics.incorrectFixtureCount,
317
- incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
318
- manualCorrectionFixtureCount: reliabilitySignals.metrics.manualCorrectionFixtureCount,
319
- manualCorrectionRate: reliabilitySignals.metrics.manualCorrectionRate,
320
- lowConfidenceFixtureCount: reliabilitySignals.metrics.lowConfidenceFixtureCount,
321
- lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
322
- flaggedFixtures: reliabilitySignals.flaggedFixtures,
323
- };
324
- }
325
-
326
- function buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator) {
327
- const forbiddenContentPassed = forbiddenContentExecution.exitCode === 0;
328
-
329
- return {
330
- forbiddenContent: {
331
- checkName: 'forbidden-content-scan',
332
- passed: forbiddenContentPassed,
333
- exitCode: forbiddenContentExecution.exitCode,
334
- details: forbiddenContentPassed
335
- ? 'No forbidden content detected'
336
- : 'Forbidden content scan found one or more violations',
337
- },
338
- vulnerabilityScan: {
339
- checkName: 'npm-audit-indicator',
340
- isAvailable: npmAuditIndicator.isAvailable,
341
- hasKnownVulnerabilities: npmAuditIndicator.hasKnownVulnerabilities,
342
- severityCounts: npmAuditIndicator.severityCounts,
343
- exitCode: npmAuditIndicator.exitCode,
344
- error: npmAuditIndicator.error,
345
- },
346
- };
347
- }
348
-
349
- function readReleaseVersion() {
350
- const packageJson = readJsonOrNull(PACKAGE_JSON_PATH);
351
- return typeof packageJson?.version === 'string' && packageJson.version.trim().length > 0
352
- ? packageJson.version.trim()
353
- : 'unknown';
354
- }
355
-
356
- function loadBenchmarkHistory() {
357
- const historyPayload = readJsonOrNull(HISTORY_OUTPUT_PATH);
358
-
359
- if (Array.isArray(historyPayload?.history)) {
360
- return historyPayload.history;
361
- }
362
-
363
- if (Array.isArray(historyPayload)) {
364
- return historyPayload;
365
- }
366
-
367
- return [];
368
- }
369
-
370
- function mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot) {
371
- const mergedHistoryEntries = [...previousHistoryEntries, currentSnapshot];
372
- if (mergedHistoryEntries.length <= MAX_HISTORY_ENTRIES) {
373
- return mergedHistoryEntries;
374
- }
375
-
376
- return mergedHistoryEntries.slice(mergedHistoryEntries.length - MAX_HISTORY_ENTRIES);
377
- }
378
-
379
- function buildHistorySnapshot({
380
- generatedAt,
381
- releaseVersion,
382
- detectionBenchmarkReport,
383
- benchmarkGateReport,
384
- benchmarkIntelligenceReport,
385
- reliabilitySignals,
386
- securityIndicators,
387
- }) {
388
- return {
389
- generatedAt,
390
- releaseVersion,
391
- fixtureCount: toFiniteNumber(detectionBenchmarkReport?.fixtureCount, 0),
392
- top1Accuracy: toFiniteNumber(detectionBenchmarkReport?.top1Accuracy, 0),
393
- manualCorrectionRate: toFiniteNumber(detectionBenchmarkReport?.manualCorrectionRate, 0),
394
- benchmarkGatePassed: benchmarkGateReport?.passed === true,
395
- intelligencePassed: benchmarkIntelligenceReport?.passed === true,
396
- staticExternalWatchlistRetired: benchmarkIntelligenceReport?.staticExternalWatchlistRetired === true,
397
- reliabilityPassed: reliabilitySignals.passed,
398
- reliabilityRiskLevel: reliabilitySignals.riskLevel,
399
- incorrectDetectionRate: reliabilitySignals.metrics.incorrectDetectionRate,
400
- lowConfidenceRate: reliabilitySignals.metrics.lowConfidenceRate,
401
- vulnerabilityTotal: securityIndicators.vulnerabilityScan.severityCounts?.total ?? null,
402
- criticalVulnerabilityCount: securityIndicators.vulnerabilityScan.severityCounts?.critical ?? null,
403
- forbiddenContentPassed: securityIndicators.forbiddenContent.passed,
404
- };
405
- }
406
-
407
- function buildReleaseDelta(historyEntries, currentSnapshot) {
408
- const previousSnapshots = historyEntries.slice(0, -1);
409
- if (previousSnapshots.length === 0) {
410
- return null;
411
- }
412
-
413
- const previousReleaseSnapshot = [...previousSnapshots].reverse().find(
414
- (historyEntry) => historyEntry.releaseVersion !== currentSnapshot.releaseVersion
415
- ) || previousSnapshots[previousSnapshots.length - 1];
416
-
417
- const top1AccuracyDelta = Number((currentSnapshot.top1Accuracy - previousReleaseSnapshot.top1Accuracy).toFixed(4));
418
- const manualCorrectionDelta = Number((currentSnapshot.manualCorrectionRate - previousReleaseSnapshot.manualCorrectionRate).toFixed(4));
419
- const vulnerabilityDelta =
420
- (toFiniteNumber(currentSnapshot.vulnerabilityTotal, 0) - toFiniteNumber(previousReleaseSnapshot.vulnerabilityTotal, 0));
421
-
422
- return {
423
- currentReleaseVersion: currentSnapshot.releaseVersion,
424
- previousReleaseVersion: previousReleaseSnapshot.releaseVersion,
425
- comparedSnapshot: {
426
- currentGeneratedAt: currentSnapshot.generatedAt,
427
- previousGeneratedAt: previousReleaseSnapshot.generatedAt,
428
- },
429
- top1AccuracyDelta,
430
- manualCorrectionRateDelta: manualCorrectionDelta,
431
- vulnerabilityTotalDelta: vulnerabilityDelta,
432
- summary: [
433
- `top1Accuracy: ${top1AccuracyDelta >= 0 ? '+' : ''}${top1AccuracyDelta}`,
434
- `manualCorrectionRate: ${manualCorrectionDelta >= 0 ? '+' : ''}${manualCorrectionDelta}`,
435
- `vulnerabilityTotal: ${vulnerabilityDelta >= 0 ? '+' : ''}${vulnerabilityDelta}`,
436
- ],
437
- };
438
- }
439
-
440
- function buildTrendTable(historyEntries) {
441
- return historyEntries.map((historyEntry, index) => ({
442
- snapshotIndex: index + 1,
443
- generatedAt: historyEntry.generatedAt,
444
- releaseVersion: historyEntry.releaseVersion,
445
- top1Accuracy: historyEntry.top1Accuracy,
446
- manualCorrectionRate: historyEntry.manualCorrectionRate,
447
- incorrectDetectionRate: historyEntry.incorrectDetectionRate,
448
- lowConfidenceRate: historyEntry.lowConfidenceRate,
449
- staticExternalWatchlistRetired: historyEntry.staticExternalWatchlistRetired === true,
450
- vulnerabilityTotal: historyEntry.vulnerabilityTotal,
451
- criticalVulnerabilityCount: historyEntry.criticalVulnerabilityCount,
452
- benchmarkGatePassed: historyEntry.benchmarkGatePassed,
453
- intelligencePassed: historyEntry.intelligencePassed,
454
- reliabilityPassed: historyEntry.reliabilityPassed,
455
- reliabilityRiskLevel: historyEntry.reliabilityRiskLevel,
456
- }));
457
- }
458
-
459
- function buildChartSeries(historyEntries) {
460
- return {
461
- generatedAt: historyEntries.map((historyEntry) => historyEntry.generatedAt),
462
- top1Accuracy: historyEntries.map((historyEntry) => historyEntry.top1Accuracy),
463
- manualCorrectionRate: historyEntries.map((historyEntry) => historyEntry.manualCorrectionRate),
464
- incorrectDetectionRate: historyEntries.map((historyEntry) => historyEntry.incorrectDetectionRate),
465
- lowConfidenceRate: historyEntries.map((historyEntry) => historyEntry.lowConfidenceRate),
466
- staticExternalWatchlistRetired: historyEntries.map((historyEntry) => historyEntry.staticExternalWatchlistRetired === true),
467
- vulnerabilityTotal: historyEntries.map((historyEntry) => historyEntry.vulnerabilityTotal),
468
- };
469
- }
470
-
471
- function convertTrendTableToCsv(trendTable) {
472
- if (trendTable.length === 0) {
473
- return '';
474
- }
475
-
476
- const headers = Object.keys(trendTable[0]);
477
- const csvRows = [headers.join(',')];
478
-
479
- for (const trendRow of trendTable) {
480
- const rowValues = headers.map((header) => {
481
- const rawValue = trendRow[header];
482
- if (rawValue === null || rawValue === undefined) {
483
- return '';
484
- }
485
-
486
- const normalizedValue = String(rawValue).replace(/"/g, '""');
487
- return `"${normalizedValue}"`;
488
- });
489
-
490
- csvRows.push(rowValues.join(','));
491
- }
492
-
493
- return `${csvRows.join('\n')}\n`;
494
- }
495
-
496
- async function runBenchmarkEvidenceBundle() {
497
- const reproducibilityProfile = readJsonOrNull(REPRO_PROFILE_PATH);
498
- const thresholdConfiguration = readJsonOrNull(BENCHMARK_THRESHOLD_PATH);
499
- const memorySchemaConfiguration = readJsonOrNull(MEMORY_SCHEMA_PATH);
500
- const memoryAdapterContractConfiguration = readJsonOrNull(MEMORY_ADAPTER_CONTRACT_PATH);
501
- const releaseVersion = readReleaseVersion();
502
-
503
- const detectionBenchmarkExecution = runJsonScript('scripts/detection-benchmark.mjs');
504
- const benchmarkGateExecution = runJsonScript('scripts/benchmark-gate.mjs');
505
- const benchmarkIntelligenceExecution = runJsonScript('scripts/benchmark-intelligence.mjs');
506
- const memoryContinuityExecution = runJsonScript('scripts/memory-continuity-benchmark.mjs');
507
- const forbiddenContentExecution = runNodeScript('scripts/forbidden-content-check.mjs');
508
- const npmAuditIndicator = runNpmAuditIndicator();
509
-
510
- const executionSummaries = [
511
- summarizeExecution(detectionBenchmarkExecution),
512
- summarizeExecution(benchmarkGateExecution),
513
- summarizeExecution(benchmarkIntelligenceExecution),
514
- summarizeExecution(memoryContinuityExecution),
515
- ];
516
-
517
- const executionFailureCount = executionSummaries.filter((executionSummary) => {
518
- if (executionSummary.parseError) {
519
- return true;
520
- }
521
-
522
- if (typeof executionSummary.passed === 'boolean') {
523
- return executionSummary.passed === false;
524
- }
525
-
526
- return executionSummary.exitCode !== 0;
527
- }).length;
528
-
529
- const reliabilitySignals = buildReliabilitySignals(detectionBenchmarkExecution.parsedReport);
530
- const reliabilityFailureCount = reliabilitySignals.failureCount > 0 ? 1 : 0;
531
- const failureCount = executionFailureCount + reliabilityFailureCount;
532
- const securityIndicators = buildSecurityIndicators(forbiddenContentExecution, npmAuditIndicator);
533
- const bugIndicators = buildBugIndicators(reliabilitySignals);
534
-
535
- const generatedAt = new Date().toISOString();
536
- const currentSnapshot = buildHistorySnapshot({
537
- generatedAt,
538
- releaseVersion,
539
- detectionBenchmarkReport: detectionBenchmarkExecution.parsedReport,
540
- benchmarkGateReport: benchmarkGateExecution.parsedReport,
541
- benchmarkIntelligenceReport: benchmarkIntelligenceExecution.parsedReport,
542
- reliabilitySignals,
543
- securityIndicators,
544
- });
545
- const previousHistoryEntries = loadBenchmarkHistory();
546
- const historyEntries = mergeBenchmarkHistory(previousHistoryEntries, currentSnapshot);
547
- const releaseDelta = buildReleaseDelta(historyEntries, currentSnapshot);
548
- const trendTable = buildTrendTable(historyEntries);
549
- const chartSeries = buildChartSeries(historyEntries);
550
-
551
- const trendReport = {
552
- generatedAt,
553
- reportName: 'benchmark-trend-report',
554
- releaseVersion,
555
- historyCount: historyEntries.length,
556
- releaseDelta,
557
- trendTable,
558
- chartSeries,
559
- artifacts: {
560
- historyPath: toRelativePath(HISTORY_OUTPUT_PATH),
561
- jsonPath: toRelativePath(TREND_JSON_OUTPUT_PATH),
562
- csvPath: toRelativePath(TREND_CSV_OUTPUT_PATH),
563
- writeMode: isStdoutOnlyMode ? 'stdout-only' : 'stdout-and-file',
564
- },
565
- };
566
-
567
- const historyPayload = {
568
- generatedAt,
569
- reportName: 'benchmark-history',
570
- maxEntries: MAX_HISTORY_ENTRIES,
571
- history: historyEntries,
572
- };
573
-
574
- const trendCsvPayload = convertTrendTableToCsv(trendTable);
575
- const baseRerunInstructions = Array.isArray(reproducibilityProfile?.rerunInstructions)
576
- ? reproducibilityProfile.rerunInstructions
577
- : [];
578
- const baseCommandExamples = Array.isArray(reproducibilityProfile?.commandExamples)
579
- ? reproducibilityProfile.commandExamples
580
- : [];
581
- const rerunInstructions = appendUniqueTextValues(baseRerunInstructions, [
582
- 'Run npm run benchmark:continuity to validate cross-agent memory hydration, privacy redaction, and token-savings behavior.',
583
- ]);
584
- const commandExamples = appendUniqueTextValues(baseCommandExamples, [
585
- 'npm run benchmark:continuity',
586
- 'node ./scripts/memory-continuity-benchmark.mjs --stdout-only',
587
- ]);
588
-
589
- const evidenceBundleReport = {
590
- generatedAt,
591
- reportName: 'benchmark-evidence-bundle',
592
- phase: 'v2.5.2',
593
- releaseVersion,
594
- passed: failureCount === 0,
595
- failureCount,
596
- methodology: {
597
- deterministicRuntime: reproducibilityProfile?.deterministicRuntime || null,
598
- scenarioCount: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios.length : 0,
599
- commandCount: commandExamples.length,
600
- },
601
- rerunInstructions,
602
- commandExamples,
603
- rawInputs: {
604
- scenarios: Array.isArray(reproducibilityProfile?.scenarios) ? reproducibilityProfile.scenarios : [],
605
- benchmarkThresholds: thresholdConfiguration,
606
- benchmarkWatchlist: {
607
- retired: true,
608
- reason: 'Static external benchmark watchlists were removed to avoid stale or biasing research inputs.',
609
- },
610
- memorySchema: memorySchemaConfiguration,
611
- memoryAdapterContract: memoryAdapterContractConfiguration,
612
- },
613
- rubric: buildRubricSummary(
614
- thresholdConfiguration,
615
- benchmarkIntelligenceExecution.parsedReport,
616
- memoryContinuityExecution.parsedReport
617
- ),
618
- bugIndicators,
619
- reliabilitySignals,
620
- securityIndicators,
621
- releaseDelta,
622
- history: historyEntries,
623
- trendReport,
624
- outputs: {
625
- detectionBenchmark: detectionBenchmarkExecution.parsedReport,
626
- benchmarkGate: benchmarkGateExecution.parsedReport,
627
- benchmarkIntelligence: benchmarkIntelligenceExecution.parsedReport,
628
- memoryContinuityBenchmark: memoryContinuityExecution.parsedReport,
629
- },
630
- executions: executionSummaries,
631
- };
632
-
633
- if (!isStdoutOnlyMode) {
634
- await fs.mkdir(join(REPOSITORY_ROOT, '.agent-context', 'state'), { recursive: true });
635
- await fs.writeFile(OUTPUT_PATH, JSON.stringify(evidenceBundleReport, null, 2) + '\n', 'utf8');
636
- await fs.writeFile(HISTORY_OUTPUT_PATH, JSON.stringify(historyPayload, null, 2) + '\n', 'utf8');
637
- await fs.writeFile(TREND_JSON_OUTPUT_PATH, JSON.stringify(trendReport, null, 2) + '\n', 'utf8');
638
- await fs.writeFile(TREND_CSV_OUTPUT_PATH, trendCsvPayload, 'utf8');
639
- }
640
-
641
- console.log(JSON.stringify(evidenceBundleReport, null, 2));
642
- process.exit(evidenceBundleReport.passed ? 0 : 1);
643
- }
644
-
645
- runBenchmarkEvidenceBundle();