@dailephd/my-dev-kit-lab 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -12
- package/benchmarks/contracts/todo-behavior.md +1 -1
- package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
- package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
- package/dist/scripts/audits/runAudit.js +28 -1
- package/dist/src/audits/core/auditConfig.js +16 -1
- package/dist/src/audits/core/auditRunner.js +6 -2
- package/dist/src/audits/report/auditReportModel.js +7 -0
- package/dist/src/audits/report/renderAuditJsonReport.js +1 -0
- package/dist/src/audits/report/renderAuditTextReport.js +37 -0
- package/dist/src/audits/security/androidAuditIntegration.js +145 -0
- package/dist/src/audits/security/mapAndroidSecurityFindingToAuditIssue.js +116 -0
- package/dist/src/audits/security/mapSecurityFindingToAuditIssue.js +15 -5
- package/dist/src/audits/security/securityAuditAdapter.js +20 -2
- package/dist/src/audits/security/securityAuditTypes.js +37 -0
- package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
- package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
- package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
- package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
- package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
- package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/validation.js +568 -0
- package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
- package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
- package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
- package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
- package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
- package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
- package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
- package/dist/src/evaluation/stageContextMetrics/index.js +10 -0
- package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
- package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
- package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +79 -0
- package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
- package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
- package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
- package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
- package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
- package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
- package/dist/src/evaluation/targetImmutability/index.js +3 -0
- package/dist/src/evaluation/targetImmutability/types.js +1 -0
- package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
- package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/index.js +3 -0
- package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
- package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
- package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +105 -0
- package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +286 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
- package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +190 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +252 -0
- package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +398 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
- package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +234 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +225 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
- package/docs/METRICS.md +56 -10
- package/package.json +1 -1
- package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
- package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
- package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
- package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
- package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
- package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
- package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
- package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
- package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
package/README.md
CHANGED
|
@@ -1,27 +1,210 @@
|
|
|
1
1
|
# my-dev-kit-lab
|
|
2
2
|
|
|
3
|
-
my-dev-kit-lab is the experiment,
|
|
3
|
+
my-dev-kit-lab is the experiment, audit, and evidence companion for [my-dev-kit](https://www.npmjs.com/package/@dailephd/my-dev-kit). It helps users compare repository-context strategies, audit project health, validate CLI/package and Android security boundaries, and turn each run into reviewable reports and visual artifacts.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
my-dev-kit provides local repository indexing and graph-guided retrieval. my-dev-kit-lab supplies the controlled benchmarks, agent adapters, metrics, security checks, and reports needed to evaluate when that retrieval is useful. Results are evidence for a specific target and configuration; they do not guarantee token savings or security.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
The latest published release is v0.4.3 (stage-specific bounded-context and workflow-instruction evaluation). See [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md).
|
|
8
8
|
|
|
9
|
-
##
|
|
9
|
+
## Current capabilities
|
|
10
|
+
|
|
11
|
+
- **Run context-strategy experiments:** compare `raw-full-file` with `my-dev-kit-guided` using deterministic fixtures or locally configured Codex and Claude CLIs.
|
|
12
|
+
- **Audit repository health:** run conservative code-rot detectors for TypeScript/JavaScript, Python, Java, and Kotlin, or adapt security findings into the common audit report.
|
|
13
|
+
- **Validate CLI/package security:** inspect dependencies, package contents, path and subprocess boundaries, malformed inputs, optional static scanners, and bounded fuzz targets.
|
|
14
|
+
- **Validate Android projects:** run nineteen static checks by default, with Gradle operations, external tools, and network access available only through explicit opt-in flags.
|
|
15
|
+
- **Review evidence:** generate JSON and HTML reports, SVG plots, optional screenshots, visualization demos, and a static gallery.
|
|
16
|
+
- **Evaluate stage-context strategies:** compare the two legacy strategies against six additional bounded stage-context strategies — `architecture-context-only`, `architecture-plus-implementation-refresh`, `architecture-plus-implementation-and-test-refresh`, `full-workflow-library`, `bounded-workflow-instruction-packet`, and `combined-bounded-stage-context` — selected through programmatic configuration, not CLI flags. Each strategy's evidence is reported through bounded `report.json`, `report.html`, and `report.txt` output with an explicit `available`/`unavailable`/`not-applicable` metric-availability model and no composite score, grade, ranking, or winning strategy.
|
|
17
|
+
|
|
18
|
+
## Quickstart
|
|
19
|
+
|
|
20
|
+
### Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
npm install
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The same command works in PowerShell and `cmd.exe`.
|
|
27
|
+
|
|
28
|
+
### Build
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
npm run build
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Verify the installation
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
npm run verify
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Run the fake-agent final demo (deterministic, no external CLIs required)
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
npm run run-final-demo -- \
|
|
44
|
+
--cases examples/token-savings-cases.json \
|
|
45
|
+
--out lab-output/final-demo \
|
|
46
|
+
--kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" \
|
|
47
|
+
--agents fake-agent \
|
|
48
|
+
--complexities short \
|
|
49
|
+
--no-screenshot
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
```powershell
|
|
53
|
+
npm run run-final-demo -- `
|
|
54
|
+
--cases examples/token-savings-cases.json `
|
|
55
|
+
--out lab-output/final-demo `
|
|
56
|
+
--kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" `
|
|
57
|
+
--agents fake-agent `
|
|
58
|
+
--complexities short `
|
|
59
|
+
--no-screenshot
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
```bat
|
|
63
|
+
npm run run-final-demo -- --cases examples/token-savings-cases.json --out lab-output/final-demo --kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" --agents fake-agent --complexities short --no-screenshot
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The lab resolves Windows `.cmd` and `.ps1` CLI shims, supports command paths with spaces, and keeps generated artifacts inside the requested output directory.
|
|
67
|
+
|
|
68
|
+
This runs a full pipeline: controlled experiment → report → plots → visualization demos → gallery.
|
|
69
|
+
|
|
70
|
+
### Run a real-agent campaign (requires Codex or Claude CLI)
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
npm run run-controlled-experiment -- \
|
|
74
|
+
--cases examples/real-agent-campaign-cases.json \
|
|
75
|
+
--agents codex,claude \
|
|
76
|
+
--strategies raw-full-file,my-dev-kit-guided \
|
|
77
|
+
--complexities medium,multi-step \
|
|
78
|
+
--out lab-output/real-agent-campaign \
|
|
79
|
+
--include-real-agents \
|
|
80
|
+
--continue-on-failure \
|
|
81
|
+
--timeout-ms 240000
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Real-agent runs require local Codex or Claude CLI setup and available usage capacity. Runs that time out, produce invalid output, or hit session limits are recorded as structured outcomes rather than failures.
|
|
85
|
+
|
|
86
|
+
### List, describe, and run experiment plugins
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
npm run experiment:list
|
|
90
|
+
npm run experiment:describe -- --experiment context-strategy-comparison
|
|
91
|
+
npm run experiment:run -- \
|
|
92
|
+
--experiment context-strategy-comparison \
|
|
93
|
+
--target /path/to/local/project \
|
|
94
|
+
--agents fake-agent \
|
|
95
|
+
--complexities short \
|
|
96
|
+
--no-screenshot
|
|
97
|
+
```
|
|
10
98
|
|
|
11
99
|
```powershell
|
|
12
|
-
npm run
|
|
100
|
+
npm run experiment:list
|
|
101
|
+
npm run experiment:describe -- --experiment context-strategy-comparison
|
|
102
|
+
npm run experiment:run -- `
|
|
103
|
+
--experiment context-strategy-comparison `
|
|
104
|
+
--target "Z:\Users\newuser\Projects\my-dev-kit-v1" `
|
|
105
|
+
--agents fake-agent `
|
|
106
|
+
--complexities short `
|
|
107
|
+
--no-screenshot
|
|
13
108
|
```
|
|
14
109
|
|
|
15
|
-
|
|
110
|
+
When `--target` is omitted, the experiment runs in self mode against my-dev-kit-lab. When `--target <path>` is provided, the lab remains the tool root and the target project is inspected separately. Generated experiment outputs stay under lab-controlled output directories by default, not inside the target project.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## Where to find outputs
|
|
115
|
+
|
|
116
|
+
| Artifact | Location |
|
|
117
|
+
|---|---|
|
|
118
|
+
| Experiment summary | `lab-output/<experiment>/experiment-summary.json` |
|
|
119
|
+
| All runs | `lab-output/<experiment>/experiment-runs.json` |
|
|
120
|
+
| Strategy comparisons | `lab-output/<experiment>/experiment-comparisons.json` |
|
|
121
|
+
| HTML report | `lab-output/<report>/experiment-report.html` |
|
|
122
|
+
| Report JSON | `lab-output/<report>/experiment-report.json` |
|
|
123
|
+
| Report screenshot | `lab-output/<report>/experiment-report.png` |
|
|
124
|
+
| Plugin experiment report JSON | `lab-output/experiments/<plugin>/<target>/<run>/report.json` |
|
|
125
|
+
| Plugin experiment report HTML | `lab-output/experiments/<plugin>/<target>/<run>/report.html` |
|
|
126
|
+
| Plugin experiment report text | `lab-output/experiments/<plugin>/<target>/<run>/report.txt` |
|
|
127
|
+
| Plot data | `lab-output/<plots>/plot-data.json` |
|
|
128
|
+
| SVG charts | `lab-output/<plots>/charts/*.svg` |
|
|
129
|
+
| Gallery manifest | `lab-output/<gallery>/gallery-manifest.json` |
|
|
130
|
+
| Gallery index | `lab-output/<gallery>/gallery-index.html` |
|
|
131
|
+
| Audit reports | `reports/audits/<type>/code-rot-audit.txt` and `.json` |
|
|
132
|
+
| Security reports | `reports/security/<prefix>-security-validation.txt` and `.json` |
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## How to read the main report
|
|
137
|
+
|
|
138
|
+
Open `experiment-report.html` in a browser. The report shows:
|
|
139
|
+
|
|
140
|
+
- **Project profile** — benchmark project name, language mix, complexity score, and file tree
|
|
141
|
+
- **Benchmark tasks** — task descriptions and answer keys
|
|
142
|
+
- **Strategy comparisons** — paired `raw-full-file` vs `my-dev-kit-guided` runs per case
|
|
143
|
+
- **Correctness scores** — deterministic answer-key scoring (not semantic LLM judging)
|
|
144
|
+
- **Token usage** — estimated or reported token totals per run
|
|
145
|
+
- **Token savings** — positive means my-dev-kit used fewer tokens; negative means it used more
|
|
146
|
+
- **Duration** — wall-clock time per run
|
|
147
|
+
- **Status** — completed, timeout, invalid-output, or limit-reached
|
|
148
|
+
- **Warnings and limitations** — notes on missing token totals or partial results
|
|
149
|
+
|
|
150
|
+
See [docs/METRICS.md](docs/METRICS.md) for full metric definitions.
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
## Current limitations
|
|
155
|
+
|
|
156
|
+
- Token savings shown in fake-agent runs are based on estimated character counts, not provider billing telemetry
|
|
157
|
+
- Claude does not expose token totals; token savings comparisons are unavailable for Claude runs
|
|
158
|
+
- Codex may expose token totals but can produce timeouts or invalid-output runs
|
|
159
|
+
- Small projects may make raw-full-file cheaper than my-dev-kit-guided; larger localized tasks are where my-dev-kit is expected to become more useful
|
|
160
|
+
- The generic experiment-plugin framework currently ships one plugin, `context-strategy-comparison`; future plugins such as warm-index reuse, incremental-change, and context-window scaling are not implemented yet
|
|
161
|
+
- The current release does not guarantee token savings; it produces auditable evidence for specific cases, targets, agents, and strategies
|
|
162
|
+
- Provider telemetry dashboards, semantic LLM judging, and cloud API billing integration are not yet implemented
|
|
163
|
+
- The six new stage-context strategies have no CLI flags yet, are configured programmatically, and do not yet include plots, screenshots, or gallery integration
|
|
164
|
+
- The published upstream artifacts the stage-context strategies read do not expose considered-but-unselected reads or unnecessary-read evidence; those metrics report `unavailable` rather than zero
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## Security validation
|
|
169
|
+
|
|
170
|
+
`npm run security:validate` is the standalone security-validation command. It checks local CLI/package boundaries and can inspect another local project with `--target <path>`. The generic audit command can reuse those results through `--types security`, but it does not replace the standalone validator or its reports.
|
|
171
|
+
|
|
172
|
+
Android validation uses `--profile android`. Its default path is static and non-destructive: it starts zero Gradle processes, zero external tools, and zero network operations. Only confirmed `SecurityFinding` records can become audit issues; Android `CandidateEvidence` remains review-only evidence.
|
|
173
|
+
|
|
174
|
+
Optional scanners are reported as `skipped` when unavailable, never as passed. The framework does not provide runtime isolation proof, device or APK/AAB analysis, signing verification, Play Console validation, automatic fixes, or manual pentesting. Manual pentest remains deferred until post-v1/version TBD.
|
|
175
|
+
|
|
176
|
+
See [COMMANDS.md](docs/COMMANDS.md) for exact syntax and [Security Validation Framework](docs/security-validation-framework.md) for checks, evidence semantics, verdicts, and limitations.
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## License
|
|
181
|
+
|
|
182
|
+
MIT License. See [LICENSE](LICENSE) for the full text.
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Support
|
|
187
|
+
|
|
188
|
+
my-dev-kit-lab is an independent project by dailephd LLC, developed and maintained by Dai Le.
|
|
16
189
|
|
|
17
|
-
|
|
190
|
+
If this project helps your workflow, you can support continued development through GitHub Sponsors or PayPal:
|
|
18
191
|
|
|
19
|
-
|
|
192
|
+
- [Sponsor on GitHub](https://github.com/sponsors/dailephd)
|
|
193
|
+
- [Support via PayPal](https://paypal.me/daile88)
|
|
20
194
|
|
|
21
|
-
|
|
195
|
+
Support is optional and does not affect access to the project.
|
|
22
196
|
|
|
23
|
-
|
|
197
|
+
---
|
|
24
198
|
|
|
25
|
-
|
|
199
|
+
## Documentation
|
|
26
200
|
|
|
27
|
-
|
|
201
|
+
- [docs/PROJECT_OVERVIEW.md](docs/PROJECT_OVERVIEW.md) — product purpose and target users
|
|
202
|
+
- [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md) — implemented, planned, validated, blocked, and next state
|
|
203
|
+
- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — current components, ownership, flows, and invariants
|
|
204
|
+
- [docs/WORKFLOWS.md](docs/WORKFLOWS.md) — step-by-step workflows with diagrams
|
|
205
|
+
- [docs/COMMANDS.md](docs/COMMANDS.md) — all commands with options and examples
|
|
206
|
+
- [docs/TUTORIAL.md](docs/TUTORIAL.md) — first-run walkthrough
|
|
207
|
+
- [docs/METRICS.md](docs/METRICS.md) — metric definitions and interpretation
|
|
208
|
+
- [docs/ROADMAP.md](docs/ROADMAP.md) — versioned plans, dependencies, exclusions, and acceptance criteria
|
|
209
|
+
- [docs/GALLERY.md](docs/GALLERY.md) — gallery output explained
|
|
210
|
+
- [docs/security-validation-framework.md](docs/security-validation-framework.md) — security evidence, verdicts, and safety boundaries
|
|
@@ -66,5 +66,5 @@ Answer key requirements:
|
|
|
66
66
|
- `answerKey.expectedFiles` lists files a correct answer may need to cite
|
|
67
67
|
- `answerKey.expectedSymbols` lists relevant symbols
|
|
68
68
|
- `answerKey.expectedFacts` lists weighted facts with unique `id`, `text`, `weight`, and `required`
|
|
69
|
-
- `answerKey.minimumCorrectFacts` defines the
|
|
69
|
+
- `answerKey.minimumCorrectFacts` defines the required-fact threshold used by deterministic correctness scoring
|
|
70
70
|
- optional `forbiddenWrongClaims` records claims that should be penalized by a later correctness scorer
|
|
@@ -1 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
# task-workflow-medium-ts
|
|
2
|
+
|
|
3
|
+
TypeScript task-workflow benchmark with multiple services, validation rules, filtering, summarization, and deterministic imports.
|
|
@@ -20,9 +20,29 @@ import { writeAuditReports } from "../../src/audits/report/writeAuditReports.js"
|
|
|
20
20
|
// ---------------------------------------------------------------------------
|
|
21
21
|
const USAGE = "Usage: npm run audit -- [--target <path>] [--types code-rot,security] " +
|
|
22
22
|
"[--include docs,tests,package,architecture,cli] [--format text,json] " +
|
|
23
|
-
"[--fail-on blocker|high|medium|low|none] [--out <path>]"
|
|
23
|
+
"[--fail-on blocker|high|medium|low|none] [--out <path>] [--android]\n" +
|
|
24
|
+
"\n" +
|
|
25
|
+
" --android opts into Batch 2's programmatic Android security integration.\n" +
|
|
26
|
+
" Requires --types to include \"security\" (e.g. --types security or --types\n" +
|
|
27
|
+
" code-rot,security). Runs the same static, read-only, nineteen-check\n" +
|
|
28
|
+
" Android validation security:validate --profile android performs --\n" +
|
|
29
|
+
" detection, manifest parsing, and internal advanced security checks --\n" +
|
|
30
|
+
" through the existing security adapter, never a subprocess. Confirmed\n" +
|
|
31
|
+
" findings are mapped into the normal issue collection; CandidateEvidence\n" +
|
|
32
|
+
" remains a separate, bounded summary and is never treated as a confirmed\n" +
|
|
33
|
+
" issue. Starts zero Gradle operations, zero external tools, and zero\n" +
|
|
34
|
+
" network operations by default; --android does not expose any of those.\n" +
|
|
35
|
+
" Example: npm run audit -- --target \"<android-project-path>\" --types security --android --format text,json --fail-on none";
|
|
24
36
|
const rawArgs = process.argv.slice(2);
|
|
25
37
|
const toolRoot = resolveToolRoot(import.meta.url);
|
|
38
|
+
// Help deliberately wins over every other argument, mirroring
|
|
39
|
+
// scripts/security/validate.ts -- no config normalization, target
|
|
40
|
+
// resolution, audit execution, or report write.
|
|
41
|
+
if (rawArgs.includes("--help") || rawArgs.includes("-h")) {
|
|
42
|
+
console.log(USAGE);
|
|
43
|
+
process.exitCode = 0;
|
|
44
|
+
process.exit(0);
|
|
45
|
+
}
|
|
26
46
|
let config;
|
|
27
47
|
try {
|
|
28
48
|
const args = parseAuditArgs(rawArgs);
|
|
@@ -89,6 +109,13 @@ if (model.securitySummary.ran) {
|
|
|
89
109
|
console.log(` Full report: ${model.securitySummary.reportPaths.text}`);
|
|
90
110
|
}
|
|
91
111
|
}
|
|
112
|
+
if (model.androidSecurity.summary.requested) {
|
|
113
|
+
const android = model.androidSecurity.summary;
|
|
114
|
+
console.log(`\nAndroid security validation: status=${android.status} applicable=${android.applicable ?? "n/a"} verdict=${android.verdict ?? "(unknown)"} checks=${android.totalChecks} confirmed=${android.confirmedFindingCount} mappedIssues=${android.mappedIssueCount} candidates=${android.candidateSummary.totalCount}`);
|
|
115
|
+
if (android.reportPaths.text) {
|
|
116
|
+
console.log(` Full report: ${android.reportPaths.text}`);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
92
119
|
console.log(`\nInventory: ${model.inventory.totalScannedFileCount} file(s) scanned, ${model.inventory.skippedFileCount} skipped`);
|
|
93
120
|
console.log(` source=${model.inventory.filesByCategory.source} tests=${model.inventory.filesByCategory.tests} docs=${model.inventory.filesByCategory.docs} package=${model.inventory.filesByCategory.package} config=${model.inventory.filesByCategory.config} scripts=${model.inventory.filesByCategory.scripts} ci=${model.inventory.filesByCategory.ci}`);
|
|
94
121
|
console.log(`Source of truth: package=${model.sourceOfTruth.packageName ?? "(none)"}@${model.sourceOfTruth.packageVersion ?? "?"} readme=${model.sourceOfTruth.hasReadme} changelog=${model.sourceOfTruth.hasChangelog} ciWorkflows=${model.sourceOfTruth.ciWorkflowCount}`);
|
|
@@ -8,10 +8,19 @@ const FLAGS_WITH_VALUE = {
|
|
|
8
8
|
"--fail-on": "failOn",
|
|
9
9
|
"--out": "out",
|
|
10
10
|
};
|
|
11
|
+
// v0.4.2 Batch 3 -- presence-only flags (no following value consumed).
|
|
12
|
+
const BOOLEAN_FLAGS = {
|
|
13
|
+
"--android": "android",
|
|
14
|
+
};
|
|
11
15
|
export function parseAuditArgs(argv) {
|
|
12
16
|
const result = {};
|
|
13
17
|
for (let i = 0; i < argv.length; i++) {
|
|
14
18
|
const arg = argv[i];
|
|
19
|
+
const booleanKey = BOOLEAN_FLAGS[arg];
|
|
20
|
+
if (booleanKey) {
|
|
21
|
+
result[booleanKey] = true;
|
|
22
|
+
continue;
|
|
23
|
+
}
|
|
15
24
|
const key = FLAGS_WITH_VALUE[arg];
|
|
16
25
|
if (!key)
|
|
17
26
|
continue;
|
|
@@ -28,6 +37,10 @@ export function normalizeAuditConfig(raw, toolRoot) {
|
|
|
28
37
|
const formats = parseFormatOption(raw.format);
|
|
29
38
|
const failOn = parseFailOnOption(raw.failOn);
|
|
30
39
|
const out = raw.out !== undefined ? resolveOutDir(raw.out) : defaultOutDir(toolRoot, types.value);
|
|
40
|
+
const android = raw.android === true;
|
|
41
|
+
if (android && !types.value.includes("security")) {
|
|
42
|
+
throw new Error(`Invalid combination: --android requires --types to include "security" (e.g. --types security or --types code-rot,security).`);
|
|
43
|
+
}
|
|
31
44
|
return {
|
|
32
45
|
targetPathArg: raw.target,
|
|
33
46
|
targetMode: raw.target === undefined ? "self" : "external",
|
|
@@ -46,7 +59,9 @@ export function normalizeAuditConfig(raw, toolRoot) {
|
|
|
46
59
|
raw.include === undefined &&
|
|
47
60
|
raw.format === undefined &&
|
|
48
61
|
raw.failOn === undefined &&
|
|
49
|
-
raw.out === undefined
|
|
62
|
+
raw.out === undefined &&
|
|
63
|
+
raw.android === undefined,
|
|
64
|
+
android,
|
|
50
65
|
};
|
|
51
66
|
}
|
|
52
67
|
function splitAndNormalizeList(raw, flagName) {
|
|
@@ -6,7 +6,7 @@ import { collectSourceOfTruth } from "./sourceOfTruth.js";
|
|
|
6
6
|
import { collectSourceFacts } from "./collectSourceFacts.js";
|
|
7
7
|
import { collectPythonProjectMetadata } from "./pythonProjectMetadata.js";
|
|
8
8
|
import { runSecurityAuditAdapter } from "../security/securityAuditAdapter.js";
|
|
9
|
-
import { SECURITY_AUDIT_NOT_RUN_SUMMARY } from "../security/securityAuditTypes.js";
|
|
9
|
+
import { ANDROID_AUDIT_NOT_REQUESTED_SUMMARY, SECURITY_AUDIT_NOT_RUN_SUMMARY, } from "../security/securityAuditTypes.js";
|
|
10
10
|
export async function runAudit(options) {
|
|
11
11
|
const { config, toolRoot } = options;
|
|
12
12
|
const registry = options.registry ?? DEFAULT_AUDIT_REGISTRY;
|
|
@@ -49,10 +49,12 @@ export async function runAudit(options) {
|
|
|
49
49
|
// code-rot detector issues (registry order) first, then security findings
|
|
50
50
|
// (security-validation check-execution order).
|
|
51
51
|
let securitySummary = SECURITY_AUDIT_NOT_RUN_SUMMARY;
|
|
52
|
+
let androidSummary = ANDROID_AUDIT_NOT_REQUESTED_SUMMARY;
|
|
52
53
|
if (config.types.includes("security")) {
|
|
53
|
-
const securityAudit = await runSecurityAuditAdapter({ toolRoot, config });
|
|
54
|
+
const securityAudit = await runSecurityAuditAdapter({ toolRoot, config, android: config.android ? { enabled: true } : undefined }, options.securityDependencies);
|
|
54
55
|
issues.push(...securityAudit.issues);
|
|
55
56
|
securitySummary = securityAudit.summary;
|
|
57
|
+
androidSummary = securityAudit.android;
|
|
56
58
|
}
|
|
57
59
|
const issueCounts = countIssuesBySeverity(issues);
|
|
58
60
|
const { exitCode, reason } = calculateAuditExitCode(issues, config.failOn);
|
|
@@ -64,6 +66,7 @@ export async function runAudit(options) {
|
|
|
64
66
|
failOn: config.failOn,
|
|
65
67
|
out: config.out,
|
|
66
68
|
isDefaultRun: config.isDefaultRun,
|
|
69
|
+
android: config.android,
|
|
67
70
|
},
|
|
68
71
|
targetSummary: {
|
|
69
72
|
rootPath: target.rootPath,
|
|
@@ -81,6 +84,7 @@ export async function runAudit(options) {
|
|
|
81
84
|
sourceFacts,
|
|
82
85
|
pythonProjectMetadata,
|
|
83
86
|
securitySummary,
|
|
87
|
+
androidSummary,
|
|
84
88
|
noDetectorsRegistered: selected.length === 0,
|
|
85
89
|
};
|
|
86
90
|
}
|
|
@@ -166,6 +166,7 @@ export function buildAuditReportModel(result, opts) {
|
|
|
166
166
|
failOn: result.configSummary.failOn,
|
|
167
167
|
out: result.configSummary.out,
|
|
168
168
|
isDefaultRun: result.configSummary.isDefaultRun,
|
|
169
|
+
android: result.configSummary.android,
|
|
169
170
|
},
|
|
170
171
|
summary: {
|
|
171
172
|
totalIssues: issues.length,
|
|
@@ -191,6 +192,10 @@ export function buildAuditReportModel(result, opts) {
|
|
|
191
192
|
sourceFacts: summarizeSourceFacts(result.sourceFacts),
|
|
192
193
|
pythonProjectMetadata: result.pythonProjectMetadata,
|
|
193
194
|
securitySummary: result.securitySummary,
|
|
195
|
+
androidSecurity: {
|
|
196
|
+
summary: result.androidSummary,
|
|
197
|
+
candidates: result.androidSummary.candidateSummary,
|
|
198
|
+
},
|
|
194
199
|
detectors,
|
|
195
200
|
issues,
|
|
196
201
|
skippedDetectors: result.skippedDetectors,
|
|
@@ -252,5 +257,7 @@ function reconstructCommand(result) {
|
|
|
252
257
|
parts.push("--format", cfg.formats.join(","));
|
|
253
258
|
parts.push("--fail-on", cfg.failOn);
|
|
254
259
|
parts.push("--out", cfg.out);
|
|
260
|
+
if (cfg.android)
|
|
261
|
+
parts.push("--android");
|
|
255
262
|
return parts.join(" ");
|
|
256
263
|
}
|
|
@@ -21,6 +21,7 @@ export function renderAuditJsonReport(model) {
|
|
|
21
21
|
sourceFacts: model.sourceFacts,
|
|
22
22
|
pythonProjectMetadata: model.pythonProjectMetadata,
|
|
23
23
|
securitySummary: model.securitySummary,
|
|
24
|
+
androidSecurity: model.androidSecurity,
|
|
24
25
|
detectors: model.detectors,
|
|
25
26
|
issues: model.issues,
|
|
26
27
|
skippedDetectors: model.skippedDetectors,
|
|
@@ -133,6 +133,43 @@ export function renderAuditTextReport(model) {
|
|
|
133
133
|
}
|
|
134
134
|
}
|
|
135
135
|
lines.push("");
|
|
136
|
+
// v0.4.2 Batch 3 -- bounded summary only, mirroring the "Security
|
|
137
|
+
// validation summary" section above exactly. Confirmed mapped Android
|
|
138
|
+
// findings are NOT re-listed here -- they already appear in the "Issues"
|
|
139
|
+
// section below like any other AuditIssue (see spec section 12: "the
|
|
140
|
+
// existing generic issues section remains the detailed presentation for
|
|
141
|
+
// confirmed mapped Android findings"). CandidateEvidence bodies never
|
|
142
|
+
// appear here either -- only counts, with a pointer to the full Android
|
|
143
|
+
// report for detail.
|
|
144
|
+
lines.push("Android security validation summary");
|
|
145
|
+
lines.push(divider("-"));
|
|
146
|
+
if (!model.androidSecurity.summary.requested) {
|
|
147
|
+
lines.push(" (not requested -- add --android alongside --types security to include it)");
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
const android = model.androidSecurity.summary;
|
|
151
|
+
const candidates = model.androidSecurity.candidates;
|
|
152
|
+
const applicabilityNote = android.applicable === false ? " (target is not an Android project)" : "";
|
|
153
|
+
lines.push(` Status: ${android.status}${applicabilityNote}`);
|
|
154
|
+
lines.push(` Verdict: ${android.verdict ?? "(unknown)"}`);
|
|
155
|
+
lines.push(` Checks: total=${android.totalChecks} passed=${android.passedChecks} withFindings=${android.checksWithFindings} candidateOnly=${android.candidateOnlyChecks} skipped=${android.skippedChecks} inconclusive=${android.inconclusiveChecks} failed=${android.failedChecks}`);
|
|
156
|
+
lines.push(` Findings: confirmed=${android.confirmedFindingCount} mappedIssues=${android.mappedIssueCount}`);
|
|
157
|
+
lines.push(` Candidates (review evidence, not confirmed vulnerabilities): total=${candidates.totalCount} checksWithCandidates=${candidates.checksWithCandidates}`);
|
|
158
|
+
if (android.warnings.length > 0) {
|
|
159
|
+
lines.push(` Warnings: ${android.warnings.length}`);
|
|
160
|
+
}
|
|
161
|
+
if (android.errors.length > 0) {
|
|
162
|
+
lines.push(` Errors: ${android.errors.length}`);
|
|
163
|
+
for (const e of android.errors.slice(0, 5)) {
|
|
164
|
+
lines.push(` ${sanitizeLine(e)}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (android.reportPaths.text || android.reportPaths.json) {
|
|
168
|
+
lines.push(` Full report: ${android.reportPaths.text ?? "(none)"} / ${android.reportPaths.json ?? "(none)"}`);
|
|
169
|
+
}
|
|
170
|
+
lines.push(" Static analysis only -- does not prove runtime behavior, APK/AAB validity, Play compliance, or pentest results.");
|
|
171
|
+
}
|
|
172
|
+
lines.push("");
|
|
136
173
|
lines.push("Issue summary");
|
|
137
174
|
lines.push(divider("-"));
|
|
138
175
|
lines.push(` Total issues: ${model.summary.totalIssues}`);
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { validateAndroidTarget } from "../../mobile/android/validate/validateAndroidTarget.js";
|
|
3
|
+
import { toAndroidReportModel } from "../../mobile/android/report/model.js";
|
|
4
|
+
import { writeAndroidReportFiles } from "../../mobile/android/report/writeAndroidReportFiles.js";
|
|
5
|
+
import { reportFilenamePrefix } from "../../securityValidation/validate/resolveTarget.js";
|
|
6
|
+
import { mapAndroidSecurityFindingToAuditIssue } from "./mapAndroidSecurityFindingToAuditIssue.js";
|
|
7
|
+
import { ANDROID_AUDIT_NOT_REQUESTED_SUMMARY, EMPTY_ANDROID_AUDIT_CANDIDATE_SUMMARY, } from "./securityAuditTypes.js";
|
|
8
|
+
const ANDROID_REPORT_SUBDIR = "android";
|
|
9
|
+
// Bounded, redacted — never the thrown value's stack, never raw child-process
|
|
10
|
+
// output (there is none at this boundary; validateAndroidTarget itself never
|
|
11
|
+
// spawns a process unless Gradle/external-tool options are requested, which
|
|
12
|
+
// Batch 2 never supplies).
|
|
13
|
+
const MAX_ANDROID_ERROR_MESSAGE_LENGTH = 500;
|
|
14
|
+
function boundErrorMessage(message) {
|
|
15
|
+
return message.length <= MAX_ANDROID_ERROR_MESSAGE_LENGTH
|
|
16
|
+
? message
|
|
17
|
+
: `${message.slice(0, MAX_ANDROID_ERROR_MESSAGE_LENGTH)}... [truncated, ${message.length} chars total]`;
|
|
18
|
+
}
|
|
19
|
+
function failedSummary(errorMessage) {
|
|
20
|
+
return {
|
|
21
|
+
...ANDROID_AUDIT_NOT_REQUESTED_SUMMARY,
|
|
22
|
+
requested: true,
|
|
23
|
+
applicable: null,
|
|
24
|
+
status: "failed",
|
|
25
|
+
errors: [boundErrorMessage(errorMessage)],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
// Exact-duplicate dedupe by AuditIssue id only, first occurrence wins —
|
|
29
|
+
// mirrors validateAndroidTarget's own findingsById convention. Iterates
|
|
30
|
+
// checks in their own fixed order (never re-sorted) and each check's own
|
|
31
|
+
// findings in their own order — never the top-level, already-flattened
|
|
32
|
+
// result.findings, which has lost per-check identity this mapper needs as
|
|
33
|
+
// context. Exported standalone so aggregation correctness (ordering,
|
|
34
|
+
// dedup, provenance) can be tested against small literal AndroidCheckResult
|
|
35
|
+
// fixtures without needing a full AndroidValidationResult.
|
|
36
|
+
export function aggregateAndroidIssues(checks, reportReference) {
|
|
37
|
+
const issuesById = new Map();
|
|
38
|
+
for (const check of checks) {
|
|
39
|
+
for (const finding of check.findings) {
|
|
40
|
+
const issue = mapAndroidSecurityFindingToAuditIssue(finding, {
|
|
41
|
+
checkId: check.id,
|
|
42
|
+
checkCategory: check.category,
|
|
43
|
+
checkConfidence: check.confidence,
|
|
44
|
+
reportReference,
|
|
45
|
+
});
|
|
46
|
+
if (!issuesById.has(issue.id)) {
|
|
47
|
+
issuesById.set(issue.id, issue);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return [...issuesById.values()];
|
|
52
|
+
}
|
|
53
|
+
export function buildCandidateSummary(checks) {
|
|
54
|
+
const byConfidence = { ...EMPTY_ANDROID_AUDIT_CANDIDATE_SUMMARY.byConfidence };
|
|
55
|
+
const byResolutionState = { ...EMPTY_ANDROID_AUDIT_CANDIDATE_SUMMARY.byResolutionState };
|
|
56
|
+
const byCategory = {};
|
|
57
|
+
let totalCount = 0;
|
|
58
|
+
let checksWithCandidates = 0;
|
|
59
|
+
for (const check of checks) {
|
|
60
|
+
const candidates = check.candidateEvidence ?? [];
|
|
61
|
+
if (candidates.length === 0)
|
|
62
|
+
continue;
|
|
63
|
+
checksWithCandidates += 1;
|
|
64
|
+
for (const candidate of candidates) {
|
|
65
|
+
totalCount += 1;
|
|
66
|
+
byConfidence[candidate.confidence] += 1;
|
|
67
|
+
byResolutionState[candidate.resolutionState] += 1;
|
|
68
|
+
byCategory[candidate.category] = (byCategory[candidate.category] ?? 0) + 1;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Deterministic key order regardless of check/candidate traversal order.
|
|
72
|
+
const sortedByCategory = Object.fromEntries(Object.entries(byCategory).sort(([a], [b]) => a.localeCompare(b)));
|
|
73
|
+
return { totalCount, checksWithCandidates, byConfidence, byResolutionState, byCategory: sortedByCategory };
|
|
74
|
+
}
|
|
75
|
+
export async function runAndroidAuditIntegration(options) {
|
|
76
|
+
const runAndroidValidation = options.runAndroidValidation ?? validateAndroidTarget;
|
|
77
|
+
let result;
|
|
78
|
+
try {
|
|
79
|
+
result = await runAndroidValidation({
|
|
80
|
+
toolRoot: options.toolRoot,
|
|
81
|
+
targetPath: options.targetPathArg,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
86
|
+
return { issues: [], summary: failedSummary(`Android validation failed to run: ${message}`) };
|
|
87
|
+
}
|
|
88
|
+
const isApplicable = result.detection.projectKind !== "non-android";
|
|
89
|
+
const reportModel = toAndroidReportModel(result, { profile: "android" });
|
|
90
|
+
const reportDir = path.join(options.toolRoot, "reports", "security", ANDROID_REPORT_SUBDIR);
|
|
91
|
+
const prefix = reportFilenamePrefix({
|
|
92
|
+
isSelf: result.target.local.isSelf,
|
|
93
|
+
packageName: result.target.local.packageName,
|
|
94
|
+
packageVersion: result.target.local.packageVersion,
|
|
95
|
+
targetRoot: result.target.local.targetRoot,
|
|
96
|
+
});
|
|
97
|
+
let reportPaths;
|
|
98
|
+
let writeErrors = [];
|
|
99
|
+
try {
|
|
100
|
+
const written = writeAndroidReportFiles({ outDir: reportDir, prefix, report: reportModel, formats: ["text", "json"] });
|
|
101
|
+
reportPaths = { text: written.textPath, json: written.jsonPath };
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
105
|
+
reportPaths = { text: null, json: null };
|
|
106
|
+
writeErrors = [boundErrorMessage(`Failed to write Android report files: ${message}`)];
|
|
107
|
+
}
|
|
108
|
+
const issues = aggregateAndroidIssues(result.checks, reportPaths);
|
|
109
|
+
const passedChecks = result.checks.filter((c) => c.status === "passed").length;
|
|
110
|
+
const skippedChecks = result.checks.filter((c) => c.status === "skipped").length;
|
|
111
|
+
const inconclusiveChecks = result.checks.filter((c) => c.status === "inconclusive").length;
|
|
112
|
+
const failedChecks = result.checks.filter((c) => c.status === "failed" || c.status === "error").length;
|
|
113
|
+
const checksWithFindings = result.checks.filter((c) => c.findings.length > 0).length;
|
|
114
|
+
const candidateOnlyChecks = result.checks.filter((c) => c.findings.length === 0 && (c.candidateEvidence?.length ?? 0) > 0).length;
|
|
115
|
+
const candidateEvidenceCount = result.checks.reduce((sum, c) => sum + (c.candidateEvidence?.length ?? 0), 0);
|
|
116
|
+
const summary = {
|
|
117
|
+
requested: true,
|
|
118
|
+
applicable: isApplicable,
|
|
119
|
+
status: "completed",
|
|
120
|
+
verdict: result.verdict,
|
|
121
|
+
totalChecks: result.checks.length,
|
|
122
|
+
passedChecks,
|
|
123
|
+
checksWithFindings,
|
|
124
|
+
candidateOnlyChecks,
|
|
125
|
+
skippedChecks,
|
|
126
|
+
inconclusiveChecks,
|
|
127
|
+
failedChecks,
|
|
128
|
+
confirmedFindingCount: result.findings.length,
|
|
129
|
+
mappedIssueCount: issues.length,
|
|
130
|
+
candidateEvidenceCount,
|
|
131
|
+
// No Gradle/external-tool programmatic passthrough exists yet in Batch 2
|
|
132
|
+
// (see securityAuditTypes.ts's AndroidAuditSummary comment) — the default
|
|
133
|
+
// request always executes zero of either.
|
|
134
|
+
requestedGradleOperationCount: 0,
|
|
135
|
+
executedGradleOperationCount: 0,
|
|
136
|
+
requestedExternalToolCount: 0,
|
|
137
|
+
executedExternalToolCount: 0,
|
|
138
|
+
skippedExternalToolCount: 0,
|
|
139
|
+
reportPaths,
|
|
140
|
+
warnings: result.warnings,
|
|
141
|
+
errors: [...result.errors, ...writeErrors],
|
|
142
|
+
candidateSummary: buildCandidateSummary(result.checks),
|
|
143
|
+
};
|
|
144
|
+
return { issues, summary };
|
|
145
|
+
}
|