@dailephd/my-dev-kit-lab 0.4.1 → 0.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +195 -12
  2. package/benchmarks/contracts/todo-behavior.md +1 -1
  3. package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
  4. package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
  5. package/dist/scripts/audits/runAudit.js +28 -1
  6. package/dist/src/audits/core/auditConfig.js +16 -1
  7. package/dist/src/audits/core/auditRunner.js +6 -2
  8. package/dist/src/audits/report/auditReportModel.js +7 -0
  9. package/dist/src/audits/report/renderAuditJsonReport.js +1 -0
  10. package/dist/src/audits/report/renderAuditTextReport.js +37 -0
  11. package/dist/src/audits/security/androidAuditIntegration.js +145 -0
  12. package/dist/src/audits/security/mapAndroidSecurityFindingToAuditIssue.js +116 -0
  13. package/dist/src/audits/security/mapSecurityFindingToAuditIssue.js +15 -5
  14. package/dist/src/audits/security/securityAuditAdapter.js +20 -2
  15. package/dist/src/audits/security/securityAuditTypes.js +37 -0
  16. package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
  17. package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
  18. package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
  19. package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
  20. package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
  21. package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
  22. package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
  23. package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
  24. package/dist/src/evaluation/stageContextExpectations/validation.js +568 -0
  25. package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
  26. package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
  27. package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
  28. package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
  29. package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
  30. package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
  31. package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
  32. package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
  33. package/dist/src/evaluation/stageContextMetrics/index.js +10 -0
  34. package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
  35. package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
  36. package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
  37. package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +79 -0
  38. package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
  39. package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
  40. package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
  41. package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
  42. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
  43. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
  44. package/dist/src/evaluation/targetImmutability/index.js +3 -0
  45. package/dist/src/evaluation/targetImmutability/types.js +1 -0
  46. package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
  47. package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
  48. package/dist/src/evaluation/upstreamArtifacts/index.js +3 -0
  49. package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
  50. package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
  51. package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
  52. package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
  53. package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
  54. package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
  55. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
  56. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
  57. package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
  58. package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
  59. package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
  60. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
  61. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +105 -0
  62. package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
  63. package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
  64. package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
  65. package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +286 -0
  66. package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
  67. package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
  68. package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
  69. package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
  70. package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +190 -0
  71. package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
  72. package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
  73. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
  74. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
  75. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
  76. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
  77. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +252 -0
  78. package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +398 -0
  79. package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
  80. package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
  81. package/dist/src/report/experiments/index.js +4 -0
  82. package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +234 -0
  83. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
  84. package/dist/src/report/experiments/renderPluginExperimentReportText.js +225 -0
  85. package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
  86. package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
  87. package/docs/METRICS.md +56 -10
  88. package/package.json +1 -1
  89. package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
  90. package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
  91. package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
  92. package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
  93. package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
  94. package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
  95. package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
  96. package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
  97. package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
  98. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
  99. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
  100. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
  101. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
  102. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
  103. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
package/README.md CHANGED
@@ -1,27 +1,210 @@
1
1
  # my-dev-kit-lab
2
2
 
3
- my-dev-kit-lab is the experiment, evidence, reporting, visualization, audit, and security-validation companion for my-dev-kit. It provides controlled retrieval experiments, deterministic demonstrations, reports, plots, galleries, generic audits, and local security-validation workflows.
3
+ my-dev-kit-lab is the experiment, audit, and evidence companion for [my-dev-kit](https://www.npmjs.com/package/@dailephd/my-dev-kit). It helps users compare repository-context strategies, audit project health, validate CLI/package and Android security boundaries, and turn each run into reviewable reports and visual artifacts.
4
4
 
5
- ## Version state
5
+ my-dev-kit provides local repository indexing and graph-guided retrieval. my-dev-kit-lab supplies the controlled benchmarks, agent adapters, metrics, security checks, and reports needed to evaluate when that retrieval is useful. Results are evidence for a specific target and configuration; they do not guarantee token savings or security.
6
6
 
7
- `v0.4.1` is the published baseline, with the advanced Android implementation released. `v0.4.0` is the previous published baseline. `v0.4.2` remains future work.
7
+ The latest published release is v0.4.3 (stage-specific bounded-context and workflow-instruction evaluation). See [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md).
8
8
 
9
- ## Android validation
9
+ ## Current capabilities
10
+
11
+ - **Run context-strategy experiments:** compare `raw-full-file` with `my-dev-kit-guided` using deterministic fixtures or locally configured Codex and Claude CLIs.
12
+ - **Audit repository health:** run conservative code-rot detectors for TypeScript/JavaScript, Python, Java, and Kotlin, or adapt security findings into the common audit report.
13
+ - **Validate CLI/package security:** inspect dependencies, package contents, path and subprocess boundaries, malformed inputs, optional static scanners, and bounded fuzz targets.
14
+ - **Validate Android projects:** run nineteen static checks by default, with Gradle operations, external tools, and network access available only through explicit opt-in flags.
15
+ - **Review evidence:** generate JSON and HTML reports, SVG plots, optional screenshots, visualization demos, and a static gallery.
16
+ - **Evaluate stage-context strategies:** compare the two legacy strategies against six additional bounded stage-context strategies — `architecture-context-only`, `architecture-plus-implementation-refresh`, `architecture-plus-implementation-and-test-refresh`, `full-workflow-library`, `bounded-workflow-instruction-packet`, and `combined-bounded-stage-context` — selected through programmatic configuration, not CLI flags. Each strategy's evidence is reported through bounded `report.json`, `report.html`, and `report.txt` output with an explicit `available`/`unavailable`/`not-applicable` metric-availability model and no composite score, grade, ranking, or winning strategy.
17
+
18
+ ## Quickstart
19
+
20
+ ### Install
21
+
22
+ ```bash
23
+ npm install
24
+ ```
25
+
26
+ The same command works in PowerShell and `cmd.exe`.
27
+
28
+ ### Build
29
+
30
+ ```bash
31
+ npm run build
32
+ ```
33
+
34
+ ### Verify the installation
35
+
36
+ ```bash
37
+ npm run verify
38
+ ```
39
+
40
+ ### Run the fake-agent final demo (deterministic, no external CLIs required)
41
+
42
+ ```bash
43
+ npm run run-final-demo -- \
44
+ --cases examples/token-savings-cases.json \
45
+ --out lab-output/final-demo \
46
+ --kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" \
47
+ --agents fake-agent \
48
+ --complexities short \
49
+ --no-screenshot
50
+ ```
51
+
52
+ ```powershell
53
+ npm run run-final-demo -- `
54
+ --cases examples/token-savings-cases.json `
55
+ --out lab-output/final-demo `
56
+ --kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" `
57
+ --agents fake-agent `
58
+ --complexities short `
59
+ --no-screenshot
60
+ ```
61
+
62
+ ```bat
63
+ npm run run-final-demo -- --cases examples/token-savings-cases.json --out lab-output/final-demo --kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" --agents fake-agent --complexities short --no-screenshot
64
+ ```
65
+
66
+ The lab resolves Windows `.cmd` and `.ps1` CLI shims, supports command paths with spaces, and keeps generated artifacts inside the requested output directory.
67
+
68
+ This runs a full pipeline: controlled experiment → report → plots → visualization demos → gallery.
69
+
70
+ ### Run a real-agent campaign (requires Codex or Claude CLI)
71
+
72
+ ```bash
73
+ npm run run-controlled-experiment -- \
74
+ --cases examples/real-agent-campaign-cases.json \
75
+ --agents codex,claude \
76
+ --strategies raw-full-file,my-dev-kit-guided \
77
+ --complexities medium,multi-step \
78
+ --out lab-output/real-agent-campaign \
79
+ --include-real-agents \
80
+ --continue-on-failure \
81
+ --timeout-ms 240000
82
+ ```
83
+
84
+ Real-agent runs require local Codex or Claude CLI setup and available usage capacity. Runs that time out, produce invalid output, or hit session limits are recorded as structured outcomes rather than failures.
85
+
86
+ ### List, describe, and run experiment plugins
87
+
88
+ ```bash
89
+ npm run experiment:list
90
+ npm run experiment:describe -- --experiment context-strategy-comparison
91
+ npm run experiment:run -- \
92
+ --experiment context-strategy-comparison \
93
+ --target /path/to/local/project \
94
+ --agents fake-agent \
95
+ --complexities short \
96
+ --no-screenshot
97
+ ```
10
98
 
11
99
  ```powershell
12
- npm run security:validate -- --target "<android-project-path>" --profile android
100
+ npm run experiment:list
101
+ npm run experiment:describe -- --experiment context-strategy-comparison
102
+ npm run experiment:run -- `
103
+ --experiment context-strategy-comparison `
104
+ --target "Z:\Users\newuser\Projects\my-dev-kit-v1" `
105
+ --agents fake-agent `
106
+ --complexities short `
107
+ --no-screenshot
13
108
  ```
14
109
 
15
- The default Android profile runs nineteen checks: the original eight checks plus eleven advanced internal checks for Network Security Config, backup/data extraction, release configuration, secret candidates, signing configuration, WebView, FileProvider, sensitive storage, sensitive logging, clipboard, and Firebase/Google services. It runs zero Gradle operations, zero external tools, and zero network operations by default.
110
+ When `--target` is omitted, the experiment runs in self mode against my-dev-kit-lab. When `--target <path>` is provided, the lab remains the tool root and the target project is inspected separately. Generated experiment outputs stay under lab-controlled output directories by default, not inside the target project.
111
+
112
+ ---
113
+
114
+ ## Where to find outputs
115
+
116
+ | Artifact | Location |
117
+ |---|---|
118
+ | Experiment summary | `lab-output/<experiment>/experiment-summary.json` |
119
+ | All runs | `lab-output/<experiment>/experiment-runs.json` |
120
+ | Strategy comparisons | `lab-output/<experiment>/experiment-comparisons.json` |
121
+ | HTML report | `lab-output/<report>/experiment-report.html` |
122
+ | Report JSON | `lab-output/<report>/experiment-report.json` |
123
+ | Report screenshot | `lab-output/<report>/experiment-report.png` |
124
+ | Plugin experiment report JSON | `lab-output/experiments/<plugin>/<target>/<run>/report.json` |
125
+ | Plugin experiment report HTML | `lab-output/experiments/<plugin>/<target>/<run>/report.html` |
126
+ | Plugin experiment report text | `lab-output/experiments/<plugin>/<target>/<run>/report.txt` |
127
+ | Plot data | `lab-output/<plots>/plot-data.json` |
128
+ | SVG charts | `lab-output/<plots>/charts/*.svg` |
129
+ | Gallery manifest | `lab-output/<gallery>/gallery-manifest.json` |
130
+ | Gallery index | `lab-output/<gallery>/gallery-index.html` |
131
+ | Audit reports | `reports/audits/<type>/code-rot-audit.txt` and `.json` |
132
+ | Security reports | `reports/security/<prefix>-security-validation.txt` and `.json` |
133
+
134
+ ---
135
+
136
+ ## How to read the main report
137
+
138
+ Open `experiment-report.html` in a browser. The report shows:
139
+
140
+ - **Project profile** — benchmark project name, language mix, complexity score, and file tree
141
+ - **Benchmark tasks** — task descriptions and answer keys
142
+ - **Strategy comparisons** — paired `raw-full-file` vs `my-dev-kit-guided` runs per case
143
+ - **Correctness scores** — deterministic answer-key scoring (not semantic LLM judging)
144
+ - **Token usage** — estimated or reported token totals per run
145
+ - **Token savings** — positive means my-dev-kit used fewer tokens; negative means it used more
146
+ - **Duration** — wall-clock time per run
147
+ - **Status** — completed, timeout, invalid-output, or limit-reached
148
+ - **Warnings and limitations** — notes on missing token totals or partial results
149
+
150
+ See [docs/METRICS.md](docs/METRICS.md) for full metric definitions.
151
+
152
+ ---
153
+
154
+ ## Current limitations
155
+
156
+ - Token savings shown in fake-agent runs are based on estimated character counts, not provider billing telemetry
157
+ - Claude does not expose token totals; token savings comparisons are unavailable for Claude runs
158
+ - Codex may expose token totals but can produce timeouts or invalid-output runs
159
+ - Small projects may make raw-full-file cheaper than my-dev-kit-guided; larger localized tasks are where my-dev-kit is expected to become more useful
160
+ - The generic experiment-plugin framework currently ships one plugin, `context-strategy-comparison`; future plugins such as warm-index reuse, incremental-change, and context-window scaling are not implemented yet
161
+ - The current release does not guarantee token savings; it produces auditable evidence for specific cases, targets, agents, and strategies
162
+ - Provider telemetry dashboards, semantic LLM judging, and cloud API billing integration are not yet implemented
163
+ - The six new stage-context strategies have no CLI flags yet, are configured programmatically, and do not yet include plots, screenshots, or gallery integration
164
+ - The published upstream artifacts the stage-context strategies read do not expose considered-but-unselected reads or unnecessary-read evidence; those metrics report `unavailable` rather than zero
165
+
166
+ ---
167
+
168
+ ## Security validation
169
+
170
+ `npm run security:validate` is the standalone security-validation command. It checks local CLI/package boundaries and can inspect another local project with `--target <path>`. The generic audit command can reuse those results through `--types security`, but it does not replace the standalone validator or its reports.
171
+
172
+ Android validation uses `--profile android`. Its default path is static and non-destructive: it starts zero Gradle processes, zero external tools, and zero network operations. Only confirmed `SecurityFinding` records can become audit issues; Android `CandidateEvidence` remains review-only evidence.
173
+
174
+ Optional scanners are reported as `skipped` when unavailable, never as passed. The framework does not provide runtime isolation proof, device or APK/AAB analysis, signing verification, Play Console validation, automatic fixes, or manual pentesting. Manual pentest remains deferred until post-v1/version TBD.
175
+
176
+ See [COMMANDS.md](docs/COMMANDS.md) for exact syntax and [Security Validation Framework](docs/security-validation-framework.md) for checks, evidence semantics, verdicts, and limitations.
177
+
178
+ ---
179
+
180
+ ## License
181
+
182
+ MIT License. See [LICENSE](LICENSE) for the full text.
183
+
184
+ ---
185
+
186
+ ## Support
187
+
188
+ my-dev-kit-lab is an independent project by dailephd LLC, developed and maintained by Dai Le.
16
189
 
17
- `SecurityFinding` records confirmed conservative evidence. `CandidateEvidence` remains separate review or unresolved evidence and is not a confirmed vulnerability. Text and JSON reports under `reports/security/` include advanced-security checks, external-tool results, candidate-evidence summaries, artifacts, mutation evidence, verdict reasons, and limitations.
190
+ If this project helps your workflow, you can support continued development through GitHub Sponsors or PayPal:
18
191
 
19
- Optional Gradle operations use `--android-gradle-operations` and only accept `wrapper-version`, `tasks`, `assemble-debug`, `unit-test-debug`, and `lint-debug`. They are environment-dependent, record mutations, and never perform cleanup or arbitrary tasks.
192
+ - [Sponsor on GitHub](https://github.com/sponsors/dailephd)
193
+ - [Support via PayPal](https://paypal.me/daile88)
20
194
 
21
- Optional external tools use `--android-external-tools semgrep,osv,android-lint,dependency-check`. Network defaults to `--android-external-network deny`; `allow-requested` authorizes only requested supported network-capable tools. Semgrep uses package-owned local rules, Android Lint reuses the closed offline Gradle path, Dependency-Check disables updates, and OSV requires explicit network authorization. Tools are never installed automatically; unavailable requested tools are skipped or inconclusive, not clean passes.
195
+ Support is optional and does not affect access to the project.
22
196
 
23
- Verdicts are `not-ready-security-blocker-remains`, `inconclusive-audit-environment-incomplete`, `ready-except-optional-manual-checks`, and `ready-for-release-preparation`. They are evidence states, not runtime, Play, publication, or pentest guarantees.
197
+ ---
24
198
 
25
- Selected local references such as supported `@xml` resources are resolved conservatively. Manifests are not merged, full overlay precedence and Gradle evaluation are not implemented, whole-program flow is not performed, and APK/AAB, signing, emulator/device, remote Firebase, live Play policy, and manual pentest validation remain outside scope.
199
+ ## Documentation
26
200
 
27
- See [Project overview](docs/PROJECT_OVERVIEW.md), [Commands](docs/COMMANDS.md), [Workflows](docs/WORKFLOWS.md), [Architecture](docs/ARCHITECTURE.md), and [Roadmap](docs/ROADMAP.md).
201
+ - [docs/PROJECT_OVERVIEW.md](docs/PROJECT_OVERVIEW.md) — product purpose and target users
202
+ - [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md) — implemented, planned, validated, blocked, and next state
203
+ - [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — current components, ownership, flows, and invariants
204
+ - [docs/WORKFLOWS.md](docs/WORKFLOWS.md) — step-by-step workflows with diagrams
205
+ - [docs/COMMANDS.md](docs/COMMANDS.md) — all commands with options and examples
206
+ - [docs/TUTORIAL.md](docs/TUTORIAL.md) — first-run walkthrough
207
+ - [docs/METRICS.md](docs/METRICS.md) — metric definitions and interpretation
208
+ - [docs/ROADMAP.md](docs/ROADMAP.md) — versioned plans, dependencies, exclusions, and acceptance criteria
209
+ - [docs/GALLERY.md](docs/GALLERY.md) — gallery output explained
210
+ - [docs/security-validation-framework.md](docs/security-validation-framework.md) — security evidence, verdicts, and safety boundaries
@@ -66,5 +66,5 @@ Answer key requirements:
66
66
  - `answerKey.expectedFiles` lists files a correct answer may need to cite
67
67
  - `answerKey.expectedSymbols` lists relevant symbols
68
68
  - `answerKey.expectedFacts` lists weighted facts with unique `id`, `text`, `weight`, and `required`
69
- - `answerKey.minimumCorrectFacts` defines the future scoring threshold, but scoring is not implemented in this prompt
69
+ - `answerKey.minimumCorrectFacts` defines the required-fact threshold used by deterministic correctness scoring
70
70
  - optional `forbiddenWrongClaims` records claims that should be penalized by a later correctness scorer
@@ -1 +1,3 @@
1
+ # task-analytics-large-mixed
2
+
1
3
  Mixed TypeScript and Python benchmark project for deterministic task analytics, quality labeling, and report generation.
@@ -1 +1,3 @@
1
- Task workflow benchmark project with multiple services, validation rules, filtering, summarization, and deterministic imports.
1
+ # task-workflow-medium-ts
2
+
3
+ TypeScript task-workflow benchmark with multiple services, validation rules, filtering, summarization, and deterministic imports.
@@ -20,9 +20,29 @@ import { writeAuditReports } from "../../src/audits/report/writeAuditReports.js"
20
20
  // ---------------------------------------------------------------------------
21
21
  const USAGE = "Usage: npm run audit -- [--target <path>] [--types code-rot,security] " +
22
22
  "[--include docs,tests,package,architecture,cli] [--format text,json] " +
23
- "[--fail-on blocker|high|medium|low|none] [--out <path>]";
23
+ "[--fail-on blocker|high|medium|low|none] [--out <path>] [--android]\n" +
24
+ "\n" +
25
+ " --android opts into Batch 2's programmatic Android security integration.\n" +
26
+ " Requires --types to include \"security\" (e.g. --types security or --types\n" +
27
+ " code-rot,security). Runs the same static, read-only, nineteen-check\n" +
28
+ " Android validation security:validate --profile android performs --\n" +
29
+ " detection, manifest parsing, and internal advanced security checks --\n" +
30
+ " through the existing security adapter, never a subprocess. Confirmed\n" +
31
+ " findings are mapped into the normal issue collection; CandidateEvidence\n" +
32
+ " remains a separate, bounded summary and is never treated as a confirmed\n" +
33
+ " issue. Starts zero Gradle operations, zero external tools, and zero\n" +
34
+ " network operations by default; --android does not expose any of those.\n" +
35
+ " Example: npm run audit -- --target \"<android-project-path>\" --types security --android --format text,json --fail-on none";
24
36
  const rawArgs = process.argv.slice(2);
25
37
  const toolRoot = resolveToolRoot(import.meta.url);
38
+ // Help deliberately wins over every other argument, mirroring
39
+ // scripts/security/validate.ts -- no config normalization, target
40
+ // resolution, audit execution, or report write.
41
+ if (rawArgs.includes("--help") || rawArgs.includes("-h")) {
42
+ console.log(USAGE);
43
+ process.exitCode = 0;
44
+ process.exit(0);
45
+ }
26
46
  let config;
27
47
  try {
28
48
  const args = parseAuditArgs(rawArgs);
@@ -89,6 +109,13 @@ if (model.securitySummary.ran) {
89
109
  console.log(` Full report: ${model.securitySummary.reportPaths.text}`);
90
110
  }
91
111
  }
112
+ if (model.androidSecurity.summary.requested) {
113
+ const android = model.androidSecurity.summary;
114
+ console.log(`\nAndroid security validation: status=${android.status} applicable=${android.applicable ?? "n/a"} verdict=${android.verdict ?? "(unknown)"} checks=${android.totalChecks} confirmed=${android.confirmedFindingCount} mappedIssues=${android.mappedIssueCount} candidates=${android.candidateSummary.totalCount}`);
115
+ if (android.reportPaths.text) {
116
+ console.log(` Full report: ${android.reportPaths.text}`);
117
+ }
118
+ }
92
119
  console.log(`\nInventory: ${model.inventory.totalScannedFileCount} file(s) scanned, ${model.inventory.skippedFileCount} skipped`);
93
120
  console.log(` source=${model.inventory.filesByCategory.source} tests=${model.inventory.filesByCategory.tests} docs=${model.inventory.filesByCategory.docs} package=${model.inventory.filesByCategory.package} config=${model.inventory.filesByCategory.config} scripts=${model.inventory.filesByCategory.scripts} ci=${model.inventory.filesByCategory.ci}`);
94
121
  console.log(`Source of truth: package=${model.sourceOfTruth.packageName ?? "(none)"}@${model.sourceOfTruth.packageVersion ?? "?"} readme=${model.sourceOfTruth.hasReadme} changelog=${model.sourceOfTruth.hasChangelog} ciWorkflows=${model.sourceOfTruth.ciWorkflowCount}`);
@@ -8,10 +8,19 @@ const FLAGS_WITH_VALUE = {
8
8
  "--fail-on": "failOn",
9
9
  "--out": "out",
10
10
  };
11
+ // v0.4.2 Batch 3 -- presence-only flags (no following value consumed).
12
+ const BOOLEAN_FLAGS = {
13
+ "--android": "android",
14
+ };
11
15
  export function parseAuditArgs(argv) {
12
16
  const result = {};
13
17
  for (let i = 0; i < argv.length; i++) {
14
18
  const arg = argv[i];
19
+ const booleanKey = BOOLEAN_FLAGS[arg];
20
+ if (booleanKey) {
21
+ result[booleanKey] = true;
22
+ continue;
23
+ }
15
24
  const key = FLAGS_WITH_VALUE[arg];
16
25
  if (!key)
17
26
  continue;
@@ -28,6 +37,10 @@ export function normalizeAuditConfig(raw, toolRoot) {
28
37
  const formats = parseFormatOption(raw.format);
29
38
  const failOn = parseFailOnOption(raw.failOn);
30
39
  const out = raw.out !== undefined ? resolveOutDir(raw.out) : defaultOutDir(toolRoot, types.value);
40
+ const android = raw.android === true;
41
+ if (android && !types.value.includes("security")) {
42
+ throw new Error(`Invalid combination: --android requires --types to include "security" (e.g. --types security or --types code-rot,security).`);
43
+ }
31
44
  return {
32
45
  targetPathArg: raw.target,
33
46
  targetMode: raw.target === undefined ? "self" : "external",
@@ -46,7 +59,9 @@ export function normalizeAuditConfig(raw, toolRoot) {
46
59
  raw.include === undefined &&
47
60
  raw.format === undefined &&
48
61
  raw.failOn === undefined &&
49
- raw.out === undefined,
62
+ raw.out === undefined &&
63
+ raw.android === undefined,
64
+ android,
50
65
  };
51
66
  }
52
67
  function splitAndNormalizeList(raw, flagName) {
@@ -6,7 +6,7 @@ import { collectSourceOfTruth } from "./sourceOfTruth.js";
6
6
  import { collectSourceFacts } from "./collectSourceFacts.js";
7
7
  import { collectPythonProjectMetadata } from "./pythonProjectMetadata.js";
8
8
  import { runSecurityAuditAdapter } from "../security/securityAuditAdapter.js";
9
- import { SECURITY_AUDIT_NOT_RUN_SUMMARY } from "../security/securityAuditTypes.js";
9
+ import { ANDROID_AUDIT_NOT_REQUESTED_SUMMARY, SECURITY_AUDIT_NOT_RUN_SUMMARY, } from "../security/securityAuditTypes.js";
10
10
  export async function runAudit(options) {
11
11
  const { config, toolRoot } = options;
12
12
  const registry = options.registry ?? DEFAULT_AUDIT_REGISTRY;
@@ -49,10 +49,12 @@ export async function runAudit(options) {
49
49
  // code-rot detector issues (registry order) first, then security findings
50
50
  // (security-validation check-execution order).
51
51
  let securitySummary = SECURITY_AUDIT_NOT_RUN_SUMMARY;
52
+ let androidSummary = ANDROID_AUDIT_NOT_REQUESTED_SUMMARY;
52
53
  if (config.types.includes("security")) {
53
- const securityAudit = await runSecurityAuditAdapter({ toolRoot, config });
54
+ const securityAudit = await runSecurityAuditAdapter({ toolRoot, config, android: config.android ? { enabled: true } : undefined }, options.securityDependencies);
54
55
  issues.push(...securityAudit.issues);
55
56
  securitySummary = securityAudit.summary;
57
+ androidSummary = securityAudit.android;
56
58
  }
57
59
  const issueCounts = countIssuesBySeverity(issues);
58
60
  const { exitCode, reason } = calculateAuditExitCode(issues, config.failOn);
@@ -64,6 +66,7 @@ export async function runAudit(options) {
64
66
  failOn: config.failOn,
65
67
  out: config.out,
66
68
  isDefaultRun: config.isDefaultRun,
69
+ android: config.android,
67
70
  },
68
71
  targetSummary: {
69
72
  rootPath: target.rootPath,
@@ -81,6 +84,7 @@ export async function runAudit(options) {
81
84
  sourceFacts,
82
85
  pythonProjectMetadata,
83
86
  securitySummary,
87
+ androidSummary,
84
88
  noDetectorsRegistered: selected.length === 0,
85
89
  };
86
90
  }
@@ -166,6 +166,7 @@ export function buildAuditReportModel(result, opts) {
166
166
  failOn: result.configSummary.failOn,
167
167
  out: result.configSummary.out,
168
168
  isDefaultRun: result.configSummary.isDefaultRun,
169
+ android: result.configSummary.android,
169
170
  },
170
171
  summary: {
171
172
  totalIssues: issues.length,
@@ -191,6 +192,10 @@ export function buildAuditReportModel(result, opts) {
191
192
  sourceFacts: summarizeSourceFacts(result.sourceFacts),
192
193
  pythonProjectMetadata: result.pythonProjectMetadata,
193
194
  securitySummary: result.securitySummary,
195
+ androidSecurity: {
196
+ summary: result.androidSummary,
197
+ candidates: result.androidSummary.candidateSummary,
198
+ },
194
199
  detectors,
195
200
  issues,
196
201
  skippedDetectors: result.skippedDetectors,
@@ -252,5 +257,7 @@ function reconstructCommand(result) {
252
257
  parts.push("--format", cfg.formats.join(","));
253
258
  parts.push("--fail-on", cfg.failOn);
254
259
  parts.push("--out", cfg.out);
260
+ if (cfg.android)
261
+ parts.push("--android");
255
262
  return parts.join(" ");
256
263
  }
@@ -21,6 +21,7 @@ export function renderAuditJsonReport(model) {
21
21
  sourceFacts: model.sourceFacts,
22
22
  pythonProjectMetadata: model.pythonProjectMetadata,
23
23
  securitySummary: model.securitySummary,
24
+ androidSecurity: model.androidSecurity,
24
25
  detectors: model.detectors,
25
26
  issues: model.issues,
26
27
  skippedDetectors: model.skippedDetectors,
@@ -133,6 +133,43 @@ export function renderAuditTextReport(model) {
133
133
  }
134
134
  }
135
135
  lines.push("");
136
+ // v0.4.2 Batch 3 -- bounded summary only, mirroring the "Security
137
+ // validation summary" section above exactly. Confirmed mapped Android
138
+ // findings are NOT re-listed here -- they already appear in the "Issues"
139
+ // section below like any other AuditIssue (see spec section 12: "the
140
+ // existing generic issues section remains the detailed presentation for
141
+ // confirmed mapped Android findings"). CandidateEvidence bodies never
142
+ // appear here either -- only counts, with a pointer to the full Android
143
+ // report for detail.
144
+ lines.push("Android security validation summary");
145
+ lines.push(divider("-"));
146
+ if (!model.androidSecurity.summary.requested) {
147
+ lines.push(" (not requested -- add --android alongside --types security to include it)");
148
+ }
149
+ else {
150
+ const android = model.androidSecurity.summary;
151
+ const candidates = model.androidSecurity.candidates;
152
+ const applicabilityNote = android.applicable === false ? " (target is not an Android project)" : "";
153
+ lines.push(` Status: ${android.status}${applicabilityNote}`);
154
+ lines.push(` Verdict: ${android.verdict ?? "(unknown)"}`);
155
+ lines.push(` Checks: total=${android.totalChecks} passed=${android.passedChecks} withFindings=${android.checksWithFindings} candidateOnly=${android.candidateOnlyChecks} skipped=${android.skippedChecks} inconclusive=${android.inconclusiveChecks} failed=${android.failedChecks}`);
156
+ lines.push(` Findings: confirmed=${android.confirmedFindingCount} mappedIssues=${android.mappedIssueCount}`);
157
+ lines.push(` Candidates (review evidence, not confirmed vulnerabilities): total=${candidates.totalCount} checksWithCandidates=${candidates.checksWithCandidates}`);
158
+ if (android.warnings.length > 0) {
159
+ lines.push(` Warnings: ${android.warnings.length}`);
160
+ }
161
+ if (android.errors.length > 0) {
162
+ lines.push(` Errors: ${android.errors.length}`);
163
+ for (const e of android.errors.slice(0, 5)) {
164
+ lines.push(` ${sanitizeLine(e)}`);
165
+ }
166
+ }
167
+ if (android.reportPaths.text || android.reportPaths.json) {
168
+ lines.push(` Full report: ${android.reportPaths.text ?? "(none)"} / ${android.reportPaths.json ?? "(none)"}`);
169
+ }
170
+ lines.push(" Static analysis only -- does not prove runtime behavior, APK/AAB validity, Play compliance, or pentest results.");
171
+ }
172
+ lines.push("");
136
173
  lines.push("Issue summary");
137
174
  lines.push(divider("-"));
138
175
  lines.push(` Total issues: ${model.summary.totalIssues}`);
@@ -0,0 +1,145 @@
1
+ import path from "node:path";
2
+ import { validateAndroidTarget } from "../../mobile/android/validate/validateAndroidTarget.js";
3
+ import { toAndroidReportModel } from "../../mobile/android/report/model.js";
4
+ import { writeAndroidReportFiles } from "../../mobile/android/report/writeAndroidReportFiles.js";
5
+ import { reportFilenamePrefix } from "../../securityValidation/validate/resolveTarget.js";
6
+ import { mapAndroidSecurityFindingToAuditIssue } from "./mapAndroidSecurityFindingToAuditIssue.js";
7
+ import { ANDROID_AUDIT_NOT_REQUESTED_SUMMARY, EMPTY_ANDROID_AUDIT_CANDIDATE_SUMMARY, } from "./securityAuditTypes.js";
8
+ const ANDROID_REPORT_SUBDIR = "android";
9
+ // Bounded, redacted — never the thrown value's stack, never raw child-process
10
+ // output (there is none at this boundary; validateAndroidTarget itself never
11
+ // spawns a process unless Gradle/external-tool options are requested, which
12
+ // Batch 2 never supplies).
13
+ const MAX_ANDROID_ERROR_MESSAGE_LENGTH = 500;
14
+ function boundErrorMessage(message) {
15
+ return message.length <= MAX_ANDROID_ERROR_MESSAGE_LENGTH
16
+ ? message
17
+ : `${message.slice(0, MAX_ANDROID_ERROR_MESSAGE_LENGTH)}... [truncated, ${message.length} chars total]`;
18
+ }
19
+ function failedSummary(errorMessage) {
20
+ return {
21
+ ...ANDROID_AUDIT_NOT_REQUESTED_SUMMARY,
22
+ requested: true,
23
+ applicable: null,
24
+ status: "failed",
25
+ errors: [boundErrorMessage(errorMessage)],
26
+ };
27
+ }
28
+ // Exact-duplicate dedupe by AuditIssue id only, first occurrence wins —
29
+ // mirrors validateAndroidTarget's own findingsById convention. Iterates
30
+ // checks in their own fixed order (never re-sorted) and each check's own
31
+ // findings in their own order — never the top-level, already-flattened
32
+ // result.findings, which has lost per-check identity this mapper needs as
33
+ // context. Exported standalone so aggregation correctness (ordering,
34
+ // dedup, provenance) can be tested against small literal AndroidCheckResult
35
+ // fixtures without needing a full AndroidValidationResult.
36
+ export function aggregateAndroidIssues(checks, reportReference) {
37
+ const issuesById = new Map();
38
+ for (const check of checks) {
39
+ for (const finding of check.findings) {
40
+ const issue = mapAndroidSecurityFindingToAuditIssue(finding, {
41
+ checkId: check.id,
42
+ checkCategory: check.category,
43
+ checkConfidence: check.confidence,
44
+ reportReference,
45
+ });
46
+ if (!issuesById.has(issue.id)) {
47
+ issuesById.set(issue.id, issue);
48
+ }
49
+ }
50
+ }
51
+ return [...issuesById.values()];
52
+ }
53
+ export function buildCandidateSummary(checks) {
54
+ const byConfidence = { ...EMPTY_ANDROID_AUDIT_CANDIDATE_SUMMARY.byConfidence };
55
+ const byResolutionState = { ...EMPTY_ANDROID_AUDIT_CANDIDATE_SUMMARY.byResolutionState };
56
+ const byCategory = {};
57
+ let totalCount = 0;
58
+ let checksWithCandidates = 0;
59
+ for (const check of checks) {
60
+ const candidates = check.candidateEvidence ?? [];
61
+ if (candidates.length === 0)
62
+ continue;
63
+ checksWithCandidates += 1;
64
+ for (const candidate of candidates) {
65
+ totalCount += 1;
66
+ byConfidence[candidate.confidence] += 1;
67
+ byResolutionState[candidate.resolutionState] += 1;
68
+ byCategory[candidate.category] = (byCategory[candidate.category] ?? 0) + 1;
69
+ }
70
+ }
71
+ // Deterministic key order regardless of check/candidate traversal order.
72
+ const sortedByCategory = Object.fromEntries(Object.entries(byCategory).sort(([a], [b]) => a.localeCompare(b)));
73
+ return { totalCount, checksWithCandidates, byConfidence, byResolutionState, byCategory: sortedByCategory };
74
+ }
75
+ export async function runAndroidAuditIntegration(options) {
76
+ const runAndroidValidation = options.runAndroidValidation ?? validateAndroidTarget;
77
+ let result;
78
+ try {
79
+ result = await runAndroidValidation({
80
+ toolRoot: options.toolRoot,
81
+ targetPath: options.targetPathArg,
82
+ });
83
+ }
84
+ catch (error) {
85
+ const message = error instanceof Error ? error.message : String(error);
86
+ return { issues: [], summary: failedSummary(`Android validation failed to run: ${message}`) };
87
+ }
88
+ const isApplicable = result.detection.projectKind !== "non-android";
89
+ const reportModel = toAndroidReportModel(result, { profile: "android" });
90
+ const reportDir = path.join(options.toolRoot, "reports", "security", ANDROID_REPORT_SUBDIR);
91
+ const prefix = reportFilenamePrefix({
92
+ isSelf: result.target.local.isSelf,
93
+ packageName: result.target.local.packageName,
94
+ packageVersion: result.target.local.packageVersion,
95
+ targetRoot: result.target.local.targetRoot,
96
+ });
97
+ let reportPaths;
98
+ let writeErrors = [];
99
+ try {
100
+ const written = writeAndroidReportFiles({ outDir: reportDir, prefix, report: reportModel, formats: ["text", "json"] });
101
+ reportPaths = { text: written.textPath, json: written.jsonPath };
102
+ }
103
+ catch (error) {
104
+ const message = error instanceof Error ? error.message : String(error);
105
+ reportPaths = { text: null, json: null };
106
+ writeErrors = [boundErrorMessage(`Failed to write Android report files: ${message}`)];
107
+ }
108
+ const issues = aggregateAndroidIssues(result.checks, reportPaths);
109
+ const passedChecks = result.checks.filter((c) => c.status === "passed").length;
110
+ const skippedChecks = result.checks.filter((c) => c.status === "skipped").length;
111
+ const inconclusiveChecks = result.checks.filter((c) => c.status === "inconclusive").length;
112
+ const failedChecks = result.checks.filter((c) => c.status === "failed" || c.status === "error").length;
113
+ const checksWithFindings = result.checks.filter((c) => c.findings.length > 0).length;
114
+ const candidateOnlyChecks = result.checks.filter((c) => c.findings.length === 0 && (c.candidateEvidence?.length ?? 0) > 0).length;
115
+ const candidateEvidenceCount = result.checks.reduce((sum, c) => sum + (c.candidateEvidence?.length ?? 0), 0);
116
+ const summary = {
117
+ requested: true,
118
+ applicable: isApplicable,
119
+ status: "completed",
120
+ verdict: result.verdict,
121
+ totalChecks: result.checks.length,
122
+ passedChecks,
123
+ checksWithFindings,
124
+ candidateOnlyChecks,
125
+ skippedChecks,
126
+ inconclusiveChecks,
127
+ failedChecks,
128
+ confirmedFindingCount: result.findings.length,
129
+ mappedIssueCount: issues.length,
130
+ candidateEvidenceCount,
131
+ // No Gradle/external-tool programmatic passthrough exists yet in Batch 2
132
+ // (see securityAuditTypes.ts's AndroidAuditSummary comment) — the default
133
+ // request always executes zero of either.
134
+ requestedGradleOperationCount: 0,
135
+ executedGradleOperationCount: 0,
136
+ requestedExternalToolCount: 0,
137
+ executedExternalToolCount: 0,
138
+ skippedExternalToolCount: 0,
139
+ reportPaths,
140
+ warnings: result.warnings,
141
+ errors: [...result.errors, ...writeErrors],
142
+ candidateSummary: buildCandidateSummary(result.checks),
143
+ };
144
+ return { issues, summary };
145
+ }