@dailephd/my-dev-kit-lab 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -113
- package/benchmarks/contracts/todo-behavior.md +1 -1
- package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
- package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
- package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
- package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
- package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
- package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
- package/dist/src/evaluation/stageContextExpectations/producerReadinessExpectationTypes.js +8 -0
- package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
- package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/validation.js +768 -0
- package/dist/src/evaluation/stageContextMetrics/calculateAllocationMetrics.js +96 -0
- package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
- package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
- package/dist/src/evaluation/stageContextMetrics/calculateCriticalityMetrics.js +112 -0
- package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
- package/dist/src/evaluation/stageContextMetrics/calculateOwnerMetrics.js +84 -0
- package/dist/src/evaluation/stageContextMetrics/calculateReadinessAgreement.js +152 -0
- package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
- package/dist/src/evaluation/stageContextMetrics/calculateSupplementalRawAgreement.js +50 -0
- package/dist/src/evaluation/stageContextMetrics/calculateTruncationClassification.js +44 -0
- package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
- package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
- package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js +127 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
- package/dist/src/evaluation/stageContextMetrics/index.js +17 -0
- package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
- package/dist/src/evaluation/stageContextMetrics/producerReadinessMetricTypes.js +1 -0
- package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
- package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +89 -0
- package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
- package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
- package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
- package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
- package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
- package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
- package/dist/src/evaluation/targetImmutability/index.js +3 -0
- package/dist/src/evaluation/targetImmutability/types.js +1 -0
- package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
- package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/index.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorContextReadinessResultV1.js +247 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/readImplementationContextPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readImplementationContextRetrievalReportV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
- package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readSupplementalContextTextFile.js +29 -0
- package/dist/src/evaluation/upstreamArtifacts/readTestContextPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readTestContextRetrievalReportV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
- package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
- package/dist/src/evaluation/upstreamArtifacts/supplementalContextTextParser.js +75 -0
- package/dist/src/evaluation/upstreamArtifacts/supplementalContextTypes.js +152 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +112 -0
- package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
- package/dist/src/evaluation/upstreamArtifacts/validateSupplementalContextDocumentV1.js +148 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +397 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
- package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +205 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +266 -0
- package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +520 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
- package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +296 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +292 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
- package/docs/METRICS.md +84 -10
- package/package.json +2 -2
- package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
- package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
- package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
- package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
- package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
- package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
- package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
- package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
- package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +13 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +7874 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +1076 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
package/README.md
CHANGED
|
@@ -1,66 +1,20 @@
|
|
|
1
1
|
# my-dev-kit-lab
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
my-dev-kit-lab is the experiment, audit, and evidence companion for [my-dev-kit](https://www.npmjs.com/package/@dailephd/my-dev-kit). It helps users compare repository-context strategies, audit project health, validate CLI/package and Android security boundaries, and turn each run into reviewable reports and visual artifacts.
|
|
4
4
|
|
|
5
|
-
my-dev-kit
|
|
5
|
+
my-dev-kit provides local repository indexing and graph-guided retrieval. my-dev-kit-lab supplies the controlled benchmarks, agent adapters, metrics, security checks, and reports needed to evaluate when that retrieval is useful. Results are evidence for a specific target and configuration; they do not guarantee token savings or security.
|
|
6
6
|
|
|
7
|
-
The
|
|
8
|
-
|
|
9
|
-
my-dev-kit is most useful when the repository is larger than the task. It helps coding agents work with large codebases through reusable structural indexing, graph-guided retrieval, targeted source slices, and auditable context selection. Results are scoped evidence; the project does not claim that my-dev-kit always saves tokens.
|
|
10
|
-
|
|
11
|
-
**my-dev-kit** is the repo indexing and graph-guided retrieval engine.
|
|
12
|
-
**my-dev-kit-lab** is the separate lab layer that feeds it benchmark inputs and records evaluation outputs.
|
|
13
|
-
|
|
14
|
-
---
|
|
7
|
+
The latest published release is v0.4.4 (producer-readiness bridge). See [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md).
|
|
15
8
|
|
|
16
9
|
## Current capabilities
|
|
17
10
|
|
|
18
|
-
-
|
|
19
|
-
-
|
|
20
|
-
-
|
|
21
|
-
-
|
|
22
|
-
-
|
|
23
|
-
-
|
|
24
|
-
-
|
|
25
|
-
- Token usage, duration, and status comparisons between matched strategy pairs
|
|
26
|
-
- HTML experiment report rendering
|
|
27
|
-
- Static SVG plot generation
|
|
28
|
-
- Optional PNG screenshot capture
|
|
29
|
-
- Gallery manifest and static gallery index output
|
|
30
|
-
- Visualization demos using my-dev-kit commands against benchmark projects
|
|
31
|
-
- Final demo workflow combining all pipeline stages
|
|
32
|
-
- Generic experiment-plugin command surface: `experiment:list`, `experiment:describe`, and `experiment:run`
|
|
33
|
-
- First experiment plugin: `context-strategy-comparison`
|
|
34
|
-
- Target-aware experiment execution for local projects via `experiment:run -- --target <path>`
|
|
35
|
-
- Plugin-aware JSON and HTML reports with plugin, target, variant, metric, artifact, warning, skip, and failure metadata
|
|
36
|
-
- Security validation framework: dependency audit, package tarball inspection, CLI adversarial tests, static scans (CodeQL/Semgrep), bounded fuzz smoke, attack-scenario checks, and structured verdict/report output — runnable against any local project via `security:validate --target <path>`. `security:validate` remains the standalone, focused security-validation command; it is not replaced by the audit adapter below.
|
|
37
|
-
- Generic audit framework: `npm run audit` runs conservative code-rot and security detectors and writes text/JSON reports under `reports/audits/` by default. `code-rot` and `security` are the currently implemented audit types (`--types code-rot`, `--types security`, or combined `--types code-rot,security`); `quality`, `project`, and `all` audit types are planned and fail cleanly instead of running. Audit does not modify target files and does not auto-fix issues. See [docs/COMMANDS.md](docs/COMMANDS.md) for full flags.
|
|
38
|
-
- Language-aware code-rot detectors for TypeScript/JavaScript, Python, Java, and Kotlin, built on a shared source-facts model and normalized language/file-role inventory. Java/Kotlin support is conservative static analysis only: no compiler parsing, type/classpath resolution, or Gradle/Maven execution.
|
|
39
|
-
- Security-validation audit adapter: `npm run audit -- --types security` runs the same `security:validate` internals through the shared audit/report surface, mapping findings into audit issues and linking back to the original `reports/security/` report. `security:validate`'s own checks and reports are unaffected.
|
|
40
|
-
- Android validation: `security:validate --profile android` runs nineteen static checks against an Android project (manifest, permissions, exported components, network security config, backup/release configuration, secrets, signing, WebView/FileProvider, sensitive storage/logging, and Firebase/Google services), with optional opt-in Gradle operations and external tools (Semgrep, OSV-Scanner, Android Lint, Dependency-Check). Default execution starts zero Gradle processes, zero external tools, and zero network operations.
|
|
41
|
-
- Android-aware generic audit: `npm run audit -- --types security --android` runs the same static Android validation through the audit adapter, mapping confirmed findings into audit issues while keeping review-only `CandidateEvidence` separate and never treated as a confirmed issue.
|
|
42
|
-
|
|
43
|
-
---
|
|
44
|
-
|
|
45
|
-
## Architecture overview
|
|
46
|
-
|
|
47
|
-
```mermaid
|
|
48
|
-
flowchart TD
|
|
49
|
-
A[scripts/experiments] --> B[Experiment plugin registry + runner]
|
|
50
|
-
B --> C[context-strategy-comparison]
|
|
51
|
-
C --> D[Controlled experiment foundations]
|
|
52
|
-
D --> E[Plugin + legacy artifacts]
|
|
53
|
-
E --> F[Plugin-aware reports]
|
|
54
|
-
E --> G[Plots / screenshots / gallery]
|
|
55
|
-
H[scripts/security] --> I[Automated security validation]
|
|
56
|
-
I --> J[Security reports + verdict]
|
|
57
|
-
I --> K[Android validation]
|
|
58
|
-
L[scripts/audits] --> M[Generic audit framework]
|
|
59
|
-
M --> N[Code-rot detectors]
|
|
60
|
-
M --> I
|
|
61
|
-
```
|
|
62
|
-
|
|
63
|
-
---
|
|
11
|
+
- **Run context-strategy experiments:** compare `raw-full-file` with `my-dev-kit-guided` using deterministic fixtures or locally configured Codex and Claude CLIs.
|
|
12
|
+
- **Audit repository health:** run conservative code-rot detectors for TypeScript/JavaScript, Python, Java, and Kotlin, or adapt security findings into the common audit report.
|
|
13
|
+
- **Validate CLI/package security:** inspect dependencies, package contents, path and subprocess boundaries, malformed inputs, optional static scanners, and bounded fuzz targets.
|
|
14
|
+
- **Validate Android projects:** run nineteen static checks by default, with Gradle operations, external tools, and network access available only through explicit opt-in flags.
|
|
15
|
+
- **Review evidence:** generate JSON and HTML reports, SVG plots, optional screenshots, visualization demos, and a static gallery.
|
|
16
|
+
- **Evaluate stage-context strategies:** compare the two legacy strategies against six additional bounded stage-context strategies — `architecture-context-only`, `architecture-plus-implementation-refresh`, `architecture-plus-implementation-and-test-refresh`, `full-workflow-library`, `bounded-workflow-instruction-packet`, and `combined-bounded-stage-context` — selected through programmatic configuration, not CLI flags. Each strategy's evidence is reported through bounded `report.json`, `report.html`, and `report.txt` output with an explicit `available`/`unavailable`/`not-applicable` metric-availability model and no composite score, grade, ranking, or winning strategy.
|
|
17
|
+
- **Evaluate the producer-readiness bridge (v0.4.4):** optionally extend `combined-bounded-stage-context` with the frozen my-dev-kit-orchestrator supplemental implementation/test-context packet and retrieval-report documents plus an observed readiness result, all supplied programmatically (there is no CLI flag), to measure owner, allocation, truncation-cause, supplemental/raw agreement, readiness-agreement, and criticality-overlay evidence without reimplementing upstream owner-selection, allocation, producer-parity, or readiness policy.
|
|
64
18
|
|
|
65
19
|
## Quickstart
|
|
66
20
|
|
|
@@ -81,9 +35,12 @@ npm run build
|
|
|
81
35
|
### Verify the installation
|
|
82
36
|
|
|
83
37
|
```bash
|
|
38
|
+
npm run test
|
|
84
39
|
npm run verify
|
|
85
40
|
```
|
|
86
41
|
|
|
42
|
+
`npm test` runs the complete test suite; `npm run verify` runs the remaining non-test verification gates. Running both is complete validation and does not execute the suite twice.
|
|
43
|
+
|
|
87
44
|
### Run the fake-agent final demo (deterministic, no external CLIs required)
|
|
88
45
|
|
|
89
46
|
```bash
|
|
@@ -170,10 +127,13 @@ When `--target` is omitted, the experiment runs in self mode against my-dev-kit-
|
|
|
170
127
|
| Report screenshot | `lab-output/<report>/experiment-report.png` |
|
|
171
128
|
| Plugin experiment report JSON | `lab-output/experiments/<plugin>/<target>/<run>/report.json` |
|
|
172
129
|
| Plugin experiment report HTML | `lab-output/experiments/<plugin>/<target>/<run>/report.html` |
|
|
130
|
+
| Plugin experiment report text | `lab-output/experiments/<plugin>/<target>/<run>/report.txt` |
|
|
173
131
|
| Plot data | `lab-output/<plots>/plot-data.json` |
|
|
174
132
|
| SVG charts | `lab-output/<plots>/charts/*.svg` |
|
|
175
133
|
| Gallery manifest | `lab-output/<gallery>/gallery-manifest.json` |
|
|
176
134
|
| Gallery index | `lab-output/<gallery>/gallery-index.html` |
|
|
135
|
+
| Audit reports | `reports/audits/<type>/code-rot-audit.txt` and `.json` |
|
|
136
|
+
| Security reports | `reports/security/<prefix>-security-validation.txt` and `.json` |
|
|
177
137
|
|
|
178
138
|
---
|
|
179
139
|
|
|
@@ -202,70 +162,23 @@ See [docs/METRICS.md](docs/METRICS.md) for full metric definitions.
|
|
|
202
162
|
- Codex may expose token totals but can produce timeouts or invalid-output runs
|
|
203
163
|
- Small projects may make raw-full-file cheaper than my-dev-kit-guided; larger localized tasks are where my-dev-kit is expected to become more useful
|
|
204
164
|
- The generic experiment-plugin framework currently ships one plugin, `context-strategy-comparison`; future plugins such as warm-index reuse, incremental-change, and context-window scaling are not implemented yet
|
|
205
|
-
- The current
|
|
165
|
+
- The current release does not guarantee token savings; it produces auditable evidence for specific cases, targets, agents, and strategies
|
|
206
166
|
- Provider telemetry dashboards, semantic LLM judging, and cloud API billing integration are not yet implemented
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
## Roadmap direction
|
|
211
|
-
|
|
212
|
-
my-dev-kit-lab is at a working baseline. The raw-vs-indexed experiment pipeline is fully implemented and produces reproducible artifacts, and real-agent campaign support exists for Codex and Claude. Recent releases added language-aware code-rot detectors (TypeScript/JavaScript, Python, Java, Kotlin), a security-validation audit adapter, and Android validation with an Android-aware extension of that same adapter.
|
|
213
|
-
|
|
214
|
-
- `v0.4.0` and `v0.4.1`: Android automated security validation (published)
|
|
215
|
-
- `v0.4.2`: Android-aware extension of the existing security audit adapter (see [CHANGELOG.md](CHANGELOG.md) for release status)
|
|
216
|
-
- manual pentest: post-v1 / version TBD
|
|
217
|
-
|
|
218
|
-
See [docs/ROADMAP.md](docs/ROADMAP.md) for the complete, versioned plan.
|
|
167
|
+
- The six new stage-context strategies have no CLI flags yet, are configured programmatically, and do not yet include plots, screenshots, or gallery integration
|
|
168
|
+
- The published upstream artifacts the stage-context strategies read do not expose considered-but-unselected reads or unnecessary-read evidence; those metrics report `unavailable` rather than zero
|
|
169
|
+
- The v0.4.4 producer-readiness bridge is released. All bridge inputs are programmatic — there is no CLI flag. The coordinated upstream releases my-dev-kit@1.10.3 and orchestrator@1.2.2 were verified published before lab publication.
|
|
219
170
|
|
|
220
171
|
---
|
|
221
172
|
|
|
222
173
|
## Security validation
|
|
223
174
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
`security:validate` remains the standalone, focused command for this validation and is unaffected by the current checked-out state. `npm run audit -- --types security` additionally adapts the same `security:validate` internals into the shared audit/report surface (mapped issues, a `securitySummary` field, and links to the original `reports/security/` output) — it does not replace or duplicate `security:validate`'s own checks or reports.
|
|
227
|
-
|
|
228
|
-
This is not a web application pentest framework. **my-dev-kit** is a local CLI/package, so the validation model is CLI/package adversarial testing focused on whether it remains:
|
|
229
|
-
|
|
230
|
-
- local-first
|
|
231
|
-
- deterministic
|
|
232
|
-
- read-only with respect to user source files
|
|
233
|
-
- network-free during normal CLI operation
|
|
234
|
-
- LLM-free
|
|
235
|
-
- database-free
|
|
236
|
-
- safe to run on local repositories
|
|
237
|
-
|
|
238
|
-
The automated validation gate is implemented. It combines static scans, dependency/package checks, adversarial CLI tests, bounded fuzz smoke tests, and attack-scenario checks with structured text/JSON reports and a four-category verdict.
|
|
239
|
-
|
|
240
|
-
### Security commands
|
|
241
|
-
|
|
242
|
-
| Command | Description |
|
|
243
|
-
|---|---|
|
|
244
|
-
| `npm run security:deps` | npm audit, OSV-Scanner (if available), outdated packages |
|
|
245
|
-
| `npm run security:package` | npm pack --dry-run, forbidden content detection |
|
|
246
|
-
| `npm run security:codeql` | CodeQL CLI availability check; skipped gracefully when absent |
|
|
247
|
-
| `npm run security:semgrep` | Semgrep scan via local binary or npx; skipped gracefully when both absent |
|
|
248
|
-
| `npm run test:security` | Automated security and adversarial CLI tests (path traversal, read-only boundaries, malformed artifacts, JSON safety, and related checks) |
|
|
249
|
-
| `npm run test:fuzz:smoke` | 9 bounded fuzz targets, seeded PRNG, completes in under 1 second |
|
|
250
|
-
| `npm run security:validate` | Runs selected security checks, applies profile-aware defaults when requested, and writes text/JSON reports according to `--format` |
|
|
251
|
-
|
|
252
|
-
CodeQL, Semgrep, and OSV-Scanner are optional. When unavailable locally, they are recorded as `skipped` in the report, not as passed. That can lead to `ready except optional manual checks`; it does not silently turn missing tooling into a clean pass.
|
|
253
|
-
|
|
254
|
-
Each security command can validate my-dev-kit-lab itself or another local project via `--target <path>`. When `--target` is omitted, the framework performs self-validation. Target projects are inspected in place: their source files are not modified by default, generated artifacts stay under `reports/security/` unless `--out` is used, and external-target reports identify the tool root, target root, target package metadata, target git metadata, command cwd, exit code, and stdout/stderr summaries.
|
|
255
|
-
|
|
256
|
-
`security:validate` supports `--checks`, `--profile`, `--format`, `--fail-on`, `--out`, and `--report-prefix`. The no-flag path remains backward compatible and still runs the classic implemented check groups: `deps`, `package`, `static`, `cli-adversarial`, and `fuzz`. Supplying `--profile` without `--checks` swaps in that profile's default checks; supplying explicit `--checks` always wins over profile defaults.
|
|
257
|
-
|
|
258
|
-
The current attack-scenario checks cover boundary, subprocess, secrets, and network assumptions. They are automated adversarial checks, not a manual pentest. Manual pentest is deferred until after `v1.0.0`. `verdictImpact` metadata from each registered scenario drives blocker categorization in report reasoning, and narrowed `--checks` runs are labeled as scoped rather than described as a full release gate.
|
|
259
|
-
|
|
260
|
-
JSON report poisoning/config-injection checks use a baseline-diff schema guard so legitimate additive JSON fields do not fail the guard while payload-created trusted top-level fields still do. Text reports are sanitized to strip ANSI/control-byte payloads before rendering.
|
|
261
|
-
|
|
262
|
-
Known limits remain intentionally explicit: secret scanning is bounded rather than exhaustive, the network/local-first check is a bounded static assumption check rather than proof of runtime isolation, package-boundary severity is still result-level rather than per-evidence-item, and profile-specific scenario behavior is limited to profile-based scenario selection/default checks.
|
|
175
|
+
`npm run security:validate` is the standalone security-validation command. It checks local CLI/package boundaries and can inspect another local project with `--target <path>`. The generic audit command can reuse those results through `--types security`, but it does not replace the standalone validator or its reports.
|
|
263
176
|
|
|
264
|
-
|
|
177
|
+
Android validation uses `--profile android`. Its default path is static and non-destructive: it starts zero Gradle processes, zero external tools, and zero network operations. Only confirmed `SecurityFinding` records can become audit issues; Android `CandidateEvidence` remains review-only evidence.
|
|
265
178
|
|
|
266
|
-
|
|
179
|
+
Optional scanners are reported as `skipped` when unavailable, never as passed. The framework does not provide runtime isolation proof, device or APK/AAB analysis, signing verification, Play Console validation, automatic fixes, or manual pentesting. Manual pentest remains deferred until post-v1/version TBD.
|
|
267
180
|
|
|
268
|
-
See [
|
|
181
|
+
See [COMMANDS.md](docs/COMMANDS.md) for exact syntax and [Security Validation Framework](docs/security-validation-framework.md) for checks, evidence semantics, verdicts, and limitations.
|
|
269
182
|
|
|
270
183
|
---
|
|
271
184
|
|
|
@@ -291,10 +204,12 @@ Support is optional and does not affect access to the project.
|
|
|
291
204
|
## Documentation
|
|
292
205
|
|
|
293
206
|
- [docs/PROJECT_OVERVIEW.md](docs/PROJECT_OVERVIEW.md) — product purpose and target users
|
|
294
|
-
- [docs/
|
|
207
|
+
- [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md) — implemented, planned, validated, blocked, and next state
|
|
208
|
+
- [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) — current components, ownership, flows, and invariants
|
|
295
209
|
- [docs/WORKFLOWS.md](docs/WORKFLOWS.md) — step-by-step workflows with diagrams
|
|
296
210
|
- [docs/COMMANDS.md](docs/COMMANDS.md) — all commands with options and examples
|
|
297
211
|
- [docs/TUTORIAL.md](docs/TUTORIAL.md) — first-run walkthrough
|
|
298
212
|
- [docs/METRICS.md](docs/METRICS.md) — metric definitions and interpretation
|
|
299
|
-
- [docs/ROADMAP.md](docs/ROADMAP.md) —
|
|
213
|
+
- [docs/ROADMAP.md](docs/ROADMAP.md) — versioned plans, dependencies, exclusions, and acceptance criteria
|
|
300
214
|
- [docs/GALLERY.md](docs/GALLERY.md) — gallery output explained
|
|
215
|
+
- [docs/security-validation-framework.md](docs/security-validation-framework.md) — security evidence, verdicts, and safety boundaries
|
|
@@ -66,5 +66,5 @@ Answer key requirements:
|
|
|
66
66
|
- `answerKey.expectedFiles` lists files a correct answer may need to cite
|
|
67
67
|
- `answerKey.expectedSymbols` lists relevant symbols
|
|
68
68
|
- `answerKey.expectedFacts` lists weighted facts with unique `id`, `text`, `weight`, and `required`
|
|
69
|
-
- `answerKey.minimumCorrectFacts` defines the
|
|
69
|
+
- `answerKey.minimumCorrectFacts` defines the required-fact threshold used by deterministic correctness scoring
|
|
70
70
|
- optional `forbiddenWrongClaims` records claims that should be penalized by a later correctness scorer
|
|
@@ -1 +1,3 @@
|
|
|
1
|
-
|
|
1
|
+
# task-workflow-medium-ts
|
|
2
|
+
|
|
3
|
+
TypeScript task-workflow benchmark with multiple services, validation rules, filtering, summarization, and deterministic imports.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { canonicalizeStageContextRun } from "./canonicalizeStageContextRun.js";
|
|
3
|
+
function sha256Hex(text) {
|
|
4
|
+
return createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex");
|
|
5
|
+
}
|
|
6
|
+
export function calculateStageContextDeterminism(candidates) {
|
|
7
|
+
if (candidates.length === 0) {
|
|
8
|
+
throw new Error("calculateStageContextDeterminism requires at least one candidate.");
|
|
9
|
+
}
|
|
10
|
+
if (candidates.length === 1) {
|
|
11
|
+
return {
|
|
12
|
+
availability: "not-applicable",
|
|
13
|
+
repeatCount: 1,
|
|
14
|
+
deterministic: null,
|
|
15
|
+
baselineSha256: null,
|
|
16
|
+
runDigests: [],
|
|
17
|
+
mismatchRunNumbers: [],
|
|
18
|
+
reason: "Repeated-run determinism requires at least two runs."
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
let runDigests;
|
|
22
|
+
try {
|
|
23
|
+
runDigests = candidates.map((candidate) => ({
|
|
24
|
+
runNumber: candidate.runNumber,
|
|
25
|
+
sha256: sha256Hex(canonicalizeStageContextRun(candidate.value))
|
|
26
|
+
}));
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return {
|
|
30
|
+
availability: "unavailable",
|
|
31
|
+
repeatCount: candidates.length,
|
|
32
|
+
deterministic: null,
|
|
33
|
+
baselineSha256: null,
|
|
34
|
+
runDigests: [],
|
|
35
|
+
mismatchRunNumbers: [],
|
|
36
|
+
reason: "Stage-context run values could not be canonicalized for determinism comparison."
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
const baselineSha256 = runDigests[0].sha256;
|
|
40
|
+
const mismatchRunNumbers = runDigests
|
|
41
|
+
.filter((digest) => digest.sha256 !== baselineSha256)
|
|
42
|
+
.map((digest) => digest.runNumber);
|
|
43
|
+
return {
|
|
44
|
+
availability: "available",
|
|
45
|
+
repeatCount: candidates.length,
|
|
46
|
+
deterministic: mismatchRunNumbers.length === 0,
|
|
47
|
+
baselineSha256,
|
|
48
|
+
runDigests,
|
|
49
|
+
mismatchRunNumbers,
|
|
50
|
+
reason: null
|
|
51
|
+
};
|
|
52
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export function canonicalizeStageContextRun(value) {
|
|
2
|
+
const seen = new WeakSet();
|
|
3
|
+
function serialize(input) {
|
|
4
|
+
if (input === null)
|
|
5
|
+
return "null";
|
|
6
|
+
const kind = typeof input;
|
|
7
|
+
if (kind === "bigint") {
|
|
8
|
+
throw new TypeError("Stage-context run values must not contain BigInt.");
|
|
9
|
+
}
|
|
10
|
+
if (kind === "function") {
|
|
11
|
+
throw new TypeError("Stage-context run values must not contain functions.");
|
|
12
|
+
}
|
|
13
|
+
if (kind === "symbol") {
|
|
14
|
+
throw new TypeError("Stage-context run values must not contain symbols.");
|
|
15
|
+
}
|
|
16
|
+
if (kind === "undefined") {
|
|
17
|
+
return "null";
|
|
18
|
+
}
|
|
19
|
+
if (kind === "number") {
|
|
20
|
+
if (!Number.isFinite(input)) {
|
|
21
|
+
throw new RangeError("Stage-context run values must not contain non-finite numbers.");
|
|
22
|
+
}
|
|
23
|
+
return JSON.stringify(input);
|
|
24
|
+
}
|
|
25
|
+
if (kind === "boolean" || kind === "string") {
|
|
26
|
+
return JSON.stringify(input);
|
|
27
|
+
}
|
|
28
|
+
if (Array.isArray(input)) {
|
|
29
|
+
if (seen.has(input)) {
|
|
30
|
+
throw new TypeError("Stage-context run values must not contain circular references.");
|
|
31
|
+
}
|
|
32
|
+
seen.add(input);
|
|
33
|
+
const items = input.map((item) => (item === undefined ? "null" : serialize(item)));
|
|
34
|
+
seen.delete(input);
|
|
35
|
+
return `[${items.join(",")}]`;
|
|
36
|
+
}
|
|
37
|
+
if (kind === "object") {
|
|
38
|
+
const obj = input;
|
|
39
|
+
if (seen.has(obj)) {
|
|
40
|
+
throw new TypeError("Stage-context run values must not contain circular references.");
|
|
41
|
+
}
|
|
42
|
+
seen.add(obj);
|
|
43
|
+
const keys = Object.keys(obj)
|
|
44
|
+
.filter((key) => obj[key] !== undefined)
|
|
45
|
+
.sort();
|
|
46
|
+
const parts = keys.map((key) => `${JSON.stringify(key)}:${serialize(obj[key])}`);
|
|
47
|
+
seen.delete(obj);
|
|
48
|
+
return `{${parts.join(",")}}`;
|
|
49
|
+
}
|
|
50
|
+
throw new TypeError("Stage-context run values contain an unsupported type.");
|
|
51
|
+
}
|
|
52
|
+
return serialize(value);
|
|
53
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// v0.4.4 Batch 2: additive, optional expectation facets covering the producer/readiness
|
|
2
|
+
// evidence Batch 1 preserves (selected owners, allocation/truncation evidence on the raw
|
|
3
|
+
// my-dev-kit artifacts, orchestrator readiness, and responsibility criticality). These are
|
|
4
|
+
// a separate axis from StageContextExpectationItemV1/expectedEvidence -- they are the
|
|
5
|
+
// deterministic lab oracle for Batch 2 metrics, not evidence-match expectations. Fully
|
|
6
|
+
// optional so every existing v0.4.3 StageContextExpectationFixtureV1 remains valid and
|
|
7
|
+
// unchanged (schema major stays 1).
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { validateStageContextExpectationFixtureV1 } from "./validation.js";
|
|
4
|
+
export async function readStageContextExpectationFixtureV1(sourcePathInput) {
|
|
5
|
+
const sourcePath = path.resolve(sourcePathInput);
|
|
6
|
+
let raw;
|
|
7
|
+
try {
|
|
8
|
+
raw = await readFile(sourcePath, "utf8");
|
|
9
|
+
}
|
|
10
|
+
catch (error) {
|
|
11
|
+
const code = error.code;
|
|
12
|
+
if (code === "ENOENT") {
|
|
13
|
+
return {
|
|
14
|
+
ok: false,
|
|
15
|
+
sourcePath,
|
|
16
|
+
code: "FILE_NOT_FOUND",
|
|
17
|
+
message: `Stage context expectation fixture was not found at "${sourcePath}".`
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
ok: false,
|
|
22
|
+
sourcePath,
|
|
23
|
+
code: "UNREADABLE_FILE",
|
|
24
|
+
message: `Stage context expectation fixture at "${sourcePath}" could not be read: ${error.message}`
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
let parsed;
|
|
28
|
+
try {
|
|
29
|
+
parsed = JSON.parse(raw);
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
return {
|
|
33
|
+
ok: false,
|
|
34
|
+
sourcePath,
|
|
35
|
+
code: "MALFORMED_JSON",
|
|
36
|
+
message: `Stage context expectation fixture at "${sourcePath}" is not valid JSON: ${error.message}`
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
40
|
+
return {
|
|
41
|
+
ok: false,
|
|
42
|
+
sourcePath,
|
|
43
|
+
code: "NON_OBJECT_ROOT",
|
|
44
|
+
message: `Stage context expectation fixture at "${sourcePath}" must parse to a JSON object at the root.`,
|
|
45
|
+
expected: "object",
|
|
46
|
+
actual: parsed
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
return validateStageContextExpectationFixtureV1(parsed, sourcePath);
|
|
50
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|