@dailephd/my-dev-kit-lab 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +468 -0
- package/README.md +3 -1
- package/dist/scripts/report-context-integrity-smoke.js +70 -0
- package/dist/src/evaluation/ecosystemFixtures/index.js +3 -0
- package/dist/src/evaluation/ecosystemFixtures/loadContextIntegrityFixture.js +102 -0
- package/dist/src/evaluation/ecosystemFixtures/manifestTypes.js +1 -0
- package/dist/src/evaluation/ecosystemFixtures/validateManifest.js +82 -0
- package/dist/src/evaluation/ecosystemFixtures/verifyFixtureHashes.js +45 -0
- package/dist/src/evaluation/stageContextMetrics/calculateAllocationMetrics.js +169 -0
- package/dist/src/evaluation/stageContextMetrics/calculateConditionCoverageMetrics.js +74 -0
- package/dist/src/evaluation/stageContextMetrics/calculateProducerConditionAgreement.js +113 -0
- package/dist/src/evaluation/stageContextMetrics/calculateReadinessAgreement.js +60 -0
- package/dist/src/evaluation/stageContextMetrics/calculateRunIntegrityAgreement.js +309 -0
- package/dist/src/evaluation/stageContextMetrics/calculateSupplementalRawAgreement.js +12 -2
- package/dist/src/evaluation/stageContextMetrics/calculateTruncationClassification.js +100 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js +76 -5
- package/dist/src/evaluation/stageContextMetrics/index.js +4 -0
- package/dist/src/evaluation/stageContextMetrics/runIntegrityAgreementTypes.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +7 -0
- package/dist/src/evaluation/stageContextSelectors/index.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/orchestratorRunIntegritySelectors.js +64 -0
- package/dist/src/evaluation/upstreamArtifacts/index.js +2 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorRunIntegrityV1.js +33 -0
- package/dist/src/evaluation/upstreamArtifacts/readOrchestratorRunIntegrityV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +59 -2
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +6 -1
- package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorRunIntegrityV1.js +177 -0
- package/dist/src/report/experiments/buildContextIntegrityReport.js +204 -0
- package/dist/src/report/experiments/contextIntegrityReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +5 -0
- package/dist/src/report/experiments/renderContextIntegrityHtml.js +98 -0
- package/dist/src/report/experiments/renderContextIntegrityJsonReport.js +3 -0
- package/dist/src/report/experiments/renderContextIntegrityText.js +111 -0
- package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +33 -0
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +11 -5
- package/docs/METRICS.md +83 -0
- package/package.json +4 -2
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/derived/run-integrity-evidence.json +99 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/producer/implementation-audit.json +1585 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/producer/implementation-capsule.json +11455 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/00-request.txt +1 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/architecture-context-packet.txt +82 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/behavior-model.txt +104 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/final-report.txt +41 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/implementation-context-packet.txt +114 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/implementation-report.txt +92 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/judge-report.txt +34 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/pseudocode-packet.txt +244 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/request-brief.txt +40 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-context-packet.txt +93 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-implementation-report.txt +69 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-strategy-packet.txt +354 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/verification-report.txt +85 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/derived/run-integrity-evidence.json +98 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/architecture-audit.json +1169 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/architecture-request.json +29 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-audit.json +1494 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-capsule.json +11364 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-request.json +33 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/test-request.json +74 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/architecture-context-retrieval-report.txt +58 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/implementation-context-retrieval-report.txt +75 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/test-context-retrieval-report.txt +84 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/run.json +62 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/tool-versions/my-dev-kit-version.txt +1 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/tool-versions/orchestrator-version.txt +1 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/corrected-replay-manifest.json +19 -0
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/failed-run-manifest.json +56 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/PROVENANCE.txt +18 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/context-capsule/complete-v1.0.0.json +556 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/retrieval-audit-record/complete-v1.0.0.json +238 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/PROVENANCE.txt +31 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/ready-pass-final-pass.json +132 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/refresh-required-need-context.json +146 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to my-dev-kit-lab are documented here.
|
|
4
|
+
|
|
5
|
+
## [0.4.5] - 2026-08-01
|
|
6
|
+
|
|
7
|
+
Context-integrity validation for the published my-dev-kit v1.10.4 and my-dev-kit-orchestrator v1.2.3 contracts.
|
|
8
|
+
|
|
9
|
+
- Added exact readers for the frozen local my-dev-kit v1.10.4 condition-aware producer contract (`roleConditionCoverage`, allocation/spillover `GroupTruncationEntry` fields, `truncation.requiredEvidenceLost`) in `src/evaluation/upstreamArtifacts`.
|
|
10
|
+
- Added allocation, spillover, condition-coverage, and producer-condition agreement metrics in `src/evaluation/stageContextMetrics`.
|
|
11
|
+
- Added exact readers and selectors for the frozen local my-dev-kit-orchestrator v1.2.3 run-integrity contract (`RunIntegrityGateResult`, `JudgeIntegrityResult`, `FinalReportEligibilityResult`, and `artifact-state.json` lifecycle records) in `src/evaluation/upstreamArtifacts` and `src/evaluation/stageContextSelectors`.
|
|
12
|
+
- Added run-integrity agreement calculators (readiness/prompt, readiness/expected-judge, expected/actual-judge, judge/correction, judge/final-eligibility, eligibility/final-artifact, and lifecycle-integrity agreement), composed additively into the existing `evaluateProducerReadinessBridge`.
|
|
13
|
+
- Added a frozen, hash-verified ecosystem regression fixture pair under `tests/fixtures/ecosystem/context-integrity/v0.4.5/`: a byte-exact real historical failed run, and a corrected-replay fixture that is a hand-distilled representation of the exact validated local v1.10.4/v1.2.3 contracts for the same request, target, and index identity — not a live capture of a complete ten-stage workflow, and not proof that every future run will behave identically. Added manifest/hash verification, determinism, and fixture self-immutability checks.
|
|
14
|
+
- Added a bounded `ContextIntegrityReportV1` JSON/text/HTML report layer in `src/report/experiments`, reusing the existing bounded-list/availability report primitives rather than duplicating them.
|
|
15
|
+
- All new inputs are read from fixtures or supplied programmatically; there is no CLI flag or command for this evaluation. No composite score, grade, ranking, or winner is produced, and no upstream producer, readiness, judge, correction, or lifecycle policy is duplicated. The orchestrator agreement evidence does not read a literal upstream `promptMode` field; `stageMayRenderNormalPrompt`, derived from structured blocked-stage evidence, is the bounded substitute used instead.
|
|
16
|
+
|
|
17
|
+
## [0.4.4] - 2026-07-29
|
|
18
|
+
|
|
19
|
+
- Added exact readers for the frozen my-dev-kit-orchestrator supplemental implementation/test-context packet and retrieval-report documents and a bounded plain-object adapter for the orchestrator readiness result.
|
|
20
|
+
- Added deterministic owner, allocation, truncation-cause, supplemental/raw agreement, readiness-agreement, and criticality-overlay metrics.
|
|
21
|
+
- Added an additive producer-readiness bridge evaluator wired into the existing `combined-bounded-stage-context` strategy and reported through the existing bounded `report.json`/`report.html`/`report.txt` pipeline as an optional section.
|
|
22
|
+
- All new inputs are optional and programmatic; existing v0.4.3 strategies, fixtures, and reports are unaffected when they are absent. No public CLI flags were added, and no upstream owner-selection, allocation, producer-parity, or readiness policy was duplicated.
|
|
23
|
+
- Release documentation finalized to post-publication state; coordinated upstream my-dev-kit@1.10.3 + orchestrator@1.2.2 verified published before lab release.
|
|
24
|
+
|
|
25
|
+
## [0.4.3] - 2026-07-22
|
|
26
|
+
|
|
27
|
+
`v0.4.3` (stage-specific bounded-context and workflow-instruction evaluation) is published. `v0.4.4` is the current release. Its complete scope, dependencies, exclusions, and acceptance criteria are preserved in [docs/ROADMAP.md](docs/ROADMAP.md).
|
|
28
|
+
|
|
29
|
+
### Added
|
|
30
|
+
|
|
31
|
+
- Added exact, non-normalizing readers for the my-dev-kit `ContextCapsule` and `RetrievalAuditRecord` and the my-dev-kit-orchestrator `WorkflowInstructionPacket`, each with schema-major validation and explicit failure on malformed or unsupported input.
|
|
32
|
+
- Extended the existing `context-strategy-comparison` plugin with six new stage-context strategies — `architecture-context-only`, `architecture-plus-implementation-refresh`, `architecture-plus-implementation-and-test-refresh`, `full-workflow-library`, `bounded-workflow-instruction-packet`, and `combined-bounded-stage-context` — selected through programmatic configuration alongside the preserved `raw-full-file` and `my-dev-kit-guided` strategies.
|
|
33
|
+
- Added an explicit stage-context expectation fixture contract (required/allowed/forbidden evidence, expected artifact states) and deterministic expectation matching against selected evidence.
|
|
34
|
+
- Added evidence-centered metrics: required-evidence recall, allowed-evidence coverage, forbidden-evidence inclusion, irrelevant-file/irrelevant-instruction inclusion, required-provenance recall, responsibility-mapping completeness, state comparisons, and context-size measurement, each reporting explicit `available`/`unavailable`/`not-applicable` status rather than coercing missing data to zero.
|
|
35
|
+
- Added read-only target-immutability snapshots (configured-file hashes and Git state) and before/after comparison, and repeated-run determinism (1–10 runs, canonical-serialization SHA-256 digests) as run-assurance evidence.
|
|
36
|
+
- Added bounded, deterministic `report.json`, `report.html`, and `report.txt` outputs for the new stage-context evidence through the existing `src/report/experiments` plugin report system, with a neutral interpretation that does not calculate a composite score, grade, ranking, or winning strategy.
|
|
37
|
+
|
|
38
|
+
### Compatibility and limitations
|
|
39
|
+
|
|
40
|
+
- No new CLI flags were added for the six new strategies; they are configured programmatically, not through `experiment:run` options.
|
|
41
|
+
- Plots, screenshots, and gallery integration for the new stage-context evidence are not part of this implementation.
|
|
42
|
+
- The published upstream artifacts do not expose considered-but-unselected reads or unnecessary-read evidence; those metrics are reported as `unavailable` with an explicit reason.
|
|
43
|
+
- Estimated token counts use `ceil(characterCount / 4)` per source and are not provider telemetry.
|
|
44
|
+
|
|
45
|
+
## [0.4.2] - 2026-07-12
|
|
46
|
+
|
|
47
|
+
Extended the existing general security audit adapter with Android-aware validation.
|
|
48
|
+
|
|
49
|
+
### Added
|
|
50
|
+
|
|
51
|
+
- Extended the existing general security audit adapter with an explicit `--android` opt-in when `--types` includes `security`, and added `--help`/`-h` to the `audit` CLI.
|
|
52
|
+
- Added direct programmatic Android validation reusing the existing `SecurityFinding`-to-`AuditIssue` mapping, with Android requested/applicable/status/verdict summaries, confirmed-finding and mapped-issue counts, and separate `CandidateEvidence` summaries that are never mapped to a confirmed `AuditIssue`.
|
|
53
|
+
- Added contained Android text/JSON report references and a bounded Android section in both the generic audit text report and the generic audit JSON report (a 17th top-level `androidSecurity` field), with text/JSON parity.
|
|
54
|
+
- Isolated Android validator failures from the rest of the audit run: a thrown Android error is captured as a bounded, stack-trace-free failure status without discarding already-collected non-Android issues.
|
|
55
|
+
- Preserved deterministic issue ordering and aggregation: non-Android issues are ordered first, Android issues follow in check/finding order, and repeated runs against unchanged fixtures produce identical issue IDs, order, and summary counts.
|
|
56
|
+
- Preserved standalone `security:validate` as the authoritative source of complete Android validation evidence; the audit adapter only reuses and links to it.
|
|
57
|
+
- Added a required `Check documentation consistency` (`npm run docs:check`) step to both the pre-release Node 26 readiness workflow and ordinary CI, running on every operating system in the existing Windows/macOS/Linux matrix, so documentation/roadmap consistency is verified cross-platform before broader validation.
|
|
58
|
+
|
|
59
|
+
### Compatibility and limitations
|
|
60
|
+
|
|
61
|
+
- The default audit path starts no Android validation. The opted-in Android path remains static and starts zero Gradle, external-tool, and network processes by default.
|
|
62
|
+
- Manual pentest, runtime/device analysis, APK/AAB inspection, signing verification, and automatic fixes remain out of scope.
|
|
63
|
+
|
|
64
|
+
## [0.4.1] - 2026-07-12
|
|
65
|
+
|
|
66
|
+
Added advanced Android security validation on top of v0.4.0. At publication, v0.4.1 superseded v0.4.0.
|
|
67
|
+
|
|
68
|
+
### Added
|
|
69
|
+
|
|
70
|
+
- Added the advanced Android rule/evidence substrate and eleven internal advanced check families: Network Security Config and cleartext traffic review, backup and data-extraction-rules review, debuggable and release configuration review, redacted hardcoded-secret detection, signing-configuration leak review, WebView and FileProvider analysis, sensitive storage/logging/clipboard analysis, and Firebase and Google services configuration and local rules review.
|
|
71
|
+
- Added opt-in Semgrep, OSV-Scanner, Android Lint, and OWASP Dependency-Check evidence with closed tool IDs, bounded artifacts, and default network denial.
|
|
72
|
+
- Activated nineteen default Android checks (eight `v0.4.0` checks plus eleven `v0.4.1` advanced internal checks) and added explicit `--android-external-tools` and `--android-external-network` CLI options.
|
|
73
|
+
- Added CandidateEvidence presentation plus advanced-security, external-tool, artifact, and candidate summaries to both text and JSON reports, with text/JSON parity.
|
|
74
|
+
|
|
75
|
+
### Compatibility and limitations
|
|
76
|
+
|
|
77
|
+
- Default Android validation starts zero Gradle operations, zero external-tool processes, and zero network operations.
|
|
78
|
+
- Advanced checks remain static-analysis-based and conservative: no APK/AAB inspection, no signing verification, no emulator/device validation, and no Play compliance or manual penetration testing.
|
|
79
|
+
- Optional Semgrep/OSV-Scanner/Android Lint/Dependency-Check evidence depends on external tool availability; missing tools are skipped, not treated as failures.
|
|
80
|
+
- Android AuditIssue mapping remains future `v0.4.2` work.
|
|
81
|
+
|
|
82
|
+
## [0.4.0] - 2026-07-11
|
|
83
|
+
|
|
84
|
+
Android/mobile validation MVP published as `v0.4.0`.
|
|
85
|
+
|
|
86
|
+
### Added
|
|
87
|
+
|
|
88
|
+
- Added the Android/mobile validation substrate and the canonical `security:validate --profile android` path.
|
|
89
|
+
- Added Android project detection and classification (application, library, multi-module, mixed, partial, and non-Android targets) and Compose/XML/mixed/uncertain UI-toolkit classification.
|
|
90
|
+
- Added independent `AndroidManifest.xml` parsing across modules, with permission, exported-component, intent-filter, and deep-link audit families producing conservative `SecurityFinding` evidence.
|
|
91
|
+
- Added static Gradle wrapper and module metadata extraction: namespace, applicationId, versionCode/versionName, minSdk/targetSdk/compileSdk, build-type/source-set metadata, Android Gradle Plugin/Kotlin Android/Compose evidence, and version-catalog evidence.
|
|
92
|
+
- Added an explicitly opt-in, five-operation allowlisted Gradle validation path (`--android-gradle-operations wrapper-version,tasks,assemble-debug,unit-test-debug,lint-debug`) that never executes arbitrary Gradle tasks and is disabled by default.
|
|
93
|
+
- Added Android text and JSON reports under the existing `reports/security/` root, an Android verdict policy, release-metadata summaries, Play-readiness checklist placeholders, and target-mutation evidence.
|
|
94
|
+
- Added `security:validate --help` / `-h` no-work help handling and unknown top-level option rejection.
|
|
95
|
+
|
|
96
|
+
### Compatibility and limitations
|
|
97
|
+
|
|
98
|
+
- `v0.4.0` preserves the existing `security:validate` command surface and report root; it adds the `android` profile and `--android-gradle-operations` option without changing any other profile's behavior.
|
|
99
|
+
- Android manifests are parsed independently and are not merged; selected local references are resolved conservatively while full overlays are not; Gradle files are parsed statically and are not evaluated; dynamic Gradle values may remain unresolved.
|
|
100
|
+
- `v0.4.0` does not inspect APK/AAB contents, validate signing, run emulators/devices, verify Digital Asset Links or domain ownership, check live Google Play policy, upload applications, or provide automatic fixes.
|
|
101
|
+
- `v0.4.1` advanced Android checks and `v0.4.2` Android audit-adapter mapping remain planned and out of scope. Manual pentest remains a deferred, post-v1 / version-TBD workflow, not assigned to v0.4.x.
|
|
102
|
+
|
|
103
|
+
## [0.3.4] - 2026-07-10
|
|
104
|
+
|
|
105
|
+
Published package version 0.3.4, superseding v0.3.3.
|
|
106
|
+
|
|
107
|
+
### Added
|
|
108
|
+
|
|
109
|
+
- Added mixed-language audit stability coverage across the committed Batch 1 fixture corpus and full detector registry.
|
|
110
|
+
- Added repeated-run audit report determinism coverage for JSON/text output, security-summary isolation, and run-order stability.
|
|
111
|
+
- Added focused cross-platform/path and CRLF/LF stability coverage in `tests/audits/crossPlatformPathStability.test.ts`, including separator normalization, paths with spaces, report output safety, security report filename-prefix sanitization, and Java/Kotlin/docs-code-mismatch line-ending robustness.
|
|
112
|
+
|
|
113
|
+
### Changed
|
|
114
|
+
|
|
115
|
+
- Hardened `collectSourceFacts()` to normalize inventory `relativePath` values before reading files or returning source-fact identities, so separator-only path variants do not leak into analyzer inputs, report evidence, or downstream lookups.
|
|
116
|
+
- Reconciled README/current-state/roadmap/architecture/commands/workflows/project-overview/security docs with the published `v0.3.4` state, with `v0.3.3` documented as the previous published baseline.
|
|
117
|
+
|
|
118
|
+
### Compatibility and limitations
|
|
119
|
+
|
|
120
|
+
- `v0.3.4` preserves the existing command surface: it does not change audit/security command names, add new CLI commands, or modify the audit report schema or `SourceFacts` schema.
|
|
121
|
+
- `v0.3.4` keeps `quality`, `project`, and `all` audit types planned and not implemented.
|
|
122
|
+
- `v0.3.4` keeps Android validation, Gradle/Maven execution, Java/Kotlin compiler execution, JVM dependency freshness checks, and JVM package/environment rot detection out of scope.
|
|
123
|
+
|
|
124
|
+
## [0.3.3] - 2026-07-10
|
|
125
|
+
|
|
126
|
+
Java/Kotlin code-rot support, published on top of the previously published `v0.3.2` baseline. Package metadata is `0.3.3`; the npm registry now contains `0.3.3`. `v0.3.2` remains the previous published baseline.
|
|
127
|
+
|
|
128
|
+
### Added
|
|
129
|
+
|
|
130
|
+
- Added dependency-free Java and Kotlin source-facts analyzers (`src/audits/core/javaAnalyzer.ts`, `src/audits/core/kotlinAnalyzer.ts`) and registered them in the language analyzer registry alongside the existing TypeScript/JavaScript and Python analyzers.
|
|
131
|
+
- Added JVM project metadata collection (`src/audits/core/jvmProjectMetadata.ts`) for static Gradle/Maven/wrapper/source-set presence and a best-effort project name. It never executes Gradle, Maven, compilers, or target tests.
|
|
132
|
+
- Added shared JVM source-facts helpers (`src/audits/codeRot/utils/jvmSourceFactsUtils.ts`) reused by Java/Kotlin detector integrations.
|
|
133
|
+
- Added Java/Kotlin source-facts-aware signals to the existing `dead-code-candidate`, `duplicate-implementation-candidate`, and `test-rot` detectors, with conservative JVM-specific evidence wording.
|
|
134
|
+
- Added Java/Kotlin docs-code-mismatch support for backtick-quoted FQCN-shaped symbol claims and static Gradle/Maven command/feature claims. Android current-state mismatch detection remains part of the existing docs-code-mismatch mechanism only.
|
|
135
|
+
- Added focused Java/Kotlin analyzer, JVM metadata, detector, and mixed-language integration coverage under `tests/audits/`.
|
|
136
|
+
|
|
137
|
+
### Changed
|
|
138
|
+
|
|
139
|
+
- Preserved the existing audit command surface: `npm run audit -- --types code-rot`, `--types security`, and `--types code-rot,security` remain the implemented audit paths, with `code-rot` still the only default audit type.
|
|
140
|
+
- Preserved the existing audit report schema: no `SourceFacts` schema change, no new top-level audit report field, and no new audit type were added for `v0.3.3`.
|
|
141
|
+
|
|
142
|
+
### Compatibility and limitations
|
|
143
|
+
|
|
144
|
+
- Java/Kotlin analysis is conservative and scanner-based. It does not perform compiler parsing, type resolution, classpath/symbol resolution, runtime reachability analysis, or target-project test execution.
|
|
145
|
+
- Java/Kotlin support does not execute Gradle, Maven, Android tooling, or network-backed dependency checks. Static JVM metadata is presence/simple-text-extraction only.
|
|
146
|
+
- Java/Kotlin field/property extraction, Java text-block special handling, Kotlin extension-function receiver semantics, and method/constructor-level dead-code detection are not implemented.
|
|
147
|
+
- JVM package/environment rot remains deferred because the existing `dependency-environment-rot` detector is still npm/Node-specific.
|
|
148
|
+
- `security:validate` remains preserved and unchanged from `v0.3.2`.
|
|
149
|
+
- `quality`, `project`, and `all` audit types remain planned; `--types quality`/`project`/`all` still fail cleanly with exit code 2.
|
|
150
|
+
- Android validation remains planned and out of scope for `v0.3.3`.
|
|
151
|
+
|
|
152
|
+
## [0.3.2] - 2026-07-09
|
|
153
|
+
|
|
154
|
+
Python code-rot support and a first security-validation audit adapter, implemented on top of the published `v0.3.1` baseline. Published.
|
|
155
|
+
|
|
156
|
+
### Added
|
|
157
|
+
|
|
158
|
+
- Added a dependency-free Python source-facts analyzer (`src/audits/core/pythonAnalyzer.ts`) that extracts imports (including relative dotted imports), `__all__`, and module-level/class-body declarations from `.py` files, registered in the language analyzer registry alongside the existing TypeScript/JavaScript analyzer.
|
|
159
|
+
- Added Python project/config metadata collection (`src/audits/core/pythonProjectMetadata.ts`): presence detection for `pyproject.toml`, `requirements.txt`, `setup.py`, `setup.cfg`, `tox.ini`, and `pytest.ini`, plus a best-effort project name and pytest-configuration flag. Never executes Python/pip/pytest/tox tooling.
|
|
160
|
+
- Added Python-aware signals to the existing `dead-code-candidate`, `duplicate-implementation-candidate`, and `test-rot` code-rot detectors, using an analyzer-id-scoped grouping key so the pre-existing TypeScript/JavaScript duplicate-declaration grouping is unaffected.
|
|
161
|
+
- Added `sourceFacts.filesWithDiagnosticsCount` to the audit report's source-facts summary — a language-agnostic count of analyzed files carrying at least one per-file diagnostic.
|
|
162
|
+
- Added `pythonProjectMetadata` as a top-level audit report field (JSON and text), populated unconditionally alongside `sourceFacts`.
|
|
163
|
+
- Added the security-validation audit adapter (`src/audits/security/securityAuditAdapter.ts`, `mapSecurityFindingToAuditIssue.ts`, `securityAuditTypes.ts`): `npm run audit -- --types security` and `npm run audit -- --types code-rot,security` now run security validation through the shared audit/report surface by calling `runSecurityValidation()` directly (no shelling out to `security:validate`, no console-text parsing).
|
|
164
|
+
- Added `securitySummary` as a top-level audit report field (JSON and text): verdict, check counts by status, finding counts by severity, mapped-issue count, and paths to the original `reports/security/*.txt`/`*.json` report.
|
|
165
|
+
- Added mapping of `SecurityFinding` results into audit-issue-shaped entries (`auditType: "security"`), preserving severity (blocker/major/minor/informational → blocker/high/medium/info), affected files, recommendation text, and release/implementation-blocking semantics. Skipped optional security checks never appear as issues or as a passed result.
|
|
166
|
+
- `security` is now an implemented `--types` value (`IMPLEMENTED_AUDIT_TYPES`), deliberately kept out of `DEFAULT_AUDIT_TYPES` so a default, no-flag `npm run audit` run is unchanged (still `code-rot` only).
|
|
167
|
+
|
|
168
|
+
### Changed
|
|
169
|
+
|
|
170
|
+
- Extracted the `SecurityReport`-object-assembly step out of `scripts/security/validate.ts` into `src/securityValidation/report/buildSecurityReport.ts` so the audit adapter can reuse it without duplicating logic. Output of `security:validate` is unchanged.
|
|
171
|
+
- Widened `reportFilenamePrefix()`'s parameter type to a narrow `Pick<SecurityValidationTarget, ...>` so the audit adapter can call it without re-resolving the validation target a second time. All existing callers are unaffected.
|
|
172
|
+
|
|
173
|
+
### Compatibility and limitations
|
|
174
|
+
|
|
175
|
+
- `npm run security:validate` (self and `--target <path>`) is unchanged — verified against the full existing security test suite plus a live smoke run.
|
|
176
|
+
- `npm run audit -- --types code-rot` is unchanged.
|
|
177
|
+
- The audit report's top-level schema grows from 14 fields (`v0.3.1`) to 16 fields: `pythonProjectMetadata` and `securitySummary` are additive; `schemaVersion` stays `"1.0"`.
|
|
178
|
+
- Python static analysis remains conservative: regex/line-based, dependency-free, single-file. It does not perform Python runtime execution, type checking, dependency resolution, or cross-file call-graph analysis. `references` are intentionally left empty for Python (no call/identifier tracking).
|
|
179
|
+
- Java and Kotlin remain fallback-only; no analyzer is registered for them.
|
|
180
|
+
- `quality`, `project`, and `all` audit types remain planned; `--types quality`/`project`/`all` still fail cleanly with exit code 2.
|
|
181
|
+
- The security audit adapter runs the same default check groups `security:validate` uses with no flags (`deps`, `package`, `static`, `cli-adversarial`, `fuzz`); there is no `--checks`/`--profile` passthrough on the `audit` command.
|
|
182
|
+
|
|
183
|
+
## [0.3.1] - 2026-07-09
|
|
184
|
+
|
|
185
|
+
Language-aware code-rot substrate plus TypeScript/JavaScript support. Published.
|
|
186
|
+
|
|
187
|
+
### Added
|
|
188
|
+
|
|
189
|
+
- Added normalized language and file-role metadata to the audit inventory scanner, including `filesByLanguage` and `filesByRole` report summaries.
|
|
190
|
+
- Added the source facts model, source facts collector, language analyzer registry, and fallback behavior for eligible source/test files without a registered analyzer.
|
|
191
|
+
- Added the syntax-only TypeScript/JavaScript analyzer for `.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, and `.cjs` files.
|
|
192
|
+
- Added source-facts-aware code-rot signals:
|
|
193
|
+
- `dead-code-candidate` merges parsed relative import/re-export basenames into reverse-reference checks.
|
|
194
|
+
- `duplicate-implementation-candidate` adds source-facts-derived duplicate exported declaration candidate signals.
|
|
195
|
+
- `test-rot` uses analyzer-recorded relative imports, including dynamic `import()`, to find missing targets missed by regex-only scanning.
|
|
196
|
+
- Added source-facts summaries to JSON and text audit reports. JSON reports include top-level `sourceFacts`; text reports include a `Source facts` section.
|
|
197
|
+
- Added end-to-end and focused tests for source facts collection, TypeScript/JavaScript analyzer behavior, detector integration, report model/rendering, and the real audit registry path.
|
|
198
|
+
- Added the latest-Node cross-platform readiness workflow for `windows-latest`, `macos-latest`, and `ubuntu-latest` on Node `26.5.0`.
|
|
199
|
+
|
|
200
|
+
### Fixed
|
|
201
|
+
|
|
202
|
+
- Fixed `.mts` and `.cts` source-role eligibility for source-facts collection.
|
|
203
|
+
- Fixed audit text rendering so an evidence entry with both `message` and `excerpt` renders both.
|
|
204
|
+
- Fixed the audit validation reliability issue by isolating the expensive `security:validate` smoke path in the audit command tests and pinning Vitest worker settings with `vitest.config.ts`.
|
|
205
|
+
|
|
206
|
+
### Compatibility and limitations
|
|
207
|
+
|
|
208
|
+
- No audit command flags were added or changed.
|
|
209
|
+
- Audit issue shape remains unchanged.
|
|
210
|
+
- Python, Java, and Kotlin remain fallback-only in `v0.3.1`; no parser/analyzer is registered for them.
|
|
211
|
+
- TypeScript/JavaScript analysis is syntax-only and single-file. It does not perform TypeScript Program semantic analysis, type checking, full module resolution, `tsconfig` path alias resolution, coverage analysis, clone detection, runtime reachability analysis, or target-file modification.
|
|
212
|
+
- Audit findings remain heuristic and conservative; they are candidates for review, not proof of a defect.
|
|
213
|
+
|
|
214
|
+
## [0.3.0] - Published
|
|
215
|
+
|
|
216
|
+
Generic audit framework and code-rot detector. Published; superseded by `v0.3.1` as the current published baseline.
|
|
217
|
+
|
|
218
|
+
### Added
|
|
219
|
+
|
|
220
|
+
- Added `npm run audit` command (`scripts/audits/runAudit.ts`) as the audit framework CLI entrypoint.
|
|
221
|
+
- Added the generic audit framework in `src/audits`: target resolution, config/flag parsing (`--target`, `--types`, `--include`, `--format`, `--fail-on`, `--out`), detector registry, and exit-code policy (`0`/`1`/`2`).
|
|
222
|
+
- Added a project inventory scanner and a source-of-truth collector so detectors read pre-collected, consistent project state instead of re-scanning.
|
|
223
|
+
- Added the code-rot detector family: `stale-command-reference`, `docs-code-mismatch`, `package-release-rot`, `duplicate-implementation-candidate`, `dead-code-candidate`, `test-rot`, `architecture-drift`, `dependency-environment-rot`, `cross-platform-rot`, `security-validation-assumption-rot` (10 detectors).
|
|
224
|
+
- Added a stable, versioned audit report schema (`schemaVersion` `"1.0"`, 13 top-level fields) with text and JSON renderers and a sanitized text-report writer.
|
|
225
|
+
- Added fail-on severity policy (`blocker`, `high`, `medium`, `low`, `none`) shared between the runner and the report model.
|
|
226
|
+
- Added external-target-safe report writing: audits against a local target project do not modify target files; generated reports stay under the tool root's `reports/audits/` unless `--out` redirects them.
|
|
227
|
+
- Added integration hardening and regression test coverage under `tests/audits/` and `tests/audits/codeRot/` (CLI options, target resolution, inventory, source-of-truth, detectors, report schema/writer, fail-on behavior, external targets, self-audit smoke, detector-error hardening).
|
|
228
|
+
- Kept the audit framework and `security:validate` independent: `npm run audit` does not call `security:validate`, and `security:validate` does not call the audit framework.
|
|
229
|
+
- Validated cross-platform behavior on the latest Node.js LTS-adjacent release (v26.4.0) across Windows, Linux, and macOS.
|
|
230
|
+
|
|
231
|
+
### Fixed
|
|
232
|
+
|
|
233
|
+
- Fixed a test flake in `securityProfileSelectionIntegration.test.ts`.
|
|
234
|
+
- Fixed a cross-platform bug in `resolveCommand` (`src/core`): the POSIX branch never checked `PATH` for a matching executable, so `security:validate` could falsely report a blocker on Linux/macOS when optional static-analysis tools (e.g. Semgrep) were absent from expected locations but present on `PATH`.
|
|
235
|
+
|
|
236
|
+
### Limitations
|
|
237
|
+
|
|
238
|
+
- Only the `code-rot` audit type is implemented. `quality`, `security`, `project`, and `all` audit types are recognized as valid `--types` identifiers but are rejected with a clear message and exit code `2` rather than running.
|
|
239
|
+
- Audit findings are heuristic and conservative; they are candidates for review, not proof of a defect.
|
|
240
|
+
|
|
241
|
+
## [0.2.2] - 2026-07-07
|
|
242
|
+
|
|
243
|
+
### Added
|
|
244
|
+
|
|
245
|
+
- Implemented the `v0.2.2` automated security-validation fortification work; version bumped and release-preparation validation completed.
|
|
246
|
+
- Added `security:validate` support for `--checks`, `--profile`, `--format`, `--fail-on`, and `--out` while preserving backward-compatible no-flag and `--target` behavior.
|
|
247
|
+
- Added the attack-scenario framework, reusable security profiles, payload/evidence models, and concrete boundary, subprocess, secrets, and network scenarios.
|
|
248
|
+
- Added fail-on threshold behavior, profile-aware default check selection, scoped-run reporting, and verdict-reason summaries.
|
|
249
|
+
- Added metadata-driven `verdictImpact` categorization and `reportSchemaGuard` baseline-diff structural-injection protection.
|
|
250
|
+
- Added schema/report hardening, text-report sanitization regression coverage, and output format/location consistency validation.
|
|
251
|
+
|
|
252
|
+
### Fixed
|
|
253
|
+
|
|
254
|
+
- Fixed a cross-platform bug in the path-traversal attack scenario where a backslash-style traversal payload was incorrectly expected to be rejected on POSIX platforms (Linux/macOS), where `\` is a literal filename character rather than a path separator. Verified via the project's own CI matrix (ubuntu-latest/macos-latest/windows-latest x Node 20/22).
|
|
255
|
+
|
|
256
|
+
### Documentation
|
|
257
|
+
|
|
258
|
+
- Synchronized current-state, architecture, command, security-validation, and project-overview documentation with the `v0.2.2` implementation and package version.
|
|
259
|
+
- Replaced the stale roadmap with a semantically ordered plan from v0.2.1 through v1.4.0, placing stable v1.0.0 after all prerequisite v0.x work.
|
|
260
|
+
- Clarified that automated security validation is implemented, while the generic audit framework, code rot and quality detectors, unified audits, and manual pentest framework are planned.
|
|
261
|
+
|
|
262
|
+
## [0.2.1] - 2026-06-25
|
|
263
|
+
|
|
264
|
+
Target-aware security-validation correction.
|
|
265
|
+
|
|
266
|
+
### Fixed
|
|
267
|
+
|
|
268
|
+
- Corrected external-target `test:security` execution so it runs in the selected target project, including installed-package validation paths.
|
|
269
|
+
|
|
270
|
+
## [0.2.0] - 2026-06-23
|
|
271
|
+
|
|
272
|
+
Generic experiment-plugin foundation, first plugin conversion, target-aware execution, plugin-aware reports, and user-facing command surface.
|
|
273
|
+
|
|
274
|
+
### Added
|
|
275
|
+
|
|
276
|
+
- Added generic experiment plugin contracts, registry, execution context, target model, config validation result, and normalized result model.
|
|
277
|
+
- Added shared local target metadata resolution used by both experiment targets and security validation targets.
|
|
278
|
+
- Added `context-strategy-comparison` as the first registered experiment plugin for the existing raw-full-file vs my-dev-kit-guided workflow.
|
|
279
|
+
- Added plugin result metadata output at `experiment-plugin-result.json` while preserving the existing `experiment-summary.json`, `experiment-runs.json`, and `experiment-comparisons.json` artifacts.
|
|
280
|
+
- Added target-aware experiment execution for explicit local projects via `experiment:run -- --target <path>`.
|
|
281
|
+
- Added plugin-aware JSON and HTML reports with plugin, target, variant, case, metric, artifact, warning, skip, and failure metadata.
|
|
282
|
+
- Added `experiment:list`.
|
|
283
|
+
- Added `experiment:describe`.
|
|
284
|
+
- Added `experiment:run`.
|
|
285
|
+
- Documented `context-strategy-comparison` as the first experiment plugin.
|
|
286
|
+
|
|
287
|
+
### Changed
|
|
288
|
+
|
|
289
|
+
- Routed the existing controlled experiment command wrapper through the `context-strategy-comparison` plugin without changing the public command arguments or legacy artifact behavior.
|
|
290
|
+
- Preserved existing raw-vs-indexed workflow behavior through the plugin runner and kept old controlled-experiment commands available.
|
|
291
|
+
|
|
292
|
+
### Validation
|
|
293
|
+
|
|
294
|
+
- `npm install`
|
|
295
|
+
- `npm run build`
|
|
296
|
+
- `npm run test`
|
|
297
|
+
- `npm run verify`
|
|
298
|
+
- `npm run security:deps`
|
|
299
|
+
- `npm run security:package`
|
|
300
|
+
- `npm run security:codeql`
|
|
301
|
+
- `npm run security:semgrep`
|
|
302
|
+
- `npm run test:security`
|
|
303
|
+
- `npm run test:fuzz:smoke`
|
|
304
|
+
- `npm run security:validate`
|
|
305
|
+
- `npm run security:validate -- --target "Z:\Users\newuser\Projects\my-dev-kit-v1"`
|
|
306
|
+
- `npm run security:validate -- --target "Z:\Users\newuser\Projects\scientific-literature-explorer-v1"`
|
|
307
|
+
- `npm run experiment:list`
|
|
308
|
+
- `npm run experiment:describe -- --experiment context-strategy-comparison`
|
|
309
|
+
- `npm run experiment:describe -- --experiment does-not-exist`
|
|
310
|
+
- `npm run experiment:run -- --experiment context-strategy-comparison --target "Z:\Users\newuser\Projects\my-dev-kit-v1" --case todo-ts-create-task --agents fake-agent --complexities short --no-screenshot`
|
|
311
|
+
- `npm run experiment:run -- --experiment context-strategy-comparison --target "Z:\Users\newuser\Projects\scientific-literature-explorer-v1" --case todo-ts-create-task --agents fake-agent --complexities short --no-screenshot`
|
|
312
|
+
- `npm run run-controlled-experiment -- --cases examples/token-savings-cases.json --out lab-output/legacy-v0.2.0-smoke --kit-command "node tests/fixtures/fake-my-dev-kit-cli.js" --agents fake-agent --complexities short --no-screenshot`
|
|
313
|
+
- `npm pack --dry-run`
|
|
314
|
+
|
|
315
|
+
## [0.1.4] - 2026-06-22
|
|
316
|
+
|
|
317
|
+
Final security-validation gate with reusable multi-project target support.
|
|
318
|
+
|
|
319
|
+
### Added (reusable target support)
|
|
320
|
+
|
|
321
|
+
- Added `--target <path>` option to `security:validate`, `security:deps`, `security:package`, `security:semgrep`, and `security:codeql` to validate any local project, not just my-dev-kit-lab itself.
|
|
322
|
+
- Added `SecurityValidationTarget` type separating the validation target from the tool root (my-dev-kit-lab).
|
|
323
|
+
- Added `resolveValidationTarget(targetPath?, toolRoot)` — validates target path, reads package.json/lockfile/git metadata gracefully; returns null fields on missing metadata rather than crashing.
|
|
324
|
+
- Added `reportFilenamePrefix(target)` — generates collision-free filename prefixes: `v0.1.4` for self, `my-dev-kit-v1.2.0` for scoped packages, `biolit-v1` for name-only packages, directory basename for projects with no package.json.
|
|
325
|
+
- Added `targetDescription(target)` — human-readable label for report headers.
|
|
326
|
+
- Added `isSelf` flag for backward-compatible self-validation when no `--target` is given.
|
|
327
|
+
- Added `--out <dir>` option to control report output directory (default: `reports/security/`).
|
|
328
|
+
- Added `--report-prefix <name>` option to override the generated filename prefix.
|
|
329
|
+
- Report headers now distinguish self-validation (`Package: ...`) from external target validation (`Tool: ..., Target: ...`).
|
|
330
|
+
- CLI adversarial tests and fuzz smoke always run against the tool root and are labeled clearly as "tool self-tests, not target-specific."
|
|
331
|
+
- Added 23 new unit tests for target resolution in `tests/security/securityValidationTarget.test.ts`.
|
|
332
|
+
- Exported `resolveValidationTarget`, `reportFilenamePrefix`, `targetDescription`, `SecurityValidationTarget` from `src/securityValidation/index.ts`.
|
|
333
|
+
|
|
334
|
+
### Added (security-validation gate)
|
|
335
|
+
|
|
336
|
+
- Added static scan integration for CodeQL and Semgrep.
|
|
337
|
+
- CodeQL: local CLI availability check; full analysis delegates to GitHub Actions.
|
|
338
|
+
- Semgrep: local or npx fallback; `.semgrep.yml` covers subprocess safety, path traversal, unsafe fs.rm, and secret leakage.
|
|
339
|
+
- Both scanners return structured `skipped` results when unavailable — never crash.
|
|
340
|
+
- Added bounded fuzz smoke tests (`test:fuzz:smoke`) for 9 security-sensitive parsers and helpers.
|
|
341
|
+
- Seeded PRNG (`0xDEADBEEF`) for deterministic CI reproduction.
|
|
342
|
+
- Completes in under 1 second at default settings (50 iterations per target).
|
|
343
|
+
- Added `security:validate` release-gate command that orchestrates all checks and produces a verdict.
|
|
344
|
+
- Added text and JSON security-validation report generation to `reports/security/<prefix>-security-validation.{txt,json}`.
|
|
345
|
+
- Added release verdict calculation from normalized check results (four verdict categories).
|
|
346
|
+
- Added documentation for mandatory checks, optional scanner availability, findings, skipped checks, and release verdicts in `docs/COMMANDS.md`.
|
|
347
|
+
- Fixed defensive null guard in `parseNpmOutdated` when JSON value entries contain null.
|
|
348
|
+
|
|
349
|
+
### Validation
|
|
350
|
+
|
|
351
|
+
- `npm run build`
|
|
352
|
+
- `npm run test` (407 tests, 88 files)
|
|
353
|
+
- `npm run verify`
|
|
354
|
+
- `npm run security:deps`
|
|
355
|
+
- `npm run security:package`
|
|
356
|
+
- `npm run security:codeql`
|
|
357
|
+
- `npm run security:semgrep`
|
|
358
|
+
- `npm run test:security` (165 tests, 12 files)
|
|
359
|
+
- `npm run test:fuzz:smoke` (9 targets, 450 iterations)
|
|
360
|
+
- `npm run security:validate` (verdict: ready except optional manual checks)
|
|
361
|
+
- `npm pack --dry-run`
|
|
362
|
+
|
|
363
|
+
### Notes
|
|
364
|
+
|
|
365
|
+
- CodeQL, Semgrep, and OSV-Scanner may be skipped locally if unavailable. Absence is reported as `ready except optional manual checks`, not a blocker.
|
|
366
|
+
- Generated security reports are not committed by default (`.gitignore` excludes `reports/security/` outputs).
|
|
367
|
+
- This release completes the initial security-validation release-gate track.
|
|
368
|
+
|
|
369
|
+
## [0.1.2] - 2026-06-22
|
|
370
|
+
|
|
371
|
+
Security validation foundation and package checks.
|
|
372
|
+
|
|
373
|
+
### Added
|
|
374
|
+
|
|
375
|
+
- Added security-validation planning and framework documentation for the staged release-security track.
|
|
376
|
+
- Added security validation core types, configuration, command runner, and test matrix scaffolding.
|
|
377
|
+
- Added dependency validation through `security:deps`.
|
|
378
|
+
- Added package-content validation through `security:package`.
|
|
379
|
+
- Added initial security unit coverage through `test:security`.
|
|
380
|
+
- Added Windows-safe npm command resolution for security scripts.
|
|
381
|
+
- Added parsers and runners for `npm audit`, `npm audit --omit=dev`, `npm ls --all`, `npm outdated`, and optional OSV-Scanner execution.
|
|
382
|
+
- Added forbidden package-content detection and structured `npm pack --dry-run` parsing.
|
|
383
|
+
|
|
384
|
+
### Validation
|
|
385
|
+
|
|
386
|
+
- `npm run build`
|
|
387
|
+
- `npm run test`
|
|
388
|
+
- `npm run verify`
|
|
389
|
+
- `npm run security:deps`
|
|
390
|
+
- `npm run security:package`
|
|
391
|
+
- `npm pack --dry-run`
|
|
392
|
+
|
|
393
|
+
### Limitations
|
|
394
|
+
|
|
395
|
+
- full CLI adversarial harness is planned next
|
|
396
|
+
- malformed artifact, JSON stdout/stderr, and Graphviz safety tests are not complete yet
|
|
397
|
+
- CodeQL and Semgrep integration is planned
|
|
398
|
+
- fuzz smoke tests are planned
|
|
399
|
+
- `security:validate` and final release security report generation are planned
|
|
400
|
+
|
|
401
|
+
## [0.1.1] - 2026-06-22
|
|
402
|
+
|
|
403
|
+
Public release hygiene and cross-platform assurance patch.
|
|
404
|
+
|
|
405
|
+
### Changed
|
|
406
|
+
|
|
407
|
+
- cleaned package metadata for the public npm release, including repository, funding, keyword, engine, and bin metadata
|
|
408
|
+
- restricted published package contents to runtime assets and the deterministic fake my-dev-kit fixture used by local demos
|
|
409
|
+
- improved cross-platform command parsing and Windows `.cmd` shim execution, especially for commands and script paths that contain spaces
|
|
410
|
+
- added cross-platform CI coverage for `windows-latest`, `macos-latest`, and `ubuntu-latest`
|
|
411
|
+
- updated quickstart and workflow docs with PowerShell guidance and clearer notes about current baseline capabilities versus future roadmap items
|
|
412
|
+
|
|
413
|
+
### Validation
|
|
414
|
+
|
|
415
|
+
- `npm run build`
|
|
416
|
+
- `npm run test`
|
|
417
|
+
- `npm run verify`
|
|
418
|
+
- `npm pack --dry-run`
|
|
419
|
+
|
|
420
|
+
### Notes
|
|
421
|
+
|
|
422
|
+
- this patch does not add the full security-validation framework
|
|
423
|
+
- generic experiment-plugin architecture remains planned for a later release
|
|
424
|
+
|
|
425
|
+
## [0.1.0] - 2026-06-16
|
|
426
|
+
|
|
427
|
+
### Initial public baseline
|
|
428
|
+
|
|
429
|
+
**Experiment pipeline**
|
|
430
|
+
- Raw-vs-indexed context experiment: compares `raw-full-file` and `my-dev-kit-guided` strategies across benchmark cases
|
|
431
|
+
- Controlled experiment runner with paired run comparison by case, project, agent, and prompt complexity
|
|
432
|
+
- Deterministic correctness scoring from benchmark answer keys (not semantic LLM judging)
|
|
433
|
+
- Token usage, duration, and status comparisons between matched strategy pairs
|
|
434
|
+
|
|
435
|
+
**Benchmark projects**
|
|
436
|
+
- `todo-ts`, `todo-js`, `todo-python`, `todo-mixed-ts-py` — small baseline projects
|
|
437
|
+
- `task-workflow-medium-ts` — medium TypeScript project
|
|
438
|
+
- `task-analytics-large-mixed` — large mixed TypeScript/Python project
|
|
439
|
+
- Project complexity metrics: file count, lines of code, language count, import count, exported symbol estimate, complexity score (0–100 weighted)
|
|
440
|
+
|
|
441
|
+
**Agent adapters**
|
|
442
|
+
- Fake-agent adapter for deterministic smoke and demo validation (no external CLI required)
|
|
443
|
+
- Codex adapter for real-agent campaigns (requires local Codex CLI and usage capacity)
|
|
444
|
+
- Claude adapter for real-agent campaigns (requires local Claude CLI; token totals not exposed)
|
|
445
|
+
- Windows-safe CLI command resolution
|
|
446
|
+
|
|
447
|
+
**Report, plot, screenshot, and gallery pipeline**
|
|
448
|
+
- HTML experiment report rendering with project profile, strategy comparisons, correctness, token savings, and warnings
|
|
449
|
+
- Static SVG plot generation
|
|
450
|
+
- Optional PNG screenshot capture via headless browser
|
|
451
|
+
- Gallery manifest and static gallery index output
|
|
452
|
+
|
|
453
|
+
**Documentation**
|
|
454
|
+
- Full user-facing documentation: README, PROJECT_OVERVIEW, ARCHITECTURE, WORKFLOWS, COMMANDS, TUTORIAL, METRICS, ROADMAP, GALLERY, CURRENT_STATE
|
|
455
|
+
- Metric glossary (docs/METRICS.md)
|
|
456
|
+
- Real-agent campaign report and final batch handoff notes
|
|
457
|
+
|
|
458
|
+
### Known limitations
|
|
459
|
+
|
|
460
|
+
- Token savings in fake-agent runs are estimated from character counts, not provider billing telemetry
|
|
461
|
+
- Claude does not expose token totals; token savings comparisons are unavailable for Claude runs
|
|
462
|
+
- Codex runs may produce timeouts, invalid output, or hit session limits
|
|
463
|
+
- Small projects may show negative token savings because raw-full-file is cheaper when the entire project fits in context
|
|
464
|
+
- Current baseline does not yet prove every future value claim; stronger evidence requires future experiment types
|
|
465
|
+
|
|
466
|
+
### Roadmap at release time
|
|
467
|
+
|
|
468
|
+
At v0.1.0 the generic experiment-plugin framework was planned. It was subsequently implemented in v0.2.0 with `context-strategy-comparison` as the first plugin. Later planned plugins cover warm-index reuse, incremental-change staleness, context-window scaling, retrieval precision/recall, and agent-success-rate experiments.
|
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@ my-dev-kit-lab is the experiment, audit, and evidence companion for [my-dev-kit]
|
|
|
4
4
|
|
|
5
5
|
my-dev-kit provides local repository indexing and graph-guided retrieval. my-dev-kit-lab supplies the controlled benchmarks, agent adapters, metrics, security checks, and reports needed to evaluate when that retrieval is useful. Results are evidence for a specific target and configuration; they do not guarantee token savings or security.
|
|
6
6
|
|
|
7
|
-
The latest published release is v0.4.
|
|
7
|
+
The latest published release is v0.4.5 (context-integrity validation), compatible with published `@dailephd/my-dev-kit@1.10.4` and `@dailephd/my-dev-kit-orchestrator@1.2.3`. See [docs/CURRENT_STATE.md](docs/CURRENT_STATE.md).
|
|
8
8
|
|
|
9
9
|
## Current capabilities
|
|
10
10
|
|
|
@@ -15,6 +15,7 @@ The latest published release is v0.4.4 (producer-readiness bridge). See [docs/CU
|
|
|
15
15
|
- **Review evidence:** generate JSON and HTML reports, SVG plots, optional screenshots, visualization demos, and a static gallery.
|
|
16
16
|
- **Evaluate stage-context strategies:** compare the two legacy strategies against six additional bounded stage-context strategies — `architecture-context-only`, `architecture-plus-implementation-refresh`, `architecture-plus-implementation-and-test-refresh`, `full-workflow-library`, `bounded-workflow-instruction-packet`, and `combined-bounded-stage-context` — selected through programmatic configuration, not CLI flags. Each strategy's evidence is reported through bounded `report.json`, `report.html`, and `report.txt` output with an explicit `available`/`unavailable`/`not-applicable` metric-availability model and no composite score, grade, ranking, or winning strategy.
|
|
17
17
|
- **Evaluate the producer-readiness bridge (v0.4.4):** optionally extend `combined-bounded-stage-context` with the frozen my-dev-kit-orchestrator supplemental implementation/test-context packet and retrieval-report documents plus an observed readiness result, all supplied programmatically (there is no CLI flag), to measure owner, allocation, truncation-cause, supplemental/raw agreement, readiness-agreement, and criticality-overlay evidence without reimplementing upstream owner-selection, allocation, producer-parity, or readiness policy.
|
|
18
|
+
- **Evaluate context integrity (v0.4.5):** compare condition-aware producer evidence from my-dev-kit v1.10.4 (role condition coverage, allocation/spillover, required-evidence-loss) against my-dev-kit-orchestrator v1.2.3 run-integrity evidence (run-integrity gate, judge integrity, final-report eligibility, artifact lifecycle state), reporting agreement or contradiction between them rather than re-deriving a verdict. Evaluation runs against a frozen, hash-verified regression fixture pair — a byte-exact real historical failed run and a hand-distilled corrected-replay counterpart representing the same validated contracts — and is programmatic/test-driven only. `npm run report:context-integrity-smoke` renders both fixtures' reports for manual inspection; it takes no arguments and is a developer convenience, not a configurable evaluation CLI.
|
|
18
19
|
|
|
19
20
|
## Quickstart
|
|
20
21
|
|
|
@@ -167,6 +168,7 @@ See [docs/METRICS.md](docs/METRICS.md) for full metric definitions.
|
|
|
167
168
|
- The six new stage-context strategies have no CLI flags yet, are configured programmatically, and do not yet include plots, screenshots, or gallery integration
|
|
168
169
|
- The published upstream artifacts the stage-context strategies read do not expose considered-but-unselected reads or unnecessary-read evidence; those metrics report `unavailable` rather than zero
|
|
169
170
|
- The v0.4.4 producer-readiness bridge is released. All bridge inputs are programmatic — there is no CLI flag. The coordinated upstream releases my-dev-kit@1.10.3 and orchestrator@1.2.2 were verified published before lab publication.
|
|
171
|
+
- The v0.4.5 context-integrity evaluation is released. It has no CLI flags, no plots/screenshot/gallery integration, and produces no composite score, grade, ranking, or winner. Its corrected-replay fixture is a hand-distilled representation of the validated my-dev-kit v1.10.4 and my-dev-kit-orchestrator v1.2.3 contracts, not a live capture of a complete ten-stage workflow. The orchestrator agreement evidence does not read a literal upstream `promptMode` field; `stageMayRenderNormalPrompt`, derived from structured blocked-stage evidence, is the bounded substitute used instead.
|
|
170
172
|
|
|
171
173
|
---
|
|
172
174
|
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// v0.4.5 context-integrity evaluation: report smoke script. Loads the frozen failed-run and
|
|
2
|
+
// corrected-replay fixtures, evaluates them through the existing producer-readiness bridge,
|
|
3
|
+
// and writes JSON/text/HTML reports for manual inspection. Not part of any release-readiness
|
|
4
|
+
// gate; a developer convenience for inspecting ContextIntegrityReportV1 output.
|
|
5
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { loadContextIntegrityFixture } from "../src/evaluation/ecosystemFixtures/loadContextIntegrityFixture.js";
|
|
7
|
+
import { evaluateProducerReadinessBridge } from "../src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js";
|
|
8
|
+
import { calculateStageContextDeterminism } from "../src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js";
|
|
9
|
+
import { buildContextIntegrityReport } from "../src/report/experiments/buildContextIntegrityReport.js";
|
|
10
|
+
import { renderContextIntegrityJsonReport } from "../src/report/experiments/renderContextIntegrityJsonReport.js";
|
|
11
|
+
import { renderContextIntegrityText } from "../src/report/experiments/renderContextIntegrityText.js";
|
|
12
|
+
import { renderContextIntegrityHtml } from "../src/report/experiments/renderContextIntegrityHtml.js";
|
|
13
|
+
const CASES = [
|
|
14
|
+
{
|
|
15
|
+
kind: "failed-run",
|
|
16
|
+
root: "tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run",
|
|
17
|
+
manifest: "tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/failed-run-manifest.json"
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
kind: "corrected-replay",
|
|
21
|
+
root: "tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay",
|
|
22
|
+
manifest: "tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/corrected-replay-manifest.json"
|
|
23
|
+
}
|
|
24
|
+
];
|
|
25
|
+
async function main() {
|
|
26
|
+
const outDir = "lab-output/context-integrity-report-smoke";
|
|
27
|
+
mkdirSync(outDir, { recursive: true });
|
|
28
|
+
for (const c of CASES) {
|
|
29
|
+
const fixture = await loadContextIntegrityFixture(c.root, c.manifest);
|
|
30
|
+
if (!fixture.ok) {
|
|
31
|
+
console.error(`Fixture load failed for ${c.kind}:`, fixture.issues);
|
|
32
|
+
process.exitCode = 1;
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
const bridgeResult = evaluateProducerReadinessBridge({
|
|
36
|
+
implementation: { role: "implementation", contextCapsule: fixture.implementationCapsule, retrievalAuditRecord: fixture.implementationAudit },
|
|
37
|
+
readiness: fixture.runIntegrityEvidence?.gate.implementationContext,
|
|
38
|
+
runIntegrityEvidence: fixture.runIntegrityEvidence,
|
|
39
|
+
expectations: {
|
|
40
|
+
schemaVersion: "1.0.0",
|
|
41
|
+
caseId: "CASE-SMOKE",
|
|
42
|
+
title: "smoke",
|
|
43
|
+
description: "smoke",
|
|
44
|
+
expectedEvidence: [],
|
|
45
|
+
expectedStates: {},
|
|
46
|
+
warnings: [],
|
|
47
|
+
producerReadinessExpectations: {}
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
const determinism = calculateStageContextDeterminism([
|
|
51
|
+
{ runNumber: 1, value: bridgeResult },
|
|
52
|
+
{ runNumber: 2, value: bridgeResult }
|
|
53
|
+
]);
|
|
54
|
+
const report = buildContextIntegrityReport({
|
|
55
|
+
fixtureKind: c.kind,
|
|
56
|
+
manifest: fixture.manifest,
|
|
57
|
+
hashVerification: fixture.hashVerification,
|
|
58
|
+
capsule: fixture.implementationCapsule,
|
|
59
|
+
audit: fixture.implementationAudit,
|
|
60
|
+
bridgeResult,
|
|
61
|
+
determinism,
|
|
62
|
+
fixtureSelfImmutable: true
|
|
63
|
+
});
|
|
64
|
+
writeFileSync(`${outDir}/${c.kind}.report.json`, renderContextIntegrityJsonReport(report), "utf8");
|
|
65
|
+
writeFileSync(`${outDir}/${c.kind}.report.txt`, renderContextIntegrityText(report), "utf8");
|
|
66
|
+
writeFileSync(`${outDir}/${c.kind}.report.html`, renderContextIntegrityHtml(report), "utf8");
|
|
67
|
+
console.log(`Wrote ${c.kind} reports. endToEnd=${report.endToEnd?.category}`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
main();
|