@dailephd/my-dev-kit-lab 0.4.4 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +484 -0
- package/README.md +39 -3
- package/dist/scripts/cli.js +9 -0
- package/dist/src/audits/core/typescriptJavaScriptAnalyzer.js +30 -12
- package/dist/src/cli/help.js +300 -0
- package/dist/src/cli/index.js +2 -0
- package/dist/src/cli/runLabCli.js +229 -0
- package/dist/src/commands/runAuditCommand.js +132 -0
- package/dist/src/commands/runControlledExperimentCommand.js +7 -3
- package/dist/{scripts/experiments/describeExperiment.js → src/commands/runExperimentDescribeCommand.js} +14 -10
- package/dist/{scripts/experiments/listExperiments.js → src/commands/runExperimentListCommand.js} +3 -6
- package/dist/{scripts/experiments/runExperiment.js → src/commands/runExperimentRunCommand.js} +80 -22
- package/dist/src/commands/runFinalDemoCommand.js +2 -3
- package/dist/src/commands/runSecurityValidationCommand.js +271 -0
- package/dist/src/evaluation/ecosystemFixtures/index.js +3 -0
- package/dist/src/evaluation/ecosystemFixtures/loadContextIntegrityFixture.js +102 -0
- package/dist/src/evaluation/ecosystemFixtures/manifestTypes.js +1 -0
- package/dist/src/evaluation/ecosystemFixtures/validateManifest.js +82 -0
- package/dist/src/evaluation/ecosystemFixtures/verifyFixtureHashes.js +45 -0
- package/dist/src/evaluation/stageContextMetrics/calculateAllocationMetrics.js +169 -0
- package/dist/src/evaluation/stageContextMetrics/calculateConditionCoverageMetrics.js +74 -0
- package/dist/src/evaluation/stageContextMetrics/calculateProducerConditionAgreement.js +113 -0
- package/dist/src/evaluation/stageContextMetrics/calculateReadinessAgreement.js +60 -0
- package/dist/src/evaluation/stageContextMetrics/calculateRunIntegrityAgreement.js +309 -0
- package/dist/src/evaluation/stageContextMetrics/calculateSupplementalRawAgreement.js +12 -2
- package/dist/src/evaluation/stageContextMetrics/calculateTruncationClassification.js +100 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js +76 -5
- package/dist/src/evaluation/stageContextMetrics/index.js +4 -0
- package/dist/src/evaluation/stageContextMetrics/runIntegrityAgreementTypes.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +7 -0
- package/dist/src/evaluation/stageContextSelectors/index.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/orchestratorRunIntegritySelectors.js +64 -0
- package/dist/src/evaluation/upstreamArtifacts/index.js +2 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorRunIntegrityV1.js +33 -0
- package/dist/src/evaluation/upstreamArtifacts/readOrchestratorRunIntegrityV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +59 -2
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +6 -1
- package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorRunIntegrityV1.js +177 -0
- package/dist/src/report/experiments/buildContextIntegrityReport.js +204 -0
- package/dist/src/report/experiments/contextIntegrityReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +5 -0
- package/dist/src/report/experiments/renderContextIntegrityHtml.js +98 -0
- package/dist/src/report/experiments/renderContextIntegrityJsonReport.js +3 -0
- package/dist/src/report/experiments/renderContextIntegrityText.js +111 -0
- package/dist/src/runtime/index.js +3 -0
- package/dist/src/runtime/labExecutionContext.js +22 -0
- package/dist/src/runtime/packageResource.js +22 -0
- package/dist/src/runtime/packageRoot.js +37 -0
- package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +46 -0
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +11 -5
- package/docs/METRICS.md +83 -0
- package/package.json +10 -8
- package/dist/scripts/audits/runAudit.js +0 -135
- package/dist/scripts/build-gallery.js +0 -3
- package/dist/scripts/capture-demo-report.js +0 -3
- package/dist/scripts/evaluate-token-savings.js +0 -2
- package/dist/scripts/generate-experiment-plots.js +0 -3
- package/dist/scripts/generate-prompt-variants.js +0 -2
- package/dist/scripts/render-experiment-report.js +0 -2
- package/dist/scripts/run-agent-prompt.js +0 -2
- package/dist/scripts/run-controlled-experiment.js +0 -2
- package/dist/scripts/run-lab-demo.js +0 -5
- package/dist/scripts/run-visualization-demos.js +0 -3
- package/dist/scripts/security/resolveToolRoot.js +0 -16
- package/dist/scripts/security/runCodeql.js +0 -58
- package/dist/scripts/security/runDependencyChecks.js +0 -58
- package/dist/scripts/security/runFuzzSmoke.js +0 -29
- package/dist/scripts/security/runPackageChecks.js +0 -57
- package/dist/scripts/security/runSemgrep.js +0 -64
- package/dist/scripts/security/validate.js +0 -266
- package/dist/scripts/verify-benchmarks.js +0 -202
- package/examples/demo-report-input.json +0 -78
- package/examples/lab-demo-cases.json +0 -35
- package/examples/real-agent-campaign-cases.json +0 -118
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/debug/res/xml/ambiguous_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/ambiguous_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/backup_rules.xml +0 -5
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/broad_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/data_extraction_rules.xml +0 -10
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/unsupported_domain_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-b/src/main/res/xml/malformed_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-b/src/main/res/xml/unsupported_root.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/debug/res/xml/ambiguous_config.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/ambiguous_config.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/base_cleartext_false.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/broad_and_duplicate_domains.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/empty_domain.xml +0 -6
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/malformed_pin.xml +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/network_security_config.xml +0 -26
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-b/src/main/res/xml/malformed_network_security_config.xml +0 -6
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-b/src/main/res/xml/unsupported_root.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-dynamic.gradle +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-false.gradle +0 -9
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-true.gradle +0 -13
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/kotlin-release-debuggable-false.gradle.kts +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/kotlin-release-debuggable-true.gradle.kts +0 -10
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/missing-release-build-type.gradle.kts +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/debug/res/xml/network_security_config.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/main/res/xml/backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/main/res/xml/network_security_config.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-b/src/main/res/xml/malformed_network_security_config.xml +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/SampleActivity.java +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/SampleActivity.kt +0 -9
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/build.gradle +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/build.gradle.kts +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/google-services.json +0 -16
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/local.properties +0 -3
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates-bounds/module/binary.properties +0 -0
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates-bounds/module/oversized.json +0 -1
- package/tests/fixtures/android/advanced-security-fixtures/signing-configuration/app/fake-release-keystore.jks +0 -1
- package/tests/fixtures/android/advanced-security-fixtures/signing-configuration/build/generated-fake.keystore +0 -1
- package/tests/fixtures/android/compose-app/app/build.gradle.kts +0 -21
- package/tests/fixtures/android/compose-app/app/src/main/AndroidManifest.xml +0 -22
- package/tests/fixtures/android/compose-app/app/src/main/java/com/example/composeapp/MainActivity.kt +0 -11
- package/tests/fixtures/android/compose-app/build.gradle.kts +0 -4
- package/tests/fixtures/android/compose-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/compose-app/settings.gradle.kts +0 -2
- package/tests/fixtures/android/gradle-metadata-fixtures/catalog-only-plugin/build.gradle.kts +0 -3
- package/tests/fixtures/android/gradle-metadata-fixtures/catalog-only-plugin/gradle/libs.versions.toml +0 -5
- package/tests/fixtures/android/gradle-metadata-fixtures/comment-safety.gradle.kts +0 -21
- package/tests/fixtures/android/gradle-metadata-fixtures/dynamic-values.gradle.kts +0 -17
- package/tests/fixtures/android/gradle-metadata-fixtures/groovy-literal.gradle +0 -23
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-malformed.properties +0 -4
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-missing-url.properties +0 -3
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-with-checksum.properties +0 -5
- package/tests/fixtures/android/java-app/app/build.gradle +0 -16
- package/tests/fixtures/android/java-app/app/src/main/AndroidManifest.xml +0 -19
- package/tests/fixtures/android/java-app/app/src/main/java/com/example/javaapp/MainActivity.java +0 -4
- package/tests/fixtures/android/java-app/build.gradle +0 -3
- package/tests/fixtures/android/java-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/java-app/settings.gradle +0 -2
- package/tests/fixtures/android/library/build.gradle.kts +0 -3
- package/tests/fixtures/android/library/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/library/mylibrary/build.gradle.kts +0 -13
- package/tests/fixtures/android/library/mylibrary/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/library/settings.gradle.kts +0 -2
- package/tests/fixtures/android/manifest-audit-fixtures/incomplete-intent-filter.xml +0 -14
- package/tests/fixtures/android/manifest-audit-fixtures/malformed-exported.xml +0 -10
- package/tests/fixtures/android/manifest-audit-fixtures/malformed.xml +0 -5
- package/tests/fixtures/android/manifest-audit-fixtures/missing-application.xml +0 -7
- package/tests/fixtures/android/manifest-audit-fixtures/missing-component-name.xml +0 -11
- package/tests/fixtures/android/manifest-audit-fixtures/permission-coverage.xml +0 -29
- package/tests/fixtures/android/manifest-audit-fixtures/risky-components.xml +0 -62
- package/tests/fixtures/android/manifest-audit-fixtures/risky-deep-links.xml +0 -45
- package/tests/fixtures/android/manifest-audit-fixtures/unresolved-placeholder.xml +0 -14
- package/tests/fixtures/android/manifest-audit-fixtures/unusual-namespace-prefix.xml +0 -11
- package/tests/fixtures/android/multi-module/app/build.gradle.kts +0 -22
- package/tests/fixtures/android/multi-module/app/src/androidTest/java/com/example/multimodule/app/ExampleInstrumentedTest.kt +0 -3
- package/tests/fixtures/android/multi-module/app/src/main/AndroidManifest.xml +0 -11
- package/tests/fixtures/android/multi-module/app/src/test/java/com/example/multimodule/app/ExampleUnitTest.kt +0 -3
- package/tests/fixtures/android/multi-module/build.gradle.kts +0 -5
- package/tests/fixtures/android/multi-module/core/build.gradle.kts +0 -14
- package/tests/fixtures/android/multi-module/core/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/multi-module/feature-login/build.gradle.kts +0 -18
- package/tests/fixtures/android/multi-module/feature-login/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/multi-module/gradle/libs.versions.toml +0 -8
- package/tests/fixtures/android/multi-module/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/multi-module/settings.gradle.kts +0 -4
- package/tests/fixtures/android/non-android-gradle/build.gradle.kts +0 -3
- package/tests/fixtures/android/non-android-gradle/core/build.gradle.kts +0 -3
- package/tests/fixtures/android/non-android-gradle/core/src/main/java/com/example/core/Core.java +0 -4
- package/tests/fixtures/android/non-android-gradle/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/non-android-gradle/settings.gradle.kts +0 -2
- package/tests/fixtures/android/partial/app/build.gradle.kts +0 -7
- package/tests/fixtures/android/partial/settings.gradle.kts +0 -2
- package/tests/fixtures/android/xml-view-app/app/build.gradle.kts +0 -17
- package/tests/fixtures/android/xml-view-app/app/src/main/AndroidManifest.xml +0 -31
- package/tests/fixtures/android/xml-view-app/app/src/main/kotlin/com/example/xmlviewapp/MainActivity.kt +0 -7
- package/tests/fixtures/android/xml-view-app/app/src/main/res/layout/activity_main.xml +0 -4
- package/tests/fixtures/android/xml-view-app/build.gradle.kts +0 -4
- package/tests/fixtures/android/xml-view-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/xml-view-app/settings.gradle.kts +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/README.md +0 -16
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/build.gradle.kts +0 -4
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/src/main/java/com/example/Widget.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/src/main/kotlin/com/example/Gadget.kt +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/build/Main.class +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/dist/index.js +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/generated/schema.ts +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/vendor/lib.py +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/build.gradle.kts +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/settings.gradle.kts +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/main/java/com/example/FooService.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/main/kotlin/com/example/BarService.kt +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/test/java/com/example/FooServiceTest.java +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/test/kotlin/com/example/BarServiceTest.kt +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/README.md +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/src/app.py +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/tests/test_app.py +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/README.md +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/src/helper.js +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/tests/index.sample.ts +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/build.gradle.kts +0 -4
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/pytests/test_app.py +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/app.py +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/helper.js +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/main/java/com/example/Multiplier.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/main/kotlin/com/example/Divider.kt +0 -7
- package/tests/fixtures/fake-adversarial-cli.js +0 -152
- package/tests/fixtures/fake-my-dev-kit-cli.js +0 -83
- package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +0 -24
- package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +0 -25
- package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +0 -230
- package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +0 -232
- package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +0 -1
- package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +0 -1
- package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +0 -18
- package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +0 -1
- package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +0 -1
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +0 -504
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +0 -955
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +0 -190
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +0 -327
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +0 -13
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +0 -7874
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +0 -1076
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +0 -160
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +0 -186
|
@@ -1,266 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { runSecurityValidation } from "../../src/securityValidation/validate/runSecurityValidation.js";
|
|
4
|
-
import { resolveValidationTarget, reportFilenamePrefix } from "../../src/securityValidation/validate/resolveTarget.js";
|
|
5
|
-
import { buildSecurityReportFromSummary } from "../../src/securityValidation/report/buildSecurityReport.js";
|
|
6
|
-
import { renderTextReport } from "../../src/securityValidation/report/renderSecurityReport.js";
|
|
7
|
-
import { writeSecurityReportFiles } from "../../src/securityValidation/report/writeSecurityReportFiles.js";
|
|
8
|
-
import { parseSecurityValidateArgs, normalizeSecurityValidateConfig, applyProfileDefaultChecksIfApplicable, } from "../../src/securityValidation/validate/cliOptions.js";
|
|
9
|
-
import { resolveAttackProfile } from "../../src/securityValidation/attackScenarios/attackProfile.js";
|
|
10
|
-
import { findingsBreachFailOnThreshold } from "../../src/securityValidation/validate/verdict.js";
|
|
11
|
-
import { resolveToolRoot } from "./resolveToolRoot.js";
|
|
12
|
-
import { validateAndroidTarget } from "../../src/mobile/android/validate/validateAndroidTarget.js";
|
|
13
|
-
import { toAndroidReportModel } from "../../src/mobile/android/report/model.js";
|
|
14
|
-
import { renderAndroidTextReport } from "../../src/mobile/android/report/renderAndroidReport.js";
|
|
15
|
-
import { writeAndroidReportFiles } from "../../src/mobile/android/report/writeAndroidReportFiles.js";
|
|
16
|
-
const USAGE = "Usage: npm run security:validate -- [--target <path>] [--out <dir>] [--report-prefix <name>] " +
|
|
17
|
-
"[--checks deps,package,static,cli-adversarial,fuzz,boundary,subprocess,secrets,network] " +
|
|
18
|
-
"[--profile node-cli-package|local-tool|npm-package|android] [--format text,json] [--fail-on blocker|high|medium|low] " +
|
|
19
|
-
"[--android-gradle-operations wrapper-version,tasks,assemble-debug,unit-test-debug,lint-debug] " +
|
|
20
|
-
"[--android-external-tools semgrep,osv,android-lint,dependency-check] " +
|
|
21
|
-
"[--android-external-network deny|allow-requested]\n" +
|
|
22
|
-
"\n" +
|
|
23
|
-
"Android profile:\n" +
|
|
24
|
-
" --profile android performs static, read-only Android project detection, manifest parsing,\n" +
|
|
25
|
-
" permission/exported-component/intent-filter/deep-link audits, static Gradle metadata\n" +
|
|
26
|
-
" extraction, and (as of v0.4.1 Batch 8) eleven internal advanced security checks (network\n" +
|
|
27
|
-
" security config, backup/release configuration, hardcoded secrets, signing configuration,\n" +
|
|
28
|
-
" WebView, FileProvider, sensitive storage/logging, clipboard, and Firebase/Google services).\n" +
|
|
29
|
-
" Gradle and external tools are never executed by default.\n" +
|
|
30
|
-
" --android-gradle-operations explicitly opts into running ONLY the listed allowlisted Gradle\n" +
|
|
31
|
-
" operations (wrapper-version, tasks, assemble-debug, unit-test-debug, lint-debug) via the\n" +
|
|
32
|
-
" project's own Gradle wrapper. Arbitrary Gradle tasks cannot be passed. This option is only\n" +
|
|
33
|
-
" valid together with --profile android.\n" +
|
|
34
|
-
" --android-external-tools explicitly opts into running ONLY the listed allowlisted optional\n" +
|
|
35
|
-
" external security tools (semgrep, osv, android-lint, dependency-check). No arbitrary tool\n" +
|
|
36
|
-
" names, commands, or arguments can be passed. This option is only valid together with\n" +
|
|
37
|
-
" --profile android.\n" +
|
|
38
|
-
" --android-external-network controls whether a requested tool may use network access.\n" +
|
|
39
|
-
" Defaults to deny (osv is skipped unless supported offline data is available; semgrep,\n" +
|
|
40
|
-
" android-lint, and dependency-check always run offline/no-update regardless of this flag).\n" +
|
|
41
|
-
" allow-requested authorizes network only for an explicitly requested tool whose Batch 7\n" +
|
|
42
|
-
" contract supports it (currently only osv). This option is only valid together with\n" +
|
|
43
|
-
" --profile android.\n" +
|
|
44
|
-
" Example: npm run security:validate -- --target \"<android-project-path>\" --profile android\n" +
|
|
45
|
-
" Example: npm run security:validate -- --target \"<android-project-path>\" --profile android --android-gradle-operations wrapper-version,tasks\n" +
|
|
46
|
-
" Example: npm run security:validate -- --target \"<android-project-path>\" --profile android --android-external-tools semgrep,android-lint\n" +
|
|
47
|
-
" Example: npm run security:validate -- --target \"<android-project-path>\" --profile android --android-external-tools osv --android-external-network allow-requested";
|
|
48
|
-
// Parse CLI arguments from process.argv (after the node/tsx and script path).
|
|
49
|
-
const rawArgs = process.argv.slice(2);
|
|
50
|
-
const toolRoot = resolveToolRoot(import.meta.url);
|
|
51
|
-
// Help deliberately wins over every other argument. It must remain a
|
|
52
|
-
// no-work path: no target/default resolution, option normalization, report
|
|
53
|
-
// directory creation, validation orchestration, or subprocess execution.
|
|
54
|
-
if (rawArgs.includes("--help") || rawArgs.includes("-h")) {
|
|
55
|
-
console.log(USAGE);
|
|
56
|
-
process.exitCode = 0;
|
|
57
|
-
process.exit(0);
|
|
58
|
-
}
|
|
59
|
-
let args;
|
|
60
|
-
let config;
|
|
61
|
-
try {
|
|
62
|
-
args = parseSecurityValidateArgs(rawArgs);
|
|
63
|
-
config = normalizeSecurityValidateConfig(args, toolRoot);
|
|
64
|
-
// v0.2.2 Batch 5: --profile with no --checks uses that profile's declared
|
|
65
|
-
// default checks. No-op unless the user passed --profile without --checks
|
|
66
|
-
// (see applyProfileDefaultChecksIfApplicable's guard) — the true no-flag
|
|
67
|
-
// case and any explicit --checks case are always unaffected. Skipped for
|
|
68
|
-
// "android": that profile does not use the classic --checks pipeline.
|
|
69
|
-
if (config.profile !== "android") {
|
|
70
|
-
config = applyProfileDefaultChecksIfApplicable(config, resolveAttackProfile(config.profile).defaultCheckIds);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
catch (err) {
|
|
74
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
75
|
-
console.error(`\nERROR: ${msg}`);
|
|
76
|
-
console.error(USAGE);
|
|
77
|
-
process.exitCode = 2;
|
|
78
|
-
process.exit(2);
|
|
79
|
-
}
|
|
80
|
-
// Resolve and validate target early so we can fail fast with a clean error.
|
|
81
|
-
let target;
|
|
82
|
-
try {
|
|
83
|
-
target = resolveValidationTarget(args.target, toolRoot);
|
|
84
|
-
}
|
|
85
|
-
catch (err) {
|
|
86
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
87
|
-
console.error(`\nERROR: ${msg}`);
|
|
88
|
-
console.error(USAGE);
|
|
89
|
-
process.exitCode = 1;
|
|
90
|
-
process.exit(1);
|
|
91
|
-
}
|
|
92
|
-
console.log("=".repeat(60));
|
|
93
|
-
console.log("my-dev-kit-lab security:validate");
|
|
94
|
-
console.log("=".repeat(60));
|
|
95
|
-
console.log(`Tool root : ${toolRoot}`);
|
|
96
|
-
if (!target.isSelf) {
|
|
97
|
-
console.log(`Target : ${target.targetRoot}`);
|
|
98
|
-
if (target.packageName)
|
|
99
|
-
console.log(`Package : ${target.packageName}${target.packageVersion ? `@${target.packageVersion}` : ""}`);
|
|
100
|
-
}
|
|
101
|
-
else {
|
|
102
|
-
console.log(`Mode : self-validation`);
|
|
103
|
-
}
|
|
104
|
-
console.log(`Profile : ${config.profile}`);
|
|
105
|
-
// ---------------------------------------------------------------------------
|
|
106
|
-
// v0.4.0 Batch 5 — Android profile path.
|
|
107
|
-
//
|
|
108
|
-
// A separate orchestrator (not the classic check-group/attack-scenario
|
|
109
|
-
// pipeline below) because the Android check/finding/status vocabulary
|
|
110
|
-
// established in Batches 1-4 does not fit SecurityCheckId/SecurityCheckResult.
|
|
111
|
-
// Same command, same --profile flag, same reports/security/ output root.
|
|
112
|
-
// ---------------------------------------------------------------------------
|
|
113
|
-
if (config.profile === "android") {
|
|
114
|
-
if (config.androidGradleOperationIds.length > 0) {
|
|
115
|
-
console.log(`Gradle ops : ${config.androidGradleOperationIds.join(", ")} (explicitly requested — will execute via the target's Gradle wrapper)`);
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
console.log(`Gradle ops : (none requested — static-only validation, zero Gradle process execution)`);
|
|
119
|
-
}
|
|
120
|
-
// v0.4.1 Batch 8 — external tools remain explicit opt-in, mirroring Gradle
|
|
121
|
-
// operations exactly. "allow-requested" (the CLI-facing value) maps to the
|
|
122
|
-
// Batch 7 dispatcher's "allow-for-requested-tool" network policy value.
|
|
123
|
-
const externalNetworkPolicy = config.androidExternalNetworkPolicy === "allow-requested" ? "allow-for-requested-tool" : "deny";
|
|
124
|
-
if (config.androidExternalToolIds.length > 0) {
|
|
125
|
-
console.log(`External : ${config.androidExternalToolIds.join(", ")} (explicitly requested — network policy: ${config.androidExternalNetworkPolicy})`);
|
|
126
|
-
}
|
|
127
|
-
else {
|
|
128
|
-
console.log(`External : (none requested — zero external-tool process execution)`);
|
|
129
|
-
}
|
|
130
|
-
console.log("");
|
|
131
|
-
const reportsDir = config.out;
|
|
132
|
-
let result;
|
|
133
|
-
try {
|
|
134
|
-
result = await validateAndroidTarget({
|
|
135
|
-
toolRoot,
|
|
136
|
-
targetPath: args.target,
|
|
137
|
-
requestedGradleOperationIds: config.androidGradleOperationIds,
|
|
138
|
-
requestedExternalToolIds: config.androidExternalToolIds,
|
|
139
|
-
externalNetworkPolicy: externalNetworkPolicy,
|
|
140
|
-
externalToolArtifactRoot: path.join(reportsDir, "external-tools"),
|
|
141
|
-
});
|
|
142
|
-
}
|
|
143
|
-
catch (err) {
|
|
144
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
145
|
-
console.error(`\nERROR: Android validation failed to run: ${msg}`);
|
|
146
|
-
process.exitCode = 3;
|
|
147
|
-
process.exit(3);
|
|
148
|
-
}
|
|
149
|
-
const reportModel = toAndroidReportModel(result, {
|
|
150
|
-
profile: config.profile,
|
|
151
|
-
requestedGradleOperations: config.androidGradleOperationIds,
|
|
152
|
-
requestedExternalTools: config.androidExternalToolIds,
|
|
153
|
-
externalNetworkPolicy: config.androidExternalNetworkPolicy,
|
|
154
|
-
});
|
|
155
|
-
const textReport = renderAndroidTextReport(reportModel);
|
|
156
|
-
const prefix = args.reportPrefix ??
|
|
157
|
-
reportFilenamePrefix({
|
|
158
|
-
isSelf: result.target.local.isSelf,
|
|
159
|
-
packageName: result.target.local.packageName,
|
|
160
|
-
packageVersion: result.target.local.packageVersion,
|
|
161
|
-
targetRoot: result.target.local.targetRoot,
|
|
162
|
-
});
|
|
163
|
-
let writtenPaths;
|
|
164
|
-
try {
|
|
165
|
-
({ writtenPaths } = writeAndroidReportFiles({ outDir: reportsDir, prefix, report: reportModel, formats: config.formats }));
|
|
166
|
-
}
|
|
167
|
-
catch (err) {
|
|
168
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
169
|
-
console.error(`\nERROR: Failed to write Android report files: ${msg}`);
|
|
170
|
-
console.error("Validation completed but the report could not be written. The in-memory result was not persisted.");
|
|
171
|
-
process.exitCode = 4;
|
|
172
|
-
process.exit(4);
|
|
173
|
-
}
|
|
174
|
-
console.log(textReport);
|
|
175
|
-
console.log(`\nReports written:`);
|
|
176
|
-
for (const p of writtenPaths) {
|
|
177
|
-
console.log(` ${p}`);
|
|
178
|
-
}
|
|
179
|
-
const blockerExists = result.verdict === "not-ready-security-blocker-remains";
|
|
180
|
-
const inconclusive = result.verdict === "inconclusive-audit-environment-incomplete";
|
|
181
|
-
if (blockerExists) {
|
|
182
|
-
console.error("\nExit 1 — security blocker remains.");
|
|
183
|
-
process.exitCode = 1;
|
|
184
|
-
}
|
|
185
|
-
else if (inconclusive) {
|
|
186
|
-
console.warn("\nExit 2 — android environment incomplete.");
|
|
187
|
-
process.exitCode = 2;
|
|
188
|
-
}
|
|
189
|
-
else {
|
|
190
|
-
console.log("\nExit 0 — validation completed.");
|
|
191
|
-
process.exitCode = 0;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
else {
|
|
195
|
-
// ---------------------------------------------------------------------------
|
|
196
|
-
// Classic check-group/attack-scenario pipeline (unchanged from prior versions).
|
|
197
|
-
// ---------------------------------------------------------------------------
|
|
198
|
-
if (!config.checksWereDefault) {
|
|
199
|
-
console.log(`Checks : ${config.checks.join(", ")}`);
|
|
200
|
-
}
|
|
201
|
-
if (config.plannedChecksRequested.length > 0) {
|
|
202
|
-
console.log(`Note : ${config.plannedChecksRequested.join(", ")} are routed through the attack-scenario framework rather than the classic check-group runners. Review the attack-scenario section in the report for their concrete results.`);
|
|
203
|
-
}
|
|
204
|
-
console.log("");
|
|
205
|
-
const summary = await runSecurityValidation({
|
|
206
|
-
cwd: toolRoot,
|
|
207
|
-
targetPath: args.target,
|
|
208
|
-
fuzzIterations: parseInt(process.env["FUZZ_ITERATIONS"] ?? "50", 10),
|
|
209
|
-
fuzzSeed: parseInt(process.env["FUZZ_SEED"] ?? "0xDEADBEEF", 16),
|
|
210
|
-
// Full selection (implemented + planned/attack-scenario ids) — the runner
|
|
211
|
-
// gates both implemented check groups and attack-scenario checks off the
|
|
212
|
-
// same selectedChecks set.
|
|
213
|
-
selectedChecks: config.checks,
|
|
214
|
-
profile: config.profile,
|
|
215
|
-
});
|
|
216
|
-
// v0.2.2 Batch 5: --fail-on breach is additive to (never a substitute for)
|
|
217
|
-
// the existing verdict-based exit decision below — it can only escalate
|
|
218
|
-
// exit 0 to exit 1 (e.g. --fail-on medium/low catching minor/informational
|
|
219
|
-
// findings the verdict policy doesn't already treat as blocking), never
|
|
220
|
-
// downgrade an existing blocker.
|
|
221
|
-
const failOnBreached = findingsBreachFailOnThreshold(summary.findings, config.failOnThreshold);
|
|
222
|
-
// Build report object
|
|
223
|
-
const report = buildSecurityReportFromSummary(summary, {
|
|
224
|
-
profile: config.profile,
|
|
225
|
-
selectedChecks: config.checks,
|
|
226
|
-
failOnThreshold: config.failOnThreshold,
|
|
227
|
-
formats: config.formats,
|
|
228
|
-
failOnBreached,
|
|
229
|
-
});
|
|
230
|
-
const textReport = renderTextReport(report);
|
|
231
|
-
// Determine output directory
|
|
232
|
-
const reportsDir = config.out;
|
|
233
|
-
// Determine report filename prefix
|
|
234
|
-
const prefix = args.reportPrefix ?? reportFilenamePrefix(target);
|
|
235
|
-
const { writtenPaths } = writeSecurityReportFiles({
|
|
236
|
-
outDir: reportsDir,
|
|
237
|
-
prefix,
|
|
238
|
-
report,
|
|
239
|
-
formats: config.formats,
|
|
240
|
-
});
|
|
241
|
-
// Print report to stdout
|
|
242
|
-
console.log(textReport);
|
|
243
|
-
console.log(`\nReports written:`);
|
|
244
|
-
for (const p of writtenPaths) {
|
|
245
|
-
console.log(` ${p}`);
|
|
246
|
-
}
|
|
247
|
-
// Exit code based on verdict, escalated (never downgraded) by --fail-on.
|
|
248
|
-
const blockerExists = summary.verdict === "not-ready-security-blocker-remains";
|
|
249
|
-
const inconclusive = summary.verdict === "inconclusive-audit-environment-incomplete";
|
|
250
|
-
if (blockerExists) {
|
|
251
|
-
console.error("\nExit 1 — security blocker remains.");
|
|
252
|
-
process.exitCode = 1;
|
|
253
|
-
}
|
|
254
|
-
else if (inconclusive) {
|
|
255
|
-
console.warn("\nExit 2 — audit environment incomplete.");
|
|
256
|
-
process.exitCode = 2;
|
|
257
|
-
}
|
|
258
|
-
else if (failOnBreached) {
|
|
259
|
-
console.error(`\nExit 1 — --fail-on ${config.failOnThreshold} threshold breached.`);
|
|
260
|
-
process.exitCode = 1;
|
|
261
|
-
}
|
|
262
|
-
else {
|
|
263
|
-
console.log("\nExit 0 — validation completed.");
|
|
264
|
-
process.exitCode = 0;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { REQUIRED_BENCHMARK_PROJECT_IDS, parseBenchmarkProjectProfiles, validateAnswerKey, validateBenchmarkProjectProfiles } from "../src/evaluation/benchmarkMetadata.js";
|
|
5
|
-
const requiredProjects = REQUIRED_BENCHMARK_PROJECT_IDS;
|
|
6
|
-
const projectRequiredPaths = {
|
|
7
|
-
"todo-ts": [
|
|
8
|
-
"README.md",
|
|
9
|
-
"package.json",
|
|
10
|
-
"tsconfig.json",
|
|
11
|
-
"src/taskStore.ts",
|
|
12
|
-
"src/taskService.ts",
|
|
13
|
-
"src/index.ts",
|
|
14
|
-
"tests/taskService.test.ts"
|
|
15
|
-
],
|
|
16
|
-
"todo-python": [
|
|
17
|
-
"README.md",
|
|
18
|
-
"src/task_store.py",
|
|
19
|
-
"src/task_service.py",
|
|
20
|
-
"src/__init__.py",
|
|
21
|
-
"tests/test_task_service.py"
|
|
22
|
-
],
|
|
23
|
-
"todo-js": [
|
|
24
|
-
"README.md",
|
|
25
|
-
"package.json",
|
|
26
|
-
"src/taskStore.js",
|
|
27
|
-
"src/taskService.js",
|
|
28
|
-
"src/index.js",
|
|
29
|
-
"tests/taskService.test.js"
|
|
30
|
-
],
|
|
31
|
-
"todo-mixed-ts-py": [
|
|
32
|
-
"README.md",
|
|
33
|
-
"package.json",
|
|
34
|
-
"tsconfig.json",
|
|
35
|
-
"src/taskCli.ts",
|
|
36
|
-
"python/task_service.py",
|
|
37
|
-
"tests/mixedBoundary.test.ts"
|
|
38
|
-
],
|
|
39
|
-
"task-workflow-medium-ts": [
|
|
40
|
-
"README.md",
|
|
41
|
-
"package.json",
|
|
42
|
-
"tsconfig.json",
|
|
43
|
-
"src/store/taskStore.ts",
|
|
44
|
-
"src/services/createTask.ts",
|
|
45
|
-
"src/services/importTasks.ts",
|
|
46
|
-
"src/services/summarizeTasks.ts",
|
|
47
|
-
"tests/importTasks.test.ts"
|
|
48
|
-
],
|
|
49
|
-
"task-analytics-large-mixed": [
|
|
50
|
-
"README.md",
|
|
51
|
-
"ts/package.json",
|
|
52
|
-
"ts/tsconfig.json",
|
|
53
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
54
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
55
|
-
"ts/tests/buildAnalyticsSnapshot.test.ts",
|
|
56
|
-
"py/task_analytics/metrics.py",
|
|
57
|
-
"py/task_analytics/quality.py",
|
|
58
|
-
"py/tests/test_reporting.py"
|
|
59
|
-
]
|
|
60
|
-
};
|
|
61
|
-
function walk(dir) {
|
|
62
|
-
const entries = readdirSync(dir, { withFileTypes: true });
|
|
63
|
-
const files = [];
|
|
64
|
-
for (const entry of entries) {
|
|
65
|
-
const fullPath = path.join(dir, entry.name);
|
|
66
|
-
if (entry.isDirectory()) {
|
|
67
|
-
files.push(...walk(fullPath));
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
70
|
-
files.push(fullPath);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
return files;
|
|
74
|
-
}
|
|
75
|
-
export function validateBenchmarks(rootDir = process.cwd()) {
|
|
76
|
-
const checks = [];
|
|
77
|
-
const errors = [];
|
|
78
|
-
const contractsDir = path.join(rootDir, "benchmarks", "contracts");
|
|
79
|
-
const projectsDir = path.join(rootDir, "benchmarks", "projects");
|
|
80
|
-
const behaviorPath = path.join(contractsDir, "todo-behavior.md");
|
|
81
|
-
const casesPath = path.join(contractsDir, "todo-benchmark-case.json");
|
|
82
|
-
const profilesPath = path.join(contractsDir, "benchmark-project-profiles.json");
|
|
83
|
-
if (!existsSync(behaviorPath)) {
|
|
84
|
-
errors.push("Missing contract file: benchmarks/contracts/todo-behavior.md");
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
checks.push("found todo-behavior.md");
|
|
88
|
-
}
|
|
89
|
-
let cases = [];
|
|
90
|
-
if (!existsSync(casesPath)) {
|
|
91
|
-
errors.push("Missing contract file: benchmarks/contracts/todo-benchmark-case.json");
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
try {
|
|
95
|
-
cases = JSON.parse(readFileSync(casesPath, "utf8"));
|
|
96
|
-
checks.push("parsed todo-benchmark-case.json");
|
|
97
|
-
}
|
|
98
|
-
catch (error) {
|
|
99
|
-
errors.push(`Invalid JSON in todo-benchmark-case.json: ${error.message}`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
if (!existsSync(profilesPath)) {
|
|
103
|
-
errors.push("Missing contract file: benchmarks/contracts/benchmark-project-profiles.json");
|
|
104
|
-
}
|
|
105
|
-
else {
|
|
106
|
-
try {
|
|
107
|
-
const profiles = parseBenchmarkProjectProfiles(JSON.parse(readFileSync(profilesPath, "utf8")));
|
|
108
|
-
const profileErrors = validateBenchmarkProjectProfiles(profiles, rootDir);
|
|
109
|
-
errors.push(...profileErrors);
|
|
110
|
-
if (profileErrors.length === 0) {
|
|
111
|
-
checks.push("validated benchmark-project-profiles.json");
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
catch (error) {
|
|
115
|
-
errors.push(`Invalid benchmark-project-profiles.json: ${error.message}`);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
const ids = new Set();
|
|
119
|
-
for (const benchmarkCase of cases) {
|
|
120
|
-
if (ids.has(benchmarkCase.id)) {
|
|
121
|
-
errors.push(`Duplicate benchmark case id: ${benchmarkCase.id}`);
|
|
122
|
-
}
|
|
123
|
-
ids.add(benchmarkCase.id);
|
|
124
|
-
}
|
|
125
|
-
if (cases.length > 0 && errors.every((error) => !error.startsWith("Duplicate benchmark case id:"))) {
|
|
126
|
-
checks.push("benchmark case ids are unique");
|
|
127
|
-
}
|
|
128
|
-
for (const project of requiredProjects) {
|
|
129
|
-
const projectDir = path.join(projectsDir, project);
|
|
130
|
-
if (!existsSync(projectDir) || !statSync(projectDir).isDirectory()) {
|
|
131
|
-
errors.push(`Missing benchmark project: benchmarks/projects/${project}`);
|
|
132
|
-
continue;
|
|
133
|
-
}
|
|
134
|
-
checks.push(`found benchmark project ${project}`);
|
|
135
|
-
for (const relPath of projectRequiredPaths[project]) {
|
|
136
|
-
const fullPath = path.join(projectDir, relPath);
|
|
137
|
-
if (!existsSync(fullPath)) {
|
|
138
|
-
errors.push(`Missing required file for ${project}: benchmarks/projects/${project}/${relPath}`);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
for (const benchmarkCase of cases) {
|
|
143
|
-
if (!benchmarkCase.answerKey) {
|
|
144
|
-
errors.push(`Case ${benchmarkCase.id} does not define answerKey`);
|
|
145
|
-
}
|
|
146
|
-
else {
|
|
147
|
-
errors.push(...validateAnswerKey(benchmarkCase.answerKey, `Case ${benchmarkCase.id}`));
|
|
148
|
-
}
|
|
149
|
-
if (!Array.isArray(benchmarkCase.expectedSymbols) || benchmarkCase.expectedSymbols.length === 0) {
|
|
150
|
-
errors.push(`Case ${benchmarkCase.id} does not define expectedSymbols`);
|
|
151
|
-
}
|
|
152
|
-
for (const [project, expectedFiles] of Object.entries(benchmarkCase.expectedFilesByProject ?? {})) {
|
|
153
|
-
if (!requiredProjects.includes(project)) {
|
|
154
|
-
errors.push(`Case ${benchmarkCase.id} references unknown project id: ${project}`);
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (!Array.isArray(expectedFiles) || expectedFiles.length === 0) {
|
|
158
|
-
errors.push(`Case ${benchmarkCase.id} does not define expected files for ${project}`);
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
for (const expectedFile of expectedFiles) {
|
|
162
|
-
const fullPath = path.join(rootDir, "benchmarks", "projects", project, expectedFile);
|
|
163
|
-
if (!existsSync(fullPath)) {
|
|
164
|
-
errors.push(`Case ${benchmarkCase.id} references missing file: benchmarks/projects/${project}/${expectedFile}`);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
for (const project of requiredProjects) {
|
|
170
|
-
const projectDir = path.join(projectsDir, project);
|
|
171
|
-
if (!existsSync(projectDir)) {
|
|
172
|
-
continue;
|
|
173
|
-
}
|
|
174
|
-
const forbidden = walk(projectDir).filter((fullPath) => {
|
|
175
|
-
const rel = path.relative(projectDir, fullPath).replace(/\\/g, "/");
|
|
176
|
-
return /(^|\/)(node_modules|dist|build|coverage|lab-output)(\/|$)/.test(rel);
|
|
177
|
-
});
|
|
178
|
-
if (forbidden.length > 0) {
|
|
179
|
-
errors.push(`Forbidden generated output found in ${project}: ${forbidden[0]}`);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
return { ok: errors.length === 0, errors, checks };
|
|
183
|
-
}
|
|
184
|
-
function printSummary(result) {
|
|
185
|
-
console.log(`Benchmark verification ${result.ok ? "passed" : "failed"}.`);
|
|
186
|
-
console.log(`Checks: ${result.checks.length}`);
|
|
187
|
-
if (result.errors.length > 0) {
|
|
188
|
-
console.log(`Errors: ${result.errors.length}`);
|
|
189
|
-
for (const error of result.errors) {
|
|
190
|
-
console.log(`- ${error}`);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
const currentFile = fileURLToPath(import.meta.url);
|
|
195
|
-
const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : "";
|
|
196
|
-
if (invokedPath === currentFile) {
|
|
197
|
-
const result = validateBenchmarks();
|
|
198
|
-
printSummary(result);
|
|
199
|
-
if (!result.ok) {
|
|
200
|
-
process.exitCode = 1;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"reportId": "demo-report",
|
|
3
|
-
"title": "Benchmark Validation Demo Report",
|
|
4
|
-
"projectName": "my-dev-kit-lab",
|
|
5
|
-
"benchmarkProject": "todo-ts",
|
|
6
|
-
"workflowName": "benchmark validation demo",
|
|
7
|
-
"generatedAt": "2026-06-10T15:00:00.000Z",
|
|
8
|
-
"summary": "Deterministic sample report for Prompt 2 report rendering and screenshot capture. This report does not claim token-savings evaluation.",
|
|
9
|
-
"steps": [
|
|
10
|
-
{
|
|
11
|
-
"id": "load-contracts",
|
|
12
|
-
"label": "Load benchmark contracts",
|
|
13
|
-
"command": "npm run verify:benchmarks",
|
|
14
|
-
"status": "pass",
|
|
15
|
-
"durationMs": 42,
|
|
16
|
-
"notes": "Validated benchmark contract structure."
|
|
17
|
-
},
|
|
18
|
-
{
|
|
19
|
-
"id": "run-project-tests",
|
|
20
|
-
"label": "Run benchmark project tests",
|
|
21
|
-
"command": "npm run test:benchmarks",
|
|
22
|
-
"status": "pass",
|
|
23
|
-
"durationMs": 118,
|
|
24
|
-
"notes": "Todo benchmark behavior stayed aligned."
|
|
25
|
-
},
|
|
26
|
-
{
|
|
27
|
-
"id": "prepare-report",
|
|
28
|
-
"label": "Prepare report artifacts",
|
|
29
|
-
"command": "npm run capture-demo-report",
|
|
30
|
-
"status": "pass",
|
|
31
|
-
"durationMs": 35,
|
|
32
|
-
"notes": "No token-savings metrics are reported in this demo."
|
|
33
|
-
}
|
|
34
|
-
],
|
|
35
|
-
"metrics": [
|
|
36
|
-
{
|
|
37
|
-
"id": "benchmark-project-count",
|
|
38
|
-
"label": "Benchmark projects",
|
|
39
|
-
"value": 4,
|
|
40
|
-
"interpretation": "Prompt 1 benchmark suite is available."
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
"id": "step-count",
|
|
44
|
-
"label": "Workflow steps",
|
|
45
|
-
"value": 3,
|
|
46
|
-
"interpretation": "Demo workflow uses a small deterministic sequence."
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
"id": "warning-count",
|
|
50
|
-
"label": "Warnings",
|
|
51
|
-
"value": 1,
|
|
52
|
-
"interpretation": "This demo explicitly notes that token comparison is not implemented yet."
|
|
53
|
-
},
|
|
54
|
-
{
|
|
55
|
-
"id": "artifact-count",
|
|
56
|
-
"label": "Artifacts declared",
|
|
57
|
-
"value": 2,
|
|
58
|
-
"interpretation": "JSON and HTML are always expected before optional PNG capture."
|
|
59
|
-
}
|
|
60
|
-
],
|
|
61
|
-
"artifacts": [
|
|
62
|
-
{
|
|
63
|
-
"id": "report-json",
|
|
64
|
-
"label": "Demo report JSON",
|
|
65
|
-
"path": "lab-output/demo-report/demo-report.json",
|
|
66
|
-
"kind": "json"
|
|
67
|
-
},
|
|
68
|
-
{
|
|
69
|
-
"id": "report-html",
|
|
70
|
-
"label": "Demo report HTML",
|
|
71
|
-
"path": "lab-output/demo-report/demo-report.html",
|
|
72
|
-
"kind": "html"
|
|
73
|
-
}
|
|
74
|
-
],
|
|
75
|
-
"warnings": [
|
|
76
|
-
"Token-savings evaluation is not implemented in Prompt 2."
|
|
77
|
-
]
|
|
78
|
-
}
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"id": "todo-ts-create-task-demo",
|
|
4
|
-
"title": "Todo TS create task demo retrieval",
|
|
5
|
-
"benchmarkProject": "todo-ts",
|
|
6
|
-
"projectProfileRef": "todo-ts",
|
|
7
|
-
"targetRoot": "benchmarks/projects/todo-ts",
|
|
8
|
-
"sourceRoots": ["src", "tests"],
|
|
9
|
-
"query": "create task deterministic id task-1",
|
|
10
|
-
"expectedFiles": ["src/taskService.ts", "src/taskStore.ts"],
|
|
11
|
-
"expectedSymbols": ["createTask", "TaskService"],
|
|
12
|
-
"answerKey": {
|
|
13
|
-
"expectedFiles": ["src/taskService.ts", "src/taskStore.ts"],
|
|
14
|
-
"expectedSymbols": ["createTask", "TaskService"],
|
|
15
|
-
"expectedFacts": [
|
|
16
|
-
{
|
|
17
|
-
"id": "create-deterministic-id",
|
|
18
|
-
"text": "createTask assigns deterministic IDs such as task-1 and task-2.",
|
|
19
|
-
"weight": 1,
|
|
20
|
-
"required": true
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
"id": "create-stores-incomplete-task",
|
|
24
|
-
"text": "createTask stores new tasks with completed set to false.",
|
|
25
|
-
"weight": 1,
|
|
26
|
-
"required": true
|
|
27
|
-
}
|
|
28
|
-
],
|
|
29
|
-
"minimumCorrectFacts": 2,
|
|
30
|
-
"notes": "Demo metadata prepares later correctness scoring without changing the current lab-demo command."
|
|
31
|
-
},
|
|
32
|
-
"rawIncludeGlobs": ["src/**/*", "tests/**/*"],
|
|
33
|
-
"notes": "Small single-case lab demo fixture."
|
|
34
|
-
}
|
|
35
|
-
]
|