@dailephd/my-dev-kit-lab 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -1
- package/README.md +43 -3
- package/dist/scripts/cli.js +9 -0
- package/dist/src/audits/core/typescriptJavaScriptAnalyzer.js +30 -12
- package/dist/src/browser/index.js +2 -0
- package/dist/src/browser/playwrightRuntime.js +74 -0
- package/dist/src/browser/types.js +10 -0
- package/dist/src/cli/help.js +376 -0
- package/dist/src/cli/index.js +2 -0
- package/dist/src/cli/runLabCli.js +257 -0
- package/dist/src/commands/runAuditCommand.js +132 -0
- package/dist/src/commands/runControlledExperimentCommand.js +7 -3
- package/dist/{scripts/experiments/describeExperiment.js → src/commands/runExperimentDescribeCommand.js} +14 -10
- package/dist/{scripts/experiments/listExperiments.js → src/commands/runExperimentListCommand.js} +3 -6
- package/dist/{scripts/experiments/runExperiment.js → src/commands/runExperimentRunCommand.js} +80 -22
- package/dist/src/commands/runFinalDemoCommand.js +2 -3
- package/dist/src/commands/runSecurityValidationCommand.js +271 -0
- package/dist/src/commands/runTutorialRunCommand.js +170 -0
- package/dist/src/commands/runTutorialValidateCommand.js +138 -0
- package/dist/src/core/processTree.js +52 -0
- package/dist/src/core/resolveCommand.js +34 -0
- package/dist/src/core/runMeasuredCommand.js +39 -31
- package/dist/src/runtime/index.js +4 -0
- package/dist/src/runtime/labExecutionContext.js +22 -0
- package/dist/src/runtime/managedProcess.js +380 -0
- package/dist/src/runtime/packageResource.js +22 -0
- package/dist/src/runtime/packageRoot.js +37 -0
- package/dist/src/screenshot/captureReportScreenshot.js +30 -27
- package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +14 -1
- package/dist/src/tutorial/index.js +16 -0
- package/dist/src/tutorial/loadTutorialContracts.js +60 -0
- package/dist/src/tutorial/markdownTutorialWriter.js +53 -0
- package/dist/src/tutorial/runTutorial.js +465 -0
- package/dist/src/tutorial/scenarioValidation.js +383 -0
- package/dist/src/tutorial/subtitleWriter.js +88 -0
- package/dist/src/tutorial/targetContractValidation.js +212 -0
- package/dist/src/tutorial/tutorialActions.js +110 -0
- package/dist/src/tutorial/tutorialArtifacts.js +176 -0
- package/dist/src/tutorial/tutorialAssertions.js +249 -0
- package/dist/src/tutorial/tutorialCursor.js +149 -0
- package/dist/src/tutorial/tutorialLocators.js +37 -0
- package/dist/src/tutorial/tutorialManifest.js +74 -0
- package/dist/src/tutorial/tutorialOverlay.js +180 -0
- package/dist/src/tutorial/tutorialPaths.js +86 -0
- package/dist/src/tutorial/tutorialSession.js +398 -0
- package/dist/src/tutorial/types.js +48 -0
- package/dist/src/tutorial/validationPrimitives.js +141 -0
- package/examples/tutorial-browser/index.html +47 -0
- package/examples/tutorial-browser/prepare.mjs +14 -0
- package/examples/tutorial-browser/scenario.json +17 -0
- package/examples/tutorial-browser/server.mjs +14 -0
- package/package.json +13 -9
- package/dist/scripts/audits/runAudit.js +0 -135
- package/dist/scripts/build-gallery.js +0 -3
- package/dist/scripts/capture-demo-report.js +0 -3
- package/dist/scripts/evaluate-token-savings.js +0 -2
- package/dist/scripts/generate-experiment-plots.js +0 -3
- package/dist/scripts/generate-prompt-variants.js +0 -2
- package/dist/scripts/render-experiment-report.js +0 -2
- package/dist/scripts/report-context-integrity-smoke.js +0 -70
- package/dist/scripts/run-agent-prompt.js +0 -2
- package/dist/scripts/run-controlled-experiment.js +0 -2
- package/dist/scripts/run-lab-demo.js +0 -5
- package/dist/scripts/run-visualization-demos.js +0 -3
- package/dist/scripts/security/resolveToolRoot.js +0 -16
- package/dist/scripts/security/runCodeql.js +0 -58
- package/dist/scripts/security/runDependencyChecks.js +0 -58
- package/dist/scripts/security/runFuzzSmoke.js +0 -29
- package/dist/scripts/security/runPackageChecks.js +0 -57
- package/dist/scripts/security/runSemgrep.js +0 -64
- package/dist/scripts/security/validate.js +0 -266
- package/dist/scripts/verify-benchmarks.js +0 -202
- package/examples/demo-report-input.json +0 -78
- package/examples/lab-demo-cases.json +0 -35
- package/examples/real-agent-campaign-cases.json +0 -118
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/debug/res/xml/ambiguous_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/ambiguous_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/backup_rules.xml +0 -5
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/broad_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/data_extraction_rules.xml +0 -10
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/unsupported_domain_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-b/src/main/res/xml/malformed_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-b/src/main/res/xml/unsupported_root.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/debug/res/xml/ambiguous_config.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/ambiguous_config.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/base_cleartext_false.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/broad_and_duplicate_domains.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/empty_domain.xml +0 -6
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/malformed_pin.xml +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/network_security_config.xml +0 -26
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-b/src/main/res/xml/malformed_network_security_config.xml +0 -6
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-b/src/main/res/xml/unsupported_root.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-dynamic.gradle +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-false.gradle +0 -9
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-true.gradle +0 -13
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/kotlin-release-debuggable-false.gradle.kts +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/kotlin-release-debuggable-true.gradle.kts +0 -10
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/missing-release-build-type.gradle.kts +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/debug/res/xml/network_security_config.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/main/res/xml/backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/main/res/xml/network_security_config.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-b/src/main/res/xml/malformed_network_security_config.xml +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/SampleActivity.java +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/SampleActivity.kt +0 -9
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/build.gradle +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/build.gradle.kts +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/google-services.json +0 -16
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/local.properties +0 -3
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates-bounds/module/binary.properties +0 -0
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates-bounds/module/oversized.json +0 -1
- package/tests/fixtures/android/advanced-security-fixtures/signing-configuration/app/fake-release-keystore.jks +0 -1
- package/tests/fixtures/android/advanced-security-fixtures/signing-configuration/build/generated-fake.keystore +0 -1
- package/tests/fixtures/android/compose-app/app/build.gradle.kts +0 -21
- package/tests/fixtures/android/compose-app/app/src/main/AndroidManifest.xml +0 -22
- package/tests/fixtures/android/compose-app/app/src/main/java/com/example/composeapp/MainActivity.kt +0 -11
- package/tests/fixtures/android/compose-app/build.gradle.kts +0 -4
- package/tests/fixtures/android/compose-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/compose-app/settings.gradle.kts +0 -2
- package/tests/fixtures/android/gradle-metadata-fixtures/catalog-only-plugin/build.gradle.kts +0 -3
- package/tests/fixtures/android/gradle-metadata-fixtures/catalog-only-plugin/gradle/libs.versions.toml +0 -5
- package/tests/fixtures/android/gradle-metadata-fixtures/comment-safety.gradle.kts +0 -21
- package/tests/fixtures/android/gradle-metadata-fixtures/dynamic-values.gradle.kts +0 -17
- package/tests/fixtures/android/gradle-metadata-fixtures/groovy-literal.gradle +0 -23
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-malformed.properties +0 -4
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-missing-url.properties +0 -3
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-with-checksum.properties +0 -5
- package/tests/fixtures/android/java-app/app/build.gradle +0 -16
- package/tests/fixtures/android/java-app/app/src/main/AndroidManifest.xml +0 -19
- package/tests/fixtures/android/java-app/app/src/main/java/com/example/javaapp/MainActivity.java +0 -4
- package/tests/fixtures/android/java-app/build.gradle +0 -3
- package/tests/fixtures/android/java-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/java-app/settings.gradle +0 -2
- package/tests/fixtures/android/library/build.gradle.kts +0 -3
- package/tests/fixtures/android/library/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/library/mylibrary/build.gradle.kts +0 -13
- package/tests/fixtures/android/library/mylibrary/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/library/settings.gradle.kts +0 -2
- package/tests/fixtures/android/manifest-audit-fixtures/incomplete-intent-filter.xml +0 -14
- package/tests/fixtures/android/manifest-audit-fixtures/malformed-exported.xml +0 -10
- package/tests/fixtures/android/manifest-audit-fixtures/malformed.xml +0 -5
- package/tests/fixtures/android/manifest-audit-fixtures/missing-application.xml +0 -7
- package/tests/fixtures/android/manifest-audit-fixtures/missing-component-name.xml +0 -11
- package/tests/fixtures/android/manifest-audit-fixtures/permission-coverage.xml +0 -29
- package/tests/fixtures/android/manifest-audit-fixtures/risky-components.xml +0 -62
- package/tests/fixtures/android/manifest-audit-fixtures/risky-deep-links.xml +0 -45
- package/tests/fixtures/android/manifest-audit-fixtures/unresolved-placeholder.xml +0 -14
- package/tests/fixtures/android/manifest-audit-fixtures/unusual-namespace-prefix.xml +0 -11
- package/tests/fixtures/android/multi-module/app/build.gradle.kts +0 -22
- package/tests/fixtures/android/multi-module/app/src/androidTest/java/com/example/multimodule/app/ExampleInstrumentedTest.kt +0 -3
- package/tests/fixtures/android/multi-module/app/src/main/AndroidManifest.xml +0 -11
- package/tests/fixtures/android/multi-module/app/src/test/java/com/example/multimodule/app/ExampleUnitTest.kt +0 -3
- package/tests/fixtures/android/multi-module/build.gradle.kts +0 -5
- package/tests/fixtures/android/multi-module/core/build.gradle.kts +0 -14
- package/tests/fixtures/android/multi-module/core/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/multi-module/feature-login/build.gradle.kts +0 -18
- package/tests/fixtures/android/multi-module/feature-login/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/multi-module/gradle/libs.versions.toml +0 -8
- package/tests/fixtures/android/multi-module/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/multi-module/settings.gradle.kts +0 -4
- package/tests/fixtures/android/non-android-gradle/build.gradle.kts +0 -3
- package/tests/fixtures/android/non-android-gradle/core/build.gradle.kts +0 -3
- package/tests/fixtures/android/non-android-gradle/core/src/main/java/com/example/core/Core.java +0 -4
- package/tests/fixtures/android/non-android-gradle/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/non-android-gradle/settings.gradle.kts +0 -2
- package/tests/fixtures/android/partial/app/build.gradle.kts +0 -7
- package/tests/fixtures/android/partial/settings.gradle.kts +0 -2
- package/tests/fixtures/android/xml-view-app/app/build.gradle.kts +0 -17
- package/tests/fixtures/android/xml-view-app/app/src/main/AndroidManifest.xml +0 -31
- package/tests/fixtures/android/xml-view-app/app/src/main/kotlin/com/example/xmlviewapp/MainActivity.kt +0 -7
- package/tests/fixtures/android/xml-view-app/app/src/main/res/layout/activity_main.xml +0 -4
- package/tests/fixtures/android/xml-view-app/build.gradle.kts +0 -4
- package/tests/fixtures/android/xml-view-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/xml-view-app/settings.gradle.kts +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/README.md +0 -16
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/build.gradle.kts +0 -4
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/src/main/java/com/example/Widget.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/src/main/kotlin/com/example/Gadget.kt +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/build/Main.class +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/dist/index.js +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/generated/schema.ts +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/vendor/lib.py +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/build.gradle.kts +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/settings.gradle.kts +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/main/java/com/example/FooService.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/main/kotlin/com/example/BarService.kt +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/test/java/com/example/FooServiceTest.java +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/test/kotlin/com/example/BarServiceTest.kt +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/README.md +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/src/app.py +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/tests/test_app.py +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/README.md +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/src/helper.js +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/tests/index.sample.ts +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/build.gradle.kts +0 -4
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/pytests/test_app.py +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/app.py +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/helper.js +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/main/java/com/example/Multiplier.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/main/kotlin/com/example/Divider.kt +0 -7
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/derived/run-integrity-evidence.json +0 -99
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/producer/implementation-audit.json +0 -1585
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/producer/implementation-capsule.json +0 -11455
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/00-request.txt +0 -1
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/architecture-context-packet.txt +0 -82
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/behavior-model.txt +0 -104
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/final-report.txt +0 -41
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/implementation-context-packet.txt +0 -114
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/implementation-report.txt +0 -92
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/judge-report.txt +0 -34
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/pseudocode-packet.txt +0 -244
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/request-brief.txt +0 -40
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-context-packet.txt +0 -93
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-implementation-report.txt +0 -69
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-strategy-packet.txt +0 -354
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/verification-report.txt +0 -85
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/derived/run-integrity-evidence.json +0 -98
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/architecture-audit.json +0 -1169
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/architecture-request.json +0 -29
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-audit.json +0 -1494
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-capsule.json +0 -11364
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-request.json +0 -33
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/test-request.json +0 -74
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/architecture-context-retrieval-report.txt +0 -58
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/implementation-context-retrieval-report.txt +0 -75
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/test-context-retrieval-report.txt +0 -84
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/run.json +0 -62
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/tool-versions/my-dev-kit-version.txt +0 -1
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/tool-versions/orchestrator-version.txt +0 -1
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/corrected-replay-manifest.json +0 -19
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/failed-run-manifest.json +0 -56
- package/tests/fixtures/fake-adversarial-cli.js +0 -152
- package/tests/fixtures/fake-my-dev-kit-cli.js +0 -83
- package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +0 -24
- package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +0 -25
- package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +0 -230
- package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +0 -232
- package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +0 -1
- package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +0 -1
- package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +0 -18
- package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +0 -1
- package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +0 -1
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +0 -504
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +0 -955
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +0 -190
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +0 -327
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/PROVENANCE.txt +0 -18
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/context-capsule/complete-v1.0.0.json +0 -556
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/retrieval-audit-record/complete-v1.0.0.json +0 -238
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +0 -13
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +0 -7874
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +0 -1076
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +0 -160
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +0 -186
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/PROVENANCE.txt +0 -31
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/ready-pass-final-pass.json +0 -132
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/refresh-required-need-context.json +0 -146
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { REQUIRED_BENCHMARK_PROJECT_IDS, parseBenchmarkProjectProfiles, validateAnswerKey, validateBenchmarkProjectProfiles } from "../src/evaluation/benchmarkMetadata.js";
|
|
5
|
-
const requiredProjects = REQUIRED_BENCHMARK_PROJECT_IDS;
|
|
6
|
-
const projectRequiredPaths = {
|
|
7
|
-
"todo-ts": [
|
|
8
|
-
"README.md",
|
|
9
|
-
"package.json",
|
|
10
|
-
"tsconfig.json",
|
|
11
|
-
"src/taskStore.ts",
|
|
12
|
-
"src/taskService.ts",
|
|
13
|
-
"src/index.ts",
|
|
14
|
-
"tests/taskService.test.ts"
|
|
15
|
-
],
|
|
16
|
-
"todo-python": [
|
|
17
|
-
"README.md",
|
|
18
|
-
"src/task_store.py",
|
|
19
|
-
"src/task_service.py",
|
|
20
|
-
"src/__init__.py",
|
|
21
|
-
"tests/test_task_service.py"
|
|
22
|
-
],
|
|
23
|
-
"todo-js": [
|
|
24
|
-
"README.md",
|
|
25
|
-
"package.json",
|
|
26
|
-
"src/taskStore.js",
|
|
27
|
-
"src/taskService.js",
|
|
28
|
-
"src/index.js",
|
|
29
|
-
"tests/taskService.test.js"
|
|
30
|
-
],
|
|
31
|
-
"todo-mixed-ts-py": [
|
|
32
|
-
"README.md",
|
|
33
|
-
"package.json",
|
|
34
|
-
"tsconfig.json",
|
|
35
|
-
"src/taskCli.ts",
|
|
36
|
-
"python/task_service.py",
|
|
37
|
-
"tests/mixedBoundary.test.ts"
|
|
38
|
-
],
|
|
39
|
-
"task-workflow-medium-ts": [
|
|
40
|
-
"README.md",
|
|
41
|
-
"package.json",
|
|
42
|
-
"tsconfig.json",
|
|
43
|
-
"src/store/taskStore.ts",
|
|
44
|
-
"src/services/createTask.ts",
|
|
45
|
-
"src/services/importTasks.ts",
|
|
46
|
-
"src/services/summarizeTasks.ts",
|
|
47
|
-
"tests/importTasks.test.ts"
|
|
48
|
-
],
|
|
49
|
-
"task-analytics-large-mixed": [
|
|
50
|
-
"README.md",
|
|
51
|
-
"ts/package.json",
|
|
52
|
-
"ts/tsconfig.json",
|
|
53
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
54
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
55
|
-
"ts/tests/buildAnalyticsSnapshot.test.ts",
|
|
56
|
-
"py/task_analytics/metrics.py",
|
|
57
|
-
"py/task_analytics/quality.py",
|
|
58
|
-
"py/tests/test_reporting.py"
|
|
59
|
-
]
|
|
60
|
-
};
|
|
61
|
-
function walk(dir) {
|
|
62
|
-
const entries = readdirSync(dir, { withFileTypes: true });
|
|
63
|
-
const files = [];
|
|
64
|
-
for (const entry of entries) {
|
|
65
|
-
const fullPath = path.join(dir, entry.name);
|
|
66
|
-
if (entry.isDirectory()) {
|
|
67
|
-
files.push(...walk(fullPath));
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
70
|
-
files.push(fullPath);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
return files;
|
|
74
|
-
}
|
|
75
|
-
export function validateBenchmarks(rootDir = process.cwd()) {
|
|
76
|
-
const checks = [];
|
|
77
|
-
const errors = [];
|
|
78
|
-
const contractsDir = path.join(rootDir, "benchmarks", "contracts");
|
|
79
|
-
const projectsDir = path.join(rootDir, "benchmarks", "projects");
|
|
80
|
-
const behaviorPath = path.join(contractsDir, "todo-behavior.md");
|
|
81
|
-
const casesPath = path.join(contractsDir, "todo-benchmark-case.json");
|
|
82
|
-
const profilesPath = path.join(contractsDir, "benchmark-project-profiles.json");
|
|
83
|
-
if (!existsSync(behaviorPath)) {
|
|
84
|
-
errors.push("Missing contract file: benchmarks/contracts/todo-behavior.md");
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
checks.push("found todo-behavior.md");
|
|
88
|
-
}
|
|
89
|
-
let cases = [];
|
|
90
|
-
if (!existsSync(casesPath)) {
|
|
91
|
-
errors.push("Missing contract file: benchmarks/contracts/todo-benchmark-case.json");
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
try {
|
|
95
|
-
cases = JSON.parse(readFileSync(casesPath, "utf8"));
|
|
96
|
-
checks.push("parsed todo-benchmark-case.json");
|
|
97
|
-
}
|
|
98
|
-
catch (error) {
|
|
99
|
-
errors.push(`Invalid JSON in todo-benchmark-case.json: ${error.message}`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
if (!existsSync(profilesPath)) {
|
|
103
|
-
errors.push("Missing contract file: benchmarks/contracts/benchmark-project-profiles.json");
|
|
104
|
-
}
|
|
105
|
-
else {
|
|
106
|
-
try {
|
|
107
|
-
const profiles = parseBenchmarkProjectProfiles(JSON.parse(readFileSync(profilesPath, "utf8")));
|
|
108
|
-
const profileErrors = validateBenchmarkProjectProfiles(profiles, rootDir);
|
|
109
|
-
errors.push(...profileErrors);
|
|
110
|
-
if (profileErrors.length === 0) {
|
|
111
|
-
checks.push("validated benchmark-project-profiles.json");
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
catch (error) {
|
|
115
|
-
errors.push(`Invalid benchmark-project-profiles.json: ${error.message}`);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
const ids = new Set();
|
|
119
|
-
for (const benchmarkCase of cases) {
|
|
120
|
-
if (ids.has(benchmarkCase.id)) {
|
|
121
|
-
errors.push(`Duplicate benchmark case id: ${benchmarkCase.id}`);
|
|
122
|
-
}
|
|
123
|
-
ids.add(benchmarkCase.id);
|
|
124
|
-
}
|
|
125
|
-
if (cases.length > 0 && errors.every((error) => !error.startsWith("Duplicate benchmark case id:"))) {
|
|
126
|
-
checks.push("benchmark case ids are unique");
|
|
127
|
-
}
|
|
128
|
-
for (const project of requiredProjects) {
|
|
129
|
-
const projectDir = path.join(projectsDir, project);
|
|
130
|
-
if (!existsSync(projectDir) || !statSync(projectDir).isDirectory()) {
|
|
131
|
-
errors.push(`Missing benchmark project: benchmarks/projects/${project}`);
|
|
132
|
-
continue;
|
|
133
|
-
}
|
|
134
|
-
checks.push(`found benchmark project ${project}`);
|
|
135
|
-
for (const relPath of projectRequiredPaths[project]) {
|
|
136
|
-
const fullPath = path.join(projectDir, relPath);
|
|
137
|
-
if (!existsSync(fullPath)) {
|
|
138
|
-
errors.push(`Missing required file for ${project}: benchmarks/projects/${project}/${relPath}`);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
for (const benchmarkCase of cases) {
|
|
143
|
-
if (!benchmarkCase.answerKey) {
|
|
144
|
-
errors.push(`Case ${benchmarkCase.id} does not define answerKey`);
|
|
145
|
-
}
|
|
146
|
-
else {
|
|
147
|
-
errors.push(...validateAnswerKey(benchmarkCase.answerKey, `Case ${benchmarkCase.id}`));
|
|
148
|
-
}
|
|
149
|
-
if (!Array.isArray(benchmarkCase.expectedSymbols) || benchmarkCase.expectedSymbols.length === 0) {
|
|
150
|
-
errors.push(`Case ${benchmarkCase.id} does not define expectedSymbols`);
|
|
151
|
-
}
|
|
152
|
-
for (const [project, expectedFiles] of Object.entries(benchmarkCase.expectedFilesByProject ?? {})) {
|
|
153
|
-
if (!requiredProjects.includes(project)) {
|
|
154
|
-
errors.push(`Case ${benchmarkCase.id} references unknown project id: ${project}`);
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (!Array.isArray(expectedFiles) || expectedFiles.length === 0) {
|
|
158
|
-
errors.push(`Case ${benchmarkCase.id} does not define expected files for ${project}`);
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
for (const expectedFile of expectedFiles) {
|
|
162
|
-
const fullPath = path.join(rootDir, "benchmarks", "projects", project, expectedFile);
|
|
163
|
-
if (!existsSync(fullPath)) {
|
|
164
|
-
errors.push(`Case ${benchmarkCase.id} references missing file: benchmarks/projects/${project}/${expectedFile}`);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
for (const project of requiredProjects) {
|
|
170
|
-
const projectDir = path.join(projectsDir, project);
|
|
171
|
-
if (!existsSync(projectDir)) {
|
|
172
|
-
continue;
|
|
173
|
-
}
|
|
174
|
-
const forbidden = walk(projectDir).filter((fullPath) => {
|
|
175
|
-
const rel = path.relative(projectDir, fullPath).replace(/\\/g, "/");
|
|
176
|
-
return /(^|\/)(node_modules|dist|build|coverage|lab-output)(\/|$)/.test(rel);
|
|
177
|
-
});
|
|
178
|
-
if (forbidden.length > 0) {
|
|
179
|
-
errors.push(`Forbidden generated output found in ${project}: ${forbidden[0]}`);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
return { ok: errors.length === 0, errors, checks };
|
|
183
|
-
}
|
|
184
|
-
function printSummary(result) {
|
|
185
|
-
console.log(`Benchmark verification ${result.ok ? "passed" : "failed"}.`);
|
|
186
|
-
console.log(`Checks: ${result.checks.length}`);
|
|
187
|
-
if (result.errors.length > 0) {
|
|
188
|
-
console.log(`Errors: ${result.errors.length}`);
|
|
189
|
-
for (const error of result.errors) {
|
|
190
|
-
console.log(`- ${error}`);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
const currentFile = fileURLToPath(import.meta.url);
|
|
195
|
-
const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : "";
|
|
196
|
-
if (invokedPath === currentFile) {
|
|
197
|
-
const result = validateBenchmarks();
|
|
198
|
-
printSummary(result);
|
|
199
|
-
if (!result.ok) {
|
|
200
|
-
process.exitCode = 1;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"reportId": "demo-report",
|
|
3
|
-
"title": "Benchmark Validation Demo Report",
|
|
4
|
-
"projectName": "my-dev-kit-lab",
|
|
5
|
-
"benchmarkProject": "todo-ts",
|
|
6
|
-
"workflowName": "benchmark validation demo",
|
|
7
|
-
"generatedAt": "2026-06-10T15:00:00.000Z",
|
|
8
|
-
"summary": "Deterministic sample report for Prompt 2 report rendering and screenshot capture. This report does not claim token-savings evaluation.",
|
|
9
|
-
"steps": [
|
|
10
|
-
{
|
|
11
|
-
"id": "load-contracts",
|
|
12
|
-
"label": "Load benchmark contracts",
|
|
13
|
-
"command": "npm run verify:benchmarks",
|
|
14
|
-
"status": "pass",
|
|
15
|
-
"durationMs": 42,
|
|
16
|
-
"notes": "Validated benchmark contract structure."
|
|
17
|
-
},
|
|
18
|
-
{
|
|
19
|
-
"id": "run-project-tests",
|
|
20
|
-
"label": "Run benchmark project tests",
|
|
21
|
-
"command": "npm run test:benchmarks",
|
|
22
|
-
"status": "pass",
|
|
23
|
-
"durationMs": 118,
|
|
24
|
-
"notes": "Todo benchmark behavior stayed aligned."
|
|
25
|
-
},
|
|
26
|
-
{
|
|
27
|
-
"id": "prepare-report",
|
|
28
|
-
"label": "Prepare report artifacts",
|
|
29
|
-
"command": "npm run capture-demo-report",
|
|
30
|
-
"status": "pass",
|
|
31
|
-
"durationMs": 35,
|
|
32
|
-
"notes": "No token-savings metrics are reported in this demo."
|
|
33
|
-
}
|
|
34
|
-
],
|
|
35
|
-
"metrics": [
|
|
36
|
-
{
|
|
37
|
-
"id": "benchmark-project-count",
|
|
38
|
-
"label": "Benchmark projects",
|
|
39
|
-
"value": 4,
|
|
40
|
-
"interpretation": "Prompt 1 benchmark suite is available."
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
"id": "step-count",
|
|
44
|
-
"label": "Workflow steps",
|
|
45
|
-
"value": 3,
|
|
46
|
-
"interpretation": "Demo workflow uses a small deterministic sequence."
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
"id": "warning-count",
|
|
50
|
-
"label": "Warnings",
|
|
51
|
-
"value": 1,
|
|
52
|
-
"interpretation": "This demo explicitly notes that token comparison is not implemented yet."
|
|
53
|
-
},
|
|
54
|
-
{
|
|
55
|
-
"id": "artifact-count",
|
|
56
|
-
"label": "Artifacts declared",
|
|
57
|
-
"value": 2,
|
|
58
|
-
"interpretation": "JSON and HTML are always expected before optional PNG capture."
|
|
59
|
-
}
|
|
60
|
-
],
|
|
61
|
-
"artifacts": [
|
|
62
|
-
{
|
|
63
|
-
"id": "report-json",
|
|
64
|
-
"label": "Demo report JSON",
|
|
65
|
-
"path": "lab-output/demo-report/demo-report.json",
|
|
66
|
-
"kind": "json"
|
|
67
|
-
},
|
|
68
|
-
{
|
|
69
|
-
"id": "report-html",
|
|
70
|
-
"label": "Demo report HTML",
|
|
71
|
-
"path": "lab-output/demo-report/demo-report.html",
|
|
72
|
-
"kind": "html"
|
|
73
|
-
}
|
|
74
|
-
],
|
|
75
|
-
"warnings": [
|
|
76
|
-
"Token-savings evaluation is not implemented in Prompt 2."
|
|
77
|
-
]
|
|
78
|
-
}
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"id": "todo-ts-create-task-demo",
|
|
4
|
-
"title": "Todo TS create task demo retrieval",
|
|
5
|
-
"benchmarkProject": "todo-ts",
|
|
6
|
-
"projectProfileRef": "todo-ts",
|
|
7
|
-
"targetRoot": "benchmarks/projects/todo-ts",
|
|
8
|
-
"sourceRoots": ["src", "tests"],
|
|
9
|
-
"query": "create task deterministic id task-1",
|
|
10
|
-
"expectedFiles": ["src/taskService.ts", "src/taskStore.ts"],
|
|
11
|
-
"expectedSymbols": ["createTask", "TaskService"],
|
|
12
|
-
"answerKey": {
|
|
13
|
-
"expectedFiles": ["src/taskService.ts", "src/taskStore.ts"],
|
|
14
|
-
"expectedSymbols": ["createTask", "TaskService"],
|
|
15
|
-
"expectedFacts": [
|
|
16
|
-
{
|
|
17
|
-
"id": "create-deterministic-id",
|
|
18
|
-
"text": "createTask assigns deterministic IDs such as task-1 and task-2.",
|
|
19
|
-
"weight": 1,
|
|
20
|
-
"required": true
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
"id": "create-stores-incomplete-task",
|
|
24
|
-
"text": "createTask stores new tasks with completed set to false.",
|
|
25
|
-
"weight": 1,
|
|
26
|
-
"required": true
|
|
27
|
-
}
|
|
28
|
-
],
|
|
29
|
-
"minimumCorrectFacts": 2,
|
|
30
|
-
"notes": "Demo metadata prepares later correctness scoring without changing the current lab-demo command."
|
|
31
|
-
},
|
|
32
|
-
"rawIncludeGlobs": ["src/**/*", "tests/**/*"],
|
|
33
|
-
"notes": "Small single-case lab demo fixture."
|
|
34
|
-
}
|
|
35
|
-
]
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"id": "task-workflow-import-dedup",
|
|
4
|
-
"title": "Task workflow import and dedupe retrieval",
|
|
5
|
-
"benchmarkProject": "task-workflow-medium-ts",
|
|
6
|
-
"projectProfileRef": "task-workflow-medium-ts",
|
|
7
|
-
"targetRoot": "benchmarks/projects/task-workflow-medium-ts",
|
|
8
|
-
"sourceRoots": ["src", "tests"],
|
|
9
|
-
"query": "import tasks duplicate normalized title deterministic id summarize workflow project",
|
|
10
|
-
"expectedFiles": [
|
|
11
|
-
"src/services/importTasks.ts",
|
|
12
|
-
"src/store/taskStore.ts",
|
|
13
|
-
"src/validation/taskValidation.ts"
|
|
14
|
-
],
|
|
15
|
-
"expectedSymbols": ["importTasks", "validateImportInput", "findDuplicate", "createTask"],
|
|
16
|
-
"answerKey": {
|
|
17
|
-
"expectedFiles": [
|
|
18
|
-
"src/services/importTasks.ts",
|
|
19
|
-
"src/store/taskStore.ts",
|
|
20
|
-
"src/validation/taskValidation.ts"
|
|
21
|
-
],
|
|
22
|
-
"expectedSymbols": ["importTasks", "validateImportInput", "findDuplicate", "createTask"],
|
|
23
|
-
"expectedFacts": [
|
|
24
|
-
{
|
|
25
|
-
"id": "workflow-import-validates-title-and-project",
|
|
26
|
-
"text": "importTasks validates normalized task titles and project ids before inserting records.",
|
|
27
|
-
"weight": 1,
|
|
28
|
-
"required": true
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
"id": "workflow-import-skips-duplicates-by-project-and-normalized-title",
|
|
32
|
-
"text": "importTasks skips duplicates when a task with the same normalized title already exists in the same project.",
|
|
33
|
-
"weight": 1,
|
|
34
|
-
"required": true
|
|
35
|
-
},
|
|
36
|
-
{
|
|
37
|
-
"id": "workflow-import-keeps-deterministic-task-sequences",
|
|
38
|
-
"text": "New imported tasks still receive deterministic ids such as task-1 and task-2 from the shared store sequence.",
|
|
39
|
-
"weight": 1,
|
|
40
|
-
"required": true
|
|
41
|
-
}
|
|
42
|
-
],
|
|
43
|
-
"minimumCorrectFacts": 3,
|
|
44
|
-
"notes": "This case is intended to require multiple files: service, validation, and store logic."
|
|
45
|
-
},
|
|
46
|
-
"rawIncludeGlobs": ["src/**/*", "tests/**/*"],
|
|
47
|
-
"notes": "Medium project case for real-agent comparison."
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
"id": "task-analytics-health-label",
|
|
51
|
-
"title": "Task analytics health label retrieval",
|
|
52
|
-
"benchmarkProject": "task-analytics-large-mixed",
|
|
53
|
-
"projectProfileRef": "task-analytics-large-mixed",
|
|
54
|
-
"targetRoot": "benchmarks/projects/task-analytics-large-mixed",
|
|
55
|
-
"sourceRoots": ["ts/src", "ts/tests", "py/task_analytics", "py/tests"],
|
|
56
|
-
"query": "completion rate stale tasks quality label health report build analytics snapshot mixed language",
|
|
57
|
-
"expectedFiles": [
|
|
58
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
59
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
60
|
-
"py/task_analytics/metrics.py",
|
|
61
|
-
"py/task_analytics/quality.py",
|
|
62
|
-
"py/task_analytics/reporting.py"
|
|
63
|
-
],
|
|
64
|
-
"expectedSymbols": [
|
|
65
|
-
"buildAnalyticsSnapshot",
|
|
66
|
-
"formatTaskHealthReport",
|
|
67
|
-
"calculate_project_metrics",
|
|
68
|
-
"determine_quality_label",
|
|
69
|
-
"build_health_report"
|
|
70
|
-
],
|
|
71
|
-
"answerKey": {
|
|
72
|
-
"expectedFiles": [
|
|
73
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
74
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
75
|
-
"py/task_analytics/metrics.py",
|
|
76
|
-
"py/task_analytics/quality.py",
|
|
77
|
-
"py/task_analytics/reporting.py"
|
|
78
|
-
],
|
|
79
|
-
"expectedSymbols": [
|
|
80
|
-
"buildAnalyticsSnapshot",
|
|
81
|
-
"formatTaskHealthReport",
|
|
82
|
-
"calculate_project_metrics",
|
|
83
|
-
"determine_quality_label",
|
|
84
|
-
"build_health_report"
|
|
85
|
-
],
|
|
86
|
-
"expectedFacts": [
|
|
87
|
-
{
|
|
88
|
-
"id": "analytics-snapshot-computes-completion-rate",
|
|
89
|
-
"text": "The analytics snapshot computes completionRate as completedTasks divided by totalTasks times 100, rounded to two decimals.",
|
|
90
|
-
"weight": 1,
|
|
91
|
-
"required": true
|
|
92
|
-
},
|
|
93
|
-
{
|
|
94
|
-
"id": "analytics-snapshot-counts-stale-open-tasks",
|
|
95
|
-
"text": "Stale tasks count only open tasks whose updated day is at least ten days behind the current day.",
|
|
96
|
-
"weight": 1,
|
|
97
|
-
"required": true
|
|
98
|
-
},
|
|
99
|
-
{
|
|
100
|
-
"id": "python-quality-labels-healthy-watch-risk",
|
|
101
|
-
"text": "The Python quality layer maps metrics to healthy, watch, or risk labels based on completion rate and stale task thresholds.",
|
|
102
|
-
"weight": 1,
|
|
103
|
-
"required": true
|
|
104
|
-
},
|
|
105
|
-
{
|
|
106
|
-
"id": "python-reporting-renders-quality-lines",
|
|
107
|
-
"text": "The Python report writer emits one line per project with the quality label plus completion, open, and stale counts.",
|
|
108
|
-
"weight": 1,
|
|
109
|
-
"required": true
|
|
110
|
-
}
|
|
111
|
-
],
|
|
112
|
-
"minimumCorrectFacts": 3,
|
|
113
|
-
"notes": "This case intentionally spans TypeScript snapshot building and Python quality reporting."
|
|
114
|
-
},
|
|
115
|
-
"rawIncludeGlobs": ["ts/src/**/*", "ts/tests/**/*", "py/task_analytics/**/*", "py/tests/**/*"],
|
|
116
|
-
"notes": "Large mixed-language case for real-agent comparison."
|
|
117
|
-
}
|
|
118
|
-
]
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
-
<data-extraction-rules>
|
|
3
|
-
<cloud-backup disableIfNoEncryptionCapabilities="true">
|
|
4
|
-
<include domain="sharedpref" path="." />
|
|
5
|
-
<exclude domain="database" path="fake-secrets.db" />
|
|
6
|
-
</cloud-backup>
|
|
7
|
-
<device-transfer>
|
|
8
|
-
<include domain="root" path="." />
|
|
9
|
-
</device-transfer>
|
|
10
|
-
</data-extraction-rules>
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
-
<network-security-config>
|
|
3
|
-
<domain-config>
|
|
4
|
-
<domain>example.org</domain>
|
|
5
|
-
<pin-set expiration="not-a-date">
|
|
6
|
-
<pin>MISSING_DIGEST_ATTRIBUTE_FAKE_PIN=</pin>
|
|
7
|
-
<pin digest="SHA-1">UNSUPPORTED_DIGEST_FAKE_PIN=</pin>
|
|
8
|
-
<pin digest="SHA-256"></pin>
|
|
9
|
-
</pin-set>
|
|
10
|
-
</domain-config>
|
|
11
|
-
</network-security-config>
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
-
<network-security-config>
|
|
3
|
-
<base-config cleartextTrafficPermitted="true">
|
|
4
|
-
<trust-anchors>
|
|
5
|
-
<certificates src="system" />
|
|
6
|
-
<certificates src="user" />
|
|
7
|
-
</trust-anchors>
|
|
8
|
-
</base-config>
|
|
9
|
-
<domain-config cleartextTrafficPermitted="true">
|
|
10
|
-
<domain includeSubdomains="true">example.com</domain>
|
|
11
|
-
<pin-set expiration="2030-01-01">
|
|
12
|
-
<pin digest="SHA-256">FAKEPINHASHFAKEPINHASHFAKEPINHASHFAKEPINHASH=</pin>
|
|
13
|
-
</pin-set>
|
|
14
|
-
<domain-config>
|
|
15
|
-
<domain includeSubdomains="false">nested.example.com</domain>
|
|
16
|
-
<trust-anchors>
|
|
17
|
-
<certificates src="@raw/my_custom_ca" />
|
|
18
|
-
</trust-anchors>
|
|
19
|
-
</domain-config>
|
|
20
|
-
</domain-config>
|
|
21
|
-
<debug-overrides>
|
|
22
|
-
<trust-anchors>
|
|
23
|
-
<certificates src="user" overridePins="true" />
|
|
24
|
-
</trust-anchors>
|
|
25
|
-
</debug-overrides>
|
|
26
|
-
</network-security-config>
|