@dailephd/my-dev-kit-lab 0.4.5 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +37 -3
- package/dist/scripts/cli.js +9 -0
- package/dist/src/audits/core/typescriptJavaScriptAnalyzer.js +30 -12
- package/dist/src/cli/help.js +300 -0
- package/dist/src/cli/index.js +2 -0
- package/dist/src/cli/runLabCli.js +229 -0
- package/dist/src/commands/runAuditCommand.js +132 -0
- package/dist/src/commands/runControlledExperimentCommand.js +7 -3
- package/dist/{scripts/experiments/describeExperiment.js → src/commands/runExperimentDescribeCommand.js} +14 -10
- package/dist/{scripts/experiments/listExperiments.js → src/commands/runExperimentListCommand.js} +3 -6
- package/dist/{scripts/experiments/runExperiment.js → src/commands/runExperimentRunCommand.js} +80 -22
- package/dist/src/commands/runFinalDemoCommand.js +2 -3
- package/dist/src/commands/runSecurityValidationCommand.js +271 -0
- package/dist/src/runtime/index.js +3 -0
- package/dist/src/runtime/labExecutionContext.js +22 -0
- package/dist/src/runtime/packageResource.js +22 -0
- package/dist/src/runtime/packageRoot.js +37 -0
- package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +14 -1
- package/package.json +7 -7
- package/dist/scripts/audits/runAudit.js +0 -135
- package/dist/scripts/build-gallery.js +0 -3
- package/dist/scripts/capture-demo-report.js +0 -3
- package/dist/scripts/evaluate-token-savings.js +0 -2
- package/dist/scripts/generate-experiment-plots.js +0 -3
- package/dist/scripts/generate-prompt-variants.js +0 -2
- package/dist/scripts/render-experiment-report.js +0 -2
- package/dist/scripts/report-context-integrity-smoke.js +0 -70
- package/dist/scripts/run-agent-prompt.js +0 -2
- package/dist/scripts/run-controlled-experiment.js +0 -2
- package/dist/scripts/run-lab-demo.js +0 -5
- package/dist/scripts/run-visualization-demos.js +0 -3
- package/dist/scripts/security/resolveToolRoot.js +0 -16
- package/dist/scripts/security/runCodeql.js +0 -58
- package/dist/scripts/security/runDependencyChecks.js +0 -58
- package/dist/scripts/security/runFuzzSmoke.js +0 -29
- package/dist/scripts/security/runPackageChecks.js +0 -57
- package/dist/scripts/security/runSemgrep.js +0 -64
- package/dist/scripts/security/validate.js +0 -266
- package/dist/scripts/verify-benchmarks.js +0 -202
- package/examples/demo-report-input.json +0 -78
- package/examples/lab-demo-cases.json +0 -35
- package/examples/real-agent-campaign-cases.json +0 -118
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/debug/res/xml/ambiguous_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/ambiguous_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/backup_rules.xml +0 -5
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/broad_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/data_extraction_rules.xml +0 -10
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-a/src/main/res/xml/unsupported_domain_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-b/src/main/res/xml/malformed_backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/backup-configuration/module-b/src/main/res/xml/unsupported_root.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/debug/res/xml/ambiguous_config.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/ambiguous_config.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/base_cleartext_false.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/broad_and_duplicate_domains.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/empty_domain.xml +0 -6
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/malformed_pin.xml +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-a/src/main/res/xml/network_security_config.xml +0 -26
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-b/src/main/res/xml/malformed_network_security_config.xml +0 -6
- package/tests/fixtures/android/advanced-security-fixtures/network-security/module-b/src/main/res/xml/unsupported_root.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-dynamic.gradle +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-false.gradle +0 -9
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/groovy-release-debuggable-true.gradle +0 -13
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/kotlin-release-debuggable-false.gradle.kts +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/kotlin-release-debuggable-true.gradle.kts +0 -10
- package/tests/fixtures/android/advanced-security-fixtures/release-configuration/missing-release-build-type.gradle.kts +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/debug/res/xml/network_security_config.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/main/res/xml/backup_rules.xml +0 -4
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-a/src/main/res/xml/network_security_config.xml +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/resource-resolution/module-b/src/main/res/xml/malformed_network_security_config.xml +0 -7
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/SampleActivity.java +0 -8
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/SampleActivity.kt +0 -9
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/build.gradle +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/build.gradle.kts +0 -11
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/google-services.json +0 -16
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates/local.properties +0 -3
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates-bounds/module/binary.properties +0 -0
- package/tests/fixtures/android/advanced-security-fixtures/secret-candidates-bounds/module/oversized.json +0 -1
- package/tests/fixtures/android/advanced-security-fixtures/signing-configuration/app/fake-release-keystore.jks +0 -1
- package/tests/fixtures/android/advanced-security-fixtures/signing-configuration/build/generated-fake.keystore +0 -1
- package/tests/fixtures/android/compose-app/app/build.gradle.kts +0 -21
- package/tests/fixtures/android/compose-app/app/src/main/AndroidManifest.xml +0 -22
- package/tests/fixtures/android/compose-app/app/src/main/java/com/example/composeapp/MainActivity.kt +0 -11
- package/tests/fixtures/android/compose-app/build.gradle.kts +0 -4
- package/tests/fixtures/android/compose-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/compose-app/settings.gradle.kts +0 -2
- package/tests/fixtures/android/gradle-metadata-fixtures/catalog-only-plugin/build.gradle.kts +0 -3
- package/tests/fixtures/android/gradle-metadata-fixtures/catalog-only-plugin/gradle/libs.versions.toml +0 -5
- package/tests/fixtures/android/gradle-metadata-fixtures/comment-safety.gradle.kts +0 -21
- package/tests/fixtures/android/gradle-metadata-fixtures/dynamic-values.gradle.kts +0 -17
- package/tests/fixtures/android/gradle-metadata-fixtures/groovy-literal.gradle +0 -23
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-malformed.properties +0 -4
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-missing-url.properties +0 -3
- package/tests/fixtures/android/gradle-metadata-fixtures/wrapper-with-checksum.properties +0 -5
- package/tests/fixtures/android/java-app/app/build.gradle +0 -16
- package/tests/fixtures/android/java-app/app/src/main/AndroidManifest.xml +0 -19
- package/tests/fixtures/android/java-app/app/src/main/java/com/example/javaapp/MainActivity.java +0 -4
- package/tests/fixtures/android/java-app/build.gradle +0 -3
- package/tests/fixtures/android/java-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/java-app/settings.gradle +0 -2
- package/tests/fixtures/android/library/build.gradle.kts +0 -3
- package/tests/fixtures/android/library/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/library/mylibrary/build.gradle.kts +0 -13
- package/tests/fixtures/android/library/mylibrary/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/library/settings.gradle.kts +0 -2
- package/tests/fixtures/android/manifest-audit-fixtures/incomplete-intent-filter.xml +0 -14
- package/tests/fixtures/android/manifest-audit-fixtures/malformed-exported.xml +0 -10
- package/tests/fixtures/android/manifest-audit-fixtures/malformed.xml +0 -5
- package/tests/fixtures/android/manifest-audit-fixtures/missing-application.xml +0 -7
- package/tests/fixtures/android/manifest-audit-fixtures/missing-component-name.xml +0 -11
- package/tests/fixtures/android/manifest-audit-fixtures/permission-coverage.xml +0 -29
- package/tests/fixtures/android/manifest-audit-fixtures/risky-components.xml +0 -62
- package/tests/fixtures/android/manifest-audit-fixtures/risky-deep-links.xml +0 -45
- package/tests/fixtures/android/manifest-audit-fixtures/unresolved-placeholder.xml +0 -14
- package/tests/fixtures/android/manifest-audit-fixtures/unusual-namespace-prefix.xml +0 -11
- package/tests/fixtures/android/multi-module/app/build.gradle.kts +0 -22
- package/tests/fixtures/android/multi-module/app/src/androidTest/java/com/example/multimodule/app/ExampleInstrumentedTest.kt +0 -3
- package/tests/fixtures/android/multi-module/app/src/main/AndroidManifest.xml +0 -11
- package/tests/fixtures/android/multi-module/app/src/test/java/com/example/multimodule/app/ExampleUnitTest.kt +0 -3
- package/tests/fixtures/android/multi-module/build.gradle.kts +0 -5
- package/tests/fixtures/android/multi-module/core/build.gradle.kts +0 -14
- package/tests/fixtures/android/multi-module/core/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/multi-module/feature-login/build.gradle.kts +0 -18
- package/tests/fixtures/android/multi-module/feature-login/src/main/AndroidManifest.xml +0 -2
- package/tests/fixtures/android/multi-module/gradle/libs.versions.toml +0 -8
- package/tests/fixtures/android/multi-module/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/multi-module/settings.gradle.kts +0 -4
- package/tests/fixtures/android/non-android-gradle/build.gradle.kts +0 -3
- package/tests/fixtures/android/non-android-gradle/core/build.gradle.kts +0 -3
- package/tests/fixtures/android/non-android-gradle/core/src/main/java/com/example/core/Core.java +0 -4
- package/tests/fixtures/android/non-android-gradle/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/non-android-gradle/settings.gradle.kts +0 -2
- package/tests/fixtures/android/partial/app/build.gradle.kts +0 -7
- package/tests/fixtures/android/partial/settings.gradle.kts +0 -2
- package/tests/fixtures/android/xml-view-app/app/build.gradle.kts +0 -17
- package/tests/fixtures/android/xml-view-app/app/src/main/AndroidManifest.xml +0 -31
- package/tests/fixtures/android/xml-view-app/app/src/main/kotlin/com/example/xmlviewapp/MainActivity.kt +0 -7
- package/tests/fixtures/android/xml-view-app/app/src/main/res/layout/activity_main.xml +0 -4
- package/tests/fixtures/android/xml-view-app/build.gradle.kts +0 -4
- package/tests/fixtures/android/xml-view-app/gradle/wrapper/gradle-wrapper.properties +0 -4
- package/tests/fixtures/android/xml-view-app/settings.gradle.kts +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/README.md +0 -16
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/build.gradle.kts +0 -4
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/src/main/java/com/example/Widget.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-docs-claims/src/main/kotlin/com/example/Gadget.kt +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/build/Main.class +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/dist/index.js +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/generated/schema.ts +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-generated-vendor/vendor/lib.py +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/build.gradle.kts +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/settings.gradle.kts +0 -1
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/main/java/com/example/FooService.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/main/kotlin/com/example/BarService.kt +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/test/java/com/example/FooServiceTest.java +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-java-kotlin/src/test/kotlin/com/example/BarServiceTest.kt +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/README.md +0 -8
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/src/app.py +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-python-ts/tests/test_app.py +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/README.md +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/src/helper.js +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-ts-js/tests/index.sample.ts +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/README.md +0 -10
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/build.gradle.kts +0 -4
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/pytests/test_app.py +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/app.py +0 -2
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/helper.js +0 -5
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/index.ts +0 -3
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/main/java/com/example/Multiplier.java +0 -7
- package/tests/fixtures/audits/mixed-language/mixed-ts-python-jvm/src/main/kotlin/com/example/Divider.kt +0 -7
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/derived/run-integrity-evidence.json +0 -99
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/producer/implementation-audit.json +0 -1585
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/corrected-replay/producer/implementation-capsule.json +0 -11455
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/00-request.txt +0 -1
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/architecture-context-packet.txt +0 -82
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/behavior-model.txt +0 -104
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/final-report.txt +0 -41
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/implementation-context-packet.txt +0 -114
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/implementation-report.txt +0 -92
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/judge-report.txt +0 -34
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/pseudocode-packet.txt +0 -244
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/request-brief.txt +0 -40
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-context-packet.txt +0 -93
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-implementation-report.txt +0 -69
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/test-strategy-packet.txt +0 -354
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/artifacts/verification-report.txt +0 -85
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/derived/run-integrity-evidence.json +0 -98
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/architecture-audit.json +0 -1169
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/architecture-request.json +0 -29
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-audit.json +0 -1494
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-capsule.json +0 -11364
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/implementation-request.json +0 -33
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/producer/test-request.json +0 -74
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/architecture-context-retrieval-report.txt +0 -58
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/implementation-context-retrieval-report.txt +0 -75
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/reports/test-context-retrieval-report.txt +0 -84
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/run.json +0 -62
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/tool-versions/my-dev-kit-version.txt +0 -1
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/failed-run/tool-versions/orchestrator-version.txt +0 -1
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/corrected-replay-manifest.json +0 -19
- package/tests/fixtures/ecosystem/context-integrity/v0.4.5/manifests/failed-run-manifest.json +0 -56
- package/tests/fixtures/fake-adversarial-cli.js +0 -152
- package/tests/fixtures/fake-my-dev-kit-cli.js +0 -83
- package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +0 -24
- package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +0 -25
- package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +0 -230
- package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +0 -232
- package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +0 -1
- package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +0 -1
- package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +0 -18
- package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +0 -1
- package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +0 -1
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +0 -504
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +0 -955
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +0 -190
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +0 -327
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/PROVENANCE.txt +0 -18
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/context-capsule/complete-v1.0.0.json +0 -556
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.4/retrieval-audit-record/complete-v1.0.0.json +0 -238
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +0 -13
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +0 -7874
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +0 -1076
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +0 -160
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +0 -186
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/PROVENANCE.txt +0 -31
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/ready-pass-final-pass.json +0 -132
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.3/run-integrity/refresh-required-need-context.json +0 -146
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { fileURLToPath } from "node:url";
|
|
4
|
-
import { REQUIRED_BENCHMARK_PROJECT_IDS, parseBenchmarkProjectProfiles, validateAnswerKey, validateBenchmarkProjectProfiles } from "../src/evaluation/benchmarkMetadata.js";
|
|
5
|
-
const requiredProjects = REQUIRED_BENCHMARK_PROJECT_IDS;
|
|
6
|
-
const projectRequiredPaths = {
|
|
7
|
-
"todo-ts": [
|
|
8
|
-
"README.md",
|
|
9
|
-
"package.json",
|
|
10
|
-
"tsconfig.json",
|
|
11
|
-
"src/taskStore.ts",
|
|
12
|
-
"src/taskService.ts",
|
|
13
|
-
"src/index.ts",
|
|
14
|
-
"tests/taskService.test.ts"
|
|
15
|
-
],
|
|
16
|
-
"todo-python": [
|
|
17
|
-
"README.md",
|
|
18
|
-
"src/task_store.py",
|
|
19
|
-
"src/task_service.py",
|
|
20
|
-
"src/__init__.py",
|
|
21
|
-
"tests/test_task_service.py"
|
|
22
|
-
],
|
|
23
|
-
"todo-js": [
|
|
24
|
-
"README.md",
|
|
25
|
-
"package.json",
|
|
26
|
-
"src/taskStore.js",
|
|
27
|
-
"src/taskService.js",
|
|
28
|
-
"src/index.js",
|
|
29
|
-
"tests/taskService.test.js"
|
|
30
|
-
],
|
|
31
|
-
"todo-mixed-ts-py": [
|
|
32
|
-
"README.md",
|
|
33
|
-
"package.json",
|
|
34
|
-
"tsconfig.json",
|
|
35
|
-
"src/taskCli.ts",
|
|
36
|
-
"python/task_service.py",
|
|
37
|
-
"tests/mixedBoundary.test.ts"
|
|
38
|
-
],
|
|
39
|
-
"task-workflow-medium-ts": [
|
|
40
|
-
"README.md",
|
|
41
|
-
"package.json",
|
|
42
|
-
"tsconfig.json",
|
|
43
|
-
"src/store/taskStore.ts",
|
|
44
|
-
"src/services/createTask.ts",
|
|
45
|
-
"src/services/importTasks.ts",
|
|
46
|
-
"src/services/summarizeTasks.ts",
|
|
47
|
-
"tests/importTasks.test.ts"
|
|
48
|
-
],
|
|
49
|
-
"task-analytics-large-mixed": [
|
|
50
|
-
"README.md",
|
|
51
|
-
"ts/package.json",
|
|
52
|
-
"ts/tsconfig.json",
|
|
53
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
54
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
55
|
-
"ts/tests/buildAnalyticsSnapshot.test.ts",
|
|
56
|
-
"py/task_analytics/metrics.py",
|
|
57
|
-
"py/task_analytics/quality.py",
|
|
58
|
-
"py/tests/test_reporting.py"
|
|
59
|
-
]
|
|
60
|
-
};
|
|
61
|
-
function walk(dir) {
|
|
62
|
-
const entries = readdirSync(dir, { withFileTypes: true });
|
|
63
|
-
const files = [];
|
|
64
|
-
for (const entry of entries) {
|
|
65
|
-
const fullPath = path.join(dir, entry.name);
|
|
66
|
-
if (entry.isDirectory()) {
|
|
67
|
-
files.push(...walk(fullPath));
|
|
68
|
-
}
|
|
69
|
-
else {
|
|
70
|
-
files.push(fullPath);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
return files;
|
|
74
|
-
}
|
|
75
|
-
export function validateBenchmarks(rootDir = process.cwd()) {
|
|
76
|
-
const checks = [];
|
|
77
|
-
const errors = [];
|
|
78
|
-
const contractsDir = path.join(rootDir, "benchmarks", "contracts");
|
|
79
|
-
const projectsDir = path.join(rootDir, "benchmarks", "projects");
|
|
80
|
-
const behaviorPath = path.join(contractsDir, "todo-behavior.md");
|
|
81
|
-
const casesPath = path.join(contractsDir, "todo-benchmark-case.json");
|
|
82
|
-
const profilesPath = path.join(contractsDir, "benchmark-project-profiles.json");
|
|
83
|
-
if (!existsSync(behaviorPath)) {
|
|
84
|
-
errors.push("Missing contract file: benchmarks/contracts/todo-behavior.md");
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
checks.push("found todo-behavior.md");
|
|
88
|
-
}
|
|
89
|
-
let cases = [];
|
|
90
|
-
if (!existsSync(casesPath)) {
|
|
91
|
-
errors.push("Missing contract file: benchmarks/contracts/todo-benchmark-case.json");
|
|
92
|
-
}
|
|
93
|
-
else {
|
|
94
|
-
try {
|
|
95
|
-
cases = JSON.parse(readFileSync(casesPath, "utf8"));
|
|
96
|
-
checks.push("parsed todo-benchmark-case.json");
|
|
97
|
-
}
|
|
98
|
-
catch (error) {
|
|
99
|
-
errors.push(`Invalid JSON in todo-benchmark-case.json: ${error.message}`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
if (!existsSync(profilesPath)) {
|
|
103
|
-
errors.push("Missing contract file: benchmarks/contracts/benchmark-project-profiles.json");
|
|
104
|
-
}
|
|
105
|
-
else {
|
|
106
|
-
try {
|
|
107
|
-
const profiles = parseBenchmarkProjectProfiles(JSON.parse(readFileSync(profilesPath, "utf8")));
|
|
108
|
-
const profileErrors = validateBenchmarkProjectProfiles(profiles, rootDir);
|
|
109
|
-
errors.push(...profileErrors);
|
|
110
|
-
if (profileErrors.length === 0) {
|
|
111
|
-
checks.push("validated benchmark-project-profiles.json");
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
catch (error) {
|
|
115
|
-
errors.push(`Invalid benchmark-project-profiles.json: ${error.message}`);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
const ids = new Set();
|
|
119
|
-
for (const benchmarkCase of cases) {
|
|
120
|
-
if (ids.has(benchmarkCase.id)) {
|
|
121
|
-
errors.push(`Duplicate benchmark case id: ${benchmarkCase.id}`);
|
|
122
|
-
}
|
|
123
|
-
ids.add(benchmarkCase.id);
|
|
124
|
-
}
|
|
125
|
-
if (cases.length > 0 && errors.every((error) => !error.startsWith("Duplicate benchmark case id:"))) {
|
|
126
|
-
checks.push("benchmark case ids are unique");
|
|
127
|
-
}
|
|
128
|
-
for (const project of requiredProjects) {
|
|
129
|
-
const projectDir = path.join(projectsDir, project);
|
|
130
|
-
if (!existsSync(projectDir) || !statSync(projectDir).isDirectory()) {
|
|
131
|
-
errors.push(`Missing benchmark project: benchmarks/projects/${project}`);
|
|
132
|
-
continue;
|
|
133
|
-
}
|
|
134
|
-
checks.push(`found benchmark project ${project}`);
|
|
135
|
-
for (const relPath of projectRequiredPaths[project]) {
|
|
136
|
-
const fullPath = path.join(projectDir, relPath);
|
|
137
|
-
if (!existsSync(fullPath)) {
|
|
138
|
-
errors.push(`Missing required file for ${project}: benchmarks/projects/${project}/${relPath}`);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
for (const benchmarkCase of cases) {
|
|
143
|
-
if (!benchmarkCase.answerKey) {
|
|
144
|
-
errors.push(`Case ${benchmarkCase.id} does not define answerKey`);
|
|
145
|
-
}
|
|
146
|
-
else {
|
|
147
|
-
errors.push(...validateAnswerKey(benchmarkCase.answerKey, `Case ${benchmarkCase.id}`));
|
|
148
|
-
}
|
|
149
|
-
if (!Array.isArray(benchmarkCase.expectedSymbols) || benchmarkCase.expectedSymbols.length === 0) {
|
|
150
|
-
errors.push(`Case ${benchmarkCase.id} does not define expectedSymbols`);
|
|
151
|
-
}
|
|
152
|
-
for (const [project, expectedFiles] of Object.entries(benchmarkCase.expectedFilesByProject ?? {})) {
|
|
153
|
-
if (!requiredProjects.includes(project)) {
|
|
154
|
-
errors.push(`Case ${benchmarkCase.id} references unknown project id: ${project}`);
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (!Array.isArray(expectedFiles) || expectedFiles.length === 0) {
|
|
158
|
-
errors.push(`Case ${benchmarkCase.id} does not define expected files for ${project}`);
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
for (const expectedFile of expectedFiles) {
|
|
162
|
-
const fullPath = path.join(rootDir, "benchmarks", "projects", project, expectedFile);
|
|
163
|
-
if (!existsSync(fullPath)) {
|
|
164
|
-
errors.push(`Case ${benchmarkCase.id} references missing file: benchmarks/projects/${project}/${expectedFile}`);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
for (const project of requiredProjects) {
|
|
170
|
-
const projectDir = path.join(projectsDir, project);
|
|
171
|
-
if (!existsSync(projectDir)) {
|
|
172
|
-
continue;
|
|
173
|
-
}
|
|
174
|
-
const forbidden = walk(projectDir).filter((fullPath) => {
|
|
175
|
-
const rel = path.relative(projectDir, fullPath).replace(/\\/g, "/");
|
|
176
|
-
return /(^|\/)(node_modules|dist|build|coverage|lab-output)(\/|$)/.test(rel);
|
|
177
|
-
});
|
|
178
|
-
if (forbidden.length > 0) {
|
|
179
|
-
errors.push(`Forbidden generated output found in ${project}: ${forbidden[0]}`);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
return { ok: errors.length === 0, errors, checks };
|
|
183
|
-
}
|
|
184
|
-
function printSummary(result) {
|
|
185
|
-
console.log(`Benchmark verification ${result.ok ? "passed" : "failed"}.`);
|
|
186
|
-
console.log(`Checks: ${result.checks.length}`);
|
|
187
|
-
if (result.errors.length > 0) {
|
|
188
|
-
console.log(`Errors: ${result.errors.length}`);
|
|
189
|
-
for (const error of result.errors) {
|
|
190
|
-
console.log(`- ${error}`);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
const currentFile = fileURLToPath(import.meta.url);
|
|
195
|
-
const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : "";
|
|
196
|
-
if (invokedPath === currentFile) {
|
|
197
|
-
const result = validateBenchmarks();
|
|
198
|
-
printSummary(result);
|
|
199
|
-
if (!result.ok) {
|
|
200
|
-
process.exitCode = 1;
|
|
201
|
-
}
|
|
202
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"reportId": "demo-report",
|
|
3
|
-
"title": "Benchmark Validation Demo Report",
|
|
4
|
-
"projectName": "my-dev-kit-lab",
|
|
5
|
-
"benchmarkProject": "todo-ts",
|
|
6
|
-
"workflowName": "benchmark validation demo",
|
|
7
|
-
"generatedAt": "2026-06-10T15:00:00.000Z",
|
|
8
|
-
"summary": "Deterministic sample report for Prompt 2 report rendering and screenshot capture. This report does not claim token-savings evaluation.",
|
|
9
|
-
"steps": [
|
|
10
|
-
{
|
|
11
|
-
"id": "load-contracts",
|
|
12
|
-
"label": "Load benchmark contracts",
|
|
13
|
-
"command": "npm run verify:benchmarks",
|
|
14
|
-
"status": "pass",
|
|
15
|
-
"durationMs": 42,
|
|
16
|
-
"notes": "Validated benchmark contract structure."
|
|
17
|
-
},
|
|
18
|
-
{
|
|
19
|
-
"id": "run-project-tests",
|
|
20
|
-
"label": "Run benchmark project tests",
|
|
21
|
-
"command": "npm run test:benchmarks",
|
|
22
|
-
"status": "pass",
|
|
23
|
-
"durationMs": 118,
|
|
24
|
-
"notes": "Todo benchmark behavior stayed aligned."
|
|
25
|
-
},
|
|
26
|
-
{
|
|
27
|
-
"id": "prepare-report",
|
|
28
|
-
"label": "Prepare report artifacts",
|
|
29
|
-
"command": "npm run capture-demo-report",
|
|
30
|
-
"status": "pass",
|
|
31
|
-
"durationMs": 35,
|
|
32
|
-
"notes": "No token-savings metrics are reported in this demo."
|
|
33
|
-
}
|
|
34
|
-
],
|
|
35
|
-
"metrics": [
|
|
36
|
-
{
|
|
37
|
-
"id": "benchmark-project-count",
|
|
38
|
-
"label": "Benchmark projects",
|
|
39
|
-
"value": 4,
|
|
40
|
-
"interpretation": "Prompt 1 benchmark suite is available."
|
|
41
|
-
},
|
|
42
|
-
{
|
|
43
|
-
"id": "step-count",
|
|
44
|
-
"label": "Workflow steps",
|
|
45
|
-
"value": 3,
|
|
46
|
-
"interpretation": "Demo workflow uses a small deterministic sequence."
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
"id": "warning-count",
|
|
50
|
-
"label": "Warnings",
|
|
51
|
-
"value": 1,
|
|
52
|
-
"interpretation": "This demo explicitly notes that token comparison is not implemented yet."
|
|
53
|
-
},
|
|
54
|
-
{
|
|
55
|
-
"id": "artifact-count",
|
|
56
|
-
"label": "Artifacts declared",
|
|
57
|
-
"value": 2,
|
|
58
|
-
"interpretation": "JSON and HTML are always expected before optional PNG capture."
|
|
59
|
-
}
|
|
60
|
-
],
|
|
61
|
-
"artifacts": [
|
|
62
|
-
{
|
|
63
|
-
"id": "report-json",
|
|
64
|
-
"label": "Demo report JSON",
|
|
65
|
-
"path": "lab-output/demo-report/demo-report.json",
|
|
66
|
-
"kind": "json"
|
|
67
|
-
},
|
|
68
|
-
{
|
|
69
|
-
"id": "report-html",
|
|
70
|
-
"label": "Demo report HTML",
|
|
71
|
-
"path": "lab-output/demo-report/demo-report.html",
|
|
72
|
-
"kind": "html"
|
|
73
|
-
}
|
|
74
|
-
],
|
|
75
|
-
"warnings": [
|
|
76
|
-
"Token-savings evaluation is not implemented in Prompt 2."
|
|
77
|
-
]
|
|
78
|
-
}
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"id": "todo-ts-create-task-demo",
|
|
4
|
-
"title": "Todo TS create task demo retrieval",
|
|
5
|
-
"benchmarkProject": "todo-ts",
|
|
6
|
-
"projectProfileRef": "todo-ts",
|
|
7
|
-
"targetRoot": "benchmarks/projects/todo-ts",
|
|
8
|
-
"sourceRoots": ["src", "tests"],
|
|
9
|
-
"query": "create task deterministic id task-1",
|
|
10
|
-
"expectedFiles": ["src/taskService.ts", "src/taskStore.ts"],
|
|
11
|
-
"expectedSymbols": ["createTask", "TaskService"],
|
|
12
|
-
"answerKey": {
|
|
13
|
-
"expectedFiles": ["src/taskService.ts", "src/taskStore.ts"],
|
|
14
|
-
"expectedSymbols": ["createTask", "TaskService"],
|
|
15
|
-
"expectedFacts": [
|
|
16
|
-
{
|
|
17
|
-
"id": "create-deterministic-id",
|
|
18
|
-
"text": "createTask assigns deterministic IDs such as task-1 and task-2.",
|
|
19
|
-
"weight": 1,
|
|
20
|
-
"required": true
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
"id": "create-stores-incomplete-task",
|
|
24
|
-
"text": "createTask stores new tasks with completed set to false.",
|
|
25
|
-
"weight": 1,
|
|
26
|
-
"required": true
|
|
27
|
-
}
|
|
28
|
-
],
|
|
29
|
-
"minimumCorrectFacts": 2,
|
|
30
|
-
"notes": "Demo metadata prepares later correctness scoring without changing the current lab-demo command."
|
|
31
|
-
},
|
|
32
|
-
"rawIncludeGlobs": ["src/**/*", "tests/**/*"],
|
|
33
|
-
"notes": "Small single-case lab demo fixture."
|
|
34
|
-
}
|
|
35
|
-
]
|
|
@@ -1,118 +0,0 @@
|
|
|
1
|
-
[
|
|
2
|
-
{
|
|
3
|
-
"id": "task-workflow-import-dedup",
|
|
4
|
-
"title": "Task workflow import and dedupe retrieval",
|
|
5
|
-
"benchmarkProject": "task-workflow-medium-ts",
|
|
6
|
-
"projectProfileRef": "task-workflow-medium-ts",
|
|
7
|
-
"targetRoot": "benchmarks/projects/task-workflow-medium-ts",
|
|
8
|
-
"sourceRoots": ["src", "tests"],
|
|
9
|
-
"query": "import tasks duplicate normalized title deterministic id summarize workflow project",
|
|
10
|
-
"expectedFiles": [
|
|
11
|
-
"src/services/importTasks.ts",
|
|
12
|
-
"src/store/taskStore.ts",
|
|
13
|
-
"src/validation/taskValidation.ts"
|
|
14
|
-
],
|
|
15
|
-
"expectedSymbols": ["importTasks", "validateImportInput", "findDuplicate", "createTask"],
|
|
16
|
-
"answerKey": {
|
|
17
|
-
"expectedFiles": [
|
|
18
|
-
"src/services/importTasks.ts",
|
|
19
|
-
"src/store/taskStore.ts",
|
|
20
|
-
"src/validation/taskValidation.ts"
|
|
21
|
-
],
|
|
22
|
-
"expectedSymbols": ["importTasks", "validateImportInput", "findDuplicate", "createTask"],
|
|
23
|
-
"expectedFacts": [
|
|
24
|
-
{
|
|
25
|
-
"id": "workflow-import-validates-title-and-project",
|
|
26
|
-
"text": "importTasks validates normalized task titles and project ids before inserting records.",
|
|
27
|
-
"weight": 1,
|
|
28
|
-
"required": true
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
"id": "workflow-import-skips-duplicates-by-project-and-normalized-title",
|
|
32
|
-
"text": "importTasks skips duplicates when a task with the same normalized title already exists in the same project.",
|
|
33
|
-
"weight": 1,
|
|
34
|
-
"required": true
|
|
35
|
-
},
|
|
36
|
-
{
|
|
37
|
-
"id": "workflow-import-keeps-deterministic-task-sequences",
|
|
38
|
-
"text": "New imported tasks still receive deterministic ids such as task-1 and task-2 from the shared store sequence.",
|
|
39
|
-
"weight": 1,
|
|
40
|
-
"required": true
|
|
41
|
-
}
|
|
42
|
-
],
|
|
43
|
-
"minimumCorrectFacts": 3,
|
|
44
|
-
"notes": "This case is intended to require multiple files: service, validation, and store logic."
|
|
45
|
-
},
|
|
46
|
-
"rawIncludeGlobs": ["src/**/*", "tests/**/*"],
|
|
47
|
-
"notes": "Medium project case for real-agent comparison."
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
"id": "task-analytics-health-label",
|
|
51
|
-
"title": "Task analytics health label retrieval",
|
|
52
|
-
"benchmarkProject": "task-analytics-large-mixed",
|
|
53
|
-
"projectProfileRef": "task-analytics-large-mixed",
|
|
54
|
-
"targetRoot": "benchmarks/projects/task-analytics-large-mixed",
|
|
55
|
-
"sourceRoots": ["ts/src", "ts/tests", "py/task_analytics", "py/tests"],
|
|
56
|
-
"query": "completion rate stale tasks quality label health report build analytics snapshot mixed language",
|
|
57
|
-
"expectedFiles": [
|
|
58
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
59
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
60
|
-
"py/task_analytics/metrics.py",
|
|
61
|
-
"py/task_analytics/quality.py",
|
|
62
|
-
"py/task_analytics/reporting.py"
|
|
63
|
-
],
|
|
64
|
-
"expectedSymbols": [
|
|
65
|
-
"buildAnalyticsSnapshot",
|
|
66
|
-
"formatTaskHealthReport",
|
|
67
|
-
"calculate_project_metrics",
|
|
68
|
-
"determine_quality_label",
|
|
69
|
-
"build_health_report"
|
|
70
|
-
],
|
|
71
|
-
"answerKey": {
|
|
72
|
-
"expectedFiles": [
|
|
73
|
-
"ts/src/services/buildAnalyticsSnapshot.ts",
|
|
74
|
-
"ts/src/reporting/formatTaskHealthReport.ts",
|
|
75
|
-
"py/task_analytics/metrics.py",
|
|
76
|
-
"py/task_analytics/quality.py",
|
|
77
|
-
"py/task_analytics/reporting.py"
|
|
78
|
-
],
|
|
79
|
-
"expectedSymbols": [
|
|
80
|
-
"buildAnalyticsSnapshot",
|
|
81
|
-
"formatTaskHealthReport",
|
|
82
|
-
"calculate_project_metrics",
|
|
83
|
-
"determine_quality_label",
|
|
84
|
-
"build_health_report"
|
|
85
|
-
],
|
|
86
|
-
"expectedFacts": [
|
|
87
|
-
{
|
|
88
|
-
"id": "analytics-snapshot-computes-completion-rate",
|
|
89
|
-
"text": "The analytics snapshot computes completionRate as completedTasks divided by totalTasks times 100, rounded to two decimals.",
|
|
90
|
-
"weight": 1,
|
|
91
|
-
"required": true
|
|
92
|
-
},
|
|
93
|
-
{
|
|
94
|
-
"id": "analytics-snapshot-counts-stale-open-tasks",
|
|
95
|
-
"text": "Stale tasks count only open tasks whose updated day is at least ten days behind the current day.",
|
|
96
|
-
"weight": 1,
|
|
97
|
-
"required": true
|
|
98
|
-
},
|
|
99
|
-
{
|
|
100
|
-
"id": "python-quality-labels-healthy-watch-risk",
|
|
101
|
-
"text": "The Python quality layer maps metrics to healthy, watch, or risk labels based on completion rate and stale task thresholds.",
|
|
102
|
-
"weight": 1,
|
|
103
|
-
"required": true
|
|
104
|
-
},
|
|
105
|
-
{
|
|
106
|
-
"id": "python-reporting-renders-quality-lines",
|
|
107
|
-
"text": "The Python report writer emits one line per project with the quality label plus completion, open, and stale counts.",
|
|
108
|
-
"weight": 1,
|
|
109
|
-
"required": true
|
|
110
|
-
}
|
|
111
|
-
],
|
|
112
|
-
"minimumCorrectFacts": 3,
|
|
113
|
-
"notes": "This case intentionally spans TypeScript snapshot building and Python quality reporting."
|
|
114
|
-
},
|
|
115
|
-
"rawIncludeGlobs": ["ts/src/**/*", "ts/tests/**/*", "py/task_analytics/**/*", "py/tests/**/*"],
|
|
116
|
-
"notes": "Large mixed-language case for real-agent comparison."
|
|
117
|
-
}
|
|
118
|
-
]
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
-
<data-extraction-rules>
|
|
3
|
-
<cloud-backup disableIfNoEncryptionCapabilities="true">
|
|
4
|
-
<include domain="sharedpref" path="." />
|
|
5
|
-
<exclude domain="database" path="fake-secrets.db" />
|
|
6
|
-
</cloud-backup>
|
|
7
|
-
<device-transfer>
|
|
8
|
-
<include domain="root" path="." />
|
|
9
|
-
</device-transfer>
|
|
10
|
-
</data-extraction-rules>
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
-
<network-security-config>
|
|
3
|
-
<domain-config>
|
|
4
|
-
<domain>example.org</domain>
|
|
5
|
-
<pin-set expiration="not-a-date">
|
|
6
|
-
<pin>MISSING_DIGEST_ATTRIBUTE_FAKE_PIN=</pin>
|
|
7
|
-
<pin digest="SHA-1">UNSUPPORTED_DIGEST_FAKE_PIN=</pin>
|
|
8
|
-
<pin digest="SHA-256"></pin>
|
|
9
|
-
</pin-set>
|
|
10
|
-
</domain-config>
|
|
11
|
-
</network-security-config>
|
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
<?xml version="1.0" encoding="utf-8"?>
|
|
2
|
-
<network-security-config>
|
|
3
|
-
<base-config cleartextTrafficPermitted="true">
|
|
4
|
-
<trust-anchors>
|
|
5
|
-
<certificates src="system" />
|
|
6
|
-
<certificates src="user" />
|
|
7
|
-
</trust-anchors>
|
|
8
|
-
</base-config>
|
|
9
|
-
<domain-config cleartextTrafficPermitted="true">
|
|
10
|
-
<domain includeSubdomains="true">example.com</domain>
|
|
11
|
-
<pin-set expiration="2030-01-01">
|
|
12
|
-
<pin digest="SHA-256">FAKEPINHASHFAKEPINHASHFAKEPINHASHFAKEPINHASH=</pin>
|
|
13
|
-
</pin-set>
|
|
14
|
-
<domain-config>
|
|
15
|
-
<domain includeSubdomains="false">nested.example.com</domain>
|
|
16
|
-
<trust-anchors>
|
|
17
|
-
<certificates src="@raw/my_custom_ca" />
|
|
18
|
-
</trust-anchors>
|
|
19
|
-
</domain-config>
|
|
20
|
-
</domain-config>
|
|
21
|
-
<debug-overrides>
|
|
22
|
-
<trust-anchors>
|
|
23
|
-
<certificates src="user" overridePins="true" />
|
|
24
|
-
</trust-anchors>
|
|
25
|
-
</debug-overrides>
|
|
26
|
-
</network-security-config>
|