@complior/engine 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.well-known/ai-compliance.json +16 -0
- package/COMPLIANCE.md +64 -0
- package/data/data-integrity.test.ts +75 -0
- package/data/eval/eval-mappings.json +33 -0
- package/data/llm/model-pricing.json +15 -0
- package/data/llm/model-routing.json +36 -0
- package/data/onboarding/risk-profile.json +17 -0
- package/data/regulations/eu-ai-act/README.md +245 -0
- package/data/regulations/eu-ai-act/applicability-tree.json +160 -0
- package/data/regulations/eu-ai-act/cross-mapping.json +175 -0
- package/data/regulations/eu-ai-act/localization.json +186 -0
- package/data/regulations/eu-ai-act/obligations.json +3981 -0
- package/data/regulations/eu-ai-act/regulation-meta.json +482 -0
- package/data/regulations/eu-ai-act/scoring.json +342 -0
- package/data/regulations/eu-ai-act/technical-requirements.json +2590 -0
- package/data/regulations/eu-ai-act/timeline.json +160 -0
- package/data/regulations/jurisdictions/at.json +15 -0
- package/data/regulations/jurisdictions/be.json +15 -0
- package/data/regulations/jurisdictions/bg.json +15 -0
- package/data/regulations/jurisdictions/cy.json +15 -0
- package/data/regulations/jurisdictions/cz.json +15 -0
- package/data/regulations/jurisdictions/de.json +15 -0
- package/data/regulations/jurisdictions/dk.json +15 -0
- package/data/regulations/jurisdictions/ee.json +15 -0
- package/data/regulations/jurisdictions/es.json +15 -0
- package/data/regulations/jurisdictions/fi.json +15 -0
- package/data/regulations/jurisdictions/fr.json +15 -0
- package/data/regulations/jurisdictions/gr.json +15 -0
- package/data/regulations/jurisdictions/hr.json +15 -0
- package/data/regulations/jurisdictions/hu.json +15 -0
- package/data/regulations/jurisdictions/ie.json +15 -0
- package/data/regulations/jurisdictions/is.json +15 -0
- package/data/regulations/jurisdictions/it.json +15 -0
- package/data/regulations/jurisdictions/li.json +15 -0
- package/data/regulations/jurisdictions/lt.json +15 -0
- package/data/regulations/jurisdictions/lu.json +15 -0
- package/data/regulations/jurisdictions/lv.json +15 -0
- package/data/regulations/jurisdictions/mt.json +15 -0
- package/data/regulations/jurisdictions/nl.json +15 -0
- package/data/regulations/jurisdictions/no.json +15 -0
- package/data/regulations/jurisdictions/pl.json +15 -0
- package/data/regulations/jurisdictions/pt.json +15 -0
- package/data/regulations/jurisdictions/ro.json +15 -0
- package/data/regulations/jurisdictions/se.json +15 -0
- package/data/regulations/jurisdictions/si.json +15 -0
- package/data/regulations/jurisdictions/sk.json +15 -0
- package/data/scanner/check-id-categories.json +81 -0
- package/data/scanner/confidence-params.json +16 -0
- package/data/scanner/limits.json +4 -0
- package/data/schemas/http-contract-sample.json +79 -0
- package/data/schemas/http-contract.json +144 -0
- package/data/semgrep-rules/bare-call.yaml +37 -0
- package/data/semgrep-rules/injection.yaml +73 -0
- package/data/semgrep-rules/missing-error-handling.yaml +58 -0
- package/data/semgrep-rules/unsafe-deser.yaml +65 -0
- package/data/templates/eu-ai-act/ai-literacy.md +184 -0
- package/data/templates/eu-ai-act/art5-screening.md +131 -0
- package/data/templates/eu-ai-act/data-governance.md +145 -0
- package/data/templates/eu-ai-act/declaration-of-conformity.md +161 -0
- package/data/templates/eu-ai-act/fria.md +127 -0
- package/data/templates/eu-ai-act/gpai-systemic-risk.md +150 -0
- package/data/templates/eu-ai-act/gpai-transparency.md +166 -0
- package/data/templates/eu-ai-act/incident-report.md +188 -0
- package/data/templates/eu-ai-act/instructions-for-use.md +202 -0
- package/data/templates/eu-ai-act/monitoring-policy.md +110 -0
- package/data/templates/eu-ai-act/qms.md +180 -0
- package/data/templates/eu-ai-act/risk-management-system.md +123 -0
- package/data/templates/eu-ai-act/technical-documentation.md +287 -0
- package/data/templates/eu-ai-act/worker-notification.md +143 -0
- package/data/templates/policies/biometrics-ai-policy.md +214 -0
- package/data/templates/policies/critical-infra-ai-policy.md +228 -0
- package/data/templates/policies/education-ai-policy.md +184 -0
- package/data/templates/policies/finance-ai-policy.md +191 -0
- package/data/templates/policies/healthcare-ai-policy.md +197 -0
- package/data/templates/policies/hr-ai-policy.md +178 -0
- package/data/templates/policies/legal-ai-policy.md +189 -0
- package/data/templates/policies/migration-ai-policy.md +239 -0
- package/engine.log +7 -0
- package/package.json +74 -0
- package/src/composition-root.ts +791 -0
- package/src/data/eval/conformity-tests.test.ts +122 -0
- package/src/data/eval/ct-1-transparency.ts +106 -0
- package/src/data/eval/ct-10-gpai.ts +25 -0
- package/src/data/eval/ct-11-industry.ts +42 -0
- package/src/data/eval/ct-2-oversight.ts +41 -0
- package/src/data/eval/ct-3-explanation.ts +14 -0
- package/src/data/eval/ct-4-bias.ts +83 -0
- package/src/data/eval/ct-5-accuracy.ts +41 -0
- package/src/data/eval/ct-6-robustness.ts +81 -0
- package/src/data/eval/ct-7-prohibited.ts +52 -0
- package/src/data/eval/ct-8-logging.ts +68 -0
- package/src/data/eval/ct-9-risk-awareness.ts +33 -0
- package/src/data/eval/deterministic-evaluator.ts +120 -0
- package/src/data/eval/index.ts +55 -0
- package/src/data/eval/judge-prompts.ts +146 -0
- package/src/data/eval/llm-judged-tests.ts +279 -0
- package/src/data/eval/llm-tests.test.ts +83 -0
- package/src/data/eval/remediation/ct-1-transparency.ts +91 -0
- package/src/data/eval/remediation/ct-10-gpai.ts +94 -0
- package/src/data/eval/remediation/ct-11-industry.ts +94 -0
- package/src/data/eval/remediation/ct-2-oversight.ts +71 -0
- package/src/data/eval/remediation/ct-3-explanation.ts +70 -0
- package/src/data/eval/remediation/ct-4-bias.ts +70 -0
- package/src/data/eval/remediation/ct-5-accuracy.ts +70 -0
- package/src/data/eval/remediation/ct-6-robustness.ts +70 -0
- package/src/data/eval/remediation/ct-7-prohibited.ts +94 -0
- package/src/data/eval/remediation/ct-8-logging.ts +94 -0
- package/src/data/eval/remediation/ct-9-risk-awareness.ts +94 -0
- package/src/data/eval/remediation/index.ts +89 -0
- package/src/data/eval/remediation/owasp-art5.ts +15 -0
- package/src/data/eval/remediation/owasp-llm01.ts +72 -0
- package/src/data/eval/remediation/owasp-llm02.ts +72 -0
- package/src/data/eval/remediation/owasp-llm03.ts +15 -0
- package/src/data/eval/remediation/owasp-llm04.ts +15 -0
- package/src/data/eval/remediation/owasp-llm05.ts +15 -0
- package/src/data/eval/remediation/owasp-llm06.ts +15 -0
- package/src/data/eval/remediation/owasp-llm07.ts +15 -0
- package/src/data/eval/remediation/owasp-llm08.ts +15 -0
- package/src/data/eval/remediation/owasp-llm09.ts +15 -0
- package/src/data/eval/remediation/owasp-llm10.ts +15 -0
- package/src/data/eval/remediation/remediation.test.ts +229 -0
- package/src/data/eval/remediation/test-mapping.ts +290 -0
- package/src/data/eval/security-rubrics.ts +381 -0
- package/src/data/finding-explanations.json +453 -0
- package/src/data/industry-patterns.ts +161 -0
- package/src/data/registry-cards.ts +368 -0
- package/src/data/regulation/index.ts +5 -0
- package/src/data/regulation/jurisdiction-data.test.ts +73 -0
- package/src/data/regulation/jurisdiction-data.ts +65 -0
- package/src/data/regulation/regulation-data.ts +19 -0
- package/src/data/regulation/regulation-loader.test.ts +107 -0
- package/src/data/regulation/regulation-loader.ts +56 -0
- package/src/data/scanner-constants.ts +46 -0
- package/src/data/schemas/schemas-core.ts +140 -0
- package/src/data/schemas/schemas-supplementary.ts +211 -0
- package/src/data/schemas/schemas.ts +28 -0
- package/src/data/security/attack-probes.test.ts +62 -0
- package/src/data/security/attack-probes.ts +496 -0
- package/src/data/security/eu-ai-act-security.ts +40 -0
- package/src/data/security/index.ts +19 -0
- package/src/data/security/mitre-atlas.test.ts +43 -0
- package/src/data/security/mitre-atlas.ts +93 -0
- package/src/data/security/nist-ai-rmf.ts +43 -0
- package/src/data/security/owasp-llm-top10.test.ts +60 -0
- package/src/data/security/owasp-llm-top10.ts +138 -0
- package/src/data/template-registry.ts +53 -0
- package/src/data/tool-versions.json +22 -0
- package/src/domain/audit/audit-package.test.ts +152 -0
- package/src/domain/audit/audit-package.ts +166 -0
- package/src/domain/audit/audit-trail.test.ts +121 -0
- package/src/domain/audit/audit-trail.ts +174 -0
- package/src/domain/audit/index.ts +8 -0
- package/src/domain/audit/permissions-matrix.test.ts +136 -0
- package/src/domain/audit/permissions-matrix.ts +121 -0
- package/src/domain/certification/adversarial/bias-tests.ts +95 -0
- package/src/domain/certification/adversarial/evaluators.ts +304 -0
- package/src/domain/certification/adversarial/index.ts +11 -0
- package/src/domain/certification/adversarial/prompt-injection.ts +103 -0
- package/src/domain/certification/adversarial/safety-boundary.ts +132 -0
- package/src/domain/certification/aiuc1-readiness.test.ts +236 -0
- package/src/domain/certification/aiuc1-readiness.ts +298 -0
- package/src/domain/certification/aiuc1-requirements.ts +235 -0
- package/src/domain/certification/index.ts +10 -0
- package/src/domain/certification/redteam-runner.test.ts +97 -0
- package/src/domain/certification/redteam-runner.ts +205 -0
- package/src/domain/certification/test-runner.test.ts +232 -0
- package/src/domain/certification/test-runner.ts +289 -0
- package/src/domain/cost/cost-estimator.test.ts +187 -0
- package/src/domain/cost/cost-estimator.ts +133 -0
- package/src/domain/disclaimer.test.ts +52 -0
- package/src/domain/disclaimer.ts +39 -0
- package/src/domain/documents/ai-enricher.test.ts +120 -0
- package/src/domain/documents/ai-enricher.ts +159 -0
- package/src/domain/documents/document-generator.test.ts +318 -0
- package/src/domain/documents/document-generator.ts +239 -0
- package/src/domain/documents/index.ts +9 -0
- package/src/domain/documents/passport-helpers.ts +25 -0
- package/src/domain/documents/policy-generator.test.ts +252 -0
- package/src/domain/documents/policy-generator.ts +94 -0
- package/src/domain/documents/worker-notification-generator.test.ts +162 -0
- package/src/domain/documents/worker-notification-generator.ts +141 -0
- package/src/domain/eval/adapters/adapter-port.ts +94 -0
- package/src/domain/eval/adapters/adapters.test.ts +303 -0
- package/src/domain/eval/adapters/anthropic-adapter.ts +57 -0
- package/src/domain/eval/adapters/auto-detect.ts +104 -0
- package/src/domain/eval/adapters/create-chat-adapter.ts +106 -0
- package/src/domain/eval/adapters/custom-adapter.ts +74 -0
- package/src/domain/eval/adapters/http-adapter.ts +66 -0
- package/src/domain/eval/adapters/index.ts +7 -0
- package/src/domain/eval/adapters/ollama-adapter.ts +48 -0
- package/src/domain/eval/adapters/openai-adapter.ts +58 -0
- package/src/domain/eval/adapters/with-timeout.ts +25 -0
- package/src/domain/eval/conformity-score.test.ts +161 -0
- package/src/domain/eval/conformity-score.ts +135 -0
- package/src/domain/eval/eval-constants.ts +55 -0
- package/src/domain/eval/eval-evidence.test.ts +85 -0
- package/src/domain/eval/eval-evidence.ts +103 -0
- package/src/domain/eval/eval-fix-generator.test.ts +421 -0
- package/src/domain/eval/eval-fix-generator.ts +205 -0
- package/src/domain/eval/eval-passport.test.ts +82 -0
- package/src/domain/eval/eval-passport.ts +89 -0
- package/src/domain/eval/eval-remediation-report.test.ts +682 -0
- package/src/domain/eval/eval-remediation-report.ts +170 -0
- package/src/domain/eval/eval-report.ts +108 -0
- package/src/domain/eval/eval-runner.test.ts +609 -0
- package/src/domain/eval/eval-runner.ts +593 -0
- package/src/domain/eval/eval-to-findings.test.ts +293 -0
- package/src/domain/eval/eval-to-findings.ts +83 -0
- package/src/domain/eval/index.ts +31 -0
- package/src/domain/eval/llm-judge.test.ts +139 -0
- package/src/domain/eval/llm-judge.ts +168 -0
- package/src/domain/eval/remediation-types.ts +90 -0
- package/src/domain/eval/security-integration.test.ts +196 -0
- package/src/domain/eval/security-integration.ts +136 -0
- package/src/domain/eval/types.test.ts +173 -0
- package/src/domain/eval/types.ts +244 -0
- package/src/domain/eval/verdict-utils.ts +45 -0
- package/src/domain/fixer/create-fixer.ts +101 -0
- package/src/domain/fixer/diff.ts +70 -0
- package/src/domain/fixer/fix-history.ts +23 -0
- package/src/domain/fixer/fixer.test.ts +306 -0
- package/src/domain/fixer/index.ts +9 -0
- package/src/domain/fixer/strategies/bandit-fix.ts +61 -0
- package/src/domain/fixer/strategies/bias-testing.ts +49 -0
- package/src/domain/fixer/strategies/ci-compliance.ts +57 -0
- package/src/domain/fixer/strategies/content-marking.ts +45 -0
- package/src/domain/fixer/strategies/cve-upgrade.ts +66 -0
- package/src/domain/fixer/strategies/data-governance.ts +65 -0
- package/src/domain/fixer/strategies/disclosure.ts +69 -0
- package/src/domain/fixer/strategies/doc-code-sync.ts +53 -0
- package/src/domain/fixer/strategies/documentation.ts +59 -0
- package/src/domain/fixer/strategies/error-handler.ts +63 -0
- package/src/domain/fixer/strategies/hitl-gate.ts +67 -0
- package/src/domain/fixer/strategies/index.ts +61 -0
- package/src/domain/fixer/strategies/kill-switch-test.ts +85 -0
- package/src/domain/fixer/strategies/kill-switch.ts +53 -0
- package/src/domain/fixer/strategies/license-fix.ts +57 -0
- package/src/domain/fixer/strategies/log-retention.ts +40 -0
- package/src/domain/fixer/strategies/logging.ts +59 -0
- package/src/domain/fixer/strategies/metadata.ts +45 -0
- package/src/domain/fixer/strategies/permission-guard.ts +84 -0
- package/src/domain/fixer/strategies/record-keeping.ts +69 -0
- package/src/domain/fixer/strategies/secret-rotation.ts +52 -0
- package/src/domain/fixer/strategies.test.ts +341 -0
- package/src/domain/fixer/template-engine.test.ts +64 -0
- package/src/domain/fixer/template-engine.ts +38 -0
- package/src/domain/fixer/types.ts +88 -0
- package/src/domain/frameworks/aiuc1-framework.test.ts +159 -0
- package/src/domain/frameworks/aiuc1-framework.ts +126 -0
- package/src/domain/frameworks/collect-foundation-metrics.test.ts +96 -0
- package/src/domain/frameworks/collect-foundation-metrics.ts +34 -0
- package/src/domain/frameworks/eu-ai-act-framework.test.ts +117 -0
- package/src/domain/frameworks/eu-ai-act-framework.ts +100 -0
- package/src/domain/frameworks/framework-registry.test.ts +91 -0
- package/src/domain/frameworks/framework-registry.ts +38 -0
- package/src/domain/frameworks/index.ts +8 -0
- package/src/domain/frameworks/mitre-atlas-framework.test.ts +53 -0
- package/src/domain/frameworks/mitre-atlas-framework.ts +53 -0
- package/src/domain/frameworks/owasp-llm-framework.test.ts +77 -0
- package/src/domain/frameworks/owasp-llm-framework.ts +54 -0
- package/src/domain/frameworks/score-plugin-framework.ts +117 -0
- package/src/domain/fria/fria-generator.test.ts +273 -0
- package/src/domain/fria/fria-generator.ts +366 -0
- package/src/domain/import/promptfoo-importer.test.ts +103 -0
- package/src/domain/import/promptfoo-importer.ts +151 -0
- package/src/domain/onboarding/guided-onboarding.test.ts +144 -0
- package/src/domain/onboarding/guided-onboarding.ts +135 -0
- package/src/domain/passport/builder/domain-mapper.ts +9 -0
- package/src/domain/passport/builder/manifest-builder.test.ts +546 -0
- package/src/domain/passport/builder/manifest-builder.ts +535 -0
- package/src/domain/passport/builder/manifest-diff.test.ts +105 -0
- package/src/domain/passport/builder/manifest-diff.ts +89 -0
- package/src/domain/passport/builder/manifest-files.ts +17 -0
- package/src/domain/passport/crypto-signer.test.ts +93 -0
- package/src/domain/passport/crypto-signer.ts +157 -0
- package/src/domain/passport/discovery/agent-discovery.test.ts +296 -0
- package/src/domain/passport/discovery/agent-discovery.ts +325 -0
- package/src/domain/passport/discovery/autonomy-analyzer.test.ts +141 -0
- package/src/domain/passport/discovery/autonomy-analyzer.ts +113 -0
- package/src/domain/passport/discovery/permission-scanner.test.ts +191 -0
- package/src/domain/passport/discovery/permission-scanner.ts +414 -0
- package/src/domain/passport/export/a2a-mapper.ts +75 -0
- package/src/domain/passport/export/aiuc1-mapper.ts +126 -0
- package/src/domain/passport/export/export.test.ts +207 -0
- package/src/domain/passport/export/index.ts +41 -0
- package/src/domain/passport/export/nist-mapper.ts +227 -0
- package/src/domain/passport/import/a2a-importer.test.ts +133 -0
- package/src/domain/passport/import/a2a-importer.ts +156 -0
- package/src/domain/passport/import/index.ts +2 -0
- package/src/domain/passport/index.ts +32 -0
- package/src/domain/passport/obligation-field-map.test.ts +113 -0
- package/src/domain/passport/obligation-field-map.ts +117 -0
- package/src/domain/passport/passport-validator.test.ts +156 -0
- package/src/domain/passport/passport-validator.ts +126 -0
- package/src/domain/passport/scan-to-compliance.test.ts +336 -0
- package/src/domain/passport/scan-to-compliance.ts +166 -0
- package/src/domain/passport/test-generator.test.ts +93 -0
- package/src/domain/passport/test-generator.ts +136 -0
- package/src/domain/proxy/index.ts +11 -0
- package/src/domain/proxy/json-rpc.test.ts +72 -0
- package/src/domain/proxy/json-rpc.ts +53 -0
- package/src/domain/proxy/policy-engine.test.ts +259 -0
- package/src/domain/proxy/policy-engine.ts +137 -0
- package/src/domain/proxy/proxy-bridge.ts +125 -0
- package/src/domain/proxy/proxy-interceptor.test.ts +184 -0
- package/src/domain/proxy/proxy-interceptor.ts +120 -0
- package/src/domain/proxy/proxy-types.ts +35 -0
- package/src/domain/registry/compute-agent-score.test.ts +279 -0
- package/src/domain/registry/compute-agent-score.ts +162 -0
- package/src/domain/reporter/audit-report.test.ts +87 -0
- package/src/domain/reporter/audit-report.ts +116 -0
- package/src/domain/reporter/badge-generator.test.ts +54 -0
- package/src/domain/reporter/badge-generator.ts +40 -0
- package/src/domain/reporter/compliance-md.ts +45 -0
- package/src/domain/reporter/index.ts +7 -0
- package/src/domain/reporter/pdf-renderer.ts +282 -0
- package/src/domain/reporter/share.test.ts +92 -0
- package/src/domain/reporter/share.ts +80 -0
- package/src/domain/scanner/ast/swc-analyzer.test.ts +49 -0
- package/src/domain/scanner/ast/swc-analyzer.ts +124 -0
- package/src/domain/scanner/attestations.ts +97 -0
- package/src/domain/scanner/checks/ai-disclosure.test.ts +90 -0
- package/src/domain/scanner/checks/ai-disclosure.ts +54 -0
- package/src/domain/scanner/checks/ai-literacy.ts +163 -0
- package/src/domain/scanner/checks/behavioral-constraints.test.ts +167 -0
- package/src/domain/scanner/checks/behavioral-constraints.ts +86 -0
- package/src/domain/scanner/checks/compliance-metadata.ts +63 -0
- package/src/domain/scanner/checks/content-marking.ts +74 -0
- package/src/domain/scanner/checks/dep-deep-scan.test.ts +318 -0
- package/src/domain/scanner/checks/dep-deep-scan.ts +137 -0
- package/src/domain/scanner/checks/documentation.test.ts +88 -0
- package/src/domain/scanner/checks/documentation.ts +79 -0
- package/src/domain/scanner/checks/git-history.test.ts +120 -0
- package/src/domain/scanner/checks/git-history.ts +163 -0
- package/src/domain/scanner/checks/gpai-systemic-risk.test.ts +84 -0
- package/src/domain/scanner/checks/gpai-systemic-risk.ts +98 -0
- package/src/domain/scanner/checks/gpai-transparency.ts +94 -0
- package/src/domain/scanner/checks/index.ts +28 -0
- package/src/domain/scanner/checks/industry/index.ts +40 -0
- package/src/domain/scanner/checks/industry/industry.test.ts +287 -0
- package/src/domain/scanner/checks/interaction-logging.test.ts +113 -0
- package/src/domain/scanner/checks/interaction-logging.ts +142 -0
- package/src/domain/scanner/checks/nhi-scanner.test.ts +158 -0
- package/src/domain/scanner/checks/nhi-scanner.ts +78 -0
- package/src/domain/scanner/checks/passport-completeness.test.ts +127 -0
- package/src/domain/scanner/checks/passport-completeness.ts +82 -0
- package/src/domain/scanner/checks/passport-presence.test.ts +56 -0
- package/src/domain/scanner/checks/passport-presence.ts +78 -0
- package/src/domain/scanner/checks/pattern-check-factory.ts +70 -0
- package/src/domain/scanner/checks/permission-scanner.test.ts +279 -0
- package/src/domain/scanner/checks/permission-scanner.ts +90 -0
- package/src/domain/scanner/checks/presence-check-factory.test.ts +124 -0
- package/src/domain/scanner/checks/presence-check-factory.ts +275 -0
- package/src/domain/scanner/compliance-diff.test.ts +165 -0
- package/src/domain/scanner/compliance-diff.ts +138 -0
- package/src/domain/scanner/confidence.test.ts +235 -0
- package/src/domain/scanner/confidence.ts +156 -0
- package/src/domain/scanner/constants.ts +13 -0
- package/src/domain/scanner/create-scanner.ts +573 -0
- package/src/domain/scanner/cross-layer.test.ts +372 -0
- package/src/domain/scanner/cross-layer.ts +232 -0
- package/src/domain/scanner/data/ai-packages.ts +82 -0
- package/src/domain/scanner/debt-calculator.test.ts +89 -0
- package/src/domain/scanner/debt-calculator.ts +111 -0
- package/src/domain/scanner/drift.test.ts +191 -0
- package/src/domain/scanner/drift.ts +73 -0
- package/src/domain/scanner/evidence-store.test.ts +207 -0
- package/src/domain/scanner/evidence-store.ts +195 -0
- package/src/domain/scanner/evidence.test.ts +104 -0
- package/src/domain/scanner/evidence.ts +71 -0
- package/src/domain/scanner/external/bandit-runner.test.ts +45 -0
- package/src/domain/scanner/external/bandit-runner.ts +90 -0
- package/src/domain/scanner/external/checks.ts +321 -0
- package/src/domain/scanner/external/dedup.test.ts +79 -0
- package/src/domain/scanner/external/dedup.ts +94 -0
- package/src/domain/scanner/external/detect-secrets-runner.test.ts +58 -0
- package/src/domain/scanner/external/detect-secrets-runner.ts +81 -0
- package/src/domain/scanner/external/external-scanner.test.ts +221 -0
- package/src/domain/scanner/external/external-scanner.ts +36 -0
- package/src/domain/scanner/external/finding-mapper.test.ts +95 -0
- package/src/domain/scanner/external/finding-mapper.ts +138 -0
- package/src/domain/scanner/external/index.ts +15 -0
- package/src/domain/scanner/external/mappings.ts +93 -0
- package/src/domain/scanner/external/modelscan-runner.test.ts +35 -0
- package/src/domain/scanner/external/modelscan-runner.ts +101 -0
- package/src/domain/scanner/external/path-utils.ts +8 -0
- package/src/domain/scanner/external/runner-port.ts +45 -0
- package/src/domain/scanner/external/semgrep-runner.test.ts +52 -0
- package/src/domain/scanner/external/semgrep-runner.ts +94 -0
- package/src/domain/scanner/external/types.ts +32 -0
- package/src/domain/scanner/finding-attribution.test.ts +444 -0
- package/src/domain/scanner/finding-attribution.ts +195 -0
- package/src/domain/scanner/finding-explainer.test.ts +157 -0
- package/src/domain/scanner/finding-explainer.ts +73 -0
- package/src/domain/scanner/fix-diff-builder.test.ts +272 -0
- package/src/domain/scanner/fix-diff-builder.ts +477 -0
- package/src/domain/scanner/import-graph.test.ts +162 -0
- package/src/domain/scanner/import-graph.ts +198 -0
- package/src/domain/scanner/languages/adapter.test.ts +105 -0
- package/src/domain/scanner/languages/adapter.ts +239 -0
- package/src/domain/scanner/layers/index.ts +24 -0
- package/src/domain/scanner/layers/layer1-files.ts +54 -0
- package/src/domain/scanner/layers/layer2-docs.test.ts +1207 -0
- package/src/domain/scanner/layers/layer2-docs.ts +297 -0
- package/src/domain/scanner/layers/layer2-parsing.ts +217 -0
- package/src/domain/scanner/layers/layer3-config.test.ts +187 -0
- package/src/domain/scanner/layers/layer3-config.ts +279 -0
- package/src/domain/scanner/layers/layer3-parsers.ts +73 -0
- package/src/domain/scanner/layers/layer4-patterns.test.ts +397 -0
- package/src/domain/scanner/layers/layer4-patterns.ts +216 -0
- package/src/domain/scanner/layers/layer5-docs.test.ts +99 -0
- package/src/domain/scanner/layers/layer5-docs.ts +250 -0
- package/src/domain/scanner/layers/layer5-llm.test.ts +146 -0
- package/src/domain/scanner/layers/layer5-llm.ts +262 -0
- package/src/domain/scanner/layers/layer5-targeted.test.ts +93 -0
- package/src/domain/scanner/layers/layer5-targeted.ts +233 -0
- package/src/domain/scanner/layers/lockfile-parsers.test.ts +320 -0
- package/src/domain/scanner/layers/lockfile-parsers.ts +184 -0
- package/src/domain/scanner/regulation-version.test.ts +54 -0
- package/src/domain/scanner/regulation-version.ts +23 -0
- package/src/domain/scanner/role-filter.test.ts +116 -0
- package/src/domain/scanner/role-filter.ts +51 -0
- package/src/domain/scanner/rules/banned-packages-data.ts +553 -0
- package/src/domain/scanner/rules/banned-packages-sdk.ts +65 -0
- package/src/domain/scanner/rules/banned-packages.test.ts +249 -0
- package/src/domain/scanner/rules/banned-packages.ts +55 -0
- package/src/domain/scanner/rules/comment-filter.test.ts +115 -0
- package/src/domain/scanner/rules/comment-filter.ts +297 -0
- package/src/domain/scanner/rules/index.ts +9 -0
- package/src/domain/scanner/rules/nhi-patterns.test.ts +128 -0
- package/src/domain/scanner/rules/nhi-patterns.ts +60 -0
- package/src/domain/scanner/rules/pattern-rules.ts +1152 -0
- package/src/domain/scanner/sbom.test.ts +136 -0
- package/src/domain/scanner/sbom.ts +103 -0
- package/src/domain/scanner/scan-cache.test.ts +136 -0
- package/src/domain/scanner/scan-cache.ts +115 -0
- package/src/domain/scanner/scanner.test.ts +125 -0
- package/src/domain/scanner/score-calculator.test.ts +363 -0
- package/src/domain/scanner/score-calculator.ts +189 -0
- package/src/domain/scanner/security-score.test.ts +107 -0
- package/src/domain/scanner/security-score.ts +116 -0
- package/src/domain/scanner/source-filter.ts +24 -0
- package/src/domain/scanner/validators.ts +223 -0
- package/src/domain/shared/compliance-constants.ts +48 -0
- package/src/domain/shared/disclosure-patterns.ts +16 -0
- package/src/domain/shared/index.ts +6 -0
- package/src/domain/shared/parse-dependencies.ts +21 -0
- package/src/domain/supply-chain/dependency-analyzer.ts +138 -0
- package/src/domain/supply-chain/index.ts +3 -0
- package/src/domain/supply-chain/supply-chain.test.ts +211 -0
- package/src/domain/supply-chain/types.ts +32 -0
- package/src/domain/whatif/config-fixer.ts +187 -0
- package/src/domain/whatif/index.ts +6 -0
- package/src/domain/whatif/scenario-engine.ts +121 -0
- package/src/domain/whatif/simulate-actions.test.ts +161 -0
- package/src/domain/whatif/simulate-actions.ts +114 -0
- package/src/domain/whatif/whatif.test.ts +135 -0
- package/src/e2e/gaps-e2e.test.ts +259 -0
- package/src/e2e/smoke.test.ts +101 -0
- package/src/hooks/hooks-export.test.ts +81 -0
- package/src/hooks/installer.ts +113 -0
- package/src/http/cors.test.ts +38 -0
- package/src/http/create-router.ts +259 -0
- package/src/http/routes/agent.route.ts +380 -0
- package/src/http/routes/audit.route.ts +66 -0
- package/src/http/routes/badge.route.ts +23 -0
- package/src/http/routes/cert.route.ts +66 -0
- package/src/http/routes/chat.route.ts +228 -0
- package/src/http/routes/cost.route.ts +33 -0
- package/src/http/routes/debt.route.ts +29 -0
- package/src/http/routes/disclaimer.route.ts +64 -0
- package/src/http/routes/eval.route.ts +161 -0
- package/src/http/routes/events.route.test.ts +108 -0
- package/src/http/routes/events.route.ts +71 -0
- package/src/http/routes/external-scan.route.ts +24 -0
- package/src/http/routes/file.route.ts +54 -0
- package/src/http/routes/fix.route.ts +219 -0
- package/src/http/routes/frameworks.route.test.ts +66 -0
- package/src/http/routes/frameworks.route.ts +36 -0
- package/src/http/routes/git.route.ts +27 -0
- package/src/http/routes/guided-onboarding.route.ts +65 -0
- package/src/http/routes/import.route.ts +64 -0
- package/src/http/routes/jurisdiction.route.ts +22 -0
- package/src/http/routes/obligations.route.test.ts +122 -0
- package/src/http/routes/obligations.route.ts +110 -0
- package/src/http/routes/onboarding.route.ts +53 -0
- package/src/http/routes/provider.route.ts +42 -0
- package/src/http/routes/proxy.route.ts +40 -0
- package/src/http/routes/redteam.route.ts +84 -0
- package/src/http/routes/report.route.ts +29 -0
- package/src/http/routes/scan.route.ts +104 -0
- package/src/http/routes/share.route.ts +44 -0
- package/src/http/routes/shell.route.ts +27 -0
- package/src/http/routes/status.route.ts +66 -0
- package/src/http/routes/supply-chain.route.ts +121 -0
- package/src/http/routes/sync.route.ts +328 -0
- package/src/http/routes/tools.route.ts +29 -0
- package/src/http/routes/whatif.route.ts +96 -0
- package/src/http/utils/validation.ts +31 -0
- package/src/index.ts +1 -0
- package/src/infra/bundle-fetcher.ts +77 -0
- package/src/infra/cache-storage.ts +34 -0
- package/src/infra/event-bus.ts +31 -0
- package/src/infra/file-collector.ts +61 -0
- package/src/infra/file-ops-adapter.ts +95 -0
- package/src/infra/file-watcher.test.ts +90 -0
- package/src/infra/file-watcher.ts +106 -0
- package/src/infra/git-adapter.ts +93 -0
- package/src/infra/git-history-adapter.ts +41 -0
- package/src/infra/headless-browser.ts +178 -0
- package/src/infra/llm-adapter.test.ts +83 -0
- package/src/infra/llm-adapter.ts +86 -0
- package/src/infra/logger.ts +27 -0
- package/src/infra/project-config.test.ts +74 -0
- package/src/infra/project-config.ts +35 -0
- package/src/infra/rate-limiter.test.ts +36 -0
- package/src/infra/rate-limiter.ts +34 -0
- package/src/infra/retry.ts +46 -0
- package/src/infra/saas-client.ts +123 -0
- package/src/infra/search-adapter.ts +113 -0
- package/src/infra/shell-adapter.ts +68 -0
- package/src/infra/tool-manager.test.ts +99 -0
- package/src/infra/tool-manager.ts +197 -0
- package/src/llm/agents/agent-modes.test.ts +44 -0
- package/src/llm/agents/modes.ts +68 -0
- package/src/llm/routing/cost-routing.test.ts +37 -0
- package/src/llm/routing/cost-tracker.ts +74 -0
- package/src/llm/routing/model-routing.test.ts +79 -0
- package/src/llm/routing/model-routing.ts +38 -0
- package/src/llm/routing/pricing.ts +19 -0
- package/src/llm/sse-protocol.ts +77 -0
- package/src/llm/tool-definitions.ts +83 -0
- package/src/llm/tool-executors.ts +80 -0
- package/src/llm/tools/types.ts +13 -0
- package/src/mcp/create-mcp-stack.ts +82 -0
- package/src/mcp/handlers.ts +245 -0
- package/src/mcp/index.ts +28 -0
- package/src/mcp/mcp-server.test.ts +80 -0
- package/src/mcp/server.ts +79 -0
- package/src/mcp/tools.ts +48 -0
- package/src/onboarding/auto-detect.ts +164 -0
- package/src/onboarding/onboarding.test.ts +89 -0
- package/src/onboarding/profile.ts +169 -0
- package/src/onboarding/questions.ts +112 -0
- package/src/onboarding/wizard.ts +66 -0
- package/src/output/github-issue.ts +32 -0
- package/src/output/json-output.ts +67 -0
- package/src/ports/browser.port.ts +23 -0
- package/src/ports/events.port.ts +28 -0
- package/src/ports/llm.port.ts +23 -0
- package/src/ports/logger.port.ts +6 -0
- package/src/ports/process.port.ts +6 -0
- package/src/ports/scanner.port.ts +15 -0
- package/src/server.ts +134 -0
- package/src/services/badge-service.ts +67 -0
- package/src/services/chat-service.test.ts +162 -0
- package/src/services/chat-service.ts +152 -0
- package/src/services/cost-service.ts +52 -0
- package/src/services/debt-service.ts +65 -0
- package/src/services/eval-integration.test.ts +132 -0
- package/src/services/eval-service.test.ts +373 -0
- package/src/services/eval-service.ts +463 -0
- package/src/services/external-scan-service.ts +60 -0
- package/src/services/file-service.ts +37 -0
- package/src/services/fix-service.test.ts +470 -0
- package/src/services/fix-service.ts +648 -0
- package/src/services/framework-service.test.ts +159 -0
- package/src/services/framework-service.ts +67 -0
- package/src/services/onboarding-service.ts +165 -0
- package/src/services/passport-audit.ts +244 -0
- package/src/services/passport-documents.ts +258 -0
- package/src/services/passport-service-utils.ts +72 -0
- package/src/services/passport-service.test.ts +251 -0
- package/src/services/passport-service.ts +339 -0
- package/src/services/proxy-service.ts +81 -0
- package/src/services/report-service.ts +72 -0
- package/src/services/scan-service.test.ts +470 -0
- package/src/services/scan-service.ts +335 -0
- package/src/services/share-service.ts +108 -0
- package/src/services/shared/backup.ts +23 -0
- package/src/services/status-service.ts +38 -0
- package/src/services/undo-service.test.ts +190 -0
- package/src/services/undo-service.ts +144 -0
- package/src/test-helpers/factories.ts +116 -0
- package/src/types/common.schemas.ts +147 -0
- package/src/types/common.types.ts +292 -0
- package/src/types/contract.test.ts +217 -0
- package/src/types/errors.ts +52 -0
- package/src/types/framework.types.ts +87 -0
- package/src/types/passport-schemas.ts +241 -0
- package/src/types/passport.types.ts +296 -0
- package/src/version.ts +1 -0
- package/tsconfig.json +20 -0
- package/vitest.config.ts +9 -0
|
@@ -0,0 +1,1207 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach } from 'vitest';
|
|
2
|
+
import { runLayer2, validateDocument, loadValidators, measureSectionDepth, measureSemanticDepth, layer2ToCheckResults } from './layer2-docs.js';
|
|
3
|
+
import { hasAiReviewMarker, extractReviewDate } from './layer2-parsing.js';
|
|
4
|
+
import { createScanFile, createScanCtx } from '../../../test-helpers/factories.js';
|
|
5
|
+
import type { DocumentValidator } from './layer2-docs.js';
|
|
6
|
+
|
|
7
|
+
describe('loadValidators', () => {
|
|
8
|
+
it('loads all 15 validators', () => {
|
|
9
|
+
const validators = loadValidators();
|
|
10
|
+
expect(validators).toHaveLength(15);
|
|
11
|
+
|
|
12
|
+
const documents = validators.map((v) => v.document).sort();
|
|
13
|
+
expect(documents).toEqual([
|
|
14
|
+
'ai-literacy',
|
|
15
|
+
'art5-screening',
|
|
16
|
+
'biometrics-ai-policy',
|
|
17
|
+
'critical-infra-ai-policy',
|
|
18
|
+
'data-governance',
|
|
19
|
+
'declaration-conformity',
|
|
20
|
+
'fria',
|
|
21
|
+
'incident-report',
|
|
22
|
+
'instructions-for-use',
|
|
23
|
+
'migration-ai-policy',
|
|
24
|
+
'monitoring-policy',
|
|
25
|
+
'qms',
|
|
26
|
+
'risk-management',
|
|
27
|
+
'tech-documentation',
|
|
28
|
+
'worker-notification',
|
|
29
|
+
]);
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe('validateDocument', () => {
|
|
34
|
+
const validator: DocumentValidator = {
|
|
35
|
+
document: 'ai-literacy',
|
|
36
|
+
obligation: 'eu-ai-act-OBL-001',
|
|
37
|
+
article: 'Art. 4',
|
|
38
|
+
file_patterns: ['AI-LITERACY.md'],
|
|
39
|
+
required_sections: [
|
|
40
|
+
{ title: 'Training Program', required: true },
|
|
41
|
+
{ title: 'Training Levels', required: true },
|
|
42
|
+
{ title: 'Assessment Methods', required: true },
|
|
43
|
+
{ title: 'Record Keeping', required: false },
|
|
44
|
+
],
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
it('returns VALID when all required sections present with substantive content', () => {
|
|
48
|
+
const content = `# AI Literacy Policy
|
|
49
|
+
|
|
50
|
+
## Training Program
|
|
51
|
+
The company has established a comprehensive AI literacy training program that covers all employees
|
|
52
|
+
who interact with AI systems in their daily work. The training covers EU AI Act obligations,
|
|
53
|
+
risk assessment procedures, and practical compliance measures. Training sessions are conducted
|
|
54
|
+
quarterly with updated materials reflecting the latest regulatory guidance from August 2026.
|
|
55
|
+
- Module 1: AI fundamentals and EU AI Act overview
|
|
56
|
+
- Module 2: Risk classification and prohibited practices
|
|
57
|
+
- Module 3: Transparency and disclosure requirements
|
|
58
|
+
|
|
59
|
+
## Training Levels
|
|
60
|
+
Training is delivered at three levels based on the employee's role and interaction with AI systems.
|
|
61
|
+
Level 1 is for general staff awareness, Level 2 for technical teams who develop or maintain AI,
|
|
62
|
+
and Level 3 for compliance officers and management who oversee AI governance per Art. 4.
|
|
63
|
+
Each level has specific learning objectives, materials, and certification requirements in the HR system.
|
|
64
|
+
Completion rates must reach 95% within 30 days of onboarding. Monthly reports to compliance committee.
|
|
65
|
+
- Level 1: General awareness (4 hours)
|
|
66
|
+
- Level 2: Technical depth (16 hours)
|
|
67
|
+
- Level 3: Governance certification (40 hours)
|
|
68
|
+
|
|
69
|
+
## Assessment Methods
|
|
70
|
+
Assessment is conducted through a combination of online quizzes, practical exercises, and
|
|
71
|
+
scenario-based evaluations. Pass threshold is 80% for all levels. Employees who fail are
|
|
72
|
+
given additional training and must re-test within 30 days. Records of all assessments
|
|
73
|
+
are maintained for a minimum of 5 years as required by Art. 4 of the EU AI Act.
|
|
74
|
+
`;
|
|
75
|
+
const result = validateDocument(validator, content);
|
|
76
|
+
|
|
77
|
+
expect(result.status).toBe('VALID');
|
|
78
|
+
expect(result.matchedRequired).toBe(3);
|
|
79
|
+
expect(result.totalRequired).toBe(3);
|
|
80
|
+
expect(result.missingSections).toHaveLength(0);
|
|
81
|
+
expect(result.obligationId).toBe('eu-ai-act-OBL-001');
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('returns PARTIAL when some required sections missing', () => {
|
|
85
|
+
const content = `# AI Literacy Policy
|
|
86
|
+
|
|
87
|
+
## Training Program
|
|
88
|
+
Details here.
|
|
89
|
+
|
|
90
|
+
## Record Keeping
|
|
91
|
+
Some records.
|
|
92
|
+
`;
|
|
93
|
+
const result = validateDocument(validator, content);
|
|
94
|
+
|
|
95
|
+
expect(result.status).toBe('PARTIAL');
|
|
96
|
+
expect(result.matchedRequired).toBe(1);
|
|
97
|
+
expect(result.missingSections).toContain('Training Levels');
|
|
98
|
+
expect(result.missingSections).toContain('Assessment Methods');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('returns EMPTY when document has no content', () => {
|
|
102
|
+
const result = validateDocument(validator, '');
|
|
103
|
+
|
|
104
|
+
expect(result.status).toBe('EMPTY');
|
|
105
|
+
expect(result.matchedRequired).toBe(0);
|
|
106
|
+
expect(result.missingSections).toHaveLength(3);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('returns EMPTY when document has no headings', () => {
|
|
110
|
+
const content = 'Some text without any headings whatsoever.';
|
|
111
|
+
const result = validateDocument(validator, content);
|
|
112
|
+
|
|
113
|
+
expect(result.status).toBe('EMPTY');
|
|
114
|
+
expect(result.matchedRequired).toBe(0);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('aggregates child section content under parent headings', () => {
|
|
118
|
+
const techDocValidator: DocumentValidator = {
|
|
119
|
+
document: 'tech-documentation',
|
|
120
|
+
obligation: 'eu-ai-act-OBL-010',
|
|
121
|
+
article: 'Art. 11',
|
|
122
|
+
file_patterns: ['TECH-DOCUMENTATION.md'],
|
|
123
|
+
required_sections: [
|
|
124
|
+
{ title: 'General Description', required: true },
|
|
125
|
+
{ title: 'System Elements', required: true },
|
|
126
|
+
],
|
|
127
|
+
};
|
|
128
|
+
|
|
129
|
+
const content = `# Technical Documentation
|
|
130
|
+
|
|
131
|
+
## General Description
|
|
132
|
+
This AI system is a classification engine used for high-risk compliance assessment.
|
|
133
|
+
It processes documents against the EU AI Act requirements per Art. 11 obligations.
|
|
134
|
+
Quarterly reviews ensure continued compliance with all applicable standards.
|
|
135
|
+
- Input: regulatory documents
|
|
136
|
+
- Output: compliance scores with 95% accuracy target
|
|
137
|
+
|
|
138
|
+
## System Elements
|
|
139
|
+
|
|
140
|
+
### 2.1 Architecture
|
|
141
|
+
The system uses a layered architecture with deterministic scanning pipelines.
|
|
142
|
+
Each layer handles a specific level of analysis from file presence to deep AST inspection.
|
|
143
|
+
Components are orchestrated by a daemon process that monitors file changes in real-time.
|
|
144
|
+
The architecture is designed per ISO 25010 quality model with documented SLA targets.
|
|
145
|
+
- Scanner engine with 5 detection layers
|
|
146
|
+
- Evidence chain with ed25519 signatures
|
|
147
|
+
|
|
148
|
+
### 2.2 Components
|
|
149
|
+
The core components include the file watcher, scanner engine, and reporting module.
|
|
150
|
+
All components are independently testable and communicate via typed event bus.
|
|
151
|
+
Performance monitoring tracks KPI metrics with automated alerts at >5% deviation.
|
|
152
|
+
- File watcher: <200ms detection latency
|
|
153
|
+
- Scanner: processes 1000 files/minute
|
|
154
|
+
`;
|
|
155
|
+
const result = validateDocument(techDocValidator, content);
|
|
156
|
+
|
|
157
|
+
// With grouped extraction, ### subsections aggregate under ## System Elements
|
|
158
|
+
expect(result.status).toBe('VALID');
|
|
159
|
+
expect(result.shallowSections).toBeUndefined();
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
it('handles mixed heading levels — only marks truly shallow sections', () => {
|
|
163
|
+
const mixedValidator: DocumentValidator = {
|
|
164
|
+
document: 'tech-documentation',
|
|
165
|
+
obligation: 'eu-ai-act-OBL-010',
|
|
166
|
+
article: 'Art. 11',
|
|
167
|
+
file_patterns: ['TECH-DOCUMENTATION.md'],
|
|
168
|
+
required_sections: [
|
|
169
|
+
{ title: 'Section A', required: true },
|
|
170
|
+
{ title: 'Section B', required: true },
|
|
171
|
+
],
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
const content = `# Document
|
|
175
|
+
|
|
176
|
+
## Section A
|
|
177
|
+
|
|
178
|
+
### A.1 Details
|
|
179
|
+
This subsection provides comprehensive documentation as required by the EU AI Act.
|
|
180
|
+
Assessment conducted on 2026-01-15 covering all applicable requirements per Art. 9.
|
|
181
|
+
Key findings include a 95% compliance rate across all evaluated criteria.
|
|
182
|
+
- Item 1: Detailed analysis
|
|
183
|
+
- Item 2: Evidence of compliance
|
|
184
|
+
|
|
185
|
+
### A.2 More Details
|
|
186
|
+
Additional documentation with quarterly review schedule and SLA targets.
|
|
187
|
+
Monitoring dashboard tracks KPIs with automated threshold alerts.
|
|
188
|
+
All measures documented per ISO 42001 requirements for ongoing compliance.
|
|
189
|
+
- Monthly audits
|
|
190
|
+
- Annual comprehensive review
|
|
191
|
+
|
|
192
|
+
## Section B
|
|
193
|
+
This section provides additional configuration and deployment documentation as required by Art. 11.
|
|
194
|
+
The system must be deployed with proper logging and monitoring. Reviews conducted quarterly per ISO 42001.
|
|
195
|
+
All deployment records are maintained for 5 years with 99.9% availability SLA targets.
|
|
196
|
+
- Deployment checklist verified before each release
|
|
197
|
+
- Rollback procedures documented and tested
|
|
198
|
+
`;
|
|
199
|
+
const result = validateDocument(mixedValidator, content);
|
|
200
|
+
|
|
201
|
+
// Section A has rich subsections → not shallow
|
|
202
|
+
// Section B now has substantive content → not shallow
|
|
203
|
+
// Both sections have completenessScore >= 50 → VALID
|
|
204
|
+
expect(result.status).toBe('VALID');
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('works with single-level headings (no regression)', () => {
|
|
208
|
+
const content = `# AI Literacy Policy
|
|
209
|
+
|
|
210
|
+
## Training Program
|
|
211
|
+
The company has established a comprehensive AI literacy training program that covers all employees.
|
|
212
|
+
Training covers EU AI Act obligations, risk assessment procedures, and practical compliance per Art. 4.
|
|
213
|
+
Sessions are conducted quarterly with updated materials. Completion target is 95% annually.
|
|
214
|
+
- Module 1: AI fundamentals and EU AI Act overview
|
|
215
|
+
- Module 2: Risk classification and prohibited practices
|
|
216
|
+
|
|
217
|
+
## Training Levels
|
|
218
|
+
Training is delivered at three levels based on the employee's role. Level 1 for general staff,
|
|
219
|
+
Level 2 for technical teams, Level 3 for compliance officers. Each level has specific learning
|
|
220
|
+
objectives and certification requirements documented in the HR system per GDPR requirements.
|
|
221
|
+
Completion rates tracked monthly with SLA target of 90% within 30 days of onboarding.
|
|
222
|
+
|
|
223
|
+
## Assessment Methods
|
|
224
|
+
Assessment uses online quizzes, practical exercises, and scenario-based evaluations.
|
|
225
|
+
Pass threshold is 80% for all levels. Re-test within 30 days per Art. 4 requirements.
|
|
226
|
+
Records maintained for minimum 5 years. Annual review of assessment materials.
|
|
227
|
+
- Knowledge tests with >80% pass rate
|
|
228
|
+
- Practical scenario exercises quarterly
|
|
229
|
+
`;
|
|
230
|
+
const result = validateDocument(validator, content);
|
|
231
|
+
|
|
232
|
+
expect(result.status).toBe('VALID');
|
|
233
|
+
expect(result.matchedRequired).toBe(3);
|
|
234
|
+
expect(result.missingSections).toHaveLength(0);
|
|
235
|
+
});
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
describe('runLayer2', () => {
|
|
239
|
+
// Helper to create doc content that passes depth check (>50 words per section with specifics)
|
|
240
|
+
const makeRichContent = (title: string, sections: readonly string[]): string => {
|
|
241
|
+
const body = sections.map((s) => `## ${s}\nThis section provides comprehensive documentation as required by the EU AI Act regulation.\nThe assessment was conducted on 2026-01-15 and covers all applicable requirements.\nKey findings include a 95% compliance rate across all evaluated criteria.\n- Item 1: Detailed analysis of requirements\n- Item 2: Evidence of compliance measures\n- Item 3: Documented procedures and policies`).join('\n\n');
|
|
242
|
+
return `# ${title}\n\n${body}`;
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
it('validates all documents found by L1 (full compliance)', () => {
|
|
246
|
+
const ctx = createScanCtx([
|
|
247
|
+
createScanFile('AI-LITERACY.md', makeRichContent('AI Literacy', ['Training Program', 'Training Levels', 'Assessment Methods'])),
|
|
248
|
+
createScanFile('ART5-SCREENING.md', makeRichContent('Screening', ['Prohibited Practices', 'Screening Results', 'Mitigations'])),
|
|
249
|
+
createScanFile('FRIA.md', makeRichContent('FRIA', ['Risk Assessment', 'Impact Analysis', 'Mitigation Measures'])),
|
|
250
|
+
createScanFile('WORKER-NOTIFICATION.md', makeRichContent('Workers', ['Notification Scope', 'Affected Workers', 'Timeline', 'Delivery Tracking'])),
|
|
251
|
+
createScanFile('TECH-DOCUMENTATION.md', makeRichContent('Tech', ['General Description', 'System Elements', 'Monitoring, Functioning and Control', 'Validation and Testing', 'Accuracy, Robustness and Cybersecurity'])),
|
|
252
|
+
createScanFile('INCIDENT-REPORT.md', makeRichContent('Incident', ['Incident Description', 'Root Cause', 'Corrective Measures', 'Timeline of Events'])),
|
|
253
|
+
createScanFile('DECLARATION-OF-CONFORMITY.md', makeRichContent('DoC', ['Conformity Statement', 'Standards Applied', 'Evidence'])),
|
|
254
|
+
createScanFile('MONITORING-POLICY.md', makeRichContent('Monitoring', ['Monitoring Scope', 'Frequency', 'Escalation Procedures'])),
|
|
255
|
+
createScanFile('RISK-MANAGEMENT.md', makeRichContent('Risk', ['Known Risks', 'Misuse Scenarios', 'Residual Risk Assessment'])),
|
|
256
|
+
createScanFile('DATA-GOVERNANCE.md', makeRichContent('Data', ['Data Sources', 'Collection Methods', 'Quality Metrics'])),
|
|
257
|
+
createScanFile('QMS.md', makeRichContent('QMS', ['Compliance Strategy', 'Design Control', 'Testing Procedures'])),
|
|
258
|
+
createScanFile('INSTRUCTIONS-FOR-USE.md', makeRichContent('Instructions', ['Intended Purpose', 'Capabilities', 'Limitations'])),
|
|
259
|
+
]);
|
|
260
|
+
|
|
261
|
+
const results = runLayer2(ctx);
|
|
262
|
+
|
|
263
|
+
expect(results).toHaveLength(12);
|
|
264
|
+
expect(results.every((r) => r.status === 'VALID')).toBe(true);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
it('skips documents not found by L1', () => {
|
|
268
|
+
const ctx = createScanCtx([
|
|
269
|
+
createScanFile('AI-LITERACY.md', makeRichContent('AI Literacy', ['Training Program', 'Training Levels', 'Assessment Methods'])),
|
|
270
|
+
createScanFile('src/app.ts', 'function main() {}'),
|
|
271
|
+
]);
|
|
272
|
+
|
|
273
|
+
const results = runLayer2(ctx);
|
|
274
|
+
|
|
275
|
+
// Only AI-LITERACY.md matched a validator
|
|
276
|
+
expect(results).toHaveLength(1);
|
|
277
|
+
expect(results[0].document).toBe('ai-literacy');
|
|
278
|
+
expect(results[0].status).toBe('VALID');
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
it('handles partial documents correctly', () => {
|
|
282
|
+
const ctx = createScanCtx([
|
|
283
|
+
createScanFile('AI-LITERACY.md', `# AI Literacy\n## Training Program\nContent only.`),
|
|
284
|
+
]);
|
|
285
|
+
|
|
286
|
+
const results = runLayer2(ctx);
|
|
287
|
+
|
|
288
|
+
expect(results).toHaveLength(1);
|
|
289
|
+
expect(results[0].status).toBe('PARTIAL');
|
|
290
|
+
expect(results[0].matchedRequired).toBe(1);
|
|
291
|
+
expect(results[0].missingSections).toContain('Training Levels');
|
|
292
|
+
expect(results[0].missingSections).toContain('Assessment Methods');
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
it('handles empty documents correctly', () => {
|
|
296
|
+
const ctx = createScanCtx([
|
|
297
|
+
createScanFile('AI-LITERACY.md', ''),
|
|
298
|
+
]);
|
|
299
|
+
|
|
300
|
+
const results = runLayer2(ctx);
|
|
301
|
+
|
|
302
|
+
expect(results).toHaveLength(1);
|
|
303
|
+
expect(results[0].status).toBe('EMPTY');
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
it('validates documents in nested directories', () => {
|
|
307
|
+
const ctx = createScanCtx([
|
|
308
|
+
createScanFile('docs/compliance/AI-LITERACY.md', makeRichContent('AI Literacy', ['Training Program', 'Training Levels', 'Assessment Methods'])),
|
|
309
|
+
]);
|
|
310
|
+
|
|
311
|
+
const results = runLayer2(ctx);
|
|
312
|
+
|
|
313
|
+
expect(results).toHaveLength(1);
|
|
314
|
+
expect(results[0].status).toBe('VALID');
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
it('detects SHALLOW documents with heading-only content', () => {
|
|
318
|
+
const ctx = createScanCtx([
|
|
319
|
+
createScanFile('AI-LITERACY.md', `# AI Literacy\n## Training Program\nBrief.\n## Training Levels\nTODO.\n## Assessment Methods\nPending.`),
|
|
320
|
+
]);
|
|
321
|
+
|
|
322
|
+
const results = runLayer2(ctx);
|
|
323
|
+
|
|
324
|
+
expect(results).toHaveLength(1);
|
|
325
|
+
expect(results[0].status).toBe('SHALLOW');
|
|
326
|
+
expect(results[0].shallowSections).toBeDefined();
|
|
327
|
+
expect(results[0].shallowSections?.length).toBeGreaterThan(0);
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
it('returns VALID when some sections shallow but not over 50%', () => {
|
|
331
|
+
const richSection = `This section provides comprehensive documentation as required by the EU AI Act regulation.
|
|
332
|
+
The assessment was conducted on 2026-01-15 and covers all applicable requirements per Art. 72.
|
|
333
|
+
Key findings include a 95% compliance rate across all evaluated criteria and objectives.
|
|
334
|
+
The SLA target is within 24 hours response time. Monthly audits are conducted quarterly.
|
|
335
|
+
All measures documented per ISO 42001 requirements for ongoing regulatory compliance.
|
|
336
|
+
- Item 1: Detailed analysis of requirements
|
|
337
|
+
- Item 2: Evidence of compliance measures
|
|
338
|
+
- Item 3: Documented procedures and policies`;
|
|
339
|
+
|
|
340
|
+
const ctx = createScanCtx([
|
|
341
|
+
createScanFile('AI-LITERACY.md', `# AI Literacy\n## Training Program\n${richSection}\n## Training Levels\n${richSection}\n## Assessment Methods\nBrief.`),
|
|
342
|
+
]);
|
|
343
|
+
|
|
344
|
+
const results = runLayer2(ctx);
|
|
345
|
+
|
|
346
|
+
expect(results).toHaveLength(1);
|
|
347
|
+
// 1 out of 3 shallow = 33% < 50% → shallowRatio OK
|
|
348
|
+
// 2 rich sections + 1 shallow → avg completenessScore >= 50 → VALID
|
|
349
|
+
expect(results[0].status).toBe('VALID');
|
|
350
|
+
});
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
describe('measureSectionDepth', () => {
|
|
354
|
+
it('detects shallow content (few words, no structure)', () => {
|
|
355
|
+
const depth = measureSectionDepth('Brief content here.');
|
|
356
|
+
expect(depth.isShallow).toBe(true);
|
|
357
|
+
expect(depth.wordCount).toBeLessThan(50);
|
|
358
|
+
expect(depth.hasLists).toBe(false);
|
|
359
|
+
expect(depth.hasTables).toBe(false);
|
|
360
|
+
expect(depth.hasSpecifics).toBe(false);
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
it('detects rich content with enough words', () => {
|
|
364
|
+
const words = Array(60).fill('compliance').join(' ');
|
|
365
|
+
const depth = measureSectionDepth(words);
|
|
366
|
+
expect(depth.isShallow).toBe(false);
|
|
367
|
+
expect(depth.wordCount).toBeGreaterThanOrEqual(50);
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
it('detects lists as substantive', () => {
|
|
371
|
+
const depth = measureSectionDepth('Overview:\n- Item one\n- Item two\n- Item three');
|
|
372
|
+
expect(depth.hasLists).toBe(true);
|
|
373
|
+
expect(depth.isShallow).toBe(false);
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
it('detects numbered lists', () => {
|
|
377
|
+
const depth = measureSectionDepth('Steps:\n1. First step\n2. Second step');
|
|
378
|
+
expect(depth.hasLists).toBe(true);
|
|
379
|
+
expect(depth.isShallow).toBe(false);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
it('detects tables as substantive', () => {
|
|
383
|
+
const depth = measureSectionDepth('| Name | Value |\n| --- | --- |\n| Test | 42 |');
|
|
384
|
+
expect(depth.hasTables).toBe(true);
|
|
385
|
+
expect(depth.isShallow).toBe(false);
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
it('detects dates as specifics', () => {
|
|
389
|
+
const depth = measureSectionDepth('Completed on 2026-01-15.');
|
|
390
|
+
expect(depth.hasSpecifics).toBe(true);
|
|
391
|
+
expect(depth.isShallow).toBe(false);
|
|
392
|
+
});
|
|
393
|
+
|
|
394
|
+
it('detects percentages as specifics', () => {
|
|
395
|
+
const depth = measureSectionDepth('Achieved 95% compliance rate.');
|
|
396
|
+
expect(depth.hasSpecifics).toBe(true);
|
|
397
|
+
expect(depth.isShallow).toBe(false);
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
it('detects article references as specifics', () => {
|
|
401
|
+
const depth = measureSectionDepth('As required by Art. 50 of the EU AI Act.');
|
|
402
|
+
expect(depth.hasSpecifics).toBe(true);
|
|
403
|
+
expect(depth.isShallow).toBe(false);
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
it('detects euro amounts as specifics', () => {
|
|
407
|
+
const depth = measureSectionDepth('Maximum fine: €35M or 7% of turnover.');
|
|
408
|
+
expect(depth.hasSpecifics).toBe(true);
|
|
409
|
+
expect(depth.isShallow).toBe(false);
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
it('handles empty content', () => {
|
|
413
|
+
const depth = measureSectionDepth('');
|
|
414
|
+
expect(depth.isShallow).toBe(true);
|
|
415
|
+
expect(depth.wordCount).toBe(0);
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
it('counts sentences correctly', () => {
|
|
419
|
+
const depth = measureSectionDepth('First sentence. Second sentence! Third sentence?');
|
|
420
|
+
expect(depth.sentenceCount).toBe(3);
|
|
421
|
+
});
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
describe('measureSemanticDepth', () => {
|
|
425
|
+
it('detects numeric metrics (percentages)', () => {
|
|
426
|
+
const depth = measureSemanticDepth('System achieves 99.5% uptime with less than 50ms latency.', 'Performance');
|
|
427
|
+
expect(depth.hasNumericMetrics).toBe(true);
|
|
428
|
+
});
|
|
429
|
+
|
|
430
|
+
it('detects numeric metrics (unit-based counts)', () => {
|
|
431
|
+
const depth = measureSemanticDepth('Processing up to 1000 requests per second with 5 errors maximum.', 'Capacity');
|
|
432
|
+
expect(depth.hasNumericMetrics).toBe(true);
|
|
433
|
+
});
|
|
434
|
+
|
|
435
|
+
it('detects legal references (Art. numbers)', () => {
|
|
436
|
+
const depth = measureSemanticDepth('As required by Art. 9 of the EU AI Act, risk management is mandatory.', 'Legal');
|
|
437
|
+
expect(depth.hasLegalReferences).toBe(true);
|
|
438
|
+
});
|
|
439
|
+
|
|
440
|
+
it('detects legal references (GDPR)', () => {
|
|
441
|
+
const depth = measureSemanticDepth('Data processing complies with GDPR requirements for personal data.', 'Data');
|
|
442
|
+
expect(depth.hasLegalReferences).toBe(true);
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
it('detects legal references (ISO standards)', () => {
|
|
446
|
+
const depth = measureSemanticDepth('The system is certified under ISO 27001 information security standard.', 'Security');
|
|
447
|
+
expect(depth.hasLegalReferences).toBe(true);
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
it('detects legal references (NIST)', () => {
|
|
451
|
+
const depth = measureSemanticDepth('Following NIST AI Risk Management Framework guidelines.', 'Framework');
|
|
452
|
+
expect(depth.hasLegalReferences).toBe(true);
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it('detects date references (ISO dates)', () => {
|
|
456
|
+
const depth = measureSemanticDepth('Last review completed on 2026-03-15 with next review scheduled.', 'Timeline');
|
|
457
|
+
expect(depth.hasDateReferences).toBe(true);
|
|
458
|
+
});
|
|
459
|
+
|
|
460
|
+
it('detects date references (frequency terms)', () => {
|
|
461
|
+
const depth = measureSemanticDepth('Reviews are conducted quarterly with annual comprehensive audits.', 'Schedule');
|
|
462
|
+
expect(depth.hasDateReferences).toBe(true);
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
it('detects date references (quarter notation)', () => {
|
|
466
|
+
const depth = measureSemanticDepth('Target completion by Q2 2026 with interim milestones.', 'Timeline');
|
|
467
|
+
expect(depth.hasDateReferences).toBe(true);
|
|
468
|
+
});
|
|
469
|
+
|
|
470
|
+
it('detects action items (must/shall)', () => {
|
|
471
|
+
const depth = measureSemanticDepth('The deployer must ensure human oversight. The system shall log all decisions.', 'Requirements');
|
|
472
|
+
expect(depth.hasActionItems).toBe(true);
|
|
473
|
+
});
|
|
474
|
+
|
|
475
|
+
it('detects action items (required to)', () => {
|
|
476
|
+
const depth = measureSemanticDepth('Operators are required to maintain records for 5 years.', 'Obligations');
|
|
477
|
+
expect(depth.hasActionItems).toBe(true);
|
|
478
|
+
});
|
|
479
|
+
|
|
480
|
+
it('detects measurable targets (SLA/KPI)', () => {
|
|
481
|
+
const depth = measureSemanticDepth('The SLA specifies 99.9% availability with KPI tracking dashboard.', 'Targets');
|
|
482
|
+
expect(depth.hasMeasurableTargets).toBe(true);
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
it('detects measurable targets (within timeframe)', () => {
|
|
486
|
+
const depth = measureSemanticDepth('All incidents must be reported within 72 hours of detection.', 'Response');
|
|
487
|
+
expect(depth.hasMeasurableTargets).toBe(true);
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
it('generates actionable feedback for missing signals', () => {
|
|
491
|
+
const depth = measureSemanticDepth('Some generic text about the system.', 'Overview');
|
|
492
|
+
expect(depth.feedback).toContain('Section "Overview"');
|
|
493
|
+
expect(depth.feedback).toContain('Add numeric metrics');
|
|
494
|
+
expect(depth.feedback).toContain('Add legal references');
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
it('returns adequate feedback when all signals present', () => {
|
|
498
|
+
const content = `As required by Art. 9 of the EU AI Act, the system must achieve 99.9% accuracy.
|
|
499
|
+
Reviews are conducted quarterly starting from 2026-01-01.
|
|
500
|
+
The SLA target is within 24 hours response time for all incidents.
|
|
501
|
+
- Monitoring dashboard tracks KPIs
|
|
502
|
+
- Automated alerts for threshold breaches
|
|
503
|
+
| Metric | Target | Current |
|
|
504
|
+
| --- | --- | --- |
|
|
505
|
+
| Uptime | 99.9% | 99.95% |`;
|
|
506
|
+
const depth = measureSemanticDepth(content, 'Monitoring');
|
|
507
|
+
expect(depth.feedback).toBe('Section "Monitoring": adequate');
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it('gives low qualityScore for shallow generic text', () => {
|
|
511
|
+
const depth = measureSemanticDepth('Brief placeholder text.', 'Overview');
|
|
512
|
+
expect(depth.qualityScore).toBeLessThan(30);
|
|
513
|
+
expect(depth.isShallow).toBe(true);
|
|
514
|
+
});
|
|
515
|
+
|
|
516
|
+
it('gives high qualityScore for rich content with metrics, legal refs, dates', () => {
|
|
517
|
+
const content = `As required by Art. 9 of the EU AI Act, the system must achieve 99.9% accuracy.
|
|
518
|
+
Reviews are conducted quarterly starting from 2026-01-01.
|
|
519
|
+
The SLA target is within 24 hours response time for all incidents.
|
|
520
|
+
This comprehensive section documents all compliance requirements and obligations.
|
|
521
|
+
The risk management framework covers identification, assessment, and mitigation of all known risks.
|
|
522
|
+
Ongoing monitoring ensures continuous compliance with regulatory requirements.
|
|
523
|
+
- Monitoring dashboard tracks KPIs
|
|
524
|
+
- Automated alerts for threshold breaches
|
|
525
|
+
| Metric | Target | Current |
|
|
526
|
+
| --- | --- | --- |
|
|
527
|
+
| Uptime | 99.9% | 99.95% |`;
|
|
528
|
+
const depth = measureSemanticDepth(content, 'Monitoring');
|
|
529
|
+
expect(depth.qualityScore).toBeGreaterThanOrEqual(70);
|
|
530
|
+
expect(depth.isShallow).toBe(false);
|
|
531
|
+
});
|
|
532
|
+
|
|
533
|
+
it('inherits base SectionDepth properties', () => {
|
|
534
|
+
const depth = measureSemanticDepth('- Item 1\n- Item 2\n- Item 3', 'List Section');
|
|
535
|
+
expect(depth.hasLists).toBe(true);
|
|
536
|
+
expect(depth.wordCount).toBeGreaterThan(0);
|
|
537
|
+
expect(depth.sentenceCount).toBeGreaterThan(0);
|
|
538
|
+
});
|
|
539
|
+
|
|
540
|
+
it('handles empty content', () => {
|
|
541
|
+
const depth = measureSemanticDepth('', 'Empty');
|
|
542
|
+
expect(depth.qualityScore).toBe(0);
|
|
543
|
+
expect(depth.isShallow).toBe(true);
|
|
544
|
+
expect(depth.wordCount).toBe(0);
|
|
545
|
+
});
|
|
546
|
+
|
|
547
|
+
it('detects scaffold placeholders and penalizes qualityScore', () => {
|
|
548
|
+
// Scaffold content: good structure but [TODO] placeholders
|
|
549
|
+
const content = `| Field | Value |
|
|
550
|
+
| --- | --- |
|
|
551
|
+
| Provider | [Company Name] |
|
|
552
|
+
| Assessor | [Name, Title] |
|
|
553
|
+
| DPO Consulted | [Name, Date] |
|
|
554
|
+
- System name: [Name]
|
|
555
|
+
- Version: [Number]
|
|
556
|
+
- Deployment context: [Where and how the system is used]
|
|
557
|
+
As required by Art. 9 of the EU AI Act, the deployer must ensure compliance.`;
|
|
558
|
+
const depth = measureSemanticDepth(content, 'Header');
|
|
559
|
+
expect(depth.placeholderCount).toBeGreaterThanOrEqual(5);
|
|
560
|
+
expect(depth.isShallow).toBe(true);
|
|
561
|
+
expect(depth.feedback).toContain('placeholder');
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
it('does not count markdown links as placeholders', () => {
|
|
565
|
+
const content = `See [EU AI Act](https://eur-lex.europa.eu) and [GDPR](https://gdpr-info.eu) for details.
|
|
566
|
+
This comprehensive section documents all compliance requirements with Art. 9 references.`;
|
|
567
|
+
const depth = measureSemanticDepth(content, 'References');
|
|
568
|
+
expect(depth.placeholderCount).toBe(0);
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
it('marks scaffold table sections as shallow despite good structure', () => {
|
|
572
|
+
const content = `| Fundamental Right | Risk Level | Description | Affected Group | Mitigation |
|
|
573
|
+
|---|---|---|---|---|
|
|
574
|
+
| Non-discrimination (Charter Art. 21) | [H/M/L/N] | [Description of risk] | [Group] | [Measures] |
|
|
575
|
+
| Privacy (Charter Art. 7-8) | [H/M/L/N] | [Description of risk] | [Group] | [Measures] |
|
|
576
|
+
| Expression (Charter Art. 11) | [H/M/L/N] | [Description of risk] | [Group] | [Measures] |`;
|
|
577
|
+
const depth = measureSemanticDepth(content, 'Risk Assessment');
|
|
578
|
+
expect(depth.placeholderCount).toBeGreaterThanOrEqual(9);
|
|
579
|
+
expect(depth.isShallow).toBe(true);
|
|
580
|
+
});
|
|
581
|
+
|
|
582
|
+
it('overrides isShallow based on qualityScore not just word count', () => {
|
|
583
|
+
// Content with few words but has lists + specifics + legal ref = qualityScore >= 30
|
|
584
|
+
const content = '- Art. 9 compliance check\n- 95% accuracy achieved';
|
|
585
|
+
const depth = measureSemanticDepth(content, 'Checks');
|
|
586
|
+
// Has lists (10) + specifics (10) + numeric metrics (15) + legal ref (10) + measurable (10) = 55
|
|
587
|
+
expect(depth.wordCount).toBeLessThan(50);
|
|
588
|
+
expect(depth.isShallow).toBe(false);
|
|
589
|
+
});
|
|
590
|
+
});
|
|
591
|
+
|
|
592
|
+
describe('validateDocument semantic features', () => {
|
|
593
|
+
const validator: DocumentValidator = {
|
|
594
|
+
document: 'monitoring-policy',
|
|
595
|
+
obligation: 'eu-ai-act-OBL-020',
|
|
596
|
+
article: 'Art. 72',
|
|
597
|
+
file_patterns: ['MONITORING-POLICY.md'],
|
|
598
|
+
required_sections: [
|
|
599
|
+
{ title: 'Monitoring Scope', required: true },
|
|
600
|
+
{ title: 'Frequency', required: true },
|
|
601
|
+
{ title: 'Escalation Procedures', required: true },
|
|
602
|
+
],
|
|
603
|
+
};
|
|
604
|
+
|
|
605
|
+
it('returns sectionFeedback for sections missing semantic signals', () => {
|
|
606
|
+
// Sections have 50+ words and lists (qualityScore >= 30 → not shallow),
|
|
607
|
+
// but lack numeric metrics, legal references, dates, and measurable targets → feedback generated
|
|
608
|
+
const content = `# Monitoring Policy
|
|
609
|
+
|
|
610
|
+
## Monitoring Scope
|
|
611
|
+
This section describes the monitoring scope for the AI system. The monitoring covers
|
|
612
|
+
all production deployments and ensures continuous compliance. Regular checks are performed
|
|
613
|
+
to maintain system integrity. Additional monitoring is applied to high-risk components.
|
|
614
|
+
Feedback loops help improve the monitoring process over time. Documentation is maintained
|
|
615
|
+
for audit purposes across the entire lifecycle.
|
|
616
|
+
- Production environment coverage
|
|
617
|
+
- Development environment spot checks
|
|
618
|
+
- Staging environment validation
|
|
619
|
+
|
|
620
|
+
## Frequency
|
|
621
|
+
Monitoring is performed regularly according to the schedule. The frequency is determined
|
|
622
|
+
by risk level and regulatory requirements. Higher risk systems are monitored more often.
|
|
623
|
+
The schedule is reviewed periodically to ensure adequacy. All monitoring activities are
|
|
624
|
+
documented and tracked. Adjustments are made based on incident trends.
|
|
625
|
+
- Continuous automated monitoring
|
|
626
|
+
- Periodic manual reviews
|
|
627
|
+
- Ad-hoc assessments on demand
|
|
628
|
+
|
|
629
|
+
## Escalation Procedures
|
|
630
|
+
When issues are detected, the escalation procedures are followed. The team is notified
|
|
631
|
+
promptly and corrective actions are taken. Escalation paths are clearly defined. Senior
|
|
632
|
+
management is involved for critical issues. All escalations are tracked and documented.
|
|
633
|
+
Post-incident reviews help improve the process.
|
|
634
|
+
- Level 1: Team lead notification
|
|
635
|
+
- Level 2: Department head escalation
|
|
636
|
+
- Level 3: Executive escalation
|
|
637
|
+
`;
|
|
638
|
+
const result = validateDocument(validator, content);
|
|
639
|
+
|
|
640
|
+
// P17: completenessScore ≈ 30 (words+lists only) < 50 → PARTIAL
|
|
641
|
+
expect(result.status).toBe('PARTIAL');
|
|
642
|
+
expect(result.sectionFeedback).toBeDefined();
|
|
643
|
+
expect(result.sectionFeedback!.length).toBeGreaterThan(0);
|
|
644
|
+
// Should suggest adding numeric metrics, legal references, etc.
|
|
645
|
+
expect(result.sectionFeedback!.some((f) => f.includes('Add numeric metrics'))).toBe(true);
|
|
646
|
+
});
|
|
647
|
+
|
|
648
|
+
it('returns completenessScore for validated documents', () => {
|
|
649
|
+
const content = `# Monitoring Policy
|
|
650
|
+
|
|
651
|
+
## Monitoring Scope
|
|
652
|
+
Brief.
|
|
653
|
+
|
|
654
|
+
## Frequency
|
|
655
|
+
TODO.
|
|
656
|
+
|
|
657
|
+
## Escalation Procedures
|
|
658
|
+
Pending.
|
|
659
|
+
`;
|
|
660
|
+
const result = validateDocument(validator, content);
|
|
661
|
+
|
|
662
|
+
expect(result.completenessScore).toBeDefined();
|
|
663
|
+
expect(typeof result.completenessScore).toBe('number');
|
|
664
|
+
});
|
|
665
|
+
|
|
666
|
+
it('returns high completenessScore for rich semantic content', () => {
|
|
667
|
+
const content = `# Monitoring Policy
|
|
668
|
+
|
|
669
|
+
## Monitoring Scope
|
|
670
|
+
As required by Art. 72 of the EU AI Act, monitoring must cover all production deployments.
|
|
671
|
+
The system shall maintain 99.9% uptime with SLA targets reviewed quarterly starting 2026-01-01.
|
|
672
|
+
The monitoring scope encompasses all AI system components including model performance, data quality,
|
|
673
|
+
and user interaction patterns. Regular assessments are conducted to ensure continued compliance.
|
|
674
|
+
Automated dashboards track KPIs across all monitored systems.
|
|
675
|
+
- Real-time performance tracking
|
|
676
|
+
- Anomaly detection with threshold alerts at >5% deviation
|
|
677
|
+
| Metric | Target | Review Frequency |
|
|
678
|
+
| --- | --- | --- |
|
|
679
|
+
| Accuracy | >95% | Monthly |
|
|
680
|
+
| Latency | <200ms | Weekly |
|
|
681
|
+
|
|
682
|
+
## Frequency
|
|
683
|
+
The system must be monitored continuously with formal reviews conducted quarterly.
|
|
684
|
+
Art. 9 requires ongoing risk assessment. Annual comprehensive audits are mandatory.
|
|
685
|
+
Incident reports must be filed within 72 hours as per GDPR and EU AI Act requirements.
|
|
686
|
+
The monitoring frequency is calibrated to the risk level as specified in ISO 42001.
|
|
687
|
+
Monthly automated scans ensure compliance with all applicable standards and benchmarks.
|
|
688
|
+
- Daily automated health checks
|
|
689
|
+
- Weekly performance reviews with KPI tracking
|
|
690
|
+
- Monthly comprehensive compliance audits
|
|
691
|
+
|
|
692
|
+
## Escalation Procedures
|
|
693
|
+
All incidents shall be escalated within 24 hours to the designated compliance officer.
|
|
694
|
+
As required by Art. 62, serious incidents must be reported to national authorities.
|
|
695
|
+
The SLA for critical issues requires resolution within 4 hours.
|
|
696
|
+
Escalation procedures are reviewed annually and updated based on lessons learned.
|
|
697
|
+
Documentation of all escalation events must be maintained for at least 5 years.
|
|
698
|
+
- Level 1: Automated alert (within 5 minutes)
|
|
699
|
+
- Level 2: Team notification (within 1 hour)
|
|
700
|
+
- Level 3: Management escalation (within 4 hours)
|
|
701
|
+
`;
|
|
702
|
+
const result = validateDocument(validator, content);
|
|
703
|
+
|
|
704
|
+
expect(result.status).toBe('VALID');
|
|
705
|
+
expect(result.completenessScore).toBeDefined();
|
|
706
|
+
expect(result.completenessScore!).toBeGreaterThanOrEqual(60);
|
|
707
|
+
});
|
|
708
|
+
|
|
709
|
+
it('returns low completenessScore for shallow content', () => {
|
|
710
|
+
const content = `# Monitoring Policy
|
|
711
|
+
|
|
712
|
+
## Monitoring Scope
|
|
713
|
+
Brief overview.
|
|
714
|
+
|
|
715
|
+
## Frequency
|
|
716
|
+
Regular checks.
|
|
717
|
+
|
|
718
|
+
## Escalation Procedures
|
|
719
|
+
Contact support.
|
|
720
|
+
`;
|
|
721
|
+
const result = validateDocument(validator, content);
|
|
722
|
+
|
|
723
|
+
expect(result.completenessScore).toBeDefined();
|
|
724
|
+
expect(result.completenessScore!).toBeLessThan(30);
|
|
725
|
+
});
|
|
726
|
+
|
|
727
|
+
it('does not include sectionFeedback when all sections are adequate', () => {
|
|
728
|
+
const richSection = `As required by Art. 72 of the EU AI Act, the system must achieve 99.9% accuracy.
|
|
729
|
+
Reviews are conducted quarterly starting from 2026-01-01.
|
|
730
|
+
The SLA target is within 24 hours response time for all incidents.
|
|
731
|
+
This comprehensive section documents all compliance requirements and obligations.
|
|
732
|
+
The risk management framework covers identification, assessment, and mitigation of all known risks.
|
|
733
|
+
Ongoing monitoring ensures continuous compliance with regulatory requirements.
|
|
734
|
+
- Monitoring dashboard tracks KPIs
|
|
735
|
+
- Automated alerts for threshold breaches
|
|
736
|
+
| Metric | Target | Current |
|
|
737
|
+
| --- | --- | --- |
|
|
738
|
+
| Uptime | 99.9% | 99.95% |`;
|
|
739
|
+
|
|
740
|
+
const content = `# Monitoring Policy
|
|
741
|
+
|
|
742
|
+
## Monitoring Scope
|
|
743
|
+
${richSection}
|
|
744
|
+
|
|
745
|
+
## Frequency
|
|
746
|
+
${richSection}
|
|
747
|
+
|
|
748
|
+
## Escalation Procedures
|
|
749
|
+
${richSection}
|
|
750
|
+
`;
|
|
751
|
+
const result = validateDocument(validator, content);
|
|
752
|
+
|
|
753
|
+
expect(result.status).toBe('VALID');
|
|
754
|
+
expect(result.sectionFeedback).toBeUndefined();
|
|
755
|
+
expect(result.completenessScore).toBeDefined();
|
|
756
|
+
expect(result.completenessScore!).toBeGreaterThanOrEqual(70);
|
|
757
|
+
});
|
|
758
|
+
|
|
759
|
+
it('does not return semantic fields for EMPTY documents', () => {
|
|
760
|
+
const result = validateDocument(validator, '');
|
|
761
|
+
|
|
762
|
+
expect(result.status).toBe('EMPTY');
|
|
763
|
+
expect(result.sectionFeedback).toBeUndefined();
|
|
764
|
+
expect(result.completenessScore).toBeUndefined();
|
|
765
|
+
});
|
|
766
|
+
|
|
767
|
+
it('detects scaffold documents with placeholders as SHALLOW', () => {
|
|
768
|
+
const scaffoldValidator: DocumentValidator = {
|
|
769
|
+
document: 'fria',
|
|
770
|
+
obligation: 'eu-ai-act-OBL-013',
|
|
771
|
+
article: 'Art. 27',
|
|
772
|
+
file_patterns: ['FRIA.md'],
|
|
773
|
+
required_sections: [
|
|
774
|
+
{ title: 'Risk Assessment', required: true },
|
|
775
|
+
{ title: 'Impact Analysis', required: true },
|
|
776
|
+
{ title: 'Mitigation Measures', required: true },
|
|
777
|
+
],
|
|
778
|
+
};
|
|
779
|
+
|
|
780
|
+
const content = `# Fundamental Rights Impact Assessment
|
|
781
|
+
|
|
782
|
+
## Risk Assessment
|
|
783
|
+
| Right | Risk Level | Description | Group | Mitigation |
|
|
784
|
+
|---|---|---|---|---|
|
|
785
|
+
| Non-discrimination (Art. 21) | [H/M/L/N] | [Description of risk] | [Group] | [Measures] |
|
|
786
|
+
| Privacy (Art. 7-8) | [H/M/L/N] | [Description of risk] | [Group] | [Measures] |
|
|
787
|
+
|
|
788
|
+
## Impact Analysis
|
|
789
|
+
- System name: [Name]
|
|
790
|
+
- Provider: [Company Name]
|
|
791
|
+
- Version: [Number]
|
|
792
|
+
- Intended purpose: [Description of purpose]
|
|
793
|
+
- Deployment context: [Where and how the system is used]
|
|
794
|
+
- Categories of persons affected: [List]
|
|
795
|
+
|
|
796
|
+
## Mitigation Measures
|
|
797
|
+
| # | Measure | Responsible | Timeline | Status |
|
|
798
|
+
|---|---------|------------|----------|--------|
|
|
799
|
+
| 1 | [Measure description] | [Name, Title] | [Date] | [Status] |
|
|
800
|
+
| 2 | [Measure description] | [Name, Title] | [Date] | [Status] |
|
|
801
|
+
`;
|
|
802
|
+
const result = validateDocument(scaffoldValidator, content);
|
|
803
|
+
|
|
804
|
+
expect(result.status).toBe('SHALLOW');
|
|
805
|
+
expect(result.shallowSections).toBeDefined();
|
|
806
|
+
expect(result.shallowSections!.length).toBeGreaterThan(0);
|
|
807
|
+
expect(result.sectionFeedback).toBeDefined();
|
|
808
|
+
expect(result.sectionFeedback!.some(f => f.includes('placeholder'))).toBe(true);
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
it('does not return semantic fields for PARTIAL documents', () => {
|
|
812
|
+
const content = `# Monitoring Policy
|
|
813
|
+
|
|
814
|
+
## Monitoring Scope
|
|
815
|
+
Some content here about monitoring.
|
|
816
|
+
`;
|
|
817
|
+
const result = validateDocument(validator, content);
|
|
818
|
+
|
|
819
|
+
expect(result.status).toBe('PARTIAL');
|
|
820
|
+
expect(result.sectionFeedback).toBeUndefined();
|
|
821
|
+
expect(result.completenessScore).toBeUndefined();
|
|
822
|
+
});
|
|
823
|
+
});
|
|
824
|
+
|
|
825
|
+
// --- AI Review Marker ---
|
|
826
|
+
|
|
827
|
+
describe('hasAiReviewMarker', () => {
|
|
828
|
+
it('detects valid review marker', () => {
|
|
829
|
+
expect(hasAiReviewMarker('some content\n<!-- complior:reviewed 2026-03-26T10:00:00.000Z -->')).toBe(true);
|
|
830
|
+
});
|
|
831
|
+
|
|
832
|
+
it('detects marker with flexible whitespace', () => {
|
|
833
|
+
expect(hasAiReviewMarker('<!-- complior:reviewed 2026-03-26T10:00:00Z -->')).toBe(true);
|
|
834
|
+
});
|
|
835
|
+
|
|
836
|
+
it('returns false when no marker present', () => {
|
|
837
|
+
expect(hasAiReviewMarker('# Just a document\n\nSome content.')).toBe(false);
|
|
838
|
+
});
|
|
839
|
+
|
|
840
|
+
it('returns false for similar but different comments', () => {
|
|
841
|
+
expect(hasAiReviewMarker('<!-- reviewed 2026-03-26T10:00:00Z -->')).toBe(false);
|
|
842
|
+
expect(hasAiReviewMarker('<!-- complior:draft 2026-03-26T10:00:00Z -->')).toBe(false);
|
|
843
|
+
});
|
|
844
|
+
});
|
|
845
|
+
|
|
846
|
+
describe('extractReviewDate', () => {
|
|
847
|
+
it('extracts ISO timestamp from marker', () => {
|
|
848
|
+
expect(extractReviewDate('<!-- complior:reviewed 2026-03-26T10:00:00.000Z -->')).toBe('2026-03-26T10:00:00.000Z');
|
|
849
|
+
});
|
|
850
|
+
|
|
851
|
+
it('returns undefined when no marker', () => {
|
|
852
|
+
expect(extractReviewDate('no marker here')).toBeUndefined();
|
|
853
|
+
});
|
|
854
|
+
});
|
|
855
|
+
|
|
856
|
+
// --- docQuality classification ---
|
|
857
|
+
|
|
858
|
+
describe('validateDocument docQuality', () => {
|
|
859
|
+
const validator: DocumentValidator = {
|
|
860
|
+
document: 'risk-management',
|
|
861
|
+
obligation: 'eu-ai-act-OBL-005',
|
|
862
|
+
article: 'Art. 9',
|
|
863
|
+
file_patterns: ['RISK-MANAGEMENT.md'],
|
|
864
|
+
required_sections: [
|
|
865
|
+
{ title: 'Risk Assessment', required: true },
|
|
866
|
+
{ title: 'Mitigation Measures', required: true },
|
|
867
|
+
],
|
|
868
|
+
};
|
|
869
|
+
|
|
870
|
+
it('returns scaffold for EMPTY document', () => {
|
|
871
|
+
const result = validateDocument(validator, '');
|
|
872
|
+
expect(result.docQuality).toBe('scaffold');
|
|
873
|
+
expect(result.status).toBe('EMPTY');
|
|
874
|
+
});
|
|
875
|
+
|
|
876
|
+
it('returns scaffold for SHALLOW document', () => {
|
|
877
|
+
const content = `# Risk Management
|
|
878
|
+
|
|
879
|
+
## Risk Assessment
|
|
880
|
+
[TODO: Describe risks]
|
|
881
|
+
|
|
882
|
+
## Mitigation Measures
|
|
883
|
+
[TODO: Describe mitigations]
|
|
884
|
+
`;
|
|
885
|
+
const result = validateDocument(validator, content);
|
|
886
|
+
expect(result.docQuality).toBe('scaffold');
|
|
887
|
+
expect(result.status).toBe('SHALLOW');
|
|
888
|
+
});
|
|
889
|
+
|
|
890
|
+
it('returns draft for VALID document without review marker', () => {
|
|
891
|
+
const content = `# Risk Management
|
|
892
|
+
|
|
893
|
+
## Risk Assessment
|
|
894
|
+
The company has conducted a thorough risk assessment covering all aspects of the AI system deployment.
|
|
895
|
+
We identified 12 risk categories including bias, accuracy degradation, and data drift. Each risk is rated
|
|
896
|
+
on a 5-point scale for likelihood and impact per Art. 9 requirements. Reviews are conducted quarterly
|
|
897
|
+
starting from 2026-01-01. The SLA target is 99.9% uptime with KPI tracking for all risk indicators.
|
|
898
|
+
All identified risks must be documented and reviewed within 30 days of detection.
|
|
899
|
+
- Bias monitoring: <2% demographic parity gap
|
|
900
|
+
- Accuracy tracking: >95% precision target
|
|
901
|
+
|
|
902
|
+
## Mitigation Measures
|
|
903
|
+
For each identified risk, specific mitigation controls have been implemented including automated monitoring
|
|
904
|
+
with 99.9% uptime SLA, quarterly bias audits achieving <2% demographic parity gap, and incident response
|
|
905
|
+
procedures with 24-hour resolution targets. All measures documented per ISO 42001 requirements.
|
|
906
|
+
The deployer must ensure continuous compliance with Art. 9 obligations and report within 72 hours.
|
|
907
|
+
- Automated monitoring dashboards with threshold alerts
|
|
908
|
+
- Incident response within 4 hours for critical issues
|
|
909
|
+
`;
|
|
910
|
+
const result = validateDocument(validator, content);
|
|
911
|
+
expect(result.docQuality).toBe('draft');
|
|
912
|
+
expect(result.status).toBe('VALID');
|
|
913
|
+
});
|
|
914
|
+
|
|
915
|
+
it('returns reviewed for document with AI review marker', () => {
|
|
916
|
+
const content = `# Risk Management
|
|
917
|
+
|
|
918
|
+
## Risk Assessment
|
|
919
|
+
The company has conducted a thorough risk assessment covering all aspects of the AI system deployment.
|
|
920
|
+
We identified 12 risk categories including bias, accuracy degradation, and data drift. Each risk is rated
|
|
921
|
+
on a 5-point scale for likelihood and impact per Art. 9 requirements. Annual reviews are conducted.
|
|
922
|
+
|
|
923
|
+
## Mitigation Measures
|
|
924
|
+
For each identified risk, specific mitigation controls have been implemented including automated monitoring
|
|
925
|
+
with 99.9% uptime SLA, quarterly bias audits achieving <2% demographic parity gap, and incident response
|
|
926
|
+
procedures with 24-hour resolution targets. All measures documented per ISO 42001 requirements.
|
|
927
|
+
|
|
928
|
+
<!-- complior:reviewed 2026-03-26T10:00:00.000Z -->
|
|
929
|
+
`;
|
|
930
|
+
const result = validateDocument(validator, content);
|
|
931
|
+
expect(result.docQuality).toBe('reviewed');
|
|
932
|
+
});
|
|
933
|
+
|
|
934
|
+
it('returns scaffold for PARTIAL document (missing sections)', () => {
|
|
935
|
+
const content = `# Risk Management
|
|
936
|
+
|
|
937
|
+
## Risk Assessment
|
|
938
|
+
Some basic content about risk assessment.
|
|
939
|
+
`;
|
|
940
|
+
const result = validateDocument(validator, content);
|
|
941
|
+
expect(result.docQuality).toBe('scaffold');
|
|
942
|
+
expect(result.status).toBe('PARTIAL');
|
|
943
|
+
});
|
|
944
|
+
|
|
945
|
+
it('reviewed overrides SHALLOW status when marker present', () => {
|
|
946
|
+
const content = `# Risk Management
|
|
947
|
+
|
|
948
|
+
## Risk Assessment
|
|
949
|
+
[TODO: Fill in]
|
|
950
|
+
|
|
951
|
+
## Mitigation Measures
|
|
952
|
+
[TODO: Fill in]
|
|
953
|
+
|
|
954
|
+
<!-- complior:reviewed 2026-03-26T10:00:00.000Z -->
|
|
955
|
+
`;
|
|
956
|
+
const result = validateDocument(validator, content);
|
|
957
|
+
expect(result.docQuality).toBe('reviewed');
|
|
958
|
+
});
|
|
959
|
+
});
|
|
960
|
+
|
|
961
|
+
// --- P17: Low-quality VALID → PARTIAL ---
|
|
962
|
+
|
|
963
|
+
describe('P17: quality-based PARTIAL status', () => {
|
|
964
|
+
const validator: DocumentValidator = {
|
|
965
|
+
document: 'monitoring-policy',
|
|
966
|
+
obligation: 'eu-ai-act-OBL-020',
|
|
967
|
+
article: 'Art. 72',
|
|
968
|
+
file_patterns: ['MONITORING-POLICY.md'],
|
|
969
|
+
required_sections: [
|
|
970
|
+
{ title: 'Monitoring Scope', required: true },
|
|
971
|
+
{ title: 'Frequency', required: true },
|
|
972
|
+
{ title: 'Escalation Procedures', required: true },
|
|
973
|
+
],
|
|
974
|
+
};
|
|
975
|
+
|
|
976
|
+
it('marks PARTIAL when completenessScore < 50 despite all headings present', () => {
|
|
977
|
+
// Each section has lists + specifics → qualityScore ~30-40 (not shallow individually)
|
|
978
|
+
// but no numeric metrics, legal refs, action items → avg < 50 → PARTIAL
|
|
979
|
+
const content = `# Monitoring Policy
|
|
980
|
+
|
|
981
|
+
## Monitoring Scope
|
|
982
|
+
This section describes the monitoring scope for the AI system and its main components.
|
|
983
|
+
The monitoring covers all production deployments and ensures broad compliance coverage.
|
|
984
|
+
Regular checks are performed to maintain system integrity across all environments today.
|
|
985
|
+
Assessment was last updated on 2026-01-15 with full scope reviewed for accuracy.
|
|
986
|
+
- Production environment coverage
|
|
987
|
+
- Development environment spot checks
|
|
988
|
+
|
|
989
|
+
## Frequency
|
|
990
|
+
Monitoring is performed regularly according to the defined schedule and requirements.
|
|
991
|
+
The frequency is determined by risk level and applicable compliance requirements here.
|
|
992
|
+
Higher risk systems are monitored more often. Last review completed on 2026-02-01.
|
|
993
|
+
The schedule is documented and tracked consistently for all monitored systems today.
|
|
994
|
+
- Continuous automated monitoring
|
|
995
|
+
- Periodic manual reviews
|
|
996
|
+
|
|
997
|
+
## Escalation Procedures
|
|
998
|
+
When issues are detected the escalation procedures are followed for proper resolution.
|
|
999
|
+
The team is notified promptly and appropriate corrective actions are taken for issues.
|
|
1000
|
+
Escalation paths are clearly defined for all severity levels. Updated on 2026-03-01.
|
|
1001
|
+
All escalation events are documented and tracked for future reference and audit needs.
|
|
1002
|
+
- Level 1: Team lead notification
|
|
1003
|
+
- Level 2: Department head escalation
|
|
1004
|
+
`;
|
|
1005
|
+
const result = validateDocument(validator, content);
|
|
1006
|
+
|
|
1007
|
+
expect(result.status).toBe('PARTIAL');
|
|
1008
|
+
expect(result.completenessScore).toBeDefined();
|
|
1009
|
+
expect(result.completenessScore!).toBeLessThan(50);
|
|
1010
|
+
expect(result.missingSections).toHaveLength(0);
|
|
1011
|
+
});
|
|
1012
|
+
|
|
1013
|
+
it('marks VALID when completenessScore >= 50', () => {
|
|
1014
|
+
const richSection = `As required by Art. 72 of the EU AI Act, the system must achieve 99.9% accuracy.
|
|
1015
|
+
Reviews are conducted quarterly starting from 2026-01-01.
|
|
1016
|
+
The SLA target is within 24 hours response time for all incidents.
|
|
1017
|
+
This comprehensive section documents all compliance requirements and obligations.
|
|
1018
|
+
The risk management framework covers identification, assessment, and mitigation of all known risks.
|
|
1019
|
+
Ongoing monitoring ensures continuous compliance with regulatory requirements.
|
|
1020
|
+
- Monitoring dashboard tracks KPIs
|
|
1021
|
+
- Automated alerts for threshold breaches
|
|
1022
|
+
| Metric | Target | Current |
|
|
1023
|
+
| --- | --- | --- |
|
|
1024
|
+
| Uptime | 99.9% | 99.95% |`;
|
|
1025
|
+
|
|
1026
|
+
const content = `# Monitoring Policy
|
|
1027
|
+
|
|
1028
|
+
## Monitoring Scope
|
|
1029
|
+
${richSection}
|
|
1030
|
+
|
|
1031
|
+
## Frequency
|
|
1032
|
+
${richSection}
|
|
1033
|
+
|
|
1034
|
+
## Escalation Procedures
|
|
1035
|
+
${richSection}
|
|
1036
|
+
`;
|
|
1037
|
+
const result = validateDocument(validator, content);
|
|
1038
|
+
|
|
1039
|
+
expect(result.status).toBe('VALID');
|
|
1040
|
+
expect(result.completenessScore).toBeDefined();
|
|
1041
|
+
expect(result.completenessScore!).toBeGreaterThanOrEqual(50);
|
|
1042
|
+
});
|
|
1043
|
+
|
|
1044
|
+
it('layer2ToCheckResults generates low-severity fail for quality-based PARTIAL', () => {
|
|
1045
|
+
const l2Result = {
|
|
1046
|
+
obligationId: 'eu-ai-act-OBL-020',
|
|
1047
|
+
article: 'Art. 72',
|
|
1048
|
+
document: 'monitoring-policy',
|
|
1049
|
+
status: 'PARTIAL' as const,
|
|
1050
|
+
foundSections: ['Monitoring Scope', 'Frequency', 'Escalation Procedures'],
|
|
1051
|
+
missingSections: [] as string[],
|
|
1052
|
+
totalRequired: 3,
|
|
1053
|
+
matchedRequired: 3,
|
|
1054
|
+
completenessScore: 30,
|
|
1055
|
+
sectionFeedback: ['Section "Monitoring Scope": Add numeric metrics'],
|
|
1056
|
+
docQuality: 'scaffold' as const,
|
|
1057
|
+
};
|
|
1058
|
+
|
|
1059
|
+
const results = layer2ToCheckResults([l2Result]);
|
|
1060
|
+
|
|
1061
|
+
expect(results).toHaveLength(1);
|
|
1062
|
+
expect(results[0].type).toBe('fail');
|
|
1063
|
+
expect(results[0].severity).toBe('low');
|
|
1064
|
+
expect(results[0].message).toContain('content needs enrichment');
|
|
1065
|
+
expect(results[0].message).toContain('quality: 30/100');
|
|
1066
|
+
expect(results[0].fix).toContain('Enrich content');
|
|
1067
|
+
expect(results[0].fix).toContain('Suggestions:');
|
|
1068
|
+
});
|
|
1069
|
+
|
|
1070
|
+
it('includes scaffold removal hint when docQuality is scaffold', () => {
|
|
1071
|
+
const l2Result = {
|
|
1072
|
+
obligationId: 'eu-ai-act-OBL-020',
|
|
1073
|
+
article: 'Art. 72',
|
|
1074
|
+
document: 'monitoring-policy',
|
|
1075
|
+
status: 'PARTIAL' as const,
|
|
1076
|
+
foundSections: ['Monitoring Scope', 'Frequency', 'Escalation Procedures'],
|
|
1077
|
+
missingSections: [] as string[],
|
|
1078
|
+
totalRequired: 3,
|
|
1079
|
+
matchedRequired: 3,
|
|
1080
|
+
completenessScore: 30,
|
|
1081
|
+
sectionFeedback: ['Section "Monitoring Scope": Add numeric metrics'],
|
|
1082
|
+
docQuality: 'scaffold' as const,
|
|
1083
|
+
};
|
|
1084
|
+
|
|
1085
|
+
const results = layer2ToCheckResults([l2Result]);
|
|
1086
|
+
|
|
1087
|
+
expect(results).toHaveLength(1);
|
|
1088
|
+
expect(results[0].fix).toContain('COMPLIOR:SCAFFOLD');
|
|
1089
|
+
expect(results[0].fix).toContain('Remove <!-- COMPLIOR:SCAFFOLD -->');
|
|
1090
|
+
});
|
|
1091
|
+
});
|
|
1092
|
+
|
|
1093
|
+
describe('P5: scaffold VALID → always fail in layer2ToCheckResults', () => {
|
|
1094
|
+
it('scaffold document with all sections fails L2 instead of passing', () => {
|
|
1095
|
+
const l2Result = {
|
|
1096
|
+
obligationId: 'eu-ai-act-OBL-013',
|
|
1097
|
+
article: 'Art. 27',
|
|
1098
|
+
document: 'fria',
|
|
1099
|
+
status: 'VALID' as const,
|
|
1100
|
+
foundSections: ['Risk Assessment', 'Impact Analysis', 'Mitigation Measures'],
|
|
1101
|
+
missingSections: [] as string[],
|
|
1102
|
+
totalRequired: 3,
|
|
1103
|
+
matchedRequired: 3,
|
|
1104
|
+
docQuality: 'scaffold' as const,
|
|
1105
|
+
};
|
|
1106
|
+
|
|
1107
|
+
const results = layer2ToCheckResults([l2Result]);
|
|
1108
|
+
|
|
1109
|
+
expect(results).toHaveLength(1);
|
|
1110
|
+
expect(results[0].type).toBe('fail');
|
|
1111
|
+
expect(results[0].severity).toBe('low');
|
|
1112
|
+
expect(results[0].message).toContain('scaffold/template quality');
|
|
1113
|
+
expect(results[0].fix).toContain('replace [bracketed] placeholders');
|
|
1114
|
+
});
|
|
1115
|
+
|
|
1116
|
+
it('non-scaffold VALID document still passes L2', () => {
|
|
1117
|
+
const l2Result = {
|
|
1118
|
+
obligationId: 'eu-ai-act-OBL-013',
|
|
1119
|
+
article: 'Art. 27',
|
|
1120
|
+
document: 'fria',
|
|
1121
|
+
status: 'VALID' as const,
|
|
1122
|
+
foundSections: ['Risk Assessment', 'Impact Analysis', 'Mitigation Measures'],
|
|
1123
|
+
missingSections: [] as string[],
|
|
1124
|
+
totalRequired: 3,
|
|
1125
|
+
matchedRequired: 3,
|
|
1126
|
+
docQuality: 'draft' as const,
|
|
1127
|
+
};
|
|
1128
|
+
|
|
1129
|
+
const results = layer2ToCheckResults([l2Result]);
|
|
1130
|
+
|
|
1131
|
+
expect(results).toHaveLength(1);
|
|
1132
|
+
expect(results[0].type).toBe('pass');
|
|
1133
|
+
});
|
|
1134
|
+
});
|
|
1135
|
+
|
|
1136
|
+
describe('N1: scaffold marker detection in docQuality', () => {
|
|
1137
|
+
const friaValidator: DocumentValidator = {
|
|
1138
|
+
document: 'fria',
|
|
1139
|
+
obligation: 'eu-ai-act-OBL-013',
|
|
1140
|
+
article: 'Art. 27',
|
|
1141
|
+
file_patterns: ['fria.md'],
|
|
1142
|
+
required_sections: [
|
|
1143
|
+
{ title: 'Risk Assessment', required: true },
|
|
1144
|
+
{ title: 'Impact Analysis', required: true },
|
|
1145
|
+
{ title: 'Mitigation Measures', required: true },
|
|
1146
|
+
],
|
|
1147
|
+
};
|
|
1148
|
+
|
|
1149
|
+
it('returns scaffold quality for VALID document with COMPLIOR:SCAFFOLD marker', () => {
|
|
1150
|
+
const content = `<!-- COMPLIOR:SCAFFOLD -->
|
|
1151
|
+
# FRIA
|
|
1152
|
+
|
|
1153
|
+
## Risk Assessment
|
|
1154
|
+
This section covers the fundamental rights risk assessment process for AI systems under Art. 27.
|
|
1155
|
+
Analysis includes bias evaluation metrics, SLA compliance at 95%, and quarterly reviews per ISO 42001.
|
|
1156
|
+
- Risk identification methodology based on NIST AI RMF
|
|
1157
|
+
- Impact scoring using standardized scales (1-5)
|
|
1158
|
+
- Annual review cycle with 30-day SLA for updates
|
|
1159
|
+
|
|
1160
|
+
## Impact Analysis
|
|
1161
|
+
Comprehensive impact analysis covering all affected stakeholders per Art. 27(3)(c).
|
|
1162
|
+
Charter Art. 21 non-discrimination metrics tracked monthly with 98% fairness threshold.
|
|
1163
|
+
| Right | Risk Level | Mitigation |
|
|
1164
|
+
|-------|-----------|------------|
|
|
1165
|
+
| Non-discrimination | Medium | Bias audit quarterly |
|
|
1166
|
+
- Stakeholder mapping completed 2026-01-15
|
|
1167
|
+
- Impact categories defined per EU AI Act Annex III
|
|
1168
|
+
|
|
1169
|
+
## Mitigation Measures
|
|
1170
|
+
Mitigation strategy implements Art. 14 human oversight requirements.
|
|
1171
|
+
Override mechanism tested monthly, SLA response within 2 hours per incident protocol.
|
|
1172
|
+
- Kill switch tested on 2026-02-01 with 100% success rate
|
|
1173
|
+
- Escalation procedure: L1 → L2 → DPO within 4-hour window
|
|
1174
|
+
- Quarterly review by compliance team, next review 2026-06-01
|
|
1175
|
+
`;
|
|
1176
|
+
const result = validateDocument(friaValidator, content);
|
|
1177
|
+
expect(result.status).toBe('VALID'); // All headings present, quality >= 50
|
|
1178
|
+
expect(result.docQuality).toBe('scaffold'); // But scaffold marker → scaffold quality
|
|
1179
|
+
});
|
|
1180
|
+
|
|
1181
|
+
it('returns draft quality for VALID document without scaffold marker', () => {
|
|
1182
|
+
const content = `# FRIA
|
|
1183
|
+
|
|
1184
|
+
## Risk Assessment
|
|
1185
|
+
This section covers the fundamental rights risk assessment process for AI systems under Art. 27.
|
|
1186
|
+
Analysis includes bias evaluation metrics, SLA compliance at 95%, and quarterly reviews per ISO 42001.
|
|
1187
|
+
- Risk identification methodology based on NIST AI RMF
|
|
1188
|
+
- Impact scoring using standardized scales (1-5)
|
|
1189
|
+
|
|
1190
|
+
## Impact Analysis
|
|
1191
|
+
Comprehensive impact analysis covering all affected stakeholders per Art. 27(3)(c).
|
|
1192
|
+
Charter Art. 21 non-discrimination metrics tracked monthly with 98% fairness threshold.
|
|
1193
|
+
| Right | Risk Level | Mitigation |
|
|
1194
|
+
|-------|-----------|------------|
|
|
1195
|
+
| Non-discrimination | Medium | Bias audit quarterly |
|
|
1196
|
+
|
|
1197
|
+
## Mitigation Measures
|
|
1198
|
+
Mitigation strategy implements Art. 14 human oversight requirements.
|
|
1199
|
+
Override mechanism tested monthly, SLA response within 2 hours per incident protocol.
|
|
1200
|
+
- Kill switch tested on 2026-02-01 with 100% success rate
|
|
1201
|
+
- Escalation procedure documented and reviewed quarterly
|
|
1202
|
+
`;
|
|
1203
|
+
const result = validateDocument(friaValidator, content);
|
|
1204
|
+
expect(result.status).toBe('VALID');
|
|
1205
|
+
expect(result.docQuality).toBe('draft');
|
|
1206
|
+
});
|
|
1207
|
+
});
|