@xenos1996/usat 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +335 -0
  3. package/USAT.md +596 -0
  4. package/dist/cli.d.ts +3 -0
  5. package/dist/cli.d.ts.map +1 -0
  6. package/dist/cli.js +438 -0
  7. package/dist/cli.js.map +1 -0
  8. package/dist/config.d.ts +7 -0
  9. package/dist/config.d.ts.map +1 -0
  10. package/dist/config.js +72 -0
  11. package/dist/config.js.map +1 -0
  12. package/dist/detect/index.d.ts +58 -0
  13. package/dist/detect/index.d.ts.map +1 -0
  14. package/dist/detect/index.js +329 -0
  15. package/dist/detect/index.js.map +1 -0
  16. package/dist/engine/audit.d.ts +21 -0
  17. package/dist/engine/audit.d.ts.map +1 -0
  18. package/dist/engine/audit.js +295 -0
  19. package/dist/engine/audit.js.map +1 -0
  20. package/dist/engine/diff.d.ts +24 -0
  21. package/dist/engine/diff.d.ts.map +1 -0
  22. package/dist/engine/diff.js +136 -0
  23. package/dist/engine/diff.js.map +1 -0
  24. package/dist/engine/evaluate.d.ts +19 -0
  25. package/dist/engine/evaluate.d.ts.map +1 -0
  26. package/dist/engine/evaluate.js +377 -0
  27. package/dist/engine/evaluate.js.map +1 -0
  28. package/dist/engine/gate.d.ts +19 -0
  29. package/dist/engine/gate.d.ts.map +1 -0
  30. package/dist/engine/gate.js +47 -0
  31. package/dist/engine/gate.js.map +1 -0
  32. package/dist/engine/loader.d.ts +22 -0
  33. package/dist/engine/loader.d.ts.map +1 -0
  34. package/dist/engine/loader.js +360 -0
  35. package/dist/engine/loader.js.map +1 -0
  36. package/dist/engine/maturity.d.ts +20 -0
  37. package/dist/engine/maturity.d.ts.map +1 -0
  38. package/dist/engine/maturity.js +157 -0
  39. package/dist/engine/maturity.js.map +1 -0
  40. package/dist/engine/score.d.ts +19 -0
  41. package/dist/engine/score.d.ts.map +1 -0
  42. package/dist/engine/score.js +141 -0
  43. package/dist/engine/score.js.map +1 -0
  44. package/dist/engine/sections.d.ts +14 -0
  45. package/dist/engine/sections.d.ts.map +1 -0
  46. package/dist/engine/sections.js +50 -0
  47. package/dist/engine/sections.js.map +1 -0
  48. package/dist/index.d.ts +23 -0
  49. package/dist/index.d.ts.map +1 -0
  50. package/dist/index.js +17 -0
  51. package/dist/index.js.map +1 -0
  52. package/dist/report/markdown.d.ts +16 -0
  53. package/dist/report/markdown.d.ts.map +1 -0
  54. package/dist/report/markdown.js +488 -0
  55. package/dist/report/markdown.js.map +1 -0
  56. package/dist/types.d.ts +282 -0
  57. package/dist/types.d.ts.map +1 -0
  58. package/dist/types.js +16 -0
  59. package/dist/types.js.map +1 -0
  60. package/dist/util/glob.d.ts +11 -0
  61. package/dist/util/glob.d.ts.map +1 -0
  62. package/dist/util/glob.js +86 -0
  63. package/dist/util/glob.js.map +1 -0
  64. package/dist/util/project.d.ts +72 -0
  65. package/dist/util/project.d.ts.map +1 -0
  66. package/dist/util/project.js +586 -0
  67. package/dist/util/project.js.map +1 -0
  68. package/dist/util/yaml.d.ts +28 -0
  69. package/dist/util/yaml.d.ts.map +1 -0
  70. package/dist/util/yaml.js +48 -0
  71. package/dist/util/yaml.js.map +1 -0
  72. package/docs/ARCHITECTURE.md +119 -0
  73. package/docs/README.md +27 -0
  74. package/docs/adr/0001-record-architecture-decisions.md +35 -0
  75. package/docs/adr/0002-typescript-engine-over-python.md +49 -0
  76. package/docs/adr/0003-rules-are-data-not-code.md +50 -0
  77. package/docs/adr/0004-markdown-only-output.md +41 -0
  78. package/docs/adr/0005-maturity-dampens-severity.md +56 -0
  79. package/docs/adr/0006-severity-status-two-axes.md +48 -0
  80. package/docs/adr/0007-suppressions-are-visible-and-expiring.md +46 -0
  81. package/docs/adr/0008-report-trailer-and-diff.md +48 -0
  82. package/docs/adr/0009-fail-closed-on-malformed-input.md +56 -0
  83. package/docs/adr/0010-detector-design.md +54 -0
  84. package/docs/adr/0011-coexist-with-deep-scanners.md +52 -0
  85. package/docs/adr/README.md +32 -0
  86. package/docs/agent-integration.md +129 -0
  87. package/docs/ci-integration.md +163 -0
  88. package/docs/concepts.md +196 -0
  89. package/docs/configuration.md +105 -0
  90. package/docs/detectors.md +186 -0
  91. package/docs/getting-started.md +99 -0
  92. package/docs/maturity-profiles.md +156 -0
  93. package/docs/reference/api.md +193 -0
  94. package/docs/reference/cli.md +412 -0
  95. package/docs/rule-packs.md +278 -0
  96. package/docs/standards-mapping.md +171 -0
  97. package/package.json +96 -0
  98. package/rules/core/architecture.yaml +190 -0
  99. package/rules/core/cicd.yaml +212 -0
  100. package/rules/core/code-quality.yaml +261 -0
  101. package/rules/core/dependencies.yaml +132 -0
  102. package/rules/core/documentation.yaml +119 -0
  103. package/rules/core/future-readiness.yaml +91 -0
  104. package/rules/core/release.yaml +131 -0
  105. package/rules/core/repo.yaml +303 -0
  106. package/rules/core/security.yaml +494 -0
  107. package/rules/core/supply-chain.yaml +357 -0
  108. package/rules/core/testing.yaml +260 -0
  109. package/rules/detectors.yaml +2255 -0
  110. package/rules/index.yaml +41 -0
  111. package/rules/profiles/maturity.yaml +135 -0
  112. package/rules/stacks/ai-era.yaml +193 -0
  113. package/rules/stacks/api-backend.yaml +121 -0
  114. package/rules/stacks/cli.yaml +105 -0
  115. package/rules/stacks/compliance.yaml +157 -0
  116. package/rules/stacks/containers.yaml +124 -0
  117. package/rules/stacks/data.yaml +165 -0
  118. package/rules/stacks/go.yaml +134 -0
  119. package/rules/stacks/iac.yaml +128 -0
  120. package/rules/stacks/jvm.yaml +150 -0
  121. package/rules/stacks/ml-ai.yaml +140 -0
  122. package/rules/stacks/mobile.yaml +121 -0
  123. package/rules/stacks/node-typescript.yaml +166 -0
  124. package/rules/stacks/python.yaml +175 -0
  125. package/rules/stacks/rust.yaml +129 -0
  126. package/rules/stacks/solidity.yaml +139 -0
  127. package/rules/stacks/web-frontend.yaml +159 -0
  128. package/templates/AGENTS.audit.md +135 -0
  129. package/templates/AUDIT_REPORT.md +155 -0
@@ -0,0 +1,357 @@
1
+ id: core/supply-chain
2
+ title: Supply Chain & Build Provenance
3
+ section: S3
4
+ section_title: Supply Chain & Build Provenance
5
+ description: >
6
+ Promoted to its own section in USAT. You can write perfect code and still
7
+ ship someone else's backdoor: these controls cover the path from a
8
+ contributor's editor to the artifact your users run.
9
+ Aligned with SLSA v1.2, NIST SSDF (SP 800-218), and the OpenSSF Scorecard.
10
+ version: '1.0'
11
+
12
+ rules:
13
+ - id: SUP-001
14
+ title: A lockfile is committed
15
+ section: S3
16
+ section_title: Supply Chain & Build Provenance
17
+ severity: HIGH
18
+ class: supply-chain
19
+ check:
20
+ kind: any_file
21
+ patterns:
22
+ - 'package-lock.json'
23
+ - 'yarn.lock'
24
+ - 'pnpm-lock.yaml'
25
+ - 'bun.lockb'
26
+ - 'bun.lock'
27
+ - 'poetry.lock'
28
+ - 'uv.lock'
29
+ - 'pdm.lock'
30
+ - 'Pipfile.lock'
31
+ - 'Cargo.lock'
32
+ - 'go.sum'
33
+ - 'composer.lock'
34
+ - 'Gemfile.lock'
35
+ - 'flake.lock'
36
+ why: 'Without a lockfile every install resolves fresh, so your build is not reproducible and transitive hijacks go unnoticed.'
37
+ remediation: 'Commit the lockfile for your package manager and install with the frozen/immutable flag in CI.'
38
+ references: ['SLSA-Source', 'OpenSSF-Scorecard:Pinned-Dependencies', 'NIST-SSDF-PS.3.2']
39
+
40
+ - id: SUP-002
41
+ title: CI installs with a frozen lockfile
42
+ section: S3
43
+ section_title: Supply Chain & Build Provenance
44
+ severity: MEDIUM
45
+ class: supply-chain
46
+ applies_when: { fact: 'has:ci' }
47
+ check:
48
+ kind: grep_present
49
+ pattern: '(npm ci|npm install --frozen|--frozen-lockfile|--immutable|pip install --require-hashes|pipenv sync|cargo build --locked|--locked)'
50
+ include: ['**/*.{yml,yaml,sh,bash,toml,Makefile,mk,justfile,just}']
51
+ exclude: ['**/node_modules/**']
52
+ flags: i
53
+ why: '`npm install` in CI silently upgrades you. A frozen install fails loudly instead.'
54
+ remediation: 'Use `npm ci` (or `--frozen-lockfile` / `--immutable` / `--locked`) everywhere in CI.'
55
+ references: ['NIST-SSDF-PS.3.2', 'SLSA-Build-L1']
56
+
57
+ - id: SUP-003
58
+ title: Dependency versions are pinned, not floating
59
+ section: S3
60
+ section_title: Supply Chain & Build Provenance
61
+ severity: MEDIUM
62
+ class: supply-chain
63
+ check:
64
+ kind: grep_absent
65
+ # NOTE: every range operator must be listed, not just the `0.0` forms —
66
+ # an earlier revision missed `^1.2.3`, `~1.2`, `>=1.0` and `1.x`, so the
67
+ # most common floating ranges passed as "pinned". Exact versions
68
+ # (`"18.3.1"`) still pass: no alternative matches a bare digit run.
69
+ pattern: ':\s*[''"]?(\*|latest|x|~|\^|>=?|>|<=?|<|\d+\.[xX])[''"]?\s*[,}]'
70
+ include:
71
+ ['package.json', 'bower.json', 'composer.json', 'pyproject.toml', 'requirements*.txt']
72
+ exclude: []
73
+ flags: i
74
+ why: '`*` and `latest` mean a compromised publish lands in your build with no code review.'
75
+ remediation: 'Pin exact versions for applications. Ranges are acceptable for published libraries with a lockfile.'
76
+ references: ['OpenSSF-Scorecard:Pinned-Dependencies', 'NIST-SSDF-PS.3.2']
77
+
78
+ - id: SUP-004
79
+ title: Automated dependency updates are configured
80
+ section: S3
81
+ section_title: Supply Chain & Build Provenance
82
+ severity: MEDIUM
83
+ class: supply-chain
84
+ check:
85
+ kind: any_file
86
+ patterns:
87
+ [
88
+ '.github/dependabot.yml',
89
+ '.gitlab/dependabot.yml',
90
+ 'renovate.json',
91
+ 'renovate.json5',
92
+ '.github/renovate.json',
93
+ '.renovaterc',
94
+ '.renovaterc.json',
95
+ ]
96
+ why: 'The only thing worse than updating dependencies is updating them two years late, all at once.'
97
+ remediation: 'Enable Dependabot or Renovate with grouped PRs and a weekly schedule.'
98
+ references: ['OpenSSF-Scorecard:Dependency-Update-Tool', 'NIST-SSDF-PW.4']
99
+
100
+ - id: SUP-005
101
+ title: Dependency vulnerability scanning runs in CI
102
+ section: S3
103
+ section_title: Supply Chain & Build Provenance
104
+ severity: HIGH
105
+ class: supply-chain
106
+ applies_when: { fact: 'has:ci' }
107
+ check:
108
+ kind: grep_present
109
+ pattern: '(npm audit|pnpm audit|yarn audit|pip-audit|osv-scanner|snyk (test|monitor)|trivy (fs|repo)|grype|dependency-check|govulncheck|cargo audit|bundler-audit)'
110
+ include:
111
+ [
112
+ '.github/workflows/*.yml',
113
+ '.github/workflows/*.yaml',
114
+ '.gitlab-ci.yml',
115
+ 'Jenkinsfile',
116
+ '.circleci/config.yml',
117
+ 'azure-pipelines.yml',
118
+ ]
119
+ flags: i
120
+ why: 'Most breaches in modern stacks arrive through a transitive dependency, not your own code.'
121
+ remediation: 'Add a scanner step that fails on HIGH/CRITICAL advisories; wire it to your advisory source (OSV/GHSA).'
122
+ references: ['OpenSSF-Scorecard:Vulnerabilities', 'NIST-SSDF-RV.1.1', 'SLSA']
123
+
124
+ - id: SUP-006
125
+ title: SAST runs in CI
126
+ section: S3
127
+ section_title: Supply Chain & Build Provenance
128
+ severity: MEDIUM
129
+ class: supply-chain
130
+ applies_when: { fact: 'has:ci' }
131
+ check:
132
+ kind: grep_present
133
+ pattern: '(codeql|semgrep|snyk code|sonar|bandit|gosec|brakeman|clippy-sarif|eslint-plugin-security|spotbugs|detekt)'
134
+ include:
135
+ [
136
+ '.github/workflows/*.yml',
137
+ '.github/workflows/*.yaml',
138
+ '.gitlab-ci.yml',
139
+ 'Jenkinsfile',
140
+ '.circleci/config.yml',
141
+ ]
142
+ flags: i
143
+ why: 'Static analysis catches the bug class a reviewer skips at 5pm on a Friday.'
144
+ remediation: 'Add CodeQL or Semgrep with a ruleset tuned to your stack; block on new high-confidence findings only.'
145
+ references: ['OpenSSF-Scorecard:SAST', 'NIST-SSDF-RV.1.1']
146
+
147
+ - id: SUP-007
148
+ title: Secret scanning runs in CI
149
+ section: S3
150
+ section_title: Supply Chain & Build Provenance
151
+ severity: MEDIUM
152
+ class: supply-chain
153
+ applies_when: { fact: 'has:ci' }
154
+ check:
155
+ kind: grep_present
156
+ pattern: '(gitleaks|trufflehog|detect-secrets|secretlint|git-secrets|secret scanning)'
157
+ include:
158
+ [
159
+ '.github/workflows/*.yml',
160
+ '.github/workflows/*.yaml',
161
+ '.gitlab-ci.yml',
162
+ '.pre-commit-config.yaml',
163
+ ]
164
+ flags: i
165
+ why: 'A pre-commit secret scan is the cheapest control in this entire template.'
166
+ remediation: 'Add gitleaks as a pre-commit hook and a CI job on pull_request.'
167
+ references: ['OpenSSF-Scorecard:Binary-Artifacts', 'NIST-SSDF-PO.3']
168
+
169
+ - id: SUP-008
170
+ title: CI workflows avoid script-injection patterns
171
+ section: S3
172
+ section_title: Supply Chain & Build Provenance
173
+ severity: HIGH
174
+ class: supply-chain
175
+ applies_when: { fact: 'ci:github-actions' }
176
+ check:
177
+ kind: grep_absent
178
+ pattern: '\$\{\{\s*github\.event\.(pull_request|issue|comment|discussion|review)\.[^}]*\}\}'
179
+ include: ['.github/workflows/*.yml', '.github/workflows/*.yaml']
180
+ why: 'Interpolating untrusted event data into a `run:` block is arbitrary code execution on your runner — and on your secrets.'
181
+ remediation: 'Pass event data through `env:` and reference it as a shell variable instead of ${{ }} inside run.'
182
+ references: ['OpenSSF-Scorecard:Dangerous-Workflow', 'CWE-94']
183
+
184
+ - id: SUP-009
185
+ title: Workflows declare least-privilege token permissions
186
+ section: S3
187
+ section_title: Supply Chain & Build Provenance
188
+ severity: MEDIUM
189
+ class: supply-chain
190
+ applies_when: { fact: 'ci:github-actions' }
191
+ check:
192
+ kind: grep_present
193
+ # NOTE: a bare `permissions:` line proves nothing — the grants live on
194
+ # the following lines, which a per-line check cannot see. Only an
195
+ # explicit same-line `read-all`, or a scoped `contents:`/`id-token:` /
196
+ # `pull-requests:` grant, counts as evidence. `write-all` and bare
197
+ # `permissions:` yield MISSING (unverifiable), never PASS.
198
+ pattern: '^\s*permissions\s*:\s*read-all\s*$|^\s{2,}(contents|id-token|pull-requests)\s*:\s*(read|write)\s*$'
199
+ include: ['.github/workflows/*.yml', '.github/workflows/*.yaml']
200
+ why: 'A default GITHUB_TOKEN with write-all turns any compromised action into a supply-chain incident.'
201
+ remediation: 'Set `permissions: contents: read` at the top of each workflow and grant write only where needed.'
202
+ references: ['OpenSSF-Scorecard:Token-Permissions', 'SLSA-Build-L2']
203
+ tags: [github-actions]
204
+
205
+ - id: SUP-010
206
+ title: Third-party actions are pinned to a version or SHA
207
+ section: S3
208
+ section_title: Supply Chain & Build Provenance
209
+ severity: MEDIUM
210
+ class: supply-chain
211
+ applies_when: { fact: 'ci:github-actions' }
212
+ check:
213
+ kind: grep_absent
214
+ pattern: 'uses:\s*[^@\s]+@(main|master|latest)\s*$'
215
+ include: ['.github/workflows/*.yml', '.github/workflows/*.yaml']
216
+ flags: i
217
+ why: '`@main` on a third-party action means their next commit runs with your secrets.'
218
+ remediation: 'Pin to a full commit SHA; enable Dependabot for github-actions to keep them current.'
219
+ references: ['OpenSSF-Scorecard:Pinned-Dependencies']
220
+
221
+ - id: SUP-011
222
+ title: No secrets in CI configuration
223
+ section: S3
224
+ section_title: Supply Chain & Build Provenance
225
+ severity: CRITICAL
226
+ class: supply-chain
227
+ applies_when: { fact: 'has:ci' }
228
+ check:
229
+ kind: grep_absent
230
+ pattern: '(password|passwd|token|secret|api[_-]?key|access[_-]?key)\s*[:=]\s*[''"]?[A-Za-z0-9/+_-]{16,}'
231
+ include:
232
+ [
233
+ '.github/workflows/*.yml',
234
+ '.github/workflows/*.yaml',
235
+ '.gitlab-ci.yml',
236
+ 'Jenkinsfile',
237
+ '.circleci/config.yml',
238
+ 'azure-pipelines.yml',
239
+ '.travis.yml',
240
+ ]
241
+ flags: i
242
+ why: 'Values in workflow files are visible to every contributor and preserved in every fork.'
243
+ remediation: 'Move to repository/environment secrets and reference them as ${{ secrets.NAME }}.'
244
+ references: ['CWE-798', 'OpenSSF-Scorecard:Dangerous-Workflow']
245
+
246
+ - id: SUP-012
247
+ title: Default branch is protected
248
+ section: S3
249
+ section_title: Supply Chain & Build Provenance
250
+ severity: HIGH
251
+ class: supply-chain
252
+ applies_when:
253
+ not: { fact: 'maturity:prototype' }
254
+ check: { kind: manual }
255
+ why: >-
256
+ Without protection, anyone with push access can rewrite history, skip review, and ship untested code straight to main.
257
+ evidence: 'Branch protection settings: required review, required status checks, no force-push, no bypass for admins.'
258
+ remediation: 'Require ≥1 review, dismiss stale approvals, require CI to pass, and block force-push on the default branch.'
259
+ references: ['OpenSSF-Scorecard:Branch-Protection', 'SLSA-Source-L2', 'NIST-SSDF-PO.2']
260
+
261
+ - id: SUP-013
262
+ title: Commits are signed or attested
263
+ section: S3
264
+ section_title: Supply Chain & Build Provenance
265
+ severity: LOW
266
+ class: supply-chain
267
+ applies_when:
268
+ any:
269
+ - { fact: 'maturity:production' }
270
+ - { fact: 'maturity:beta' }
271
+ check:
272
+ kind: grep_present
273
+ pattern: '(actions/attest|sigstore|slsa-framework|gpg-sign|commit\.gpgsign|ssh-sign|gh attestation)'
274
+ include: ['.github/workflows/*.yml', '.github/workflows/*.yaml', '.git/config', '.gitconfig']
275
+ flags: i
276
+ why: 'Unsigned history means anyone with push access can forge a commit attributed to someone else.'
277
+ remediation: 'Enable commit signing, then add build provenance attestation to your release workflow.'
278
+ references: ['SLSA-Build-L2', 'OpenSSF-Scorecard:Signed-Releases']
279
+
280
+ - id: SUP-014
281
+ title: An SBOM is generated for releases
282
+ section: S3
283
+ section_title: Supply Chain & Build Provenance
284
+ severity: LOW
285
+ class: supply-chain
286
+ applies_when:
287
+ any:
288
+ - { fact: 'maturity:production' }
289
+ - { fact: 'maturity:beta' }
290
+ check:
291
+ kind: grep_present
292
+ pattern: '(cyclonedx|spdx|sbom|anchore/sbom-action|syft)'
293
+ include:
294
+ [
295
+ '.github/workflows/*.yml',
296
+ '.github/workflows/*.yaml',
297
+ 'Makefile',
298
+ 'justfile',
299
+ 'Taskfile.yml',
300
+ ]
301
+ exclude: ['**/node_modules/**']
302
+ flags: i
303
+ why: 'When the next Log4Shell drops, the first question is "are we affected?" An SBOM is the answer in seconds.'
304
+ remediation: 'Generate a CycloneDX or SPDX SBOM in CI and publish it with the release.'
305
+ references: ['SLSA', 'NIST-SSDF-PS.3.2', 'EU-CRA-Annex-I']
306
+
307
+ - id: SUP-015
308
+ title: Release artifacts carry provenance attestation
309
+ section: S3
310
+ section_title: Supply Chain & Build Provenance
311
+ severity: FUTURE
312
+ class: supply-chain
313
+ applies_when: { fact: 'maturity:production' }
314
+ check: { kind: manual }
315
+ why: >-
316
+ When an artifact is compromised, provenance is the only thing that answers "was this built by us, from this source?"
317
+ evidence: 'Release workflow producing a SLSA/in-toto attestation, verified at deploy time by consumers.'
318
+ remediation: 'Enable GitHub Artifact Attestations (SLSA Build L2 by default) or npm trusted publishing; verify with slsa-verifier.'
319
+ references: ['SLSA-Build-L2', 'SLSA-v1.2']
320
+
321
+ - id: SUP-016
322
+ title: Container images are scanned before publish
323
+ section: S3
324
+ section_title: Supply Chain & Build Provenance
325
+ severity: MEDIUM
326
+ class: supply-chain
327
+ applies_when: { fact: 'has:containers' }
328
+ check:
329
+ kind: grep_present
330
+ pattern: '(trivy|grype|snyk container|docker scan|anchore|cosign)'
331
+ include:
332
+ [
333
+ '.github/workflows/*.yml',
334
+ '.github/workflows/*.yaml',
335
+ '.gitlab-ci.yml',
336
+ 'Dockerfile*',
337
+ 'Makefile',
338
+ 'justfile',
339
+ ]
340
+ flags: i
341
+ why: 'Base images ship CVEs. Scanning at build time is the last cheap moment to catch them.'
342
+ remediation: 'Add trivy/grype to the image pipeline; fail on HIGH/CRITICAL; sign images with cosign.'
343
+ references: ['SLSA-Build-L2', 'NIST-SSDF-RV.1.1']
344
+
345
+ - id: SUP-017
346
+ title: Build pipeline is reproducible from source
347
+ section: S3
348
+ section_title: Supply Chain & Build Provenance
349
+ severity: FUTURE
350
+ class: supply-chain
351
+ depths: [deep]
352
+ check: { kind: manual }
353
+ why: >-
354
+ A build that cannot be reproduced cannot be verified, and an unverifiable build is an unverifiable supply chain.
355
+ evidence: 'A clean-machine build from a fresh clone produces a byte-identical (or digest-stable) artifact.'
356
+ remediation: 'Pin toolchains, vendor dependencies, remove timestamps, and document the build environment.'
357
+ references: ['SLSA-Build-L3', 'Reproducible-Builds']
@@ -0,0 +1,260 @@
1
+ id: core/testing
2
+ title: Testing & Quality Assurance
3
+ section: S7
4
+ section_title: Testing & Quality Assurance
5
+ description: >
6
+ Coverage is a weak proxy for confidence; USAT separates "tests exist",
7
+ "tests run in CI", and "tests actually cover the paths that matter".
8
+ version: '1.0'
9
+
10
+ rules:
11
+ - id: TEST-001
12
+ title: A test suite exists
13
+ section: S7
14
+ section_title: Testing & Quality Assurance
15
+ severity: HIGH
16
+ class: correctness
17
+ check:
18
+ kind: any_file
19
+ patterns:
20
+ - '**/*.test.{ts,tsx,js,jsx,mjs,cjs}'
21
+ - '**/*.spec.{ts,tsx,js,jsx,mjs,cjs}'
22
+ - '**/test_*.py'
23
+ - '**/*_test.py'
24
+ - '**/*_test.go'
25
+ - '**/*_spec.rb'
26
+ - '**/*Test.java'
27
+ - '**/*Test.kt'
28
+ - '**/*Test.php'
29
+ - '**/tests/**/*.rs'
30
+ - '**/__tests__/**'
31
+ why: 'A project with zero tests has no definition of "working" that a machine can check.'
32
+ remediation: 'Start with the happy path of the main flow and every bug you have already fixed once.'
33
+ references: ['OpenSSF-Scorecard:CI-Tests', 'NIST-SSDF-RV.1.1']
34
+
35
+ - id: TEST-002
36
+ title: Tests run in CI on every pull request
37
+ section: S7
38
+ section_title: Testing & Quality Assurance
39
+ severity: HIGH
40
+ class: operations
41
+ applies_when: { fact: 'has:ci' }
42
+ check:
43
+ kind: grep_present
44
+ pattern: '(npm (run )?test|npm (run )?test:ci|vitest run|jest --ci|pytest|go test|cargo test|mvn test|gradle test|rspec|phpunit|dotnet test|make test)'
45
+ include:
46
+ [
47
+ '.github/workflows/*.yml',
48
+ '.github/workflows/*.yaml',
49
+ '.gitlab-ci.yml',
50
+ 'Jenkinsfile',
51
+ '.circleci/config.yml',
52
+ 'azure-pipelines.yml',
53
+ '.buildkite/**',
54
+ ]
55
+ flags: i
56
+ why: 'Tests that only run on a laptop are a memoir, not a control.'
57
+ remediation: 'Run the full suite on pull_request and on push to the default branch; block merge on failure.'
58
+ references: ['OpenSSF-Scorecard:CI-Tests', 'NIST-SSDF-PW.7']
59
+
60
+ - id: TEST-003
61
+ title: There is a meaningful number of tests
62
+ section: S7
63
+ section_title: Testing & Quality Assurance
64
+ severity: MEDIUM
65
+ class: correctness
66
+ applies_when: { fact: 'has:tests' }
67
+ check:
68
+ kind: count_min
69
+ patterns:
70
+ - '**/*.test.{ts,tsx,js,jsx,mjs,cjs}'
71
+ - '**/*.spec.{ts,tsx,js,jsx,mjs,cjs}'
72
+ - '**/test_*.py'
73
+ - '**/*_test.py'
74
+ - '**/*_test.go'
75
+ - '**/*_spec.rb'
76
+ - '**/*Test.java'
77
+ min: 5
78
+ why: 'One test file is a gesture. Five is a habit.'
79
+ remediation: 'Target the paths that carry money or data first: auth, payments, persistence, permissions.'
80
+ references: ['OpenSSF-Scorecard:CI-Tests']
81
+
82
+ - id: TEST-004
83
+ title: Coverage is measured
84
+ section: S7
85
+ section_title: Testing & Quality Assurance
86
+ severity: MEDIUM
87
+ class: correctness
88
+ applies_when: { fact: 'has:tests' }
89
+ check:
90
+ kind: grep_present
91
+ pattern: '(--coverage|--cov\b|coverage[_-]?threshold|coverageThreshold|--fail-under|codecov|coveralls|lcov|c8|nyc|go test -cover|tarpaulin|llvm-cov)'
92
+ include:
93
+ [
94
+ 'package.json',
95
+ 'pyproject.toml',
96
+ 'setup.cfg',
97
+ '.coveragerc',
98
+ 'Makefile',
99
+ 'justfile',
100
+ 'Taskfile.yml',
101
+ '.github/workflows/*.yml',
102
+ '.github/workflows/*.yaml',
103
+ 'vitest.config.*',
104
+ 'jest.config.*',
105
+ ]
106
+ flags: i
107
+ why: 'Unmeasured coverage drifts down silently, one "I''ll add tests later" at a time.'
108
+ remediation: 'Turn on coverage reporting; publish the number on every PR so it is visible when it moves.'
109
+ references: ['NIST-SSDF-RV.1.1']
110
+
111
+ - id: TEST-005
112
+ title: Coverage thresholds are enforced
113
+ section: S7
114
+ section_title: Testing & Quality Assurance
115
+ severity: LOW
116
+ class: correctness
117
+ applies_when:
118
+ any:
119
+ - { fact: 'maturity:production' }
120
+ - { fact: 'maturity:beta' }
121
+ check:
122
+ kind: grep_present
123
+ pattern: '(coverageThreshold|failUnder|--fail-under|min-coverage|coverage\s*\{|\[tool\.coverage\.report\]\s*\n[^\n]*fail_under)'
124
+ include:
125
+ [
126
+ 'package.json',
127
+ 'vitest.config.*',
128
+ 'jest.config.*',
129
+ 'pyproject.toml',
130
+ 'setup.cfg',
131
+ '.coveragerc',
132
+ 'codecov.yml',
133
+ '.codecov.yml',
134
+ 'Cargo.toml',
135
+ ]
136
+ flags: i
137
+ why: "A gate stops the slide. Set it at today's number and ratchet upward."
138
+ remediation: 'Set a floor slightly below current coverage and raise it whenever it is comfortably met.'
139
+ references: ['NIST-SSDF-RV.1.1']
140
+
141
+ - id: TEST-006
142
+ title: Integration tests cover the seams
143
+ section: S7
144
+ section_title: Testing & Quality Assurance
145
+ severity: MEDIUM
146
+ class: correctness
147
+ applies_when:
148
+ any:
149
+ - { fact: 'has:database' }
150
+ - { fact: 'platform:server' }
151
+ - { fact: 'project:api' }
152
+ check: { kind: manual }
153
+ evidence: 'Tests that exercise a real database/queue/HTTP boundary (or a containerised test double) rather than mocking every collaborator.'
154
+ why: 'Unit tests with everything mocked verify your mock, not your integration.'
155
+ remediation: 'Add tests that run against a real Postgres/Redis via testcontainers or a docker-compose test profile.'
156
+ references: ['Testing-Trophy']
157
+
158
+ - id: TEST-007
159
+ title: End-to-end tests cover critical user journeys
160
+ section: S7
161
+ section_title: Testing & Quality Assurance
162
+ severity: MEDIUM
163
+ class: correctness
164
+ applies_when:
165
+ any:
166
+ - { fact: 'platform:web' }
167
+ - { fact: 'platform:mobile' }
168
+ - { fact: 'maturity:production' }
169
+ check:
170
+ kind: any_file
171
+ patterns:
172
+ [
173
+ 'e2e/**',
174
+ 'cypress/**',
175
+ 'playwright.config.*',
176
+ 'tests/e2e/**',
177
+ '**/*.e2e.{ts,js,py}',
178
+ '**/*.cy.{ts,js}',
179
+ '**/spec/features/**',
180
+ ]
181
+ why: 'Unit tests cannot tell you that checkout is broken because a button moved.'
182
+ remediation: 'Three journeys: sign up/log in, the core money path, and the path that would lose data if it failed.'
183
+ references: ['Testing-Trophy']
184
+
185
+ - id: TEST-008
186
+ title: Security-relevant behaviour is tested
187
+ section: S7
188
+ section_title: Testing & Quality Assurance
189
+ severity: MEDIUM
190
+ class: security
191
+ applies_when:
192
+ any:
193
+ - { fact: 'auth:jwt' }
194
+ - { fact: 'auth:session' }
195
+ - { fact: 'auth:oauth' }
196
+ - { fact: 'has:database' }
197
+ check: { kind: manual }
198
+ evidence: "Tests asserting that user A cannot read or modify user B's resources, and that unauthenticated calls are rejected."
199
+ why: 'Authorization regressions are silent: nothing crashes, someone just sees data they should not.'
200
+ remediation: 'Add negative tests for every permission boundary — 401/403 paths and cross-tenant access attempts.'
201
+ references: ['ASVS-4.2.1', 'OWASP-A01:2021']
202
+
203
+ - id: TEST-009
204
+ title: Tests are independent and deterministic
205
+ section: S7
206
+ section_title: Testing & Quality Assurance
207
+ severity: MEDIUM
208
+ class: correctness
209
+ applies_when: { fact: 'has:tests' }
210
+ check: { kind: manual }
211
+ evidence: 'Run the suite in random order and twice in a row. Report failures and any test that depends on wall-clock time or network.'
212
+ why: 'Order-dependent tests are a distributed system with extra steps, and flaky tests train people to ignore red builds.'
213
+ remediation: 'Reset state per test; freeze the clock; seed randomness. Quarantine flaky tests and fix them within a sprint.'
214
+ references: ['Google-Testing-Blog:Flaky']
215
+
216
+ - id: TEST-010
217
+ title: Performance/load tests exist for critical paths
218
+ section: S7
219
+ section_title: Testing & Quality Assurance
220
+ severity: FUTURE
221
+ class: performance
222
+ applies_when:
223
+ any:
224
+ - { fact: 'maturity:production' }
225
+ - { fact: 'maturity:beta' }
226
+ check: { kind: manual }
227
+ evidence: 'A load-test script (k6, Locust, wrk, artillery) and the last recorded p95/p99 for the main endpoint.'
228
+ why: 'You do not know your ceiling until you have hit it.'
229
+ remediation: 'Script the top three endpoints; run them before each release; record the numbers.'
230
+ references: ['ISO-25010']
231
+
232
+ - id: TEST-011
233
+ title: Mutation testing or equivalent
234
+ section: S7
235
+ section_title: Testing & Quality Assurance
236
+ severity: FUTURE
237
+ class: correctness
238
+ depths: [deep]
239
+ check: { kind: manual }
240
+ evidence: 'Astryx/Stryker/mutmut/cargo-mutants score, or a documented decision not to use it.'
241
+ why: 'Coverage tells you which lines ran, not whether the test would have failed if the line were wrong.'
242
+ remediation: 'Run mutation testing on the highest-risk module first.'
243
+ references: ['Mutation-Testing']
244
+
245
+ - id: TEST-012
246
+ title: Test data comes from fixtures or factories
247
+ section: S7
248
+ section_title: Testing & Quality Assurance
249
+ severity: LOW
250
+ class: maintainability
251
+ applies_when: { fact: 'has:tests' }
252
+ check:
253
+ kind: grep_present
254
+ pattern: '(factory|fixture|faker|builder|testcontainers|factories)'
255
+ include: ['**/*.{ts,tsx,js,jsx,py,go,rs,java,kt,rb}']
256
+ exclude: ['**/node_modules/**']
257
+ flags: i
258
+ why: 'Hand-built object literals in 200 tests make every schema change a 200-file migration.'
259
+ remediation: 'Add a factory (fishery, factory_boy, faker) and generate data at the edges of tests.'
260
+ references: ['Testing-Patterns']