@xenos1996/usat 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +335 -0
- package/USAT.md +596 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +438 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +7 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +72 -0
- package/dist/config.js.map +1 -0
- package/dist/detect/index.d.ts +58 -0
- package/dist/detect/index.d.ts.map +1 -0
- package/dist/detect/index.js +329 -0
- package/dist/detect/index.js.map +1 -0
- package/dist/engine/audit.d.ts +21 -0
- package/dist/engine/audit.d.ts.map +1 -0
- package/dist/engine/audit.js +295 -0
- package/dist/engine/audit.js.map +1 -0
- package/dist/engine/diff.d.ts +24 -0
- package/dist/engine/diff.d.ts.map +1 -0
- package/dist/engine/diff.js +136 -0
- package/dist/engine/diff.js.map +1 -0
- package/dist/engine/evaluate.d.ts +19 -0
- package/dist/engine/evaluate.d.ts.map +1 -0
- package/dist/engine/evaluate.js +377 -0
- package/dist/engine/evaluate.js.map +1 -0
- package/dist/engine/gate.d.ts +19 -0
- package/dist/engine/gate.d.ts.map +1 -0
- package/dist/engine/gate.js +47 -0
- package/dist/engine/gate.js.map +1 -0
- package/dist/engine/loader.d.ts +22 -0
- package/dist/engine/loader.d.ts.map +1 -0
- package/dist/engine/loader.js +360 -0
- package/dist/engine/loader.js.map +1 -0
- package/dist/engine/maturity.d.ts +20 -0
- package/dist/engine/maturity.d.ts.map +1 -0
- package/dist/engine/maturity.js +157 -0
- package/dist/engine/maturity.js.map +1 -0
- package/dist/engine/score.d.ts +19 -0
- package/dist/engine/score.d.ts.map +1 -0
- package/dist/engine/score.js +141 -0
- package/dist/engine/score.js.map +1 -0
- package/dist/engine/sections.d.ts +14 -0
- package/dist/engine/sections.d.ts.map +1 -0
- package/dist/engine/sections.js +50 -0
- package/dist/engine/sections.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -0
- package/dist/index.js.map +1 -0
- package/dist/report/markdown.d.ts +16 -0
- package/dist/report/markdown.d.ts.map +1 -0
- package/dist/report/markdown.js +488 -0
- package/dist/report/markdown.js.map +1 -0
- package/dist/types.d.ts +282 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +16 -0
- package/dist/types.js.map +1 -0
- package/dist/util/glob.d.ts +11 -0
- package/dist/util/glob.d.ts.map +1 -0
- package/dist/util/glob.js +86 -0
- package/dist/util/glob.js.map +1 -0
- package/dist/util/project.d.ts +72 -0
- package/dist/util/project.d.ts.map +1 -0
- package/dist/util/project.js +586 -0
- package/dist/util/project.js.map +1 -0
- package/dist/util/yaml.d.ts +28 -0
- package/dist/util/yaml.d.ts.map +1 -0
- package/dist/util/yaml.js +48 -0
- package/dist/util/yaml.js.map +1 -0
- package/docs/ARCHITECTURE.md +119 -0
- package/docs/README.md +27 -0
- package/docs/adr/0001-record-architecture-decisions.md +35 -0
- package/docs/adr/0002-typescript-engine-over-python.md +49 -0
- package/docs/adr/0003-rules-are-data-not-code.md +50 -0
- package/docs/adr/0004-markdown-only-output.md +41 -0
- package/docs/adr/0005-maturity-dampens-severity.md +56 -0
- package/docs/adr/0006-severity-status-two-axes.md +48 -0
- package/docs/adr/0007-suppressions-are-visible-and-expiring.md +46 -0
- package/docs/adr/0008-report-trailer-and-diff.md +48 -0
- package/docs/adr/0009-fail-closed-on-malformed-input.md +56 -0
- package/docs/adr/0010-detector-design.md +54 -0
- package/docs/adr/0011-coexist-with-deep-scanners.md +52 -0
- package/docs/adr/README.md +32 -0
- package/docs/agent-integration.md +129 -0
- package/docs/ci-integration.md +163 -0
- package/docs/concepts.md +196 -0
- package/docs/configuration.md +105 -0
- package/docs/detectors.md +186 -0
- package/docs/getting-started.md +99 -0
- package/docs/maturity-profiles.md +156 -0
- package/docs/reference/api.md +193 -0
- package/docs/reference/cli.md +412 -0
- package/docs/rule-packs.md +278 -0
- package/docs/standards-mapping.md +171 -0
- package/package.json +96 -0
- package/rules/core/architecture.yaml +190 -0
- package/rules/core/cicd.yaml +212 -0
- package/rules/core/code-quality.yaml +261 -0
- package/rules/core/dependencies.yaml +132 -0
- package/rules/core/documentation.yaml +119 -0
- package/rules/core/future-readiness.yaml +91 -0
- package/rules/core/release.yaml +131 -0
- package/rules/core/repo.yaml +303 -0
- package/rules/core/security.yaml +494 -0
- package/rules/core/supply-chain.yaml +357 -0
- package/rules/core/testing.yaml +260 -0
- package/rules/detectors.yaml +2255 -0
- package/rules/index.yaml +41 -0
- package/rules/profiles/maturity.yaml +135 -0
- package/rules/stacks/ai-era.yaml +193 -0
- package/rules/stacks/api-backend.yaml +121 -0
- package/rules/stacks/cli.yaml +105 -0
- package/rules/stacks/compliance.yaml +157 -0
- package/rules/stacks/containers.yaml +124 -0
- package/rules/stacks/data.yaml +165 -0
- package/rules/stacks/go.yaml +134 -0
- package/rules/stacks/iac.yaml +128 -0
- package/rules/stacks/jvm.yaml +150 -0
- package/rules/stacks/ml-ai.yaml +140 -0
- package/rules/stacks/mobile.yaml +121 -0
- package/rules/stacks/node-typescript.yaml +166 -0
- package/rules/stacks/python.yaml +175 -0
- package/rules/stacks/rust.yaml +129 -0
- package/rules/stacks/solidity.yaml +139 -0
- package/rules/stacks/web-frontend.yaml +159 -0
- package/templates/AGENTS.audit.md +135 -0
- package/templates/AUDIT_REPORT.md +155 -0
package/rules/index.yaml
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# USAT rule pack registry (schema: usat-rules-index-v1)
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# Each entry is a path (relative to this file) or { file, enabled }.
|
|
5
|
+
# Packs self-select via `skip_when` / per-rule `applies_when`, so the list below
|
|
6
|
+
# is just "what exists" — USAT decides "what applies" at audit time.
|
|
7
|
+
# =============================================================================
|
|
8
|
+
|
|
9
|
+
schema: usat-rules-index-v1
|
|
10
|
+
|
|
11
|
+
packs:
|
|
12
|
+
# ---- universal: every project gets these ---------------------------------
|
|
13
|
+
- core/repo.yaml
|
|
14
|
+
- core/security.yaml
|
|
15
|
+
- core/supply-chain.yaml
|
|
16
|
+
- core/architecture.yaml
|
|
17
|
+
- core/code-quality.yaml
|
|
18
|
+
- core/testing.yaml
|
|
19
|
+
- core/cicd.yaml
|
|
20
|
+
- core/release.yaml
|
|
21
|
+
- core/dependencies.yaml
|
|
22
|
+
- core/documentation.yaml
|
|
23
|
+
- core/future-readiness.yaml
|
|
24
|
+
|
|
25
|
+
# ---- conditional: activated by detection ---------------------------------
|
|
26
|
+
- stacks/node-typescript.yaml
|
|
27
|
+
- stacks/python.yaml
|
|
28
|
+
- stacks/go.yaml
|
|
29
|
+
- stacks/rust.yaml
|
|
30
|
+
- stacks/jvm.yaml
|
|
31
|
+
- stacks/web-frontend.yaml
|
|
32
|
+
- stacks/mobile.yaml
|
|
33
|
+
- stacks/containers.yaml
|
|
34
|
+
- stacks/iac.yaml
|
|
35
|
+
- stacks/solidity.yaml
|
|
36
|
+
- stacks/ml-ai.yaml
|
|
37
|
+
- stacks/cli.yaml
|
|
38
|
+
- stacks/data.yaml
|
|
39
|
+
- stacks/api-backend.yaml
|
|
40
|
+
- stacks/compliance.yaml
|
|
41
|
+
- stacks/ai-era.yaml
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# =============================================================================
|
|
2
|
+
# USAT — Maturity profiles (schema: usat-profiles-v1)
|
|
3
|
+
# -----------------------------------------------------------------------------
|
|
4
|
+
# The same repo at two different stages deserves two different reports.
|
|
5
|
+
# A 3-day prototype should not be graded against a production bar, and a
|
|
6
|
+
# production system should not be excused because it "moves fast".
|
|
7
|
+
#
|
|
8
|
+
# `dampen` = how many severity steps a failing rule is reduced by, per rule
|
|
9
|
+
# class. Severity ladder: FUTURE < LOW < MEDIUM < HIGH < CRITICAL.
|
|
10
|
+
#
|
|
11
|
+
# HARD RULE: CRITICAL is never dampened, at any stage. A leaked credential is
|
|
12
|
+
# a leaked credential. Everything else is negotiable with the calendar.
|
|
13
|
+
# =============================================================================
|
|
14
|
+
|
|
15
|
+
schema: usat-profiles-v1
|
|
16
|
+
ladder: [FUTURE, LOW, MEDIUM, HIGH, CRITICAL]
|
|
17
|
+
|
|
18
|
+
profiles:
|
|
19
|
+
prototype:
|
|
20
|
+
label: Prototype / Spike
|
|
21
|
+
summary: >
|
|
22
|
+
Proving an idea. Optimise for learning speed, but never for leaking
|
|
23
|
+
secrets or shipping unrecoverable data loss.
|
|
24
|
+
dampen:
|
|
25
|
+
security: 1
|
|
26
|
+
supply-chain: 1
|
|
27
|
+
correctness: 1
|
|
28
|
+
maintainability: 2
|
|
29
|
+
operations: 2
|
|
30
|
+
performance: 2
|
|
31
|
+
compliance: 2
|
|
32
|
+
documentation: 2
|
|
33
|
+
style: 2
|
|
34
|
+
expected_band: [30, 65]
|
|
35
|
+
focus:
|
|
36
|
+
- Secret hygiene and .gitignore (non-negotiable, costs 5 minutes)
|
|
37
|
+
- A README that says what this is and how to run it
|
|
38
|
+
- One smoke test so the happy path is known-good
|
|
39
|
+
defer:
|
|
40
|
+
- Kubernetes, multi-region, formal ADRs, 80% coverage gates
|
|
41
|
+
|
|
42
|
+
mvp:
|
|
43
|
+
label: MVP / Early product
|
|
44
|
+
summary: >
|
|
45
|
+
Real users, real data. Security and recovery basics must exist;
|
|
46
|
+
process ceremony can wait.
|
|
47
|
+
dampen:
|
|
48
|
+
security: 0
|
|
49
|
+
supply-chain: 1
|
|
50
|
+
correctness: 1
|
|
51
|
+
maintainability: 1
|
|
52
|
+
operations: 1
|
|
53
|
+
performance: 1
|
|
54
|
+
compliance: 1
|
|
55
|
+
documentation: 2
|
|
56
|
+
style: 2
|
|
57
|
+
expected_band: [45, 75]
|
|
58
|
+
focus:
|
|
59
|
+
- Input validation and output encoding on every user-facing path
|
|
60
|
+
- Automated backup + a restore you have actually performed
|
|
61
|
+
- CI that runs lint + tests on every PR
|
|
62
|
+
- Dependency scanning (Dependabot/Renovate) before the graph grows
|
|
63
|
+
defer:
|
|
64
|
+
- Formal threat model, multi-region, chaos engineering, SBOM/VEX
|
|
65
|
+
|
|
66
|
+
beta:
|
|
67
|
+
label: Beta / Growing
|
|
68
|
+
summary: >
|
|
69
|
+
Scaling users and contributors. Close the process gaps before they
|
|
70
|
+
compound.
|
|
71
|
+
dampen:
|
|
72
|
+
security: 0
|
|
73
|
+
supply-chain: 0
|
|
74
|
+
correctness: 0
|
|
75
|
+
maintainability: 1
|
|
76
|
+
operations: 1
|
|
77
|
+
performance: 1
|
|
78
|
+
compliance: 1
|
|
79
|
+
documentation: 1
|
|
80
|
+
style: 1
|
|
81
|
+
expected_band: [60, 85]
|
|
82
|
+
focus:
|
|
83
|
+
- Branch protection + mandatory review on the default branch
|
|
84
|
+
- Structured logging with correlation IDs; real alerting
|
|
85
|
+
- Integration test suite around critical flows
|
|
86
|
+
- Documented rollback procedure
|
|
87
|
+
defer:
|
|
88
|
+
- Full DR site, formal verification, mutation testing
|
|
89
|
+
|
|
90
|
+
production:
|
|
91
|
+
label: Production / GA
|
|
92
|
+
summary: >
|
|
93
|
+
Full bar. Nothing is dampened — this is the profile USAT grades
|
|
94
|
+
against when you pass --profile production.
|
|
95
|
+
dampen:
|
|
96
|
+
security: 0
|
|
97
|
+
supply-chain: 0
|
|
98
|
+
correctness: 0
|
|
99
|
+
maintainability: 0
|
|
100
|
+
operations: 0
|
|
101
|
+
performance: 0
|
|
102
|
+
compliance: 0
|
|
103
|
+
documentation: 0
|
|
104
|
+
style: 0
|
|
105
|
+
expected_band: [75, 95]
|
|
106
|
+
focus:
|
|
107
|
+
- Close every HIGH; timebox a plan for each MEDIUM
|
|
108
|
+
- Provenance/attestation on release artifacts (SLSA)
|
|
109
|
+
- Tested disaster recovery with a real RTO/RPO
|
|
110
|
+
- Observability that answers "is it broken?" in under 5 minutes
|
|
111
|
+
defer: []
|
|
112
|
+
|
|
113
|
+
legacy:
|
|
114
|
+
label: Legacy / Maintenance
|
|
115
|
+
summary: >
|
|
116
|
+
Still running, no longer evolving. Graded on risk containment and
|
|
117
|
+
replacement cost, not on modernisation ambition.
|
|
118
|
+
dampen:
|
|
119
|
+
security: 0
|
|
120
|
+
supply-chain: 0
|
|
121
|
+
correctness: 0
|
|
122
|
+
maintainability: 1
|
|
123
|
+
operations: 0
|
|
124
|
+
performance: 1
|
|
125
|
+
compliance: 0
|
|
126
|
+
documentation: 1
|
|
127
|
+
style: 1
|
|
128
|
+
expected_band: [40, 70]
|
|
129
|
+
focus:
|
|
130
|
+
- Inventory EOL dependencies and pin a replacement date for each
|
|
131
|
+
- Confirm backups restore, and that someone still knows how
|
|
132
|
+
- Keep the lights on: monitoring, alerting, and an owner per service
|
|
133
|
+
- Document the system well enough to hand over or retire
|
|
134
|
+
defer:
|
|
135
|
+
- Rewrites without a strangler plan, framework migrations for their own sake
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
id: stacks/ai-era
|
|
2
|
+
title: AI / LLM-Era Risks
|
|
3
|
+
section: S14
|
|
4
|
+
section_title: AI / LLM-Era Risks
|
|
5
|
+
description: >
|
|
6
|
+
New in USAT. Your application may be deterministic while depending on a
|
|
7
|
+
component that is not. Mapped to the OWASP Top 10 for LLM Applications
|
|
8
|
+
(2026) and the OWASP Top 10 for Agentic AI (ASI, 2026).
|
|
9
|
+
version: '1.0'
|
|
10
|
+
skip_when:
|
|
11
|
+
all:
|
|
12
|
+
- { fact: 'ai:llm-sdk', op: 'absent' }
|
|
13
|
+
- { fact: 'ai:agents', op: 'absent' }
|
|
14
|
+
- { fact: 'ai:mcp', op: 'absent' }
|
|
15
|
+
- { fact: 'ai:rag', op: 'absent' }
|
|
16
|
+
- { fact: 'ai:prompts', op: 'absent' }
|
|
17
|
+
|
|
18
|
+
rules:
|
|
19
|
+
- id: AI-001
|
|
20
|
+
title: Untrusted content cannot override instructions
|
|
21
|
+
section: S14
|
|
22
|
+
section_title: AI / LLM-Era Risks
|
|
23
|
+
severity: CRITICAL
|
|
24
|
+
class: security
|
|
25
|
+
applies_when:
|
|
26
|
+
any:
|
|
27
|
+
- { fact: 'ai:llm-sdk' }
|
|
28
|
+
- { fact: 'ai:rag' }
|
|
29
|
+
check: { kind: manual }
|
|
30
|
+
evidence: 'How retrieved/web/user content is separated from instructions (delimiters, role separation, or a distinct tool channel).'
|
|
31
|
+
why: 'LLM01:2026 Prompt Injection is still the top risk. Any text your model reads — a PDF, a ticket, a web page — is a potential instruction.'
|
|
32
|
+
remediation: 'Separate trusted and untrusted content by role; strip or neutralise instruction-like patterns from retrieved text; never let retrieval feed a privileged tool call directly.'
|
|
33
|
+
references: ['OWASP-LLM01:2026', 'OWASP-ASI01:2026']
|
|
34
|
+
|
|
35
|
+
- id: AI-002
|
|
36
|
+
title: Model output is validated before it is executed
|
|
37
|
+
section: S14
|
|
38
|
+
section_title: AI / LLM-Era Risks
|
|
39
|
+
severity: CRITICAL
|
|
40
|
+
class: security
|
|
41
|
+
applies_when:
|
|
42
|
+
any:
|
|
43
|
+
- { fact: 'ai:llm-sdk' }
|
|
44
|
+
- { fact: 'ai:agents' }
|
|
45
|
+
check:
|
|
46
|
+
kind: grep_wrong
|
|
47
|
+
pattern: '(exec\s*\(\s*(?:completion|response|result|output|llm|content)|eval\s*\(\s*(?:completion|response|result|output)|dangerouslySetInnerHTML\s*=\s*\{\{\s*__html:\s*(?:completion|response|result|output)|subprocess.*(?:completion|response|llm_output))'
|
|
48
|
+
include: ['**/*.{ts,tsx,js,jsx,py,go,rs}']
|
|
49
|
+
exclude: ['**/*.test.*', '**/node_modules/**']
|
|
50
|
+
flags: i
|
|
51
|
+
why: 'LLM10:2026 Improper Output Handling. Treating generated text as code is remote code execution with a random number generator in charge.'
|
|
52
|
+
remediation: 'Parse model output into a constrained schema (JSON mode / tool calling) and validate before any use in a shell, SQL query, or DOM.'
|
|
53
|
+
references: ['OWASP-LLM10:2026', 'OWASP-ASI05:2026', 'CWE-94']
|
|
54
|
+
|
|
55
|
+
- id: AI-003
|
|
56
|
+
title: Agents and tools run with least privilege
|
|
57
|
+
section: S14
|
|
58
|
+
section_title: AI / LLM-Era Risks
|
|
59
|
+
severity: HIGH
|
|
60
|
+
class: security
|
|
61
|
+
applies_when:
|
|
62
|
+
any:
|
|
63
|
+
- { fact: 'ai:agents' }
|
|
64
|
+
- { fact: 'ai:mcp' }
|
|
65
|
+
check: { kind: manual }
|
|
66
|
+
evidence: 'The tool/permission allowlist: what an agent can call, with which credentials, and what is blocked by default.'
|
|
67
|
+
why: 'LLM03:2026 Excessive Agency jumped to #3 for a reason: agents with broad tool access turn prompt injection into arbitrary action.'
|
|
68
|
+
remediation: 'Allowlist tools explicitly; scope credentials per tool; require human approval for irreversible actions; sandbox filesystem and network.'
|
|
69
|
+
references: ['OWASP-LLM03:2026', 'OWASP-ASI02:2026', 'OWASP-ASI03:2026']
|
|
70
|
+
|
|
71
|
+
- id: AI-004
|
|
72
|
+
title: Secrets and PII do not reach model providers unchecked
|
|
73
|
+
section: S14
|
|
74
|
+
section_title: AI / LLM-Era Risks
|
|
75
|
+
severity: HIGH
|
|
76
|
+
class: security
|
|
77
|
+
applies_when: { fact: 'ai:llm-sdk' }
|
|
78
|
+
check: { kind: manual }
|
|
79
|
+
evidence: 'What is sent in the prompt/context: any credentials, customer PII, or regulated data? Is redaction applied first?'
|
|
80
|
+
why: 'LLM02:2026 Sensitive Information Disclosure now explicitly covers traces, embeddings, and telemetry — not just chat text.'
|
|
81
|
+
remediation: 'Redact before sending; use a provider agreement with no-training/no-retention; keep PII out of embeddings.'
|
|
82
|
+
references: ['OWASP-LLM02:2026', 'GDPR-Art.44']
|
|
83
|
+
|
|
84
|
+
- id: AI-005
|
|
85
|
+
title: Inference spend and token usage are bounded
|
|
86
|
+
section: S14
|
|
87
|
+
section_title: AI / LLM-Era Risks
|
|
88
|
+
severity: MEDIUM
|
|
89
|
+
class: performance
|
|
90
|
+
applies_when: { fact: 'ai:llm-sdk' }
|
|
91
|
+
check:
|
|
92
|
+
kind: grep_present
|
|
93
|
+
pattern: '(max_tokens|maxTokens|max_output_tokens|timeout|maxRetries|max_turns|max_steps|budget|abortSignal|stopAfterAttempt)'
|
|
94
|
+
include: ['**/*.{ts,tsx,js,jsx,py,go,rs}']
|
|
95
|
+
exclude: ['**/*.test.*', '**/node_modules/**']
|
|
96
|
+
flags: i
|
|
97
|
+
why: 'LLM06:2026 Unbounded Consumption — denial-of-wallet is a real attack, and runaway agent loops are a real bug.'
|
|
98
|
+
remediation: 'Set max tokens, request timeouts, retry caps, step limits, and a hard per-user spend ceiling with alerting.'
|
|
99
|
+
references: ['OWASP-LLM06:2026', 'OWASP-ASI08:2026']
|
|
100
|
+
|
|
101
|
+
- id: AI-006
|
|
102
|
+
title: System prompts and tool schemas are not exposed
|
|
103
|
+
section: S14
|
|
104
|
+
section_title: AI / LLM-Era Risks
|
|
105
|
+
severity: MEDIUM
|
|
106
|
+
class: security
|
|
107
|
+
applies_when: { fact: 'ai:prompts' }
|
|
108
|
+
check: { kind: manual }
|
|
109
|
+
evidence: 'Whether the system prompt, tool definitions, and policies are retrievable by the end user (directly or via an error/debug path).'
|
|
110
|
+
why: 'LLM08:2026 Hidden Context Exposure broadened from system-prompt leakage: tool schemas and roles enable reconnaissance.'
|
|
111
|
+
remediation: 'Keep prompts server-side; never echo them in errors or debug output; assume they will leak and put no secret in them.'
|
|
112
|
+
references: ['OWASP-LLM08:2026']
|
|
113
|
+
|
|
114
|
+
- id: AI-007
|
|
115
|
+
title: RAG retrieval is access-controlled per tenant
|
|
116
|
+
section: S14
|
|
117
|
+
section_title: AI / LLM-Era Risks
|
|
118
|
+
severity: HIGH
|
|
119
|
+
class: security
|
|
120
|
+
applies_when: { fact: 'ai:rag' }
|
|
121
|
+
check: { kind: manual }
|
|
122
|
+
evidence: 'How documents are scoped at query time — metadata filters, per-tenant indexes, or a post-retrieval permission check.'
|
|
123
|
+
why: "LLM09:2026 Vector and Embedding Weaknesses: similarity search happily returns another tenant's documents."
|
|
124
|
+
remediation: 'Filter by tenant/ACL at query time (not after generation); partition indexes per tenant where the risk is high.'
|
|
125
|
+
references: ['OWASP-LLM09:2026', 'OWASP-ASI06:2026']
|
|
126
|
+
|
|
127
|
+
- id: AI-008
|
|
128
|
+
title: Model and prompt supply chain is pinned
|
|
129
|
+
section: S14
|
|
130
|
+
section_title: AI / LLM-Era Risks
|
|
131
|
+
severity: MEDIUM
|
|
132
|
+
class: supply-chain
|
|
133
|
+
applies_when: { fact: 'ai:llm-sdk' }
|
|
134
|
+
check: { kind: manual }
|
|
135
|
+
evidence: 'Model IDs/versions pinned (not "latest"), prompts versioned in the repo, and third-party agents/MCP servers reviewed.'
|
|
136
|
+
why: 'LLM04:2026 Supply Chain now covers model provenance, adapters, quantisation, and conversion tooling.'
|
|
137
|
+
remediation: 'Pin exact model versions and digests; version prompts in git; treat MCP servers as production dependencies.'
|
|
138
|
+
references: ['OWASP-LLM04:2026', 'OWASP-ASI04:2026', 'SLSA']
|
|
139
|
+
|
|
140
|
+
- id: AI-009
|
|
141
|
+
title: An eval suite exists for model behaviour
|
|
142
|
+
section: S14
|
|
143
|
+
section_title: AI / LLM-Era Risks
|
|
144
|
+
severity: MEDIUM
|
|
145
|
+
class: correctness
|
|
146
|
+
applies_when: { fact: 'ai:llm-sdk' }
|
|
147
|
+
check: { kind: manual }
|
|
148
|
+
evidence: 'A regression suite of prompts with expected properties, run in CI on model or prompt changes.'
|
|
149
|
+
why: 'Without evals, a prompt tweak is a silent behaviour change shipped to production.'
|
|
150
|
+
remediation: 'Start with 20–50 cases covering the critical paths; fail CI when the pass rate drops.'
|
|
151
|
+
references: ['OWASP-LLM07:2026', 'Evals-Best-Practices']
|
|
152
|
+
|
|
153
|
+
- id: AI-010
|
|
154
|
+
title: Consequential actions require human confirmation
|
|
155
|
+
section: S14
|
|
156
|
+
section_title: AI / LLM-Era Risks
|
|
157
|
+
severity: HIGH
|
|
158
|
+
class: security
|
|
159
|
+
applies_when:
|
|
160
|
+
any:
|
|
161
|
+
- { fact: 'ai:agents' }
|
|
162
|
+
- { fact: 'ai:mcp' }
|
|
163
|
+
check: { kind: manual }
|
|
164
|
+
evidence: 'For each irreversible action (payment, deletion, deployment, outbound message): who or what approves it?'
|
|
165
|
+
why: 'ASI09:2026 Human-Agent Trust Exploitation — users approve faster than they read, and agents exploit that by default.'
|
|
166
|
+
remediation: 'Gate irreversible actions behind explicit confirmation showing the concrete effect; keep an audit log of approvals.'
|
|
167
|
+
references: ['OWASP-ASI09:2026', 'NIST-AI-RMF']
|
|
168
|
+
|
|
169
|
+
- id: AI-011
|
|
170
|
+
title: Agent instruction files are scoped and safe
|
|
171
|
+
section: S14
|
|
172
|
+
section_title: AI / LLM-Era Risks
|
|
173
|
+
severity: MEDIUM
|
|
174
|
+
class: security
|
|
175
|
+
applies_when: { fact: 'ai:agents' }
|
|
176
|
+
check: { kind: manual }
|
|
177
|
+
evidence: 'AGENTS.md / CLAUDE.md / .cursorrules contents: do they forbid destructive commands, secret reads, and unreviewed pushes?'
|
|
178
|
+
why: 'These files are executable-ish instructions that commit as easily as code, and agents follow them literally.'
|
|
179
|
+
remediation: 'Add an explicit denylist (no force push, no rm -rf, no reading .env), plus the exact build/test commands to use.'
|
|
180
|
+
references: ['OWASP-ASI04:2026', 'Agent-Skills']
|
|
181
|
+
|
|
182
|
+
- id: AI-012
|
|
183
|
+
title: Model inputs and outputs are logged for audit
|
|
184
|
+
section: S14
|
|
185
|
+
section_title: AI / LLM-Era Risks
|
|
186
|
+
severity: LOW
|
|
187
|
+
class: compliance
|
|
188
|
+
applies_when: { fact: 'maturity:production' }
|
|
189
|
+
check: { kind: manual }
|
|
190
|
+
evidence: 'A record per request: model version, prompt template id, redacted input, output, tokens, latency, and cost.'
|
|
191
|
+
why: 'You cannot investigate a hallucination, an abuse report, or a cost spike without the trace.'
|
|
192
|
+
remediation: 'Log request/response metadata with a trace ID; redact PII; retain per your data policy.'
|
|
193
|
+
references: ['NIST-AI-RMF', 'EU-AI-Act-Art.12', 'OWASP-LLM02:2026']
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
id: stacks/api-backend
|
|
2
|
+
title: Backend / API
|
|
3
|
+
section: S15
|
|
4
|
+
section_title: Platform-Specific
|
|
5
|
+
description: 'Activated for server-side applications and API services.'
|
|
6
|
+
version: '1.0'
|
|
7
|
+
skip_when:
|
|
8
|
+
all:
|
|
9
|
+
- { fact: 'platform:server', op: 'absent' }
|
|
10
|
+
- { fact: 'project:api', op: 'absent' }
|
|
11
|
+
|
|
12
|
+
rules:
|
|
13
|
+
- id: API-001
|
|
14
|
+
title: The API is versioned
|
|
15
|
+
section: S15
|
|
16
|
+
section_title: Platform-Specific
|
|
17
|
+
severity: MEDIUM
|
|
18
|
+
class: maintainability
|
|
19
|
+
applies_when:
|
|
20
|
+
not: { fact: 'maturity:prototype' }
|
|
21
|
+
check:
|
|
22
|
+
kind: grep_present
|
|
23
|
+
pattern: '(/(v[0-9]+|api/v[0-9]+)/|version\s*:\s*[''"]?[0-9]|Accept.*version|setGlobalPrefix)'
|
|
24
|
+
include: ['**/*.{ts,js,py,go,rs,java,kt,rb,yml,yaml,json}']
|
|
25
|
+
exclude: ['**/node_modules/**', 'package.json']
|
|
26
|
+
flags: i
|
|
27
|
+
why: 'Without a version in the path or header, every breaking change is a coordinated deploy across every client.'
|
|
28
|
+
remediation: 'Prefix routes with /v1 (or use a version header); never break an existing version in place.'
|
|
29
|
+
references: ['ASVS-14.1.3', 'API-Design']
|
|
30
|
+
|
|
31
|
+
- id: API-002
|
|
32
|
+
title: Request timeouts are configured
|
|
33
|
+
section: S15
|
|
34
|
+
section_title: Platform-Specific
|
|
35
|
+
severity: HIGH
|
|
36
|
+
class: performance
|
|
37
|
+
check: { kind: manual }
|
|
38
|
+
evidence: 'Server request timeout, upstream client timeouts, and DB statement timeouts — all three, with values.'
|
|
39
|
+
why: 'No timeout means one slow dependency exhausts your entire worker pool.'
|
|
40
|
+
remediation: 'Set a request timeout at the edge, per-upstream client timeouts, and statement_timeout in the database.'
|
|
41
|
+
references: ['CWE-400', 'Release-It']
|
|
42
|
+
|
|
43
|
+
- id: API-003
|
|
44
|
+
title: Retries are bounded with jitter
|
|
45
|
+
section: S15
|
|
46
|
+
section_title: Platform-Specific
|
|
47
|
+
severity: MEDIUM
|
|
48
|
+
class: performance
|
|
49
|
+
check: { kind: manual }
|
|
50
|
+
evidence: 'Retry configuration: max attempts, backoff strategy, and whether retries are applied to non-idempotent calls.'
|
|
51
|
+
why: 'Retrying a POST three times can create three charges. Retrying without jitter synchronises your thundering herd.'
|
|
52
|
+
remediation: 'Exponential backoff with jitter, capped attempts, and retry only on idempotent/transient failures.'
|
|
53
|
+
references: ['AWS-Builders-Library:Timeouts']
|
|
54
|
+
|
|
55
|
+
- id: API-004
|
|
56
|
+
title: Write operations are idempotent
|
|
57
|
+
section: S15
|
|
58
|
+
section_title: Platform-Specific
|
|
59
|
+
severity: MEDIUM
|
|
60
|
+
class: correctness
|
|
61
|
+
check: { kind: manual }
|
|
62
|
+
evidence: 'An idempotency-key mechanism (or natural-key upsert) on POST/PUT endpoints that create resources.'
|
|
63
|
+
why: 'Networks retry. Clients double-click. Only one of those should create two orders.'
|
|
64
|
+
remediation: 'Accept an Idempotency-Key header, store the result, and replay it on duplicate requests.'
|
|
65
|
+
references: ['Stripe-Idempotency', 'API-Design']
|
|
66
|
+
|
|
67
|
+
- id: API-005
|
|
68
|
+
title: Response format is consistent
|
|
69
|
+
section: S15
|
|
70
|
+
section_title: Platform-Specific
|
|
71
|
+
severity: LOW
|
|
72
|
+
class: maintainability
|
|
73
|
+
check: { kind: manual }
|
|
74
|
+
evidence: 'One success envelope, one error envelope (code, message, details, requestId) used everywhere.'
|
|
75
|
+
why: 'Clients should not need per-endpoint parsing logic.'
|
|
76
|
+
remediation: 'Define the envelopes once in middleware and enforce with contract tests.'
|
|
77
|
+
references: ['API-Design', 'RFC-7807']
|
|
78
|
+
|
|
79
|
+
- id: API-006
|
|
80
|
+
title: An OpenAPI/contract spec exists and is current
|
|
81
|
+
section: S15
|
|
82
|
+
section_title: Platform-Specific
|
|
83
|
+
severity: MEDIUM
|
|
84
|
+
class: documentation
|
|
85
|
+
applies_when:
|
|
86
|
+
not: { fact: 'maturity:prototype' }
|
|
87
|
+
check:
|
|
88
|
+
kind: any_file
|
|
89
|
+
patterns: ['**/openapi.{yaml,yml,json}', '**/swagger.{yaml,yml,json}', 'docs/api/**']
|
|
90
|
+
why: 'A spec that is generated and published is documentation. One that is written once is a memory.'
|
|
91
|
+
remediation: 'Generate the spec from code (or code from the spec) in CI and publish it with the release.'
|
|
92
|
+
references: ['OpenAPI', 'NIST-SSDF-PS.1']
|
|
93
|
+
|
|
94
|
+
- id: API-007
|
|
95
|
+
title: GraphQL depth/complexity limits are enforced
|
|
96
|
+
section: S15
|
|
97
|
+
section_title: Platform-Specific
|
|
98
|
+
severity: HIGH
|
|
99
|
+
class: security
|
|
100
|
+
applies_when: { fact: 'api:graphql' }
|
|
101
|
+
check:
|
|
102
|
+
kind: grep_present
|
|
103
|
+
pattern: '(depthLimit|depth-limit|complexity|maxDepth|costAnalysis|graphql-armor|rate-limit)'
|
|
104
|
+
include: ['**/*.{ts,js,py,go,rs}']
|
|
105
|
+
exclude: ['**/node_modules/**']
|
|
106
|
+
flags: i
|
|
107
|
+
why: 'A cyclic query can bring a GraphQL server to its knees with a single request.'
|
|
108
|
+
remediation: 'Add depth and complexity limits; disable introspection in production unless the API is public.'
|
|
109
|
+
references: ['OWASP-API4:2023', 'CWE-770']
|
|
110
|
+
|
|
111
|
+
- id: API-008
|
|
112
|
+
title: Health and readiness endpoints are unauthenticated but minimal
|
|
113
|
+
section: S15
|
|
114
|
+
section_title: Platform-Specific
|
|
115
|
+
severity: LOW
|
|
116
|
+
class: operations
|
|
117
|
+
check: { kind: manual }
|
|
118
|
+
evidence: '/health returns no version strings, dependency names, or configuration details to anonymous callers.'
|
|
119
|
+
why: 'A verbose /health endpoint is a free architecture diagram for an attacker.'
|
|
120
|
+
remediation: 'Return {"status":"ok"}; put diagnostics behind authentication or a separate internal port.'
|
|
121
|
+
references: ['CWE-200', 'K8s-Probes']
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
id: stacks/cli
|
|
2
|
+
title: CLI Tools
|
|
3
|
+
section: S15
|
|
4
|
+
section_title: Platform-Specific
|
|
5
|
+
description: 'Activated when the project exposes a binary or console script entry point.'
|
|
6
|
+
version: '1.0'
|
|
7
|
+
skip_when: { fact: 'project:cli', op: 'absent' }
|
|
8
|
+
|
|
9
|
+
rules:
|
|
10
|
+
- id: CLI-001
|
|
11
|
+
title: --help is implemented and useful
|
|
12
|
+
section: S15
|
|
13
|
+
section_title: Platform-Specific
|
|
14
|
+
severity: MEDIUM
|
|
15
|
+
class: documentation
|
|
16
|
+
check:
|
|
17
|
+
kind: grep_present
|
|
18
|
+
pattern: "(--help|'help'|describe\\(|usage:|commander|yargs|argparse|clap|click|cobra|oclif|cac)"
|
|
19
|
+
include: ['**/*.{ts,js,py,go,rs,java,kt,rb}', 'package.json', 'Cargo.toml', 'pyproject.toml']
|
|
20
|
+
exclude: ['**/node_modules/**', '**/*.test.*']
|
|
21
|
+
flags: i
|
|
22
|
+
why: 'A CLI without --help forces users to read the source, which is the least friendly documentation format.'
|
|
23
|
+
remediation: 'Use an argument parser that generates help automatically; write a one-line description per command.'
|
|
24
|
+
references: ['CLIG']
|
|
25
|
+
|
|
26
|
+
- id: CLI-002
|
|
27
|
+
title: Exit codes are meaningful
|
|
28
|
+
section: S15
|
|
29
|
+
section_title: Platform-Specific
|
|
30
|
+
severity: MEDIUM
|
|
31
|
+
class: correctness
|
|
32
|
+
check:
|
|
33
|
+
kind: grep_present
|
|
34
|
+
# Matches both styles: inline `process.exit(n)`, and the cleaner
|
|
35
|
+
# `process.exitCode = main(argv)` used by single-entry-point CLIs.
|
|
36
|
+
pattern: '(process\.exit(Code)?\s*=|process\.exit\s*\(\s*[0-3]\s*\)|(sys\.)?exit\s*\(\s*[0-3]\s*\)|os\.Exit\s*\(\s*[0-3]\s*\)|ExitCode|SystemExit)'
|
|
37
|
+
include: ['**/*.{ts,js,py,go,rs,java,kt,rb}']
|
|
38
|
+
exclude: ['**/node_modules/**', '**/*.test.*', '**/tests/**']
|
|
39
|
+
why: 'Scripts pipe your CLI into `&&` and `set -e`. Exit 0 on failure corrupts everything downstream.'
|
|
40
|
+
remediation: 'Exit 0 on success, 1 on expected failure, 2 on usage error. Document the codes in --help.'
|
|
41
|
+
references: ['CLIG', 'POSIX-Exit-Codes']
|
|
42
|
+
|
|
43
|
+
- id: CLI-003
|
|
44
|
+
title: Destructive operations support --dry-run
|
|
45
|
+
section: S15
|
|
46
|
+
section_title: Platform-Specific
|
|
47
|
+
severity: HIGH
|
|
48
|
+
class: correctness
|
|
49
|
+
check: { kind: manual }
|
|
50
|
+
evidence: 'For every command that deletes, overwrites, or publishes: a --dry-run flag, or a confirmation prompt.'
|
|
51
|
+
why: 'A CLI that deletes without asking will eventually be run in the wrong directory by someone having a bad day.'
|
|
52
|
+
remediation: 'Add --dry-run that prints the intended actions and exits 0 without performing them.'
|
|
53
|
+
references: ['CLIG']
|
|
54
|
+
|
|
55
|
+
- id: CLI-004
|
|
56
|
+
title: Input from arguments and stdin is validated
|
|
57
|
+
section: S15
|
|
58
|
+
section_title: Platform-Specific
|
|
59
|
+
severity: MEDIUM
|
|
60
|
+
class: security
|
|
61
|
+
check: { kind: manual }
|
|
62
|
+
evidence: 'Path arguments resolved and checked; stdin length bounded; no unchecked interpolation into shell commands.'
|
|
63
|
+
why: 'CLI input is user input. Path traversal and command injection apply here too.'
|
|
64
|
+
remediation: 'Validate and canonicalise paths; cap stdin size; never pass raw input to a shell.'
|
|
65
|
+
references: ['CWE-22', 'CWE-78']
|
|
66
|
+
|
|
67
|
+
- id: CLI-005
|
|
68
|
+
title: Operations are idempotent where expected
|
|
69
|
+
section: S15
|
|
70
|
+
section_title: Platform-Specific
|
|
71
|
+
severity: MEDIUM
|
|
72
|
+
class: correctness
|
|
73
|
+
check: { kind: manual }
|
|
74
|
+
evidence: 'Running the same command twice produces the same end state (or a clear "already done" message).'
|
|
75
|
+
why: 'CLIs get run twice — by retries, by scripts, and by humans who forgot they already ran it.'
|
|
76
|
+
remediation: 'Check before acting; make writes atomic; report no-op as success.'
|
|
77
|
+
references: ['CLIG']
|
|
78
|
+
|
|
79
|
+
- id: CLI-006
|
|
80
|
+
title: Errors go to stderr, output to stdout
|
|
81
|
+
section: S15
|
|
82
|
+
section_title: Platform-Specific
|
|
83
|
+
severity: LOW
|
|
84
|
+
class: correctness
|
|
85
|
+
check: { kind: manual }
|
|
86
|
+
evidence: '`cli > out.txt` captures only machine-readable output; errors remain on the terminal.'
|
|
87
|
+
why: 'Mixing them breaks every pipe and every parser downstream.'
|
|
88
|
+
remediation: 'Log diagnostics to stderr; keep stdout clean for piping (optionally JSON with --json).'
|
|
89
|
+
references: ['CLIG']
|
|
90
|
+
|
|
91
|
+
- id: CLI-007
|
|
92
|
+
title: Machine-readable output mode exists
|
|
93
|
+
section: S15
|
|
94
|
+
section_title: Platform-Specific
|
|
95
|
+
severity: LOW
|
|
96
|
+
class: maintainability
|
|
97
|
+
check:
|
|
98
|
+
kind: grep_present
|
|
99
|
+
pattern: '(--json|--format|--output\s+(json|yaml)|jsonOutput|as_json)'
|
|
100
|
+
include: ['**/*.{ts,js,py,go,rs}', 'README*']
|
|
101
|
+
exclude: ['**/node_modules/**']
|
|
102
|
+
flags: i
|
|
103
|
+
why: 'Agents and scripts are now the primary consumers of most CLIs. Human text is not an API.'
|
|
104
|
+
remediation: 'Add --json with a stable schema; document it.'
|
|
105
|
+
references: ['CLIG', 'Agent-Skills']
|