@shomra/agent 0.3.17 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +1 -1
- package/README.md +57 -57
- package/package.json +3 -9
- package/shomra.mjs +9 -7168
- package/src/agents/hook-command.mjs +19 -0
- package/src/agents/hook-files.mjs +41 -0
- package/src/agents/installers.mjs +203 -0
- package/src/artifacts/matchers.mjs +59 -0
- package/src/artifacts/report.mjs +50 -0
- package/src/cli/flags.mjs +68 -0
- package/src/cli/help-sections.mjs +309 -0
- package/src/cli/help.mjs +27 -0
- package/src/cli/main.mjs +55 -0
- package/src/cli/registry.mjs +80 -0
- package/src/cli/suggestions.mjs +33 -0
- package/src/commands/add.mjs +149 -0
- package/src/commands/agent-identity.mjs +46 -0
- package/src/commands/check.mjs +194 -0
- package/src/commands/corpus.mjs +126 -0
- package/src/commands/design.mjs +168 -0
- package/src/commands/doctor.mjs +209 -0
- package/src/commands/fix.mjs +115 -0
- package/src/commands/gate.mjs +154 -0
- package/src/commands/git-hooks.mjs +163 -0
- package/src/commands/init.mjs +36 -0
- package/src/commands/install-hook.mjs +51 -0
- package/src/commands/llm-proxy.mjs +153 -0
- package/src/commands/mcp-add.mjs +185 -0
- package/src/commands/mcp.mjs +143 -0
- package/src/commands/memory-scan.mjs +181 -0
- package/src/commands/model-scan.mjs +99 -0
- package/src/commands/models.mjs +145 -0
- package/src/commands/new.mjs +64 -0
- package/src/commands/plan.mjs +87 -0
- package/src/commands/pr.mjs +249 -0
- package/src/commands/protect.mjs +38 -0
- package/src/commands/provenance.mjs +91 -0
- package/src/commands/redteam.mjs +166 -0
- package/src/commands/rules.mjs +220 -0
- package/src/commands/run.mjs +128 -0
- package/src/commands/scan-zip.mjs +118 -0
- package/src/commands/scan.mjs +102 -0
- package/src/commands/secrets.mjs +99 -0
- package/src/commands/status.mjs +50 -0
- package/src/commands/why.mjs +88 -0
- package/src/core/api-client.mjs +66 -0
- package/src/core/api-key.mjs +6 -0
- package/src/core/circuit-breaker.mjs +42 -0
- package/src/core/config.mjs +37 -0
- package/src/core/exit-codes.mjs +9 -0
- package/src/core/json-file.mjs +13 -0
- package/src/core/numbers.mjs +4 -0
- package/src/core/package-root.mjs +10 -0
- package/src/core/terminal.mjs +16 -0
- package/src/core/version.mjs +14 -0
- package/src/core/wire-limits.mjs +53 -0
- package/src/corpus/screening.mjs +127 -0
- package/{ai-usage.mjs → src/detect/ai-usage.mjs} +0 -27
- package/src/detect/code-sast.mjs +2 -0
- package/{design.mjs → src/detect/design.mjs} +17 -106
- package/src/detect/guard-signals.mjs +18 -0
- package/{model-refs.mjs → src/detect/model-refs.mjs} +18 -77
- package/src/detect/sast/chains.mjs +30 -0
- package/src/detect/sast/path-expressions.mjs +76 -0
- package/src/detect/sast/rules-chains.mjs +33 -0
- package/src/detect/sast/rules-config.mjs +51 -0
- package/src/detect/sast/rules-javascript.mjs +109 -0
- package/src/detect/sast/rules-python.mjs +292 -0
- package/src/detect/sast/scanner.mjs +104 -0
- package/src/detect/sast/source-lines.mjs +115 -0
- package/src/detect/sast/taint.mjs +71 -0
- package/src/detect/signals/artifacts.mjs +113 -0
- package/src/detect/signals/autonomy.mjs +55 -0
- package/src/detect/signals/config-markers.mjs +28 -0
- package/src/detect/signals/credential-harvest.mjs +64 -0
- package/src/detect/signals/durable-claims.mjs +73 -0
- package/src/detect/signals/egress.mjs +56 -0
- package/src/detect/signals/execution-hijack.mjs +128 -0
- package/src/detect/signals/gate.mjs +91 -0
- package/src/detect/signals/injection.mjs +55 -0
- package/src/detect/signals/lines.mjs +42 -0
- package/src/detect/signals/masking.mjs +99 -0
- package/src/detect/signals/memory.mjs +357 -0
- package/src/detect/signals/packages.mjs +45 -0
- package/src/detect/signals/propagation.mjs +86 -0
- package/src/detect/signals/prose-context.mjs +82 -0
- package/src/detect/signals/scan.mjs +91 -0
- package/src/detect/signals/secrets.mjs +85 -0
- package/src/detect/signals/sensitive.mjs +9 -0
- package/src/detect/signals/severity.mjs +10 -0
- package/src/detect/signals/shell.mjs +96 -0
- package/src/detect/signals/staged-fetch.mjs +66 -0
- package/src/detect/signals/text-match.mjs +35 -0
- package/src/gate/batch.mjs +157 -0
- package/src/gate/environment.mjs +122 -0
- package/src/gate/repo-policy.mjs +65 -0
- package/src/gate/result.mjs +53 -0
- package/src/gate/sarif.mjs +33 -0
- package/src/gate/sast.mjs +64 -0
- package/src/gate/suppressions.mjs +0 -0
- package/src/guard/classify.mjs +50 -0
- package/src/guard/emit.mjs +51 -0
- package/src/guard/ignore.mjs +24 -0
- package/src/guard/ledger.mjs +112 -0
- package/src/guard/model-load.mjs +50 -0
- package/src/guard/normalize.mjs +77 -0
- package/src/guard/options.mjs +10 -0
- package/src/guard/prompt-guard.mjs +184 -0
- package/src/guard/report.mjs +35 -0
- package/src/guard/result-guard.mjs +140 -0
- package/src/guard/tool-guard.mjs +166 -0
- package/src/inventory/agent-artifacts.mjs +5 -0
- package/src/inventory/agent-posture.mjs +249 -0
- package/src/inventory/artifacts/classify.mjs +27 -0
- package/src/inventory/artifacts/discover.mjs +187 -0
- package/src/inventory/artifacts/file-read.mjs +42 -0
- package/src/inventory/artifacts/hooks.mjs +14 -0
- package/src/inventory/artifacts/limits.mjs +37 -0
- package/src/inventory/artifacts/marketplaces.mjs +45 -0
- package/src/inventory/artifacts/roots.mjs +20 -0
- package/src/inventory/artifacts/walk.mjs +36 -0
- package/src/inventory/discovery/ai-dependencies.mjs +161 -0
- package/src/inventory/discovery/ai-tools.mjs +23 -0
- package/src/inventory/discovery/all.mjs +40 -0
- package/src/inventory/discovery/coding-agents.mjs +77 -0
- package/src/inventory/discovery/fs-read.mjs +36 -0
- package/src/inventory/discovery/local-runtimes.mjs +53 -0
- package/src/inventory/discovery/mcp-clients.mjs +67 -0
- package/src/inventory/discovery/mcp-servers.mjs +78 -0
- package/src/inventory/discovery/model-keys.mjs +97 -0
- package/src/inventory/discovery/platform.mjs +16 -0
- package/src/inventory/discovery/rules-files.mjs +25 -0
- package/src/inventory/discovery/vector-stores.mjs +176 -0
- package/src/inventory/discovery/workspace.mjs +124 -0
- package/src/inventory/discovery.mjs +10 -0
- package/src/mcp/child-process.mjs +50 -0
- package/src/mcp/config-wrapping.mjs +75 -0
- package/src/mcp/connect-gate.mjs +45 -0
- package/src/mcp/hosts.mjs +16 -0
- package/src/mcp/jsonrpc.mjs +48 -0
- package/src/mcp/lookup.mjs +50 -0
- package/src/mcp/screening.mjs +103 -0
- package/src/mcp/server-tools.mjs +97 -0
- package/src/mcp/server.mjs +102 -0
- package/src/mcp/shim.mjs +205 -0
- package/src/models/lookup.mjs +79 -0
- package/src/models/references.mjs +103 -0
- package/src/rules/context.mjs +98 -0
- package/src/rules/generate.mjs +103 -0
- package/src/rules/sections.mjs +145 -0
- package/src/scaffold/agent-project.mjs +185 -0
- package/src/scaffold/artifact-templates.mjs +35 -0
- package/code-sast.mjs +0 -1063
- package/discovery.mjs +0 -977
- package/guard-ledger.mjs +0 -239
- package/guard-signals.mjs +0 -2055
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export const CONFIG_RULES = [
|
|
2
|
+
{
|
|
3
|
+
id: 'json.automodel_usage',
|
|
4
|
+
|
|
5
|
+
title: 'AutoModel bound to repo-shipped code (trust_remote_code)',
|
|
6
|
+
severity: 'MEDIUM',
|
|
7
|
+
category: 'remote-code',
|
|
8
|
+
confidence: 0.8,
|
|
9
|
+
re: /"(AutoModel[A-Za-z]*|AutoConfig)"\s*:\s*"([^"]+)"/,
|
|
10
|
+
sink: (m) => `auto_map.${m[1]}`,
|
|
11
|
+
message: 'config.json maps an Auto* class to code shipped in this repo. Loading with trust_remote_code imports and runs that code before any weights - review the publisher and the referenced module. (A dangerous sink in that module is reported separately at higher severity.)',
|
|
12
|
+
remediation: 'Review the referenced module before loading, pin revision= to a reviewed commit, or use a model with native transformers support.',
|
|
13
|
+
cwe: 'CWE-829',
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
id: 'json.autotokenizer_usage',
|
|
17
|
+
title: 'AutoTokenizer bound to remote code',
|
|
18
|
+
severity: 'HIGH',
|
|
19
|
+
category: 'remote-code',
|
|
20
|
+
confidence: 0.8,
|
|
21
|
+
re: /"(AutoTokenizer|AutoProcessor|AutoFeatureExtractor|AutoImageProcessor)"\s*:\s*"([^"]+)"/,
|
|
22
|
+
sink: (m) => `auto_map.${m[1]}`,
|
|
23
|
+
message: 'config maps a tokenizer/processor class to repo-shipped code, executed under trust_remote_code when the tokenizer loads.',
|
|
24
|
+
remediation: 'Review the referenced tokenizer code before loading; prefer a model whose tokenizer ships with transformers.',
|
|
25
|
+
cwe: 'CWE-829',
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
id: 'json.trust_remote_code',
|
|
29
|
+
title: 'Config declares trust_remote_code',
|
|
30
|
+
severity: 'HIGH',
|
|
31
|
+
category: 'remote-code',
|
|
32
|
+
confidence: 0.85,
|
|
33
|
+
re: /"trust_remote_code"\s*:\s*true/i,
|
|
34
|
+
sink: () => '"trust_remote_code": true',
|
|
35
|
+
message: 'The config pins trust_remote_code on, so any loader that honours it (transformers, sentence-transformers) will import and run the repo\'s custom code without the caller opting in.',
|
|
36
|
+
remediation: 'Remove the trust_remote_code flag from the config and require callers to opt in explicitly against a reviewed, pinned revision.',
|
|
37
|
+
cwe: 'CWE-94',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'json.custom_pipeline',
|
|
41
|
+
title: 'Config binds a custom pipeline to remote code',
|
|
42
|
+
severity: 'HIGH',
|
|
43
|
+
category: 'remote-code',
|
|
44
|
+
confidence: 0.8,
|
|
45
|
+
re: /"custom_pipelines?"\s*:\s*[{"]/,
|
|
46
|
+
sink: (m) => m[0].replace(/\s*:\s*[{"]$/, '').trim(),
|
|
47
|
+
message: 'config.json declares a custom_pipeline, which binds the pipeline loader to code shipped in this repo - executed under trust_remote_code, before any weights, exactly like auto_map.',
|
|
48
|
+
remediation: 'Remove the custom_pipeline entry, or pin revision= to a reviewed commit and read the referenced pipeline code before loading.',
|
|
49
|
+
cwe: 'CWE-829',
|
|
50
|
+
},
|
|
51
|
+
];
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { isNotAModuleLoad, isStaticPathExpr } from './path-expressions.mjs';
|
|
2
|
+
import { callArgText } from './source-lines.mjs';
|
|
3
|
+
|
|
4
|
+
export const JS_RULES = [
|
|
5
|
+
{
|
|
6
|
+
id: 'js.code_exec',
|
|
7
|
+
title: 'Dynamic code execution',
|
|
8
|
+
severity: 'CRITICAL',
|
|
9
|
+
category: 'code-exec',
|
|
10
|
+
confidence: 0.85,
|
|
11
|
+
re: /(?<![.\w])eval\s*\(|\bnew\s+Function\s*\(|\bvm\.(runInContext|runInNewContext|runInThisContext|compileFunction)\s*\(|\bnew\s+vm\.Script\s*\(|\b(setTimeout|setInterval)\s*\(\s*['"`]/,
|
|
12
|
+
sink: (m) => m[0].replace(/\s*\($/, '').replace(/\s*\(\s*['"`]$/, '').trim(),
|
|
13
|
+
source: 'tool input / model output',
|
|
14
|
+
message: 'Runs a string as code via eval / new Function / vm / a string-valued setTimeout|setInterval. In an MCP server or agent tool this turns any attacker-influenced string into host code execution.',
|
|
15
|
+
remediation: 'Never eval strings. Parse structured input explicitly (JSON.parse) and dispatch on a fixed allowlist of handlers.',
|
|
16
|
+
cwe: 'CWE-94',
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
id: 'js.command_exec',
|
|
20
|
+
title: 'Shell / process execution',
|
|
21
|
+
severity: 'CRITICAL',
|
|
22
|
+
category: 'code-exec',
|
|
23
|
+
confidence: 0.8,
|
|
24
|
+
re: /\bchild_process\b|require\(\s*['"]child_process['"]\s*\)|\bfrom\s+['"]child_process['"]|\b(execSync|execFileSync|spawnSync|execFile)\s*\(/,
|
|
25
|
+
sink: (m) => m[0].trim(),
|
|
26
|
+
source: 'tool input / model output',
|
|
27
|
+
message: 'Spawns a shell or child process. If any argument derives from tool input or model output this is command injection / RCE in the agent host.',
|
|
28
|
+
remediation: 'Avoid shelling out. If unavoidable, use execFile with a fixed binary and an argument array (never a shell string), and validate every argument.',
|
|
29
|
+
cwe: 'CWE-78',
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
id: 'js.mcp_client',
|
|
33
|
+
title: 'MCP client integration (untrusted tool-output ingress)',
|
|
34
|
+
|
|
35
|
+
severity: 'LOW',
|
|
36
|
+
category: 'agentic',
|
|
37
|
+
confidence: 0.55,
|
|
38
|
+
|
|
39
|
+
re: /@modelcontextprotocol\/sdk\/client|\b(StdioClientTransport|SSEClientTransport|StreamableHTTPClientTransport|WebSocketClientTransport)\b/,
|
|
40
|
+
sink: (m) => m[0].trim(),
|
|
41
|
+
source: 'MCP server tool output',
|
|
42
|
+
message: 'Acts as an MCP client/host: connects to MCP servers and passes their tool descriptions and results back to a model. Every server it reaches is an untrusted-input ingress - a poisoned tool description or result can hijack the agent (prompt injection / tool poisoning).',
|
|
43
|
+
remediation: 'Pin exactly which MCP servers this client may connect to and run each through governance before trusting it. Treat all server output as untrusted data, never instructions, and screen it (runtime firewall) before it reaches the model.',
|
|
44
|
+
cwe: 'CWE-829',
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
id: 'js.decode_and_run',
|
|
48
|
+
title: 'Encoded payload decode-and-run',
|
|
49
|
+
severity: 'CRITICAL',
|
|
50
|
+
category: 'obfuscation',
|
|
51
|
+
confidence: 0.85,
|
|
52
|
+
re: /(?<![.\w])(eval|Function)\s*\(\s*(atob|unescape|decodeURIComponent|Buffer\.from)\b/,
|
|
53
|
+
sink: (m) => m[0].replace(/\s*$/, '').trim(),
|
|
54
|
+
message: 'Decodes an encoded string and immediately executes it - the packer pattern used to hide malicious code inside an otherwise innocuous-looking tool.',
|
|
55
|
+
remediation: 'Decode the blob offline and inspect it. Remove any decode-and-execute path from shipped tool code.',
|
|
56
|
+
cwe: 'CWE-506',
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
id: 'js.dynamic_require',
|
|
60
|
+
title: 'Dynamic / obfuscated module load',
|
|
61
|
+
severity: 'HIGH',
|
|
62
|
+
category: 'obfuscation',
|
|
63
|
+
confidence: 0.6,
|
|
64
|
+
re: /(?<![.\w])require\s*\(\s*[^'"\s)]|(?<![.\w])import\s*\(\s*[^'"\s)]/,
|
|
65
|
+
|
|
66
|
+
suppress: (m, unitText, ctx) => {
|
|
67
|
+
const arg = callArgText(unitText, m.index);
|
|
68
|
+
return isNotAModuleLoad(m, unitText, arg) || isStaticPathExpr(arg, ctx.pathNs, ctx.constPaths);
|
|
69
|
+
},
|
|
70
|
+
sink: (m) => m[0].trim(),
|
|
71
|
+
message: 'Loads a module chosen at runtime rather than a string literal, often to conceal which dangerous module is imported.',
|
|
72
|
+
remediation: 'Import modules by string literal so the dependency is statically reviewable; remove runtime-computed requires.',
|
|
73
|
+
cwe: 'CWE-829',
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
id: 'js.network_egress',
|
|
77
|
+
title: 'Network egress',
|
|
78
|
+
|
|
79
|
+
severity: 'MEDIUM',
|
|
80
|
+
category: 'egress',
|
|
81
|
+
confidence: 0.5,
|
|
82
|
+
re: /\baxios\s*\.\s*(get|post|put|request)\s*\(|\bhttps?\.request\s*\(|\bnet\.(connect|createConnection)\s*\(|\bnew\s+WebSocket\s*\(|require\(\s*['"](node-fetch|got|undici|axios)['"]/,
|
|
83
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
84
|
+
source: 'network',
|
|
85
|
+
message: 'Opens an outbound network connection. Normal in application code; chained with a reads-secrets or remote-code-load finding this is the exfiltration / second-stage-download shape.',
|
|
86
|
+
remediation: 'Confirm the destination is expected and necessary; agent tools should not phone out to arbitrary hosts.',
|
|
87
|
+
cwe: 'CWE-913',
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
id: 'js.hardcoded_ai_key',
|
|
91
|
+
title: 'Hardcoded AI-provider API key',
|
|
92
|
+
severity: 'MEDIUM',
|
|
93
|
+
category: 'secret',
|
|
94
|
+
confidence: 0.85,
|
|
95
|
+
re: /['"](sk-ant-[A-Za-z0-9_-]{20,}|sk-[A-Za-z0-9]{20,}|hf_[A-Za-z0-9]{20,}|AIza[A-Za-z0-9_-]{20,}|gsk_[A-Za-z0-9]{20,})['"]/,
|
|
96
|
+
sink: (m) => m[1].slice(0, 12) + '…',
|
|
97
|
+
source: 'source literal',
|
|
98
|
+
message: 'An AI-provider API key is hardcoded as a string literal. Anyone with read access to this repo can drain the account; committed keys are scraped within minutes.',
|
|
99
|
+
remediation: 'Remove the literal and load the key from an environment variable / secret manager at runtime. Rotate the exposed key immediately.',
|
|
100
|
+
cwe: 'CWE-798',
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
|
|
104
|
+
export const JS_TAINT = {
|
|
105
|
+
lang: 'js',
|
|
106
|
+
ruleId: 'js.llm_output_to_sink',
|
|
107
|
+
aiCall: /\.(generate|invoke|run|complete|stream|predict|call)\s*\(|\.chat\.completions\.create\s*\(|\.messages\.create\s*\(|\.create(Chat)?Completion\s*\(/,
|
|
108
|
+
execSink: /(?<![.\w])eval\s*\(|\bnew\s+Function\s*\(|\b(execSync|execFileSync|spawnSync|execFile|exec|spawn)\s*\(|\bvm\.\w+\s*\(/g,
|
|
109
|
+
};
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
export const PY_RULES = [
|
|
2
|
+
{
|
|
3
|
+
id: 'python.dangerous_sinks',
|
|
4
|
+
title: 'Dangerous code-execution sink',
|
|
5
|
+
severity: 'CRITICAL',
|
|
6
|
+
category: 'code-exec',
|
|
7
|
+
confidence: 0.85,
|
|
8
|
+
re: /(?<![.\w])(eval|exec|compile)\s*\(|\bos\.(system|popen|exec[lv]?[pe]*)\s*\(|\bsubprocess\.(run|call|check_output|check_call|Popen)\s*\(|(?<![.\w])__import__\s*\(|(?<![.\w])getattr\s*\(\s*__builtins__/,
|
|
9
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
10
|
+
source: 'model load / forward()',
|
|
11
|
+
message: 'Model code invokes an arbitrary code-execution primitive. Under trust_remote_code this runs in the host process the moment the model is imported.',
|
|
12
|
+
remediation: 'Remove the eval/exec/os.system/subprocess call. Load this model only after reviewing the pinned revision; never with trust_remote_code=True from an untrusted publisher.',
|
|
13
|
+
cwe: 'CWE-94',
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
id: 'python.pickle_deserialization',
|
|
17
|
+
title: 'Unsafe deserialization',
|
|
18
|
+
severity: 'CRITICAL',
|
|
19
|
+
category: 'deserialization',
|
|
20
|
+
confidence: 0.9,
|
|
21
|
+
|
|
22
|
+
re: /\b(pickle|cpickle|dill|_pickle|cloudpickle)\.(loads?|Unpickler)\s*\(|\bjsonpickle\.(decode|loads)\s*\(|\bmarshal\.loads?\s*\(|\btorch\.(load|jit\.load)\s*\(|\byaml\.(unsafe_load|load\s*\((?![^)]*Loader\s*=\s*yaml\.(Safe|Full)Loader))|\bjoblib\.load\s*\(|\bskops\.io\.load\s*\(|\bshelve\.open\s*\(|\bnumpy\.load\s*\([^)]*allow_pickle\s*=\s*True|\b(pandas|pd)\.read_pickle\s*\(|\bmlflow\.[\w.]+\.load_model\s*\(/,
|
|
23
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
24
|
+
source: 'weight / config file',
|
|
25
|
+
message: 'Deserializes data with a pickle-backed loader. A crafted file runs arbitrary code via __reduce__ on load - the primary model-hub malware vector.',
|
|
26
|
+
remediation: 'Load weights from safetensors (use_safetensors=True). For YAML use yaml.safe_load; for numpy set allow_pickle=False; avoid torch.load / cloudpickle / pandas.read_pickle / mlflow.load_model on untrusted files.',
|
|
27
|
+
cwe: 'CWE-502',
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
id: 'python.trust_remote_code',
|
|
31
|
+
title: 'Model loaded with trust_remote_code',
|
|
32
|
+
severity: 'CRITICAL',
|
|
33
|
+
category: 'remote-code',
|
|
34
|
+
confidence: 0.95,
|
|
35
|
+
|
|
36
|
+
codeOnly: true,
|
|
37
|
+
re: /trust_remote_code\s*=\s*True/,
|
|
38
|
+
sink: () => 'trust_remote_code=True',
|
|
39
|
+
source: 'model repository',
|
|
40
|
+
message: 'Loads a model/tokenizer/embedder with trust_remote_code=True, which imports and runs code shipped in the model repo inside the host process before any weights load - an instant RCE if the publisher, or a later silent revision, is malicious.',
|
|
41
|
+
remediation: 'Remove trust_remote_code=True. Prefer a model with native transformers support, or pin revision= to a specific reviewed commit hash and read the custom modeling code first.',
|
|
42
|
+
cwe: 'CWE-94',
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
id: 'python.dataset_remote_code',
|
|
46
|
+
title: 'Dataset loaded with trust_remote_code',
|
|
47
|
+
severity: 'CRITICAL',
|
|
48
|
+
category: 'remote-code',
|
|
49
|
+
confidence: 0.9,
|
|
50
|
+
codeOnly: true,
|
|
51
|
+
|
|
52
|
+
re: /\bload_dataset\s*\((?=[^)]*trust_remote_code\s*=\s*True)/,
|
|
53
|
+
sink: () => 'load_dataset(trust_remote_code=True)',
|
|
54
|
+
source: 'dataset repository',
|
|
55
|
+
message: 'Loads a Hugging Face dataset with trust_remote_code=True, which imports and runs the dataset\'s Python loading script in the host process - arbitrary code execution from the dataset publisher.',
|
|
56
|
+
remediation: 'Remove trust_remote_code=True. Use a non-script dataset format (Parquet/Arrow/CSV/JSON), or pin revision= to a reviewed commit and read the loading script first.',
|
|
57
|
+
cwe: 'CWE-94',
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
id: 'python.native_code_load',
|
|
61
|
+
title: 'Loads a native library (ctypes)',
|
|
62
|
+
severity: 'HIGH',
|
|
63
|
+
category: 'code-exec',
|
|
64
|
+
confidence: 0.85,
|
|
65
|
+
re: /\bctypes\.(CDLL|WinDLL|OleDLL|PyDLL|cdll|windll|oledll)\b|\bcdll\.LoadLibrary\s*\(|\bwindll\.LoadLibrary\s*\(/,
|
|
66
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
67
|
+
source: 'native library',
|
|
68
|
+
message: 'Loads a native/shared library via ctypes. Model code has no reason to dlopen libc or an arbitrary .so/.dll - it is an execution primitive (e.g. ctypes.CDLL("libc.so.6").system(cmd)).',
|
|
69
|
+
remediation: 'Remove the ctypes native-library load from model/loader code. Treat any model that dlopens libraries on load as hostile.',
|
|
70
|
+
cwe: 'CWE-94',
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
id: 'python.dynamic_file_exec',
|
|
74
|
+
title: 'Loads and executes a Python file at runtime',
|
|
75
|
+
severity: 'HIGH',
|
|
76
|
+
category: 'code-exec',
|
|
77
|
+
confidence: 0.85,
|
|
78
|
+
re: /\bSourceFileLoader\s*\(|\bspec_from_file_location\s*\(|\bimp\.load_source\s*\(|\bexec_module\s*\(/,
|
|
79
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
80
|
+
source: 'file path',
|
|
81
|
+
message: 'Imports and executes a Python file from a path at runtime (SourceFileLoader / spec_from_file_location / imp.load_source). This runs code that is not statically visible as an import - a common way to hide an execution path.',
|
|
82
|
+
remediation: 'Remove runtime file-based module loading from model code. Import only reviewed, statically-visible modules.',
|
|
83
|
+
cwe: 'CWE-94',
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
id: 'python.torch_remote_code',
|
|
87
|
+
title: 'Loads/executes remote or native code via torch',
|
|
88
|
+
severity: 'CRITICAL',
|
|
89
|
+
category: 'remote-code',
|
|
90
|
+
confidence: 0.9,
|
|
91
|
+
re: /\btorch\.hub\.load\s*\(|\btorch\.hub\.load_state_dict_from_url\s*\(|\btorch\.package\.PackageImporter\s*\(|\btorch\.classes\.load_library\s*\(/,
|
|
92
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
93
|
+
source: 'remote repo / packaged code',
|
|
94
|
+
message: 'Fetches and executes code that is not in this repository (torch.hub.load runs a remote hubconf.py; load_state_dict_from_url pulls a pickle; PackageImporter / load_library run packaged or native code) - an instant RCE at load time.',
|
|
95
|
+
remediation: 'Do not torch.hub.load untrusted repos (and never with trust_repo=True on an unreviewed source). Load a local, reviewed safetensors checkpoint instead.',
|
|
96
|
+
cwe: 'CWE-494',
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
id: 'python.weights_only_false',
|
|
100
|
+
title: 'torch.load with weights_only=False',
|
|
101
|
+
severity: 'CRITICAL',
|
|
102
|
+
category: 'deserialization',
|
|
103
|
+
confidence: 0.95,
|
|
104
|
+
re: /\btorch\.load\s*\([^)]*weights_only\s*=\s*False/,
|
|
105
|
+
sink: () => 'torch.load(..., weights_only=False)',
|
|
106
|
+
source: 'weight file',
|
|
107
|
+
message: 'torch.load is called with weights_only=False, which turns the safe (default since torch 2.6) tensor-only loader back into the full pickle unpickler - a crafted checkpoint then runs arbitrary code via __reduce__ on load.',
|
|
108
|
+
remediation: 'Remove weights_only=False (let it default to True), or load from safetensors. Only ever disable it for a checkpoint you built yourself.',
|
|
109
|
+
cwe: 'CWE-502',
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
id: 'python.keras_unsafe_load',
|
|
113
|
+
title: 'Keras load with safe_mode disabled',
|
|
114
|
+
severity: 'HIGH',
|
|
115
|
+
category: 'deserialization',
|
|
116
|
+
confidence: 0.85,
|
|
117
|
+
re: /\bsafe_mode\s*=\s*False/,
|
|
118
|
+
sink: () => 'safe_mode=False',
|
|
119
|
+
source: 'model file',
|
|
120
|
+
message: 'A Keras/TensorFlow model is loaded with safe_mode=False, which allows deserialization of Lambda layers - arbitrary Python bytecode that executes the moment the model loads (CVE-2024-3660 / CVE-2025-1550 class).',
|
|
121
|
+
remediation: 'Remove safe_mode=False. Load only models you trust; a Lambda layer in an untrusted model is remote code execution regardless of format (.h5 ignores safe_mode entirely).',
|
|
122
|
+
cwe: 'CWE-502',
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
id: 'python.langchain_code_exec',
|
|
126
|
+
title: 'LLM-driven code-execution component',
|
|
127
|
+
severity: 'HIGH',
|
|
128
|
+
category: 'agentic',
|
|
129
|
+
confidence: 0.85,
|
|
130
|
+
|
|
131
|
+
re: /\b(PythonREPL|PythonREPLTool|PythonAstREPLTool|PALChain|CPALChain|LLMMathChain|create_pandas_dataframe_agent|create_spark_dataframe_agent|create_csv_agent|PandasQueryEngine|PandasInstructionParser|CodeInterpreterToolSpec|CodeAgent|LocalPythonExecutor|local_python_executor|PythonInterpreterTool)\b|\bload_tools\s*\([^)]*['"](python_repl|terminal|shell|bash)/,
|
|
132
|
+
sink: (m) => m[0].trim(),
|
|
133
|
+
source: 'LLM output',
|
|
134
|
+
message: 'Uses an agent-framework component that executes LLM-generated code (exec/eval on model output). If the model can be steered (prompt injection), this is remote code execution in the agent host (CVE-2023-29374 / CVE-2024-4181 class).',
|
|
135
|
+
remediation: 'Avoid code-executing chains/tools on untrusted input. If unavoidable, run them in a locked-down sandbox with no host/network access and a strict output validator.',
|
|
136
|
+
cwe: 'CWE-94',
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
id: 'python.autogen_local_exec',
|
|
140
|
+
title: 'Agent executes LLM-written code locally',
|
|
141
|
+
severity: 'HIGH',
|
|
142
|
+
category: 'agentic',
|
|
143
|
+
confidence: 0.75,
|
|
144
|
+
|
|
145
|
+
re: /code_execution_config\s*=\s*\{|use_docker\s*=\s*False/,
|
|
146
|
+
sink: (m) => m[0].replace(/\s*=\s*\{$/, '').trim(),
|
|
147
|
+
source: 'LLM output',
|
|
148
|
+
message: 'An AutoGen-style agent is configured to execute LLM-written code on the host (a dict code_execution_config / use_docker=False). Any prompt-injected instruction the model follows becomes code execution in the agent process.',
|
|
149
|
+
remediation: 'Set code_execution_config=False, or require use_docker=True (an isolated container) with no host mounts and a locked-down image. Never run model-authored code directly on the host.',
|
|
150
|
+
cwe: 'CWE-94',
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
id: 'python.langchain_serialized_load',
|
|
154
|
+
title: 'Loads a serialized chain / prompt / agent',
|
|
155
|
+
severity: 'HIGH',
|
|
156
|
+
category: 'deserialization',
|
|
157
|
+
confidence: 0.75,
|
|
158
|
+
|
|
159
|
+
re: /\b(load_chain|load_prompt|load_agent)\s*\(|\bhub\.pull\s*\(/,
|
|
160
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
161
|
+
source: 'serialized chain / hub',
|
|
162
|
+
message: 'Deserializes a LangChain chain/prompt/agent from a file or the hub. The serialized config can name arbitrary classes to construct - a poisoned artifact is code execution when the object is built.',
|
|
163
|
+
remediation: 'Build chains in code from reviewed source, not from an untrusted serialized artifact; if you must load one, pin and review it and never load from a user-supplied path/URL.',
|
|
164
|
+
cwe: 'CWE-502',
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
id: 'python.rag_unsafe_deser',
|
|
168
|
+
title: 'Unsafe vector-store / RAG deserialization',
|
|
169
|
+
severity: 'CRITICAL',
|
|
170
|
+
category: 'deserialization',
|
|
171
|
+
confidence: 0.9,
|
|
172
|
+
re: /allow_dangerous_deserialization\s*=\s*True|\bFAISS\.load_local\s*\(|\b(pickle|joblib)\.load\s*\([^)]*(index|faiss|embedding|vector|chroma)/i,
|
|
173
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
174
|
+
source: 'vector store / embedding index',
|
|
175
|
+
message: 'Loads a RAG vector store or embedding index through a pickle-backed path (allow_dangerous_deserialization / FAISS.load_local / a pickled index). A poisoned index file executes arbitrary code the moment it is loaded - the embedding-store supply-chain vector.',
|
|
176
|
+
remediation: 'Never set allow_dangerous_deserialization=True on an index you did not build. Rebuild the vector store from source documents in your own environment, or use a non-pickle store format.',
|
|
177
|
+
cwe: 'CWE-502',
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
id: 'python.mcp_client',
|
|
181
|
+
title: 'MCP client integration (untrusted tool-output ingress)',
|
|
182
|
+
|
|
183
|
+
severity: 'LOW',
|
|
184
|
+
category: 'agentic',
|
|
185
|
+
confidence: 0.55,
|
|
186
|
+
|
|
187
|
+
re: /\bfrom\s+mcp\.client[\w.]*\s+import\b|\bimport\s+mcp\.client\b|\bmcp\.client\.\w+|\bfrom\s+mcp\s+import\b[^\n]*\bClientSession\b/,
|
|
188
|
+
codeOnly: true,
|
|
189
|
+
sink: (m) => m[0].trim(),
|
|
190
|
+
source: 'MCP server tool output',
|
|
191
|
+
message: 'Acts as an MCP client/host: opens a ClientSession to MCP servers and passes their tool descriptions and results back to a model. Every server it reaches is an untrusted-input ingress - a poisoned tool description or result can hijack the agent (prompt injection / tool poisoning).',
|
|
192
|
+
remediation: 'Pin exactly which MCP servers this client may connect to and run each through governance before trusting it. Treat all server output as untrusted data, never instructions, and screen it (runtime firewall) before it reaches the model.',
|
|
193
|
+
cwe: 'CWE-829',
|
|
194
|
+
},
|
|
195
|
+
{
|
|
196
|
+
id: 'python.reduce_payload',
|
|
197
|
+
title: 'Custom __reduce__ (pickle RCE gadget)',
|
|
198
|
+
severity: 'CRITICAL',
|
|
199
|
+
category: 'deserialization',
|
|
200
|
+
confidence: 0.8,
|
|
201
|
+
re: /def\s+__reduce__\s*\(|def\s+__reduce_ex__\s*\(|def\s+__setstate__\s*\(/,
|
|
202
|
+
sink: (m) => m[0].replace(/^def\s+/, '').replace(/\s*\($/, '').trim(),
|
|
203
|
+
message: 'Defines a pickle reduction hook. These execute on unpickling and are the classic gadget used to hide code-exec inside a serialized model object.',
|
|
204
|
+
remediation: 'Verify why the class needs custom pickling. Do not unpickle objects from this repo; prefer safetensors serialization which has no code path.',
|
|
205
|
+
cwe: 'CWE-502',
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
id: 'python.network_egress',
|
|
209
|
+
title: 'Network egress',
|
|
210
|
+
|
|
211
|
+
severity: 'MEDIUM',
|
|
212
|
+
category: 'egress',
|
|
213
|
+
confidence: 0.5,
|
|
214
|
+
re: /\b(requests|httpx)\.(get|post|put|request)\s*\(|\burllib\.request\.(urlopen|urlretrieve)\s*\(|\bsocket\.(socket|create_connection)\s*\(|\baiohttp\.ClientSession\s*\(/,
|
|
215
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
216
|
+
source: 'network',
|
|
217
|
+
message: 'Opens an outbound network connection. Normal in application code; in model/tokenizer code that should never phone out, or chained with a remote-code load, this is the exfiltration / second-stage-download shape.',
|
|
218
|
+
remediation: 'Confirm the destination and payload. Model inference code should never make outbound requests; treat a phoning-out model as hostile until proven otherwise.',
|
|
219
|
+
cwe: 'CWE-913',
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
id: 'python.dynamic_import',
|
|
223
|
+
title: 'Dynamic / obfuscated import',
|
|
224
|
+
severity: 'HIGH',
|
|
225
|
+
category: 'obfuscation',
|
|
226
|
+
confidence: 0.7,
|
|
227
|
+
re: /\bimportlib\.import_module\s*\(|\b__import__\s*\(\s*['"]?\s*(os|subprocess|socket|base64|marshal|ctypes)|\bexec\s*\(\s*(base64|bytes|marshal|codecs)/,
|
|
228
|
+
sink: (m) => m[0].trim(),
|
|
229
|
+
message: 'Imports or executes a module chosen at runtime, often to hide os/subprocess/socket usage from a quick read.',
|
|
230
|
+
remediation: 'Resolve what is imported and why. Obfuscated dynamic imports in model code are a strong malware tell.',
|
|
231
|
+
cwe: 'CWE-94',
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
id: 'python.encoded_payload',
|
|
235
|
+
title: 'Encoded blob decode',
|
|
236
|
+
|
|
237
|
+
severity: 'LOW',
|
|
238
|
+
category: 'obfuscation',
|
|
239
|
+
confidence: 0.5,
|
|
240
|
+
re: /\b(base64|codecs|binascii)\.(b64decode|decode|unhexlify)\s*\(|bytes\.fromhex\s*\(/,
|
|
241
|
+
sink: (m) => m[0].replace(/\s*\($/, '').trim(),
|
|
242
|
+
message: 'Decodes an encoded blob. Benign on its own (common in tokenizers/quantization), but the "decode then run it" packer combines this with eval/exec - see any decode-and-run chain finding on this file.',
|
|
243
|
+
remediation: 'If this decode feeds eval/exec/import, decode the blob offline and inspect it. A lone decode of vocab/kernel data is expected.',
|
|
244
|
+
cwe: 'CWE-506',
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
id: 'python.gradio_public_share',
|
|
248
|
+
title: 'Model UI exposed via public share tunnel',
|
|
249
|
+
severity: 'MEDIUM',
|
|
250
|
+
category: 'exposure',
|
|
251
|
+
confidence: 0.8,
|
|
252
|
+
re: /\.launch\s*\([^)]*share\s*=\s*True|\.queue\s*\([^)]*\)\.launch\s*\([^)]*share\s*=\s*True/,
|
|
253
|
+
sink: () => 'launch(share=True)',
|
|
254
|
+
message: 'Launches a Gradio/model UI with share=True, publishing a public tunnel URL to a locally-running model - anyone with the link can drive inference (and any tools wired to it) with no auth.',
|
|
255
|
+
remediation: 'Remove share=True for anything beyond a throwaway demo. Bind to localhost or put the app behind authenticated ingress; never expose a tool-enabled agent this way.',
|
|
256
|
+
cwe: 'CWE-668',
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
id: 'python.hardcoded_ai_key',
|
|
260
|
+
title: 'Hardcoded AI-provider API key',
|
|
261
|
+
severity: 'MEDIUM',
|
|
262
|
+
category: 'secret',
|
|
263
|
+
confidence: 0.85,
|
|
264
|
+
|
|
265
|
+
re: /['"](sk-ant-[A-Za-z0-9_-]{20,}|sk-[A-Za-z0-9]{20,}|hf_[A-Za-z0-9]{20,}|AIza[A-Za-z0-9_-]{20,}|gsk_[A-Za-z0-9]{20,})['"]/,
|
|
266
|
+
sink: (m) => m[1].slice(0, 12) + '…',
|
|
267
|
+
source: 'source literal',
|
|
268
|
+
message: 'An AI-provider API key is hardcoded as a string literal. Anyone with read access to this repo can drain the account; committed keys are scraped within minutes.',
|
|
269
|
+
remediation: 'Remove the literal and load the key from an environment variable / secret manager at runtime. Rotate the exposed key immediately.',
|
|
270
|
+
cwe: 'CWE-798',
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
id: 'python.env_exfil',
|
|
274
|
+
title: 'Reads environment / secrets',
|
|
275
|
+
severity: 'MEDIUM',
|
|
276
|
+
category: 'secret',
|
|
277
|
+
confidence: 0.5,
|
|
278
|
+
re: /\bos\.environ\b|\bos\.getenv\s*\(|\bParameterStore|\bboto3\.client\s*\(\s*['"]s(ts|ecretsmanager)/,
|
|
279
|
+
sink: (m) => m[0].trim(),
|
|
280
|
+
source: 'process environment',
|
|
281
|
+
message: 'Reads environment variables or a secrets store. Paired with network egress this is credential exfiltration.',
|
|
282
|
+
remediation: 'Confirm the code has a legitimate need for the variable; model inference code generally should not read the environment.',
|
|
283
|
+
cwe: 'CWE-200',
|
|
284
|
+
},
|
|
285
|
+
];
|
|
286
|
+
|
|
287
|
+
export const PY_TAINT = {
|
|
288
|
+
lang: 'python',
|
|
289
|
+
ruleId: 'python.llm_output_to_sink',
|
|
290
|
+
aiCall: /\.(a?generate|a?predict|a?invoke|a?run|complete|acomplete|chat|stream|__call__|predict_messages)\s*\(|\.(chat\.)?completions\.create\s*\(|\.messages\.create\s*\(|\bllm\s*\(/,
|
|
291
|
+
execSink: /(?<![.\w])(eval|exec|compile)\s*\(|\bos\.(system|popen)\s*\(|\bsubprocess\.(run|call|check_output|check_call|Popen)\s*\(/g,
|
|
292
|
+
};
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { chainFindings } from './chains.mjs';
|
|
2
|
+
import { constPathBindings, pathBindings } from './path-expressions.mjs';
|
|
3
|
+
import { CONFIG_RULES } from './rules-config.mjs';
|
|
4
|
+
import { JS_RULES, JS_TAINT } from './rules-javascript.mjs';
|
|
5
|
+
import { PY_RULES, PY_TAINT } from './rules-python.mjs';
|
|
6
|
+
import { contextChunk, isCommentLine, isInsideString, logicalLines, physicalIdx } from './source-lines.mjs';
|
|
7
|
+
import { taintFindings } from './taint.mjs';
|
|
8
|
+
|
|
9
|
+
function scanLines(text, file, rules, taintCfg) {
|
|
10
|
+
const lines = text.split(/\r?\n/);
|
|
11
|
+
const units = logicalLines(lines);
|
|
12
|
+
const out = [];
|
|
13
|
+
const seen = new Set();
|
|
14
|
+
|
|
15
|
+
const pathNs = pathBindings(text);
|
|
16
|
+
const ctx = { pathNs, constPaths: constPathBindings(text, pathNs) };
|
|
17
|
+
for (const unit of units) {
|
|
18
|
+
for (const rule of rules) {
|
|
19
|
+
rule.re.lastIndex = 0;
|
|
20
|
+
const m = rule.re.exec(unit.text);
|
|
21
|
+
if (!m) continue;
|
|
22
|
+
|
|
23
|
+
if (rule.codeOnly && isInsideString(unit.text, m.index)) continue;
|
|
24
|
+
|
|
25
|
+
if (rule.suppress && rule.suppress(m, unit.text, ctx)) continue;
|
|
26
|
+
const idx = physicalIdx(unit, m.index);
|
|
27
|
+
const trimmed = (lines[idx] ?? '').trim();
|
|
28
|
+
if (!trimmed || isCommentLine(trimmed)) continue;
|
|
29
|
+
const key = `${rule.id}@${idx}`;
|
|
30
|
+
if (seen.has(key)) continue;
|
|
31
|
+
seen.add(key);
|
|
32
|
+
out.push({
|
|
33
|
+
ruleId: rule.id,
|
|
34
|
+
title: rule.title,
|
|
35
|
+
severity: rule.severity,
|
|
36
|
+
category: rule.category,
|
|
37
|
+
confidence: rule.confidence,
|
|
38
|
+
file,
|
|
39
|
+
line: idx + 1,
|
|
40
|
+
sink: (rule.sink ? rule.sink(m) : m[0]).slice(0, 120),
|
|
41
|
+
source: rule.source || undefined,
|
|
42
|
+
...contextChunk(lines, idx),
|
|
43
|
+
message: rule.message,
|
|
44
|
+
remediation: rule.remediation,
|
|
45
|
+
cwe: rule.cwe,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (taintCfg) out.push(...taintFindings(lines, units, file, taintCfg));
|
|
50
|
+
out.push(...chainFindings(lines, out, file));
|
|
51
|
+
return out;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function scanPythonSource(text, file) { return text ? scanLines(text, file, PY_RULES, PY_TAINT) : []; }
|
|
55
|
+
|
|
56
|
+
export function scanJsSource(text, file) { return text ? scanLines(text, file, JS_RULES, JS_TAINT) : []; }
|
|
57
|
+
|
|
58
|
+
export function scanModelConfig(text, file) { return text ? scanLines(text, file, CONFIG_RULES, null) : []; }
|
|
59
|
+
|
|
60
|
+
export function scanNotebook(text, file) {
|
|
61
|
+
if (!text) return [];
|
|
62
|
+
let nb;
|
|
63
|
+
try { nb = JSON.parse(text); } catch { return []; }
|
|
64
|
+
const cells = Array.isArray(nb?.cells) ? nb.cells : [];
|
|
65
|
+
const lang = String(nb?.metadata?.kernelspec?.language || nb?.metadata?.language_info?.name || 'python').toLowerCase();
|
|
66
|
+
const isJs = /javascript|typescript|deno|node|^js$|^ts$/.test(lang);
|
|
67
|
+
const rules = isJs ? JS_RULES : PY_RULES;
|
|
68
|
+
const taintCfg = isJs ? JS_TAINT : PY_TAINT;
|
|
69
|
+
const out = [];
|
|
70
|
+
let codeCell = 0;
|
|
71
|
+
for (const cell of cells) {
|
|
72
|
+
if (cell?.cell_type !== 'code') continue;
|
|
73
|
+
codeCell++;
|
|
74
|
+
const src = Array.isArray(cell.source) ? cell.source.join('') : String(cell.source ?? '');
|
|
75
|
+
if (!src.trim()) continue;
|
|
76
|
+
out.push(...scanLines(src, `${file}#cell${codeCell}`, rules, taintCfg));
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const PY_EXT = /\.py$/i;
|
|
82
|
+
|
|
83
|
+
const JS_EXT = /\.(m|c)?[jt]sx?$/i;
|
|
84
|
+
|
|
85
|
+
const NB_EXT = /\.ipynb$/i;
|
|
86
|
+
|
|
87
|
+
const MODEL_CONFIG_RE = /(^|\/)(config|tokenizer_config|generation_config|preprocessor_config)\.json$/i;
|
|
88
|
+
|
|
89
|
+
export function isScannableSource(path) {
|
|
90
|
+
return PY_EXT.test(path) || JS_EXT.test(path) || NB_EXT.test(path);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function isModelConfig(path) {
|
|
94
|
+
return MODEL_CONFIG_RE.test(String(path ?? '').split(/[\\/]+/).join('/'));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function scanSourceFile(text, file) {
|
|
98
|
+
if (!text) return [];
|
|
99
|
+
if (NB_EXT.test(file)) return scanNotebook(text, file);
|
|
100
|
+
if (PY_EXT.test(file)) return scanPythonSource(text, file);
|
|
101
|
+
if (JS_EXT.test(file)) return scanJsSource(text, file);
|
|
102
|
+
if (isModelConfig(file)) return scanModelConfig(text, file);
|
|
103
|
+
return [];
|
|
104
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
const MAX_SNIPPET = 400;
|
|
2
|
+
|
|
3
|
+
const CONTEXT_RADIUS = 3;
|
|
4
|
+
|
|
5
|
+
const MAX_JOIN_LINES = 40;
|
|
6
|
+
|
|
7
|
+
export function contextChunk(lines, idx) {
|
|
8
|
+
const start = Math.max(0, idx - CONTEXT_RADIUS);
|
|
9
|
+
const end = Math.min(lines.length - 1, idx + CONTEXT_RADIUS);
|
|
10
|
+
const snippet = lines.slice(start, end + 1).join('\n').slice(0, MAX_SNIPPET);
|
|
11
|
+
return { snippet, snippetStartLine: start + 1 };
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function isCommentLine(trimmed) {
|
|
15
|
+
return trimmed.startsWith('#') || trimmed.startsWith('//') || trimmed.startsWith('*') || trimmed.startsWith('/*');
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function lineDepthDelta(line) {
|
|
19
|
+
let delta = 0;
|
|
20
|
+
let quote = null;
|
|
21
|
+
for (let i = 0; i < line.length; i++) {
|
|
22
|
+
const c = line[i];
|
|
23
|
+
if (quote) {
|
|
24
|
+
if (c === '\\') { i++; continue; }
|
|
25
|
+
if (c === quote) quote = null;
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
if (c === '"' || c === "'" || c === '`') { quote = c; continue; }
|
|
29
|
+
if (c === '#') break;
|
|
30
|
+
if (c === '/' && line[i + 1] === '/') break;
|
|
31
|
+
if (c === '(' || c === '[' || c === '{') delta++;
|
|
32
|
+
else if (c === ')' || c === ']' || c === '}') delta--;
|
|
33
|
+
}
|
|
34
|
+
return { delta, backslash: !quote && /\\\s*$/.test(line) };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function logicalLines(lines) {
|
|
38
|
+
const out = [];
|
|
39
|
+
let i = 0;
|
|
40
|
+
while (i < lines.length) {
|
|
41
|
+
const startLine = i + 1;
|
|
42
|
+
const buf = [];
|
|
43
|
+
let depth = 0;
|
|
44
|
+
while (i < lines.length) {
|
|
45
|
+
const line = lines[i];
|
|
46
|
+
buf.push(line);
|
|
47
|
+
const { delta, backslash } = lineDepthDelta(line);
|
|
48
|
+
depth += delta;
|
|
49
|
+
i++;
|
|
50
|
+
if ((depth <= 0 && !backslash) || buf.length >= MAX_JOIN_LINES) break;
|
|
51
|
+
}
|
|
52
|
+
out.push({ text: buf.join('\n'), startLine });
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function escapeRe(s) {
|
|
58
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function callArgText(text, from) {
|
|
62
|
+
const open = text.indexOf('(', from);
|
|
63
|
+
if (open < 0) return '';
|
|
64
|
+
let depth = 0;
|
|
65
|
+
let quote = '';
|
|
66
|
+
for (let i = open; i < text.length; i++) {
|
|
67
|
+
const ch = text[i];
|
|
68
|
+
if (quote) {
|
|
69
|
+
if (ch === '\\') i++;
|
|
70
|
+
else if (ch === quote) quote = '';
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (ch === '"' || ch === "'" || ch === '`') quote = ch;
|
|
74
|
+
else if (ch === '(') depth++;
|
|
75
|
+
else if (ch === ')') {
|
|
76
|
+
depth--;
|
|
77
|
+
if (depth === 0) return text.slice(open + 1, i);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return '';
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function isInsideString(text, offset) {
|
|
84
|
+
let quote = null;
|
|
85
|
+
let triple = false;
|
|
86
|
+
for (let i = 0; i < offset && i < text.length; i++) {
|
|
87
|
+
const c = text[i];
|
|
88
|
+
if (quote) {
|
|
89
|
+
if (c === '\\') { i++; continue; }
|
|
90
|
+
if (triple) {
|
|
91
|
+
if (c === quote && text[i + 1] === quote && text[i + 2] === quote) { i += 2; quote = null; triple = false; }
|
|
92
|
+
} else if (c === quote) {
|
|
93
|
+
quote = null;
|
|
94
|
+
}
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
if (c === '#') {
|
|
98
|
+
const nl = text.indexOf('\n', i);
|
|
99
|
+
if (nl === -1) return false;
|
|
100
|
+
i = nl;
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (c === '"' || c === "'" || c === '`') {
|
|
104
|
+
quote = c;
|
|
105
|
+
triple = text[i + 1] === c && text[i + 2] === c;
|
|
106
|
+
if (triple) i += 2;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return quote !== null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export function physicalIdx(unit, offset) {
|
|
113
|
+
const newlines = unit.text.slice(0, offset).match(/\n/g);
|
|
114
|
+
return unit.startLine - 1 + (newlines ? newlines.length : 0);
|
|
115
|
+
}
|