@openai/codex-security 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +306 -0
- package/_bundled_plugin/.app.json +20 -0
- package/_bundled_plugin/.codex-plugin/plugin.json +41 -0
- package/_bundled_plugin/.mcp.json +20 -0
- package/_bundled_plugin/assets/logo.png +0 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +22 -0
- package/_bundled_plugin/examples/completed-scan/findings.json +51 -0
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +43 -0
- package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
- package/_bundled_plugin/mcp/server.mjs +24 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +179 -0
- package/_bundled_plugin/references/config-preflight.md +115 -0
- package/_bundled_plugin/references/final-report.md +205 -0
- package/_bundled_plugin/references/finding-detail-fields.md +162 -0
- package/_bundled_plugin/references/sarif-adapter.md +26 -0
- package/_bundled_plugin/references/scan-artifacts.md +116 -0
- package/_bundled_plugin/references/scan-contract.md +164 -0
- package/_bundled_plugin/references/security-guidance.md +15 -0
- package/_bundled_plugin/references/shared-hard-rules.md +15 -0
- package/_bundled_plugin/references/static-finding-assessment.md +75 -0
- package/_bundled_plugin/schemas/coverage.schema.json +228 -0
- package/_bundled_plugin/schemas/findings.schema.json +359 -0
- package/_bundled_plugin/schemas/scan-manifest.schema.json +332 -0
- package/_bundled_plugin/scripts/config_preflight.py +901 -0
- package/_bundled_plugin/scripts/deep_scan_config.py +108 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +1547 -0
- package/_bundled_plugin/scripts/filesystem_identity.py +30 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +2208 -0
- package/_bundled_plugin/scripts/finding_preview.py +270 -0
- package/_bundled_plugin/scripts/generate_rank_input.py +1013 -0
- package/_bundled_plugin/scripts/normalize_candidates.py +320 -0
- package/_bundled_plugin/scripts/rank_preview.py +968 -0
- package/_bundled_plugin/scripts/report_projection.py +844 -0
- package/_bundled_plugin/scripts/resolve_security_md.py +158 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +26 -0
- package/_bundled_plugin/scripts/validate_report_format.py +138 -0
- package/_bundled_plugin/scripts/validate_scan_contract.py +108 -0
- package/_bundled_plugin/scripts/validate_tracking_source.py +92 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +612 -0
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -0
- package/_bundled_plugin/scripts/workbench/handoff.py +238 -0
- package/_bundled_plugin/scripts/workbench_cli.py +325 -0
- package/_bundled_plugin/scripts/workbench_constants.py +81 -0
- package/_bundled_plugin/scripts/workbench_db.py +3728 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +242 -0
- package/_bundled_plugin/scripts/workbench_progress.py +226 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +125 -0
- package/_bundled_plugin/scripts/workbench_scan_history.py +829 -0
- package/_bundled_plugin/scripts/workbench_scan_start.py +149 -0
- package/_bundled_plugin/scripts/workbench_schema.py +714 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +99 -0
- package/_bundled_plugin/scripts/workbench_target.py +469 -0
- package/_bundled_plugin/scripts/workbench_target_state.py +63 -0
- package/_bundled_plugin/scripts/workbench_validation.py +294 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +117 -0
- package/_bundled_plugin/skills/attack-path-analysis/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +76 -0
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +153 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +166 -0
- package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +93 -0
- package/_bundled_plugin/skills/define-security-policy/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +155 -0
- package/_bundled_plugin/skills/finding-discovery/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +112 -0
- package/_bundled_plugin/skills/fix-finding/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +240 -0
- package/_bundled_plugin/skills/propose-security-hardening/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +424 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +167 -0
- package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/security-scan/SKILL.md +42 -0
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +48 -0
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +85 -0
- package/_bundled_plugin/skills/threat-model/SKILL.md +59 -0
- package/_bundled_plugin/skills/threat-model/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/threat-model/references/threat-model-guidance.md +48 -0
- package/_bundled_plugin/skills/track-findings/SKILL.md +222 -0
- package/_bundled_plugin/skills/track-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +54 -0
- package/_bundled_plugin/skills/track-findings/references/jira.md +77 -0
- package/_bundled_plugin/skills/triage-finding/SKILL.md +342 -0
- package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +168 -0
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +62 -0
- package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +108 -0
- package/_bundled_plugin/skills/validation/SKILL.md +110 -0
- package/_bundled_plugin/skills/validation/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +122 -0
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +263 -0
- package/_bundled_plugin/skills/vulnerability-writeup/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +104 -0
- package/bin/codex-security.mjs +26 -0
- package/dist/api.d.ts +112 -0
- package/dist/api.d.ts.map +1 -0
- package/dist/api.js +1051 -0
- package/dist/api.js.map +1 -0
- package/dist/auth.d.ts +29 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +246 -0
- package/dist/auth.js.map +1 -0
- package/dist/bulk-scan-discovery.d.ts +35 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -0
- package/dist/bulk-scan-discovery.js +217 -0
- package/dist/bulk-scan-discovery.js.map +1 -0
- package/dist/cli.d.ts +67 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2128 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +199 -0
- package/dist/config.js.map +1 -0
- package/dist/contract.d.ts +21 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +848 -0
- package/dist/contract.js.map +1 -0
- package/dist/cost.d.ts +30 -0
- package/dist/cost.d.ts.map +1 -0
- package/dist/cost.js +280 -0
- package/dist/cost.js.map +1 -0
- package/dist/errors.d.ts +38 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +52 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/knowledge-base.d.ts +7 -0
- package/dist/knowledge-base.d.ts.map +1 -0
- package/dist/knowledge-base.js +181 -0
- package/dist/knowledge-base.js.map +1 -0
- package/dist/models.d.ts +231 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +3 -0
- package/dist/models.js.map +1 -0
- package/dist/multiscan.d.ts +30 -0
- package/dist/multiscan.d.ts.map +1 -0
- package/dist/multiscan.js +397 -0
- package/dist/multiscan.js.map +1 -0
- package/dist/result.d.ts +39 -0
- package/dist/result.d.ts.map +1 -0
- package/dist/result.js +71 -0
- package/dist/result.js.map +1 -0
- package/dist/runtime.d.ts +69 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +997 -0
- package/dist/runtime.js.map +1 -0
- package/dist/scan-comparison.d.ts +42 -0
- package/dist/scan-comparison.d.ts.map +1 -0
- package/dist/scan-comparison.js +153 -0
- package/dist/scan-comparison.js.map +1 -0
- package/dist/scan-history-renderer.d.ts +13 -0
- package/dist/scan-history-renderer.d.ts.map +1 -0
- package/dist/scan-history-renderer.js +297 -0
- package/dist/scan-history-renderer.js.map +1 -0
- package/dist/targets.d.ts +39 -0
- package/dist/targets.d.ts.map +1 -0
- package/dist/targets.js +295 -0
- package/dist/targets.js.map +1 -0
- package/dist/trusted-executable.d.ts +6 -0
- package/dist/trusted-executable.d.ts.map +1 -0
- package/dist/trusted-executable.js +77 -0
- package/dist/trusted-executable.js.map +1 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +42 -0
- package/dist/version.js.map +1 -0
- package/dist/worker-progress.d.ts +13 -0
- package/dist/worker-progress.d.ts.map +1 -0
- package/dist/worker-progress.js +113 -0
- package/dist/worker-progress.js.map +1 -0
- package/package.json +74 -0
|
@@ -0,0 +1,968 @@
|
|
|
1
|
+
"""Build deterministic structural or sampled source previews for rank inputs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import ast
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
DEFAULT_PREVIEW_BYTES = 1024
|
|
12
|
+
PREVIEW_HEAD_LINES = 12
|
|
13
|
+
PREVIEW_SAMPLE_LINES = 10
|
|
14
|
+
|
|
15
|
+
TEXT_CODE_EXTENSIONS = {
|
|
16
|
+
".c",
|
|
17
|
+
".cc",
|
|
18
|
+
".cfg",
|
|
19
|
+
".clj",
|
|
20
|
+
".cpp",
|
|
21
|
+
".cs",
|
|
22
|
+
".css",
|
|
23
|
+
".cue",
|
|
24
|
+
".cxx",
|
|
25
|
+
".dart",
|
|
26
|
+
".ex",
|
|
27
|
+
".exs",
|
|
28
|
+
".go",
|
|
29
|
+
".graphql",
|
|
30
|
+
".h",
|
|
31
|
+
".hpp",
|
|
32
|
+
".hs",
|
|
33
|
+
".html",
|
|
34
|
+
".java",
|
|
35
|
+
".js",
|
|
36
|
+
".json",
|
|
37
|
+
".jsx",
|
|
38
|
+
".kt",
|
|
39
|
+
".kts",
|
|
40
|
+
".lua",
|
|
41
|
+
".mjs",
|
|
42
|
+
".mm",
|
|
43
|
+
".php",
|
|
44
|
+
".proto",
|
|
45
|
+
".py",
|
|
46
|
+
".rb",
|
|
47
|
+
".rs",
|
|
48
|
+
".scala",
|
|
49
|
+
".sh",
|
|
50
|
+
".sql",
|
|
51
|
+
".swift",
|
|
52
|
+
".toml",
|
|
53
|
+
".ts",
|
|
54
|
+
".tsx",
|
|
55
|
+
".vue",
|
|
56
|
+
".xml",
|
|
57
|
+
".yaml",
|
|
58
|
+
".yml",
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
JAVASCRIPT_EXTENSIONS = {".js", ".jsx", ".mjs", ".ts", ".tsx", ".vue"}
|
|
62
|
+
JAVA_LIKE_EXTENSIONS = {".c", ".cc", ".cpp", ".cs", ".cxx", ".h", ".hpp", ".java", ".mm"}
|
|
63
|
+
BRACE_LANGUAGE_EXTENSIONS = {
|
|
64
|
+
*JAVASCRIPT_EXTENSIONS,
|
|
65
|
+
*JAVA_LIKE_EXTENSIONS,
|
|
66
|
+
".dart",
|
|
67
|
+
".go",
|
|
68
|
+
".kt",
|
|
69
|
+
".kts",
|
|
70
|
+
".php",
|
|
71
|
+
".rs",
|
|
72
|
+
".scala",
|
|
73
|
+
".swift",
|
|
74
|
+
}
|
|
75
|
+
NESTED_BLOCK_COMMENT_EXTENSIONS = {".kt", ".kts", ".rs", ".scala", ".swift"}
|
|
76
|
+
RUST_RAW_STRING_RE = re.compile(r'(?:br|r)(#{0,16})"')
|
|
77
|
+
RUST_LIFETIME_RE = re.compile(r"'[A-Za-z_][A-Za-z0-9_]*")
|
|
78
|
+
PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def is_binary_sample(data: bytes) -> bool:
|
|
82
|
+
return b"\0" in data
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def compact_preview_line(line: str) -> str:
|
|
86
|
+
return " ".join(line.split())
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def truncate_utf8(text: str, max_bytes: int) -> str:
|
|
90
|
+
if max_bytes <= 0:
|
|
91
|
+
return ""
|
|
92
|
+
encoded = text.encode("utf-8")
|
|
93
|
+
if len(encoded) <= max_bytes:
|
|
94
|
+
return text
|
|
95
|
+
return encoded[:max_bytes].decode("utf-8", errors="ignore")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def select_preview_lines(lines: list[str]) -> list[str]:
|
|
99
|
+
compact = [value for line in lines if (value := compact_preview_line(line))]
|
|
100
|
+
head = compact[:PREVIEW_HEAD_LINES]
|
|
101
|
+
remainder = compact[PREVIEW_HEAD_LINES:]
|
|
102
|
+
if len(remainder) <= PREVIEW_SAMPLE_LINES:
|
|
103
|
+
return [*head, *remainder]
|
|
104
|
+
|
|
105
|
+
last_index = len(remainder) - 1
|
|
106
|
+
sampled = [
|
|
107
|
+
remainder[index * last_index // (PREVIEW_SAMPLE_LINES - 1)]
|
|
108
|
+
for index in range(PREVIEW_SAMPLE_LINES)
|
|
109
|
+
]
|
|
110
|
+
return [*head, "...", *sampled]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def fit_preview_lines(lines: list[str], max_bytes: int) -> str:
|
|
114
|
+
if not lines or max_bytes <= 0:
|
|
115
|
+
return ""
|
|
116
|
+
|
|
117
|
+
full_preview = "\n".join(lines)
|
|
118
|
+
if len(full_preview.encode("utf-8")) <= max_bytes:
|
|
119
|
+
return full_preview
|
|
120
|
+
|
|
121
|
+
def render(line_bytes: int) -> str:
|
|
122
|
+
return "\n".join(
|
|
123
|
+
line if line == "..." else truncate_utf8(line, line_bytes).rstrip() for line in lines
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
content_lines = [line for line in lines if line != "..."]
|
|
127
|
+
if not content_lines:
|
|
128
|
+
return truncate_utf8(full_preview, max_bytes)
|
|
129
|
+
low = 0
|
|
130
|
+
high = max(len(line.encode("utf-8")) for line in content_lines)
|
|
131
|
+
best = ""
|
|
132
|
+
while low <= high:
|
|
133
|
+
middle = (low + high) // 2
|
|
134
|
+
candidate = render(middle)
|
|
135
|
+
if len(candidate.encode("utf-8")) <= max_bytes:
|
|
136
|
+
best = candidate
|
|
137
|
+
low = middle + 1
|
|
138
|
+
else:
|
|
139
|
+
high = middle - 1
|
|
140
|
+
if best:
|
|
141
|
+
return best
|
|
142
|
+
return truncate_utf8(full_preview, max_bytes)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def python_decorators(node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef) -> str:
|
|
146
|
+
values: list[str] = []
|
|
147
|
+
for decorator in node.decorator_list:
|
|
148
|
+
try:
|
|
149
|
+
rendered = compact_preview_line(ast.unparse(decorator))
|
|
150
|
+
except (AttributeError, RecursionError, ValueError):
|
|
151
|
+
continue
|
|
152
|
+
if rendered:
|
|
153
|
+
values.append(f"@{rendered}")
|
|
154
|
+
return " ".join(values)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def python_arguments(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
|
|
158
|
+
arguments = [argument.arg for argument in (*node.args.posonlyargs, *node.args.args)]
|
|
159
|
+
if node.args.vararg is not None:
|
|
160
|
+
arguments.append(f"*{node.args.vararg.arg}")
|
|
161
|
+
arguments.extend(argument.arg for argument in node.args.kwonlyargs)
|
|
162
|
+
if node.args.kwarg is not None:
|
|
163
|
+
arguments.append(f"**{node.args.kwarg.arg}")
|
|
164
|
+
return ", ".join(arguments)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def python_outline(text: str) -> list[str]:
|
|
168
|
+
try:
|
|
169
|
+
tree = ast.parse(text)
|
|
170
|
+
except (SyntaxError, ValueError, TypeError, MemoryError):
|
|
171
|
+
return []
|
|
172
|
+
|
|
173
|
+
outline: list[str] = []
|
|
174
|
+
for node in tree.body:
|
|
175
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
176
|
+
prefix = python_decorators(node)
|
|
177
|
+
kind = "async function" if isinstance(node, ast.AsyncFunctionDef) else "function"
|
|
178
|
+
declaration = f"{kind} {node.name}({python_arguments(node)})"
|
|
179
|
+
outline.append(f"{prefix} {declaration}".strip())
|
|
180
|
+
elif isinstance(node, ast.ClassDef):
|
|
181
|
+
prefix = python_decorators(node)
|
|
182
|
+
bases: list[str] = []
|
|
183
|
+
for base in node.bases:
|
|
184
|
+
try:
|
|
185
|
+
bases.append(compact_preview_line(ast.unparse(base)))
|
|
186
|
+
except (AttributeError, RecursionError, ValueError):
|
|
187
|
+
continue
|
|
188
|
+
suffix = f"({', '.join(bases)})" if bases else ""
|
|
189
|
+
outline.append(f"{prefix} class {node.name}{suffix}".strip())
|
|
190
|
+
for member in node.body:
|
|
191
|
+
if not isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
192
|
+
continue
|
|
193
|
+
member_prefix = python_decorators(member)
|
|
194
|
+
kind = "async method" if isinstance(member, ast.AsyncFunctionDef) else "method"
|
|
195
|
+
declaration = f"{kind} {node.name}.{member.name}"
|
|
196
|
+
outline.append(f"{member_prefix} {declaration}".strip())
|
|
197
|
+
return outline
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def javascript_regex_end(text: str, start: int) -> int | None:
|
|
201
|
+
if start + 1 >= len(text) or text[start + 1] in {"/", "*"}:
|
|
202
|
+
return None
|
|
203
|
+
previous = start - 1
|
|
204
|
+
while previous >= 0 and text[previous] in " \t\r":
|
|
205
|
+
previous -= 1
|
|
206
|
+
if previous >= 0 and text[previous] not in "=(:,[!&|?{};\n":
|
|
207
|
+
prefix = text[max(0, previous - 8) : previous + 1]
|
|
208
|
+
if not re.search(r"\b(?:case|return|throw)$", prefix):
|
|
209
|
+
return None
|
|
210
|
+
|
|
211
|
+
index = start + 1
|
|
212
|
+
in_character_class = False
|
|
213
|
+
while index < len(text):
|
|
214
|
+
char = text[index]
|
|
215
|
+
if char == "\n":
|
|
216
|
+
return None
|
|
217
|
+
if char == "\\" and index + 1 < len(text):
|
|
218
|
+
index += 2
|
|
219
|
+
continue
|
|
220
|
+
if char == "[":
|
|
221
|
+
in_character_class = True
|
|
222
|
+
elif char == "]":
|
|
223
|
+
in_character_class = False
|
|
224
|
+
elif char == "/" and not in_character_class:
|
|
225
|
+
index += 1
|
|
226
|
+
while index < len(text) and text[index].isalpha():
|
|
227
|
+
index += 1
|
|
228
|
+
return index
|
|
229
|
+
index += 1
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def mask_c_style_source(text: str, suffix: str) -> str:
|
|
234
|
+
masked: list[str] = []
|
|
235
|
+
index = 0
|
|
236
|
+
block_comment_depth = 0
|
|
237
|
+
in_line_comment = False
|
|
238
|
+
quote = ""
|
|
239
|
+
raw_terminator = ""
|
|
240
|
+
heredoc_terminator = ""
|
|
241
|
+
while index < len(text):
|
|
242
|
+
char = text[index]
|
|
243
|
+
next_char = text[index + 1] if index + 1 < len(text) else ""
|
|
244
|
+
if char == "\n":
|
|
245
|
+
masked.append(char)
|
|
246
|
+
in_line_comment = False
|
|
247
|
+
if quote and quote not in {"`", '"""', "'''", '@"'}:
|
|
248
|
+
quote = ""
|
|
249
|
+
index += 1
|
|
250
|
+
continue
|
|
251
|
+
if in_line_comment:
|
|
252
|
+
masked.append(" ")
|
|
253
|
+
index += 1
|
|
254
|
+
continue
|
|
255
|
+
if block_comment_depth:
|
|
256
|
+
if char == "/" and next_char == "*" and suffix in NESTED_BLOCK_COMMENT_EXTENSIONS:
|
|
257
|
+
masked.extend((" ", " "))
|
|
258
|
+
block_comment_depth += 1
|
|
259
|
+
index += 2
|
|
260
|
+
elif char == "*" and next_char == "/":
|
|
261
|
+
masked.extend((" ", " "))
|
|
262
|
+
block_comment_depth -= 1
|
|
263
|
+
index += 2
|
|
264
|
+
else:
|
|
265
|
+
masked.append(" ")
|
|
266
|
+
index += 1
|
|
267
|
+
continue
|
|
268
|
+
if heredoc_terminator:
|
|
269
|
+
if index == 0 or text[index - 1] == "\n":
|
|
270
|
+
line_end = text.find("\n", index)
|
|
271
|
+
if line_end < 0:
|
|
272
|
+
line_end = len(text)
|
|
273
|
+
candidate = text[index:line_end].strip().removesuffix(";")
|
|
274
|
+
if candidate == heredoc_terminator:
|
|
275
|
+
masked.extend(" " * (line_end - index))
|
|
276
|
+
index = line_end
|
|
277
|
+
heredoc_terminator = ""
|
|
278
|
+
continue
|
|
279
|
+
masked.append(" ")
|
|
280
|
+
index += 1
|
|
281
|
+
continue
|
|
282
|
+
if raw_terminator:
|
|
283
|
+
if text.startswith(raw_terminator, index):
|
|
284
|
+
masked.extend(" " * len(raw_terminator))
|
|
285
|
+
index += len(raw_terminator)
|
|
286
|
+
raw_terminator = ""
|
|
287
|
+
else:
|
|
288
|
+
masked.append(" ")
|
|
289
|
+
index += 1
|
|
290
|
+
continue
|
|
291
|
+
if quote in {'"""', "'''"}:
|
|
292
|
+
if text.startswith(quote, index):
|
|
293
|
+
masked.extend(" " * len(quote))
|
|
294
|
+
index += len(quote)
|
|
295
|
+
quote = ""
|
|
296
|
+
else:
|
|
297
|
+
masked.append(" ")
|
|
298
|
+
index += 1
|
|
299
|
+
continue
|
|
300
|
+
if quote == '@"':
|
|
301
|
+
masked.append(" ")
|
|
302
|
+
if char == '"' and next_char == '"':
|
|
303
|
+
masked.append(" ")
|
|
304
|
+
index += 2
|
|
305
|
+
else:
|
|
306
|
+
if char == '"':
|
|
307
|
+
quote = ""
|
|
308
|
+
index += 1
|
|
309
|
+
continue
|
|
310
|
+
if quote:
|
|
311
|
+
if char == "\\" and next_char:
|
|
312
|
+
masked.append(" ")
|
|
313
|
+
if next_char == "\n":
|
|
314
|
+
index += 1
|
|
315
|
+
else:
|
|
316
|
+
masked.append(" ")
|
|
317
|
+
index += 2
|
|
318
|
+
else:
|
|
319
|
+
masked.append(" ")
|
|
320
|
+
if char == quote:
|
|
321
|
+
quote = ""
|
|
322
|
+
index += 1
|
|
323
|
+
continue
|
|
324
|
+
if suffix == ".rs":
|
|
325
|
+
raw_match = RUST_RAW_STRING_RE.match(text, index)
|
|
326
|
+
if raw_match:
|
|
327
|
+
token = raw_match.group(0)
|
|
328
|
+
masked.extend(" " * len(token))
|
|
329
|
+
raw_terminator = f'"{raw_match.group(1)}'
|
|
330
|
+
index += len(token)
|
|
331
|
+
continue
|
|
332
|
+
if char == "'":
|
|
333
|
+
lifetime_match = RUST_LIFETIME_RE.match(text, index)
|
|
334
|
+
if lifetime_match:
|
|
335
|
+
token = lifetime_match.group(0)
|
|
336
|
+
following = text[index + len(token) : index + len(token) + 1]
|
|
337
|
+
if following != "'":
|
|
338
|
+
masked.extend(token)
|
|
339
|
+
index += len(token)
|
|
340
|
+
continue
|
|
341
|
+
if suffix == ".cs" and char == "@" and next_char == '"':
|
|
342
|
+
masked.extend((" ", " "))
|
|
343
|
+
quote = '@"'
|
|
344
|
+
index += 2
|
|
345
|
+
continue
|
|
346
|
+
triple_quote = text[index : index + 3]
|
|
347
|
+
if triple_quote in {'"""', "'''"}:
|
|
348
|
+
masked.extend((" ", " ", " "))
|
|
349
|
+
quote = triple_quote
|
|
350
|
+
index += 3
|
|
351
|
+
continue
|
|
352
|
+
if suffix == ".php" and text.startswith("<<<", index):
|
|
353
|
+
heredoc_match = PHP_HEREDOC_RE.match(text, index)
|
|
354
|
+
if heredoc_match:
|
|
355
|
+
token = heredoc_match.group(0)
|
|
356
|
+
masked.extend(" " * len(token))
|
|
357
|
+
heredoc_terminator = heredoc_match.group(1)
|
|
358
|
+
index += len(token)
|
|
359
|
+
continue
|
|
360
|
+
if suffix in JAVASCRIPT_EXTENSIONS and char == "/":
|
|
361
|
+
regex_end = javascript_regex_end(text, index)
|
|
362
|
+
if regex_end is not None:
|
|
363
|
+
masked.extend(" " * (regex_end - index))
|
|
364
|
+
index = regex_end
|
|
365
|
+
continue
|
|
366
|
+
if char == "/" and next_char == "/":
|
|
367
|
+
masked.extend((" ", " "))
|
|
368
|
+
in_line_comment = True
|
|
369
|
+
index += 2
|
|
370
|
+
continue
|
|
371
|
+
if char == "/" and next_char == "*":
|
|
372
|
+
masked.extend((" ", " "))
|
|
373
|
+
block_comment_depth = 1
|
|
374
|
+
index += 2
|
|
375
|
+
continue
|
|
376
|
+
if char == "#" and suffix == ".php" and next_char != "[":
|
|
377
|
+
masked.append(" ")
|
|
378
|
+
in_line_comment = True
|
|
379
|
+
index += 1
|
|
380
|
+
continue
|
|
381
|
+
if char in {'"', "'", "`"}:
|
|
382
|
+
quote = char
|
|
383
|
+
masked.append(" ")
|
|
384
|
+
index += 1
|
|
385
|
+
continue
|
|
386
|
+
masked.append(char)
|
|
387
|
+
index += 1
|
|
388
|
+
return "".join(masked)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def match_type_declaration(line: str, suffix: str) -> tuple[str, str] | None:
|
|
392
|
+
match: re.Match[str] | None = None
|
|
393
|
+
if suffix in JAVASCRIPT_EXTENSIONS:
|
|
394
|
+
match = re.match(
|
|
395
|
+
r"^(?:(?:export|default|declare|abstract)\s+)*(class|interface|enum|namespace)\s+"
|
|
396
|
+
r"([A-Za-z_$][\w$]*)",
|
|
397
|
+
line,
|
|
398
|
+
)
|
|
399
|
+
elif suffix == ".java":
|
|
400
|
+
match = re.search(r"\b(class|interface|enum|record|@interface)\s+([A-Za-z_$][\w$]*)", line)
|
|
401
|
+
elif suffix == ".cs":
|
|
402
|
+
match = re.search(
|
|
403
|
+
r"\b(class|interface|struct|enum|record(?:\s+(?:class|struct))?)\s+"
|
|
404
|
+
r"(@?[A-Za-z_]\w*)",
|
|
405
|
+
line,
|
|
406
|
+
)
|
|
407
|
+
elif suffix == ".php":
|
|
408
|
+
match = re.match(
|
|
409
|
+
r"^(?:(?:abstract|final|readonly)\s+)*(class|interface|trait|enum)\s+"
|
|
410
|
+
r"([A-Za-z_]\w*)",
|
|
411
|
+
line,
|
|
412
|
+
)
|
|
413
|
+
elif suffix == ".go":
|
|
414
|
+
match = re.match(r"^type\s+([A-Za-z_]\w*)\s+(struct|interface)\b", line)
|
|
415
|
+
if match:
|
|
416
|
+
return match.group(2), match.group(1)
|
|
417
|
+
elif suffix in {".kt", ".kts"}:
|
|
418
|
+
companion_match = re.search(r"\bcompanion\s+object(?:\s+([A-Za-z_]\w*))?", line)
|
|
419
|
+
if companion_match:
|
|
420
|
+
return "object", companion_match.group(1) or "Companion"
|
|
421
|
+
match = re.search(
|
|
422
|
+
r"\b((?:(?:data|enum|sealed|annotation|value)\s+)?(?:class|interface|object))\s+"
|
|
423
|
+
r"([A-Za-z_]\w*)",
|
|
424
|
+
line,
|
|
425
|
+
)
|
|
426
|
+
elif suffix == ".scala":
|
|
427
|
+
match = re.match(
|
|
428
|
+
r"^(?:(?:case|sealed|abstract)\s+)*(class|trait|object|enum)\s+([A-Za-z_]\w*)", line
|
|
429
|
+
)
|
|
430
|
+
elif suffix == ".rs":
|
|
431
|
+
match = re.search(r"\b(struct|enum|trait|union)\s+([A-Za-z_]\w*)", line)
|
|
432
|
+
if not match:
|
|
433
|
+
impl_match = re.search(
|
|
434
|
+
r"\bimpl(?:\s*<[^>{}]*>)?\s+(?:[\w:<>]+\s+for\s+)?([A-Za-z_]\w*)",
|
|
435
|
+
line,
|
|
436
|
+
)
|
|
437
|
+
if impl_match:
|
|
438
|
+
return "impl", impl_match.group(1)
|
|
439
|
+
elif suffix == ".swift":
|
|
440
|
+
match = re.search(r"\b(class|struct|enum|protocol|actor|extension)\s+([A-Za-z_]\w*)", line)
|
|
441
|
+
elif suffix == ".dart":
|
|
442
|
+
match = re.search(r"\b(class|mixin|enum|extension)\s+([A-Za-z_]\w*)", line)
|
|
443
|
+
elif suffix in {".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".mm"}:
|
|
444
|
+
match = re.search(
|
|
445
|
+
r"\b(class|struct|union|enum(?:\s+class)?)\s+([A-Za-z_]\w*)",
|
|
446
|
+
line,
|
|
447
|
+
)
|
|
448
|
+
if not match:
|
|
449
|
+
return None
|
|
450
|
+
return match.group(1), match.group(2)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
CONTROL_NAMES = {
|
|
454
|
+
"assert",
|
|
455
|
+
"catch",
|
|
456
|
+
"for",
|
|
457
|
+
"foreach",
|
|
458
|
+
"if",
|
|
459
|
+
"lock",
|
|
460
|
+
"new",
|
|
461
|
+
"return",
|
|
462
|
+
"sizeof",
|
|
463
|
+
"static_assert",
|
|
464
|
+
"switch",
|
|
465
|
+
"synchronized",
|
|
466
|
+
"throw",
|
|
467
|
+
"typeof",
|
|
468
|
+
"while",
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def match_java_like_function(
|
|
473
|
+
line: str, suffix: str, type_name: str | None
|
|
474
|
+
) -> tuple[str, str] | None:
|
|
475
|
+
paren = line.find("(")
|
|
476
|
+
if paren < 0:
|
|
477
|
+
return None
|
|
478
|
+
before = line[:paren].rstrip()
|
|
479
|
+
if not before or "=" in before or before.startswith("#"):
|
|
480
|
+
return None
|
|
481
|
+
before = re.sub(r"<[^<>]*>$", "", before).rstrip()
|
|
482
|
+
name_match = re.search(
|
|
483
|
+
r"(~?[A-Za-z_$@][\w$@]*(?:(?:::|\.)~?[A-Za-z_$@][\w$@]*)?)$",
|
|
484
|
+
before,
|
|
485
|
+
)
|
|
486
|
+
if not name_match:
|
|
487
|
+
return None
|
|
488
|
+
qualified_name = name_match.group(1)
|
|
489
|
+
name = re.split(r"::|\.", qualified_name)[-1].lstrip("~")
|
|
490
|
+
if name in CONTROL_NAMES:
|
|
491
|
+
return None
|
|
492
|
+
prefix = before[: name_match.start()].strip()
|
|
493
|
+
if not prefix and name != (type_name or ""):
|
|
494
|
+
return None
|
|
495
|
+
if suffix in {".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".mm"} and prefix.startswith(
|
|
496
|
+
("typedef", "using")
|
|
497
|
+
):
|
|
498
|
+
return None
|
|
499
|
+
if (
|
|
500
|
+
suffix in {".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".mm"}
|
|
501
|
+
and type_name is None
|
|
502
|
+
and line.endswith(";")
|
|
503
|
+
and not re.search(
|
|
504
|
+
r"(?:\b(?:auto|bool|char|consteval|constexpr|double|extern|float|inline|int|long|"
|
|
505
|
+
r"short|signed|static|unsigned|void)\b|[*&:]|<)",
|
|
506
|
+
prefix,
|
|
507
|
+
)
|
|
508
|
+
):
|
|
509
|
+
return None
|
|
510
|
+
return ("method" if type_name else "function"), qualified_name
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def match_function_declaration(
|
|
514
|
+
line: str, suffix: str, type_name: str | None
|
|
515
|
+
) -> tuple[str, str] | None:
|
|
516
|
+
if suffix in JAVASCRIPT_EXTENSIONS:
|
|
517
|
+
function_match = re.match(
|
|
518
|
+
r"^(?:(?:export|default|declare)\s+)*(async\s+)?function\s*\*?\s*"
|
|
519
|
+
r"([A-Za-z_$][\w$]*)\s*(?:<[^>{}]*>)?\s*\(",
|
|
520
|
+
line,
|
|
521
|
+
)
|
|
522
|
+
if function_match:
|
|
523
|
+
kind = "async function" if function_match.group(1) else "function"
|
|
524
|
+
return kind, function_match.group(2)
|
|
525
|
+
arrow_match = re.match(
|
|
526
|
+
r"^(?:(?:export|declare)\s+)*(?:const|let|var)\s+([A-Za-z_$][\w$]*)"
|
|
527
|
+
r"\s*(?::[^=]+)?=\s*(?:async\s+)?(?:function\b|\([^)]*\)\s*(?::[^=]+)?=>|"
|
|
528
|
+
r"[A-Za-z_$][\w$]*\s*=>)",
|
|
529
|
+
line,
|
|
530
|
+
)
|
|
531
|
+
if arrow_match:
|
|
532
|
+
return "function", arrow_match.group(1)
|
|
533
|
+
if type_name:
|
|
534
|
+
field_arrow_match = re.match(
|
|
535
|
+
r"^(?:(?:public|private|protected|static|abstract|override|readonly|declare|"
|
|
536
|
+
r"accessor)\s+)*(#?[A-Za-z_$][\w$]*)(?:[?!])?\s*(?::[^=]+)?=\s*"
|
|
537
|
+
r"(?:async\s+)?(?:\([^)]*\)\s*(?::[^=]+)?=>|[A-Za-z_$][\w$]*\s*=>)",
|
|
538
|
+
line,
|
|
539
|
+
)
|
|
540
|
+
if field_arrow_match:
|
|
541
|
+
return "method", field_arrow_match.group(1)
|
|
542
|
+
method_match = re.match(
|
|
543
|
+
r"^(?:(?:public|private|protected|static|abstract|async|override|readonly|"
|
|
544
|
+
r"declare|get|set)\s+)*(#?[A-Za-z_$][\w$]*|constructor)\s*"
|
|
545
|
+
r"(?:<[^>{}]*>)?\s*\(",
|
|
546
|
+
line,
|
|
547
|
+
)
|
|
548
|
+
if method_match and method_match.group(1) not in CONTROL_NAMES:
|
|
549
|
+
return "method", method_match.group(1)
|
|
550
|
+
return None
|
|
551
|
+
if suffix == ".php":
|
|
552
|
+
match = re.match(
|
|
553
|
+
r"^(?:(?:public|protected|private|static|final|abstract|readonly)\s+)*"
|
|
554
|
+
r"function\s*&?\s*([A-Za-z_]\w*)\s*\(",
|
|
555
|
+
line,
|
|
556
|
+
)
|
|
557
|
+
if match:
|
|
558
|
+
return ("method" if type_name else "function"), match.group(1)
|
|
559
|
+
return None
|
|
560
|
+
if suffix == ".go":
|
|
561
|
+
match = re.match(
|
|
562
|
+
r"^func\s+(?:\(([^)]*)\)\s*)?([A-Za-z_]\w*)\s*(?:\[[^]]+\]\s*)?\(",
|
|
563
|
+
line,
|
|
564
|
+
)
|
|
565
|
+
if match:
|
|
566
|
+
receiver = match.group(1)
|
|
567
|
+
name = match.group(2)
|
|
568
|
+
if receiver:
|
|
569
|
+
receiver_match = re.search(r"\*?([A-Za-z_]\w*)(?:\[[^]]*])?\s*$", receiver)
|
|
570
|
+
if receiver_match:
|
|
571
|
+
return "method", f"{receiver_match.group(1)}.{name}"
|
|
572
|
+
return "method", name
|
|
573
|
+
return ("method" if type_name else "function"), name
|
|
574
|
+
return None
|
|
575
|
+
if suffix in {".kt", ".kts"}:
|
|
576
|
+
match = re.search(r"\bfun\s+(?:<[^>{}]*>\s*)?(?:[\w.<>?]+\.)?([A-Za-z_]\w*)\s*\(", line)
|
|
577
|
+
if match:
|
|
578
|
+
return ("method" if type_name else "function"), match.group(1)
|
|
579
|
+
return None
|
|
580
|
+
if suffix == ".scala":
|
|
581
|
+
match = re.search(r"\bdef\s+([A-Za-z_]\w*)\b", line)
|
|
582
|
+
if match:
|
|
583
|
+
return ("method" if type_name else "function"), match.group(1)
|
|
584
|
+
return None
|
|
585
|
+
if suffix == ".rs":
|
|
586
|
+
match = re.search(
|
|
587
|
+
r"\b(?:async\s+)?(?:unsafe\s+)?(?:extern\s+\"[^\"]+\"\s+)?(?:const\s+)?"
|
|
588
|
+
r"fn\s+([A-Za-z_]\w*)",
|
|
589
|
+
line,
|
|
590
|
+
)
|
|
591
|
+
if match:
|
|
592
|
+
return ("method" if type_name else "function"), match.group(1)
|
|
593
|
+
return None
|
|
594
|
+
if suffix == ".swift":
|
|
595
|
+
match = re.search(r"\bfunc\s+([A-Za-z_]\w*)\s*(?:<[^>{}]*>)?\s*\(", line)
|
|
596
|
+
if match:
|
|
597
|
+
return ("method" if type_name else "function"), match.group(1)
|
|
598
|
+
if type_name and re.search(r"\binit\s*\(", line):
|
|
599
|
+
return "method", "init"
|
|
600
|
+
return None
|
|
601
|
+
if suffix == ".dart":
|
|
602
|
+
return match_java_like_function(line, suffix, type_name)
|
|
603
|
+
if suffix in JAVA_LIKE_EXTENSIONS:
|
|
604
|
+
return match_java_like_function(line, suffix, type_name)
|
|
605
|
+
return None
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def is_annotation_line(line: str) -> bool:
|
|
609
|
+
return bool(
|
|
610
|
+
re.match(r"^@[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:\(.*\))?$", line)
|
|
611
|
+
or re.match(r"^\[[A-Za-z_][^]]*]$", line)
|
|
612
|
+
or re.match(r"^#\[[A-Za-z_][^]]*]$", line)
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
LEADING_ANNOTATION_RE = re.compile(
|
|
617
|
+
r"^(?:@[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*(?:\([^)]*\))?|"
|
|
618
|
+
r"\[[A-Za-z_][^]]*]|#\[[A-Za-z_][^]]*])\s*"
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def strip_leading_annotations(original: str, masked: str) -> tuple[str, str, list[str]]:
|
|
623
|
+
annotations: list[str] = []
|
|
624
|
+
if masked.startswith("@interface "):
|
|
625
|
+
return original, masked, annotations
|
|
626
|
+
while masked_match := LEADING_ANNOTATION_RE.match(masked):
|
|
627
|
+
original_match = LEADING_ANNOTATION_RE.match(original)
|
|
628
|
+
if original_match:
|
|
629
|
+
annotations.append(compact_preview_line(original_match.group(0)))
|
|
630
|
+
original = original[original_match.end() :].lstrip()
|
|
631
|
+
masked = masked[masked_match.end() :].lstrip()
|
|
632
|
+
return original, masked, annotations
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def brace_language_outline(text: str, suffix: str) -> list[str]:
|
|
636
|
+
original_lines = text.splitlines()
|
|
637
|
+
masked_lines = mask_c_style_source(text, suffix).splitlines()
|
|
638
|
+
outline: list[str] = []
|
|
639
|
+
seen: set[str] = set()
|
|
640
|
+
type_stack: list[tuple[str, int]] = []
|
|
641
|
+
function_depths: list[int] = []
|
|
642
|
+
blocked_depths: list[int] = []
|
|
643
|
+
pending_type: str | None = None
|
|
644
|
+
pending_function = False
|
|
645
|
+
pending_annotations: list[str] = []
|
|
646
|
+
depth = 0
|
|
647
|
+
|
|
648
|
+
def add(value: str) -> None:
|
|
649
|
+
decorated = f"{' '.join(pending_annotations[-2:])} {value}".strip()
|
|
650
|
+
if decorated not in seen:
|
|
651
|
+
seen.add(decorated)
|
|
652
|
+
outline.append(decorated)
|
|
653
|
+
|
|
654
|
+
def next_lines_open_body(line_index: int) -> bool:
|
|
655
|
+
inspected = 0
|
|
656
|
+
for future_index in range(line_index + 1, len(masked_lines)):
|
|
657
|
+
future = masked_lines[future_index]
|
|
658
|
+
future_line = compact_preview_line(future)
|
|
659
|
+
if not future_line:
|
|
660
|
+
continue
|
|
661
|
+
inspected += 1
|
|
662
|
+
if "{" in future_line:
|
|
663
|
+
return True
|
|
664
|
+
if "}" in future_line or future_line.endswith(";") or inspected >= 6:
|
|
665
|
+
return False
|
|
666
|
+
return False
|
|
667
|
+
|
|
668
|
+
for line_index, (original, masked) in enumerate(
|
|
669
|
+
zip(original_lines, masked_lines, strict=False)
|
|
670
|
+
):
|
|
671
|
+
while function_depths and depth < function_depths[-1]:
|
|
672
|
+
function_depths.pop()
|
|
673
|
+
while type_stack and depth < type_stack[-1][1]:
|
|
674
|
+
type_stack.pop()
|
|
675
|
+
while blocked_depths and depth < blocked_depths[-1]:
|
|
676
|
+
blocked_depths.pop()
|
|
677
|
+
|
|
678
|
+
masked_line = compact_preview_line(masked)
|
|
679
|
+
original_line = compact_preview_line(original)
|
|
680
|
+
opens = masked.count("{")
|
|
681
|
+
closes = masked.count("}")
|
|
682
|
+
|
|
683
|
+
opened_pending_scope = False
|
|
684
|
+
if pending_type and "{" in masked:
|
|
685
|
+
type_stack.append((pending_type, depth + 1))
|
|
686
|
+
pending_type = None
|
|
687
|
+
opened_pending_scope = True
|
|
688
|
+
if pending_function and "{" in masked:
|
|
689
|
+
function_depths.append(depth + 1)
|
|
690
|
+
pending_function = False
|
|
691
|
+
opened_pending_scope = True
|
|
692
|
+
|
|
693
|
+
if (
|
|
694
|
+
masked_line
|
|
695
|
+
and original_line
|
|
696
|
+
and is_annotation_line(original_line)
|
|
697
|
+
and not function_depths
|
|
698
|
+
and not blocked_depths
|
|
699
|
+
):
|
|
700
|
+
pending_annotations.append(original_line)
|
|
701
|
+
elif masked_line and not function_depths and not blocked_depths:
|
|
702
|
+
_, masked_line, inline_annotations = strip_leading_annotations(
|
|
703
|
+
original_line, masked_line
|
|
704
|
+
)
|
|
705
|
+
pending_annotations.extend(inline_annotations)
|
|
706
|
+
current_type = type_stack[-1][0] if type_stack else None
|
|
707
|
+
direct_type_body = not type_stack or depth == type_stack[-1][1]
|
|
708
|
+
type_match = match_type_declaration(masked_line, suffix) if direct_type_body else None
|
|
709
|
+
if type_match:
|
|
710
|
+
kind, name = type_match
|
|
711
|
+
if (
|
|
712
|
+
current_type
|
|
713
|
+
and suffix in {".kt", ".kts"}
|
|
714
|
+
and kind == "object"
|
|
715
|
+
and name == "Companion"
|
|
716
|
+
):
|
|
717
|
+
name = f"{current_type}.{name}"
|
|
718
|
+
add(f"{kind} {name}")
|
|
719
|
+
pending_annotations.clear()
|
|
720
|
+
if "{" in masked:
|
|
721
|
+
type_stack.append((name, depth + 1))
|
|
722
|
+
elif not masked_line.endswith(";") and next_lines_open_body(line_index):
|
|
723
|
+
pending_type = name
|
|
724
|
+
elif direct_type_body:
|
|
725
|
+
function_match = match_function_declaration(masked_line, suffix, current_type)
|
|
726
|
+
if function_match:
|
|
727
|
+
kind, name = function_match
|
|
728
|
+
qualified_name = (
|
|
729
|
+
f"{current_type}.{name}"
|
|
730
|
+
if current_type and "." not in name and "::" not in name
|
|
731
|
+
else name
|
|
732
|
+
)
|
|
733
|
+
add(f"{kind} {qualified_name}")
|
|
734
|
+
pending_annotations.clear()
|
|
735
|
+
if "{" in masked:
|
|
736
|
+
function_depths.append(depth + 1)
|
|
737
|
+
elif (
|
|
738
|
+
not masked_line.endswith(";")
|
|
739
|
+
and "=>" not in masked_line
|
|
740
|
+
and "=" not in masked_line
|
|
741
|
+
and next_lines_open_body(line_index)
|
|
742
|
+
):
|
|
743
|
+
pending_function = True
|
|
744
|
+
else:
|
|
745
|
+
pending_annotations.clear()
|
|
746
|
+
is_transparent_container = bool(
|
|
747
|
+
re.match(
|
|
748
|
+
r"^(?:(?:export|inline)\s+)?namespace\b|"
|
|
749
|
+
r"^(?:pub\s+)?mod\b|^(?:declare\s+)?module\b|"
|
|
750
|
+
r"^(?:unsafe\s+)?extern\b",
|
|
751
|
+
masked_line,
|
|
752
|
+
)
|
|
753
|
+
)
|
|
754
|
+
if opens and not opened_pending_scope and not is_transparent_container:
|
|
755
|
+
blocked_depths.append(depth + 1)
|
|
756
|
+
|
|
757
|
+
depth = max(0, depth + opens - closes)
|
|
758
|
+
while function_depths and depth < function_depths[-1]:
|
|
759
|
+
function_depths.pop()
|
|
760
|
+
while type_stack and depth < type_stack[-1][1]:
|
|
761
|
+
type_stack.pop()
|
|
762
|
+
while blocked_depths and depth < blocked_depths[-1]:
|
|
763
|
+
blocked_depths.pop()
|
|
764
|
+
return outline
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def ruby_outline(text: str) -> list[str]:
|
|
768
|
+
outline: list[str] = []
|
|
769
|
+
seen: set[str] = set()
|
|
770
|
+
type_stack: list[tuple[int, str]] = []
|
|
771
|
+
function_indents: list[int] = []
|
|
772
|
+
|
|
773
|
+
def add(value: str) -> None:
|
|
774
|
+
if value not in seen:
|
|
775
|
+
seen.add(value)
|
|
776
|
+
outline.append(value)
|
|
777
|
+
|
|
778
|
+
for raw_line in text.splitlines():
|
|
779
|
+
stripped = raw_line.strip()
|
|
780
|
+
if not stripped or stripped.startswith("#"):
|
|
781
|
+
continue
|
|
782
|
+
indent = len(raw_line.expandtabs(4)) - len(raw_line.lstrip().expandtabs(4))
|
|
783
|
+
while function_indents and indent <= function_indents[-1]:
|
|
784
|
+
function_indents.pop()
|
|
785
|
+
while type_stack and indent <= type_stack[-1][0]:
|
|
786
|
+
type_stack.pop()
|
|
787
|
+
|
|
788
|
+
type_match = re.match(r"^(class|module)\s+([A-Z]\w*(?:::[A-Z]\w*)*)", stripped)
|
|
789
|
+
if type_match and not function_indents:
|
|
790
|
+
kind, name = type_match.groups()
|
|
791
|
+
add(f"{kind} {name}")
|
|
792
|
+
type_stack.append((indent, name))
|
|
793
|
+
continue
|
|
794
|
+
|
|
795
|
+
function_match = re.match(r"^def\s+(?:self\.)?([^\s(]+)", stripped)
|
|
796
|
+
if function_match and not function_indents:
|
|
797
|
+
name = function_match.group(1)
|
|
798
|
+
if type_stack:
|
|
799
|
+
add(f"method {type_stack[-1][1]}.{name}")
|
|
800
|
+
else:
|
|
801
|
+
add(f"function {name}")
|
|
802
|
+
function_indents.append(indent)
|
|
803
|
+
return outline
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def simple_language_outline(text: str, suffix: str) -> list[str]:
|
|
807
|
+
outline: list[str] = []
|
|
808
|
+
seen: set[str] = set()
|
|
809
|
+
|
|
810
|
+
def add(value: str) -> None:
|
|
811
|
+
if value not in seen:
|
|
812
|
+
seen.add(value)
|
|
813
|
+
outline.append(value)
|
|
814
|
+
|
|
815
|
+
for raw_line in text.splitlines():
|
|
816
|
+
line = compact_preview_line(raw_line)
|
|
817
|
+
if not line:
|
|
818
|
+
continue
|
|
819
|
+
match: re.Match[str] | None = None
|
|
820
|
+
if suffix == ".py":
|
|
821
|
+
match = re.match(r"^(async\s+)?def\s+([A-Za-z_]\w*)\s*\(", line)
|
|
822
|
+
if match:
|
|
823
|
+
add(f"{'async function' if match.group(1) else 'function'} {match.group(2)}")
|
|
824
|
+
continue
|
|
825
|
+
match = re.match(r"^class\s+([A-Za-z_]\w*)", line)
|
|
826
|
+
if match:
|
|
827
|
+
add(f"class {match.group(1)}")
|
|
828
|
+
elif suffix == ".rb":
|
|
829
|
+
match = re.match(r"^(class|module)\s+([A-Z]\w*(?:::[A-Z]\w*)*)", line)
|
|
830
|
+
if match:
|
|
831
|
+
add(f"{match.group(1)} {match.group(2)}")
|
|
832
|
+
continue
|
|
833
|
+
match = re.match(r"^def\s+(?:self\.)?([^\s(]+)", line)
|
|
834
|
+
if match:
|
|
835
|
+
add(f"function {match.group(1)}")
|
|
836
|
+
elif suffix in {".ex", ".exs"}:
|
|
837
|
+
match = re.match(r"^defmodule\s+([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", line)
|
|
838
|
+
if match:
|
|
839
|
+
add(f"module {match.group(1)}")
|
|
840
|
+
continue
|
|
841
|
+
match = re.match(r"^(?:def|defp|defmacro)\s+([A-Za-z_]\w*[!?]?)", line)
|
|
842
|
+
if match:
|
|
843
|
+
add(f"function {match.group(1)}")
|
|
844
|
+
elif suffix == ".clj":
|
|
845
|
+
match = re.match(
|
|
846
|
+
r"^\((defn|defmacro|defprotocol|defrecord|deftype|defmulti|defmethod)\s+([^\s)]+)",
|
|
847
|
+
line,
|
|
848
|
+
)
|
|
849
|
+
if match:
|
|
850
|
+
add(f"{match.group(1)} {match.group(2)}")
|
|
851
|
+
elif suffix == ".lua":
|
|
852
|
+
match = re.match(r"^(?:local\s+)?function\s+([A-Za-z_]\w*(?:[.:][A-Za-z_]\w*)*)", line)
|
|
853
|
+
if match:
|
|
854
|
+
add(f"function {match.group(1)}")
|
|
855
|
+
elif suffix == ".sh":
|
|
856
|
+
match = re.match(r"^(?:function\s+)?([A-Za-z_]\w*)\s*\(\s*\)\s*\{?", line)
|
|
857
|
+
if match:
|
|
858
|
+
add(f"function {match.group(1)}")
|
|
859
|
+
elif suffix == ".hs":
|
|
860
|
+
match = re.match(r"^(data|newtype|type|class)\s+([A-Z]\w*)", line)
|
|
861
|
+
if match:
|
|
862
|
+
add(f"{match.group(1)} {match.group(2)}")
|
|
863
|
+
continue
|
|
864
|
+
match = re.match(r"^([a-z_]\w*)\s*::", line)
|
|
865
|
+
if match:
|
|
866
|
+
add(f"function {match.group(1)}")
|
|
867
|
+
elif suffix == ".proto":
|
|
868
|
+
match = re.match(r"^(message|service|enum)\s+([A-Za-z_]\w*)", line)
|
|
869
|
+
if match:
|
|
870
|
+
add(f"{match.group(1)} {match.group(2)}")
|
|
871
|
+
continue
|
|
872
|
+
match = re.match(r"^rpc\s+([A-Za-z_]\w*)\s*\(", line)
|
|
873
|
+
if match:
|
|
874
|
+
add(f"rpc {match.group(1)}")
|
|
875
|
+
elif suffix == ".graphql":
|
|
876
|
+
match = re.match(
|
|
877
|
+
r"^(type|interface|input|enum|union|scalar|directive)\s+([A-Za-z_]\w*)",
|
|
878
|
+
line,
|
|
879
|
+
)
|
|
880
|
+
if match:
|
|
881
|
+
add(f"{match.group(1)} {match.group(2)}")
|
|
882
|
+
continue
|
|
883
|
+
match = re.match(r"^(query|mutation|subscription)\s+([A-Za-z_]\w*)", line)
|
|
884
|
+
if match:
|
|
885
|
+
add(f"{match.group(1)} {match.group(2)}")
|
|
886
|
+
elif suffix == ".sql":
|
|
887
|
+
match = re.match(
|
|
888
|
+
r"^CREATE\s+(?:OR\s+REPLACE\s+)?(FUNCTION|PROCEDURE|TABLE|VIEW|TRIGGER)\s+"
|
|
889
|
+
r"([^\s(;]+)",
|
|
890
|
+
line,
|
|
891
|
+
flags=re.IGNORECASE,
|
|
892
|
+
)
|
|
893
|
+
if match:
|
|
894
|
+
add(f"{match.group(1).lower()} {match.group(2)}")
|
|
895
|
+
elif suffix in {".yaml", ".yml"}:
|
|
896
|
+
match = re.match(r"^([A-Za-z0-9_.-]+):(?:\s|$)", raw_line)
|
|
897
|
+
if match:
|
|
898
|
+
add(f"key {match.group(1)}")
|
|
899
|
+
elif suffix in {".toml", ".cfg"}:
|
|
900
|
+
match = re.match(r"^\[([^]]+)]$", line)
|
|
901
|
+
if match:
|
|
902
|
+
add(f"section {match.group(1)}")
|
|
903
|
+
return outline
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def json_outline(text: str) -> list[str]:
|
|
907
|
+
try:
|
|
908
|
+
parsed = json.loads(text)
|
|
909
|
+
except (json.JSONDecodeError, RecursionError, MemoryError):
|
|
910
|
+
return []
|
|
911
|
+
if not isinstance(parsed, dict):
|
|
912
|
+
return []
|
|
913
|
+
outline: list[str] = []
|
|
914
|
+
for key, value in parsed.items():
|
|
915
|
+
if isinstance(value, dict) and value:
|
|
916
|
+
children = ", ".join(str(child) for child in value)
|
|
917
|
+
outline.append(f"key {key} [{children}]")
|
|
918
|
+
else:
|
|
919
|
+
outline.append(f"key {key}")
|
|
920
|
+
return outline
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def structural_outline(path: Path, text: str) -> list[str]:
|
|
924
|
+
suffix = path.suffix.lower()
|
|
925
|
+
if suffix == ".py":
|
|
926
|
+
return python_outline(text) or simple_language_outline(text, suffix)
|
|
927
|
+
if suffix == ".rb":
|
|
928
|
+
return ruby_outline(text)
|
|
929
|
+
if suffix in BRACE_LANGUAGE_EXTENSIONS:
|
|
930
|
+
return brace_language_outline(text, suffix)
|
|
931
|
+
if suffix == ".json":
|
|
932
|
+
return json_outline(text)
|
|
933
|
+
return simple_language_outline(text, suffix)
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def preview_for(
|
|
937
|
+
path: Path, preview_bytes: int, *, max_read_bytes: int | None = None
|
|
938
|
+
) -> tuple[str, bool]:
|
|
939
|
+
try:
|
|
940
|
+
with path.open("rb") as source:
|
|
941
|
+
sample = source.read(4096)
|
|
942
|
+
if is_binary_sample(sample):
|
|
943
|
+
return "", True
|
|
944
|
+
remaining = (
|
|
945
|
+
source.read()
|
|
946
|
+
if max_read_bytes is None
|
|
947
|
+
else source.read(max(0, max_read_bytes - len(sample)))
|
|
948
|
+
)
|
|
949
|
+
data = sample + remaining
|
|
950
|
+
except OSError:
|
|
951
|
+
return "", True
|
|
952
|
+
if is_binary_sample(data):
|
|
953
|
+
return "", True
|
|
954
|
+
text = data.decode("utf-8", errors="ignore")
|
|
955
|
+
outline = structural_outline(path, text)
|
|
956
|
+
preview_lines = select_preview_lines(outline or text.splitlines())
|
|
957
|
+
return fit_preview_lines(preview_lines, preview_bytes), False
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def main() -> None:
|
|
961
|
+
parser = argparse.ArgumentParser(
|
|
962
|
+
description="Build structural or sampled previews for Codex Security rank inputs."
|
|
963
|
+
)
|
|
964
|
+
parser.parse_args()
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
if __name__ == "__main__":
|
|
968
|
+
main()
|