@jenga-ai/agent 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/agents/developer.md +82 -2
- package/agents/scrum-master.md +140 -21
- package/agents/tester.md +90 -8
- package/hooks/on_session_end.sh +171 -20
- package/package.json +1 -1
- package/scripts/check-permission-level.sh +107 -0
- package/scripts/check-publicignore-match.sh +122 -0
- package/scripts/check-worktree-liveness.sh +193 -0
- package/scripts/generate-rapport-manifest.sh +43 -0
- package/scripts/idea_manager.sh +47 -0
- package/scripts/install-worktree-commit-guard.sh +134 -0
- package/scripts/jenga-permission-level-switch.sh +109 -0
- package/scripts/smoke-harness.sh +139 -0
- package/scripts/validate-board.sh +62 -0
- package/scripts/with-lock.sh +158 -0
- package/scripts/worktree-remove-guard.sh +204 -0
- package/skills/clearify/SKILL.md +52 -0
- package/skills/commit/SKILL.md +13 -4
- package/skills/distribute/CONFIG_SCHEMA.md +60 -2
- package/skills/do/SKILL.md +48 -11
- package/skills/doc-sync/SKILL.md +16 -0
- package/skills/doc-sync/assets/doc_targets.md +11 -0
- package/skills/idea/SKILL.md +56 -0
- package/skills/idea/assets/idea_handoff_template.md +26 -0
- package/skills/idea/assets/idea_template.md +3 -0
- package/skills/init/SKILL.md +100 -7
- package/skills/init/assets/directory_structure.txt +1 -0
- package/skills/init/assets/workflow_template.json +1 -1
- package/skills/init/scripts/apply-project-visibility.sh +176 -0
- package/skills/init/scripts/detect-existing-codebase.sh +166 -0
- package/skills/init/scripts/init.sh +30 -1
- package/skills/jenga/SKILL.md +160 -17
- package/skills/jenga/scripts/board-scan.sh +238 -0
- package/skills/jenga/scripts/cascade-resolve.sh +297 -0
- package/skills/jenga/scripts/render-confirmation.sh +679 -0
- package/skills/jenga/scripts/render-picker.sh +439 -0
- package/skills/jenga/scripts/resolve-id.sh +367 -0
- package/skills/jenga-permission-level/SKILL.md +81 -0
- package/skills/proceed/SKILL.md +1 -1
- package/skills/publish/SKILL.md +8 -5
- package/skills/publish/assets/ci-contract.md +2 -2
- package/skills/publish/assets/ownership-matrix.md +1 -1
- package/skills/publish/scripts/finalize_changelog.sh +115 -0
- package/skills/publish/scripts/generate_release_notes.sh +475 -28
- package/skills/publish/scripts/npm_ci_pipeline.sh +44 -6
- package/skills/publish/scripts/publish_deploy.sh +38 -8
- package/skills/publish/scripts/run_gates.sh +2 -2
- package/skills/reconcile/SKILL.md +117 -5
- package/skills/reconcile/scripts/detect-unlinked-code.sh +741 -0
- package/skills/skillify/assets/init-new/assets/directory_structure.txt +5 -1
- package/skills/spinoff/SKILL.md +12 -7
- package/skills/todo/SKILL.md +2 -0
- package/skills/uncharted/SKILL.md +711 -0
- package/skills/uncharted/assets/SEGMENT_PROPOSAL_TEMPLATE.md +129 -0
- package/skills/uncharted/assets/UNDERSTANDING_DOC_TEMPLATE.md +160 -0
- package/skills/uncharted/scripts/apply-subsystem-cap.sh +573 -0
- package/skills/uncharted/scripts/detect-dependencies.sh +732 -0
- package/skills/uncharted/scripts/detect-tests.sh +553 -0
- package/skills/uncharted/scripts/discover-subsystems.sh +1029 -0
- package/skills/uncharted/scripts/enumerate-target.sh +470 -0
- package/skills/uncharted/scripts/import-source.sh +517 -0
- package/skills/uncharted/scripts/inspect-provenance.sh +573 -0
- package/skills/uncharted/scripts/resolve-segment-target.sh +640 -0
- package/skills/uncharted/scripts/run-engine.sh +655 -0
- package/skills/uncharted/scripts/validate-proposed-items.sh +125 -0
- package/skills/uncharted/scripts/write-backfilled-epics.sh +498 -0
- package/skills/wtf/SKILL.md +20 -0
- package/templates/CHANGELOG_TEMPLATE.md +13 -0
- package/templates/PROBLEM_RAPPORT_TEMPLATE.md +4 -1
- package/templates/SCRUM_BOARD_SCHEMA.md +157 -10
- package/templates/permission-levels/README.md +73 -0
- package/templates/permission-levels/level-1-locked.json +71 -0
- package/templates/permission-levels/level-2-guarded.json +64 -0
- package/templates/permission-levels/level-3-standard.json +62 -0
- package/templates/permission-levels/level-4-elevated.json +60 -0
- package/templates/permission-levels/level-5-unrestricted.json +58 -0
- package/skills/convert/SKILL.md +0 -124
- package/skills/convert/convert_cli.py +0 -235
- package/skills/convert/tests/sample.csv +0 -4
- package/skills/convert/tests/sample.json +0 -5
- package/skills/convert/tests/sample.jsonl +0 -3
- package/skills/convert/tests/sample.yaml +0 -18
- package/skills/convert/tests/sample_obj.csv +0 -2
- package/skills/convert/tests/sample_obj.json +0 -9
- package/skills/mirror-public/SKILL.md +0 -237
- package/skills/mirror-public/assets/config.json +0 -5
- package/skills/mirror-public/scripts/mirror.sh +0 -374
- package/skills/self-sync/SKILL.md +0 -73
- package/skills/self-sync/scripts/run.js +0 -136
- package/skills/strategy/SKILL.md +0 -312
|
@@ -0,0 +1,732 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# detect-dependencies.sh — dependency detection for the /uncharted investigative engine
|
|
3
|
+
#
|
|
4
|
+
# Usage: detect-dependencies.sh <target-path>
|
|
5
|
+
# detect-dependencies.sh --help
|
|
6
|
+
#
|
|
7
|
+
# Scans <target-path> (a file or a directory) for import/require/include statements
|
|
8
|
+
# and reports two buckets on stdout as JSON:
|
|
9
|
+
#
|
|
10
|
+
# internal — dependencies that resolve to a path inside the target's own repository
|
|
11
|
+
# external — third-party packages, stdlib modules, and notable external commands
|
|
12
|
+
#
|
|
13
|
+
# It additionally lists every dependency manifest found in, or in any directory above,
|
|
14
|
+
# the target (package.json, pyproject.toml, requirements.txt, go.mod, Cargo.toml,
|
|
15
|
+
# Gemfile, and friends) with a best-effort parse of the dependency names they declare,
|
|
16
|
+
# so the calling agent can reconcile grepped imports against declared dependencies.
|
|
17
|
+
#
|
|
18
|
+
# This script is STRICTLY READ-ONLY. It creates no files, no temp files, and mutates
|
|
19
|
+
# no git state. Its only outputs are JSON on stdout and diagnostics on stderr.
|
|
20
|
+
#
|
|
21
|
+
# ---------------------------------------------------------------------------
|
|
22
|
+
# Output contract (stdout, JSON) — consumed by skills/uncharted/scripts/run-engine.sh
|
|
23
|
+
# ---------------------------------------------------------------------------
|
|
24
|
+
# {
|
|
25
|
+
# "script": "detect-dependencies.sh",
|
|
26
|
+
# "version": 1,
|
|
27
|
+
# "target": "<repo-relative path to the target>",
|
|
28
|
+
# "target_absolute": "<absolute path to the target>",
|
|
29
|
+
# "target_type": "file" | "directory",
|
|
30
|
+
# "repo_root": "<absolute path to the repository root>",
|
|
31
|
+
# "files_scanned": <int>,
|
|
32
|
+
# "files_truncated": <bool>, // true when the walk cap was hit
|
|
33
|
+
# "unrecognised": <bool>, // true when no source language was recognised
|
|
34
|
+
# "languages": [ { "language": "shell", "files": 12 }, ... ],
|
|
35
|
+
# "unrecognised_extensions": [ { "extension": ".md", "files": 4 }, ... ],
|
|
36
|
+
# "counts": { "internal": <int>, "external": <int> },
|
|
37
|
+
# "internal": [
|
|
38
|
+
# {
|
|
39
|
+
# "raw": "<the specifier exactly as written in source>",
|
|
40
|
+
# "language": "<language of the referencing file>",
|
|
41
|
+
# "resolved": "<repo-relative path it resolves to, or null>",
|
|
42
|
+
# "occurrences": <int>,
|
|
43
|
+
# "sources": [ "<up to 3 repo-relative referencing files>" ]
|
|
44
|
+
# }
|
|
45
|
+
# ],
|
|
46
|
+
# "external": [
|
|
47
|
+
# {
|
|
48
|
+
# "name": "<package / module / command name>",
|
|
49
|
+
# "language": "<language of the referencing file>",
|
|
50
|
+
# "kind": "package" | "stdlib" | "command" | "unknown",
|
|
51
|
+
# "occurrences": <int>,
|
|
52
|
+
# "sources": [ "<up to 3 repo-relative referencing files>" ]
|
|
53
|
+
# }
|
|
54
|
+
# ],
|
|
55
|
+
# "manifests": [
|
|
56
|
+
# {
|
|
57
|
+
# "name": "package.json",
|
|
58
|
+
# "path": "<repo-relative path>",
|
|
59
|
+
# "location": "target" | "ancestor",
|
|
60
|
+
# "distance": <int>, // directory levels above the target (0 = in it)
|
|
61
|
+
# "declared_dependencies": [ "<name>", ... ] | null // null = not parsed / parse failed
|
|
62
|
+
# }
|
|
63
|
+
# ],
|
|
64
|
+
# "notices": [ "<non-fatal diagnostic>", ... ]
|
|
65
|
+
# }
|
|
66
|
+
#
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
# Behaviour notes
|
|
69
|
+
# ---------------------------------------------------------------------------
|
|
70
|
+
# - Unrecognised language: NOT an error. The script exits 0 with empty internal/external
|
|
71
|
+
# arrays, "unrecognised": true, and an explanatory entry in "notices".
|
|
72
|
+
# - Target outside a git repository: NOT an error. repo_root falls back to the target's
|
|
73
|
+
# own directory and a notice is emitted.
|
|
74
|
+
# - Regex-based extraction is deliberately approximate — this output feeds an agent's
|
|
75
|
+
# understanding document, not a build system. Every entry carries its raw specifier
|
|
76
|
+
# and example source files so the reader can verify.
|
|
77
|
+
#
|
|
78
|
+
# Exit codes:
|
|
79
|
+
# 0 — success (including every graceful-degradation path above)
|
|
80
|
+
# 1 — usage error (wrong number of arguments)
|
|
81
|
+
# 2 — target path does not exist
|
|
82
|
+
|
|
83
|
+
set -euo pipefail
|
|
84
|
+
|
|
85
|
+
usage() {
|
|
86
|
+
cat <<EOF
|
|
87
|
+
Usage: $(basename "$0") <target-path>
|
|
88
|
+
|
|
89
|
+
Scans a file or directory for imports/requires/includes and prints a JSON report
|
|
90
|
+
separating internal (in-repo) from external (third-party/stdlib) dependencies,
|
|
91
|
+
plus any dependency manifests found in or above the target.
|
|
92
|
+
|
|
93
|
+
Arguments:
|
|
94
|
+
<target-path> File or directory to analyse.
|
|
95
|
+
|
|
96
|
+
Options:
|
|
97
|
+
-h, --help Show this help and exit.
|
|
98
|
+
|
|
99
|
+
Examples:
|
|
100
|
+
$(basename "$0") skills/reconcile/
|
|
101
|
+
$(basename "$0") scripts/board_resolver.sh
|
|
102
|
+
EOF
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
# ---------------------------------------------------------------------------
|
|
106
|
+
# Argument validation
|
|
107
|
+
# ---------------------------------------------------------------------------
|
|
108
|
+
|
|
109
|
+
case "${1:-}" in
|
|
110
|
+
-h|--help)
|
|
111
|
+
usage
|
|
112
|
+
exit 0
|
|
113
|
+
;;
|
|
114
|
+
esac
|
|
115
|
+
|
|
116
|
+
if [[ $# -ne 1 ]]; then
|
|
117
|
+
echo "Error: expected exactly 1 argument, got $#" >&2
|
|
118
|
+
usage >&2
|
|
119
|
+
exit 1
|
|
120
|
+
fi
|
|
121
|
+
|
|
122
|
+
TARGET="$1"
|
|
123
|
+
|
|
124
|
+
if [[ ! -e "$TARGET" ]]; then
|
|
125
|
+
echo "Error: target path does not exist: $TARGET" >&2
|
|
126
|
+
exit 2
|
|
127
|
+
fi
|
|
128
|
+
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
# Resolve absolute target path and repository root
|
|
131
|
+
# ---------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
if [[ -d "$TARGET" ]]; then
|
|
134
|
+
TARGET_ABS=$(cd "$TARGET" && pwd)
|
|
135
|
+
TARGET_DIR="$TARGET_ABS"
|
|
136
|
+
else
|
|
137
|
+
TARGET_DIR=$(cd "$(dirname "$TARGET")" && pwd)
|
|
138
|
+
TARGET_ABS="$TARGET_DIR/$(basename "$TARGET")"
|
|
139
|
+
fi
|
|
140
|
+
|
|
141
|
+
REPO_ROOT=$(git -C "$TARGET_DIR" rev-parse --show-toplevel 2>/dev/null || true)
|
|
142
|
+
|
|
143
|
+
# ---------------------------------------------------------------------------
|
|
144
|
+
# Scan and emit
|
|
145
|
+
# ---------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
python3 - "$TARGET_ABS" "$REPO_ROOT" <<'PY'
|
|
148
|
+
import json
|
|
149
|
+
import os
|
|
150
|
+
import re
|
|
151
|
+
import sys
|
|
152
|
+
|
|
153
|
+
TARGET = sys.argv[1]
|
|
154
|
+
REPO_ROOT = sys.argv[2]
|
|
155
|
+
|
|
156
|
+
notices = []
|
|
157
|
+
|
|
158
|
+
if not REPO_ROOT:
|
|
159
|
+
REPO_ROOT = TARGET if os.path.isdir(TARGET) else os.path.dirname(TARGET)
|
|
160
|
+
notices.append(
|
|
161
|
+
"Target is not inside a git repository; repo root fell back to '%s'. "
|
|
162
|
+
"Internal/external classification may be less accurate." % REPO_ROOT
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
MAX_FILES = 4000
|
|
166
|
+
MAX_BYTES = 1024 * 1024
|
|
167
|
+
|
|
168
|
+
SKIP_DIRS = {
|
|
169
|
+
".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build",
|
|
170
|
+
".next", "coverage", "vendor", "target", ".mypy_cache", ".pytest_cache",
|
|
171
|
+
".tox", ".gradle", ".idea", ".svn", ".hg", "bower_components", ".terraform",
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
EXT_LANG = {
|
|
175
|
+
".js": "javascript", ".mjs": "javascript", ".cjs": "javascript", ".jsx": "javascript",
|
|
176
|
+
".ts": "typescript", ".tsx": "typescript", ".mts": "typescript", ".cts": "typescript",
|
|
177
|
+
".py": "python", ".pyi": "python",
|
|
178
|
+
".go": "go",
|
|
179
|
+
".rs": "rust",
|
|
180
|
+
".rb": "ruby",
|
|
181
|
+
".sh": "shell", ".bash": "shell", ".zsh": "shell",
|
|
182
|
+
".c": "c", ".h": "c",
|
|
183
|
+
".cc": "cpp", ".cpp": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".hh": "cpp",
|
|
184
|
+
".java": "java", ".kt": "kotlin", ".kts": "kotlin",
|
|
185
|
+
".php": "php",
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
SHEBANG_LANG = [
|
|
189
|
+
(re.compile(r"^#!.*\b(?:bash|sh|zsh|dash)\b"), "shell"),
|
|
190
|
+
(re.compile(r"^#!.*\bpython[0-9.]*\b"), "python"),
|
|
191
|
+
(re.compile(r"^#!.*\bnode\b"), "javascript"),
|
|
192
|
+
(re.compile(r"^#!.*\bruby\b"), "ruby"),
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def rel(path):
|
|
197
|
+
"""Repo-relative POSIX path, falling back to the absolute path when outside the repo."""
|
|
198
|
+
try:
|
|
199
|
+
r = os.path.relpath(path, REPO_ROOT)
|
|
200
|
+
except ValueError:
|
|
201
|
+
return path
|
|
202
|
+
if r.startswith(".."):
|
|
203
|
+
return path
|
|
204
|
+
return r.replace(os.sep, "/")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def walk_files(root):
|
|
208
|
+
"""Enumerate files under root (or [root] when root is a file), honouring SKIP_DIRS."""
|
|
209
|
+
if os.path.isfile(root):
|
|
210
|
+
return [root], False
|
|
211
|
+
out = []
|
|
212
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
213
|
+
keep = []
|
|
214
|
+
for d in sorted(dirnames):
|
|
215
|
+
if d in SKIP_DIRS:
|
|
216
|
+
continue
|
|
217
|
+
# .claude/worktrees holds full copies of the repo — never descend into it.
|
|
218
|
+
if d == "worktrees" and os.path.basename(dirpath) == ".claude":
|
|
219
|
+
continue
|
|
220
|
+
keep.append(d)
|
|
221
|
+
dirnames[:] = keep
|
|
222
|
+
for fn in sorted(filenames):
|
|
223
|
+
out.append(os.path.join(dirpath, fn))
|
|
224
|
+
if len(out) >= MAX_FILES:
|
|
225
|
+
return out, True
|
|
226
|
+
return out, False
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def read_text(path):
|
|
230
|
+
try:
|
|
231
|
+
if os.path.getsize(path) > MAX_BYTES:
|
|
232
|
+
return None
|
|
233
|
+
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
|
234
|
+
return fh.read()
|
|
235
|
+
except OSError:
|
|
236
|
+
return None
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def detect_language(path, text):
|
|
240
|
+
ext = os.path.splitext(path)[1].lower()
|
|
241
|
+
if ext in EXT_LANG:
|
|
242
|
+
return EXT_LANG[ext], ext
|
|
243
|
+
if not ext and text:
|
|
244
|
+
first = text.split("\n", 1)[0]
|
|
245
|
+
for pattern, lang in SHEBANG_LANG:
|
|
246
|
+
if pattern.match(first):
|
|
247
|
+
return lang, ext
|
|
248
|
+
return None, ext
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def find_upwards(start_dir, names):
|
|
252
|
+
"""Yield (directory, distance) from start_dir up to and including REPO_ROOT."""
|
|
253
|
+
cur = os.path.abspath(start_dir)
|
|
254
|
+
root = os.path.abspath(REPO_ROOT)
|
|
255
|
+
distance = 0
|
|
256
|
+
while True:
|
|
257
|
+
yield cur, distance
|
|
258
|
+
if cur == root or os.path.dirname(cur) == cur:
|
|
259
|
+
return
|
|
260
|
+
if not cur.startswith(root + os.sep):
|
|
261
|
+
return
|
|
262
|
+
cur = os.path.dirname(cur)
|
|
263
|
+
distance += 1
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# ---------------------------------------------------------------------------
|
|
267
|
+
# Dependency accumulators
|
|
268
|
+
# ---------------------------------------------------------------------------
|
|
269
|
+
|
|
270
|
+
internal = {}
|
|
271
|
+
external = {}
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def add_internal(raw, language, resolved, source):
|
|
275
|
+
key = (raw, language)
|
|
276
|
+
entry = internal.setdefault(key, {
|
|
277
|
+
"raw": raw, "language": language, "resolved": resolved,
|
|
278
|
+
"occurrences": 0, "sources": [],
|
|
279
|
+
})
|
|
280
|
+
entry["occurrences"] += 1
|
|
281
|
+
if resolved and not entry["resolved"]:
|
|
282
|
+
entry["resolved"] = resolved
|
|
283
|
+
if source not in entry["sources"] and len(entry["sources"]) < 3:
|
|
284
|
+
entry["sources"].append(source)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def add_external(name, language, kind, source):
|
|
288
|
+
key = (name, language)
|
|
289
|
+
entry = external.setdefault(key, {
|
|
290
|
+
"name": name, "language": language, "kind": kind,
|
|
291
|
+
"occurrences": 0, "sources": [],
|
|
292
|
+
})
|
|
293
|
+
entry["occurrences"] += 1
|
|
294
|
+
if kind != "unknown" and entry["kind"] == "unknown":
|
|
295
|
+
entry["kind"] = kind
|
|
296
|
+
if source not in entry["sources"] and len(entry["sources"]) < 3:
|
|
297
|
+
entry["sources"].append(source)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def resolve_path(base_dir, spec, suffixes):
|
|
301
|
+
"""Resolve spec against base_dir (then REPO_ROOT), trying each suffix. Returns rel path or None."""
|
|
302
|
+
candidates = []
|
|
303
|
+
if spec.startswith("/"):
|
|
304
|
+
candidates.append(os.path.join(REPO_ROOT, spec.lstrip("/")))
|
|
305
|
+
else:
|
|
306
|
+
candidates.append(os.path.normpath(os.path.join(base_dir, spec)))
|
|
307
|
+
candidates.append(os.path.normpath(os.path.join(REPO_ROOT, spec)))
|
|
308
|
+
for cand in candidates:
|
|
309
|
+
for suffix in suffixes:
|
|
310
|
+
probe = cand + suffix
|
|
311
|
+
if os.path.isfile(probe):
|
|
312
|
+
return rel(probe)
|
|
313
|
+
if not suffixes and os.path.exists(cand):
|
|
314
|
+
return rel(cand)
|
|
315
|
+
return None
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
# ---------------------------------------------------------------------------
|
|
319
|
+
# Per-language extractors
|
|
320
|
+
# ---------------------------------------------------------------------------
|
|
321
|
+
|
|
322
|
+
JS_PATTERNS = [
|
|
323
|
+
re.compile(r"""\bfrom\s+['"]([^'"\n]+)['"]"""),
|
|
324
|
+
re.compile(r"""\brequire\s*\(\s*['"]([^'"\n]+)['"]\s*\)"""),
|
|
325
|
+
re.compile(r"""\bimport\s*\(\s*['"]([^'"\n]+)['"]\s*\)"""),
|
|
326
|
+
re.compile(r"""^\s*import\s+['"]([^'"\n]+)['"]""", re.M),
|
|
327
|
+
]
|
|
328
|
+
JS_SUFFIXES = ["", ".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".json",
|
|
329
|
+
"/index.js", "/index.ts", "/index.mjs"]
|
|
330
|
+
# Node.js built-in modules — reported as stdlib so the caller does not flag them as
|
|
331
|
+
# undeclared when reconciling against package.json.
|
|
332
|
+
NODE_BUILTINS = {
|
|
333
|
+
"assert", "async_hooks", "buffer", "child_process", "cluster", "console", "constants",
|
|
334
|
+
"crypto", "dgram", "diagnostics_channel", "dns", "domain", "events", "fs", "http",
|
|
335
|
+
"http2", "https", "inspector", "module", "net", "os", "path", "perf_hooks", "process",
|
|
336
|
+
"punycode", "querystring", "readline", "repl", "stream", "string_decoder", "timers",
|
|
337
|
+
"tls", "trace_events", "tty", "url", "util", "v8", "vm", "wasi", "worker_threads", "zlib",
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def scan_js(text, path, base_dir, language, source):
|
|
342
|
+
for pattern in JS_PATTERNS:
|
|
343
|
+
for spec in pattern.findall(text):
|
|
344
|
+
if spec.startswith((".", "/", "~/", "@/", "#")):
|
|
345
|
+
add_internal(spec, language, resolve_path(base_dir, spec.lstrip("~@#"), JS_SUFFIXES), source)
|
|
346
|
+
continue
|
|
347
|
+
parts = spec.split("/")
|
|
348
|
+
name = "/".join(parts[:2]) if spec.startswith("@") else parts[0]
|
|
349
|
+
resolved = resolve_path(base_dir, spec, JS_SUFFIXES)
|
|
350
|
+
if resolved:
|
|
351
|
+
add_internal(spec, language, resolved, source)
|
|
352
|
+
elif name.startswith("node:") or name in NODE_BUILTINS:
|
|
353
|
+
add_external(name.replace("node:", "", 1), language, "stdlib", source)
|
|
354
|
+
else:
|
|
355
|
+
add_external(name, language, "package", source)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
PY_IMPORT = re.compile(r"^\s*import\s+([A-Za-z_][\w.]*(?:\s*,\s*[A-Za-z_][\w.]*)*)", re.M)
|
|
359
|
+
PY_FROM = re.compile(r"^\s*from\s+(\.*[A-Za-z_][\w.]*|\.+)\s+import\s", re.M)
|
|
360
|
+
PY_STDLIB = set(getattr(sys, "stdlib_module_names", ()))
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _py_resolve(dotted, base_dir):
|
|
364
|
+
parts = dotted.split(".")
|
|
365
|
+
for root in (base_dir, REPO_ROOT):
|
|
366
|
+
stem = os.path.join(root, *parts)
|
|
367
|
+
if os.path.isfile(stem + ".py"):
|
|
368
|
+
return rel(stem + ".py")
|
|
369
|
+
if os.path.isfile(os.path.join(stem, "__init__.py")):
|
|
370
|
+
return rel(os.path.join(stem, "__init__.py"))
|
|
371
|
+
return None
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def scan_python(text, path, base_dir, language, source):
|
|
375
|
+
specs = []
|
|
376
|
+
for group in PY_IMPORT.findall(text):
|
|
377
|
+
specs.extend(s.strip() for s in group.split(","))
|
|
378
|
+
specs.extend(PY_FROM.findall(text))
|
|
379
|
+
for spec in specs:
|
|
380
|
+
if not spec:
|
|
381
|
+
continue
|
|
382
|
+
if spec.startswith("."):
|
|
383
|
+
up = len(spec) - len(spec.lstrip("."))
|
|
384
|
+
tail = spec.lstrip(".")
|
|
385
|
+
anchor = base_dir
|
|
386
|
+
for _ in range(up - 1):
|
|
387
|
+
anchor = os.path.dirname(anchor)
|
|
388
|
+
add_internal(spec, language, _py_resolve(tail, anchor) if tail else rel(anchor), source)
|
|
389
|
+
continue
|
|
390
|
+
top = spec.split(".")[0]
|
|
391
|
+
resolved = _py_resolve(spec, base_dir)
|
|
392
|
+
if resolved:
|
|
393
|
+
add_internal(spec, language, resolved, source)
|
|
394
|
+
elif top in PY_STDLIB:
|
|
395
|
+
add_external(top, language, "stdlib", source)
|
|
396
|
+
else:
|
|
397
|
+
add_external(top, language, "package", source)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
GO_BLOCK = re.compile(r"^\s*import\s*\(([^)]*)\)", re.M)
|
|
401
|
+
GO_SINGLE = re.compile(r"""^\s*import\s+(?:[\w.]+\s+)?"([^"]+)\"""", re.M)
|
|
402
|
+
GO_QUOTED = re.compile(r'"([^"]+)"')
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _go_module_path(base_dir):
|
|
406
|
+
for directory, _ in find_upwards(base_dir, None):
|
|
407
|
+
gomod = os.path.join(directory, "go.mod")
|
|
408
|
+
if os.path.isfile(gomod):
|
|
409
|
+
text = read_text(gomod) or ""
|
|
410
|
+
match = re.search(r"^\s*module\s+(\S+)", text, re.M)
|
|
411
|
+
if match:
|
|
412
|
+
return match.group(1)
|
|
413
|
+
return None
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def scan_go(text, path, base_dir, language, source):
|
|
417
|
+
module = _go_module_path(base_dir)
|
|
418
|
+
specs = []
|
|
419
|
+
for block in GO_BLOCK.findall(text):
|
|
420
|
+
specs.extend(GO_QUOTED.findall(block))
|
|
421
|
+
specs.extend(GO_SINGLE.findall(text))
|
|
422
|
+
for spec in specs:
|
|
423
|
+
if module and (spec == module or spec.startswith(module + "/")):
|
|
424
|
+
sub = spec[len(module):].lstrip("/")
|
|
425
|
+
add_internal(spec, language, rel(os.path.join(REPO_ROOT, sub)) if sub else rel(REPO_ROOT), source)
|
|
426
|
+
elif "." not in spec.split("/")[0]:
|
|
427
|
+
add_external(spec, language, "stdlib", source)
|
|
428
|
+
else:
|
|
429
|
+
add_external("/".join(spec.split("/")[:3]), language, "package", source)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
RUST_USE = re.compile(r"^\s*(?:pub\s+)?use\s+([A-Za-z_][\w:]*)", re.M)
|
|
433
|
+
RUST_MOD = re.compile(r"^\s*(?:pub\s+)?mod\s+([A-Za-z_]\w*)\s*;", re.M)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def scan_rust(text, path, base_dir, language, source):
|
|
437
|
+
for spec in RUST_USE.findall(text):
|
|
438
|
+
root = spec.split("::")[0]
|
|
439
|
+
if root in ("crate", "super", "self"):
|
|
440
|
+
add_internal(spec, language, None, source)
|
|
441
|
+
else:
|
|
442
|
+
add_external(root, language, "package", source)
|
|
443
|
+
for name in RUST_MOD.findall(text):
|
|
444
|
+
add_internal("mod " + name, language, resolve_path(base_dir, name, [".rs", "/mod.rs"]), source)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
RB_RELATIVE = re.compile(r"""\brequire_relative\s+['"]([^'"\n]+)['"]""")
|
|
448
|
+
RB_REQUIRE = re.compile(r"""\brequire\s+['"]([^'"\n]+)['"]""")
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def scan_ruby(text, path, base_dir, language, source):
|
|
452
|
+
for spec in RB_RELATIVE.findall(text):
|
|
453
|
+
add_internal(spec, language, resolve_path(base_dir, spec, [".rb", ""]), source)
|
|
454
|
+
for spec in RB_REQUIRE.findall(text):
|
|
455
|
+
resolved = resolve_path(base_dir, spec, [".rb"])
|
|
456
|
+
if resolved:
|
|
457
|
+
add_internal(spec, language, resolved, source)
|
|
458
|
+
else:
|
|
459
|
+
add_external(spec.split("/")[0], language, "package", source)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
SH_SOURCE = re.compile(r"""^\s*(?:\.|source)\s+["']?([^"'\s;&|]+)""", re.M)
|
|
463
|
+
SH_PATHLIKE = re.compile(r"""[A-Za-z0-9_./-]+\.(?:sh|bash|py|js|mjs|cjs|json|jsonl|md|tpl)\b""")
|
|
464
|
+
SH_COMMANDS = (
|
|
465
|
+
"jq", "git", "gh", "npm", "npx", "node", "python3", "pip", "pip3", "curl", "wget",
|
|
466
|
+
"docker", "rsync", "shellcheck", "pytest", "bats", "cargo", "go", "make", "ssh",
|
|
467
|
+
"openssl", "aws", "gcloud", "terraform", "sqlite3", "yq", "tar", "zip", "unzip",
|
|
468
|
+
)
|
|
469
|
+
SH_COMMAND_RE = re.compile(
|
|
470
|
+
r"(?:^|[|;&({!]|\$\(|&&|\|\|)\s*(?:!\s*)?(?:sudo\s+)?(" + "|".join(SH_COMMANDS) + r")\b", re.M
|
|
471
|
+
)
|
|
472
|
+
SH_MAX_PATHS = 40
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def scan_shell(text, path, base_dir, language, source):
|
|
476
|
+
for spec in SH_SOURCE.findall(text):
|
|
477
|
+
if "$" in spec:
|
|
478
|
+
# Contains a variable reference — record it, but do not pretend to resolve it.
|
|
479
|
+
add_internal(spec, language, None, source)
|
|
480
|
+
else:
|
|
481
|
+
add_internal(spec, language, resolve_path(base_dir, spec, [""]), source)
|
|
482
|
+
seen = set()
|
|
483
|
+
for spec in SH_PATHLIKE.findall(text):
|
|
484
|
+
if spec in seen:
|
|
485
|
+
continue
|
|
486
|
+
seen.add(spec)
|
|
487
|
+
if len(seen) > SH_MAX_PATHS:
|
|
488
|
+
break
|
|
489
|
+
resolved = resolve_path(base_dir, spec, [""])
|
|
490
|
+
if resolved and resolved != rel(path):
|
|
491
|
+
add_internal(spec, language, resolved, source)
|
|
492
|
+
for name in SH_COMMAND_RE.findall(text):
|
|
493
|
+
add_external(name, language, "command", source)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
C_LOCAL = re.compile(r"""^\s*#\s*include\s+"([^"\n]+)\"""", re.M)
|
|
497
|
+
C_SYSTEM = re.compile(r"^\s*#\s*include\s+<([^>\n]+)>", re.M)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def scan_c(text, path, base_dir, language, source):
|
|
501
|
+
for spec in C_LOCAL.findall(text):
|
|
502
|
+
add_internal(spec, language, resolve_path(base_dir, spec, [""]), source)
|
|
503
|
+
for spec in C_SYSTEM.findall(text):
|
|
504
|
+
add_external(spec, language, "stdlib", source)
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
JVM_IMPORT = re.compile(r"^\s*import\s+(?:static\s+)?([\w.]+)", re.M)
|
|
508
|
+
JVM_STDLIB_PREFIXES = ("java.", "javax.", "jakarta.", "kotlin.", "kotlinx.", "scala.")
|
|
509
|
+
JVM_SRC_ROOTS = ("", "src", "src/main/java", "src/main/kotlin", "app/src/main/java")
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def scan_jvm(text, path, base_dir, language, source):
|
|
513
|
+
for spec in JVM_IMPORT.findall(text):
|
|
514
|
+
if spec.startswith(JVM_STDLIB_PREFIXES):
|
|
515
|
+
add_external(spec, language, "stdlib", source)
|
|
516
|
+
continue
|
|
517
|
+
as_path = spec.replace(".", "/")
|
|
518
|
+
resolved = None
|
|
519
|
+
for src_root in JVM_SRC_ROOTS:
|
|
520
|
+
resolved = resolve_path(os.path.join(REPO_ROOT, src_root), as_path, [".java", ".kt"])
|
|
521
|
+
if resolved:
|
|
522
|
+
break
|
|
523
|
+
if resolved:
|
|
524
|
+
add_internal(spec, language, resolved, source)
|
|
525
|
+
else:
|
|
526
|
+
add_external(spec, language, "package", source)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
PHP_INCLUDE = re.compile(
|
|
530
|
+
r"""^\s*(?:require|require_once|include|include_once)\s*\(?\s*['"]([^'"\n]+)['"]""", re.M
|
|
531
|
+
)
|
|
532
|
+
PHP_USE = re.compile(r"^\s*use\s+([\w\\]+)", re.M)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def scan_php(text, path, base_dir, language, source):
|
|
536
|
+
for spec in PHP_INCLUDE.findall(text):
|
|
537
|
+
add_internal(spec, language, resolve_path(base_dir, spec, ["", ".php"]), source)
|
|
538
|
+
for spec in PHP_USE.findall(text):
|
|
539
|
+
add_external(spec.lstrip("\\").split("\\")[0], language, "package", source)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
SCANNERS = {
|
|
543
|
+
"javascript": scan_js, "typescript": scan_js,
|
|
544
|
+
"python": scan_python,
|
|
545
|
+
"go": scan_go,
|
|
546
|
+
"rust": scan_rust,
|
|
547
|
+
"ruby": scan_ruby,
|
|
548
|
+
"shell": scan_shell,
|
|
549
|
+
"c": scan_c, "cpp": scan_c,
|
|
550
|
+
"java": scan_jvm, "kotlin": scan_jvm,
|
|
551
|
+
"php": scan_php,
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
# ---------------------------------------------------------------------------
|
|
555
|
+
# Manifest discovery
|
|
556
|
+
# ---------------------------------------------------------------------------
|
|
557
|
+
|
|
558
|
+
MANIFEST_NAMES = [
|
|
559
|
+
"package.json", "pyproject.toml", "requirements.txt", "go.mod", "Cargo.toml", "Gemfile",
|
|
560
|
+
"setup.py", "setup.cfg", "Pipfile", "composer.json", "pom.xml",
|
|
561
|
+
"build.gradle", "build.gradle.kts", "Package.swift", "mix.exs", "pubspec.yaml",
|
|
562
|
+
]
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def _declared_package_json(text):
|
|
566
|
+
data = json.loads(text)
|
|
567
|
+
names = []
|
|
568
|
+
for key in ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies"):
|
|
569
|
+
block = data.get(key)
|
|
570
|
+
if isinstance(block, dict):
|
|
571
|
+
names.extend(block.keys())
|
|
572
|
+
return sorted(set(names))
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _declared_requirements(text):
|
|
576
|
+
names = []
|
|
577
|
+
for line in text.splitlines():
|
|
578
|
+
line = line.split("#", 1)[0].strip()
|
|
579
|
+
if not line or line.startswith("-"):
|
|
580
|
+
continue
|
|
581
|
+
names.append(re.split(r"[\s=<>!~;\[]", line, maxsplit=1)[0])
|
|
582
|
+
return sorted({n for n in names if n})
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _declared_go_mod(text):
|
|
586
|
+
names = []
|
|
587
|
+
for block in re.findall(r"^\s*require\s*\(([^)]*)\)", text, re.M):
|
|
588
|
+
for line in block.splitlines():
|
|
589
|
+
line = line.split("//", 1)[0].strip()
|
|
590
|
+
if line:
|
|
591
|
+
names.append(line.split()[0])
|
|
592
|
+
for match in re.findall(r"^\s*require\s+(\S+)\s+\S+", text, re.M):
|
|
593
|
+
if not match.startswith("("):
|
|
594
|
+
names.append(match)
|
|
595
|
+
return sorted(set(names))
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def _declared_toml_sections(text, sections):
|
|
599
|
+
names = []
|
|
600
|
+
current = None
|
|
601
|
+
for line in text.splitlines():
|
|
602
|
+
stripped = line.strip()
|
|
603
|
+
if stripped.startswith("[") and stripped.endswith("]"):
|
|
604
|
+
current = stripped.strip("[]").strip()
|
|
605
|
+
continue
|
|
606
|
+
if current in sections and "=" in stripped and not stripped.startswith("#"):
|
|
607
|
+
names.append(stripped.split("=", 1)[0].strip().split(".")[0].strip('"'))
|
|
608
|
+
return sorted({n for n in names if n})
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
def _declared_cargo(text):
|
|
612
|
+
return _declared_toml_sections(text, {"dependencies", "dev-dependencies", "build-dependencies"})
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _declared_pyproject(text):
|
|
616
|
+
names = _declared_toml_sections(text, {"tool.poetry.dependencies", "tool.poetry.dev-dependencies"})
|
|
617
|
+
for block in re.findall(r"^\s*dependencies\s*=\s*\[([^\]]*)\]", text, re.M):
|
|
618
|
+
for item in re.findall(r"""['"]([^'"]+)['"]""", block):
|
|
619
|
+
names.append(re.split(r"[\s=<>!~;\[]", item, maxsplit=1)[0])
|
|
620
|
+
return sorted({n for n in names if n})
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
DECLARED_PARSERS = {
|
|
624
|
+
"package.json": _declared_package_json,
|
|
625
|
+
"requirements.txt": _declared_requirements,
|
|
626
|
+
"go.mod": _declared_go_mod,
|
|
627
|
+
"Cargo.toml": _declared_cargo,
|
|
628
|
+
"pyproject.toml": _declared_pyproject,
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def collect_manifests(start_dir):
|
|
633
|
+
found = []
|
|
634
|
+
for directory, distance in find_upwards(start_dir, None):
|
|
635
|
+
for name in MANIFEST_NAMES:
|
|
636
|
+
candidate = os.path.join(directory, name)
|
|
637
|
+
if not os.path.isfile(candidate):
|
|
638
|
+
continue
|
|
639
|
+
declared = None
|
|
640
|
+
parser = DECLARED_PARSERS.get(name)
|
|
641
|
+
if parser:
|
|
642
|
+
text = read_text(candidate)
|
|
643
|
+
if text is not None:
|
|
644
|
+
try:
|
|
645
|
+
declared = parser(text)
|
|
646
|
+
except Exception:
|
|
647
|
+
notices.append(
|
|
648
|
+
"Could not parse declared dependencies from %s; manifest is listed "
|
|
649
|
+
"but declared_dependencies is null." % rel(candidate)
|
|
650
|
+
)
|
|
651
|
+
found.append({
|
|
652
|
+
"name": name,
|
|
653
|
+
"path": rel(candidate),
|
|
654
|
+
"location": "target" if distance == 0 else "ancestor",
|
|
655
|
+
"distance": distance,
|
|
656
|
+
"declared_dependencies": declared,
|
|
657
|
+
})
|
|
658
|
+
return found
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# ---------------------------------------------------------------------------
|
|
662
|
+
# Main scan
|
|
663
|
+
# ---------------------------------------------------------------------------
|
|
664
|
+
|
|
665
|
+
is_dir = os.path.isdir(TARGET)
|
|
666
|
+
start_dir = TARGET if is_dir else os.path.dirname(TARGET)
|
|
667
|
+
|
|
668
|
+
files, truncated = walk_files(TARGET)
|
|
669
|
+
if truncated:
|
|
670
|
+
notices.append(
|
|
671
|
+
"File walk stopped at the %d-file cap; results are partial. Narrow the target for full coverage."
|
|
672
|
+
% MAX_FILES
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
language_counts = {}
|
|
676
|
+
unknown_ext_counts = {}
|
|
677
|
+
scanned = 0
|
|
678
|
+
|
|
679
|
+
for file_path in files:
|
|
680
|
+
text = read_text(file_path)
|
|
681
|
+
if text is None:
|
|
682
|
+
continue
|
|
683
|
+
language, ext = detect_language(file_path, text)
|
|
684
|
+
if language is None:
|
|
685
|
+
unknown_ext_counts[ext or "<no extension>"] = unknown_ext_counts.get(ext or "<no extension>", 0) + 1
|
|
686
|
+
continue
|
|
687
|
+
language_counts[language] = language_counts.get(language, 0) + 1
|
|
688
|
+
scanned += 1
|
|
689
|
+
scanner = SCANNERS.get(language)
|
|
690
|
+
if scanner:
|
|
691
|
+
scanner(text, file_path, os.path.dirname(file_path), language, rel(file_path))
|
|
692
|
+
|
|
693
|
+
unrecognised = scanned == 0
|
|
694
|
+
if unrecognised:
|
|
695
|
+
notices.append(
|
|
696
|
+
"No recognised source language was found under the target; the dependency result is "
|
|
697
|
+
"explicitly empty rather than an error. Recognised languages: %s."
|
|
698
|
+
% ", ".join(sorted(set(EXT_LANG.values())))
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
manifests = collect_manifests(start_dir)
|
|
702
|
+
|
|
703
|
+
internal_list = sorted(internal.values(), key=lambda e: (e["resolved"] or e["raw"], e["language"]))
|
|
704
|
+
external_list = sorted(external.values(), key=lambda e: (e["language"], e["name"]))
|
|
705
|
+
|
|
706
|
+
result = {
|
|
707
|
+
"script": "detect-dependencies.sh",
|
|
708
|
+
"version": 1,
|
|
709
|
+
"target": rel(TARGET),
|
|
710
|
+
"target_absolute": TARGET,
|
|
711
|
+
"target_type": "directory" if is_dir else "file",
|
|
712
|
+
"repo_root": REPO_ROOT,
|
|
713
|
+
"files_scanned": scanned,
|
|
714
|
+
"files_truncated": truncated,
|
|
715
|
+
"unrecognised": unrecognised,
|
|
716
|
+
"languages": [
|
|
717
|
+
{"language": k, "files": v}
|
|
718
|
+
for k, v in sorted(language_counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
|
719
|
+
],
|
|
720
|
+
"unrecognised_extensions": [
|
|
721
|
+
{"extension": k, "files": v}
|
|
722
|
+
for k, v in sorted(unknown_ext_counts.items(), key=lambda kv: (-kv[1], kv[0]))[:10]
|
|
723
|
+
],
|
|
724
|
+
"counts": {"internal": len(internal_list), "external": len(external_list)},
|
|
725
|
+
"internal": internal_list,
|
|
726
|
+
"external": external_list,
|
|
727
|
+
"manifests": manifests,
|
|
728
|
+
"notices": notices,
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
print(json.dumps(result, indent=2))
|
|
732
|
+
PY
|