@jenga-ai/agent 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/agents/developer.md +82 -2
- package/agents/scrum-master.md +140 -21
- package/agents/tester.md +90 -8
- package/hooks/on_session_end.sh +171 -20
- package/package.json +1 -1
- package/scripts/check-permission-level.sh +107 -0
- package/scripts/check-publicignore-match.sh +122 -0
- package/scripts/check-worktree-liveness.sh +193 -0
- package/scripts/generate-rapport-manifest.sh +43 -0
- package/scripts/idea_manager.sh +47 -0
- package/scripts/install-worktree-commit-guard.sh +134 -0
- package/scripts/jenga-permission-level-switch.sh +109 -0
- package/scripts/smoke-harness.sh +139 -0
- package/scripts/validate-board.sh +62 -0
- package/scripts/with-lock.sh +158 -0
- package/scripts/worktree-remove-guard.sh +204 -0
- package/skills/clearify/SKILL.md +52 -0
- package/skills/commit/SKILL.md +13 -4
- package/skills/distribute/CONFIG_SCHEMA.md +60 -2
- package/skills/do/SKILL.md +48 -11
- package/skills/doc-sync/SKILL.md +16 -0
- package/skills/doc-sync/assets/doc_targets.md +11 -0
- package/skills/idea/SKILL.md +56 -0
- package/skills/idea/assets/idea_handoff_template.md +26 -0
- package/skills/idea/assets/idea_template.md +3 -0
- package/skills/init/SKILL.md +100 -7
- package/skills/init/assets/directory_structure.txt +1 -0
- package/skills/init/assets/workflow_template.json +1 -1
- package/skills/init/scripts/apply-project-visibility.sh +176 -0
- package/skills/init/scripts/detect-existing-codebase.sh +166 -0
- package/skills/init/scripts/init.sh +30 -1
- package/skills/jenga/SKILL.md +160 -17
- package/skills/jenga/scripts/board-scan.sh +238 -0
- package/skills/jenga/scripts/cascade-resolve.sh +297 -0
- package/skills/jenga/scripts/render-confirmation.sh +679 -0
- package/skills/jenga/scripts/render-picker.sh +439 -0
- package/skills/jenga/scripts/resolve-id.sh +367 -0
- package/skills/jenga-permission-level/SKILL.md +81 -0
- package/skills/proceed/SKILL.md +1 -1
- package/skills/publish/SKILL.md +8 -5
- package/skills/publish/assets/ci-contract.md +2 -2
- package/skills/publish/assets/ownership-matrix.md +1 -1
- package/skills/publish/scripts/finalize_changelog.sh +115 -0
- package/skills/publish/scripts/generate_release_notes.sh +475 -28
- package/skills/publish/scripts/npm_ci_pipeline.sh +44 -6
- package/skills/publish/scripts/publish_deploy.sh +38 -8
- package/skills/publish/scripts/run_gates.sh +2 -2
- package/skills/reconcile/SKILL.md +117 -5
- package/skills/reconcile/scripts/detect-unlinked-code.sh +741 -0
- package/skills/skillify/assets/init-new/assets/directory_structure.txt +5 -1
- package/skills/spinoff/SKILL.md +12 -7
- package/skills/todo/SKILL.md +2 -0
- package/skills/uncharted/SKILL.md +711 -0
- package/skills/uncharted/assets/SEGMENT_PROPOSAL_TEMPLATE.md +129 -0
- package/skills/uncharted/assets/UNDERSTANDING_DOC_TEMPLATE.md +160 -0
- package/skills/uncharted/scripts/apply-subsystem-cap.sh +573 -0
- package/skills/uncharted/scripts/detect-dependencies.sh +732 -0
- package/skills/uncharted/scripts/detect-tests.sh +553 -0
- package/skills/uncharted/scripts/discover-subsystems.sh +1029 -0
- package/skills/uncharted/scripts/enumerate-target.sh +470 -0
- package/skills/uncharted/scripts/import-source.sh +517 -0
- package/skills/uncharted/scripts/inspect-provenance.sh +573 -0
- package/skills/uncharted/scripts/resolve-segment-target.sh +640 -0
- package/skills/uncharted/scripts/run-engine.sh +655 -0
- package/skills/uncharted/scripts/validate-proposed-items.sh +125 -0
- package/skills/uncharted/scripts/write-backfilled-epics.sh +498 -0
- package/skills/wtf/SKILL.md +20 -0
- package/templates/CHANGELOG_TEMPLATE.md +13 -0
- package/templates/PROBLEM_RAPPORT_TEMPLATE.md +4 -1
- package/templates/SCRUM_BOARD_SCHEMA.md +157 -10
- package/templates/permission-levels/README.md +73 -0
- package/templates/permission-levels/level-1-locked.json +71 -0
- package/templates/permission-levels/level-2-guarded.json +64 -0
- package/templates/permission-levels/level-3-standard.json +62 -0
- package/templates/permission-levels/level-4-elevated.json +60 -0
- package/templates/permission-levels/level-5-unrestricted.json +58 -0
- package/skills/convert/SKILL.md +0 -124
- package/skills/convert/convert_cli.py +0 -235
- package/skills/convert/tests/sample.csv +0 -4
- package/skills/convert/tests/sample.json +0 -5
- package/skills/convert/tests/sample.jsonl +0 -3
- package/skills/convert/tests/sample.yaml +0 -18
- package/skills/convert/tests/sample_obj.csv +0 -2
- package/skills/convert/tests/sample_obj.json +0 -9
- package/skills/mirror-public/SKILL.md +0 -237
- package/skills/mirror-public/assets/config.json +0 -5
- package/skills/mirror-public/scripts/mirror.sh +0 -374
- package/skills/self-sync/SKILL.md +0 -73
- package/skills/self-sync/scripts/run.js +0 -136
- package/skills/strategy/SKILL.md +0 -312
|
@@ -0,0 +1,573 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ---------------------------------------------------------------------------
|
|
3
|
+
# skills/uncharted/scripts/inspect-provenance.sh
|
|
4
|
+
#
|
|
5
|
+
# Provenance surface for `/uncharted import`'s placement-confirmation gate
|
|
6
|
+
# (E40_S03_T02). Given a staged source -- the `content_path` that
|
|
7
|
+
# `import-source.sh` (E40_S03_T01) produced -- reports what is KNOWABLE about
|
|
8
|
+
# its licensing and origin, so a human can make an informed placement
|
|
9
|
+
# decision:
|
|
10
|
+
#
|
|
11
|
+
# - presence and detected type of any LICENSE / LICENCE / COPYING /
|
|
12
|
+
# UNLICENSE / NOTICE file
|
|
13
|
+
# - SPDX-License-Identifier headers found in file contents
|
|
14
|
+
# - copyright lines found near the top of files
|
|
15
|
+
# - git remote URL and HEAD SHA, when the source is git-derived
|
|
16
|
+
#
|
|
17
|
+
# This is an INFORMATIONAL SURFACE, not an automated legal check. It reports
|
|
18
|
+
# findings; the calling skill presents them and the user decides. It NEVER
|
|
19
|
+
# implies a source is unencumbered just because nothing was found: every
|
|
20
|
+
# section below states explicitly, in its own "message" field, when it found
|
|
21
|
+
# nothing, and the top-level "overall" object restates that plainly when
|
|
22
|
+
# every section comes back empty.
|
|
23
|
+
#
|
|
24
|
+
# Why --origin-* flags exist: import-source.sh deliberately strips the
|
|
25
|
+
# top-level .git directory after cloning or copying, as part of its
|
|
26
|
+
# "acquired source is hostile" security contract -- so by the time this
|
|
27
|
+
# script runs, the ORIGINAL remote URL and HEAD SHA it read during
|
|
28
|
+
# acquisition are no longer recoverable from the staged tree itself. Pass
|
|
29
|
+
# them through from import-source.sh's own JSON summary (fields "source",
|
|
30
|
+
# "source_type", "git_ref") rather than trying to re-derive something that is
|
|
31
|
+
# no longer on disk. Independently of that, this script ALSO detects and
|
|
32
|
+
# reports any NESTED .git directories still present in the staged tree (for
|
|
33
|
+
# example a vendored submodule checkout import-source.sh did not strip,
|
|
34
|
+
# since it only strips the top-level one) -- those are read directly, live,
|
|
35
|
+
# off disk, with no flag needed.
|
|
36
|
+
#
|
|
37
|
+
# READ-ONLY CONTRACT: this script never writes to, executes, sources, or
|
|
38
|
+
# otherwise modifies anything inside the target. The only git operations it
|
|
39
|
+
# performs against a nested .git are read-only plumbing (`config --get`,
|
|
40
|
+
# `rev-parse`) -- no hooks, no checkout, no fetch, matching
|
|
41
|
+
# import-source.sh's stance that acquired content is never executed.
|
|
42
|
+
#
|
|
43
|
+
# Usage:
|
|
44
|
+
# inspect-provenance.sh [options] <staging-path>
|
|
45
|
+
#
|
|
46
|
+
# Options:
|
|
47
|
+
# --origin-source <text> The original source identifier (URL or path)
|
|
48
|
+
# that import-source.sh acquired from. Carried
|
|
49
|
+
# through verbatim -- not validated, not
|
|
50
|
+
# re-fetched.
|
|
51
|
+
# --origin-type <type> git|path|snippet -- import-source.sh's detected
|
|
52
|
+
# source_type for this acquisition.
|
|
53
|
+
# --origin-ref <sha> The HEAD short SHA import-source.sh captured
|
|
54
|
+
# before stripping .git (its "git_ref" field).
|
|
55
|
+
# --max-files N Bound on how many files are content-scanned for
|
|
56
|
+
# SPDX identifiers and copyright lines (default
|
|
57
|
+
# 4000, matching detect-dependencies.sh's cap).
|
|
58
|
+
# --json-out <file> Also write the JSON report to this file.
|
|
59
|
+
# -h, --help Show this help.
|
|
60
|
+
#
|
|
61
|
+
# Output (stdout): exactly one JSON object. Diagnostics and notices
|
|
62
|
+
# (truncation, excluded directories, git command failures) go to stderr --
|
|
63
|
+
# read it, the same as every other engine script.
|
|
64
|
+
#
|
|
65
|
+
# JSON shape (top level):
|
|
66
|
+
# {
|
|
67
|
+
# "script": "inspect-provenance.sh", "version": 1,
|
|
68
|
+
# "target": "<canonicalised staging path>",
|
|
69
|
+
# "files_scanned": <int>, "files_truncated": <bool>,
|
|
70
|
+
# "symlinked_files_excluded": <int>, // never opened; see stderr notice
|
|
71
|
+
# "license": {
|
|
72
|
+
# "found": bool,
|
|
73
|
+
# "files": [ { "path": "<relative>", "type": "MIT"|...|"unknown" } ],
|
|
74
|
+
# "message": "<set iff found is false>"
|
|
75
|
+
# },
|
|
76
|
+
# "spdx_identifiers": {
|
|
77
|
+
# "found": bool,
|
|
78
|
+
# "identifiers": [ { "id": "MIT", "occurrences": N, "sources": [...] } ],
|
|
79
|
+
# "message": "<set iff found is false>"
|
|
80
|
+
# },
|
|
81
|
+
# "copyright": {
|
|
82
|
+
# "found": bool,
|
|
83
|
+
# "lines": [ { "text": "...", "source": "<relative>" } ],
|
|
84
|
+
# "message": "<set iff found is false>"
|
|
85
|
+
# },
|
|
86
|
+
# "git_origin": {
|
|
87
|
+
# "applicable": bool,
|
|
88
|
+
# "origin": { "source": ..., "source_type": ..., "head_sha": ... } | null,
|
|
89
|
+
# "nested_repos": [ { "path": "<relative>", "remote_url": ...|null, "head_sha": ...|null } ],
|
|
90
|
+
# "message": "<set iff there is nothing git-related to report>"
|
|
91
|
+
# },
|
|
92
|
+
# "overall": { "any_signal_found": bool, "message": "<set iff false>" }
|
|
93
|
+
# }
|
|
94
|
+
#
|
|
95
|
+
# Exit codes:
|
|
96
|
+
# 0 report generated (regardless of whether any provenance signal was
|
|
97
|
+
# found -- "nothing found" is a valid, reportable result, not a failure)
|
|
98
|
+
# 1 usage error (bad flag, missing argument, missing target)
|
|
99
|
+
# 2 target does not exist or is not a directory
|
|
100
|
+
# 3 python3 unavailable, or the python3 content scan itself failed
|
|
101
|
+
# 4 --json-out could not be written
|
|
102
|
+
#
|
|
103
|
+
# Requires: bash, git (only used for read-only plumbing against any nested
|
|
104
|
+
# .git found), python3. jq is NOT required -- JSON is emitted by python3,
|
|
105
|
+
# matching detect-dependencies.sh / detect-tests.sh / discover-subsystems.sh
|
|
106
|
+
# rather than import-source.sh's flatter jq-with-fallback approach; this
|
|
107
|
+
# script's nested arrays make the python convention the better fit.
|
|
108
|
+
# ---------------------------------------------------------------------------
|
|
109
|
+
|
|
110
|
+
set -euo pipefail
|
|
111
|
+
|
|
112
|
+
SELF="$(basename "$0")"
|
|
113
|
+
|
|
114
|
+
die() {
|
|
115
|
+
local code="$1"; shift
|
|
116
|
+
printf '%s: error: %s\n' "$SELF" "$*" >&2
|
|
117
|
+
exit "$code"
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
usage() {
|
|
121
|
+
sed -n '/^# Usage:/,/^# Exit codes:/p' "$0" | sed 's/^# \{0,1\}//' >&2
|
|
122
|
+
exit 1
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
# ---------------------------------------------------------------------------
|
|
126
|
+
# Argument parsing
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
|
|
129
|
+
TARGET=""
|
|
130
|
+
TARGET_SET=0
|
|
131
|
+
ORIGIN_SOURCE=""
|
|
132
|
+
ORIGIN_TYPE=""
|
|
133
|
+
ORIGIN_REF=""
|
|
134
|
+
MAX_FILES=4000
|
|
135
|
+
JSON_OUT=""
|
|
136
|
+
|
|
137
|
+
while [ "$#" -gt 0 ]; do
|
|
138
|
+
case "$1" in
|
|
139
|
+
--origin-source)
|
|
140
|
+
[ "$#" -ge 2 ] || die 1 "--origin-source requires a value"
|
|
141
|
+
ORIGIN_SOURCE="$2"; shift 2 ;;
|
|
142
|
+
--origin-source=*)
|
|
143
|
+
ORIGIN_SOURCE="${1#--origin-source=}"; shift ;;
|
|
144
|
+
--origin-type)
|
|
145
|
+
[ "$#" -ge 2 ] || die 1 "--origin-type requires a value"
|
|
146
|
+
ORIGIN_TYPE="$2"; shift 2 ;;
|
|
147
|
+
--origin-type=*)
|
|
148
|
+
ORIGIN_TYPE="${1#--origin-type=}"; shift ;;
|
|
149
|
+
--origin-ref)
|
|
150
|
+
[ "$#" -ge 2 ] || die 1 "--origin-ref requires a value"
|
|
151
|
+
ORIGIN_REF="$2"; shift 2 ;;
|
|
152
|
+
--origin-ref=*)
|
|
153
|
+
ORIGIN_REF="${1#--origin-ref=}"; shift ;;
|
|
154
|
+
--max-files)
|
|
155
|
+
[ "$#" -ge 2 ] || die 1 "--max-files requires a value"
|
|
156
|
+
MAX_FILES="$2"; shift 2 ;;
|
|
157
|
+
--max-files=*)
|
|
158
|
+
MAX_FILES="${1#--max-files=}"; shift ;;
|
|
159
|
+
--json-out)
|
|
160
|
+
[ "$#" -ge 2 ] || die 1 "--json-out requires a value"
|
|
161
|
+
JSON_OUT="$2"; shift 2 ;;
|
|
162
|
+
--json-out=*)
|
|
163
|
+
JSON_OUT="${1#--json-out=}"; shift ;;
|
|
164
|
+
-h|--help)
|
|
165
|
+
usage ;;
|
|
166
|
+
--)
|
|
167
|
+
shift
|
|
168
|
+
if [ "$#" -gt 0 ]; then TARGET="$1"; TARGET_SET=1; shift; fi
|
|
169
|
+
[ "$#" -eq 0 ] || die 1 "unexpected extra argument: $1" ;;
|
|
170
|
+
-*)
|
|
171
|
+
die 1 "unknown option: $1 (use -- before a target that starts with '-')" ;;
|
|
172
|
+
*)
|
|
173
|
+
[ "$TARGET_SET" -eq 0 ] || die 1 "unexpected extra argument: $1"
|
|
174
|
+
TARGET="$1"; TARGET_SET=1; shift ;;
|
|
175
|
+
esac
|
|
176
|
+
done
|
|
177
|
+
|
|
178
|
+
[ "$TARGET_SET" -eq 1 ] || usage
|
|
179
|
+
|
|
180
|
+
case "$MAX_FILES" in
|
|
181
|
+
''|*[!0-9]*) die 1 "--max-files must be a positive integer, got: $MAX_FILES" ;;
|
|
182
|
+
esac
|
|
183
|
+
[ "$MAX_FILES" -gt 0 ] || die 1 "--max-files must be a positive integer, got: $MAX_FILES"
|
|
184
|
+
|
|
185
|
+
if [ -n "$ORIGIN_TYPE" ]; then
|
|
186
|
+
case "$ORIGIN_TYPE" in
|
|
187
|
+
git|path|snippet) ;;
|
|
188
|
+
*) die 1 "invalid --origin-type '$ORIGIN_TYPE' (expected git, path, or snippet)" ;;
|
|
189
|
+
esac
|
|
190
|
+
fi
|
|
191
|
+
|
|
192
|
+
command -v python3 >/dev/null 2>&1 || die 3 "python3 is required but was not found on PATH"
|
|
193
|
+
|
|
194
|
+
[ -e "$TARGET" ] || die 2 "target does not exist: $TARGET"
|
|
195
|
+
[ -d "$TARGET" ] || die 2 "target is not a directory: $TARGET (inspect the acquired content_path, not a single file)"
|
|
196
|
+
|
|
197
|
+
TARGET_ABS="$(cd -- "$TARGET" && pwd -P)"
|
|
198
|
+
|
|
199
|
+
if [ -n "$JSON_OUT" ]; then
|
|
200
|
+
JSON_OUT_DIR="$(dirname -- "$JSON_OUT")"
|
|
201
|
+
[ -d "$JSON_OUT_DIR" ] || die 1 "--json-out directory does not exist: $JSON_OUT_DIR"
|
|
202
|
+
[ -w "$JSON_OUT_DIR" ] || die 1 "--json-out directory is not writable: $JSON_OUT_DIR"
|
|
203
|
+
JSON_OUT="$(cd -- "$JSON_OUT_DIR" && pwd -P)/$(basename -- "$JSON_OUT")"
|
|
204
|
+
fi
|
|
205
|
+
|
|
206
|
+
# ---------------------------------------------------------------------------
|
|
207
|
+
# Nested .git detection (bash + git; read-only plumbing only)
|
|
208
|
+
#
|
|
209
|
+
# import-source.sh only strips the TOP-LEVEL .git of what it acquires, so a
|
|
210
|
+
# staged tree can still legitimately contain nested .git directories (a
|
|
211
|
+
# vendored checkout, an uncleaned submodule). Detected here in bash -- via
|
|
212
|
+
# git itself -- rather than in python, because shelling out to git for
|
|
213
|
+
# read-only plumbing is what git is for.
|
|
214
|
+
#
|
|
215
|
+
# Bounded to avoid a pathological walk through a huge staged tree: common
|
|
216
|
+
# build/dependency directory NAMES are pruned, matching the SKIP_DIRS set the
|
|
217
|
+
# python content scan below also uses, kept in sync deliberately.
|
|
218
|
+
# ---------------------------------------------------------------------------
|
|
219
|
+
|
|
220
|
+
NESTED_REPORT_FILE="$(mktemp "${TMPDIR:-/tmp}/inspect-provenance-nested.XXXXXXXX")"
|
|
221
|
+
trap 'rm -f "$NESTED_REPORT_FILE"' EXIT
|
|
222
|
+
|
|
223
|
+
PRUNE_NAMES=(node_modules .venv venv __pycache__ dist build .next coverage
|
|
224
|
+
vendor target .mypy_cache .pytest_cache .tox .gradle .idea
|
|
225
|
+
.svn .hg bower_components .terraform)
|
|
226
|
+
|
|
227
|
+
PRUNE_EXPR=()
|
|
228
|
+
for name in "${PRUNE_NAMES[@]}"; do
|
|
229
|
+
[ "${#PRUNE_EXPR[@]}" -eq 0 ] || PRUNE_EXPR+=(-o)
|
|
230
|
+
PRUNE_EXPR+=(-name "$name")
|
|
231
|
+
done
|
|
232
|
+
|
|
233
|
+
: > "$NESTED_REPORT_FILE"
|
|
234
|
+
while IFS= read -r -d '' git_dir; do
|
|
235
|
+
repo_dir="$(dirname -- "$git_dir")"
|
|
236
|
+
rel_path="${repo_dir#"$TARGET_ABS"/}"
|
|
237
|
+
[ "$rel_path" != "$repo_dir" ] || rel_path="."
|
|
238
|
+
remote_url="$(git -C "$repo_dir" config --get remote.origin.url 2>/dev/null || true)"
|
|
239
|
+
head_sha="$(git -C "$repo_dir" rev-parse HEAD 2>/dev/null || true)"
|
|
240
|
+
printf '%s\t%s\t%s\n' "$rel_path" "$remote_url" "$head_sha" >> "$NESTED_REPORT_FILE"
|
|
241
|
+
done < <(find "$TARGET_ABS" \( "${PRUNE_EXPR[@]}" \) -prune -o -type d -name .git -print0 2>/dev/null)
|
|
242
|
+
|
|
243
|
+
# ---------------------------------------------------------------------------
|
|
244
|
+
# License / SPDX / copyright content scan (python3 -- JSON emission, no jq)
|
|
245
|
+
#
|
|
246
|
+
# Deliberately NOT wrapped in a `$(...)` capture: the macOS-shipped bash
|
|
247
|
+
# (3.2, GPLv2-final -- still `/bin/bash` on every unmodified macOS install)
|
|
248
|
+
# has a long-standing heredoc parser bug where a `<<'quoted'` heredoc nested
|
|
249
|
+
# inside a `$( ... )` command substitution stops treating its body as fully
|
|
250
|
+
# literal once the body contains certain apostrophe/comment combinations,
|
|
251
|
+
# and misparses an ordinary contraction in a python comment as an unterminated
|
|
252
|
+
# shell string. Every sibling script in this directory
|
|
253
|
+
# (detect-dependencies.sh, detect-tests.sh, discover-subsystems.sh,
|
|
254
|
+
# apply-subsystem-cap.sh, write-backfilled-epics.sh) avoids this entirely by
|
|
255
|
+
# never wrapping its `python3 -c "$PY_SRC"` / `python3 - args <<'PY'` call in
|
|
256
|
+
# a capture -- python writes stdout (and, when requested, --json-out)
|
|
257
|
+
# directly. This script follows the same convention for the same reason.
|
|
258
|
+
# ---------------------------------------------------------------------------
|
|
259
|
+
|
|
260
|
+
python3 - "$TARGET_ABS" "$ORIGIN_SOURCE" "$ORIGIN_TYPE" "$ORIGIN_REF" "$MAX_FILES" "$NESTED_REPORT_FILE" "$JSON_OUT" <<'PY'
|
|
261
|
+
import json
|
|
262
|
+
import os
|
|
263
|
+
import re
|
|
264
|
+
import sys
|
|
265
|
+
|
|
266
|
+
TARGET, ORIGIN_SOURCE, ORIGIN_TYPE, ORIGIN_REF, MAX_FILES_S, NESTED_REPORT_FILE, JSON_OUT = sys.argv[1:8]
|
|
267
|
+
MAX_FILES = int(MAX_FILES_S)
|
|
268
|
+
MAX_BYTES = 262144 # 256KB -- generous for a license file, a hard stop for anything else
|
|
269
|
+
COPYRIGHT_HEAD_LINES = 25
|
|
270
|
+
|
|
271
|
+
notices = []
|
|
272
|
+
|
|
273
|
+
SKIP_DIRS = {
|
|
274
|
+
".git", "node_modules", ".venv", "venv", "__pycache__", "dist", "build",
|
|
275
|
+
".next", "coverage", "vendor", "target", ".mypy_cache", ".pytest_cache",
|
|
276
|
+
".tox", ".gradle", ".idea", ".svn", ".hg", "bower_components", ".terraform",
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
LICENSE_NAME_RE = re.compile(
|
|
280
|
+
r"^(LICENSE|LICENCE|COPYING|COPYING\.LESSER|UNLICENSE|NOTICE)(\.[A-Za-z0-9]+)?$",
|
|
281
|
+
re.IGNORECASE,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
# Ordered (most specific first) content signatures for well-known licenses.
|
|
285
|
+
# This is a heuristic classifier, not a legal determination -- a match means
|
|
286
|
+
# "the text strongly resembles this license", nothing stronger.
|
|
287
|
+
LICENSE_SIGNATURES = [
|
|
288
|
+
("Apache-2.0", [r"apache license", r"version\s*2\.0"]),
|
|
289
|
+
("GPL-3.0", [r"gnu general public license", r"version\s*3"]),
|
|
290
|
+
("GPL-2.0", [r"gnu general public license", r"version\s*2"]),
|
|
291
|
+
("LGPL-3.0", [r"gnu lesser general public license", r"version\s*3"]),
|
|
292
|
+
("LGPL-2.1", [r"gnu lesser general public license", r"version\s*2\.1"]),
|
|
293
|
+
("MPL-2.0", [r"mozilla public license", r"version\s*2\.0"]),
|
|
294
|
+
("BSD-3-Clause", [r"redistributions of source code must retain",
|
|
295
|
+
r"redistributions in binary form",
|
|
296
|
+
r"may be used to endorse or promote products"]),
|
|
297
|
+
("BSD-2-Clause", [r"redistributions of source code must retain",
|
|
298
|
+
r"redistributions in binary form"]),
|
|
299
|
+
("Unlicense", [r"this is free and unencumbered software released into the public domain"]),
|
|
300
|
+
("ISC", [r"permission to use, copy, modify, and/or distribute this software"]),
|
|
301
|
+
("MIT", [r"permission is hereby granted, free of charge",
|
|
302
|
+
r"the software is provided \"as is\""]),
|
|
303
|
+
]
|
|
304
|
+
|
|
305
|
+
SPDX_RE = re.compile(r"SPDX-License-Identifier:\s*([A-Za-z0-9.\-+() ]+?)\s*(?:\*/|-->|#>)?\s*$")
|
|
306
|
+
COPYRIGHT_RE = re.compile(r"copyright\b.{0,120}", re.IGNORECASE)
|
|
307
|
+
COPYRIGHT_HAS_SIGNAL_RE = re.compile(r"(\(c\)|©|\d{4})", re.IGNORECASE)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def rel(path):
|
|
311
|
+
r = os.path.relpath(path, TARGET)
|
|
312
|
+
return r.replace(os.sep, "/") if r != "." else os.path.basename(path) or "."
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def is_probably_binary(path):
|
|
316
|
+
try:
|
|
317
|
+
with open(path, "rb") as fh:
|
|
318
|
+
chunk = fh.read(1024)
|
|
319
|
+
except OSError:
|
|
320
|
+
return True
|
|
321
|
+
return b"\x00" in chunk
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def read_text(path, max_bytes=MAX_BYTES):
|
|
325
|
+
try:
|
|
326
|
+
if os.path.getsize(path) > max_bytes * 8:
|
|
327
|
+
# Absurdly large for a source file we'd scan headers of -- skip outright.
|
|
328
|
+
return None
|
|
329
|
+
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
|
330
|
+
return fh.read(max_bytes)
|
|
331
|
+
except OSError:
|
|
332
|
+
return None
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def walk_files(root):
|
|
336
|
+
# Symlinked FILES are enumerated but never opened for content scanning
|
|
337
|
+
# below (see the security note at the call site) -- a staged tree is
|
|
338
|
+
# untrusted input, and a symlink pointing outside the staging root must
|
|
339
|
+
# not let this scan read (and then surface, in its report) an arbitrary
|
|
340
|
+
# host file. `os.walk`'s default `followlinks=False` already keeps this
|
|
341
|
+
# from recursing INTO a symlinked directory as though it were part of
|
|
342
|
+
# the tree, which covers the directory half of the same concern.
|
|
343
|
+
out = []
|
|
344
|
+
symlinks = []
|
|
345
|
+
truncated = False
|
|
346
|
+
for dirpath, dirnames, filenames in os.walk(root):
|
|
347
|
+
dirnames[:] = sorted(d for d in dirnames if d not in SKIP_DIRS)
|
|
348
|
+
for fn in sorted(filenames):
|
|
349
|
+
fpath = os.path.join(dirpath, fn)
|
|
350
|
+
if os.path.islink(fpath):
|
|
351
|
+
symlinks.append(fpath)
|
|
352
|
+
continue
|
|
353
|
+
out.append(fpath)
|
|
354
|
+
if len(out) >= MAX_FILES:
|
|
355
|
+
truncated = True
|
|
356
|
+
return out, symlinks, truncated
|
|
357
|
+
return out, symlinks, truncated
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
all_files, symlinked_files, files_truncated = walk_files(TARGET)
|
|
361
|
+
if files_truncated:
|
|
362
|
+
notices.append(
|
|
363
|
+
"file walk hit the --max-files cap of %d; some files were not scanned. "
|
|
364
|
+
"Re-run with a higher --max-files for a fuller scan of a large staged tree."
|
|
365
|
+
% MAX_FILES
|
|
366
|
+
)
|
|
367
|
+
if symlinked_files:
|
|
368
|
+
notices.append(
|
|
369
|
+
"%d symlinked file(s) were found in the staged tree and were NOT opened for "
|
|
370
|
+
"content scanning -- a symlink can point outside the staged tree, and this scan "
|
|
371
|
+
"never follows one to read its target. Inspect them manually if their target "
|
|
372
|
+
"matters: %s"
|
|
373
|
+
% (len(symlinked_files),
|
|
374
|
+
", ".join(rel(p) for p in symlinked_files[:10]) +
|
|
375
|
+
(", ..." if len(symlinked_files) > 10 else ""))
|
|
376
|
+
)
|
|
377
|
+
notices.append(
|
|
378
|
+
"content scan excludes common build/dependency directories (%s); "
|
|
379
|
+
"third-party licensing nested inside them is not surfaced by this scan."
|
|
380
|
+
% ", ".join(sorted(SKIP_DIRS - {".git"}))
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
# ---------------------------------------------------------------------------
|
|
384
|
+
# License files
|
|
385
|
+
# ---------------------------------------------------------------------------
|
|
386
|
+
|
|
387
|
+
license_files = []
|
|
388
|
+
for path in all_files:
|
|
389
|
+
name = os.path.basename(path)
|
|
390
|
+
if not LICENSE_NAME_RE.match(name):
|
|
391
|
+
continue
|
|
392
|
+
if is_probably_binary(path):
|
|
393
|
+
license_files.append({"path": rel(path), "type": "unknown"})
|
|
394
|
+
continue
|
|
395
|
+
text = read_text(path) or ""
|
|
396
|
+
normalised = re.sub(r"\s+", " ", text).lower()
|
|
397
|
+
detected = "unknown"
|
|
398
|
+
for lic_id, patterns in LICENSE_SIGNATURES:
|
|
399
|
+
if all(re.search(p, normalised) for p in patterns):
|
|
400
|
+
detected = lic_id
|
|
401
|
+
break
|
|
402
|
+
license_files.append({"path": rel(path), "type": detected})
|
|
403
|
+
|
|
404
|
+
license_files.sort(key=lambda e: e["path"])
|
|
405
|
+
license_section = {
|
|
406
|
+
"found": len(license_files) > 0,
|
|
407
|
+
"files": license_files,
|
|
408
|
+
"message": None if license_files else
|
|
409
|
+
"no LICENSE, LICENCE, COPYING, UNLICENSE, or NOTICE file was found in the staged source",
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
# ---------------------------------------------------------------------------
|
|
413
|
+
# SPDX identifiers + copyright lines -- single content pass over non-license,
|
|
414
|
+
# non-binary files (license files are already fully classified above, but a
|
|
415
|
+
# copyright/SPDX line inside one is still worth surfacing, so they are not
|
|
416
|
+
# excluded from this pass).
|
|
417
|
+
# ---------------------------------------------------------------------------
|
|
418
|
+
|
|
419
|
+
spdx_index = {} # id -> {"id":..., "occurrences":..., "sources":[...]}
|
|
420
|
+
copyright_lines = []
|
|
421
|
+
copyright_seen = set()
|
|
422
|
+
files_scanned = 0
|
|
423
|
+
|
|
424
|
+
for path in all_files:
|
|
425
|
+
if is_probably_binary(path):
|
|
426
|
+
continue
|
|
427
|
+
text = read_text(path)
|
|
428
|
+
if text is None:
|
|
429
|
+
continue
|
|
430
|
+
files_scanned += 1
|
|
431
|
+
source = rel(path)
|
|
432
|
+
|
|
433
|
+
for line in text.splitlines():
|
|
434
|
+
m = SPDX_RE.search(line)
|
|
435
|
+
if m:
|
|
436
|
+
spdx_id = m.group(1).strip()
|
|
437
|
+
if not spdx_id:
|
|
438
|
+
continue
|
|
439
|
+
entry = spdx_index.setdefault(
|
|
440
|
+
spdx_id, {"id": spdx_id, "occurrences": 0, "sources": []}
|
|
441
|
+
)
|
|
442
|
+
entry["occurrences"] += 1
|
|
443
|
+
if source not in entry["sources"] and len(entry["sources"]) < 5:
|
|
444
|
+
entry["sources"].append(source)
|
|
445
|
+
|
|
446
|
+
for line in text.splitlines()[:COPYRIGHT_HEAD_LINES]:
|
|
447
|
+
m = COPYRIGHT_RE.search(line)
|
|
448
|
+
if not m or not COPYRIGHT_HAS_SIGNAL_RE.search(m.group(0)):
|
|
449
|
+
continue
|
|
450
|
+
cleaned = re.sub(r"^[\s#/*\-]+", "", line).strip()
|
|
451
|
+
cleaned = re.sub(r"[\s*/\-]+$", "", cleaned)
|
|
452
|
+
if not cleaned:
|
|
453
|
+
continue
|
|
454
|
+
key = (cleaned, source)
|
|
455
|
+
if key in copyright_seen:
|
|
456
|
+
continue
|
|
457
|
+
copyright_seen.add(key)
|
|
458
|
+
if len(copyright_lines) < 200:
|
|
459
|
+
copyright_lines.append({"text": cleaned, "source": source})
|
|
460
|
+
|
|
461
|
+
spdx_identifiers = sorted(spdx_index.values(), key=lambda e: e["id"])
|
|
462
|
+
spdx_section = {
|
|
463
|
+
"found": len(spdx_identifiers) > 0,
|
|
464
|
+
"identifiers": spdx_identifiers,
|
|
465
|
+
"message": None if spdx_identifiers else
|
|
466
|
+
"no SPDX-License-Identifier headers were found in any scanned file",
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
copyright_section = {
|
|
470
|
+
"found": len(copyright_lines) > 0,
|
|
471
|
+
"lines": copyright_lines,
|
|
472
|
+
"message": None if copyright_lines else
|
|
473
|
+
"no copyright lines were found near the top of any scanned file",
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
# ---------------------------------------------------------------------------
|
|
477
|
+
# git origin -- forwarded acquisition metadata + nested repos found on disk
|
|
478
|
+
# ---------------------------------------------------------------------------
|
|
479
|
+
|
|
480
|
+
origin = None
|
|
481
|
+
if ORIGIN_TYPE == "git" or ORIGIN_SOURCE or ORIGIN_REF:
|
|
482
|
+
origin = {
|
|
483
|
+
"source": ORIGIN_SOURCE or None,
|
|
484
|
+
"source_type": ORIGIN_TYPE or None,
|
|
485
|
+
"head_sha": ORIGIN_REF or None,
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
nested_repos = []
|
|
489
|
+
try:
|
|
490
|
+
with open(NESTED_REPORT_FILE, "r", encoding="utf-8", errors="replace") as fh:
|
|
491
|
+
for line in fh:
|
|
492
|
+
line = line.rstrip("\n")
|
|
493
|
+
if not line:
|
|
494
|
+
continue
|
|
495
|
+
parts = line.split("\t")
|
|
496
|
+
path = parts[0] if len(parts) > 0 else "."
|
|
497
|
+
remote = parts[1] if len(parts) > 1 and parts[1] else None
|
|
498
|
+
head = parts[2] if len(parts) > 2 and parts[2] else None
|
|
499
|
+
nested_repos.append({"path": path, "remote_url": remote, "head_sha": head})
|
|
500
|
+
except OSError:
|
|
501
|
+
pass
|
|
502
|
+
|
|
503
|
+
git_applicable = origin is not None or len(nested_repos) > 0
|
|
504
|
+
git_message = None
|
|
505
|
+
if not git_applicable:
|
|
506
|
+
git_message = (
|
|
507
|
+
"no git origin information is available -- either the source was not "
|
|
508
|
+
"git-acquired, or no --origin-* flags were passed and no nested .git "
|
|
509
|
+
"directory was found in the staged tree"
|
|
510
|
+
)
|
|
511
|
+
elif origin is None:
|
|
512
|
+
notices.append(
|
|
513
|
+
"nested git repositories were found but no --origin-* flags were passed; "
|
|
514
|
+
"the acquiring import-source.sh run's own summary (source/source_type/git_ref) "
|
|
515
|
+
"was not forwarded to this scan"
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
git_origin_section = {
|
|
519
|
+
"applicable": git_applicable,
|
|
520
|
+
"origin": origin,
|
|
521
|
+
"nested_repos": nested_repos,
|
|
522
|
+
"message": git_message,
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
# ---------------------------------------------------------------------------
|
|
526
|
+
# Overall
|
|
527
|
+
# ---------------------------------------------------------------------------
|
|
528
|
+
|
|
529
|
+
any_signal_found = (
|
|
530
|
+
license_section["found"]
|
|
531
|
+
or spdx_section["found"]
|
|
532
|
+
or copyright_section["found"]
|
|
533
|
+
or (git_origin_section["applicable"] and (origin is not None or nested_repos))
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
overall = {
|
|
537
|
+
"any_signal_found": any_signal_found,
|
|
538
|
+
"message": None if any_signal_found else
|
|
539
|
+
"no license information was found in this staged source -- this means the scan found "
|
|
540
|
+
"nothing, NOT that the source is unencumbered. Verify with the original source before "
|
|
541
|
+
"importing it into the repository.",
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
result = {
|
|
545
|
+
"script": "inspect-provenance.sh",
|
|
546
|
+
"version": 1,
|
|
547
|
+
"target": TARGET,
|
|
548
|
+
"files_scanned": files_scanned,
|
|
549
|
+
"files_truncated": files_truncated,
|
|
550
|
+
"symlinked_files_excluded": len(symlinked_files),
|
|
551
|
+
"license": license_section,
|
|
552
|
+
"spdx_identifiers": spdx_section,
|
|
553
|
+
"copyright": copyright_section,
|
|
554
|
+
"git_origin": git_origin_section,
|
|
555
|
+
"overall": overall,
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
rendered = json.dumps(result, indent=2, ensure_ascii=False) + "\n"
|
|
559
|
+
|
|
560
|
+
if JSON_OUT:
|
|
561
|
+
try:
|
|
562
|
+
with open(JSON_OUT, "w", encoding="utf-8") as fh:
|
|
563
|
+
fh.write(rendered)
|
|
564
|
+
except OSError as exc:
|
|
565
|
+
sys.stderr.write("%s: error: failed to write --json-out to %s: %s\n"
|
|
566
|
+
% ("inspect-provenance.sh", JSON_OUT, exc))
|
|
567
|
+
sys.exit(4)
|
|
568
|
+
|
|
569
|
+
sys.stdout.write(rendered)
|
|
570
|
+
|
|
571
|
+
for n in notices:
|
|
572
|
+
sys.stderr.write("Notice: %s\n" % n)
|
|
573
|
+
PY
|