@tduyng/pi-devos 0.0.0-stage → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +1421 -0
  2. package/LICENSE +201 -0
  3. package/README.md +206 -2
  4. package/package.json +44 -4
  5. package/plugins/devos/agents/reader.md +27 -0
  6. package/plugins/devos/agents/verifier.md +155 -0
  7. package/plugins/devos/pi/extension/devos.ts +605 -0
  8. package/plugins/devos/scripts/check-evidence-router.sh +140 -0
  9. package/plugins/devos/scripts/evidence-router.py +254 -0
  10. package/plugins/devos/scripts/gates/check-dod-core.sh +285 -0
  11. package/plugins/devos/scripts/gates/check-dod.sh +38 -0
  12. package/plugins/devos/scripts/gates/check-implementation-boundary.sh +99 -0
  13. package/plugins/devos/scripts/gates/check-port-source.sh +134 -0
  14. package/plugins/devos/scripts/gates/check-report-locations.sh +148 -0
  15. package/plugins/devos/scripts/gates/check-scenario-coverage.sh +104 -0
  16. package/plugins/devos/scripts/gates/gate-check.mjs +204 -0
  17. package/plugins/devos/scripts/gates/phase-gate.sh +208 -0
  18. package/plugins/devos/scripts/gates/plan-template-check.py +141 -0
  19. package/plugins/devos/scripts/gates/run-gate.sh +148 -0
  20. package/plugins/devos/scripts/gates/spec-contract.sh +19 -0
  21. package/plugins/devos/scripts/gates/spec-template-check.py +152 -0
  22. package/plugins/devos/scripts/gates/suite/01-dod.sh +485 -0
  23. package/plugins/devos/scripts/gates/suite/02-integrity.sh +310 -0
  24. package/plugins/devos/scripts/gates/suite/03-archetype.sh +255 -0
  25. package/plugins/devos/scripts/gates/suite/04-spec-validator.sh +183 -0
  26. package/plugins/devos/scripts/gates/suite/05-requires.sh +360 -0
  27. package/plugins/devos/scripts/gates/suite/06-clarify-adapters.sh +435 -0
  28. package/plugins/devos/scripts/gates/suite/07-lint-evals.sh +350 -0
  29. package/plugins/devos/scripts/gates/suite/08-converge.sh +591 -0
  30. package/plugins/devos/scripts/gates/suite/09-quality.sh +516 -0
  31. package/plugins/devos/scripts/gates/suite/10-eval-harness.sh +679 -0
  32. package/plugins/devos/scripts/gates/test-gates.sh +76 -0
  33. package/plugins/devos/scripts/gates/test-integrity-rubric.sh +152 -0
  34. package/plugins/devos/scripts/gates/test-phase-gate.sh +116 -0
  35. package/plugins/devos/skills/absorb/SKILL.md +75 -0
  36. package/plugins/devos/skills/absorb/references/distill.md +67 -0
  37. package/plugins/devos/skills/absorb/scripts/check-refs.sh +108 -0
  38. package/plugins/devos/skills/absorb/scripts/knowledge.py +258 -0
  39. package/plugins/devos/skills/build/SKILL.md +71 -0
  40. package/plugins/devos/skills/build/references/advanced-routing.md +19 -0
  41. package/plugins/devos/skills/build/references/defect.md +17 -0
  42. package/plugins/devos/skills/build/references/exposure.md +28 -0
  43. package/plugins/devos/skills/build/references/hard.md +39 -0
  44. package/plugins/devos/skills/build/references/refactor.md +66 -0
  45. package/plugins/devos/skills/build/references/routes.md +56 -0
  46. package/plugins/devos/skills/build/references/scale.md +84 -0
  47. package/plugins/devos/skills/build/references/spike.md +19 -0
  48. package/plugins/devos/skills/build/scripts/construct-census.py +204 -0
  49. package/plugins/devos/skills/build/scripts/dep-order.py +297 -0
  50. package/plugins/devos/skills/build/scripts/ledger.sh +228 -0
  51. package/plugins/devos/skills/build/scripts/route.sh +5 -0
  52. package/plugins/devos/skills/build/scripts/secret-scan.sh +121 -0
  53. package/plugins/devos/skills/build/scripts/toolchain-preflight.py +302 -0
  54. package/plugins/devos/skills/frame/SKILL.md +54 -0
  55. package/plugins/devos/skills/frame/references/template.md +115 -0
  56. package/plugins/devos/skills/frame/scripts/survey.sh +44 -0
  57. package/plugins/devos/skills/implement/SKILL.md +36 -0
  58. package/plugins/devos/skills/implement/references/conditional-plan.md +17 -0
  59. package/plugins/devos/skills/implement/references/debug-loop.md +7 -0
  60. package/plugins/devos/skills/implement/references/dod.md +37 -0
  61. package/plugins/devos/skills/implement/references/implement.md +54 -0
  62. package/plugins/devos/skills/implement/references/manual-gates.md +16 -0
  63. package/plugins/devos/skills/implement/references/plan.md +142 -0
  64. package/plugins/devos/skills/implement/references/resume.md +9 -0
  65. package/plugins/devos/skills/implement/references/scale.md +86 -0
  66. package/plugins/devos/skills/implement/references/verify-close.md +62 -0
  67. package/plugins/devos/skills/implement/references/visual-verification.md +7 -0
  68. package/plugins/devos/skills/implement/scripts/check-converge.sh +147 -0
  69. package/plugins/devos/skills/implement/scripts/equivalence-check.sh +140 -0
  70. package/plugins/devos/skills/implement/scripts/manifest-queue.sh +185 -0
  71. package/plugins/devos/skills/implement/scripts/ops-log.sh +58 -0
  72. package/plugins/devos/skills/implement/scripts/pilot-consistency.py +173 -0
  73. package/plugins/devos/skills/implement/scripts/validate-spec.sh +220 -0
  74. package/plugins/devos/skills/init/SKILL.md +100 -0
  75. package/plugins/devos/skills/init/references/archetype-sections.md +15 -0
  76. package/plugins/devos/skills/init/references/context.md +42 -0
  77. package/plugins/devos/skills/init/references/conventions-template.md +153 -0
  78. package/plugins/devos/skills/init/references/reverse-engineer.md +3 -0
  79. package/plugins/devos/skills/init/references/rule-format.md +58 -0
  80. package/plugins/devos/skills/init/references/shape-model.md +29 -0
  81. package/plugins/devos/skills/init/scripts/detect-archetype.py +461 -0
  82. package/plugins/devos/skills/init/scripts/render-rule.py +114 -0
  83. package/plugins/devos/skills/init/scripts/reverse-engineer.sh +162 -0
  84. package/plugins/devos/skills/learn/SKILL.md +29 -0
  85. package/plugins/devos/skills/learn/references/destinations.md +43 -0
  86. package/plugins/devos/skills/learn/references/scoring.md +26 -0
  87. package/plugins/devos/skills/review/SKILL.md +54 -0
  88. package/plugins/devos/skills/review/references/comments.md +13 -0
  89. package/plugins/devos/skills/review/references/gaps.md +26 -0
  90. package/plugins/devos/skills/review/references/reviewer.md +124 -0
  91. package/plugins/devos/skills/review/references/routing.md +35 -0
  92. package/plugins/devos/skills/review/scripts/__pycache__/prepare.cpython-313.pyc +0 -0
  93. package/plugins/devos/skills/review/scripts/prepare.py +763 -0
  94. package/plugins/devos/skills/spec/SKILL.md +57 -0
  95. package/plugins/devos/skills/spec/references/prototype.md +21 -0
  96. package/plugins/devos/skills/spec/references/question-craft.md +47 -0
  97. package/plugins/devos/skills/spec/references/roadmap-link.md +5 -0
  98. package/plugins/devos/skills/spec/references/write-spec.md +128 -0
  99. package/plugins/devos/skills/spec/scripts/clarify.sh +173 -0
  100. package/plugins/devos/skills/status/SKILL.md +67 -0
  101. package/plugins/devos/skills/status/references/cost.md +38 -0
  102. package/plugins/devos/skills/status/scripts/artifact-budgets.sh +88 -0
  103. package/plugins/devos/skills/status/scripts/check-requires.py +556 -0
  104. package/plugins/devos/skills/status/scripts/context-cost.sh +89 -0
  105. package/plugins/devos/skills/status/scripts/convention-usage.sh +184 -0
  106. package/plugins/devos/skills/status/scripts/findings-summary.sh +112 -0
  107. package/plugins/devos/skills/status/scripts/quality-correlation.sh +259 -0
  108. package/plugins/devos/skills/status/scripts/sizing-calibration.sh +71 -0
  109. package/plugins/devos/skills/status/scripts/spec-diff.sh +137 -0
@@ -0,0 +1,228 @@
1
+ #!/usr/bin/env bash
2
+ # Ledger for the build skill's H route (a hard task a direct attempt failed).
3
+ #
4
+ # Filesystem state only: no network, no model call, no harness tool, so the
5
+ # route works on every adapter DevOS installs. One run is one directory holding
6
+ # `task.md`, `plan.md`, `notes.md`, `tasks.md`, the artifact and a flat `meta`
7
+ # file. The manager (main agent) reads a compact view instead of carrying worker
8
+ # transcripts, and one fresh worker per round reads the same files. The two
9
+ # decisions the paper's own transcripts credit, and which a model left to prose
10
+ # tends to drop, are enforced here: `next` refuses to let the same task be
11
+ # reissued silently, and `notes` replaces (and caps) the file instead of
12
+ # appending to it.
13
+ #
14
+ # Usage: ledger.sh <action> --dir DIR [options]
15
+ # init --dir D (--task S | --task-file F) [--artifact NAME] [--check CMD] [--max-iters N]
16
+ # read --dir D
17
+ # plan --dir D # plan text on stdin
18
+ # tasks --dir D # task-list lines on stdin
19
+ # notes --dir D [--append] # notes on stdin; replaces unless --append
20
+ # artifact --dir D [--read] # artifact on stdin, or head of it with --read
21
+ # check --dir D [--cmd C] # run the runnable check, record the verdict
22
+ # next --dir D --task S # record the round's one task, print the guard
23
+ #
24
+ # `check` exits 0 on pass, 1 on fail. Every other action exits 0 on success,
25
+ # 2 on a usage or missing-run error.
26
+ set -eu
27
+
28
+ MAX_PLAN_CHARS=4000
29
+ MAX_NOTES_CHARS=8000
30
+ READ_ARTIFACT_CHARS=4000
31
+ READ_NOTES_TAIL=1500
32
+
33
+ die() { printf 'ledger: %s\n' "$1" >&2; exit 2; }
34
+
35
+ action="${1:-}"
36
+ [ -n "$action" ] || die "usage: ledger.sh <init|read|plan|tasks|notes|artifact|check|next> --dir DIR"
37
+ shift
38
+
39
+ DIR=""; TASK=""; TASK_FILE=""; ARTIFACT=""; CHECK=""; MAX_ITERS=10
40
+ MODE="replace"; READ_ONLY=0
41
+ while [ $# -gt 0 ]; do
42
+ case "$1" in
43
+ --dir) DIR="${2:-}"; shift 2 ;;
44
+ --task) TASK="${2:-}"; shift 2 ;;
45
+ --task-file) TASK_FILE="${2:-}"; shift 2 ;;
46
+ --artifact) ARTIFACT="${2:-}"; shift 2 ;;
47
+ --check|--cmd) CHECK="${2:-}"; shift 2 ;;
48
+ --max-iters) MAX_ITERS="${2:-}"; shift 2 ;;
49
+ --append) MODE="append"; shift ;;
50
+ --replace) MODE="replace"; shift ;;
51
+ --read) READ_ONLY=1; shift ;;
52
+ *) die "unknown argument: $1" ;;
53
+ esac
54
+ done
55
+
56
+ [ -n "$DIR" ] || die "--dir is required"
57
+
58
+ # Flat key=value file: readable with one awk, writable without jq, and the value
59
+ # may itself contain `=` (a check command), which is why only the first field is
60
+ # compared.
61
+ meta_get() {
62
+ [ -f "$DIR/meta" ] || return 0
63
+ awk -F= -v k="$1" '$1==k{sub(/^[^=]*=/,"");print;exit}' "$DIR/meta"
64
+ }
65
+ meta_set() {
66
+ tmp="$DIR/.meta.tmp"
67
+ if [ -f "$DIR/meta" ]; then
68
+ awk -F= -v k="$1" '$1!=k' "$DIR/meta" > "$tmp"
69
+ else
70
+ : > "$tmp"
71
+ fi
72
+ printf '%s=%s\n' "$1" "$2" >> "$tmp"
73
+ mv "$tmp" "$DIR/meta"
74
+ }
75
+ need_run() { [ -f "$DIR/meta" ] || die "no initialized run at $DIR"; }
76
+ artifact_name() { printf '%s' "$(meta_get artifact)"; }
77
+
78
+ case "$action" in
79
+ init)
80
+ case "$MAX_ITERS" in
81
+ ''|*[!0-9]*) die "max-iters must be a positive integer: '$MAX_ITERS'" ;;
82
+ esac
83
+ MAX_ITERS=$((10#$MAX_ITERS))
84
+ (( MAX_ITERS > 0 )) || die "max-iters must be a positive integer: '$MAX_ITERS'"
85
+ [ -n "$TASK_FILE" ] && TASK="$(cat "$TASK_FILE")"
86
+ [ -n "$TASK" ] || die "init needs --task or --task-file"
87
+ [ -f "$DIR/meta" ] && die "run already initialized: $DIR (use a new --dir)"
88
+ ARTIFACT="${ARTIFACT:-solution.py}"
89
+ case "$ARTIFACT" in
90
+ ''|.|..|*/*|/*) die "artifact must be a file name, not a path: '$ARTIFACT'" ;;
91
+ esac
92
+ mkdir -p "$DIR"
93
+ printf '%s\n' "$TASK" > "$DIR/task.md"
94
+ : > "$DIR/plan.md"
95
+ : > "$DIR/notes.md"
96
+ : > "$DIR/tasks.md"
97
+ : > "$DIR/$ARTIFACT"
98
+ {
99
+ printf 'round=0\n'
100
+ printf 'max_iters=%s\n' "$MAX_ITERS"
101
+ printf 'artifact=%s\n' "$ARTIFACT"
102
+ printf 'check=%s\n' "$CHECK"
103
+ printf 'last_task=\n'
104
+ printf 'verdict=\n'
105
+ } > "$DIR/meta"
106
+ printf 'ledger initialized: %s (artifact %s, %s rounds, check: %s)\n' \
107
+ "$DIR" "$ARTIFACT" "$MAX_ITERS" "${CHECK:-none}"
108
+ ;;
109
+ read)
110
+ need_run
111
+ art="$(artifact_name)"
112
+ size=0
113
+ [ -f "$DIR/$art" ] && size="$(wc -c < "$DIR/$art" | tr -d ' ')"
114
+ printf 'run: %s\n' "$DIR"
115
+ printf 'round: %s/%s\n' "$(meta_get round)" "$(meta_get max_iters)"
116
+ printf 'artifact: %s (%s)\n' "$art" "$([ "$size" -gt 0 ] && printf '%s chars' "$size" || printf 'empty')"
117
+ printf 'verdict: %s\n' "$(meta_get verdict)"
118
+ printf 'last task: %s\n' "$(meta_get last_task)"
119
+ printf 'check: %s\n' "$(meta_get check)"
120
+ printf '\nplan:\n'
121
+ cat "$DIR/plan.md"
122
+ printf '\ntasks:\n'
123
+ cat "$DIR/tasks.md"
124
+ printf '\nnotes (tail):\n'
125
+ tail -c "$READ_NOTES_TAIL" "$DIR/notes.md"
126
+ if [ "$size" -gt 0 ]; then
127
+ printf '\nartifact (head):\n'
128
+ head -c "$READ_ARTIFACT_CHARS" "$DIR/$art"
129
+ printf '\n'
130
+ fi
131
+ ;;
132
+ plan)
133
+ need_run
134
+ cat > "$DIR/.plan.in"
135
+ plan_in=$(wc -c < "$DIR/.plan.in" | tr -d ' ')
136
+ head -c "$MAX_PLAN_CHARS" "$DIR/.plan.in" > "$DIR/plan.md"
137
+ rm -f "$DIR/.plan.in"
138
+ printf 'plan written (%s chars on disk)\n' "$(wc -c < "$DIR/plan.md" | tr -d ' ')"
139
+ # A cap that reports success is a lie the manager cannot see: it reads the
140
+ # plan back whole and hands the worker the half that survived. Name what
141
+ # was dropped, and how much.
142
+ [ "$plan_in" -le "$MAX_PLAN_CHARS" ] || printf 'TRUNCATED: plan was %s chars, cap is %s, the last %s chars did not land. Shorten it or split the work across rounds.\n' \
143
+ "$plan_in" "$MAX_PLAN_CHARS" "$((plan_in - MAX_PLAN_CHARS))"
144
+ ;;
145
+ tasks)
146
+ need_run
147
+ cat > "$DIR/.tasks.tmp"
148
+ if grep '^- \[' "$DIR/.tasks.tmp" > "$DIR/tasks.md"; then :; else : > "$DIR/tasks.md"; fi
149
+ rm -f "$DIR/.tasks.tmp"
150
+ printf 'tasks written (%s)\n' "$(grep -c '^- \[' "$DIR/tasks.md" || true)"
151
+ ;;
152
+ notes)
153
+ need_run
154
+ cat > "$DIR/.notes.in"
155
+ notes_in=$(wc -c < "$DIR/.notes.in" | tr -d ' ')
156
+ if [ "$MODE" = "append" ] && [ -s "$DIR/notes.md" ]; then
157
+ cat "$DIR/notes.md" "$DIR/.notes.in" > "$DIR/.notes.all"
158
+ notes_in=$(wc -c < "$DIR/.notes.all" | tr -d ' ')
159
+ head -c "$MAX_NOTES_CHARS" "$DIR/.notes.all" > "$DIR/notes.md"
160
+ rm -f "$DIR/.notes.all"
161
+ else
162
+ head -c "$MAX_NOTES_CHARS" "$DIR/.notes.in" > "$DIR/notes.md"
163
+ fi
164
+ rm -f "$DIR/.notes.in"
165
+ printf 'notes %s (%s chars on disk, cap %s)\n' \
166
+ "$([ "$MODE" = "append" ] && printf appended || printf replaced)" \
167
+ "$(wc -c < "$DIR/notes.md" | tr -d ' ')" "$MAX_NOTES_CHARS"
168
+ # On append the cap cuts the end, which is the text just handed in, so
169
+ # report the append that only partly happened.
170
+ [ "$notes_in" -le "$MAX_NOTES_CHARS" ] || printf 'TRUNCATED: notes reached %s chars against a cap of %s, the last %s chars did not land. The newest findings are what the cap drops, so record them shorter.\n' \
171
+ "$notes_in" "$MAX_NOTES_CHARS" "$((notes_in - MAX_NOTES_CHARS))"
172
+ ;;
173
+ artifact)
174
+ need_run
175
+ art="$(artifact_name)"
176
+ if [ "$READ_ONLY" = "1" ]; then
177
+ if [ -s "$DIR/$art" ]; then
178
+ head -c "$READ_ARTIFACT_CHARS" "$DIR/$art"
179
+ printf '\n'
180
+ else
181
+ printf '%s is empty\n' "$art"
182
+ fi
183
+ else
184
+ head -c 200000 > "$DIR/$art"
185
+ printf '%s written (%s chars)\n' "$art" "$(wc -c < "$DIR/$art" | tr -d ' ')"
186
+ fi
187
+ ;;
188
+ check)
189
+ need_run
190
+ [ -n "$CHECK" ] || CHECK="$(meta_get check)"
191
+ [ -n "$CHECK" ] || die "no check command (pass --check, or set it at init)"
192
+ rc=0
193
+ out="$(cd "$DIR" && sh -c "$CHECK" 2>&1)" || rc=$?
194
+ if [ "$rc" -eq 0 ]; then
195
+ meta_set verdict "pass"
196
+ printf 'VERDICT: PASS\n'
197
+ else
198
+ meta_set verdict "fail (exit $rc)"
199
+ printf 'VERDICT: FAIL (exit %s)\n' "$rc"
200
+ fi
201
+ printf '%s\n' "$out" | tail -n 20
202
+ [ "$rc" -eq 0 ]
203
+ ;;
204
+ next)
205
+ need_run
206
+ [ -n "$TASK" ] || die "next needs --task"
207
+ round="$(meta_get round)"; round="${round:-0}"
208
+ max_iters="$(meta_get max_iters)"; max_iters="${max_iters:-$MAX_ITERS}"
209
+ if (( round >= max_iters )); then
210
+ printf 'STOP: iteration budget exhausted (%s/%s)\n' "$round" "$max_iters" >&2
211
+ exit 1
212
+ fi
213
+ prev="$(meta_get last_task)"
214
+ cur_n="$(printf '%s' "$TASK" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
215
+ prev_n="$(printf '%s' "$prev" | tr '[:upper:]' '[:lower:]' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
216
+ meta_set last_task "$TASK"
217
+ round="$(($round + 1))"
218
+ meta_set round "$round"
219
+ printf 'round %s/%s\n' "$round" "$(meta_get max_iters)"
220
+ printf 'task: %s\n' "$TASK"
221
+ if [ -n "$prev_n" ] && [ "$cur_n" = "$prev_n" ]; then
222
+ printf 'GUARD: this is the exact task you just handed out. Stop and finalize, or switch to a genuinely different approach.\n'
223
+ fi
224
+ ;;
225
+ *)
226
+ die "unknown action: $action"
227
+ ;;
228
+ esac
@@ -0,0 +1,5 @@
1
+ #!/bin/bash
2
+ # route.sh <F|S|M|L|R|B|C|H> "<evidence>": commit the size /devos:build picked.
3
+ # The before-tool hook (scripts/gates/phase-gate.sh) records it and refuses a
4
+ # shrink back to S. This script only prints the next step.
5
+ exec bash "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/../../../scripts/gates/phase-gate.sh" next "${1:-}"
@@ -0,0 +1,121 @@
1
+ #!/bin/bash
2
+ # secret-scan.sh: the credentials a tree still carries, with the values masked.
3
+ #
4
+ # A legacy tree keeps running while the port lands, so whatever it still
5
+ # carries is still exposed. The scan finds the shapes that credentials take and
6
+ # reports where they are, ranked by what they are worth to someone else.
7
+ #
8
+ # The value never reaches the output. A finding names the file, the line, the
9
+ # kind and a masked fingerprint, because a findings file that holds the
10
+ # credentials it found is a second copy of the problem. The secret is where the
11
+ # finding points; that is enough to remove it.
12
+ #
13
+ # Offline and deterministic: the patterns are read from the file, so the scan
14
+ # needs no network and no database, and the same tree gives the same report.
15
+ # It finds shapes, not secrets, so a shape that is not a credential is a
16
+ # finding to discard by hand, and a credential with an unusual shape is a miss.
17
+ #
18
+ # Usage:
19
+ # secret-scan.sh --tree DIR
20
+ #
21
+ # Diagnostic envelope (shared by DevOS check scripts):
22
+ # FAIL <check>: <message>
23
+ # fix: <one actionable line>
24
+ # Exit 0 = nothing found, 1 = findings, 2 = usage.
25
+ set -u
26
+ TREE=""
27
+ while [[ $# -gt 0 ]]; do
28
+ case "$1" in
29
+ --tree) TREE="${2:-}"; shift 2 ;;
30
+ -h | --help) sed -n '2,23p' "$0"; exit 0 ;;
31
+ *) echo "FAIL secret-scan: unknown argument: $1" >&2; echo " fix: pass --tree DIR" >&2; exit 2 ;;
32
+ esac
33
+ done
34
+ [[ -n "$TREE" ]] || { echo "FAIL secret-scan: no tree given" >&2; echo " fix: pass --tree DIR" >&2; exit 2; }
35
+ [[ -d "$TREE" ]] || { echo "FAIL secret-scan: no tree at $TREE" >&2; echo " fix: point --tree at the directory that holds the code" >&2; exit 2; }
36
+
37
+ # kind | severity | how the value sits in the match | case-insensitive | pattern
38
+ KINDS='
39
+ private key|critical|whole|no|-----BEGIN ([A-Z ]+)?PRIVATE KEY-----
40
+ cloud access key|critical|whole|no|(A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{16}
41
+ cloud secret key|critical|quoted|yes|aws_secret_access_key[[:space:]]*[:=][[:space:]]*['\''"][^'\''"]{20,}['\''"]
42
+ github token|critical|whole|no|(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36}
43
+ slack token|critical|whole|no|xox[abprs]-[A-Za-z0-9-]{10,}
44
+ google api key|critical|whole|no|AIza[0-9A-Za-z_-]{35}
45
+ json web token|high|whole|no|eyJ[A-Za-z0-9_-]{10,}[.][A-Za-z0-9_-]{10,}[.][A-Za-z0-9_-]{10,}
46
+ credential in url|high|urlpw|no|[a-zA-Z][a-zA-Z0-9+.-]*://[^/[:space:]@:]+:[^@[:space:]]+@
47
+ named credential|medium|quoted|yes|(api[_-]?key|secret[_-]?key|access[_-]?key|client[_-]?secret|password|passwd|auth[_-]?token)[[:space:]]*[:=][[:space:]]*['\''"][^'\''"]{8,}['\''"]
48
+ '
49
+
50
+ PRUNE_DIRS=(node_modules .git vendor target dist build .venv venv __pycache__ coverage .next .tox site-packages)
51
+ prune_flags=()
52
+ for d in "${PRUNE_DIRS[@]}"; do
53
+ prune_flags+=(--exclude-dir="$d")
54
+ done
55
+
56
+ printf '| Severity | Kind | Site | Fingerprint |\n|---|---|---|---|\n'
57
+ found=0
58
+ while IFS='|' read -r kind severity mode ci pattern; do
59
+ [[ -n "${kind:-}" ]] || continue
60
+ ci_flag=""
61
+ [[ "$ci" == "yes" ]] && ci_flag="-i"
62
+ out=$(grep -rEnI $ci_flag "${prune_flags[@]}" -e "$pattern" "$TREE" 2>/dev/null | awk -v kind="$kind" -v sev="$severity" -v mode="$mode" -v pat="$pattern" -v ci="$ci" '
63
+ function value(v, q, rest, r, p, s, e) {
64
+ if (mode == "quoted") {
65
+ q = index(v, "\47")
66
+ if (q > 0) {
67
+ rest = substr(v, q + 1)
68
+ r = index(rest, "\47")
69
+ if (r > 0) v = substr(rest, 1, r - 1)
70
+ }
71
+ } else if (mode == "urlpw") {
72
+ p = index(v, "://")
73
+ if (p > 0) {
74
+ rest = substr(v, p + 3)
75
+ s = index(rest, ":")
76
+ if (s > 0) {
77
+ e = index(rest, "@")
78
+ if (e > s) v = substr(rest, s + 1, e - s - 1)
79
+ }
80
+ }
81
+ }
82
+ return v
83
+ }
84
+ {
85
+ line = $0
86
+ c1 = index(line, ":")
87
+ rest = substr(line, c1 + 1)
88
+ c2 = index(rest, ":")
89
+ file = substr(line, 1, c1 - 1)
90
+ lineno = substr(rest, 1, c2 - 1)
91
+ text = substr(rest, c2 + 1)
92
+ # The line is not the value. Match the pattern inside it to find
93
+ # what actually matched, and mask that.
94
+ if (ci == "yes") {
95
+ if (!match(tolower(text), tolower(pat))) next
96
+ } else {
97
+ if (!match(text, pat)) next
98
+ }
99
+ v = substr(text, RSTART, RLENGTH)
100
+ v = value(v)
101
+ gsub(/^[ \t]+|[ \t]+$/, "", v)
102
+ # A stand-in is not a credential. The shapes are broad on purpose,
103
+ # so the obvious placeholders are dropped here rather than by hand.
104
+ if (v ~ /example|changeme|placeholder|your[_-]|todo|fixme|xx+|redacted|dummy|fake|^[*]+$|^<|^$/) next
105
+ if (length(v) < 4) next
106
+ print "| " sev " | " kind " | " file ":" lineno " | " substr(v, 1, 3) "**** (" length(v) " chars) |"
107
+ }')
108
+ [[ -n "$out" ]] || continue
109
+ echo "$out"
110
+ count=$(echo "$out" | grep -c '^| ')
111
+ found=$((found + count))
112
+ done <<<"$KINDS"
113
+
114
+ echo
115
+ if [[ "$found" -gt 0 ]]; then
116
+ echo "FAIL secret-scan: $found credential shape(s) in $TREE"
117
+ echo " fix: remove each one and rotate it; the value is at the site named above"
118
+ exit 1
119
+ fi
120
+ echo "PASS secret-scan: no credential shapes in $TREE"
121
+ exit 0
@@ -0,0 +1,302 @@
1
+ #!/usr/bin/env python3
2
+ """toolchain-preflight.py: can the port start at all?
3
+
4
+ A port moves code to a stack the project does not run today, and the first
5
+ unit stalls on a missing runtime, a missing package manager, or a build
6
+ command nobody wrote. The stall costs the whole port, because the units queue
7
+ behind it.
8
+
9
+ This asks the question before the first unit, and it asks it of the machine,
10
+ not of the model: the tools are looked up on PATH, the build command is looked
11
+ up in the project's own manifest, and the project's check command is run. Three
12
+ answers come back, kept apart because the fixes differ:
13
+
14
+ ready the target stack is installed and the project's check passes
15
+ no toolchain something the port needs is not there, or not declared
16
+ check failed the toolchain is fine and the project itself is red
17
+ usage the declaration is missing or malformed
18
+
19
+ The declaration is a file in the spec folder, so it is reviewable and
20
+ diffable, and the filesystem holds the answer: no tracker can disagree with
21
+ the tree.
22
+
23
+ Usage:
24
+ toolchain-preflight.py --spec SPEC_DIR [--root PROJECT_ROOT]
25
+
26
+ Declaration file, SPEC_DIR/target-stack.tsv, one `key<TAB>value` per row:
27
+
28
+ runtime the command that runs the target code (node, python3)
29
+ package_manager the command that installs dependencies (npm, uv, cargo)
30
+ build the command that builds the target tree (npm run build)
31
+ check the command that judges the target tree (npm test)
32
+
33
+ Only `runtime` and `check` are required. A port with no build step (a script
34
+ port, a config port) leaves `build` out, and a port with no dependency step
35
+ leaves `package_manager` out.
36
+
37
+ Diagnostic envelope (shared by DevOS check scripts):
38
+ FAIL <check>: <message>
39
+ fix: <one actionable line>
40
+
41
+ Exit 0 = ready, 1 = no toolchain, 2 = usage, 3 = the project's check failed.
42
+ """
43
+
44
+ import json
45
+ import os
46
+ import shutil
47
+ import subprocess
48
+ import sys
49
+
50
+ DECLARATION = "target-stack.tsv"
51
+ REQUIRED_KEYS = ("runtime", "check")
52
+ KNOWN_KEYS = ("runtime", "package_manager", "build", "check")
53
+ # The check runs through a shell, so it may be a builtin rather than a binary.
54
+ # A builtin has no PATH entry to look up, and looking one up would report a
55
+ # tool that is not installed when it is.
56
+ SHELL_BUILTINS = frozenset(
57
+ "break cd continue echo eval exec exit export false for printf pwd read return set shift test times trap true type ulimit umask unalias unset wait"
58
+ .split()
59
+ )
60
+
61
+
62
+ def fail(check, message, fix):
63
+ print(f"FAIL {check}: {message}")
64
+ print(f" fix: {fix}")
65
+
66
+
67
+ def die(message, fix):
68
+ print(f"FAIL toolchain-preflight: {message}")
69
+ print(f" fix: {fix}", file=sys.stderr)
70
+ raise SystemExit(2)
71
+
72
+
73
+ def read_declaration(path):
74
+ rows = {}
75
+ with open(path, encoding="utf-8") as handle:
76
+ for number, line in enumerate(handle, 1):
77
+ line = line.rstrip("\n").rstrip("\r")
78
+ if not line.strip() or line.lstrip().startswith("#"):
79
+ continue
80
+ if "\t" not in line:
81
+ die(
82
+ f"{path}:{number}: no tab between key and value",
83
+ "one key<TAB>value per row, for example: check<TAB>npm test",
84
+ )
85
+ key, value = line.split("\t", 1)
86
+ key = key.strip()
87
+ value = value.strip()
88
+ if key not in KNOWN_KEYS:
89
+ die(
90
+ f"{path}:{number}: unknown key: {key}",
91
+ "the keys are: " + ", ".join(KNOWN_KEYS),
92
+ )
93
+ if not value:
94
+ die(f"{path}:{number}: {key} has no value", "give the command to run")
95
+ rows[key] = value
96
+ for key in REQUIRED_KEYS:
97
+ if key not in rows:
98
+ die(
99
+ f"{path}: no {key} row",
100
+ f"the declaration needs a {key} row, for example: {key}\t<command>",
101
+ )
102
+ return rows
103
+
104
+
105
+ def first_word(command):
106
+ return command.split()[0] if command.split() else ""
107
+
108
+
109
+ def on_path(tool):
110
+ return shutil.which(tool) is not None
111
+
112
+
113
+ def tool_version(command):
114
+ """The tool's own version line, for the report. Best effort, never fatal."""
115
+ try:
116
+ done = subprocess.run(
117
+ [command, "--version"],
118
+ capture_output=True,
119
+ text=True,
120
+ timeout=20,
121
+ )
122
+ except (OSError, subprocess.SubprocessError):
123
+ return ""
124
+ out = (done.stdout or done.stderr or "").strip().splitlines()
125
+ return out[0] if out else ""
126
+
127
+
128
+ def declared_build(root, build_command):
129
+ """Is the build command one the project itself declares?
130
+
131
+ A build command that does not exist in the project's manifest is a port
132
+ that stalls on its first unit, and it stalls late, so it is worth knowing
133
+ before the units start. The manifest shapes are the ones a port actually
134
+ moves to.
135
+ """
136
+ words = build_command.split()
137
+ if not words:
138
+ return True, ""
139
+ tool = os.path.basename(words[0])
140
+
141
+ if tool in ("npm", "pnpm", "yarn", "bun"):
142
+ if len(words) < 3 or words[1] != "run":
143
+ return True, ""
144
+ script = words[2]
145
+ manifest = os.path.join(root, "package.json")
146
+ if not os.path.isfile(manifest):
147
+ return False, f"there is no package.json in {root}"
148
+ try:
149
+ with open(manifest, encoding="utf-8") as handle:
150
+ data = json.load(handle)
151
+ except (OSError, ValueError) as error:
152
+ return False, f"package.json could not be read: {error}"
153
+ scripts = data.get("scripts") or {}
154
+ if script not in scripts:
155
+ named = ", ".join(sorted(scripts)) or "none"
156
+ return False, f"package.json has no script: {script} (it has: {named})"
157
+ return True, ""
158
+
159
+ if tool == "make":
160
+ if len(words) < 2:
161
+ return True, ""
162
+ target = words[1]
163
+ for name in ("Makefile", "makefile", "GNUmakefile"):
164
+ path = os.path.join(root, name)
165
+ if not os.path.isfile(path):
166
+ continue
167
+ with open(path, encoding="utf-8", errors="replace") as handle:
168
+ for line in handle:
169
+ stripped = line.strip()
170
+ if not stripped or stripped.startswith("#"):
171
+ continue
172
+ if stripped.split(":", 1)[0].strip() == target:
173
+ return True, ""
174
+ return False, f"{name} has no target: {target}"
175
+ return False, f"there is no Makefile in {root}"
176
+
177
+ if tool == "cargo":
178
+ if not os.path.isfile(os.path.join(root, "Cargo.toml")):
179
+ return False, f"there is no Cargo.toml in {root}"
180
+ return True, ""
181
+
182
+ if tool in ("uv", "pip", "pip3", "poetry"):
183
+ if not os.path.isfile(os.path.join(root, "pyproject.toml")):
184
+ return False, f"there is no pyproject.toml in {root}"
185
+ return True, ""
186
+
187
+ if tool == "go":
188
+ if not os.path.isfile(os.path.join(root, "go.mod")):
189
+ return False, f"there is no go.mod in {root}"
190
+ return True, ""
191
+
192
+ return True, ""
193
+
194
+
195
+ def main(argv):
196
+ spec = ""
197
+ root = "."
198
+ rest = list(argv)
199
+ while rest:
200
+ flag = rest.pop(0)
201
+ if flag == "--spec":
202
+ spec = rest.pop(0) if rest else ""
203
+ elif flag == "--root":
204
+ root = rest.pop(0) if rest else ""
205
+ elif flag in ("-h", "--help"):
206
+ print(__doc__)
207
+ return 0
208
+ else:
209
+ die(f"unknown argument: {flag}", "pass --spec SPEC_DIR [--root PROJECT_ROOT]")
210
+
211
+ if not spec:
212
+ die("no spec dir given", "pass --spec SPEC_DIR")
213
+ declaration = os.path.join(spec, DECLARATION)
214
+ if not os.path.isfile(declaration):
215
+ die(
216
+ f"no {DECLARATION} in {spec}",
217
+ f"write {spec}/{DECLARATION} with the target stack's runtime and check command",
218
+ )
219
+ rows = read_declaration(declaration)
220
+ if not os.path.isdir(root):
221
+ die(f"no project at {root}", "pass --root PROJECT_ROOT")
222
+
223
+ missing = []
224
+ for key in ("runtime", "package_manager", "build", "check"):
225
+ command = rows.get(key)
226
+ if not command:
227
+ continue
228
+ tool = os.path.basename(first_word(command))
229
+ if tool in SHELL_BUILTINS:
230
+ continue
231
+ if not on_path(tool):
232
+ missing.append((key, tool))
233
+
234
+ if missing:
235
+ for key, tool in missing:
236
+ fail(
237
+ "toolchain-preflight",
238
+ f"the {key} is not installed: {tool} is not on PATH",
239
+ f"install {tool} before the port starts, or point the {key} at a command this machine has",
240
+ )
241
+ return 1
242
+
243
+ if "build" in rows:
244
+ ok, reason = declared_build(root, rows["build"])
245
+ if not ok:
246
+ fail(
247
+ "toolchain-preflight",
248
+ f"the build command is not one the project declares: {reason}",
249
+ "write the build command the project actually has, or add the script it names",
250
+ )
251
+ return 1
252
+
253
+ print(f"runtime {rows['runtime']} ({tool_version(first_word(rows['runtime']))})")
254
+ if "package_manager" in rows:
255
+ manager = first_word(rows["package_manager"])
256
+ print(f"package manager {rows['package_manager']} ({tool_version(manager)})")
257
+ if "build" in rows:
258
+ print(f"build {rows['build']}")
259
+ print(f"check {rows['check']}")
260
+
261
+ try:
262
+ done = subprocess.run(
263
+ rows["check"],
264
+ shell=True,
265
+ cwd=root,
266
+ capture_output=True,
267
+ text=True,
268
+ timeout=600,
269
+ )
270
+ except subprocess.TimeoutExpired:
271
+ fail(
272
+ "toolchain-preflight",
273
+ f"the project's check did not finish: {rows['check']}",
274
+ "raise the timeout or run the check by hand to see where it hangs",
275
+ )
276
+ return 3
277
+
278
+ if done.returncode == 127:
279
+ fail(
280
+ "toolchain-preflight",
281
+ f"the check runs a command this machine does not have: {rows['check']}",
282
+ f"install the tool the check names, or point the check at a command this machine has",
283
+ )
284
+ return 1
285
+
286
+ if done.returncode != 0:
287
+ tail = (done.stdout or "") + (done.stderr or "")
288
+ lines = [line for line in tail.strip().splitlines() if line.strip()]
289
+ excerpt = "\n".join(lines[-8:])
290
+ fail(
291
+ "toolchain-preflight",
292
+ f"the project's check failed before the port started: {rows['check']} exited {done.returncode}",
293
+ f"a red baseline is unproven, not equivalent: fix the project first, or judge on a command that is green today\n{excerpt}",
294
+ )
295
+ return 3
296
+
297
+ print("ready the target stack is installed and the project's check is green")
298
+ return 0
299
+
300
+
301
+ if __name__ == "__main__":
302
+ raise SystemExit(main(sys.argv[1:]))