@giamme/forge 0.0.0-stage → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +181 -0
- package/LICENSE +21 -0
- package/README.md +1854 -2
- package/SKILL.md +122 -0
- package/VERSION +1 -0
- package/forge +74 -0
- package/package.json +26 -4
- package/references/dataset.md +329 -0
- package/references/decompose.md +667 -0
- package/references/fractal-dashboard.md +21 -0
- package/references/fractal.md +307 -0
- package/references/harnesses.md +353 -0
- package/references/jev.md +865 -0
- package/references/memory.md +228 -0
- package/references/planning.md +42 -0
- package/references/ripwire.md +47 -0
- package/references/runtime.md +64 -0
- package/references/solo.md +113 -0
- package/references/ui.md +112 -0
- package/registry.tsv +81 -0
- package/scripts/forge-artifact.sh +78 -0
- package/scripts/forge-contract.py +430 -0
- package/scripts/forge-dataset.py +6 -0
- package/scripts/forge-dispatch.sh +665 -0
- package/scripts/forge-fail-first.py +195 -0
- package/scripts/forge-fractal-dispatch.sh +4 -0
- package/scripts/forge-fractal-options.sh +54 -0
- package/scripts/forge-fractal.py +6 -0
- package/scripts/forge-guard.py +266 -0
- package/scripts/forge-install-fractal.sh +4 -0
- package/scripts/forge-install-ripwire.sh +52 -0
- package/scripts/forge-install.sh +135 -0
- package/scripts/forge-jev-options.sh +259 -0
- package/scripts/forge-jev.py +6 -0
- package/scripts/forge-lib-plans.sh +785 -0
- package/scripts/forge-lib-review.sh +494 -0
- package/scripts/forge-lib-task.sh +302 -0
- package/scripts/forge-lib-verify.sh +30 -0
- package/scripts/forge-memory.sh +575 -0
- package/scripts/forge-metrics.sh +39 -0
- package/scripts/forge-parallel-options.sh +76 -0
- package/scripts/forge-parallel.sh +1717 -0
- package/scripts/forge-plan-check.py +118 -0
- package/scripts/forge-prompt.py +45 -0
- package/scripts/forge-ripwire.py +136 -0
- package/scripts/forge-route.py +344 -0
- package/scripts/forge-runtime.py +533 -0
- package/scripts/forge-schedule.py +312 -0
- package/scripts/forge-shadow.py +305 -0
- package/scripts/forge-solo.sh +347 -0
- package/scripts/forge-ui-options.sh +42 -0
- package/scripts/forge-ui.py +435 -0
- package/scripts/forge-update.sh +71 -0
- package/scripts/forge-verify-coverage.py +956 -0
- package/scripts/forge-version.py +198 -0
- package/scripts/forge_dataset/__init__.py +439 -0
- package/scripts/forge_dataset/capture.py +330 -0
- package/scripts/forge_dataset/checks.py +109 -0
- package/scripts/forge_dataset/cli.py +233 -0
- package/scripts/forge_dataset/export.py +1009 -0
- package/scripts/forge_dataset/harvest.py +716 -0
- package/scripts/forge_dataset/importer.py +412 -0
- package/scripts/forge_dataset/outcomes.py +275 -0
- package/scripts/forge_dataset/routing.py +278 -0
- package/scripts/forge_fractal/__init__.py +83 -0
- package/scripts/forge_fractal/bridge.py +46 -0
- package/scripts/forge_fractal/cli.py +202 -0
- package/scripts/forge_fractal/dashboard.css +138 -0
- package/scripts/forge_fractal/dashboard.html +41 -0
- package/scripts/forge_fractal/dashboard.js +687 -0
- package/scripts/forge_fractal/decomposition.py +262 -0
- package/scripts/forge_fractal/execution.py +659 -0
- package/scripts/forge_fractal/inspection.py +261 -0
- package/scripts/forge_fractal/install.py +158 -0
- package/scripts/forge_fractal/invocation.py +94 -0
- package/scripts/forge_fractal/pipeline.py +124 -0
- package/scripts/forge_fractal/selection.py +179 -0
- package/scripts/forge_jev/__init__.py +376 -0
- package/scripts/forge_jev/backtest.py +322 -0
- package/scripts/forge_jev/cli.py +980 -0
- package/scripts/forge_jev/client.py +418 -0
- package/scripts/forge_jev/corpus.py +156 -0
- package/scripts/forge_jev/coupling.py +244 -0
- package/scripts/forge_jev/features.py +24 -0
- package/scripts/forge_jev/gates.py +100 -0
- package/scripts/forge_jev/memory.py +163 -0
- package/scripts/forge_jev/outcome.py +49 -0
- package/scripts/forge_jev/predict.py +204 -0
- package/scripts/forge_jev/questions.py +661 -0
- package/scripts/forge_jev/review.py +73 -0
- package/scripts/forge_jev/routing.py +271 -0
- package/scripts/forge_jev/subset.py +123 -0
- package/scripts/forge_jev/verify.py +453 -0
- package/scripts/forge_registry.py +140 -0
- package/ui/README.md +287 -0
- package/ui/THIRD_PARTY_NOTICES.md +69 -0
- package/ui/docs/api.md +888 -0
- package/ui/forge_ui/__init__.py +20 -0
- package/ui/forge_ui/__main__.py +135 -0
- package/ui/forge_ui/diffs.py +454 -0
- package/ui/forge_ui/executions.py +436 -0
- package/ui/forge_ui/export.py +204 -0
- package/ui/forge_ui/findings.py +132 -0
- package/ui/forge_ui/fractal.py +583 -0
- package/ui/forge_ui/models.py +216 -0
- package/ui/forge_ui/reader.py +1336 -0
- package/ui/forge_ui/server.py +303 -0
- package/ui/forge_ui/static/app.js +924 -0
- package/ui/forge_ui/static/cloud3d.js +646 -0
- package/ui/forge_ui/static/cloudmodel.js +375 -0
- package/ui/forge_ui/static/cloudui.js +394 -0
- package/ui/forge_ui/static/diffview.js +216 -0
- package/ui/forge_ui/static/favicon.svg +1 -0
- package/ui/forge_ui/static/files.js +776 -0
- package/ui/forge_ui/static/gantt.js +97 -0
- package/ui/forge_ui/static/graph.js +266 -0
- package/ui/forge_ui/static/index.html +55 -0
- package/ui/forge_ui/static/iso.js +765 -0
- package/ui/forge_ui/static/leaderboard.js +315 -0
- package/ui/forge_ui/static/map3d.js +45 -0
- package/ui/forge_ui/static/modelstats.js +249 -0
- package/ui/forge_ui/static/notify.js +179 -0
- package/ui/forge_ui/static/panel.js +526 -0
- package/ui/forge_ui/static/runmodel.js +651 -0
- package/ui/forge_ui/static/scene3d.js +996 -0
- package/ui/forge_ui/static/search.js +119 -0
- package/ui/forge_ui/static/style.css +618 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Fail-first evidence: do the tests a change adds or edits fail without the change?
|
|
3
|
+
|
|
4
|
+
forge-fail-first.py run --repo <review snapshot> --base <commit> --reviewed <commit>
|
|
5
|
+
--command '<test command with {files}>' --out <failfirst.json>
|
|
6
|
+
[--deps-from <dir>] [--timeout S]
|
|
7
|
+
|
|
8
|
+
Opt-in (`--fail-first` on forge-solo.sh and `plan`) and advisory. A test that passes on the
|
|
9
|
+
code it is meant to protect, before that code changed, proves nothing about the change. For
|
|
10
|
+
every test file the diff adds or edits, this runs the project's own per-file test command
|
|
11
|
+
(`.forge/verify-subset`, which must contain `{files}`) twice, each in a throwaway export:
|
|
12
|
+
|
|
13
|
+
- with the change: the reviewed tree;
|
|
14
|
+
- without it: the base tree plus the change's test-path files only (test files, fixtures
|
|
15
|
+
and helpers under test directories), so the new tests meet the old code.
|
|
16
|
+
|
|
17
|
+
Each file ends up `proven` (passes with, fails without), `passes_without_change` (passes both
|
|
18
|
+
ways: it does not prove the change), `inconclusive` (fails even with the change: a broken test
|
|
19
|
+
or a missing dependency, no evidence either way) or `error` (could not export or timed out).
|
|
20
|
+
|
|
21
|
+
The result goes to the JSON file, and the reviewer's prompt section goes to stdout; the QA
|
|
22
|
+
prompt asks the reviewer to report every `passes_without_change` file as a P2 finding. Forge
|
|
23
|
+
never changes a status or verdict on its own from this. The review snapshot is only read:
|
|
24
|
+
exports use a private index, so no fingerprint can change. Always exits 0 except on usage
|
|
25
|
+
errors (2).
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
import re
|
|
34
|
+
import shlex
|
|
35
|
+
import shutil
|
|
36
|
+
import subprocess
|
|
37
|
+
import sys
|
|
38
|
+
import tempfile
|
|
39
|
+
import time
|
|
40
|
+
|
|
41
|
+
# The same two conditions as forge_jev/subset.py: a test path, and for the files that are run,
|
|
42
|
+
# a source suffix (README.md and JSON fixtures under tests/ are carried over, never run).
|
|
43
|
+
TEST_PATH = re.compile(r'(^|/)(tests?|spec|specs|__tests__)/'
|
|
44
|
+
r'|(^|/)test_[^/]*$|_test\.[^/]+$|\.(test|spec)\.[^/]+$')
|
|
45
|
+
CODE_SUFFIXES = ('.py', '.go', '.js', '.jsx', '.ts', '.tsx', '.rb', '.rs', '.java',
|
|
46
|
+
'.kt', '.php', '.sh', '.mjs', '.cjs')
|
|
47
|
+
DEP_DIRS = ('node_modules', '.venv', 'venv', 'vendor')
|
|
48
|
+
MAX_FILES = 20
|
|
49
|
+
TAIL = 2000
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _git(repo, *args, env=None, data=None):
|
|
53
|
+
return subprocess.run(['git', '-C', str(repo), *args], input=data, capture_output=True,
|
|
54
|
+
env=env, timeout=120, check=True).stdout
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def changed_paths(repo, base, reviewed):
|
|
58
|
+
"""[(status, path)] of the change, renames split into a delete and an add."""
|
|
59
|
+
out = _git(repo, 'diff', '--name-status', '-z', '--no-renames', base, reviewed, '--', '.', ':(exclude).forge')
|
|
60
|
+
parts = out.decode('utf-8', 'surrogateescape').split('\0')
|
|
61
|
+
return [(parts[i], parts[i + 1]) for i in range(0, len(parts) - 1, 2)]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def export(repo, tree, dest, overlay_from=None, overlay=(), deps_from=None):
|
|
65
|
+
"""Check `tree` out into `dest` through a private index; `overlay` paths come from `overlay_from`."""
|
|
66
|
+
handle, index = tempfile.mkstemp(prefix='forge-failfirst-index.')
|
|
67
|
+
os.close(handle); os.unlink(index)
|
|
68
|
+
env = dict(os.environ, GIT_INDEX_FILE=index)
|
|
69
|
+
try:
|
|
70
|
+
_git(repo, 'read-tree', tree, env=env)
|
|
71
|
+
for path in overlay:
|
|
72
|
+
entry = _git(repo, 'ls-tree', '-z', overlay_from, '--', path).decode('utf-8', 'surrogateescape')
|
|
73
|
+
if entry:
|
|
74
|
+
_git(repo, 'update-index', '-z', '--index-info', env=env, data=entry.encode('utf-8', 'surrogateescape'))
|
|
75
|
+
else:
|
|
76
|
+
_git(repo, 'update-index', '--force-remove', '--', path, env=env)
|
|
77
|
+
_git(repo, 'checkout-index', '--all', '--force', '--prefix=' + str(dest) + '/', env=env)
|
|
78
|
+
finally:
|
|
79
|
+
if os.path.exists(index):
|
|
80
|
+
os.unlink(index)
|
|
81
|
+
for name in DEP_DIRS: # dependencies are not in the tree; link them so the export can run
|
|
82
|
+
source = Path(deps_from) / name if deps_from else None
|
|
83
|
+
if source and source.is_dir() and not (Path(dest) / name).exists():
|
|
84
|
+
try:
|
|
85
|
+
os.symlink(source, Path(dest) / name)
|
|
86
|
+
except OSError:
|
|
87
|
+
pass
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def run_one(command, cwd, timeout):
|
|
91
|
+
started = time.time()
|
|
92
|
+
try:
|
|
93
|
+
done = subprocess.run(command, shell=True, cwd=str(cwd), capture_output=True, text=True,
|
|
94
|
+
stdin=subprocess.DEVNULL, timeout=timeout, errors='replace')
|
|
95
|
+
except subprocess.TimeoutExpired:
|
|
96
|
+
return dict(rc=None, seconds=round(time.time() - started, 3), tail='timed out')
|
|
97
|
+
except OSError as error:
|
|
98
|
+
return dict(rc=None, seconds=round(time.time() - started, 3), tail=str(error))
|
|
99
|
+
tail = ((done.stdout or '') + (done.stderr or ''))[-TAIL:]
|
|
100
|
+
return dict(rc=done.returncode, seconds=round(time.time() - started, 3), tail=tail)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def classify(with_change, without):
|
|
104
|
+
if with_change['rc'] is None or without['rc'] is None:
|
|
105
|
+
return 'error'
|
|
106
|
+
if with_change['rc'] != 0:
|
|
107
|
+
return 'inconclusive'
|
|
108
|
+
return 'proven' if without['rc'] != 0 else 'passes_without_change'
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def evaluate(repo, base, reviewed, command, timeout, deps_from=None):
|
|
112
|
+
result = dict(version=1, status='ran', command=command, base=base, reviewed=reviewed, tests=[],
|
|
113
|
+
skipped=[], started=time.time())
|
|
114
|
+
if '{files}' not in (command or ''):
|
|
115
|
+
result.update(status='skipped', reason='no per-file test command: put one with {files} in .forge/verify-subset')
|
|
116
|
+
return result
|
|
117
|
+
changes = changed_paths(repo, base, reviewed)
|
|
118
|
+
support = [path for _status, path in changes if TEST_PATH.search(path)]
|
|
119
|
+
tests = [path for status, path in changes
|
|
120
|
+
if status != 'D' and TEST_PATH.search(path) and path.endswith(CODE_SUFFIXES)]
|
|
121
|
+
if not tests:
|
|
122
|
+
result.update(status='skipped', reason='the change adds or edits no test files')
|
|
123
|
+
return result
|
|
124
|
+
result['skipped'] = tests[MAX_FILES:]
|
|
125
|
+
work = Path(tempfile.mkdtemp(prefix='forge-failfirst-'))
|
|
126
|
+
try:
|
|
127
|
+
with_dir, without_dir = work / 'with', work / 'without'
|
|
128
|
+
export(repo, reviewed, with_dir, deps_from=deps_from)
|
|
129
|
+
export(repo, base, without_dir, overlay_from=reviewed, overlay=support, deps_from=deps_from)
|
|
130
|
+
for path in tests[:MAX_FILES]:
|
|
131
|
+
line = command.replace('{files}', shlex.quote(path))
|
|
132
|
+
with_change = run_one(line, with_dir, timeout)
|
|
133
|
+
without = run_one(line, without_dir, timeout)
|
|
134
|
+
result['tests'].append(dict(path=path, result=classify(with_change, without),
|
|
135
|
+
with_change=with_change, without_change=without))
|
|
136
|
+
except (OSError, subprocess.SubprocessError) as error:
|
|
137
|
+
result.update(status='error', reason=f'could not export the change: {error}'[:500])
|
|
138
|
+
finally:
|
|
139
|
+
shutil.rmtree(work, ignore_errors=True)
|
|
140
|
+
result['ended'] = time.time()
|
|
141
|
+
return result
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def prompt_section(result):
|
|
145
|
+
"""What the reviewer reads; empty when there is nothing to say."""
|
|
146
|
+
if result.get('status') != 'ran' or not result.get('tests'):
|
|
147
|
+
return ''
|
|
148
|
+
words = dict(proven='fails without the change and passes with it',
|
|
149
|
+
passes_without_change='PASSES WITHOUT THE CHANGE: it does not prove the change',
|
|
150
|
+
inconclusive='fails even with the change (a broken test or the environment): no evidence either way',
|
|
151
|
+
error='could not be run (timeout or error): no evidence either way')
|
|
152
|
+
lines = ['Fail-first evidence (observed by forge, not claimed by the implementer): each test file this',
|
|
153
|
+
'change adds or edits was run with the change and against the code WITHOUT its non-test changes.',
|
|
154
|
+
'Command: ' + result['command']]
|
|
155
|
+
lines += [f' - {item["path"]}: {words[item["result"]]}' for item in result['tests']]
|
|
156
|
+
if result.get('skipped'):
|
|
157
|
+
lines.append(f' ({len(result["skipped"])} more test files were not run)')
|
|
158
|
+
if any(item['result'] == 'passes_without_change' for item in result['tests']):
|
|
159
|
+
lines += ['Report each test file that passes without the change as a [P2][CONFIRMED] finding at that file:',
|
|
160
|
+
'a test that cannot fail does not protect this change. If the test is not meant to catch it',
|
|
161
|
+
'(a refactor kept green, a pure rename), say why instead.']
|
|
162
|
+
return '\n'.join(lines) + '\n'
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def main(argv=None):
|
|
166
|
+
parser = argparse.ArgumentParser(prog='forge-fail-first.py')
|
|
167
|
+
sub = parser.add_subparsers(dest='cmd', required=True)
|
|
168
|
+
run = sub.add_parser('run')
|
|
169
|
+
run.add_argument('--repo', required=True)
|
|
170
|
+
run.add_argument('--base', required=True)
|
|
171
|
+
run.add_argument('--reviewed', required=True)
|
|
172
|
+
run.add_argument('--command', default='')
|
|
173
|
+
run.add_argument('--out', required=True)
|
|
174
|
+
run.add_argument('--deps-from')
|
|
175
|
+
run.add_argument('--timeout', type=float, default=float(os.environ.get('FORGE_FAIL_FIRST_TIMEOUT') or 600))
|
|
176
|
+
args = parser.parse_args(argv)
|
|
177
|
+
try:
|
|
178
|
+
result = evaluate(args.repo, args.base, args.reviewed, args.command.strip(), max(1.0, args.timeout),
|
|
179
|
+
deps_from=args.deps_from)
|
|
180
|
+
except Exception as error: # advisory: never break the run
|
|
181
|
+
result = dict(version=1, status='error', reason=str(error)[:500], command=args.command, tests=[])
|
|
182
|
+
counts = {}
|
|
183
|
+
for item in result.get('tests', []):
|
|
184
|
+
counts[item['result']] = counts.get(item['result'], 0) + 1
|
|
185
|
+
result['counts'] = counts
|
|
186
|
+
try:
|
|
187
|
+
Path(args.out).write_text(json.dumps(result, indent=1) + '\n', encoding='utf-8')
|
|
188
|
+
except OSError:
|
|
189
|
+
pass
|
|
190
|
+
sys.stdout.write(prompt_section(result))
|
|
191
|
+
return 0
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
if __name__ == '__main__':
|
|
195
|
+
sys.exit(main())
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Source in runners. Arrays and syntax remain compatible with macOS Bash 3.2.
|
|
3
|
+
FRACTAL_CHOICE=""; FRACTAL_ARGS=()
|
|
4
|
+
forge_fractal_help() {
|
|
5
|
+
cat <<'HELP'
|
|
6
|
+
Optional Fractal execution (solo or parallel run):
|
|
7
|
+
--fractal | --no-fractal Explicit choice; unattended defaults off
|
|
8
|
+
--fractal-auto-decompose Enable Fractal with automatic split-or-atomic planning
|
|
9
|
+
--fractal-planner SPEC Decision model (top-level planner, then task root model)
|
|
10
|
+
--fractal-depth N Nested levels below implementation (2)
|
|
11
|
+
--fractal-children N Unsettled direct children per node (3)
|
|
12
|
+
--fractal-nodes N Lifetime nodes per task (12)
|
|
13
|
+
--fractal-iterations N Iterations per node per attempt (6)
|
|
14
|
+
--fractal-concurrency N Model slots across the run, including QA (3)
|
|
15
|
+
--fractal-deadline N Task attempt seconds excluding pauses (2700)
|
|
16
|
+
--fractal-max-cost N Unsupported: costs are observational
|
|
17
|
+
Run choice, planners, limits and pools persist for resume/retry. Installation never activates it.
|
|
18
|
+
Solo child routing: --dwarf-low, --dwarf-medium, --dwarf-high (comma pools).
|
|
19
|
+
Decomposed routing is configured by the plan command.
|
|
20
|
+
Use ./forge fractal --help for installation, inspection, controls and HTML reports.
|
|
21
|
+
HELP
|
|
22
|
+
}
|
|
23
|
+
forge_fractal_flag() {
|
|
24
|
+
local choice="$1"
|
|
25
|
+
[ -z "$FRACTAL_CHOICE" ] || [ "$FRACTAL_CHOICE" = "$choice" ] || { echo '--fractal and --no-fractal are mutually exclusive' >&2; return 2; }
|
|
26
|
+
FRACTAL_CHOICE="$choice"
|
|
27
|
+
}
|
|
28
|
+
forge_fractal_select() {
|
|
29
|
+
local run="$1" repo="$2" mode="$3" dry="$4" selected
|
|
30
|
+
shift 4
|
|
31
|
+
local args=(select --run-dir "$run" --repo "$repo" --mode "$mode" --choice "$FRACTAL_CHOICE")
|
|
32
|
+
[ "$dry" != 1 ] || args+=(--dry-run)
|
|
33
|
+
selected="$(python3 "$SKILL_DIR/scripts/forge-fractal.py" "${args[@]}" ${FRACTAL_ARGS[@]+"${FRACTAL_ARGS[@]}"} "$@")" || return $?
|
|
34
|
+
if [ "$selected" != off ] && [ "$selected" != dry-fractal ]; then
|
|
35
|
+
export FORGE_FRACTAL_RUN="$selected"
|
|
36
|
+
DISPATCH="$SKILL_DIR/scripts/forge-fractal-dispatch.sh"
|
|
37
|
+
elif [ "$selected" = dry-fractal ]; then
|
|
38
|
+
unset FORGE_FRACTAL_RUN
|
|
39
|
+
echo 'backend: fractal (dry run)' >&2
|
|
40
|
+
else
|
|
41
|
+
unset FORGE_FRACTAL_RUN
|
|
42
|
+
fi
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
forge_fractal_parallel_run() {
|
|
46
|
+
local plan="$1" repo="$2" dry="$3"; shift 3
|
|
47
|
+
local flags=()
|
|
48
|
+
[ ! -f "$plan/yolo_dwarf" ] || flags+=(--yolo-dwarf)
|
|
49
|
+
[ ! -f "$plan/yolo_qa" ] || flags+=(--yolo-qa)
|
|
50
|
+
forge_fractal_select "$plan" "$repo" parallel "$dry" ${flags[@]+"${flags[@]}"} || return $?
|
|
51
|
+
if [ -n "${FORGE_FRACTAL_RUN:-}" ]; then
|
|
52
|
+
python3 "$SKILL_DIR/scripts/forge-fractal.py" _remember "$SELF" run "$plan" "$@" || return $?
|
|
53
|
+
fi
|
|
54
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Implementer-side guards for forge-dispatch.sh. Stdlib only, offline.
|
|
3
|
+
|
|
4
|
+
forge-guard.py ping
|
|
5
|
+
prints `ok`; lets the dispatcher prove the helper can start before relying on it.
|
|
6
|
+
|
|
7
|
+
forge-guard.py session -- CMD [ARGS...]
|
|
8
|
+
os.setsid(), then exec CMD. The pid does not change, so the dispatcher's $! is
|
|
9
|
+
still the harness pid, and that pid is now also the process-group id: everything
|
|
10
|
+
the harness spawns (a dwarf's `npm test &`, a dev server) shares the group and can
|
|
11
|
+
be signalled or reaped as a unit. macOS has no `setsid` binary; this is why the
|
|
12
|
+
helper exists.
|
|
13
|
+
|
|
14
|
+
forge-guard.py reap <pgid> <orphans-file>
|
|
15
|
+
After the harness exited: anything still alive in its process group is an orphan
|
|
16
|
+
the implementer abandoned. List it (only when something is found), TERM, wait a
|
|
17
|
+
bounded ~2s, KILL, print how many were found. Never exits nonzero for an ordinary
|
|
18
|
+
outcome, and never touches its own or its caller's group.
|
|
19
|
+
|
|
20
|
+
forge-guard.py promises <file> [--tail-bytes N]
|
|
21
|
+
Scan the tail of an implementer's final message for promises of later work ("it is
|
|
22
|
+
still running in the background", "I'll check back"). One `pattern<TAB>excerpt`
|
|
23
|
+
line per hit; nothing when clean.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import signal
|
|
30
|
+
import subprocess
|
|
31
|
+
import sys
|
|
32
|
+
import time
|
|
33
|
+
|
|
34
|
+
PS_CANDIDATES = ('/bin/ps', '/usr/bin/ps')
|
|
35
|
+
TERM_WAIT_S = 2.0
|
|
36
|
+
KILL_WAIT_S = 1.0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# --- session ------------------------------------------------------------------
|
|
40
|
+
def session(argv: list[str]) -> int:
|
|
41
|
+
if not argv:
|
|
42
|
+
print('forge-guard: session needs a command', file=sys.stderr)
|
|
43
|
+
return 2
|
|
44
|
+
try:
|
|
45
|
+
os.setsid()
|
|
46
|
+
except OSError:
|
|
47
|
+
# Already a group leader (job control on): pgid == pid already, which is all
|
|
48
|
+
# the reaper needs.
|
|
49
|
+
pass
|
|
50
|
+
try:
|
|
51
|
+
os.execvp(argv[0], argv)
|
|
52
|
+
except FileNotFoundError:
|
|
53
|
+
print('forge-guard: cannot execute %s: not found' % argv[0], file=sys.stderr)
|
|
54
|
+
return 127
|
|
55
|
+
except OSError as error:
|
|
56
|
+
print('forge-guard: cannot execute %s: %s' % (argv[0], error), file=sys.stderr)
|
|
57
|
+
return 126
|
|
58
|
+
return 126 # unreachable
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# --- reap ---------------------------------------------------------------------
|
|
62
|
+
def group_exists(pgid: int) -> bool:
|
|
63
|
+
try:
|
|
64
|
+
os.killpg(pgid, 0)
|
|
65
|
+
except ProcessLookupError:
|
|
66
|
+
return False
|
|
67
|
+
except PermissionError:
|
|
68
|
+
return True
|
|
69
|
+
except OSError:
|
|
70
|
+
return False
|
|
71
|
+
return True
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def list_group(pgid: int):
|
|
75
|
+
"""[(pid, command)] of live members, or None when ps is unavailable.
|
|
76
|
+
|
|
77
|
+
Zombies are skipped: they are already dead and only wait for their parent.
|
|
78
|
+
"""
|
|
79
|
+
for ps in PS_CANDIDATES:
|
|
80
|
+
if not os.access(ps, os.X_OK):
|
|
81
|
+
continue
|
|
82
|
+
try:
|
|
83
|
+
out = subprocess.run([ps, '-axo', 'pid=,pgid=,stat=,command='],
|
|
84
|
+
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
|
85
|
+
timeout=5).stdout.decode(errors='replace')
|
|
86
|
+
except (OSError, subprocess.SubprocessError):
|
|
87
|
+
continue
|
|
88
|
+
members = []
|
|
89
|
+
for line in out.splitlines():
|
|
90
|
+
parts = line.split(None, 3)
|
|
91
|
+
if len(parts) < 3:
|
|
92
|
+
continue
|
|
93
|
+
try:
|
|
94
|
+
pid, group = int(parts[0]), int(parts[1])
|
|
95
|
+
except ValueError:
|
|
96
|
+
continue
|
|
97
|
+
if group != pgid or pid == os.getpid() or parts[2].startswith('Z'):
|
|
98
|
+
continue
|
|
99
|
+
members.append((pid, parts[3].strip() if len(parts) > 3 else ''))
|
|
100
|
+
return members
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def wait_gone(pgid: int, seconds: float) -> bool:
|
|
105
|
+
deadline = time.monotonic() + seconds
|
|
106
|
+
while time.monotonic() < deadline:
|
|
107
|
+
members = list_group(pgid)
|
|
108
|
+
if members is None:
|
|
109
|
+
if not group_exists(pgid):
|
|
110
|
+
return True
|
|
111
|
+
elif not members:
|
|
112
|
+
return True
|
|
113
|
+
time.sleep(0.05)
|
|
114
|
+
members = list_group(pgid)
|
|
115
|
+
return (not members) if members is not None else not group_exists(pgid)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def reap(pgid: int, orphans_file: str) -> int:
|
|
119
|
+
# Never signal ourselves, the dispatcher that called us, or init.
|
|
120
|
+
forbidden = {0, 1, os.getpgrp()}
|
|
121
|
+
try:
|
|
122
|
+
forbidden.add(os.getpgid(os.getppid()))
|
|
123
|
+
except OSError:
|
|
124
|
+
pass
|
|
125
|
+
if pgid in forbidden or pgid < 0:
|
|
126
|
+
print(0)
|
|
127
|
+
return 0
|
|
128
|
+
if not group_exists(pgid):
|
|
129
|
+
print(0)
|
|
130
|
+
return 0
|
|
131
|
+
members = list_group(pgid)
|
|
132
|
+
if members is not None and not members:
|
|
133
|
+
# Only zombies (or nothing visible) left; nothing worth reporting or killing.
|
|
134
|
+
print(0)
|
|
135
|
+
return 0
|
|
136
|
+
found = members if members is not None else [(0, '(unknown: ps unavailable; process group %d still has live members)' % pgid)]
|
|
137
|
+
try:
|
|
138
|
+
with open(orphans_file, 'w') as handle:
|
|
139
|
+
for pid, command in found:
|
|
140
|
+
handle.write('%s\t%s\n' % (pid or '?', ' '.join(command.split()) or '?'))
|
|
141
|
+
except OSError:
|
|
142
|
+
pass
|
|
143
|
+
for sig, wait in ((signal.SIGTERM, TERM_WAIT_S), (signal.SIGKILL, KILL_WAIT_S)):
|
|
144
|
+
try:
|
|
145
|
+
os.killpg(pgid, sig)
|
|
146
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
147
|
+
break
|
|
148
|
+
if wait_gone(pgid, wait):
|
|
149
|
+
break
|
|
150
|
+
print(len(found))
|
|
151
|
+
return 0
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# --- promises -----------------------------------------------------------------
|
|
155
|
+
# A promise of later work in a FINAL message is a bug: the dispatch ends when the message
|
|
156
|
+
# does, so whatever was "still running" is abandoned and whatever "I'll check" never
|
|
157
|
+
# happens. Patterns are deliberately tight; a false positive costs a warning line, a false
|
|
158
|
+
# negative costs a silent half-finished change, but a noisy scan gets ignored.
|
|
159
|
+
DONE_WORDS = re.compile(
|
|
160
|
+
r"\b(?:finished|completed|done|exited|passed|failed|succeeded|terminated|killed|stopped|ended|returned|"
|
|
161
|
+
r"was\s+running|were\s+running|had\s+(?:finished|completed))\b", re.I)
|
|
162
|
+
Q = r"['’]"
|
|
163
|
+
PROMISES = (
|
|
164
|
+
# (name, regex, past_tense_tolerated)
|
|
165
|
+
# "in the background job scheduler" describes code, not a process left running.
|
|
166
|
+
('background', re.compile(
|
|
167
|
+
r"\bin\s+(?:the\s+)?background\b(?!\s+(?:jobs?|tasks?|workers?|threads?|queues?|refresh|sync|polling|fetch|process(?:es)?\b))", re.I), True),
|
|
168
|
+
('still_running', re.compile(r"\bstill\s+(?:running|executing|in\s+progress|working|going)\b", re.I), True),
|
|
169
|
+
('report_back', re.compile(r"\b(?:will|I%s?ll|I\s+will|going\s+to)\s+(?:report|come|get|circle|check)\s+back\b" % Q, re.I), False),
|
|
170
|
+
('once_it_finishes', re.compile(
|
|
171
|
+
r"\bonce\s+(?:it|they|that|this|the\s+[\w-]+(?:\s+[\w-]+)?)\s+(?:finish(?:es)?|completes?|is\s+done|are\s+done|ends|exits)\b", re.I), False),
|
|
172
|
+
('ill_do', re.compile(
|
|
173
|
+
r"\bI(?:%s|\s+wi)ll\s+(?:check|report|continue|follow\s+up|run|measure|verify)\b" % Q, re.I), False),
|
|
174
|
+
('next_ill', re.compile(r"\bnext,?\s+I(?:%s|\s+wi)ll\b" % Q, re.I), False),
|
|
175
|
+
('will_continue', re.compile(
|
|
176
|
+
r"\b(?:I|we)\s+will\s+(?:continue|finish)\b|\bwill\s+(?:continue|finish)\s+(?:running|in\s+the|once|after|shortly|later)\b", re.I), False),
|
|
177
|
+
('to_be_continued', re.compile(r"\bto\s+be\s+continued\b", re.I), False),
|
|
178
|
+
('waiting_for', re.compile(r"\b(?:am|I%sm|still|now|currently)\s+waiting\s+(?:for|on)\b" % Q, re.I), False),
|
|
179
|
+
)
|
|
180
|
+
_SENTENCE_BREAK = re.compile(r"[.!?\n]")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _sentence(text: str, start: int, end: int) -> str:
|
|
184
|
+
left = start
|
|
185
|
+
while left > 0 and not _SENTENCE_BREAK.match(text[left - 1]):
|
|
186
|
+
left -= 1
|
|
187
|
+
right = end
|
|
188
|
+
while right < len(text) and not _SENTENCE_BREAK.match(text[right]):
|
|
189
|
+
right += 1
|
|
190
|
+
return ' '.join(text[left:right].split())
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def scan_promises(text: str) -> list[tuple[str, str]]:
|
|
194
|
+
hits: list[tuple[str, str]] = []
|
|
195
|
+
for name, pattern, tolerate_past in PROMISES:
|
|
196
|
+
for match in pattern.finditer(text):
|
|
197
|
+
sentence = _sentence(text, match.start(), match.end())
|
|
198
|
+
if tolerate_past and DONE_WORDS.search(sentence):
|
|
199
|
+
continue
|
|
200
|
+
hit = (name, sentence[:200])
|
|
201
|
+
if hit not in hits:
|
|
202
|
+
hits.append(hit)
|
|
203
|
+
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
204
|
+
if lines and lines[-1].endswith(':') and not lines[-1].endswith('::'):
|
|
205
|
+
hits.append(('trailing_colon', lines[-1][-200:]))
|
|
206
|
+
return hits
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def promises(path: str, tail_bytes: int = 3000) -> int:
|
|
210
|
+
try:
|
|
211
|
+
with open(path, 'rb') as handle:
|
|
212
|
+
handle.seek(0, os.SEEK_END)
|
|
213
|
+
size = handle.tell()
|
|
214
|
+
handle.seek(max(0, size - tail_bytes))
|
|
215
|
+
data = handle.read()
|
|
216
|
+
except OSError:
|
|
217
|
+
return 0
|
|
218
|
+
text = data.decode('utf-8', errors='replace')
|
|
219
|
+
if size > tail_bytes:
|
|
220
|
+
# The cut usually lands mid-sentence; a fragment would only produce noise.
|
|
221
|
+
newline = text.find('\n')
|
|
222
|
+
text = text[newline + 1:] if 0 <= newline < len(text) - 1 else text
|
|
223
|
+
for name, excerpt in scan_promises(text):
|
|
224
|
+
print('%s\t%s' % (name, excerpt.replace('\t', ' ')))
|
|
225
|
+
return 0
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# --- cli ----------------------------------------------------------------------
|
|
229
|
+
def main(argv: list[str]) -> int:
|
|
230
|
+
if not argv:
|
|
231
|
+
print('usage: forge-guard.py ping | session -- CMD... | reap <pgid> <orphans-file> | '
|
|
232
|
+
'promises <file> [--tail-bytes N]', file=sys.stderr)
|
|
233
|
+
return 2
|
|
234
|
+
command, rest = argv[0], argv[1:]
|
|
235
|
+
try:
|
|
236
|
+
if command == 'ping':
|
|
237
|
+
print('ok')
|
|
238
|
+
return 0
|
|
239
|
+
if command == 'session':
|
|
240
|
+
if rest and rest[0] == '--':
|
|
241
|
+
rest = rest[1:]
|
|
242
|
+
return session(rest)
|
|
243
|
+
if command == 'reap':
|
|
244
|
+
if len(rest) != 2:
|
|
245
|
+
print('usage: forge-guard.py reap <pgid> <orphans-file>', file=sys.stderr)
|
|
246
|
+
return 2
|
|
247
|
+
return reap(int(rest[0]), rest[1])
|
|
248
|
+
if command == 'promises':
|
|
249
|
+
tail = 3000
|
|
250
|
+
if '--tail-bytes' in rest:
|
|
251
|
+
index = rest.index('--tail-bytes')
|
|
252
|
+
tail = int(rest[index + 1])
|
|
253
|
+
rest = rest[:index] + rest[index + 2:]
|
|
254
|
+
if len(rest) != 1:
|
|
255
|
+
print('usage: forge-guard.py promises <file> [--tail-bytes N]', file=sys.stderr)
|
|
256
|
+
return 2
|
|
257
|
+
return promises(rest[0], max(1, tail))
|
|
258
|
+
except (ValueError, IndexError) as error:
|
|
259
|
+
print('forge-guard: ' + str(error), file=sys.stderr)
|
|
260
|
+
return 2
|
|
261
|
+
print('forge-guard: unknown command: ' + command, file=sys.stderr)
|
|
262
|
+
return 2
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
if __name__ == '__main__':
|
|
266
|
+
sys.exit(main(sys.argv[1:]))
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Optional pinned upstream installer. No network activity before consent.
|
|
3
|
+
set -uo pipefail
|
|
4
|
+
YES=0; DRY=0
|
|
5
|
+
for arg in "$@"; do
|
|
6
|
+
case "$arg" in
|
|
7
|
+
--yes) YES=1 ;;
|
|
8
|
+
--dry-run) DRY=1 ;;
|
|
9
|
+
*) echo "forge: unknown installation option: $arg" >&2; exit 2 ;;
|
|
10
|
+
esac
|
|
11
|
+
done
|
|
12
|
+
[ "${FORGE_RIPWIRE:-}" != off ] || exit 0
|
|
13
|
+
DEST="$HOME/.local/bin/ripwire"
|
|
14
|
+
if command -v ripwire >/dev/null 2>&1 || [ -e "$DEST" ] || [ -L "$DEST" ]; then
|
|
15
|
+
echo 'forge: preserving existing Ripwire installation' >&2
|
|
16
|
+
exit 0
|
|
17
|
+
fi
|
|
18
|
+
printf 'forge: Ripwire v0.4.0 -> %s\n' "$DEST" >&2
|
|
19
|
+
[ "$DRY" = 0 ] || exit 0
|
|
20
|
+
if [ "$YES" = 0 ]; then
|
|
21
|
+
# Open the controlling terminal, never task stdin (which may be a pipe/FIFO).
|
|
22
|
+
if { exec 3<> /dev/tty; } 2>/dev/null && [ -t 3 ]; then
|
|
23
|
+
printf 'Ripwire is missing. Install it now? [Y/n] ' >&3
|
|
24
|
+
reply=''
|
|
25
|
+
if ! IFS= read -r reply <&3; then exec 3>&-; exit 0; fi
|
|
26
|
+
exec 3>&-
|
|
27
|
+
case "$reply" in ''|y|Y|yes|YES) ;; *) exit 0 ;; esac
|
|
28
|
+
else
|
|
29
|
+
printf 'forge: optional install: bash %q --yes\n' "${BASH_SOURCE[0]}" >&2
|
|
30
|
+
exit 0
|
|
31
|
+
fi
|
|
32
|
+
fi
|
|
33
|
+
work="$(mktemp -d "${TMPDIR:-/tmp}/forge-install-ripwire.XXXXXX")" || exit 1
|
|
34
|
+
trap 'rm -rf "$work"' EXIT
|
|
35
|
+
curl -fLSs --connect-timeout 10 --max-time 60 \
|
|
36
|
+
https://raw.githubusercontent.com/redhat-et/ripwire/v0.4.0/scripts/install.sh \
|
|
37
|
+
-o "$work/install.sh" || { echo 'forge: Ripwire installer download failed' >&2; exit 1; }
|
|
38
|
+
# Pin the installer bytes as well as the release; never execute a changed tag.
|
|
39
|
+
python3 - "$work/install.sh" <<'PYHASH'
|
|
40
|
+
import hashlib, pathlib, sys
|
|
41
|
+
expected = '7290f90775d333798ec9d7f1193c5950516dd0f1fea6ce5faea3923b1f60ff01'
|
|
42
|
+
if hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest() != expected:
|
|
43
|
+
sys.exit('forge: Ripwire installer checksum mismatch')
|
|
44
|
+
PYHASH
|
|
45
|
+
[ "$?" = 0 ] || exit 1
|
|
46
|
+
# The upstream installer verifies archive checksums and the binary version.
|
|
47
|
+
RIPWIRE_REPO=redhat-et/ripwire RIPWIRE_VERSION=v0.4.0 \
|
|
48
|
+
RIPWIRE_INSTALL_PREFIX="$HOME/.local" RIPWIRE_INSTALL_YES=1 RIPWIRE_NO_ACTIVATE=1 \
|
|
49
|
+
/bin/bash "$work/install.sh"
|
|
50
|
+
rc=$?
|
|
51
|
+
[ "$rc" = 0 ] || echo "forge: Ripwire installation failed (exit $rc); Forge can continue" >&2
|
|
52
|
+
exit "$rc"
|