e3d-pilot 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +659 -2
- package/bin/e3d-backend-benchmark +147 -0
- package/bin/e3d-debate +291 -0
- package/bin/e3d-grok-workers +59 -0
- package/bin/e3d-mcp +303 -0
- package/bin/e3d-pilot +6327 -0
- package/bin/e3d-provenance-agreement +137 -0
- package/config.schema.json +160 -0
- package/lib/draft/e3d-code-spec-conventions.md +42 -0
- package/lib/ideas/jev_dedup.sh +83 -0
- package/lib/ideas/ledger.sh +890 -0
- package/lib/ideas/materialize.sh +363 -0
- package/lib/ideas/project_tracking.sh +202 -0
- package/lib/negotiate/convergence +34 -0
- package/lib/negotiate/parse-status +76 -0
- package/lib/ops/collect-fleet-health.sh +399 -0
- package/lib/provenance/graph.sh +741 -0
- package/lib/providers/claude +91 -0
- package/lib/providers/codex +85 -0
- package/lib/providers/common.sh +116 -0
- package/lib/providers/devin +42 -0
- package/lib/providers/gemini +493 -0
- package/lib/providers/grok-build +91 -0
- package/lib/providers/jev.sh +69 -0
- package/lib/providers/local +54 -0
- package/lib/publish/github +68 -0
- package/lib/publish/local +24 -0
- package/lib/review/parse-verdict +41 -0
- package/lib/scoring/fanout.sh +464 -0
- package/lib/training/export.sh +528 -0
- package/lib/web/auth.js +53 -0
- package/lib/web/cli.js +213 -0
- package/lib/web/csrf.js +50 -0
- package/lib/web/render.js +226 -0
- package/lib/web/server.js +346 -0
- package/package.json +38 -4
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
5
|
+
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
6
|
+
# shellcheck source=lib/providers/common.sh
|
|
7
|
+
source "$REPO_ROOT/lib/providers/common.sh"
|
|
8
|
+
E3D_BENCHMARK_TIMEOUT="${E3D_BENCHMARK_TIMEOUT:-1800}"
|
|
9
|
+
|
|
10
|
+
usage() {
|
|
11
|
+
cat <<'EOF'
|
|
12
|
+
Usage: e3d-backend-benchmark --repo <path> --task-id <id> --prompt <file>
|
|
13
|
+
[--backends codex,claude,grok-build] [--verify <command>] [--output <json>]
|
|
14
|
+
|
|
15
|
+
Runs the identical prompt in a detached temporary worktree per backend. It
|
|
16
|
+
never commits, pushes, publishes, or merges. Missing token/cost fields are null.
|
|
17
|
+
EOF
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
|
|
21
|
+
|
|
22
|
+
repo="" task_id="" prompt_file="" backends="codex,claude,grok-build" verify_command="" output_file=""
|
|
23
|
+
while [[ $# -gt 0 ]]; do
|
|
24
|
+
case "$1" in
|
|
25
|
+
--repo) repo="${2:-}"; shift 2 ;;
|
|
26
|
+
--task-id) task_id="${2:-}"; shift 2 ;;
|
|
27
|
+
--prompt) prompt_file="${2:-}"; shift 2 ;;
|
|
28
|
+
--backends) backends="${2:-}"; shift 2 ;;
|
|
29
|
+
--verify) verify_command="${2:-}"; shift 2 ;;
|
|
30
|
+
--output) output_file="${2:-}"; shift 2 ;;
|
|
31
|
+
-h|--help) usage; exit 0 ;;
|
|
32
|
+
*) die "unknown argument: $1" ;;
|
|
33
|
+
esac
|
|
34
|
+
done
|
|
35
|
+
|
|
36
|
+
[[ -n "$repo" && -n "$task_id" && -n "$prompt_file" ]] || { usage >&2; exit 1; }
|
|
37
|
+
[[ -d "$repo" ]] || die "repository not found: $repo"
|
|
38
|
+
repo="$(cd "$repo" && pwd -P)"
|
|
39
|
+
[[ -f "$prompt_file" ]] || die "prompt file not found: $prompt_file"
|
|
40
|
+
prompt_file="$(cd "$(dirname "$prompt_file")" && pwd -P)/$(basename "$prompt_file")"
|
|
41
|
+
git -C "$repo" rev-parse --is-inside-work-tree >/dev/null 2>&1 || die "not a git repository: $repo"
|
|
42
|
+
command -v jq >/dev/null 2>&1 || die "jq is required"
|
|
43
|
+
[[ "$E3D_BENCHMARK_TIMEOUT" =~ ^[0-9]+$ ]] && (( E3D_BENCHMARK_TIMEOUT > 0 )) \
|
|
44
|
+
|| die "E3D_BENCHMARK_TIMEOUT must be a positive integer"
|
|
45
|
+
|
|
46
|
+
[[ -n "$output_file" ]] || output_file="$repo/.e3d-pilot/benchmarks/${task_id}.json"
|
|
47
|
+
mkdir -p "$(dirname "$output_file")"
|
|
48
|
+
artifact_dir="${output_file%.json}.artifacts"
|
|
49
|
+
mkdir -p "$artifact_dir"
|
|
50
|
+
results_file="$(mktemp "${TMPDIR:-/tmp}/e3d-benchmark-results.XXXXXX")"
|
|
51
|
+
worktrees_file="$(mktemp "${TMPDIR:-/tmp}/e3d-benchmark-worktrees.XXXXXX")"
|
|
52
|
+
|
|
53
|
+
cleanup() {
|
|
54
|
+
local path
|
|
55
|
+
while IFS= read -r path; do
|
|
56
|
+
[[ -n "$path" && -d "$path" ]] || continue
|
|
57
|
+
git -C "$repo" worktree remove --force "$path" >/dev/null 2>&1 || true
|
|
58
|
+
done < "$worktrees_file"
|
|
59
|
+
rm -f "$results_file" "$worktrees_file"
|
|
60
|
+
}
|
|
61
|
+
trap cleanup EXIT
|
|
62
|
+
trap 'exit 130' INT
|
|
63
|
+
trap 'exit 143' TERM HUP
|
|
64
|
+
|
|
65
|
+
IFS=',' read -r -a backend_list <<< "$backends"
|
|
66
|
+
for backend in "${backend_list[@]}"; do
|
|
67
|
+
case "$backend" in codex|claude|grok-build) ;; *) die "unsupported benchmark backend: $backend" ;; esac
|
|
68
|
+
provider="$REPO_ROOT/lib/providers/$backend"
|
|
69
|
+
[[ -x "$provider" ]] || die "provider adapter missing or not executable: $provider"
|
|
70
|
+
|
|
71
|
+
worktree="$(mktemp -d "${TMPDIR:-/tmp}/e3d-benchmark.${backend}.XXXXXX")"
|
|
72
|
+
# git worktree add requires the destination not to exist.
|
|
73
|
+
rmdir "$worktree"
|
|
74
|
+
git -C "$repo" worktree add --detach "$worktree" HEAD >/dev/null
|
|
75
|
+
printf '%s\n' "$worktree" >> "$worktrees_file"
|
|
76
|
+
stdout_file="$artifact_dir/${backend}.stdout"
|
|
77
|
+
stderr_file="$artifact_dir/${backend}.stderr"
|
|
78
|
+
verify_stdout_file="$artifact_dir/${backend}.verify.stdout"
|
|
79
|
+
verify_stderr_file="$artifact_dir/${backend}.verify.stderr"
|
|
80
|
+
|
|
81
|
+
start_seconds="$(date +%s)"
|
|
82
|
+
set +e
|
|
83
|
+
(cd "$worktree" && provider_run_with_timeout "$E3D_BENCHMARK_TIMEOUT" env \
|
|
84
|
+
E3D_PILOT_PROVIDER_MODE=execute GROK_BUILD_OUTPUT_FORMAT=json \
|
|
85
|
+
"$provider" "$prompt_file") >"$stdout_file" 2>"$stderr_file"
|
|
86
|
+
backend_status=$?
|
|
87
|
+
set -e
|
|
88
|
+
end_seconds="$(date +%s)"
|
|
89
|
+
|
|
90
|
+
additions=0 deletions=0 files_changed=0
|
|
91
|
+
while IFS=$'\t' read -r add del path; do
|
|
92
|
+
[[ -n "$path" ]] || continue
|
|
93
|
+
files_changed=$((files_changed + 1))
|
|
94
|
+
[[ "$add" == "-" ]] || additions=$((additions + add))
|
|
95
|
+
[[ "$del" == "-" ]] || deletions=$((deletions + del))
|
|
96
|
+
done < <(git -C "$worktree" diff --numstat --no-renames -- .)
|
|
97
|
+
while IFS= read -r path; do
|
|
98
|
+
[[ -n "$path" ]] || continue
|
|
99
|
+
files_changed=$((files_changed + 1))
|
|
100
|
+
[[ -f "$worktree/$path" ]] && additions=$((additions + $(wc -l < "$worktree/$path")))
|
|
101
|
+
done < <(git -C "$worktree" ls-files --others --exclude-standard)
|
|
102
|
+
|
|
103
|
+
verify_status_json=null
|
|
104
|
+
verify_exit_json=null
|
|
105
|
+
if [[ -n "$verify_command" ]]; then
|
|
106
|
+
set +e
|
|
107
|
+
(cd "$worktree" && bash -lc "$verify_command") >"$verify_stdout_file" 2>"$verify_stderr_file"
|
|
108
|
+
verify_status=$?
|
|
109
|
+
set -e
|
|
110
|
+
verify_exit_json="$verify_status"
|
|
111
|
+
if [[ $verify_status -eq 0 ]]; then verify_status_json=true; else verify_status_json=false; fi
|
|
112
|
+
fi
|
|
113
|
+
|
|
114
|
+
tokens_json=null cost_json=null model_json=null usage_partial_json=null
|
|
115
|
+
if [[ "$backend" == "grok-build" ]] && jq -e 'type == "object"' "$stdout_file" >/dev/null 2>&1; then
|
|
116
|
+
tokens_json="$(jq -c '.usage.total_tokens // .usage.totalTokens // .total_tokens // null' "$stdout_file")"
|
|
117
|
+
cost_json="$(jq -c '.total_cost_usd // .usage.total_cost_usd // null' "$stdout_file")"
|
|
118
|
+
model_json="$(jq -c '.model // ((.usage.models? // {}) | keys[0]?) // null' "$stdout_file")"
|
|
119
|
+
usage_partial_json="$(jq -c '.cost_is_partial // .usage_is_incomplete // null' "$stdout_file")"
|
|
120
|
+
fi
|
|
121
|
+
|
|
122
|
+
success_json=false
|
|
123
|
+
if [[ $backend_status -eq 0 && "$verify_status_json" != "false" ]]; then success_json=true; fi
|
|
124
|
+
stderr_text="$(tail -c 8000 "$stderr_file" 2>/dev/null || true)"
|
|
125
|
+
jq -n \
|
|
126
|
+
--arg backend "$backend" --arg task_id "$task_id" --argjson duration "$((end_seconds - start_seconds))" \
|
|
127
|
+
--argjson success "$success_json" --argjson exit_status "$backend_status" \
|
|
128
|
+
--argjson files_changed "$files_changed" --argjson lines_added "$additions" --argjson lines_removed "$deletions" \
|
|
129
|
+
--argjson tests_passed "$verify_status_json" --argjson test_exit_status "$verify_exit_json" \
|
|
130
|
+
--argjson tokens "$tokens_json" --argjson cost_usd "$cost_json" --argjson model "$model_json" \
|
|
131
|
+
--argjson usage_partial "$usage_partial_json" --arg stderr "$stderr_text" \
|
|
132
|
+
--arg stdout_path "$stdout_file" --arg stderr_path "$stderr_file" \
|
|
133
|
+
--arg verify_stdout_path "$verify_stdout_file" --arg verify_stderr_path "$verify_stderr_file" \
|
|
134
|
+
'{backend:$backend, task_id:$task_id, duration_seconds:$duration, success:$success,
|
|
135
|
+
files_changed:$files_changed, lines_added:$lines_added, lines_removed:$lines_removed,
|
|
136
|
+
tests_passed:$tests_passed, test_exit_status:$test_exit_status, exit_status:$exit_status,
|
|
137
|
+
retry_count:0, tokens:$tokens, cost_usd:$cost_usd, model:$model,
|
|
138
|
+
usage_partial:$usage_partial, backend_error:(if $exit_status == 0 or $stderr == "" then null else $stderr end),
|
|
139
|
+
stdout_path:$stdout_path, stderr_path:$stderr_path,
|
|
140
|
+
verify_stdout_path:(if $tests_passed == null then null else $verify_stdout_path end),
|
|
141
|
+
verify_stderr_path:(if $tests_passed == null then null else $verify_stderr_path end),
|
|
142
|
+
human_review_score:null, review_quality_notes:null}' >> "$results_file"
|
|
143
|
+
done
|
|
144
|
+
|
|
145
|
+
jq -s --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
146
|
+
'{schema_version:1, generated_at:$generated_at, results:.}' "$results_file" > "$output_file"
|
|
147
|
+
printf '%s\n' "$output_file"
|
package/bin/e3d-debate
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
# Prototype: run a question through a multi-round debate across several
|
|
5
|
+
# lib/providers/<name> adapters, then ask one of them to synthesize a final
|
|
6
|
+
# answer from the full transcript. Not wired into bin/e3d-pilot or
|
|
7
|
+
# config.schema.json yet -- this is a standalone experiment to see whether
|
|
8
|
+
# automating the manual "ask several models, compare, decide" loop is worth
|
|
9
|
+
# building out further.
|
|
10
|
+
|
|
11
|
+
# Resolve symlinks so this works when invoked via npm's node_modules/.bin link.
|
|
12
|
+
_src="${BASH_SOURCE[0]}"
|
|
13
|
+
while [[ -L "$_src" ]]; do
|
|
14
|
+
_dir="$(cd "$(dirname "$_src")" && pwd)"
|
|
15
|
+
_src="$(readlink "$_src")"
|
|
16
|
+
[[ "$_src" == /* ]] || _src="$_dir/$_src"
|
|
17
|
+
done
|
|
18
|
+
SCRIPT_DIR="$(cd "$(dirname "$_src")" && pwd)"
|
|
19
|
+
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
20
|
+
PROVIDERS_DIR="$REPO_ROOT/lib/providers"
|
|
21
|
+
# shellcheck source=lib/providers/jev.sh
|
|
22
|
+
source "$REPO_ROOT/lib/providers/jev.sh"
|
|
23
|
+
|
|
24
|
+
DEBATE_PROVIDERS=(claude codex)
|
|
25
|
+
DEBATE_ROUNDS=2
|
|
26
|
+
DEBATE_SYNTHESIZER="claude"
|
|
27
|
+
DEBATE_MAX_WORDS=200
|
|
28
|
+
OUT_DIR=""
|
|
29
|
+
QUESTION=""
|
|
30
|
+
QUESTION_FILE=""
|
|
31
|
+
LINK_REPO=""
|
|
32
|
+
LINK_IDEA=""
|
|
33
|
+
|
|
34
|
+
usage() {
|
|
35
|
+
cat <<'EOF'
|
|
36
|
+
e3d-debate "<question>" [--providers claude,codex] [--rounds 2] [--synthesizer claude] [--max-words 200] [--question-file <path>] [--out-dir <dir>] [--repo <path> --idea <idea-id>]
|
|
37
|
+
|
|
38
|
+
Runs the question through N rounds across the given lib/providers/<name>
|
|
39
|
+
adapters (round 1: independent answers; round 2+: each provider sees every
|
|
40
|
+
other provider's prior-round answer and is asked to rebut or concede), then
|
|
41
|
+
asks --synthesizer to read the full transcript and produce one final answer --
|
|
42
|
+
unless every participant's round-2+ status already converged on "approved",
|
|
43
|
+
in which case the synthesizer call is skipped and the first participant's
|
|
44
|
+
POSITION line is emitted directly (full agreement means everyone would act
|
|
45
|
+
the same, so there is nothing left for a synthesizer to add).
|
|
46
|
+
Prints the transcript and final answer to stdout; also writes transcript.md
|
|
47
|
+
under --out-dir (default: a mktemp dir, path printed at the end).
|
|
48
|
+
|
|
49
|
+
--question-file reads the question from a file instead of argv -- use it for
|
|
50
|
+
long or heavily quoted/formatted questions that are awkward to pass as a
|
|
51
|
+
single shell argument. --max-words caps each provider's per-round answer
|
|
52
|
+
length (raise it for questions that need real depth, e.g. a multi-part
|
|
53
|
+
strategic review).
|
|
54
|
+
|
|
55
|
+
--repo <path> and --idea <idea-id> (must be given together) link this
|
|
56
|
+
debate's verdict into that repo's provenance graph as soon as synthesis
|
|
57
|
+
succeeds, via `e3d-pilot ideas note --decision --evidence`. This requires
|
|
58
|
+
--out-dir to be a path inside --repo (e.g. docs/debates/<slug>), since the
|
|
59
|
+
evidence link is a repo-relative path and only durable once that transcript
|
|
60
|
+
is committed. This does not create the idea -- it must already exist.
|
|
61
|
+
|
|
62
|
+
If TYPESAFE_API_KEY is set, e3d-debate also asks Jev (TypeSafe System One) a
|
|
63
|
+
single Noul question up front -- "is this obvious enough that debate is
|
|
64
|
+
unnecessary?" -- and logs the answer into the transcript as a shadow-mode,
|
|
65
|
+
informational-only signal. It never skips or shortens the debate; this is
|
|
66
|
+
calibration data for a possible future live pre-screen gate (see
|
|
67
|
+
idea-82a5c2225266). No key set means this is silently skipped.
|
|
68
|
+
EOF
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
while [[ $# -gt 0 ]]; do
|
|
72
|
+
case "$1" in
|
|
73
|
+
--providers) IFS=',' read -r -a DEBATE_PROVIDERS <<< "$2"; shift 2 ;;
|
|
74
|
+
--rounds) DEBATE_ROUNDS="$2"; shift 2 ;;
|
|
75
|
+
--synthesizer) DEBATE_SYNTHESIZER="$2"; shift 2 ;;
|
|
76
|
+
--max-words) DEBATE_MAX_WORDS="$2"; shift 2 ;;
|
|
77
|
+
--question-file) QUESTION_FILE="$2"; shift 2 ;;
|
|
78
|
+
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
|
79
|
+
--repo) LINK_REPO="$2"; shift 2 ;;
|
|
80
|
+
--idea) LINK_IDEA="$2"; shift 2 ;;
|
|
81
|
+
-h|--help) usage; exit 0 ;;
|
|
82
|
+
*) QUESTION="$1"; shift ;;
|
|
83
|
+
esac
|
|
84
|
+
done
|
|
85
|
+
|
|
86
|
+
if [[ -n "$QUESTION_FILE" ]]; then
|
|
87
|
+
[[ -f "$QUESTION_FILE" ]] || { echo "error: --question-file not found: $QUESTION_FILE" >&2; exit 1; }
|
|
88
|
+
QUESTION="$(cat "$QUESTION_FILE")"
|
|
89
|
+
fi
|
|
90
|
+
|
|
91
|
+
[[ -n "$QUESTION" ]] || { echo "error: question required (see --help)" >&2; exit 1; }
|
|
92
|
+
[[ "$DEBATE_MAX_WORDS" =~ ^[0-9]+$ ]] && (( DEBATE_MAX_WORDS >= 1 )) || { echo "error: --max-words must be a positive integer" >&2; exit 1; }
|
|
93
|
+
[[ "$DEBATE_ROUNDS" =~ ^[0-9]+$ ]] && (( DEBATE_ROUNDS >= 1 )) || { echo "error: --rounds must be a positive integer" >&2; exit 1; }
|
|
94
|
+
(( ${#DEBATE_PROVIDERS[@]} >= 2 )) || { echo "error: need at least 2 --providers to debate" >&2; exit 1; }
|
|
95
|
+
|
|
96
|
+
for p in "${DEBATE_PROVIDERS[@]}" "$DEBATE_SYNTHESIZER"; do
|
|
97
|
+
[[ -x "$PROVIDERS_DIR/$p" ]] || { echo "error: no such provider adapter: $PROVIDERS_DIR/$p" >&2; exit 1; }
|
|
98
|
+
done
|
|
99
|
+
|
|
100
|
+
if [[ -n "$LINK_REPO" || -n "$LINK_IDEA" ]]; then
|
|
101
|
+
[[ -n "$LINK_REPO" && -n "$LINK_IDEA" ]] || { echo "error: --repo and --idea must be given together" >&2; exit 1; }
|
|
102
|
+
[[ -d "$LINK_REPO" ]] || { echo "error: --repo not found: $LINK_REPO" >&2; exit 1; }
|
|
103
|
+
[[ -n "$OUT_DIR" ]] || { echo "error: --out-dir is required (and must be inside --repo) when using --repo/--idea" >&2; exit 1; }
|
|
104
|
+
mkdir -p "$OUT_DIR"
|
|
105
|
+
LINK_REPO="$(cd "$LINK_REPO" && pwd)"
|
|
106
|
+
out_dir_abs="$(cd "$OUT_DIR" && pwd)"
|
|
107
|
+
case "$out_dir_abs" in
|
|
108
|
+
"$LINK_REPO"/*) LINK_TRANSCRIPT_REL="${out_dir_abs#"$LINK_REPO"/}/transcript.md" ;;
|
|
109
|
+
*) echo "error: --out-dir must be inside --repo to be evidence-linkable (got $out_dir_abs, repo $LINK_REPO)" >&2; exit 1 ;;
|
|
110
|
+
esac
|
|
111
|
+
fi
|
|
112
|
+
|
|
113
|
+
work_dir="$(mktemp -d "${TMPDIR:-/tmp}/e3d-debate.XXXXXX")"
|
|
114
|
+
transcript_md="$work_dir/transcript.md"
|
|
115
|
+
printf '# Debate\n\n**Question:** %s\n\n' "$QUESTION" > "$transcript_md"
|
|
116
|
+
|
|
117
|
+
# Jev pre-screen: shadow-mode only (idea-82a5c2225266) -- this NEVER skips or
|
|
118
|
+
# shortens the debate below, it just logs an informational signal into the
|
|
119
|
+
# transcript for later calibration of whether Jev's obviousness judgment is
|
|
120
|
+
# trustworthy enough to ever gate on. Silently skipped with no
|
|
121
|
+
# TYPESAFE_API_KEY; a failed call is logged and otherwise ignored.
|
|
122
|
+
if jev_available; then
|
|
123
|
+
jev_state="$(jq -ncS --arg q "$QUESTION" '{question:$q}')"
|
|
124
|
+
jev_questions="$(jq -ncS '{
|
|
125
|
+
obviousness: {
|
|
126
|
+
type: "noul",
|
|
127
|
+
instructions: "Is this question'\''s answer obvious enough that informed, knowledgeable people would not meaningfully disagree, or does it need genuine multi-perspective debate to reach a well-reasoned answer?",
|
|
128
|
+
criteria: {
|
|
129
|
+
"true": "The answer is clear-cut -- reasonable experts would converge on the same conclusion without needing to compare independent perspectives.",
|
|
130
|
+
"false": "The answer involves real tradeoffs, judgment calls, or contestable premises where different informed people could reasonably land in different places."
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}')"
|
|
134
|
+
jev_response=""
|
|
135
|
+
jev_response="$(jev_call "jev-latest" "$jev_state" "$jev_questions")" || jev_response=""
|
|
136
|
+
if [[ -n "$jev_response" ]]; then
|
|
137
|
+
# Noul answers carry only a `noul` probability, no `confidence` field
|
|
138
|
+
# (that's Choice/Score-only) -- see https://docs.typesafe.ai/api.md.
|
|
139
|
+
jev_noul="$(jq -r '.answers.obviousness.noul // "unknown"' <<< "$jev_response" 2>/dev/null || echo "unknown")"
|
|
140
|
+
echo "=== Jev pre-screen (shadow mode, informational only -- does not skip the debate): obviousness noul=$jev_noul ===" >&2
|
|
141
|
+
printf '**Jev pre-screen (shadow mode, informational only):** obviousness noul=%s (1.0 = clearly obvious, 0.0 = clearly needs debate) -- logged for calibration, does not affect whether this debate runs.\n\n' "$jev_noul" >> "$transcript_md"
|
|
142
|
+
fi
|
|
143
|
+
fi
|
|
144
|
+
|
|
145
|
+
declare -A LAST_ANSWER
|
|
146
|
+
converged=0
|
|
147
|
+
converged_at_round=0
|
|
148
|
+
|
|
149
|
+
for ((round = 1; round <= DEBATE_ROUNDS; round++)); do
|
|
150
|
+
echo "=== Round $round ===" >&2
|
|
151
|
+
pids=()
|
|
152
|
+
for provider in "${DEBATE_PROVIDERS[@]}"; do
|
|
153
|
+
prompt_file="$work_dir/round${round}-${provider}.prompt.md"
|
|
154
|
+
out_file="$work_dir/round${round}-${provider}.out"
|
|
155
|
+
status_file="$work_dir/round${round}-${provider}.status"
|
|
156
|
+
{
|
|
157
|
+
if (( round == 1 )); then
|
|
158
|
+
printf 'You are one of several AI models independently answering a question, before comparing notes with the others in later rounds.\n\n'
|
|
159
|
+
printf 'Question: %s\n\n' "$QUESTION"
|
|
160
|
+
printf 'Give your best direct answer with your key reasoning and any load-bearing assumptions. Be decisive, not hedgy. Under %d words.\n' "$DEBATE_MAX_WORDS"
|
|
161
|
+
else
|
|
162
|
+
printf 'You are in round %d of a structured debate among several AI models answering one question.\n\n' "$round"
|
|
163
|
+
printf 'Question: %s\n\n' "$QUESTION"
|
|
164
|
+
printf 'Here is what each participant said in the previous round:\n\n'
|
|
165
|
+
for other in "${DEBATE_PROVIDERS[@]}"; do
|
|
166
|
+
printf -- '--- %s (previous round) ---\n%s\n\n' "$other" "${LAST_ANSWER[$other]}"
|
|
167
|
+
done
|
|
168
|
+
printf 'Critically evaluate the other answers against your own. If your prior position holds up, restate it and rebut what'\''s wrong in the others. If you'\''ve changed your mind, say so plainly and why. Under %d words.\n\n' "$DEBATE_MAX_WORDS"
|
|
169
|
+
printf 'End with exactly these two lines, in this exact format:\n'
|
|
170
|
+
printf 'POSITION: <one-sentence current answer>\n'
|
|
171
|
+
printf 'status: approved|revise\n'
|
|
172
|
+
printf '("approved" = a person acting on your answer and a person acting on every other participant'\''s answer would do the same thing -- extra caveats, framing, or added nuance do NOT count as disagreement. "revise" = someone following your answer would act differently than someone following at least one other participant'\''s answer.)\n'
|
|
173
|
+
fi
|
|
174
|
+
} > "$prompt_file"
|
|
175
|
+
(
|
|
176
|
+
set +e
|
|
177
|
+
"$PROVIDERS_DIR/$provider" "$prompt_file" > "$out_file" 2>"$work_dir/round${round}-${provider}.stderr"
|
|
178
|
+
echo $? > "$status_file"
|
|
179
|
+
) &
|
|
180
|
+
pids+=($!)
|
|
181
|
+
done
|
|
182
|
+
for pid in "${pids[@]}"; do wait "$pid" || true; done
|
|
183
|
+
|
|
184
|
+
for provider in "${DEBATE_PROVIDERS[@]}"; do
|
|
185
|
+
out_file="$work_dir/round${round}-${provider}.out"
|
|
186
|
+
status_file="$work_dir/round${round}-${provider}.status"
|
|
187
|
+
status="$(cat "$status_file" 2>/dev/null || echo 1)"
|
|
188
|
+
if [[ "$status" == "0" && -s "$out_file" ]]; then
|
|
189
|
+
answer="$(cat "$out_file")"
|
|
190
|
+
else
|
|
191
|
+
answer="(provider $provider failed in round $round -- see $work_dir/round${round}-${provider}.stderr)"
|
|
192
|
+
fi
|
|
193
|
+
LAST_ANSWER[$provider]="$answer"
|
|
194
|
+
printf '## Round %d -- %s\n\n%s\n\n' "$round" "$provider" "$answer" >> "$transcript_md"
|
|
195
|
+
printf -- '--- Round %d / %s ---\n%s\n\n' "$round" "$provider" "$answer"
|
|
196
|
+
done
|
|
197
|
+
|
|
198
|
+
if (( round >= 2 )); then
|
|
199
|
+
statuses=()
|
|
200
|
+
for provider in "${DEBATE_PROVIDERS[@]}"; do
|
|
201
|
+
s="$(grep -m1 -E '^status: (approved|revise)$' <<< "${LAST_ANSWER[$provider]}" | sed -E 's/^status: //' || true)"
|
|
202
|
+
[[ "$s" == "approved" || "$s" == "revise" ]] || s="revise"
|
|
203
|
+
statuses+=("$s")
|
|
204
|
+
done
|
|
205
|
+
if "$REPO_ROOT/lib/negotiate/convergence" "${statuses[@]}" >/dev/null 2>&1; then
|
|
206
|
+
converged=1
|
|
207
|
+
converged_at_round=$round
|
|
208
|
+
echo "=== Converged after round $round -- all participants agree, skipping remaining rounds ===" >&2
|
|
209
|
+
break
|
|
210
|
+
fi
|
|
211
|
+
fi
|
|
212
|
+
done
|
|
213
|
+
|
|
214
|
+
if (( converged == 1 )); then
|
|
215
|
+
rounds_ran_note="converged after round $converged_at_round of a max $DEBATE_ROUNDS -- all participants ended that round agreeing"
|
|
216
|
+
else
|
|
217
|
+
rounds_ran_note="ran the full $DEBATE_ROUNDS rounds without every participant converging"
|
|
218
|
+
fi
|
|
219
|
+
|
|
220
|
+
synth_prompt="$work_dir/synthesis.prompt.md"
|
|
221
|
+
{
|
|
222
|
+
printf 'Below is the full transcript of a debate between multiple AI models answering one question for a human who needs a single, decisive answer (%s).\n\n' "$rounds_ran_note"
|
|
223
|
+
printf 'Question: %s\n\n' "$QUESTION"
|
|
224
|
+
cat "$transcript_md"
|
|
225
|
+
printf '\nRead the transcript and produce a final answer. Output exactly this format:\n'
|
|
226
|
+
printf 'FINAL ANSWER: <2-4 decisive sentences>\n'
|
|
227
|
+
printf 'CONSENSUS: <full | majority | split>\n'
|
|
228
|
+
printf 'DISSENT: <one line on any remaining disagreement worth flagging, or "none">\n'
|
|
229
|
+
} > "$synth_prompt"
|
|
230
|
+
|
|
231
|
+
synth_out="$work_dir/synthesis.out"
|
|
232
|
+
|
|
233
|
+
# When every participant converged on "approved" in the final round, the
|
|
234
|
+
# synthesizer call adds no information -- everyone already agreed they'd act
|
|
235
|
+
# the same as everyone else. Skip it and emit the first participant's
|
|
236
|
+
# POSITION line directly (order = --providers order, deterministic).
|
|
237
|
+
fast_synthesis=0
|
|
238
|
+
if (( converged == 1 )); then
|
|
239
|
+
first_provider="${DEBATE_PROVIDERS[0]}"
|
|
240
|
+
fast_position="$(grep -m1 -E '^POSITION: ' <<< "${LAST_ANSWER[$first_provider]}" | sed -E 's/^POSITION: //')"
|
|
241
|
+
if [[ -n "$fast_position" ]]; then
|
|
242
|
+
fast_synthesis=1
|
|
243
|
+
printf 'FINAL ANSWER: %s\nCONSENSUS: full\nDISSENT: none\n' "$fast_position" > "$synth_out"
|
|
244
|
+
fi
|
|
245
|
+
fi
|
|
246
|
+
|
|
247
|
+
if (( fast_synthesis == 1 )); then
|
|
248
|
+
synth_status=0
|
|
249
|
+
echo "=== Synthesis (fast path -- all participants converged in round $converged_at_round, no synthesizer call) ===" >&2
|
|
250
|
+
else
|
|
251
|
+
set +e
|
|
252
|
+
"$PROVIDERS_DIR/$DEBATE_SYNTHESIZER" "$synth_prompt" > "$synth_out" 2>"$work_dir/synthesis.stderr"
|
|
253
|
+
synth_status=$?
|
|
254
|
+
set -e
|
|
255
|
+
echo "=== Synthesis ($DEBATE_SYNTHESIZER) ===" >&2
|
|
256
|
+
fi
|
|
257
|
+
|
|
258
|
+
synth_ok=0
|
|
259
|
+
if [[ $synth_status -eq 0 && -s "$synth_out" ]]; then
|
|
260
|
+
synth_ok=1
|
|
261
|
+
cat "$synth_out"
|
|
262
|
+
synth_label="$DEBATE_SYNTHESIZER"
|
|
263
|
+
(( fast_synthesis == 1 )) && synth_label="fast path, no LLM call"
|
|
264
|
+
printf '\n## Synthesis (%s)\n\n%s\n' "$synth_label" "$(cat "$synth_out")" >> "$transcript_md"
|
|
265
|
+
else
|
|
266
|
+
echo "error: synthesis provider failed -- see $work_dir/synthesis.stderr" >&2
|
|
267
|
+
fi
|
|
268
|
+
|
|
269
|
+
if [[ -n "$OUT_DIR" ]]; then
|
|
270
|
+
mkdir -p "$OUT_DIR"
|
|
271
|
+
cp "$transcript_md" "$OUT_DIR/transcript.md"
|
|
272
|
+
echo "transcript: $OUT_DIR/transcript.md" >&2
|
|
273
|
+
else
|
|
274
|
+
echo "transcript: $transcript_md" >&2
|
|
275
|
+
fi
|
|
276
|
+
|
|
277
|
+
if [[ -n "$LINK_REPO" ]]; then
|
|
278
|
+
if [[ "$synth_ok" -eq 1 ]]; then
|
|
279
|
+
provider_list="$(IFS=,; echo "${DEBATE_PROVIDERS[*]}")"
|
|
280
|
+
note_text="$(printf 'e3d-debate (%s, %s) verdict:\n\n%s' "$provider_list" "$rounds_ran_note" "$(cat "$synth_out")")"
|
|
281
|
+
if "$REPO_ROOT/bin/e3d-pilot" ideas note --repo "$LINK_REPO" --decision \
|
|
282
|
+
--evidence "$LINK_TRANSCRIPT_REL" "$LINK_IDEA" "$note_text" \
|
|
283
|
+
--actor "e3d-debate:$provider_list" >&2; then
|
|
284
|
+
echo "provenance: linked to $LINK_IDEA in $LINK_REPO (evidence_ref=$LINK_TRANSCRIPT_REL)" >&2
|
|
285
|
+
else
|
|
286
|
+
echo "warning: failed to link debate evidence onto $LINK_IDEA -- transcript is still at $OUT_DIR/transcript.md, link it manually with 'e3d-pilot ideas note --evidence'" >&2
|
|
287
|
+
fi
|
|
288
|
+
else
|
|
289
|
+
echo "warning: skipping provenance link -- synthesis failed, no verdict to record" >&2
|
|
290
|
+
fi
|
|
291
|
+
fi
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
set -euo pipefail
|
|
3
|
+
|
|
4
|
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
5
|
+
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
6
|
+
# shellcheck source=lib/scoring/fanout.sh
|
|
7
|
+
source "$REPO_ROOT/lib/scoring/fanout.sh"
|
|
8
|
+
|
|
9
|
+
usage() {
|
|
10
|
+
cat <<'EOF'
|
|
11
|
+
Usage: e3d-grok-workers --repo <path> --candidates <json> --output <json>
|
|
12
|
+
[--provider <name>] [--workers <3..5>] [--top <n>]
|
|
13
|
+
|
|
14
|
+
Read-only candidate scoring. `--provider` defaults to grok-build; any
|
|
15
|
+
lib/providers adapter is accepted. Scoring never executes a candidate.
|
|
16
|
+
EOF
|
|
17
|
+
}
|
|
18
|
+
die() { printf 'error: %s\n' "$*" >&2; exit 1; }
|
|
19
|
+
|
|
20
|
+
repo="" candidates_file="" output_file="" provider="grok-build" workers=3 top=5
|
|
21
|
+
while [[ $# -gt 0 ]]; do
|
|
22
|
+
case "$1" in
|
|
23
|
+
--repo) repo="${2:-}"; shift 2 ;;
|
|
24
|
+
--candidates) candidates_file="${2:-}"; shift 2 ;;
|
|
25
|
+
--output) output_file="${2:-}"; shift 2 ;;
|
|
26
|
+
--provider) provider="${2:-}"; shift 2 ;;
|
|
27
|
+
--workers) workers="${2:-}"; shift 2 ;;
|
|
28
|
+
--top) top="${2:-}"; shift 2 ;;
|
|
29
|
+
-h|--help) usage; exit 0 ;;
|
|
30
|
+
*) die "unknown argument: $1" ;;
|
|
31
|
+
esac
|
|
32
|
+
done
|
|
33
|
+
|
|
34
|
+
[[ -n "$repo" && -n "$candidates_file" && -n "$output_file" ]] || { usage >&2; exit 1; }
|
|
35
|
+
[[ "$workers" =~ ^[0-9]+$ ]] && (( workers >= 3 && workers <= 5 )) || die "--workers must be between 3 and 5"
|
|
36
|
+
[[ "$top" =~ ^[0-9]+$ ]] && (( top > 0 )) || die "--top must be a positive integer"
|
|
37
|
+
[[ -d "$repo" ]] || die "repository not found: $repo"
|
|
38
|
+
repo="$(cd "$repo" && pwd -P)"
|
|
39
|
+
jq -e 'type == "array" and length > 0 and all(.[]; (.id | type == "string") and (.id | length > 0))' "$candidates_file" >/dev/null \
|
|
40
|
+
|| die "candidates must be a non-empty JSON array with string id fields"
|
|
41
|
+
[[ "$(jq '[.[].id] | length == (unique | length)' "$candidates_file")" == "true" ]] || die "candidate ids must be unique"
|
|
42
|
+
|
|
43
|
+
artifact_dir="${output_file%.json}.artifacts"
|
|
44
|
+
raw_output="$(mktemp "${TMPDIR:-/tmp}/e3d-scoring-raw.XXXXXX")"
|
|
45
|
+
status=0
|
|
46
|
+
if ! scoring_run_fanout "$repo" "$provider" "$candidates_file" "$workers" "$artifact_dir" "$raw_output"; then
|
|
47
|
+
status=$?
|
|
48
|
+
fi
|
|
49
|
+
[[ -s "$raw_output" ]] || { rm -f "$raw_output"; die "all scoring workers failed; inspect $artifact_dir"; }
|
|
50
|
+
|
|
51
|
+
jq --argjson top "$top" '
|
|
52
|
+
. + {
|
|
53
|
+
selected: (.ranked[:$top]),
|
|
54
|
+
downstream:"frontier review then existing negotiation; no automatic execution"
|
|
55
|
+
}
|
|
56
|
+
' "$raw_output" > "$output_file"
|
|
57
|
+
rm -f "$raw_output"
|
|
58
|
+
printf '%s\n' "$output_file"
|
|
59
|
+
exit "$status"
|