opencode-longrun-harness 1.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +390 -0
- package/docs/V1.2.20_EVIDENCE.md +114 -0
- package/docs/V1.2.21_EVIDENCE.md +68 -0
- package/docs/V1.2.22_EVIDENCE.md +52 -0
- package/harness/commissioning/README.md +16 -0
- package/harness/commissioning/inspect-copied-run.mjs +25 -0
- package/harness/commissioning/verify-copied-case.mjs +35 -0
- package/harness/plugin/longrun.js +677 -0
- package/harness/src/cli.mjs +40 -0
- package/harness/src/controller.js +1413 -0
- package/harness/src/evidence.mjs +135 -0
- package/harness/src/execution.mjs +217 -0
- package/harness/src/executor.mjs +21 -0
- package/harness/src/install.mjs +435 -0
- package/harness/src/maintenance.mjs +257 -0
- package/harness/src/memory.mjs +472 -0
- package/harness/test/candidates.test.mjs +73 -0
- package/harness/test/checkpoint.test.mjs +65 -0
- package/harness/test/controller.test.mjs +230 -0
- package/harness/test/evidence.test.mjs +57 -0
- package/harness/test/fixtures/durable-host.mjs +27 -0
- package/harness/test/fixtures/example-app-run.json +1375 -0
- package/harness/test/fixtures/notes-budget-exhausted-run.json +2070 -0
- package/harness/test/fixtures/notes-premature-complete-run.json +1496 -0
- package/harness/test/fixtures/notes-recovery-run.json +622 -0
- package/harness/test/fixtures/presets-readout-run.json +825 -0
- package/harness/test/fixtures/routing-worker.mjs +35 -0
- package/harness/test/fixtures/vitest-failed-receipt.json +33 -0
- package/harness/test/helper.mjs +41 -0
- package/harness/test/install.test.mjs +117 -0
- package/harness/test/lifecycle.test.mjs +102 -0
- package/harness/test/maintenance.test.mjs +204 -0
- package/harness/test/memory.test.mjs +145 -0
- package/harness/test/negative-control.test.mjs +91 -0
- package/harness/test/plugin.test.mjs +169 -0
- package/harness/test/recovery-runner.test.mjs +435 -0
- package/harness/test/recovery.test.mjs +68 -0
- package/harness/test/repair-mechanics.test.mjs +122 -0
- package/harness/test/toolbehavior.test.mjs +75 -0
- package/harness/test/v121-commissioning.test.mjs +177 -0
- package/harness/test/v1210-deadline.test.mjs +134 -0
- package/harness/test/v1211-pause.test.mjs +81 -0
- package/harness/test/v1212-maintenance-pause.test.mjs +76 -0
- package/harness/test/v1213-readout.test.mjs +82 -0
- package/harness/test/v1214-durable.test.mjs +121 -0
- package/harness/test/v1215-guidance.test.mjs +57 -0
- package/harness/test/v1216-test-summary.test.mjs +39 -0
- package/harness/test/v1217-discovery.test.mjs +73 -0
- package/harness/test/v1218-completion-review.test.mjs +203 -0
- package/harness/test/v1219-budget-pause.test.mjs +134 -0
- package/harness/test/v122-lifecycle-resolver.test.mjs +218 -0
- package/harness/test/v1220-budget-amendment.test.mjs +343 -0
- package/harness/test/v1221-negative-fixture-anchor.test.mjs +65 -0
- package/harness/test/v1222-default-evidence-class.test.mjs +75 -0
- package/harness/test/v123-plugin-e2e.test.mjs +120 -0
- package/harness/test/v123-receipt-model.test.mjs +185 -0
- package/harness/test/v124-canonical.test.mjs +147 -0
- package/harness/test/v124-installed.test.mjs +48 -0
- package/harness/test/v125-stability.test.mjs +183 -0
- package/harness/test/v126-execution.test.mjs +183 -0
- package/harness/test/v127-reconciliation.test.mjs +139 -0
- package/harness/test/v128-compaction.test.mjs +156 -0
- package/harness/test/v129-routing.test.mjs +165 -0
- package/harness/tools/audit-receipts.mjs +121 -0
- package/harness/tools/recovery-runner.mjs +499 -0
- package/package.json +49 -0
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Long-run harness — operator-side recovery runner (source-repo tool, NOT installed into OpenCode).
|
|
3
|
+
//
|
|
4
|
+
// Purpose: when a real natural compaction leaves a tracked run in RECOVERY_REQUIRED and no host is
|
|
5
|
+
// active, perform the standard SUPERVISED resume dispatch automatically, with bounded attempts and a
|
|
6
|
+
// full JSON log. This is operator orchestration around the existing CLI; it does NOT enable in-harness
|
|
7
|
+
// automatic continuation, does NOT touch budgets/state, and refuses loudly whenever anything is
|
|
8
|
+
// ambiguous. Dependency-free (node builtins) like the rest of the harness.
|
|
9
|
+
//
|
|
10
|
+
// Usage:
|
|
11
|
+
// node harness/tools/recovery-runner.mjs --state-dir DIR --project DIR --run RUN_ID
|
|
12
|
+
// [--session SES_ID] [--max-attempts 2] [--config-file JSON] [--prompt-file FILE]
|
|
13
|
+
// [--log-dir DIR] [--timeout-ms N] [--dry-run] [--json]
|
|
14
|
+
// [--fresh-session [--phase resume|verify|complete]] [--allow-status CSV]
|
|
15
|
+
//
|
|
16
|
+
// --fresh-session is the REDUCED-CONTEXT path: when the bound conversation's own context is over the
|
|
17
|
+
// model window it re-compacts on every resume, so dispatching to it again cannot make progress. This
|
|
18
|
+
// mode creates a NEW conversation in the same project instead; the model's first native action must be
|
|
19
|
+
// action=resume for the SAME run, which re-binds the session+worktree (v1.2.2 RE-BIND). The run, its
|
|
20
|
+
// contract, budgets, receipts and counters are untouched, and the mode reports whether the rebind
|
|
21
|
+
// actually happened — a turn that changes nothing is NOT reported as recovered.
|
|
22
|
+
// --allow-status widens which lifecycle states may be dispatched, so it additionally requires an
|
|
23
|
+
// explicit --prompt-file: a human states the intent rather than the tool guessing.
|
|
24
|
+
//
|
|
25
|
+
// Exit codes: 0 recovered / nothing to do; 2 still stuck after the attempt cap; 3 precondition refused;
|
|
26
|
+
// 4 environment or dispatch error (endpoint/model/spawn).
|
|
27
|
+
import fs from "node:fs";
|
|
28
|
+
import path from "node:path";
|
|
29
|
+
import { spawnSync } from "node:child_process";
|
|
30
|
+
import { pathToFileURL } from "node:url";
|
|
31
|
+
|
|
32
|
+
// These are DEFAULTS, not laws. The harness is meant to drive whatever local OpenCode + model setup an
|
|
33
|
+
// operator actually runs, so the model id, the OpenCode binary and the runtime matcher are all
|
|
34
|
+
// overridable by environment. The defaults describe the local MTPLX setup this harness was built
|
|
35
|
+
// against; nothing outside this block should assume them.
|
|
36
|
+
export const DEFAULT_REQUIRED_MODEL = "mtplx-flash-next-optimized-speed";
|
|
37
|
+
export const DEFAULT_REQUIRED_PROVIDER_MODEL = "mtplx/mtplx-flash-next-optimized-speed";
|
|
38
|
+
export const DEFAULT_OPENCODE_BIN = "/opt/homebrew/bin/opencode";
|
|
39
|
+
// Matches the local model server's bundled runtime, used only to attribute a listening socket to it.
|
|
40
|
+
export const DEFAULT_RUNTIME_MATCHER = "mtplx[\\\\/].*runtime-venv|mtplx\\.app";
|
|
41
|
+
|
|
42
|
+
// Back-compat aliases for the shipped defaults, so existing callers and tests keep working.
|
|
43
|
+
export const REQUIRED_MODEL = DEFAULT_REQUIRED_MODEL;
|
|
44
|
+
export const REQUIRED_PROVIDER_MODEL = DEFAULT_REQUIRED_PROVIDER_MODEL;
|
|
45
|
+
export const OPENCODE_BIN = DEFAULT_OPENCODE_BIN;
|
|
46
|
+
|
|
47
|
+
export function requiredModel(env = process.env) { return env.LONGRUN_REQUIRED_MODEL || DEFAULT_REQUIRED_MODEL; }
|
|
48
|
+
export function requiredProviderModel(env = process.env) { return env.LONGRUN_REQUIRED_PROVIDER_MODEL || DEFAULT_REQUIRED_PROVIDER_MODEL; }
|
|
49
|
+
export function opencodeBin(env = process.env) { return env.LONGRUN_OPENCODE_BIN || DEFAULT_OPENCODE_BIN; }
|
|
50
|
+
export function runtimeMatcher(env = process.env) { return env.LONGRUN_RUNTIME_MATCHER || DEFAULT_RUNTIME_MATCHER; }
|
|
51
|
+
|
|
52
|
+
// ---- pure decision logic (unit-tested offline) ------------------------------------------------
|
|
53
|
+
// Decide what to do for a run, before any dispatch.
|
|
54
|
+
// Lifecycle states a reduced-context NEW conversation may adopt. Both are non-terminal and
|
|
55
|
+
// non-ambiguous; neither authorises anything the run itself does not already permit.
|
|
56
|
+
export const FRESH_SESSION_STATUSES = ["RECOVERY_REQUIRED", "PAUSED"];
|
|
57
|
+
|
|
58
|
+
export function decideRecovery({ status, inFlight, hostLive, sessionId, attempt = 0, maxAttempts = 2, allowStatus = null, freshSession = false } = {}) {
|
|
59
|
+
if (hostLive) return { action: "refuse", reason: "HOST_LIVE", detail: "a host already owns this project directory; do not launch a competing turn" };
|
|
60
|
+
if (inFlight) return { action: "refuse", reason: "VERIFY_IN_FLIGHT", detail: "a declared check is reserved/running; wait or reconcile instead" };
|
|
61
|
+
const okStatus = allowStatus ? allowStatus.includes(status) : (freshSession ? FRESH_SESSION_STATUSES.includes(status) : status === "RECOVERY_REQUIRED");
|
|
62
|
+
if (!okStatus) {
|
|
63
|
+
return freshSession
|
|
64
|
+
? { action: "refuse", reason: "NOT_RESUMABLE", status, detail: `reduced-context continuation only adopts a non-terminal ${FRESH_SESSION_STATUSES.join("/")} run, not ${status}` }
|
|
65
|
+
: { action: "refuse", reason: "NOT_RECOVERY_REQUIRED", status, detail: "this tool only resumes a run left in RECOVERY_REQUIRED" };
|
|
66
|
+
}
|
|
67
|
+
// A fresh conversation is exactly the case where the old binding is unusable (missing or over the
|
|
68
|
+
// model window), so the missing binding is not itself a refusal — the run identity is what matters.
|
|
69
|
+
if (!sessionId && !freshSession) return { action: "refuse", reason: "NO_SESSION_BINDING", detail: "no session binding found in runs.json; a recovery would not reach the same session" };
|
|
70
|
+
if (attempt >= maxAttempts) return { action: "give_up", reason: "ATTEMPT_LIMIT", detail: `still ${status} after ${attempt} supervised attempt(s)` };
|
|
71
|
+
return { action: "dispatch", attempt: attempt + 1, reason: freshSession ? "FRESH_SESSION_REBIND" : "SUPERVISED_RESUME" };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Decide the outcome after a dispatch turn finished. In-flight work always wins: a reserved or running
|
|
75
|
+
// check must never be retried over, whatever the lifecycle status says.
|
|
76
|
+
// States in which the run is finished and cannot legitimately be re-bound: a terminal run must never
|
|
77
|
+
// be adopted by a new conversation, so the rebind guard below must not demand one. Observed live: the
|
|
78
|
+
// completion dispatch drove the run to COMPLETE, then the guard scored that success as NO_REBIND and
|
|
79
|
+
// retried until ATTEMPT_LIMIT, so a fully completed run was reported as a runner failure.
|
|
80
|
+
export const TERMINAL_STATUSES = ["COMPLETE", "CANCELLED", "BLOCKED"];
|
|
81
|
+
|
|
82
|
+
export function evaluateOutcome({ status, inFlight } = {}) {
|
|
83
|
+
if (inFlight) return { recovered: false, retry: false, status, reason: "VERIFY_IN_FLIGHT" };
|
|
84
|
+
if (TERMINAL_STATUSES.includes(status)) return { recovered: true, retry: false, terminal: true, status };
|
|
85
|
+
if (status === "RECOVERY_REQUIRED") return { recovered: false, retry: true, status };
|
|
86
|
+
return { recovered: true, retry: false, status };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// The supervised resume prompt. Deliberately bounded and non-destructive.
|
|
90
|
+
export function resumePrompt(runId) {
|
|
91
|
+
return [
|
|
92
|
+
"Supervised same-run recovery. A natural compaction left this tracked run in RECOVERY_REQUIRED and no host is active.",
|
|
93
|
+
`Call native longrun action=next (resume-context) for runId ${runId}, then explicit action=resume for the SAME run, then a bounded action=checkpoint naming the immediate next action, then continue the already-authorized work.`,
|
|
94
|
+
"Do not start a new run, do not change budgets, limits or the contract, do not bypass guards through another tool or session, keep automatic continuation OFF, and do not COMPLETE.",
|
|
95
|
+
"If the run cannot be resumed, report the exact refusal code and stop.",
|
|
96
|
+
].join(" ");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// The reduced-context prompt: a NEW conversation that must first re-bind the SAME run. It is
|
|
100
|
+
// deliberately explicit that this is a continuation, not a new run, and that only the operator can
|
|
101
|
+
// authorise completion (which requires a review acceptance the model cannot grant itself).
|
|
102
|
+
export function freshSessionPrompt(runId, { phase = "verify" } = {}) {
|
|
103
|
+
const parts = [
|
|
104
|
+
"Reduced-context supervised continuation. The previous conversation for this tracked run exceeded the model window and re-compacted on every resume, so this is a NEW conversation continuing the SAME tracked run.",
|
|
105
|
+
`Call native longrun action=status for runId ${runId} first, then explicit action=resume for that SAME runId; resume re-binds this new session and worktree to the existing run, so report whether the rebound flag came back true.`,
|
|
106
|
+
];
|
|
107
|
+
if (phase === "verify") parts.push("Then run the checks the contract already DECLARES, by their declared ids, with longrun_verify(runId=..., checkId=...); do not add, remove or reinterpret criteria, and do not touch the contract or budgets.");
|
|
108
|
+
if (phase === "verify") parts.push("Then record a bounded action=checkpoint naming the immediate next action.");
|
|
109
|
+
if (phase === "complete") parts.push("The operator has already accepted the current evidence review. Confirm with action=status that every REQUIRED criterion is verified and nothing is in flight, then call action=complete for this run and report the canonical result verbatim.");
|
|
110
|
+
parts.push("Do not start a new run or a replacement run, do not change budgets, limits or the contract, do not bypass the lifecycle guards, and keep automatic continuation OFF.");
|
|
111
|
+
if (phase !== "complete") parts.push("Do not COMPLETE: completion additionally requires an operator review acceptance that has not been granted yet.");
|
|
112
|
+
parts.push("If the run cannot be resumed, report the exact refusal code and stop.");
|
|
113
|
+
return parts.join(" ");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// A reduced-context dispatch only counts as recovered if the new conversation actually adopted the
|
|
117
|
+
// run. A turn that merely left the status alone proves nothing and must never read as success.
|
|
118
|
+
// Adoption means BOTH that the new session is bound to this run in runs.json AND that the run's own
|
|
119
|
+
// compaction-owner marker (if it has one) moved to the new session.
|
|
120
|
+
export function applyRebindGuard(outcome, { newSessionId, boundSessions: bound = [], compactionSessionID = null } = {}) {
|
|
121
|
+
const listed = !!(newSessionId && bound.includes(newSessionId));
|
|
122
|
+
const owned = !newSessionId || !compactionSessionID || compactionSessionID === newSessionId;
|
|
123
|
+
const rebound = listed && owned;
|
|
124
|
+
const info = { newSessionId: newSessionId || null, boundSessions: bound.slice(), compactionSessionID: compactionSessionID || null, rebound };
|
|
125
|
+
// A terminal run is a SUCCESSFUL ending, not an unproven adoption: it cannot be rebound by design.
|
|
126
|
+
if (outcome.recovered && !outcome.terminal && !rebound) return { ...info, outcome: { recovered: false, retry: true, status: outcome.status, reason: "NO_REBIND" } };
|
|
127
|
+
return { ...info, outcome };
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// Raw dispatch streams are the audit trail, so their filenames must not collide between dispatches.
|
|
131
|
+
// The original stem keyed only on attempt number and run id, which meant a later dispatch silently
|
|
132
|
+
// OVERWROTE an earlier one's captured output (observed live: the completion dispatch replaced the
|
|
133
|
+
// verify dispatch's stream). The phase and a dispatch timestamp make each stream addressable.
|
|
134
|
+
export function dispatchLogStem({ logDir, attempt, runId, phase = null, at = Date.now() }) {
|
|
135
|
+
const phaseTag = phase ? `-${phase}` : "";
|
|
136
|
+
const stamp = new Date(at).toISOString().replace(/[:.]/g, "-");
|
|
137
|
+
return path.join(logDir, `recovery${phaseTag}-attempt-${attempt}-${runId}-${stamp}`);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// A fresh CLI dispatch reports its own new session id on the event stream. Reading it back from the
|
|
141
|
+
// dispatch output is how the operator proves which conversation actually did the work.
|
|
142
|
+
export function extractSessionId(stdout) {
|
|
143
|
+
for (const line of String(stdout || "").split("\n")) {
|
|
144
|
+
const trimmed = line.trim();
|
|
145
|
+
if (!trimmed.startsWith("{")) continue;
|
|
146
|
+
try {
|
|
147
|
+
const event = JSON.parse(trimmed);
|
|
148
|
+
if (event && typeof event.sessionID === "string" && event.sessionID) return event.sessionID;
|
|
149
|
+
} catch {}
|
|
150
|
+
}
|
|
151
|
+
const m = String(stdout || "").match(/"sessionID"\s*:\s*"(ses_[A-Za-z0-9]+)"/);
|
|
152
|
+
return m ? m[1] : null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// The installed operator launcher, resolved from HOME itself.
|
|
156
|
+
// An earlier version resolved the give-up path with path.dirname(HOME), which yields
|
|
157
|
+
// <dirname(HOME)>/.config/... instead of <HOME>/.config/... — the binary did not exist, the spawn
|
|
158
|
+
// failed, and the pause fallback failed silently (ok:false, exitCode:null) while a run was left in
|
|
159
|
+
// RECOVERY_REQUIRED. Both call sites
|
|
160
|
+
// now share this one definition, and the resolved binary is reported so a failure is diagnosable.
|
|
161
|
+
export function defaultMaintenanceBin(home = process.env.HOME || "") {
|
|
162
|
+
return path.join(home, ".config", "opencode", "longrun-harness", "longrun");
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Operator-side fallback: land a run in a controlled state when supervised resumes cannot clear
|
|
166
|
+
// RECOVERY_REQUIRED (e.g. the session's own context is over the model window, so it re-compacts
|
|
167
|
+
// immediately). This is a canonical, non-destructive lifecycle transition performed by the operator
|
|
168
|
+
// CLI — it resets nothing, only stops the loop. Uses the installed maintenance launcher.
|
|
169
|
+
export function pauseViaMaintenance({ bin, stateDir, project, runId }) {
|
|
170
|
+
const res = spawnSync(bin, ["pause", "--json", "--project", project, "--run", runId], {
|
|
171
|
+
env: { ...process.env, LONGRUN_STATE_DIR: stateDir }, encoding: "utf8", timeout: 60000,
|
|
172
|
+
});
|
|
173
|
+
let state = null;
|
|
174
|
+
try { state = JSON.parse(res.stdout || "{}").state || null; } catch {}
|
|
175
|
+
// A null exitCode means the process never ran (bad bin path, ENOENT). Surface that instead of
|
|
176
|
+
// reporting a bare failure, so the operator can see why the fallback did not happen.
|
|
177
|
+
return { ok: res.status === 0, exitCode: res.status, state, bin,
|
|
178
|
+
error: res.error ? String(res.error.code || res.error.message) : null,
|
|
179
|
+
stderr: String(res.stderr || "").trim().slice(0, 400) || null };
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ---- read-only state helpers -------------------------------------------------------------------
|
|
183
|
+
export function findRunKey(stateDir, runId) {
|
|
184
|
+
const stateRoot = path.join(stateDir, "state");
|
|
185
|
+
for (const name of fs.readdirSync(stateRoot)) {
|
|
186
|
+
if (!/^[a-f0-9]{32}$/.test(name)) continue;
|
|
187
|
+
try {
|
|
188
|
+
const run = JSON.parse(fs.readFileSync(path.join(stateRoot, name, "run.json"), "utf8"));
|
|
189
|
+
if (run && run.runId === runId) return { key: name, run };
|
|
190
|
+
} catch {}
|
|
191
|
+
}
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
export function sessionForRun(stateDir, runId) {
|
|
195
|
+
try {
|
|
196
|
+
const runs = JSON.parse(fs.readFileSync(path.join(stateDir, "runs.json"), "utf8"));
|
|
197
|
+
for (const [sessionId, entry] of Object.entries(runs)) if (entry && entry.runId === runId) return sessionId;
|
|
198
|
+
} catch {}
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
// A run may legitimately accumulate MORE THAN ONE bound conversation: `resume` adds the current
|
|
202
|
+
// session to runs.json rather than replacing the old entry (v1.2.2 RE-BIND). Reading only the first
|
|
203
|
+
// binding therefore shows the historical session and can hide a successful re-bind, so the operator
|
|
204
|
+
// tool reads the whole set instead of guessing from one entry.
|
|
205
|
+
export function boundSessions(stateDir, runId) {
|
|
206
|
+
try {
|
|
207
|
+
const runs = JSON.parse(fs.readFileSync(path.join(stateDir, "runs.json"), "utf8"));
|
|
208
|
+
return Object.entries(runs).filter(([, entry]) => entry && entry.runId === runId).map(([sessionId]) => sessionId);
|
|
209
|
+
} catch { return []; }
|
|
210
|
+
}
|
|
211
|
+
export function hostLive(project, bin = opencodeBin()) {
|
|
212
|
+
const out = spawnSync("pgrep", ["-f", `${bin} run --dir ${project}`], { encoding: "utf8" });
|
|
213
|
+
return out.status === 0 && String(out.stdout || "").trim().length > 0;
|
|
214
|
+
}
|
|
215
|
+
// The inference endpoint is a MOVING detail of the environment, not a constant of this tool: MTPLX has
|
|
216
|
+
// served the required model on :8000 and later on :8001, and another local router can hold :8000 at the
|
|
217
|
+
// same time. Hardcoding one port made the identity check lie — it reported MODEL_MISMATCH against a
|
|
218
|
+
// different server's model list while the required model was in fact being served on the other port.
|
|
219
|
+
// So resolve the endpoint from the live OpenCode config, which is also what the dispatched session
|
|
220
|
+
// itself will use, and keep the historic port only as a last-resort default.
|
|
221
|
+
export const DEFAULT_INFERENCE_BASE = "http://127.0.0.1:8000/v1";
|
|
222
|
+
|
|
223
|
+
export function inferenceBaseFromConfig({ configDir = process.env.OPENCODE_CONFIG_DIR || path.join(process.env.HOME || "", ".config", "opencode") } = {}) {
|
|
224
|
+
for (const name of ["opencode.json", "opencode.jsonc"]) {
|
|
225
|
+
try {
|
|
226
|
+
const cfg = JSON.parse(fs.readFileSync(path.join(configDir, name), "utf8"));
|
|
227
|
+
const url = cfg && cfg.provider && cfg.provider.mtplx && cfg.provider.mtplx.options && cfg.provider.mtplx.options.baseURL;
|
|
228
|
+
if (typeof url === "string" && url.trim()) return url.trim().replace(/\/+$/, "");
|
|
229
|
+
} catch { /* missing or unparsable: try the next candidate */ }
|
|
230
|
+
}
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// The PATH of the config file that supplies the endpoint, so a dispatch can pin that exact file
|
|
235
|
+
// instead of letting a merged sibling override the baseURL the identity check just read.
|
|
236
|
+
export function configFileSupplyingBase({ configDir = process.env.OPENCODE_CONFIG_DIR || path.join(process.env.HOME || "", ".config", "opencode") } = {}) {
|
|
237
|
+
for (const name of ["opencode.json", "opencode.jsonc"]) {
|
|
238
|
+
const p = path.join(configDir, name);
|
|
239
|
+
try {
|
|
240
|
+
const cfg = JSON.parse(fs.readFileSync(p, "utf8"));
|
|
241
|
+
const url = cfg && cfg.provider && cfg.provider.mtplx && cfg.provider.mtplx.options && cfg.provider.mtplx.options.baseURL;
|
|
242
|
+
if (typeof url === "string" && url.trim()) return p;
|
|
243
|
+
} catch { /* try the next candidate */ }
|
|
244
|
+
}
|
|
245
|
+
return null;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Precedence: explicit flag > environment override > the OpenCode config > historic default.
|
|
249
|
+
export function resolveInferenceBase({ flagValue = null, env = process.env, configDir } = {}) {
|
|
250
|
+
const explicit = flagValue || env.LONGRUN_MODEL_BASE;
|
|
251
|
+
if (explicit && String(explicit).trim()) return String(explicit).trim().replace(/\/+$/, "");
|
|
252
|
+
return inferenceBaseFromConfig({ configDir }) || DEFAULT_INFERENCE_BASE;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// A configured port can still be stale in the OTHER direction: MTPLX's own Settings say :8000 and it
|
|
256
|
+
// only fell back to :8001 because another app held :8000 ("the port in Settings is unchanged and will
|
|
257
|
+
// be tried again at the next start"), so the config can point at a port MTPLX has left. The authoritative
|
|
258
|
+
// answer is the OS: MTPLX serves from its bundled runtime, so the listening socket owned by that runtime
|
|
259
|
+
// IS MTPLX's endpoint, whatever port it landed on. Discovery is still held to the same strict single-model
|
|
260
|
+
// identity check below — it only decides WHERE to look, never WHAT is acceptable.
|
|
261
|
+
|
|
262
|
+
// Parse `lsof -nP -iTCP -sTCP:LISTEN -F pcn` output into {pid, port} pairs.
|
|
263
|
+
export function parseListeners(lsofOut) {
|
|
264
|
+
const out = [];
|
|
265
|
+
let pid = null;
|
|
266
|
+
for (const line of String(lsofOut || "").split("\n")) {
|
|
267
|
+
if (line.startsWith("p")) pid = line.slice(1).trim();
|
|
268
|
+
else if (line.startsWith("n") && pid) {
|
|
269
|
+
const m = /:(\d+)$/.exec(line.slice(1).trim());
|
|
270
|
+
if (m) out.push({ pid, port: Number(m[1]) });
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return out;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// The local model server runs from its own bundled runtime (e.g. '<support>/MTPLX/runtime-venv/bin/python'
|
|
277
|
+
// or inside the app bundle), which is what identifies its sockets. The pattern is configurable so the
|
|
278
|
+
// harness is not tied to one vendor's runtime layout.
|
|
279
|
+
export function isLocalModelRuntime(commPath, matcher = DEFAULT_RUNTIME_MATCHER) {
|
|
280
|
+
try { return new RegExp(matcher, "i").test(commPath || ""); } catch { return false; }
|
|
281
|
+
}
|
|
282
|
+
export function isMtplxRuntime(commPath, matcher = DEFAULT_RUNTIME_MATCHER) {
|
|
283
|
+
return isLocalModelRuntime(commPath, matcher);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
export function discoverMtplxBases({ exec = spawnSync, matcher = DEFAULT_RUNTIME_MATCHER } = {}) {
|
|
287
|
+
const out = [];
|
|
288
|
+
try {
|
|
289
|
+
const lsof = exec("lsof", ["-nP", "-iTCP", "-sTCP:LISTEN", "-F", "pcn"], { encoding: "utf8" });
|
|
290
|
+
for (const { pid, port } of parseListeners(lsof && lsof.stdout)) {
|
|
291
|
+
const ps = exec("ps", ["-o", "comm=", "-p", String(pid)], { encoding: "utf8" });
|
|
292
|
+
const comm = String((ps && ps.stdout) || "").trim();
|
|
293
|
+
if (isLocalModelRuntime(comm, matcher)) out.push(`http://127.0.0.1:${port}/v1`);
|
|
294
|
+
}
|
|
295
|
+
} catch { /* discovery is best-effort; the caller falls back to config */ }
|
|
296
|
+
return [...new Set(out)];
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
// Candidates in trust order: what the operator said, then where MTPLX actually is, then what the config
|
|
300
|
+
// says, then the historic default. The FIRST that serves exactly the required model wins.
|
|
301
|
+
export function inferenceCandidates({ flagValue = null, env = process.env, configDir, exec = spawnSync, matcher = runtimeMatcher(env) } = {}) {
|
|
302
|
+
const list = [];
|
|
303
|
+
for (const c of [flagValue || env.LONGRUN_MODEL_BASE, ...discoverMtplxBases({ exec, matcher }), inferenceBaseFromConfig({ configDir }), DEFAULT_INFERENCE_BASE]) {
|
|
304
|
+
if (typeof c === "string" && c.trim()) {
|
|
305
|
+
const norm = c.trim().replace(/\/+$/, "");
|
|
306
|
+
if (!list.includes(norm)) list.push(norm);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return list;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Pick the first candidate that serves EXACTLY the required model. Injected probe keeps this testable.
|
|
313
|
+
export async function pickServedBase({ bases = [], probe = servedModels, required = DEFAULT_REQUIRED_MODEL } = {}) {
|
|
314
|
+
const tried = [];
|
|
315
|
+
for (const base of bases) {
|
|
316
|
+
try {
|
|
317
|
+
const models = await probe(base);
|
|
318
|
+
tried.push({ base, models });
|
|
319
|
+
if (models.length === 1 && models[0] === required) return { base, models, tried };
|
|
320
|
+
} catch (e) {
|
|
321
|
+
tried.push({ base, error: String((e && e.message) || e) });
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return { base: null, models: null, tried };
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// The config content a dispatch should run with: the resolved file, with baseURL forced to the endpoint
|
|
328
|
+
// that was actually verified, so the session cannot talk to a different server than the check approved.
|
|
329
|
+
export function pinnedConfigContent({ file, base }) {
|
|
330
|
+
if (!file) return null;
|
|
331
|
+
try {
|
|
332
|
+
const cfg = JSON.parse(fs.readFileSync(file, "utf8"));
|
|
333
|
+
cfg.provider = cfg.provider || {};
|
|
334
|
+
cfg.provider.mtplx = cfg.provider.mtplx || {};
|
|
335
|
+
cfg.provider.mtplx.options = cfg.provider.mtplx.options || {};
|
|
336
|
+
cfg.provider.mtplx.options.baseURL = base;
|
|
337
|
+
return JSON.stringify(cfg, null, 2);
|
|
338
|
+
} catch { return null; }
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
export async function servedModels(baseUrl = DEFAULT_INFERENCE_BASE) {
|
|
342
|
+
const res = await fetch(`${baseUrl}/models`);
|
|
343
|
+
const body = await res.json();
|
|
344
|
+
return (body.data || []).map((m) => m.id);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ---- main ---------------------------------------------------------------------------------------
|
|
348
|
+
async function main() {
|
|
349
|
+
const argv = process.argv.slice(2);
|
|
350
|
+
const flag = (n) => { const i = argv.indexOf(n); return i >= 0 ? argv[i + 1] : undefined; };
|
|
351
|
+
const has = (n) => argv.includes(n);
|
|
352
|
+
const stateDir = flag("--state-dir"), project = flag("--project"), runId = flag("--run");
|
|
353
|
+
const maxAttempts = Number(flag("--max-attempts") || 2);
|
|
354
|
+
const dryRun = has("--dry-run"), json = has("--json");
|
|
355
|
+
const timeoutMs = Number(flag("--timeout-ms") || 1800000);
|
|
356
|
+
const out = { tool: "recovery-runner", runId, project, stateDir, attempts: [], ok: false };
|
|
357
|
+
const freshSession = has("--fresh-session");
|
|
358
|
+
const phase = flag("--phase") || "verify";
|
|
359
|
+
const allowStatusRaw = flag("--allow-status");
|
|
360
|
+
const allowStatus = allowStatusRaw ? allowStatusRaw.split(",").map((s) => s.trim()).filter(Boolean) : null;
|
|
361
|
+
const bail = (error, detail, code = 3) => {
|
|
362
|
+
out.error = error; out.detail = detail;
|
|
363
|
+
process.stdout.write(JSON.stringify(out, null, json ? 2 : 0) + "\n");
|
|
364
|
+
process.exit(code);
|
|
365
|
+
};
|
|
366
|
+
if (!stateDir || !project || !runId) bail("ARGUMENTS_REQUIRED", "--state-dir, --project and --run are required");
|
|
367
|
+
if (freshSession && flag("--session")) bail("ARGUMENT_CONFLICT", "--fresh-session creates a NEW conversation, so it cannot be combined with --session; pass one or the other");
|
|
368
|
+
if (!["resume", "verify", "complete"].includes(phase)) bail("INVALID_PHASE", `--phase must be resume, verify or complete (saw ${phase})`);
|
|
369
|
+
// Widening the accepted lifecycle states is an operator decision, so it must come with an explicit
|
|
370
|
+
// written prompt instead of the tool's own default wording.
|
|
371
|
+
if (allowStatus && !flag("--prompt-file")) bail("PROMPT_REQUIRED", "--allow-status overrides which lifecycle states may be dispatched; it requires an explicit --prompt-file stating the intent");
|
|
372
|
+
// Toolchain resolved from the environment, with the shipped MTPLX values as defaults. Nothing below
|
|
373
|
+
// should assume a particular model id, binary path or runtime layout.
|
|
374
|
+
const toolchain = { model: requiredModel(), providerModel: requiredProviderModel(), bin: opencodeBin(), matcher: runtimeMatcher() };
|
|
375
|
+
out.toolchain = toolchain;
|
|
376
|
+
const found = findRunKey(stateDir, runId);
|
|
377
|
+
if (!found) { out.error = "NO_RUN"; process.stdout.write(JSON.stringify(out, null, json ? 2 : 0) + "\n"); process.exit(3); }
|
|
378
|
+
const boundSession = flag("--session") || sessionForRun(stateDir, runId);
|
|
379
|
+
const sessionId = freshSession ? null : boundSession;
|
|
380
|
+
out.sessionId = sessionId || null;
|
|
381
|
+
if (freshSession) { out.freshSession = true; out.phase = phase; }
|
|
382
|
+
if (boundSession) out.boundSession = boundSession;
|
|
383
|
+
|
|
384
|
+
// Environment identity check: never dispatch to an unverified model. --skip-model-check exists only
|
|
385
|
+
// for offline unit tests of the refusal/decision paths; a real dispatch must verify the endpoint.
|
|
386
|
+
// The endpoint is DISCOVERED (MTPLX's own listening socket), not assumed, but the identity bar is
|
|
387
|
+
// unchanged: exactly one served model, and it must be the required one.
|
|
388
|
+
if (!has("--skip-model-check")) {
|
|
389
|
+
const bases = inferenceCandidates({ flagValue: flag("--model-base"), matcher: toolchain.matcher });
|
|
390
|
+
out.inferenceCandidates = bases;
|
|
391
|
+
const picked = await pickServedBase({ bases, required: toolchain.model });
|
|
392
|
+
out.inferenceTried = picked.tried;
|
|
393
|
+
if (!picked.base) {
|
|
394
|
+
out.error = "MODEL_MISMATCH";
|
|
395
|
+
out.detail = `no candidate endpoint serves exactly [${toolchain.model}]; tried ${JSON.stringify(picked.tried)}`;
|
|
396
|
+
process.stdout.write(JSON.stringify(out, null, 2) + "\n");
|
|
397
|
+
process.exit(4);
|
|
398
|
+
}
|
|
399
|
+
out.inferenceBase = picked.base;
|
|
400
|
+
out.servedModels = picked.models;
|
|
401
|
+
} else { out.modelCheck = "SKIPPED"; }
|
|
402
|
+
|
|
403
|
+
const promptFile = flag("--prompt-file");
|
|
404
|
+
const prompt = promptFile ? fs.readFileSync(promptFile, "utf8") : (freshSession ? freshSessionPrompt(runId, { phase }) : resumePrompt(runId));
|
|
405
|
+
// Pin the endpoint the check just verified. The identity check and the dispatched session must not
|
|
406
|
+
// disagree about which endpoint is in play: OpenCode merges every config file it finds, so a stale
|
|
407
|
+
// sibling can override the baseURL, and the check would then bless one server while the session talks
|
|
408
|
+
// to another and 404s. We force baseURL to the VERIFIED endpoint inside the config we hand over, so
|
|
409
|
+
// the session cannot reach anything the check did not approve. --no-pin-config opts out.
|
|
410
|
+
const pinnedFile = has("--no-pin-config") ? null : (flag("--config-file") || configFileSupplyingBase());
|
|
411
|
+
if (pinnedFile) out.configPinned = pinnedFile;
|
|
412
|
+
const inlineConfig = pinnedFile
|
|
413
|
+
? ((out.inferenceBase ? pinnedConfigContent({ file: pinnedFile, base: out.inferenceBase }) : null) || fs.readFileSync(pinnedFile, "utf8"))
|
|
414
|
+
: null;
|
|
415
|
+
let attempt = 0;
|
|
416
|
+
for (;;) {
|
|
417
|
+
const run = findRunKey(stateDir, runId)?.run || {};
|
|
418
|
+
const decision = decideRecovery({
|
|
419
|
+
status: run.status, inFlight: run.execution?.inFlight, hostLive: hostLive(project),
|
|
420
|
+
sessionId, attempt, maxAttempts, allowStatus, freshSession,
|
|
421
|
+
});
|
|
422
|
+
out.attempts.push({ attempt, status: run.status, receiptsBefore: (run.receipts || []).length, decision });
|
|
423
|
+
if (decision.action === "refuse") { out.error = decision.reason; out.detail = decision.detail; out.state = run.status; break; }
|
|
424
|
+
if (decision.action === "give_up") {
|
|
425
|
+
out.error = decision.reason; out.detail = decision.detail; out.state = run.status;
|
|
426
|
+
if (run.status === "RECOVERY_REQUIRED" && !has("--no-pause-on-give-up")) {
|
|
427
|
+
const bin = flag("--maintenance-bin") || defaultMaintenanceBin();
|
|
428
|
+
const paused = pauseViaMaintenance({ bin, stateDir, project, runId });
|
|
429
|
+
out.pausedAfterAttemptLimit = paused;
|
|
430
|
+
if (paused.ok) out.state = paused.state || "PAUSED";
|
|
431
|
+
}
|
|
432
|
+
break;
|
|
433
|
+
}
|
|
434
|
+
if (dryRun) { out.ok = true; out.dryRun = true; out.wouldDispatch = true; break; }
|
|
435
|
+
|
|
436
|
+
const env = { ...process.env, LONGRUN_STATE_DIR: stateDir };
|
|
437
|
+
if (inlineConfig) env.OPENCODE_CONFIG_CONTENT = inlineConfig;
|
|
438
|
+
for (const k of ["LONGRUN_TEST", "LONGRUN_CONTROLLER_FILE", "OPENCODE_CLIENT"]) delete env[k];
|
|
439
|
+
const started = Date.now();
|
|
440
|
+
const spawnArgs = ["run", "--dir", project, "--model", toolchain.providerModel, "--agent", "build", "--format", "json"];
|
|
441
|
+
if (sessionId) spawnArgs.push("--session", sessionId); // a fresh conversation deliberately has none
|
|
442
|
+
spawnArgs.push(prompt);
|
|
443
|
+
const res = spawnSync(toolchain.bin, spawnArgs, {
|
|
444
|
+
env, encoding: "utf8", timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024,
|
|
445
|
+
});
|
|
446
|
+
const after = findRunKey(stateDir, runId)?.run || {};
|
|
447
|
+
let outcome = evaluateOutcome({ status: after.status, inFlight: after.execution?.inFlight });
|
|
448
|
+
// Adoption is only required when this dispatch was supposed to TAKE OVER a stopped run. A later
|
|
449
|
+
// phase (e.g. the operator-authorised completion after review) continues a run that is already
|
|
450
|
+
// running, so it is judged by its own result instead.
|
|
451
|
+
if (freshSession && ["PAUSED", "RECOVERY_REQUIRED"].includes(run.status)) {
|
|
452
|
+
const newSessionId = extractSessionId(res.stdout);
|
|
453
|
+
const guarded = applyRebindGuard(outcome, { newSessionId, boundSessions: boundSessions(stateDir, runId),
|
|
454
|
+
compactionSessionID: after.compactionSessionID });
|
|
455
|
+
outcome = guarded.outcome;
|
|
456
|
+
out.attempts.at(-1).freshSession = { newSessionId: guarded.newSessionId, boundSessions: guarded.boundSessions,
|
|
457
|
+
compactionSessionID: guarded.compactionSessionID, rebound: guarded.rebound, statusBefore: run.status,
|
|
458
|
+
statusAfter: after.status, receiptsAfter: (after.receipts || []).length };
|
|
459
|
+
}
|
|
460
|
+
out.attempts.at(-1).dispatch = { exitCode: res.status, timedOut: res.error?.code === "ETIMEDOUT", ms: Date.now() - started };
|
|
461
|
+
out.attempts.at(-1).outcome = outcome;
|
|
462
|
+
// Capture the dispatched host's raw output so the recovery is auditable, not just summarised.
|
|
463
|
+
const logDirEarly = flag("--log-dir");
|
|
464
|
+
if (logDirEarly) {
|
|
465
|
+
fs.mkdirSync(logDirEarly, { recursive: true });
|
|
466
|
+
const stem = dispatchLogStem({ logDir: logDirEarly, attempt: attempt + 1, runId,
|
|
467
|
+
phase: freshSession ? phase : null, at: started });
|
|
468
|
+
try { fs.writeFileSync(`${stem}-events.jsonl`, String(res.stdout || "")); } catch {}
|
|
469
|
+
try { fs.writeFileSync(`${stem}-stderr.log`, String(res.stderr || "")); } catch {}
|
|
470
|
+
out.attempts.at(-1).dispatch.eventsFile = `${stem}-events.jsonl`;
|
|
471
|
+
}
|
|
472
|
+
if (!outcome.retry) { out.ok = outcome.recovered; out.state = after.status; break; }
|
|
473
|
+
attempt += 1;
|
|
474
|
+
if (attempt >= maxAttempts) {
|
|
475
|
+
out.error = "ATTEMPT_LIMIT"; out.state = after.status;
|
|
476
|
+
// Leave the run controlled rather than thrashing in recovery, unless told not to.
|
|
477
|
+
if (after.status === "RECOVERY_REQUIRED" && !has("--no-pause-on-give-up")) {
|
|
478
|
+
const bin = flag("--maintenance-bin") || defaultMaintenanceBin();
|
|
479
|
+
const paused = pauseViaMaintenance({ bin, stateDir, project, runId });
|
|
480
|
+
out.pausedAfterAttemptLimit = paused;
|
|
481
|
+
if (paused.ok) out.state = paused.state || "PAUSED";
|
|
482
|
+
}
|
|
483
|
+
break;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
const logDir = flag("--log-dir");
|
|
488
|
+
if (logDir) {
|
|
489
|
+
fs.mkdirSync(logDir, { recursive: true });
|
|
490
|
+
const file = path.join(logDir, `recovery-runner-${runId}-${new Date().toISOString().replace(/[:.]/g, "-")}.json`);
|
|
491
|
+
fs.writeFileSync(file, JSON.stringify(out, null, 2));
|
|
492
|
+
out.log = file;
|
|
493
|
+
}
|
|
494
|
+
process.stdout.write(JSON.stringify(out, null, 2) + "\n");
|
|
495
|
+
process.exit(out.ok ? 0 : (out.error === "ATTEMPT_LIMIT" ? 2 : 3));
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
const invokedDirectly = process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url;
|
|
499
|
+
if (invokedDirectly) main().catch((e) => { process.stdout.write("ERROR " + (e && e.message) + "\n"); process.exit(4); });
|
package/package.json
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "opencode-longrun-harness",
|
|
3
|
+
"version": "1.2.22",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Bounded, evidence-driven long-run workflow harness for OpenCode: contract-bound runs, fingerprint-bound receipts, operator-reviewed completion. Dependency-free.",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"main": "harness/plugin/longrun.js",
|
|
8
|
+
"bin": {
|
|
9
|
+
"longrun-harness": "harness/src/cli.mjs"
|
|
10
|
+
},
|
|
11
|
+
"files": [
|
|
12
|
+
"harness/src",
|
|
13
|
+
"harness/plugin",
|
|
14
|
+
"harness/tools",
|
|
15
|
+
"harness/commissioning",
|
|
16
|
+
"harness/test",
|
|
17
|
+
"docs/V1.2.20_EVIDENCE.md",
|
|
18
|
+
"docs/V1.2.21_EVIDENCE.md",
|
|
19
|
+
"docs/V1.2.22_EVIDENCE.md",
|
|
20
|
+
"LICENSE"
|
|
21
|
+
],
|
|
22
|
+
"keywords": [
|
|
23
|
+
"opencode",
|
|
24
|
+
"plugin",
|
|
25
|
+
"harness",
|
|
26
|
+
"agent",
|
|
27
|
+
"verification",
|
|
28
|
+
"evidence",
|
|
29
|
+
"long-running",
|
|
30
|
+
"llm"
|
|
31
|
+
],
|
|
32
|
+
"repository": {
|
|
33
|
+
"type": "git",
|
|
34
|
+
"url": "git+https://github.com/shamusj-create/opencode-longrun-harness.git"
|
|
35
|
+
},
|
|
36
|
+
"homepage": "https://github.com/shamusj-create/opencode-longrun-harness#readme",
|
|
37
|
+
"bugs": {
|
|
38
|
+
"url": "https://github.com/shamusj-create/opencode-longrun-harness/issues"
|
|
39
|
+
},
|
|
40
|
+
"engines": {
|
|
41
|
+
"node": ">=22"
|
|
42
|
+
},
|
|
43
|
+
"scripts": {
|
|
44
|
+
"test": "node --test harness/test/*.test.mjs",
|
|
45
|
+
"doctor": "node harness/src/cli.mjs doctor",
|
|
46
|
+
"install:global": "node harness/src/cli.mjs install",
|
|
47
|
+
"uninstall": "node harness/src/cli.mjs uninstall"
|
|
48
|
+
}
|
|
49
|
+
}
|