@gr8ful/spf 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/defaults/spf.config.yaml +2 -0
- package/assets/prompts/refiner/system.md +1 -0
- package/assets/prompts/scout/system.md +1 -0
- package/assets/skill/references/config.md +7 -1
- package/dist/chains/index.js +7 -1
- package/dist/cli/commands/watch.js +19 -29
- package/dist/core/agent_cc.js +9 -0
- package/dist/core/agent_flue.d.ts +41 -1
- package/dist/core/agent_flue.js +99 -1
- package/dist/core/utils.d.ts +24 -0
- package/dist/core/utils.js +56 -1
- package/dist/core/watch.d.ts +17 -0
- package/dist/core/watch.js +48 -2
- package/package.json +1 -1
|
@@ -246,6 +246,7 @@ agents:
|
|
|
246
246
|
- ls
|
|
247
247
|
- bash
|
|
248
248
|
- write
|
|
249
|
+
- webfetch # check current docs for a stack/tool the repo uses, rather than guessing from training data
|
|
249
250
|
|
|
250
251
|
# No tester agent: running the suite is a known command, so it is a kind="code"
|
|
251
252
|
# phase over core/quality.ts. See SKILL.md hard rule 8.
|
|
@@ -265,6 +266,7 @@ agents:
|
|
|
265
266
|
- ls
|
|
266
267
|
- bash
|
|
267
268
|
- write
|
|
269
|
+
- webfetch # look up a possible tech stack/tool/3rd-party API's current docs before decomposing around it
|
|
268
270
|
|
|
269
271
|
- name: reviewer
|
|
270
272
|
model: openai/gpt-5.6-terra
|
|
@@ -11,6 +11,7 @@ Decompose a product spec into a feature/story-or-bug tree of vertical slices the
|
|
|
11
11
|
- You inherit the operator's shell environment — their PATH, toolchains and credentials are already live. Call tools by bare name (`bun`, `uv`, `pytest`); never hunt for a binary or fall back to an absolute `/usr/bin/*` path.
|
|
12
12
|
- Judge any command you run by its exit status, never by scanning its output for words. `error` or `not found` inside passing output is text, not a failure.
|
|
13
13
|
- Write your working notes to `<context_handoff_dir>/refine_plan.md` before emitting your Report JSON.
|
|
14
|
+
- `webfetch` fetches a URL's current content — use it to check a specific library/API/tool's actual current docs when a slice depends on one you're not certain about, rather than decomposing around stale training-data assumptions. It fetches a page you name; it cannot search the web for one.
|
|
14
15
|
|
|
15
16
|
## Grounding: tie every slice to real code
|
|
16
17
|
|
|
@@ -12,6 +12,7 @@ Find and report where things live. Change nothing.
|
|
|
12
12
|
- Judge any command you run by its exit status, never by scanning its output for words. `error` or `not found` inside passing output is text, not a failure.
|
|
13
13
|
- Write your findings to `<context_handoff_dir>/scout_findings.md` for agents that follow.
|
|
14
14
|
- If you find nothing, say so plainly — an empty finding is a valid finding.
|
|
15
|
+
- `webfetch` fetches a URL's current content when the repo touches a library, API, or tool you're not certain about — use it to check, don't rely on stale training-data knowledge of a fast-moving stack. Not for general research: fetch a specific doc/reference page, not a search query.
|
|
15
16
|
|
|
16
17
|
## Subagents
|
|
17
18
|
|
|
@@ -564,11 +564,17 @@ session** — a joined run starts that agent fresh instead of resuming.
|
|
|
564
564
|
| `grep` | search file contents |
|
|
565
565
|
| `glob` (alias: `find`) | find files by pattern |
|
|
566
566
|
| `ls` | recognized name, **no built-in on either backend** — harmless to list, never mounts |
|
|
567
|
+
| `webfetch` | fetch a URL over HTTP(S), return its content as plain text — e.g. a library/API's current docs |
|
|
567
568
|
|
|
568
569
|
These names are canonical across backends — a roster entry never says
|
|
569
570
|
which; each backend module (`agent_flue.ts`, `agent_cc.ts`) maps them to
|
|
570
571
|
its own tool vocabulary (Flue's lowercase functions, Claude Code's
|
|
571
|
-
capitalized `Read`/`Bash`/...).
|
|
572
|
+
capitalized `Read`/`Bash`/...). `webfetch` maps to Claude Code's own native
|
|
573
|
+
`WebFetch` on that backend; Flue has no such built-in, so on `flue` it's a
|
|
574
|
+
custom tool that runs `curl`/`wget` through the agent's own `Sandbox.exec()`
|
|
575
|
+
— which means a remote sandbox's `egress` policy (see `sandbox` below)
|
|
576
|
+
governs it exactly like any other `bash`-issued network call, with no
|
|
577
|
+
separate rule to configure.
|
|
572
578
|
|
|
573
579
|
**Resolution order:** an agent's own `tools` wins → else `defaults.tools` →
|
|
574
580
|
else unset (all tools usable). An empty list is a tool-less agent, and it
|
package/dist/chains/index.js
CHANGED
|
@@ -80,7 +80,13 @@ export const CHAINS = [
|
|
|
80
80
|
// {{previous_envelope}}. This makes `scout` a required agent for this
|
|
81
81
|
// chain: a roster that pruned it fails agents.validate() by name at
|
|
82
82
|
// `spf watch` startup, same as any other missing required agent.
|
|
83
|
-
|
|
83
|
+
// retries: 2 (3 attempts total) — `gates.artifactsExist` failing here has
|
|
84
|
+
// historically been transient (a declared artifact briefly missing,
|
|
85
|
+
// since fixed at the source by giving `claim()` real per-issue
|
|
86
|
+
// exclusivity — see `core/watch.ts`'s `issueLockPath`), so a same-session
|
|
87
|
+
// correction retry or two is worth it before this blocks the whole spec
|
|
88
|
+
// and pages a human.
|
|
89
|
+
steps.scout({ description: "Map the subsystems this spec touches — change nothing", retries: 2 }),
|
|
84
90
|
steps.refine(),
|
|
85
91
|
steps.publishIssues(),
|
|
86
92
|
]),
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* actual state machine; this file is just the wiring: config, the GitHub
|
|
6
6
|
* provider, the chain-dispatch callback, the lockfile, and the CLI loop.
|
|
7
7
|
*/
|
|
8
|
-
import { existsSync, mkdirSync, readFileSync, symlinkSync
|
|
8
|
+
import { existsSync, mkdirSync, readFileSync, symlinkSync } from "node:fs";
|
|
9
9
|
import { homedir } from "node:os";
|
|
10
10
|
import path from "node:path";
|
|
11
11
|
import * as v from "valibot";
|
|
@@ -23,7 +23,7 @@ import { withRunScope } from "../../core/sandbox.js";
|
|
|
23
23
|
import { excludeSpfDataFromGit } from "../../core/worktree_data.js";
|
|
24
24
|
import { ReviewOutput } from "../../core/data_types.js";
|
|
25
25
|
import { SfDb } from "../../ui/server/db.js";
|
|
26
|
-
import { parseCli } from "../../core/utils.js";
|
|
26
|
+
import { acquirePidLock, parseCli, releasePidLock } from "../../core/utils.js";
|
|
27
27
|
import { isInteractive } from "../ask.js";
|
|
28
28
|
/** Kept well under Slack's own 2900-char slice on `detail` (see `slack_channel.ts`) — a reviewer can emit a lot of findings, but the PR body/notification only needs enough to tell a human whether to look closer. */
|
|
29
29
|
const MAX_REVIEW_DIGEST_CHARS = 1200;
|
|
@@ -149,34 +149,14 @@ function resolveCodeHostProvider(cfg) {
|
|
|
149
149
|
console.error(`watch.code_host ${JSON.stringify(cfg.watch.code_host)} is not supported`);
|
|
150
150
|
return null;
|
|
151
151
|
}
|
|
152
|
-
|
|
153
|
-
try {
|
|
154
|
-
process.kill(pid, 0);
|
|
155
|
-
return true;
|
|
156
|
-
}
|
|
157
|
-
catch {
|
|
158
|
-
return false;
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
/** Stale-steal, like the reference implementation's daemon lock: a dead pid never blocks a restart. */
|
|
152
|
+
/** Stale-steal, like the reference implementation's daemon lock: a dead pid never blocks a restart. Atomic — see `utils.acquirePidLock`. */
|
|
162
153
|
function acquireLock(lockPath) {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
throw new Error(`another \`spf watch\` is already running (pid ${pid}) — lock at ${lockPath}`);
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
170
|
-
writeFileSync(lockPath, String(process.pid));
|
|
171
|
-
}
|
|
172
|
-
function releaseLock(lockPath) {
|
|
173
|
-
try {
|
|
174
|
-
unlinkSync(lockPath);
|
|
175
|
-
}
|
|
176
|
-
catch {
|
|
177
|
-
// already gone — fine
|
|
154
|
+
const result = acquirePidLock(lockPath);
|
|
155
|
+
if (!result.ok) {
|
|
156
|
+
throw new Error(`another \`spf watch\` is already running (pid ${result.holderPid}) — lock at ${lockPath}`);
|
|
178
157
|
}
|
|
179
158
|
}
|
|
159
|
+
const releaseLock = releasePidLock;
|
|
180
160
|
/**
|
|
181
161
|
* `spf watch init` — idempotently seed the `<prefix>:*` labels the state
|
|
182
162
|
* machine needs, with sensible colors/descriptions. Doesn't touch git or
|
|
@@ -743,8 +723,18 @@ export async function watchCommand(argv) {
|
|
|
743
723
|
if (stopping)
|
|
744
724
|
break;
|
|
745
725
|
}
|
|
746
|
-
|
|
747
|
-
|
|
726
|
+
// BOTH lanes, not just `inflight` — `state.refining`'s specs are exactly
|
|
727
|
+
// as in-flight (a fire-and-forget `runSpec(...).finally(...)`, same
|
|
728
|
+
// shape as `runIssue`'s), and a graceful stop that only waits on
|
|
729
|
+
// `inflight` would print "stopping after the current tick" and then
|
|
730
|
+
// exit while a refine chain is still writing into its worktree — the
|
|
731
|
+
// next `spf watch` start-up would find `claimSpecs`'s per-issue lock
|
|
732
|
+
// still held by this (still-alive, still-running) process's own pid and
|
|
733
|
+
// correctly back off, but ONLY because that lock exists; before it did,
|
|
734
|
+
// this exact gap is what let a resumed spec race a still-running prior
|
|
735
|
+
// attempt for the same issue.
|
|
736
|
+
while (state.inflight.size > 0 || state.refining.size > 0) {
|
|
737
|
+
deps.log(`[spf] watch draining ${state.inflight.size} in-flight issue(s), ${state.refining.size} refining spec(s)...`); // deps.log already routes to the dashboard when one is mounted, console.log otherwise
|
|
748
738
|
await interruptibleSleep(1000);
|
|
749
739
|
if (stopping && sigints >= 2)
|
|
750
740
|
break; // stop() itself already exits on the 2nd signal; this is belt-and-suspenders
|
package/dist/core/agent_cc.js
CHANGED
|
@@ -213,6 +213,15 @@ const TOOL_NAME_MAP = {
|
|
|
213
213
|
bash: "Bash",
|
|
214
214
|
grep: "Grep",
|
|
215
215
|
glob: "Glob",
|
|
216
|
+
// CC's own built-in — fetches a URL and returns its content as markdown,
|
|
217
|
+
// already runs headlessly under --dangerously-skip-permissions like every
|
|
218
|
+
// other tool here. No sandbox/egress layer of SPF's own to route through:
|
|
219
|
+
// CC's own process makes the request directly, same as a bash `curl` an
|
|
220
|
+
// agent with `bash` could already issue — see agent_flue.ts's
|
|
221
|
+
// `createWebFetchTool` for the equivalent on the flue backend, where no
|
|
222
|
+
// native fetch tool exists and one has to route through the Sandbox
|
|
223
|
+
// instead.
|
|
224
|
+
webfetch: "WebFetch",
|
|
216
225
|
};
|
|
217
226
|
const TOOL_ALIASES = { find: "glob" };
|
|
218
227
|
const DROPPED_TOOLS = new Set(["ls"]);
|
|
@@ -22,7 +22,8 @@
|
|
|
22
22
|
* `run()`'s signature deliberately mirrors the old agent_pi.ts `run()` so
|
|
23
23
|
* agents.ts's `send()` closure changes only its imports and field names.
|
|
24
24
|
*/
|
|
25
|
-
import
|
|
25
|
+
import * as v from "valibot";
|
|
26
|
+
import { type ConversationStreamChunk, type Sandbox } from "@flue/runtime";
|
|
26
27
|
import type { AgentRequest, AgentResult } from "./data_types.ts";
|
|
27
28
|
/**
|
|
28
29
|
* Folds Flue's `tool-input` + `tool-output`/`tool-output-error` chunk pair
|
|
@@ -37,6 +38,45 @@ export declare class ToolCallTracker {
|
|
|
37
38
|
observe(chunk: ConversationStreamChunk): Record<string, any> | null;
|
|
38
39
|
private finish;
|
|
39
40
|
}
|
|
41
|
+
/**
|
|
42
|
+
* Strip `<script>`/`<style>` blocks and tags, decode the handful of entities
|
|
43
|
+
* a real docs page actually uses, and collapse whitespace. NOT a real
|
|
44
|
+
* HTML-to-markdown renderer — Claude Code's native `WebFetch` (the
|
|
45
|
+
* claude_code backend) does that; Flue has nothing equivalent built in, and
|
|
46
|
+
* a full renderer is more than this needs. A JSON/plain-text response
|
|
47
|
+
* passes through essentially unchanged, since none of these patterns match
|
|
48
|
+
* it — same "minimal, not a full parser" trade `jira_provider.ts`'s
|
|
49
|
+
* `adfToText` makes for ADF.
|
|
50
|
+
*/
|
|
51
|
+
export declare function stripHtml(text: string): string;
|
|
52
|
+
/**
|
|
53
|
+
* The flue backend's stand-in for Claude Code's native `WebFetch` tool —
|
|
54
|
+
* `@flue/runtime` ships no such built-in (its only tool factories are
|
|
55
|
+
* read/write/edit/bash/grep/glob), so this is a custom `defineTool()`.
|
|
56
|
+
*
|
|
57
|
+
* Runs the actual request THROUGH `env.exec()` rather than calling Node's
|
|
58
|
+
* own `fetch()` directly — deliberately: `exec()` is the one universal
|
|
59
|
+
* primitive every `Sandbox` implements (`local`, and the remote
|
|
60
|
+
* `opensandbox`/`cloudflare` backends via `core/sandbox.ts`), so a command
|
|
61
|
+
* run through it executes INSIDE whichever sandbox the agent is actually
|
|
62
|
+
* using. A remote sandbox's own `egress` policy (`sandbox_opensandbox.ts`'s
|
|
63
|
+
* `networkPolicy`) then governs this exactly as it already governs every
|
|
64
|
+
* `bash` call — no separate egress rule needed for this tool. Calling
|
|
65
|
+
* `fetch()` here instead would silently bypass that policy by making the
|
|
66
|
+
* request from SPF's own orchestrator process rather than the sandbox.
|
|
67
|
+
*
|
|
68
|
+
* `curl`, with a `wget` fallback in the SAME command for portability across
|
|
69
|
+
* whatever base image a remote sandbox happens to ship — both are close to
|
|
70
|
+
* universal, but neither is guaranteed; a sandbox image with neither
|
|
71
|
+
* surfaces that plainly as a failed tool call (exit code + stderr handed
|
|
72
|
+
* back to the model), not a hang. Only `http`/`https` are accepted — this is
|
|
73
|
+
* also what stops a `file://` URL from turning "fetch a page" into "read an
|
|
74
|
+
* arbitrary local file" on the `local` sandbox, where `exec()` runs with the
|
|
75
|
+
* operator's own full filesystem access.
|
|
76
|
+
*/
|
|
77
|
+
export declare function createWebFetchTool(env: Sandbox): import("@flue/runtime").ToolDefinition<v.ObjectSchema<{
|
|
78
|
+
readonly url: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.NonEmptyAction<string, "url is required">]>;
|
|
79
|
+
}, undefined>, undefined, false, false>;
|
|
40
80
|
/** Used by agents.validate() so a typo'd tool name fails before anything spawns. */
|
|
41
81
|
export declare function isKnownToolName(name: string): boolean;
|
|
42
82
|
/**
|
package/dist/core/agent_flue.js
CHANGED
|
@@ -22,7 +22,8 @@
|
|
|
22
22
|
* `run()`'s signature deliberately mirrors the old agent_pi.ts `run()` so
|
|
23
23
|
* agents.ts's `send()` closure changes only its imports and field names.
|
|
24
24
|
*/
|
|
25
|
-
import
|
|
25
|
+
import * as v from "valibot";
|
|
26
|
+
import { AgentRunError, createBashTool, createEditTool, createGlobTool, createGrepTool, createReadTool, createWriteTool, defineTool, init, observe, useDataWriter, useModel, useSandbox, useTool, } from "@flue/runtime";
|
|
26
27
|
import { local, sqlite, start } from "@flue/runtime/node";
|
|
27
28
|
import { UsageBreakdown, makeAgentResult } from "./data_types.js";
|
|
28
29
|
import { registerOllamaModel } from "./ollama_provider.js";
|
|
@@ -127,6 +128,102 @@ export class ToolCallTracker {
|
|
|
127
128
|
return record;
|
|
128
129
|
}
|
|
129
130
|
}
|
|
131
|
+
// ── webfetch: Flue's stand-in for Claude Code's native WebFetch ─────────────
|
|
132
|
+
const WEBFETCH_MAX_BYTES = 200_000; // a bounded chunk of a page, not the whole thing — matches RESULT_SNIPPET_CHARS's spirit
|
|
133
|
+
const WEBFETCH_TIMEOUT_MS = 20_000;
|
|
134
|
+
const WebFetchParams = v.object({ url: v.pipe(v.string(), v.nonEmpty("url is required")) });
|
|
135
|
+
/** POSIX-safe single-quoting for a shell argument: closes and reopens the quote around any embedded `'`. */
|
|
136
|
+
function shQuote(value) {
|
|
137
|
+
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Strip `<script>`/`<style>` blocks and tags, decode the handful of entities
|
|
141
|
+
* a real docs page actually uses, and collapse whitespace. NOT a real
|
|
142
|
+
* HTML-to-markdown renderer — Claude Code's native `WebFetch` (the
|
|
143
|
+
* claude_code backend) does that; Flue has nothing equivalent built in, and
|
|
144
|
+
* a full renderer is more than this needs. A JSON/plain-text response
|
|
145
|
+
* passes through essentially unchanged, since none of these patterns match
|
|
146
|
+
* it — same "minimal, not a full parser" trade `jira_provider.ts`'s
|
|
147
|
+
* `adfToText` makes for ADF.
|
|
148
|
+
*/
|
|
149
|
+
export function stripHtml(text) {
|
|
150
|
+
const cleaned = text
|
|
151
|
+
.replace(/<script[\s\S]*?<\/script>/gi, "")
|
|
152
|
+
.replace(/<style[\s\S]*?<\/style>/gi, "")
|
|
153
|
+
.replace(/<[^>]+>/g, " ")
|
|
154
|
+
.replace(/ /g, " ")
|
|
155
|
+
.replace(/&/g, "&")
|
|
156
|
+
.replace(/</g, "<")
|
|
157
|
+
.replace(/>/g, ">")
|
|
158
|
+
.replace(/"/g, '"')
|
|
159
|
+
.replace(/'/g, "'")
|
|
160
|
+
.replace(/[ \t]+/g, " ");
|
|
161
|
+
// Line-by-line trim BEFORE collapsing blank runs: a tag->" " substitution
|
|
162
|
+
// (above) routinely leaves a stray leading/trailing space on the line that
|
|
163
|
+
// used to hold a block-level tag's boundary — collapsing blank lines first
|
|
164
|
+
// would miss those, since they're not yet blank.
|
|
165
|
+
return cleaned
|
|
166
|
+
.split("\n")
|
|
167
|
+
.map((line) => line.trim())
|
|
168
|
+
.join("\n")
|
|
169
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
170
|
+
.trim();
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The flue backend's stand-in for Claude Code's native `WebFetch` tool —
|
|
174
|
+
* `@flue/runtime` ships no such built-in (its only tool factories are
|
|
175
|
+
* read/write/edit/bash/grep/glob), so this is a custom `defineTool()`.
|
|
176
|
+
*
|
|
177
|
+
* Runs the actual request THROUGH `env.exec()` rather than calling Node's
|
|
178
|
+
* own `fetch()` directly — deliberately: `exec()` is the one universal
|
|
179
|
+
* primitive every `Sandbox` implements (`local`, and the remote
|
|
180
|
+
* `opensandbox`/`cloudflare` backends via `core/sandbox.ts`), so a command
|
|
181
|
+
* run through it executes INSIDE whichever sandbox the agent is actually
|
|
182
|
+
* using. A remote sandbox's own `egress` policy (`sandbox_opensandbox.ts`'s
|
|
183
|
+
* `networkPolicy`) then governs this exactly as it already governs every
|
|
184
|
+
* `bash` call — no separate egress rule needed for this tool. Calling
|
|
185
|
+
* `fetch()` here instead would silently bypass that policy by making the
|
|
186
|
+
* request from SPF's own orchestrator process rather than the sandbox.
|
|
187
|
+
*
|
|
188
|
+
* `curl`, with a `wget` fallback in the SAME command for portability across
|
|
189
|
+
* whatever base image a remote sandbox happens to ship — both are close to
|
|
190
|
+
* universal, but neither is guaranteed; a sandbox image with neither
|
|
191
|
+
* surfaces that plainly as a failed tool call (exit code + stderr handed
|
|
192
|
+
* back to the model), not a hang. Only `http`/`https` are accepted — this is
|
|
193
|
+
* also what stops a `file://` URL from turning "fetch a page" into "read an
|
|
194
|
+
* arbitrary local file" on the `local` sandbox, where `exec()` runs with the
|
|
195
|
+
* operator's own full filesystem access.
|
|
196
|
+
*/
|
|
197
|
+
export function createWebFetchTool(env) {
|
|
198
|
+
return defineTool({
|
|
199
|
+
name: "webfetch",
|
|
200
|
+
description: "Fetch a URL over HTTP(S) and return its content as plain text (HTML tags stripped). " +
|
|
201
|
+
"Use it to check current documentation for a library, API, or tool before relying on prior " +
|
|
202
|
+
"knowledge that may be stale or version-specific. GET only, no custom headers/auth, response " +
|
|
203
|
+
"truncated to a safe size.",
|
|
204
|
+
input: WebFetchParams,
|
|
205
|
+
run: async ({ data }) => {
|
|
206
|
+
let parsed;
|
|
207
|
+
try {
|
|
208
|
+
parsed = new URL(data.url);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
return `webfetch: ${JSON.stringify(data.url)} is not a valid URL`;
|
|
212
|
+
}
|
|
213
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
214
|
+
return `webfetch: unsupported scheme ${JSON.stringify(parsed.protocol)} — only http/https are allowed`;
|
|
215
|
+
}
|
|
216
|
+
const url = shQuote(parsed.toString());
|
|
217
|
+
const command = `(curl -sS -L --max-time 20 --max-redirs 5 -A "spf-webfetch/1.0" ${url} ` +
|
|
218
|
+
`|| wget -qO- --timeout=20 ${url}) | head -c ${WEBFETCH_MAX_BYTES}`;
|
|
219
|
+
const result = await env.exec(command, { timeoutMs: WEBFETCH_TIMEOUT_MS });
|
|
220
|
+
if (result.exitCode !== 0 || !result.stdout.trim()) {
|
|
221
|
+
return `webfetch: fetching ${parsed.toString()} failed (exit ${result.exitCode}): ${(result.stderr || "no output").trim().slice(0, 2000)}`;
|
|
222
|
+
}
|
|
223
|
+
return stripHtml(result.stdout);
|
|
224
|
+
},
|
|
225
|
+
});
|
|
226
|
+
}
|
|
130
227
|
// ── tool-name resolution ─────────────────────────────────────────────────────
|
|
131
228
|
const BUILTIN_TOOLS = {
|
|
132
229
|
read: createReadTool,
|
|
@@ -135,6 +232,7 @@ const BUILTIN_TOOLS = {
|
|
|
135
232
|
bash: createBashTool,
|
|
136
233
|
grep: createGrepTool,
|
|
137
234
|
glob: createGlobTool,
|
|
235
|
+
webfetch: createWebFetchTool,
|
|
138
236
|
};
|
|
139
237
|
// pi's vocabulary -> Flue's. "ls" has no Flue built-in (bash/glob cover it);
|
|
140
238
|
// it is a KNOWN name that resolves to nothing, not an unknown one.
|
package/dist/core/utils.d.ts
CHANGED
|
@@ -31,6 +31,30 @@ export declare function newId(length?: number): string;
|
|
|
31
31
|
* on the first attempt so callers see it immediately).
|
|
32
32
|
*/
|
|
33
33
|
export declare function fetchRetryTransient(input: string, init?: RequestInit): Promise<Response>;
|
|
34
|
+
/** `process.kill(pid, 0)` sends no signal — it throws iff `pid` isn't running (or isn't ours to signal), the standard Node liveness probe. */
|
|
35
|
+
export declare function isPidAlive(pid: number): boolean;
|
|
36
|
+
export type PidLockResult = {
|
|
37
|
+
ok: true;
|
|
38
|
+
} | {
|
|
39
|
+
ok: false;
|
|
40
|
+
holderPid: number;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Atomic PID lockfile at `lockPath`: `{ flag: "wx" }` makes "does anyone
|
|
44
|
+
* hold this" and "claim it" ONE filesystem operation, closing the race a
|
|
45
|
+
* separate `existsSync` check followed by a later `writeFileSync` leaves
|
|
46
|
+
* open — two callers can both pass that existence check before either
|
|
47
|
+
* writes, and both walk away believing they hold the lock. A dead pid never
|
|
48
|
+
* blocks a caller: on `EEXIST`, `isPidAlive` decides whether this is a live
|
|
49
|
+
* holder (returns `ok: false`) or a stale lock from a process that never
|
|
50
|
+
* cleaned up (crash, SIGKILL, a restart that didn't wait for it to exit) —
|
|
51
|
+
* stolen via unlink-then-retry. Another caller can win that same steal
|
|
52
|
+
* race; if so, ITS write is what the retry's `wx` finds, and this caller
|
|
53
|
+
* correctly backs off against a live pid on the next pass instead of
|
|
54
|
+
* clobbering a real holder.
|
|
55
|
+
*/
|
|
56
|
+
export declare function acquirePidLock(lockPath: string): PidLockResult;
|
|
57
|
+
export declare function releasePidLock(lockPath: string): void;
|
|
34
58
|
/** Matches Python's `datetime.now(timezone.utc).isoformat(timespec="milliseconds")`. */
|
|
35
59
|
export declare function nowIso(): string;
|
|
36
60
|
export declare function ensureDir(dirPath: string): string;
|
package/dist/core/utils.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { randomBytes } from "node:crypto";
|
|
10
10
|
import { spawnSync } from "node:child_process";
|
|
11
|
-
import { existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
|
|
11
|
+
import { existsSync, mkdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
/**
|
|
14
14
|
* The engineer's own environment, as their shell would hand it over.
|
|
@@ -55,6 +55,61 @@ export async function fetchRetryTransient(input, init) {
|
|
|
55
55
|
return await fetch(input, init);
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
|
+
/** `process.kill(pid, 0)` sends no signal — it throws iff `pid` isn't running (or isn't ours to signal), the standard Node liveness probe. */
|
|
59
|
+
export function isPidAlive(pid) {
|
|
60
|
+
try {
|
|
61
|
+
process.kill(pid, 0);
|
|
62
|
+
return true;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return false;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Atomic PID lockfile at `lockPath`: `{ flag: "wx" }` makes "does anyone
|
|
70
|
+
* hold this" and "claim it" ONE filesystem operation, closing the race a
|
|
71
|
+
* separate `existsSync` check followed by a later `writeFileSync` leaves
|
|
72
|
+
* open — two callers can both pass that existence check before either
|
|
73
|
+
* writes, and both walk away believing they hold the lock. A dead pid never
|
|
74
|
+
* blocks a caller: on `EEXIST`, `isPidAlive` decides whether this is a live
|
|
75
|
+
* holder (returns `ok: false`) or a stale lock from a process that never
|
|
76
|
+
* cleaned up (crash, SIGKILL, a restart that didn't wait for it to exit) —
|
|
77
|
+
* stolen via unlink-then-retry. Another caller can win that same steal
|
|
78
|
+
* race; if so, ITS write is what the retry's `wx` finds, and this caller
|
|
79
|
+
* correctly backs off against a live pid on the next pass instead of
|
|
80
|
+
* clobbering a real holder.
|
|
81
|
+
*/
|
|
82
|
+
export function acquirePidLock(lockPath) {
|
|
83
|
+
mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
84
|
+
for (;;) {
|
|
85
|
+
try {
|
|
86
|
+
writeFileSync(lockPath, String(process.pid), { flag: "wx" });
|
|
87
|
+
return { ok: true };
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
if (error.code !== "EEXIST")
|
|
91
|
+
throw error;
|
|
92
|
+
const pid = Number.parseInt(readFileSync(lockPath, "utf-8").trim(), 10);
|
|
93
|
+
if (Number.isInteger(pid) && isPidAlive(pid))
|
|
94
|
+
return { ok: false, holderPid: pid };
|
|
95
|
+
try {
|
|
96
|
+
unlinkSync(lockPath);
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
// already gone (another caller's steal beat us to it) — loop back
|
|
100
|
+
// and let the `wx` above settle who actually gets it
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
export function releasePidLock(lockPath) {
|
|
106
|
+
try {
|
|
107
|
+
unlinkSync(lockPath);
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
// already gone — fine
|
|
111
|
+
}
|
|
112
|
+
}
|
|
58
113
|
/** Matches Python's `datetime.now(timezone.utc).isoformat(timespec="milliseconds")`. */
|
|
59
114
|
export function nowIso() {
|
|
60
115
|
const iso = new Date().toISOString(); // e.g. 2024-01-01T12:00:00.123Z
|
package/dist/core/watch.d.ts
CHANGED
|
@@ -222,6 +222,23 @@ export declare function createWatchState(): WatchRunState;
|
|
|
222
222
|
export declare function branchNameFor(issue: Issue): string;
|
|
223
223
|
/** Same idea as `branchNameFor`, for the refine lane's throwaway worktree — a spec never gets a PR, so this branch is only ever fetched-from-and-thrown-away, never pushed. */
|
|
224
224
|
export declare function refineBranchNameFor(issue: Issue): string;
|
|
225
|
+
/**
|
|
226
|
+
* A local, atomic, PID-checked lock per `adwId` (`issue-<id>`/`spec-<id>`) —
|
|
227
|
+
* sibling to `worktreesDir`, so it lives at the same per-repo root
|
|
228
|
+
* (`~/.spf/watch/<repo>/locks/`). `provider.claim()`'s own read-modify-write
|
|
229
|
+
* (see `IssueProvider.claim`'s doc comment) only verifies the END STATE
|
|
230
|
+
* looks claimed, not that no one else raced it — the real exclusivity has
|
|
231
|
+
* to live here, gating `claim()` itself, because a resumed/orphaned spec
|
|
232
|
+
* dispatches straight into the SAME deterministic worktree and
|
|
233
|
+
* `context_handoff_dir` a still-running prior attempt already owns (see
|
|
234
|
+
* `chains/steps.ts`'s `clearStaleRefineOutputFiles` doc comment). A dead
|
|
235
|
+
* holder (the daemon that owned it got killed, not stopped) never blocks a
|
|
236
|
+
* fresh claim — `acquirePidLock`'s stale-steal — but a genuinely live
|
|
237
|
+
* holder (this daemon restarted while the OLD process's chain run, spawned
|
|
238
|
+
* fire-and-forget from `claimNewWork`/`claimSpecs`, was still in flight)
|
|
239
|
+
* does, until that run's own `.finally()` releases it.
|
|
240
|
+
*/
|
|
241
|
+
export declare function issueLockPath(deps: WatchDeps, adwId: string): string;
|
|
225
242
|
/**
|
|
226
243
|
* Any issue labeled `working` that THIS process isn't tracking is an
|
|
227
244
|
* orphan — a daemon restart, or another instance's claim this process
|
package/dist/core/watch.js
CHANGED
|
@@ -59,7 +59,7 @@ import { attemptAdwId, attemptBranch, attemptWorktreePath, runBestOf } from "./f
|
|
|
59
59
|
import { PRIORITY_RANK } from "./data_types.js";
|
|
60
60
|
import { redact } from "./otel.js";
|
|
61
61
|
import { parseRefineMarker } from "./refine.js";
|
|
62
|
-
import { newId } from "./utils.js";
|
|
62
|
+
import { acquirePidLock, newId, releasePidLock } from "./utils.js";
|
|
63
63
|
const MAX_ORPHAN_ATTEMPTS = 2;
|
|
64
64
|
/** GitHub's own documented sub-issue nesting cap (see `github_provider.ts`'s `linkChild` doc comment) — `rollUp`'s own recursion bound, so a malformed/cyclic hierarchy can't spin forever. */
|
|
65
65
|
const MAX_ROLLUP_DEPTH = 8;
|
|
@@ -95,6 +95,25 @@ function worktreePathFor(deps, issue) {
|
|
|
95
95
|
function specWorktreePathFor(deps, issue) {
|
|
96
96
|
return path.join(deps.worktreesDir, `spec-${issue.id}`);
|
|
97
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* A local, atomic, PID-checked lock per `adwId` (`issue-<id>`/`spec-<id>`) —
|
|
100
|
+
* sibling to `worktreesDir`, so it lives at the same per-repo root
|
|
101
|
+
* (`~/.spf/watch/<repo>/locks/`). `provider.claim()`'s own read-modify-write
|
|
102
|
+
* (see `IssueProvider.claim`'s doc comment) only verifies the END STATE
|
|
103
|
+
* looks claimed, not that no one else raced it — the real exclusivity has
|
|
104
|
+
* to live here, gating `claim()` itself, because a resumed/orphaned spec
|
|
105
|
+
* dispatches straight into the SAME deterministic worktree and
|
|
106
|
+
* `context_handoff_dir` a still-running prior attempt already owns (see
|
|
107
|
+
* `chains/steps.ts`'s `clearStaleRefineOutputFiles` doc comment). A dead
|
|
108
|
+
* holder (the daemon that owned it got killed, not stopped) never blocks a
|
|
109
|
+
* fresh claim — `acquirePidLock`'s stale-steal — but a genuinely live
|
|
110
|
+
* holder (this daemon restarted while the OLD process's chain run, spawned
|
|
111
|
+
* fire-and-forget from `claimNewWork`/`claimSpecs`, was still in flight)
|
|
112
|
+
* does, until that run's own `.finally()` releases it.
|
|
113
|
+
*/
|
|
114
|
+
export function issueLockPath(deps, adwId) {
|
|
115
|
+
return path.join(path.dirname(deps.worktreesDir), "locks", `${adwId}.lock`);
|
|
116
|
+
}
|
|
98
117
|
function cleanupWorktree(deps, marker) {
|
|
99
118
|
if (!marker)
|
|
100
119
|
return;
|
|
@@ -1182,9 +1201,21 @@ export async function claimNewWork(deps, state) {
|
|
|
1182
1201
|
deps.log(`watch: [dry-run] would claim ${issue.id} (${issue.title}) and run chain "${deps.chain}"`);
|
|
1183
1202
|
continue;
|
|
1184
1203
|
}
|
|
1204
|
+
// Gate the tracker-side claim itself behind local exclusivity — see
|
|
1205
|
+
// `issueLockPath`'s doc comment. A held lock means a live process (this
|
|
1206
|
+
// machine, this tick or an earlier one) already owns this issue's
|
|
1207
|
+
// worktree, so skip identically to a lost tracker-side claim race
|
|
1208
|
+
// rather than ever writing the tracker label.
|
|
1209
|
+
const lockPath = issueLockPath(deps, `issue-${issue.id}`);
|
|
1210
|
+
const lock = acquirePidLock(lockPath);
|
|
1211
|
+
if (!lock.ok) {
|
|
1212
|
+
deps.log(`watch: ${issue.id} is locked by another live \`spf watch\` process (pid ${lock.holderPid}) — skipping`);
|
|
1213
|
+
continue;
|
|
1214
|
+
}
|
|
1185
1215
|
const claimed = await deps.provider.claim(issue);
|
|
1186
1216
|
if (!claimed) {
|
|
1187
1217
|
deps.log(`watch: ${issue.id} lost the claim race this tick — skipping`);
|
|
1218
|
+
releasePidLock(lockPath);
|
|
1188
1219
|
continue;
|
|
1189
1220
|
}
|
|
1190
1221
|
deps.log(`watch: claimed ${issue.id}: ${issue.title}`);
|
|
@@ -1200,6 +1231,7 @@ export async function claimNewWork(deps, state) {
|
|
|
1200
1231
|
runIssue(deps, issue).finally(() => {
|
|
1201
1232
|
state.inflight.delete(issue.id);
|
|
1202
1233
|
state.inflightParents.delete(issue.id);
|
|
1234
|
+
releasePidLock(lockPath);
|
|
1203
1235
|
});
|
|
1204
1236
|
}
|
|
1205
1237
|
}
|
|
@@ -1238,9 +1270,20 @@ export async function claimSpecs(deps, state, from = "spec-ready") {
|
|
|
1238
1270
|
deps.log(`watch: [dry-run] would claim spec ${issue.id} (${issue.title}) and run refine chain "${deps.refineChain}"`);
|
|
1239
1271
|
continue;
|
|
1240
1272
|
}
|
|
1273
|
+
// See claimNewWork's identical guard and issueLockPath's doc comment —
|
|
1274
|
+
// same local-exclusivity gate, keyed to match runSpec's own adwId
|
|
1275
|
+
// (`spec-<id>`) so it covers the SAME deterministic worktree a resumed
|
|
1276
|
+
// (`continue-refinement`) claim reruns into.
|
|
1277
|
+
const lockPath = issueLockPath(deps, `spec-${issue.id}`);
|
|
1278
|
+
const lock = acquirePidLock(lockPath);
|
|
1279
|
+
if (!lock.ok) {
|
|
1280
|
+
deps.log(`watch: spec ${issue.id} is locked by another live \`spf watch\` process (pid ${lock.holderPid}) — skipping`);
|
|
1281
|
+
continue;
|
|
1282
|
+
}
|
|
1241
1283
|
const claimed = await deps.provider.claim(issue, { from, to: "refining" });
|
|
1242
1284
|
if (!claimed) {
|
|
1243
1285
|
deps.log(`watch: spec ${issue.id} lost the claim race this tick — skipping`);
|
|
1286
|
+
releasePidLock(lockPath);
|
|
1244
1287
|
continue;
|
|
1245
1288
|
}
|
|
1246
1289
|
deps.log(`watch: claimed spec ${issue.id}: ${issue.title}`);
|
|
@@ -1251,7 +1294,10 @@ export async function claimSpecs(deps, state, from = "spec-ready") {
|
|
|
1251
1294
|
fields: [["issue", issue.id], ["title", issue.title], ["chain", deps.refineChain]],
|
|
1252
1295
|
});
|
|
1253
1296
|
state.refining.add(issue.id);
|
|
1254
|
-
runSpec(deps, issue).finally(() =>
|
|
1297
|
+
runSpec(deps, issue).finally(() => {
|
|
1298
|
+
state.refining.delete(issue.id);
|
|
1299
|
+
releasePidLock(lockPath);
|
|
1300
|
+
});
|
|
1255
1301
|
}
|
|
1256
1302
|
}
|
|
1257
1303
|
function tickErrorHandler(deps, stage) {
|