gemcatch 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -1
- package/README.md +18 -3
- package/db.js +4 -1
- package/index.js +280 -36
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,72 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] - 2026-07-19
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `gemcatch export [--tag <t>] [--status <s>] [--format md|json] [-o <file>]` —
|
|
15
|
+
concatenate finished results into one document, each under a heading with its
|
|
16
|
+
prompt, id and date. Markdown by default (or JSON for `jq`), to stdout or a
|
|
17
|
+
file. This is the "gather" step that pairs with `batch`'s "scatter": where
|
|
18
|
+
`get` prints one result at a time, `export` collects a whole tag at once.
|
|
19
|
+
- `gemcatch digest --tag <t>` — feed a tag's completed results back through a
|
|
20
|
+
single Gemini call to synthesise one summary. Submits like `research` and
|
|
21
|
+
watches to completion; the summary lands under `<tag>-digest`.
|
|
22
|
+
- `GEMCATCH_WATCH_MAX_FAILS` (default 10) — the consecutive-poll-failure bound
|
|
23
|
+
at which `watch` and `batch -w` give up rather than loop forever.
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- `research --watch` no longer marks a **successfully submitted, server-running**
|
|
28
|
+
task `failed` when a poll errors during the watch. A transient poll failure (a
|
|
29
|
+
5xx past its retries, a network blip) or an expiry mid-watch would propagate to
|
|
30
|
+
the submit handler and overwrite the status to `failed`, dropping the task from
|
|
31
|
+
the active set so the daemon abandoned it and the result was lost. The watch
|
|
32
|
+
loop now rides out poll errors (retrying on the next interval), and only a
|
|
33
|
+
failed *submit* — a task with no `interaction_id` yet — is ever marked `failed`.
|
|
34
|
+
- A wedged or expired interaction no longer keeps a task in flight forever. When a
|
|
35
|
+
poll returns **404** (the free tier drops interactions after 24h, or one was
|
|
36
|
+
deleted), the task is retired locally to `incomplete` with a recorded reason, so
|
|
37
|
+
it leaves the active set and `daemon --exit-when-idle` converges. `watch` and
|
|
38
|
+
`batch -w` additionally stop after `GEMCATCH_WATCH_MAX_FAILS` consecutive poll
|
|
39
|
+
failures (or a stalled batch), with a clear message and a non-zero exit, instead
|
|
40
|
+
of spinning.
|
|
41
|
+
- A **completed-but-empty** result is now served from the local cache. `get` and
|
|
42
|
+
`watch` gated the cache hit on the result being truthy, so a task that completed
|
|
43
|
+
with empty text (`''`) skipped the cache, re-polled, and 404'd after 24h — the
|
|
44
|
+
exact loss the daemon exists to prevent. The gate is now on presence
|
|
45
|
+
(`result != null`), not truthiness.
|
|
46
|
+
- `prune -d <negative>` (or a non-numeric `--days`) is rejected instead of putting
|
|
47
|
+
the cutoff in the future and deleting **every** finished task. `--days` must now
|
|
48
|
+
be a non-negative number.
|
|
49
|
+
- `batch` no longer silently drops a prompt line that starts with `#`. A `#` is a
|
|
50
|
+
comment only when followed by whitespace (`# like this`); a line such as
|
|
51
|
+
`#1 cause of X?` is a real prompt and survives. When comment or blank lines are
|
|
52
|
+
skipped, a one-line count is noted on stderr.
|
|
53
|
+
- `watch -i` / `daemon -i` reject a non-positive interval (`-i -5` busy-looped,
|
|
54
|
+
`-i 0` silently fell back to the default). `list -n 0` now returns zero rows
|
|
55
|
+
instead of all of them, and `-n` rejects negatives (which SQLite reads as "no
|
|
56
|
+
limit").
|
|
57
|
+
- A second `Ctrl-C` to the daemon now force-exits (130) instead of doing nothing
|
|
58
|
+
while a long paced pass finishes.
|
|
59
|
+
- One-shot commands close the SQLite store on exit, so they no longer leave
|
|
60
|
+
`-wal`/`-shm` sidecar files lingering next to `tasks.db`.
|
|
61
|
+
- Colour written to **stderr** (the status chatter from `watch`, `daemon` and
|
|
62
|
+
`research -w`) is now keyed to `process.stderr.isTTY`, not stdout's. Redirecting
|
|
63
|
+
one stream no longer strips colour from the other, nor leaks raw ANSI into a
|
|
64
|
+
redirected file.
|
|
65
|
+
|
|
66
|
+
### Notes
|
|
67
|
+
|
|
68
|
+
- The default `@google/genai` SDK transport is now covered by the offline suite
|
|
69
|
+
(previously every test forced `GEMCATCH_FORCE_REST=1`): a stubbed client drives
|
|
70
|
+
submit → poll → completed and one unwrapped SDK error, confirming `shape()`
|
|
71
|
+
reads an SDK-shaped response and `friendly()` surfaces Google's real message.
|
|
72
|
+
- `GEMCATCH_RPM` pacing is **per process**. Two concurrent `gemcatch` processes
|
|
73
|
+
each keep their own counter and can together exceed the ceiling; run a single
|
|
74
|
+
daemon if the limit must hold. Documented in the README.
|
|
75
|
+
|
|
10
76
|
## [0.2.0] - 2026-07-19
|
|
11
77
|
|
|
12
78
|
### Added
|
|
@@ -96,7 +162,8 @@ seen a task complete, the text is cached locally and survives that expiry — bu
|
|
|
96
162
|
something has to poll inside that window for it to be seen at all, which is what
|
|
97
163
|
`gemcatch daemon` exists to do.
|
|
98
164
|
|
|
99
|
-
[Unreleased]: https://github.com/Booyaka101/gemcatch/compare/v0.
|
|
165
|
+
[Unreleased]: https://github.com/Booyaka101/gemcatch/compare/v0.3.0...HEAD
|
|
166
|
+
[0.3.0]: https://github.com/Booyaka101/gemcatch/compare/v0.2.0...v0.3.0
|
|
100
167
|
[0.2.0]: https://github.com/Booyaka101/gemcatch/compare/v0.1.1...v0.2.0
|
|
101
168
|
[0.1.1]: https://github.com/Booyaka101/gemcatch/compare/v0.1.0...v0.1.1
|
|
102
169
|
[0.1.0]: https://github.com/Booyaka101/gemcatch/releases/tag/v0.1.0
|
package/README.md
CHANGED
|
@@ -83,6 +83,8 @@ This week in AI: ...
|
|
|
83
83
|
| `gemcatch status <id>` | Polls the API and prints the current state. |
|
|
84
84
|
| `gemcatch get <id>` | Prints the full response if complete, otherwise the current status. |
|
|
85
85
|
| `gemcatch list` | All tasks, newest first: id, age, status, prompt. |
|
|
86
|
+
| `gemcatch export` | Concatenates finished results, each under its prompt, to stdout or a file (Markdown or JSON). |
|
|
87
|
+
| `gemcatch digest` | Feeds a tag's completed results through one Gemini call into a single summary. |
|
|
86
88
|
| `gemcatch watch <id>` | Polls until the task finishes, then prints the result. |
|
|
87
89
|
| `gemcatch sync` | Refreshes every in-flight task in one pass. |
|
|
88
90
|
| `gemcatch daemon` | Keeps polling in-flight tasks on a loop, so results are cached before they expire. |
|
|
@@ -102,9 +104,12 @@ Useful flags:
|
|
|
102
104
|
| `-t, --tag <tag>` | `research`, `batch`, `list` | Label tasks and filter them. |
|
|
103
105
|
| `-w, --watch` | `research`, `batch` | Submit and wait, in one command. |
|
|
104
106
|
| `--separator <str>` | `batch` | Split the file on this delimiter line for multi-line prompts. |
|
|
105
|
-
| `-i, --interval <s>` | `watch`, `daemon` | Poll rate. Default 10s for `watch`, 300s for `daemon`. |
|
|
107
|
+
| `-i, --interval <s>` | `watch`, `daemon` | Poll rate in seconds; must be > 0. Default 10s for `watch`, 300s for `daemon`. |
|
|
106
108
|
| `--exit-when-idle` | `daemon` | Stop once nothing is left in flight. |
|
|
107
|
-
|
|
|
109
|
+
| `--status <s>` | `list`, `export` | Only tasks in this status. |
|
|
110
|
+
| `-n, --limit <n>` | `list` | Cap the rows (non-negative; `0` shows none). |
|
|
111
|
+
| `--format <md\|json>` | `export` | Output format. Default `md`. |
|
|
112
|
+
| `-o, --out <file>` | `export` | Write to a file instead of stdout. |
|
|
108
113
|
| `--dry-run` | `batch`, `prune` | Show what would go; submit/delete nothing. |
|
|
109
114
|
| `--raw` | `get` | Dump the raw interaction JSON. |
|
|
110
115
|
|
|
@@ -117,10 +122,15 @@ Statuses come straight from the API: `in_progress`, `requires_action`, `complete
|
|
|
117
122
|
```bash
|
|
118
123
|
# Fire off a whole file of prompts in one command, then collect later.
|
|
119
124
|
# Every task shares one auto-generated tag (batch-xxxxxx), printed on submit.
|
|
120
|
-
$ gemcatch batch questions.txt # one prompt per line; # and blanks skipped
|
|
125
|
+
$ gemcatch batch questions.txt # one prompt per line; "# " and blanks skipped
|
|
121
126
|
$ gemcatch daemon --exit-when-idle # keep polling until they're all in
|
|
122
127
|
$ gemcatch list --tag batch-1a2b3c --status completed
|
|
123
128
|
|
|
129
|
+
# Collect a whole batch into one document (the "gather" for batch's "scatter").
|
|
130
|
+
$ gemcatch export --tag batch-1a2b3c -o results.md # Markdown, one section per prompt
|
|
131
|
+
$ gemcatch export --tag batch-1a2b3c --format json | jq -r '.[].result'
|
|
132
|
+
$ gemcatch digest --tag batch-1a2b3c # or synthesize them into one summary
|
|
133
|
+
|
|
124
134
|
# Multi-line prompts: split the file on a delimiter line instead of per-line
|
|
125
135
|
$ gemcatch batch briefs.md --separator ---
|
|
126
136
|
$ gemcatch batch - < questions.txt # or pipe the list in on stdin
|
|
@@ -182,10 +192,14 @@ $ gemcatch daemon --exit-when-idle -i 30
|
|
|
182
192
|
$ gemcatch list --tag batch1 --status completed
|
|
183
193
|
```
|
|
184
194
|
|
|
195
|
+
If a task's interaction has vanished server-side — the free tier dropped it after 24h, or it was deleted — polling it returns a 404. Rather than chase a task that can never resolve, `gemcatch` retires it locally to `incomplete`, so it leaves the in-flight set and `--exit-when-idle` still converges. `watch` and `batch -w` also give up after a bounded run of consecutive poll failures (`GEMCATCH_WATCH_MAX_FAILS`, default 10) instead of looping forever.
|
|
196
|
+
|
|
185
197
|
## Rate limits and retries
|
|
186
198
|
|
|
187
199
|
The free tier allows roughly 15 requests a minute, which a wide `gemcatch sync` or a busy daemon would otherwise blow straight through. Every outbound call is paced to `GEMCATCH_RPM` (default 15) — set it higher on a paid key, or `0` to disable pacing entirely.
|
|
188
200
|
|
|
201
|
+
Pacing is **per process**: each `gemcatch` invocation keeps its own counter, so two running at once (a `daemon` in one terminal and a one-off `sync` in another) can together exceed the ceiling. If you need the limit to hold, run a single daemon and let it do the polling.
|
|
202
|
+
|
|
189
203
|
Transient failures are retried with exponential backoff and full jitter, honouring `Retry-After` when the server sends it. A rate limit, a timeout or a 5xx gets `GEMCATCH_MAX_RETRIES` more attempts (default 4); a 4xx does not, because a bad key or a bad model id fails identically forever and retrying it only burns your quota.
|
|
190
204
|
|
|
191
205
|
## Environment variables
|
|
@@ -199,6 +213,7 @@ Transient failures are retried with exponential backoff and full jitter, honouri
|
|
|
199
213
|
| `GEMCATCH_DAEMON_S` | `daemon` interval in seconds. Default `300`. |
|
|
200
214
|
| `GEMCATCH_RPM` | Requests/minute ceiling. Default `15` (the free tier). `0` disables pacing. |
|
|
201
215
|
| `GEMCATCH_MAX_RETRIES` | Extra attempts on a transient failure. Default `4`. `0` disables retries. |
|
|
216
|
+
| `GEMCATCH_WATCH_MAX_FAILS` | Consecutive poll failures before `watch`/`batch -w` give up. Default `10`. |
|
|
202
217
|
| `GEMCATCH_BASE_URL` | Override the API endpoint (proxy/gateway/testing). |
|
|
203
218
|
| `GEMCATCH_FORCE_REST` | `1` bypasses the SDK and uses raw `fetch`. |
|
|
204
219
|
| `NO_COLOR` | Disable colour output. |
|
package/db.js
CHANGED
|
@@ -125,7 +125,10 @@ function listTasks(opts) {
|
|
|
125
125
|
let sql = 'SELECT * FROM tasks';
|
|
126
126
|
if (where.length) sql += ` WHERE ${where.join(' AND ')}`;
|
|
127
127
|
sql += ' ORDER BY created_at DESC';
|
|
128
|
-
|
|
128
|
+
// Presence, not truthiness: `--limit 0` is a real cap (return nothing), so it
|
|
129
|
+
// must not be treated the same as "no limit given". The caller validates that
|
|
130
|
+
// it is a non-negative integer before we get here.
|
|
131
|
+
if (o.limit != null) {
|
|
129
132
|
sql += ' LIMIT @limit';
|
|
130
133
|
params.limit = o.limit;
|
|
131
134
|
}
|
package/index.js
CHANGED
|
@@ -13,20 +13,41 @@ const DEFAULT_POLL_MS = Number(process.env.GEMCATCH_POLL_MS) || 10000;
|
|
|
13
13
|
// comfortably faster than that. Five minutes is far inside the margin and
|
|
14
14
|
// costs a handful of requests an hour.
|
|
15
15
|
const DEFAULT_DAEMON_S = Number(process.env.GEMCATCH_DAEMON_S) || 300;
|
|
16
|
+
// A watch loop must not spin forever on a task the server can no longer resolve
|
|
17
|
+
// -- a wedged in_progress, or transient poll errors that never clear. `watch`
|
|
18
|
+
// and `batch -w` give up after this many *consecutive* poll failures (a clean
|
|
19
|
+
// poll resets the run), surfacing a clear message and a non-zero exit instead
|
|
20
|
+
// of hanging. The daemon, meant to run for days, is bounded differently: a 404
|
|
21
|
+
// retires the task locally (see refresh) so it simply leaves the active set.
|
|
22
|
+
const WATCH_MAX_FAILS = Number(process.env.GEMCATCH_WATCH_MAX_FAILS) || 10;
|
|
16
23
|
const ALL_STATUSES = [PENDING].concat(ACTIVE, TERMINAL);
|
|
17
24
|
|
|
18
25
|
// --- output ---------------------------------------------------------------
|
|
19
26
|
|
|
20
|
-
|
|
21
|
-
|
|
27
|
+
// Colour is decided per stream. Progress and status chatter go to stderr
|
|
28
|
+
// (watch/daemon/research -w); results and tables go to stdout. Each stream keys
|
|
29
|
+
// its ANSI on its *own* TTY-ness, so redirecting one (`gemcatch watch x > out.txt`)
|
|
30
|
+
// neither strips colour from the other nor leaks raw escape codes into the
|
|
31
|
+
// redirected file. NO_COLOR disables both.
|
|
32
|
+
const NO_COLOR = !!process.env.NO_COLOR;
|
|
33
|
+
const useColor = process.stdout.isTTY && !NO_COLOR; // stdout-bound colour
|
|
34
|
+
const useColorErr = process.stderr.isTTY && !NO_COLOR; // stderr-bound colour
|
|
35
|
+
|
|
36
|
+
const wrap = (on) => (code, s) => (on ? `[${code}m${s}[0m` : s);
|
|
37
|
+
const paint = wrap(useColor); // paints for stdout
|
|
38
|
+
const epaint = wrap(useColorErr); // paints for stderr
|
|
22
39
|
const dim = (s) => paint('2', s);
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
if (s
|
|
28
|
-
return
|
|
40
|
+
const edim = (s) => epaint('2', s);
|
|
41
|
+
|
|
42
|
+
// Status colour keyed to a given painter, so one rule set serves both streams.
|
|
43
|
+
function tint(pnt, s) {
|
|
44
|
+
if (isSuccess(s)) return pnt('32', s); // green
|
|
45
|
+
if (s === 'in_progress' || s === PENDING) return pnt('36', s); // cyan
|
|
46
|
+
if (s === 'requires_action') return pnt('33', s); // yellow
|
|
47
|
+
return pnt('31', s); // red: failed/cancelled/incomplete/budget_exceeded
|
|
29
48
|
}
|
|
49
|
+
const colorStatus = (s) => tint(paint, s); // for stdout
|
|
50
|
+
const ecolorStatus = (s) => tint(epaint, s); // for stderr
|
|
30
51
|
|
|
31
52
|
const hhmmss = () => new Date().toISOString().slice(11, 19);
|
|
32
53
|
|
|
@@ -100,7 +121,22 @@ async function resolvePrompt(arg, opts) {
|
|
|
100
121
|
// Poll one task and persist whatever came back.
|
|
101
122
|
async function refresh(task) {
|
|
102
123
|
if (!task.interaction_id) return { status: task.status, text: null, usage: null };
|
|
103
|
-
|
|
124
|
+
let r;
|
|
125
|
+
try {
|
|
126
|
+
r = await gemini.poll(task.interaction_id);
|
|
127
|
+
} catch (err) {
|
|
128
|
+
// A 404 is genuine and permanent: the interaction is gone -- dropped after
|
|
129
|
+
// the free tier's 24h retention, or deleted -- and it will 404 identically
|
|
130
|
+
// forever (a 4xx never retries). Retire the task locally so it leaves the
|
|
131
|
+
// active set, instead of the daemon or a watch loop polling a ghost until
|
|
132
|
+
// the end of time. Any other error (5xx, network) is transient and is
|
|
133
|
+
// re-thrown for the caller to retry on its next pass.
|
|
134
|
+
if (err && err.httpStatus === 404) {
|
|
135
|
+
store.setStatus(task.id, 'incomplete', { error: 'interaction not found (expired or deleted)' });
|
|
136
|
+
return { status: 'incomplete', text: null, usage: null, raw: null };
|
|
137
|
+
}
|
|
138
|
+
throw err;
|
|
139
|
+
}
|
|
104
140
|
const extra = {};
|
|
105
141
|
if (isDone(r.status)) {
|
|
106
142
|
if (isSuccess(r.status)) extra.result = r.text;
|
|
@@ -162,7 +198,7 @@ program
|
|
|
162
198
|
// Under --watch the submit line is progress, not the answer, so it
|
|
163
199
|
// goes to stderr -- `gemcatch research -w "..." > out.txt` then captures
|
|
164
200
|
// only the result.
|
|
165
|
-
if (!opts.json) console.error(
|
|
201
|
+
if (!opts.json) console.error(edim(`Task ${id} submitted.`));
|
|
166
202
|
await watchTask(store.getTask(id), DEFAULT_POLL_MS, opts.json);
|
|
167
203
|
return;
|
|
168
204
|
}
|
|
@@ -170,16 +206,29 @@ program
|
|
|
170
206
|
console.log(`Task ${id} submitted. Run: gemcatch get ${id} when ready.`)
|
|
171
207
|
);
|
|
172
208
|
} catch (err) {
|
|
173
|
-
|
|
209
|
+
// Only a failed *submit* should mark the task failed. Once it has an
|
|
210
|
+
// interaction_id it is live on the server, and a later watch/poll error
|
|
211
|
+
// must never overwrite it to failed -- that would drop it from the active
|
|
212
|
+
// set and the daemon would abandon a task whose result is still coming.
|
|
213
|
+
// Leave it active; the daemon (or a later `get`) collects it.
|
|
214
|
+
if (id) {
|
|
215
|
+
const t = store.getTask(id);
|
|
216
|
+
if (!t || !t.interaction_id) store.setStatus(id, 'failed', { error: err.message });
|
|
217
|
+
}
|
|
174
218
|
die(err);
|
|
175
219
|
}
|
|
176
220
|
});
|
|
177
221
|
|
|
178
222
|
// --- batch ----------------------------------------------------------------
|
|
179
223
|
|
|
180
|
-
// Turn a prompts file into a list of prompts
|
|
181
|
-
//
|
|
224
|
+
// Turn a prompts file into a list of prompts, plus a count of the lines it
|
|
225
|
+
// dropped so the caller can note them. Default: one per line, skipping blank
|
|
226
|
+
// lines and `#` comments. With --separator, split the whole file on that
|
|
182
227
|
// delimiter line instead, so a single prompt can span multiple lines.
|
|
228
|
+
//
|
|
229
|
+
// A `#` is a comment only when followed by whitespace (`# like this`). A line
|
|
230
|
+
// such as `#1 cause of X?` is a real prompt, not a comment, and must survive --
|
|
231
|
+
// treating every leading `#` as a comment silently swallowed those.
|
|
183
232
|
function parsePrompts(text, separator) {
|
|
184
233
|
if (separator) {
|
|
185
234
|
const blocks = [];
|
|
@@ -193,12 +242,20 @@ function parsePrompts(text, separator) {
|
|
|
193
242
|
}
|
|
194
243
|
}
|
|
195
244
|
blocks.push(cur.join('\n').trim());
|
|
196
|
-
|
|
245
|
+
const prompts = blocks.filter(Boolean);
|
|
246
|
+
return { prompts, skipped: blocks.length - prompts.length };
|
|
247
|
+
}
|
|
248
|
+
// The file almost always ends in a newline; that trailing empty line is not a
|
|
249
|
+
// blank the user wrote, so it does not count towards the skipped tally.
|
|
250
|
+
const lines = text.split(/\r?\n/).map((l) => l.trim());
|
|
251
|
+
while (lines.length && lines[lines.length - 1] === '') lines.pop();
|
|
252
|
+
const prompts = [];
|
|
253
|
+
let skipped = 0;
|
|
254
|
+
for (const l of lines) {
|
|
255
|
+
if (!l || /^#\s/.test(l)) skipped += 1;
|
|
256
|
+
else prompts.push(l);
|
|
197
257
|
}
|
|
198
|
-
return
|
|
199
|
-
.split(/\r?\n/)
|
|
200
|
-
.map((l) => l.trim())
|
|
201
|
-
.filter((l) => l && !l.startsWith('#'));
|
|
258
|
+
return { prompts, skipped };
|
|
202
259
|
}
|
|
203
260
|
|
|
204
261
|
// Poll just this batch until nothing tagged with it is still in flight, then
|
|
@@ -207,11 +264,25 @@ function parsePrompts(text, separator) {
|
|
|
207
264
|
async function watchBatch(tag, intervalMs, json) {
|
|
208
265
|
const inFlight = () => store.listTasks({ tag }).filter((t) => t.interaction_id && !isDone(t.status));
|
|
209
266
|
let pending = inFlight();
|
|
267
|
+
let stalls = 0; // consecutive passes that resolved nothing
|
|
210
268
|
while (pending.length) {
|
|
211
269
|
// A poll that throws keeps the task's old status; the next pass retries it.
|
|
270
|
+
// A 404 retires the task inside refresh, so it drops out of `inFlight`.
|
|
212
271
|
await mapLimit(pending, 4, (t) => refresh(t).catch(() => {}));
|
|
213
|
-
|
|
214
|
-
|
|
272
|
+
const next = inFlight();
|
|
273
|
+
// Forward progress = the in-flight set shrank. A pass that resolves nothing
|
|
274
|
+
// -- every poll erroring, or a wedged in_progress that never moves -- is a
|
|
275
|
+
// stall; enough of those in a row means give up rather than loop forever.
|
|
276
|
+
stalls = next.length < pending.length ? 0 : stalls + 1;
|
|
277
|
+
pending = next;
|
|
278
|
+
if (!pending.length) break;
|
|
279
|
+
if (stalls >= WATCH_MAX_FAILS) {
|
|
280
|
+
const msg = `Batch ${tag}: gave up after ${stalls} passes with no progress; ${pending.length} task(s) unresolved.`;
|
|
281
|
+
emit(json, { tag, error: msg, unresolved: pending.length }, () => console.error(edim(msg)));
|
|
282
|
+
process.exitCode = 1;
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
215
286
|
}
|
|
216
287
|
const tasks = store.listTasks({ tag });
|
|
217
288
|
const completed = tasks.filter((t) => isSuccess(t.status)).length;
|
|
@@ -235,8 +306,13 @@ program
|
|
|
235
306
|
.action(async (file, opts) => {
|
|
236
307
|
try {
|
|
237
308
|
const text = file === '-' ? await readStdin() : fs.readFileSync(file, 'utf8');
|
|
238
|
-
const prompts = parsePrompts(text, opts.separator);
|
|
309
|
+
const { prompts, skipped } = parsePrompts(text, opts.separator);
|
|
239
310
|
if (!prompts.length) throw new Error(`no prompts found in ${file === '-' ? 'stdin' : file}`);
|
|
311
|
+
// A one-line heads-up so a swallowed prompt (or a stray comment) is never a
|
|
312
|
+
// silent mystery. Goes to stderr so it can't corrupt --json on stdout.
|
|
313
|
+
if (skipped) {
|
|
314
|
+
console.error(edim(`(skipped ${skipped} blank/comment line${skipped === 1 ? '' : 's'})`));
|
|
315
|
+
}
|
|
240
316
|
// Auto-tag so the batch is collectable as a unit; a user tag wins.
|
|
241
317
|
const tag = opts.tag || `batch-${crypto.randomUUID().slice(0, 6)}`;
|
|
242
318
|
|
|
@@ -321,10 +397,13 @@ program
|
|
|
321
397
|
const task = needTask(id);
|
|
322
398
|
try {
|
|
323
399
|
// Completed tasks are served from SQLite -- no network, and it still
|
|
324
|
-
// works after the free tier drops the interaction at 24h.
|
|
325
|
-
|
|
400
|
+
// works after the free tier drops the interaction at 24h. Gate on the
|
|
401
|
+
// result being *present*, not truthy: a task that completes with empty
|
|
402
|
+
// text stores `''`, which is exactly the case the cache must still serve
|
|
403
|
+
// -- re-polling it would 404 after 24h, the very thing we cache to avoid.
|
|
404
|
+
if (isSuccess(task.status) && task.result != null && !opts.raw) {
|
|
326
405
|
emit(opts.json, { id: task.id, status: task.status, result: task.result }, () =>
|
|
327
|
-
console.log(task.result)
|
|
406
|
+
console.log(task.result || '(empty response)')
|
|
328
407
|
);
|
|
329
408
|
return;
|
|
330
409
|
}
|
|
@@ -360,6 +439,11 @@ program
|
|
|
360
439
|
.option('--json', 'machine-readable output')
|
|
361
440
|
.description('all tasks, newest first')
|
|
362
441
|
.action((opts) => {
|
|
442
|
+
// `-n 0` is a valid cap (show nothing); a negative would become SQLite's
|
|
443
|
+
// "no limit" (LIMIT -1 = all rows), so reject anything but a non-negative int.
|
|
444
|
+
if (opts.limit != null && (!Number.isInteger(opts.limit) || opts.limit < 0)) {
|
|
445
|
+
return die(new Error(`--limit must be a non-negative integer (got ${opts.limit})`));
|
|
446
|
+
}
|
|
363
447
|
const tasks = store.listTasks({ status: opts.status, tag: opts.tag, limit: opts.limit });
|
|
364
448
|
if (opts.json) return console.log(JSON.stringify(tasks, null, 2));
|
|
365
449
|
if (!tasks.length) {
|
|
@@ -378,6 +462,117 @@ program
|
|
|
378
462
|
}
|
|
379
463
|
});
|
|
380
464
|
|
|
465
|
+
// --- export ---------------------------------------------------------------
|
|
466
|
+
|
|
467
|
+
// Collect many finished results into one document -- the "gather" that pairs
|
|
468
|
+
// with `batch`'s "scatter". Where `get` prints one result at a time, `export`
|
|
469
|
+
// concatenates a whole tag (or status) under prompt headings, to stdout or a
|
|
470
|
+
// file, as Markdown (default) or JSON.
|
|
471
|
+
program
|
|
472
|
+
.command('export')
|
|
473
|
+
.option('-t, --tag <tag>', 'only this tag')
|
|
474
|
+
.addOption(new Option('--status <status>', 'only this status').choices(ALL_STATUSES).default('completed'))
|
|
475
|
+
.addOption(new Option('--format <fmt>', 'output format').choices(['md', 'json']).default('md'))
|
|
476
|
+
.option('-o, --out <file>', 'write to a file instead of stdout')
|
|
477
|
+
.description('concatenate finished results, each under its prompt, to stdout or a file')
|
|
478
|
+
.action((opts) => {
|
|
479
|
+
const tasks = store.listTasks({ tag: opts.tag, status: opts.status });
|
|
480
|
+
// Newest-first suits a listing, but an export reads top-to-bottom like a
|
|
481
|
+
// document, so oldest-first is the natural order here.
|
|
482
|
+
tasks.reverse();
|
|
483
|
+
// Only rows that actually carry a result are worth exporting: a status
|
|
484
|
+
// filter other than `completed` can match tasks that never stored text.
|
|
485
|
+
const rows = tasks.filter((t) => t.result != null);
|
|
486
|
+
if (!rows.length) {
|
|
487
|
+
// Nothing to write isn't an error, but say why so an empty -o file (or an
|
|
488
|
+
// empty pipe) isn't a mystery. The note goes to stderr, never the output.
|
|
489
|
+
console.error(`No ${opts.status} results to export${opts.tag ? ` for tag '${opts.tag}'` : ''}.`);
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
let output;
|
|
494
|
+
if (opts.format === 'json') {
|
|
495
|
+
output = JSON.stringify(
|
|
496
|
+
rows.map((t) => ({
|
|
497
|
+
id: t.id,
|
|
498
|
+
tag: t.tag,
|
|
499
|
+
status: t.status,
|
|
500
|
+
prompt: t.prompt,
|
|
501
|
+
result: t.result,
|
|
502
|
+
created_at: t.created_at,
|
|
503
|
+
})),
|
|
504
|
+
null,
|
|
505
|
+
2
|
|
506
|
+
);
|
|
507
|
+
} else {
|
|
508
|
+
output = rows
|
|
509
|
+
.map((t) => {
|
|
510
|
+
const when = new Date(t.created_at).toISOString().replace('T', ' ').slice(0, 16);
|
|
511
|
+
const head = (t.prompt || '(no prompt)').replace(/\s+/g, ' ').trim();
|
|
512
|
+
const body = t.result && t.result.trim() ? t.result : '_(empty result)_';
|
|
513
|
+
return `## ${head}\n\n\`${t.id}\` · ${t.status} · ${when} UTC\n\n${body}`;
|
|
514
|
+
})
|
|
515
|
+
.join('\n\n---\n\n');
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
if (opts.out) {
|
|
519
|
+
fs.writeFileSync(opts.out, output.endsWith('\n') ? output : `${output}\n`);
|
|
520
|
+
console.error(`Wrote ${rows.length} result(s) to ${opts.out}.`);
|
|
521
|
+
} else {
|
|
522
|
+
console.log(output);
|
|
523
|
+
}
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
// --- digest ---------------------------------------------------------------
|
|
527
|
+
|
|
528
|
+
// One step past `export`: instead of concatenating a tag's results, feed them
|
|
529
|
+
// back through a single Gemini call and synthesise one summary. It is `research`
|
|
530
|
+
// with a prompt built from what you have already collected, so it submits, then
|
|
531
|
+
// watches to completion just like `research -w`.
|
|
532
|
+
program
|
|
533
|
+
.command('digest')
|
|
534
|
+
.requiredOption('-t, --tag <tag>', 'synthesize the completed results under this tag')
|
|
535
|
+
.option('-m, --model <id>', 'model to use', gemini.DEFAULT_MODEL)
|
|
536
|
+
.option('-s, --system <text>', 'system instruction for the synthesis')
|
|
537
|
+
.option('--json', 'machine-readable output')
|
|
538
|
+
.description("feed a tag's completed results through one Gemini call into a single summary")
|
|
539
|
+
.action(async (opts) => {
|
|
540
|
+
let id;
|
|
541
|
+
try {
|
|
542
|
+
const done = store
|
|
543
|
+
.listTasks({ tag: opts.tag, status: 'completed' })
|
|
544
|
+
.filter((t) => t.result != null && t.result.trim());
|
|
545
|
+
if (!done.length) {
|
|
546
|
+
throw new Error(
|
|
547
|
+
`no completed results tagged '${opts.tag}' to digest.` +
|
|
548
|
+
' Collect them first: gemcatch daemon --exit-when-idle'
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
done.reverse(); // oldest first, so the sources read in submission order
|
|
552
|
+
const sources = done
|
|
553
|
+
.map((t, i) => `## Source ${i + 1}: ${(t.prompt || '').replace(/\s+/g, ' ').trim()}\n\n${t.result}`)
|
|
554
|
+
.join('\n\n');
|
|
555
|
+
const prompt =
|
|
556
|
+
`Synthesize the following ${done.length} research result(s) into one coherent summary.` +
|
|
557
|
+
' Note where they agree and disagree, and do not simply repeat each verbatim.\n\n' +
|
|
558
|
+
sources;
|
|
559
|
+
// The digest is itself a task, tagged so it is findable but kept out of
|
|
560
|
+
// the source tag so a later digest never digests its own output.
|
|
561
|
+
id = store.createTask({ prompt, model: opts.model, systemInstruction: opts.system, tag: `${opts.tag}-digest` });
|
|
562
|
+
const r = await gemini.submit(prompt, { model: opts.model, systemInstruction: opts.system });
|
|
563
|
+
store.setInteraction(id, r.interactionId, r.status);
|
|
564
|
+
if (!opts.json) console.error(edim(`Digesting ${done.length} result(s) tagged ${opts.tag} -> task ${id}.`));
|
|
565
|
+
await watchTask(store.getTask(id), DEFAULT_POLL_MS, opts.json);
|
|
566
|
+
} catch (err) {
|
|
567
|
+
// Same rule as `research`: only a failed *submit* marks the task failed.
|
|
568
|
+
if (id) {
|
|
569
|
+
const t = store.getTask(id);
|
|
570
|
+
if (!t || !t.interaction_id) store.setStatus(id, 'failed', { error: err.message });
|
|
571
|
+
}
|
|
572
|
+
die(err);
|
|
573
|
+
}
|
|
574
|
+
});
|
|
575
|
+
|
|
381
576
|
// --- sync -----------------------------------------------------------------
|
|
382
577
|
|
|
383
578
|
// One refresh pass over everything in flight. Never throws: a task that fails
|
|
@@ -422,12 +617,18 @@ program
|
|
|
422
617
|
.option('--json', 'newline-delimited JSON events on stdout')
|
|
423
618
|
.description('poll in-flight tasks on a loop so results are cached before they expire')
|
|
424
619
|
.action(async (opts) => {
|
|
425
|
-
|
|
620
|
+
if (!Number.isFinite(opts.interval) || opts.interval <= 0) {
|
|
621
|
+
return die(new Error(`--interval must be a positive number of seconds (got ${opts.interval})`));
|
|
622
|
+
}
|
|
623
|
+
const intervalMs = Math.max(1000, opts.interval * 1000);
|
|
426
624
|
let stopping = false;
|
|
427
625
|
let wake = null;
|
|
428
626
|
// Finish the pass in progress, then exit cleanly -- never leave a polled
|
|
429
|
-
// result unwritten because someone hit Ctrl-C.
|
|
627
|
+
// result unwritten because someone hit Ctrl-C. A *second* signal, though,
|
|
628
|
+
// means "I don't want to wait for this pass" -- force-exit immediately with
|
|
629
|
+
// the conventional 130 (128 + SIGINT) so a long paced pass can't trap you.
|
|
430
630
|
const stop = () => {
|
|
631
|
+
if (stopping) process.exit(130);
|
|
431
632
|
stopping = true;
|
|
432
633
|
if (wake) wake();
|
|
433
634
|
};
|
|
@@ -440,7 +641,7 @@ program
|
|
|
440
641
|
|
|
441
642
|
if (!opts.json) {
|
|
442
643
|
console.error(
|
|
443
|
-
|
|
644
|
+
edim(`gemcatch daemon: polling every ${intervalMs / 1000}s. Store: ${store.DB_PATH}. Ctrl-C to stop.`)
|
|
444
645
|
);
|
|
445
646
|
}
|
|
446
647
|
event({ event: 'start', interval_s: intervalMs / 1000, db: store.DB_PATH });
|
|
@@ -455,7 +656,7 @@ program
|
|
|
455
656
|
// the next pass may well succeed, and a daemon that dies silently is
|
|
456
657
|
// worse than one that complains.
|
|
457
658
|
if (opts.json) event({ event: 'error', error: err.message });
|
|
458
|
-
else console.error(`${
|
|
659
|
+
else console.error(`${edim(`[${hhmmss()}]`)} Error: ${err.message}`);
|
|
459
660
|
}
|
|
460
661
|
|
|
461
662
|
// Quiet by default: only transitions and failures are worth a line.
|
|
@@ -465,7 +666,7 @@ program
|
|
|
465
666
|
event({ event: r.error ? 'error' : 'update', id: r.id, status: r.status, error: r.error || null });
|
|
466
667
|
} else {
|
|
467
668
|
console.error(
|
|
468
|
-
|
|
669
|
+
edim(`[${hhmmss()}] ${r.id}: `) + ecolorStatus(r.status) + (r.error ? ` ${edim(r.error)}` : '')
|
|
469
670
|
);
|
|
470
671
|
}
|
|
471
672
|
}
|
|
@@ -488,7 +689,7 @@ program
|
|
|
488
689
|
}
|
|
489
690
|
|
|
490
691
|
event({ event: 'stop' });
|
|
491
|
-
if (!opts.json) console.error(
|
|
692
|
+
if (!opts.json) console.error(edim('gemcatch daemon: stopped.'));
|
|
492
693
|
store.close();
|
|
493
694
|
});
|
|
494
695
|
|
|
@@ -496,12 +697,32 @@ program
|
|
|
496
697
|
|
|
497
698
|
async function watchTask(task, intervalMs, json) {
|
|
498
699
|
let last = null;
|
|
700
|
+
let fails = 0;
|
|
499
701
|
for (;;) {
|
|
500
|
-
|
|
702
|
+
let r;
|
|
703
|
+
try {
|
|
704
|
+
r = await refresh(task);
|
|
705
|
+
fails = 0; // a clean poll resets the failure run
|
|
706
|
+
} catch (err) {
|
|
707
|
+
// A poll error must not sink a live task: keep its old status and try
|
|
708
|
+
// again next interval, exactly like watchBatch. Give up only once the
|
|
709
|
+
// failures pile up, so a task the server can't answer for can't hang the
|
|
710
|
+
// watch forever. (A 404 doesn't reach here -- refresh retires it and
|
|
711
|
+
// returns a terminal status, handled below.)
|
|
712
|
+
fails += 1;
|
|
713
|
+
if (fails >= WATCH_MAX_FAILS) {
|
|
714
|
+
const msg = `Gave up watching ${task.id} after ${fails} consecutive poll failures: ${err.message}`;
|
|
715
|
+
emit(json, { id: task.id, status: task.status, error: msg }, () => console.error(edim(msg)));
|
|
716
|
+
process.exitCode = 1;
|
|
717
|
+
return;
|
|
718
|
+
}
|
|
719
|
+
await new Promise((r2) => setTimeout(r2, intervalMs));
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
501
722
|
// Status chatter goes to stderr so `gemcatch watch x > out.txt` captures only
|
|
502
723
|
// the result.
|
|
503
724
|
if (r.status !== last && !json) {
|
|
504
|
-
console.error(
|
|
725
|
+
console.error(edim(`[${new Date().toISOString().slice(11, 19)}] ${task.id}: `) + ecolorStatus(r.status));
|
|
505
726
|
last = r.status;
|
|
506
727
|
}
|
|
507
728
|
if (isSuccess(r.status)) {
|
|
@@ -512,7 +733,7 @@ async function watchTask(task, intervalMs, json) {
|
|
|
512
733
|
}
|
|
513
734
|
if (isDone(r.status)) {
|
|
514
735
|
emit(json, { id: task.id, status: r.status, error: r.text || null }, () => {
|
|
515
|
-
console.error(`Task ${task.id} ended: ${
|
|
736
|
+
console.error(`Task ${task.id} ended: ${ecolorStatus(r.status)}`);
|
|
516
737
|
if (r.text) console.log(r.text);
|
|
517
738
|
});
|
|
518
739
|
process.exitCode = 1;
|
|
@@ -531,12 +752,17 @@ program
|
|
|
531
752
|
.action(async (id, opts) => {
|
|
532
753
|
const task = needTask(id);
|
|
533
754
|
try {
|
|
534
|
-
|
|
755
|
+
// Serve a completed result from cache -- present, not merely truthy, so an
|
|
756
|
+
// empty-text completion is served instead of re-polled (and lost at 24h).
|
|
757
|
+
if (isSuccess(task.status) && task.result != null) {
|
|
535
758
|
emit(opts.json, { id: task.id, status: task.status, result: task.result }, () =>
|
|
536
|
-
console.log(task.result)
|
|
759
|
+
console.log(task.result || '(empty response)')
|
|
537
760
|
);
|
|
538
761
|
return;
|
|
539
762
|
}
|
|
763
|
+
if (opts.interval != null && (!Number.isFinite(opts.interval) || opts.interval <= 0)) {
|
|
764
|
+
return die(new Error(`--interval must be a positive number of seconds (got ${opts.interval})`));
|
|
765
|
+
}
|
|
540
766
|
await watchTask(task, opts.interval ? opts.interval * 1000 : DEFAULT_POLL_MS, opts.json);
|
|
541
767
|
} catch (err) {
|
|
542
768
|
die(err);
|
|
@@ -579,7 +805,7 @@ program
|
|
|
579
805
|
} catch (err) {
|
|
580
806
|
// Free-tier interactions vanish after 24h, so a missing remote is
|
|
581
807
|
// normal -- never block the local delete on it.
|
|
582
|
-
console.error(
|
|
808
|
+
console.error(edim(` (remote delete failed for ${task.id}: ${err.message})`));
|
|
583
809
|
}
|
|
584
810
|
}
|
|
585
811
|
if (store.removeTask(task.id)) removed += 1;
|
|
@@ -595,6 +821,12 @@ program
|
|
|
595
821
|
.option('--dry-run', 'list what would go, delete nothing')
|
|
596
822
|
.description('drop old finished tasks (in-flight work is never touched)')
|
|
597
823
|
.action((opts) => {
|
|
824
|
+
// A negative (or non-numeric) --days puts the cutoff in the *future*, which
|
|
825
|
+
// would match every finished task and quietly wipe the lot. Refuse it: the
|
|
826
|
+
// cutoff must be at or before now.
|
|
827
|
+
if (!Number.isFinite(opts.days) || opts.days < 0) {
|
|
828
|
+
return die(new Error(`--days must be a non-negative number (got ${opts.days})`));
|
|
829
|
+
}
|
|
598
830
|
const cutoff = Date.now() - opts.days * 86400000;
|
|
599
831
|
const doomed = store.prunableTasks(cutoff);
|
|
600
832
|
if (!doomed.length) {
|
|
@@ -626,4 +858,16 @@ program
|
|
|
626
858
|
});
|
|
627
859
|
});
|
|
628
860
|
|
|
861
|
+
// Close the store on the way out so a one-shot command doesn't leave the
|
|
862
|
+
// SQLite -wal/-shm sidecars lingering. The store opens lazily, so if a command
|
|
863
|
+
// never touched it this is a no-op; the daemon closes explicitly too, and a
|
|
864
|
+
// second close is harmless.
|
|
865
|
+
process.on('exit', () => {
|
|
866
|
+
try {
|
|
867
|
+
store.close();
|
|
868
|
+
} catch (_) {
|
|
869
|
+
/* best effort on the way out */
|
|
870
|
+
}
|
|
871
|
+
});
|
|
872
|
+
|
|
629
873
|
program.parseAsync(process.argv).catch(die);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gemcatch",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Fire-and-forget CLI for Gemini's Interactions API background execution. Submit long-running research prompts, close your laptop, collect results later.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"bin": {
|