gemcatch 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -1
- package/README.md +179 -9
- package/db.js +71 -4
- package/gemini.js +179 -48
- package/index.js +706 -114
- package/package.json +3 -2
- package/sources.js +580 -0
package/index.js
CHANGED
|
@@ -2,10 +2,13 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
const fs = require('fs');
|
|
5
|
+
const path = require('path');
|
|
6
|
+
const { pathToFileURL } = require('url');
|
|
5
7
|
const crypto = require('crypto');
|
|
6
8
|
const { Command, Option } = require('commander');
|
|
7
9
|
const store = require('./db');
|
|
8
10
|
const gemini = require('./gemini');
|
|
11
|
+
const sources = require('./sources');
|
|
9
12
|
const { TERMINAL, ACTIVE, PENDING, isDone, isSuccess } = require('./status');
|
|
10
13
|
|
|
11
14
|
const DEFAULT_POLL_MS = Number(process.env.GEMCATCH_POLL_MS) || 10000;
|
|
@@ -90,22 +93,26 @@ function needTask(id) {
|
|
|
90
93
|
}
|
|
91
94
|
|
|
92
95
|
// Citations ride along with an agent's report -- the docs tell users to review
|
|
93
|
-
// them to verify the sources
|
|
94
|
-
//
|
|
95
|
-
function withSources(text, citations) {
|
|
96
|
-
const
|
|
97
|
-
if (
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
96
|
+
// them to verify the sources -- so they are printed under the result, and so are
|
|
97
|
+
// the paths of any charts it drew.
|
|
98
|
+
function withSources(text, citations, images) {
|
|
99
|
+
const parts = [text || '(empty response)'];
|
|
100
|
+
if (Array.isArray(citations) && citations.length) {
|
|
101
|
+
const lines = citations.map((c, i) => {
|
|
102
|
+
const title = (c && (c.title || c.text)) || '';
|
|
103
|
+
const url = (c && (c.url || c.uri)) || '';
|
|
104
|
+
return ` [${i + 1}] ${[title, url].filter(Boolean).join(' — ') || JSON.stringify(c)}`;
|
|
105
|
+
});
|
|
106
|
+
parts.push(`Sources:\n${lines.join('\n')}`);
|
|
107
|
+
}
|
|
108
|
+
if (images && images.length) parts.push(`Images:\n${images.map((p) => ` ${p}`).join('\n')}`);
|
|
109
|
+
return parts.join('\n\n');
|
|
104
110
|
}
|
|
105
111
|
|
|
106
|
-
// The citations
|
|
107
|
-
// degrades to "
|
|
108
|
-
|
|
112
|
+
// The citations and images_json columns hold JSON arrays (or NULL). Parsed
|
|
113
|
+
// defensively: a corrupt row degrades to "none", never a crash in the middle of
|
|
114
|
+
// printing a result.
|
|
115
|
+
function parseList(raw) {
|
|
109
116
|
if (!raw) return null;
|
|
110
117
|
try {
|
|
111
118
|
const v = JSON.parse(raw);
|
|
@@ -115,6 +122,40 @@ function parseCitations(raw) {
|
|
|
115
122
|
}
|
|
116
123
|
}
|
|
117
124
|
|
|
125
|
+
// --- images ---------------------------------------------------------------
|
|
126
|
+
|
|
127
|
+
const IMAGE_DIR = path.join(store.HOME, 'images');
|
|
128
|
+
function imageExt(mime) {
|
|
129
|
+
if (mime === 'image/jpeg') return '.jpg';
|
|
130
|
+
const sub = /^image\/([a-z0-9]+)/i.exec(mime || '');
|
|
131
|
+
return sub ? `.${sub[1].toLowerCase()}` : '.png';
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// A --visualize run's charts arrive as base64 in the result, which the store
|
|
135
|
+
// keeps as text only, so they are written out as <task-id>-<n>.<ext>. They can
|
|
136
|
+
// chart private data, so they get the store's permissions.
|
|
137
|
+
function saveImages(task, images) {
|
|
138
|
+
fs.mkdirSync(IMAGE_DIR, { recursive: true, mode: 0o700 });
|
|
139
|
+
return images.map((img, i) => {
|
|
140
|
+
const file = path.join(IMAGE_DIR, `${task.id}-${i + 1}${imageExt(img.mime_type)}`);
|
|
141
|
+
fs.writeFileSync(file, Buffer.from(img.data, 'base64'), { mode: 0o600 });
|
|
142
|
+
return file;
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Only files named for this task go: a digest lists its sources' images too,
|
|
147
|
+
// and forgetting the digest must not take them with it.
|
|
148
|
+
function removeImages(task) {
|
|
149
|
+
for (const p of parseList(task.images_json) || []) {
|
|
150
|
+
if (!path.basename(p).startsWith(`${task.id}-`)) continue;
|
|
151
|
+
try {
|
|
152
|
+
fs.rmSync(p, { force: true });
|
|
153
|
+
} catch (err) {
|
|
154
|
+
console.error(`Note: could not delete ${p} (${err.message}).`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
118
159
|
// --- spend guard ----------------------------------------------------------
|
|
119
160
|
|
|
120
161
|
// Deep Research agents are billed PER TASK, not per token -- the docs put
|
|
@@ -126,6 +167,12 @@ function parseCitations(raw) {
|
|
|
126
167
|
// The bands are quoted with the docs' own hedge ("estimates based on preview
|
|
127
168
|
// rates and subject to change"), never as authoritative.
|
|
128
169
|
|
|
170
|
+
// A planning turn is a task and is billed as one. The docs publish ONE band per
|
|
171
|
+
// task and price no planning turn separately, so it is quoted at the same band
|
|
172
|
+
// and the line says so outright. Planning buys you a look at the plan before you
|
|
173
|
+
// commit to the research run; it does not buy you a discount.
|
|
174
|
+
const PLAN_NOTE = 'the docs price per task and do not price a planning turn separately';
|
|
175
|
+
|
|
129
176
|
function bandText(agentId, count) {
|
|
130
177
|
const band = gemini.AGENT_PRICE_BANDS[agentId];
|
|
131
178
|
if (!band) return 'no published price band for this agent';
|
|
@@ -135,9 +182,53 @@ function bandText(agentId, count) {
|
|
|
135
182
|
: `estimated ${money(band[0])}–${money(band[1])} for this task`;
|
|
136
183
|
}
|
|
137
184
|
|
|
138
|
-
function spendLine(agentId, count) {
|
|
185
|
+
function spendLine(agentId, count, planning) {
|
|
139
186
|
const head = count > 1 ? `${count} prompts × ${agentId}` : `Agent ${agentId}`;
|
|
140
|
-
return `${head} — ${bandText(agentId, count)}`;
|
|
187
|
+
return `${head}${planning ? ' (planning turn)' : ''} — ${bandText(agentId, count)}`;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// The parenthetical after the band. `hedge` is the docs' "preview rates" caveat,
|
|
191
|
+
// which the confirmation carries; --dry-run already reads as a projection.
|
|
192
|
+
function spendNote(planning, hedge) {
|
|
193
|
+
const parts = [];
|
|
194
|
+
if (hedge) parts.push('preview rates, subject to change');
|
|
195
|
+
if (planning) parts.push(PLAN_NOTE);
|
|
196
|
+
return parts.length ? ` (${parts.join('; ')})` : '';
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// What the agent runs in the store have plausibly cost, from the same per-task
|
|
200
|
+
// bands the guard quotes before each one. A plan chain bills per turn, so this
|
|
201
|
+
// counts plans and refinements alongside reports -- that is the number worth
|
|
202
|
+
// knowing. Agents with no published band are counted separately rather than
|
|
203
|
+
// silently priced at zero. Null when nothing has been billed at all.
|
|
204
|
+
function estimatedSpend(agentRows) {
|
|
205
|
+
let low = 0;
|
|
206
|
+
let high = 0;
|
|
207
|
+
let tasks = 0;
|
|
208
|
+
let unpriced = 0;
|
|
209
|
+
for (const a of agentRows) {
|
|
210
|
+
const band = gemini.AGENT_PRICE_BANDS[a.agent];
|
|
211
|
+
if (!band) {
|
|
212
|
+
unpriced += a.n;
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
low += band[0] * a.n;
|
|
216
|
+
high += band[1] * a.n;
|
|
217
|
+
tasks += a.n;
|
|
218
|
+
}
|
|
219
|
+
if (!tasks && !unpriced) return null;
|
|
220
|
+
// Nothing priced means the total is unknown, NOT zero. Reporting $0.00 for
|
|
221
|
+
// runs that cost real money is the exact dishonesty this guard exists to
|
|
222
|
+
// avoid, so low/high are null and the caller says "unknown" instead.
|
|
223
|
+
if (!tasks) return { low: null, high: null, tasks: 0, unpriced };
|
|
224
|
+
return { low, high, tasks, unpriced };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// A --dry-run's lines: the sources it would have sent, then the band, the
|
|
228
|
+
// honesty note and that nothing went, in the confirmation's order.
|
|
229
|
+
function dryRunSpend(agentId, count, planning, src) {
|
|
230
|
+
const line = `${spendLine(agentId, count, planning)}${spendNote(planning, false)}. Nothing submitted (--dry-run).`;
|
|
231
|
+
return [...sources.describe(src), line].join('\n');
|
|
141
232
|
}
|
|
142
233
|
|
|
143
234
|
function askYesNo(question) {
|
|
@@ -154,8 +245,9 @@ function askYesNo(question) {
|
|
|
154
245
|
// Returns only when the submission is confirmed; otherwise it exits (declined)
|
|
155
246
|
// or throws (no way to ask). Runs BEFORE any row is written, so a declined or
|
|
156
247
|
// refused submission leaves the tasks table untouched.
|
|
157
|
-
async function confirmSpend(agentId, count, opts) {
|
|
158
|
-
|
|
248
|
+
async function confirmSpend(agentId, count, opts, planning, src) {
|
|
249
|
+
for (const line of sources.describe(src)) console.error(line);
|
|
250
|
+
console.error(`${spendLine(agentId, count, planning)}${spendNote(planning, true)}.`);
|
|
159
251
|
if (opts.yes) return;
|
|
160
252
|
// GEMCATCH_ASSUME_TTY lets the offline suite drive the interactive branch
|
|
161
253
|
// through a pipe; real non-TTY callers (cron, CI, scripts) must say --yes.
|
|
@@ -177,7 +269,18 @@ async function confirmSpend(agentId, count, opts) {
|
|
|
177
269
|
// only when the user actually typed it -- commander fills in the default
|
|
178
270
|
// otherwise, and the default must not poison every agent run.
|
|
179
271
|
function resolveAgentOpts(opts, cmd) {
|
|
180
|
-
if (!opts.agent)
|
|
272
|
+
if (!opts.agent) {
|
|
273
|
+
// Collaborative planning is an agent_config field on an agent run. A model
|
|
274
|
+
// run has no plan turn at all, so --plan without --agent is a mistake worth
|
|
275
|
+
// naming rather than a flag that quietly does nothing.
|
|
276
|
+
if (opts.plan) {
|
|
277
|
+
throw new Error(
|
|
278
|
+
'--plan is a research-agent feature: collaborative planning applies to an agent, not a model, ' +
|
|
279
|
+
'and a model run has no plan turn.\n Try: --agent deep-research --plan'
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
return null;
|
|
283
|
+
}
|
|
181
284
|
if (cmd.getOptionValueSource('model') === 'cli') {
|
|
182
285
|
throw new Error(
|
|
183
286
|
'--model and --agent are mutually exclusive: an agent run is submitted with `agent` ' +
|
|
@@ -187,6 +290,271 @@ function resolveAgentOpts(opts, cmd) {
|
|
|
187
290
|
return gemini.resolveAgent(opts.agent);
|
|
188
291
|
}
|
|
189
292
|
|
|
293
|
+
// --- sources --------------------------------------------------------------
|
|
294
|
+
|
|
295
|
+
// --mcp, --file-search, --no-web, --attach and --visualize, on research and
|
|
296
|
+
// batch alike. Each command gets its own MCP parsers: the modifier flags attach
|
|
297
|
+
// to the --mcp before them, which only works if they share one list.
|
|
298
|
+
function addSourceOptions(cmd) {
|
|
299
|
+
const mcp = sources.mcpOptionParsers();
|
|
300
|
+
const collect = (v, list) => (list || []).concat(v);
|
|
301
|
+
return cmd
|
|
302
|
+
.optionsGroup('Your own data (needs --agent):')
|
|
303
|
+
.option('--mcp <url>', 'let the agent call this remote MCP server (repeatable)', mcp.mcp)
|
|
304
|
+
.option('--mcp-name <name>', 'name for the --mcp before it (default: its host)', mcp.name)
|
|
305
|
+
.option('--mcp-header <header>', "'Name: value' header for the --mcp before it; ${VAR} reads the environment", mcp.header)
|
|
306
|
+
.option('--mcp-allow <tools>', 'comma-separated tools the --mcp before it may call', mcp.allow)
|
|
307
|
+
.option('--file-search <store>', 'let the agent search this File Search store (repeatable)', collect)
|
|
308
|
+
.option('--no-web', 'drop Google Search and URL Context; needs --mcp, --file-search or --attach')
|
|
309
|
+
.option('--attach <path|url>', 'attach a PDF, CSV or image to the first turn (repeatable)', collect)
|
|
310
|
+
.option('--visualize', 'let the agent draw charts; they are saved as image files');
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// Reads the inline attachments and uploads the rest. Runs after the spend is
|
|
314
|
+
// confirmed and before any row is written, so a failed upload costs nothing.
|
|
315
|
+
// A plan is continued later, so it is told when its uploads expire.
|
|
316
|
+
async function attachFiles(files, planning) {
|
|
317
|
+
const attached = await sources.materialize(files, gemini.upload, (a) => {
|
|
318
|
+
console.error(edim(`Uploading ${a.source} (${sources.size(a.bytes)}) to the Files API...`));
|
|
319
|
+
});
|
|
320
|
+
const expiry = Math.min(...attached.record.map((a) => a.expires_at || Infinity));
|
|
321
|
+
if (planning && expiry !== Infinity) {
|
|
322
|
+
const when = new Date(expiry).toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short' });
|
|
323
|
+
console.error(`Note: the uploaded files expire at ${when}; a refine or approve after that will probably fail.`);
|
|
324
|
+
}
|
|
325
|
+
return attached;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// What a submitted turn records about its sources. Tools are stored with their
|
|
329
|
+
// header values because a later turn of the chain has to send them again.
|
|
330
|
+
function sourceColumns(src, record) {
|
|
331
|
+
return {
|
|
332
|
+
toolsJson: src.tools ? JSON.stringify(src.tools) : null,
|
|
333
|
+
attachmentsJson: record && record.length ? JSON.stringify(record) : null,
|
|
334
|
+
visualization: src.visualization || null,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// The source fields of a gemini.submit call, with ${VAR} header values filled in.
|
|
339
|
+
function sourceArgs(src, items) {
|
|
340
|
+
return { tools: sources.withEnv(src.tools), attachments: items, visualization: src.visualization };
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// The sources a later turn inherits from the row it continues. Attachments went
|
|
344
|
+
// with the first turn and are already in the conversation, so none are resent.
|
|
345
|
+
function storedSources(task) {
|
|
346
|
+
let tools;
|
|
347
|
+
try {
|
|
348
|
+
tools = task.tools_json ? JSON.parse(task.tools_json) : undefined;
|
|
349
|
+
} catch (_) {
|
|
350
|
+
throw new Error(`task ${task.id} has an unreadable tools_json column, so its tools can't be sent again`);
|
|
351
|
+
}
|
|
352
|
+
sources.withEnv(tools);
|
|
353
|
+
return { tools, files: [], visualization: task.visualization || undefined };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Server text about a task can quote its MCP headers or attachment URLs back.
|
|
357
|
+
function redactFor(task, text) {
|
|
358
|
+
const urls = (parseList(chainRoot(task).attachments_json) || []).filter((a) => a.via === 'url').map((a) => a.source);
|
|
359
|
+
return sources.redactText(text, parseList(task.tools_json), urls);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function maskRaw(raw, task) {
|
|
363
|
+
const shown = raw && raw.tools ? { ...raw, tools: sources.redactTools(raw.tools) } : raw;
|
|
364
|
+
return redactFor(task, JSON.stringify(shown, null, 2));
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// --- plan chains ----------------------------------------------------------
|
|
368
|
+
|
|
369
|
+
// `collaborative_planning: true` makes the agent return a research plan instead
|
|
370
|
+
// of a report. That plan is a decision point, not a deliverable: you read it,
|
|
371
|
+
// optionally `refine` it, and `approve` it to spend on the research run itself.
|
|
372
|
+
// Each turn is its own task row, linked to the one it continues by parent_id
|
|
373
|
+
// locally and by previous_interaction_id on the wire.
|
|
374
|
+
|
|
375
|
+
// What the approval turn sends as `input`. The plan is already in the
|
|
376
|
+
// conversation via previous_interaction_id, so this turn only has to say yes --
|
|
377
|
+
// the docs' own example sends a one-line confirmation, not the question again.
|
|
378
|
+
const APPROVE_INPUT = 'Plan looks good, proceed with the research.';
|
|
379
|
+
|
|
380
|
+
// The reason refresh() records when a poll 404s. Named because `approve` reads
|
|
381
|
+
// it back to tell an expired plan apart from any other terminal one.
|
|
382
|
+
const EXPIRED_ERROR = 'interaction not found (expired or deleted)';
|
|
383
|
+
|
|
384
|
+
const RETENTION_NOTE =
|
|
385
|
+
'The Interactions API retains interactions for 1 day on the free tier (55 days on paid).';
|
|
386
|
+
|
|
387
|
+
// A finished plan is a decision point, so the next command is spelled out under
|
|
388
|
+
// it. Guidance, so it goes to stderr like the rest -- `gemcatch get <plan> >
|
|
389
|
+
// plan.md` still captures only the plan.
|
|
390
|
+
function planFooter(task) {
|
|
391
|
+
return `Approve with: gemcatch approve ${task.id} · Refine with: gemcatch refine ${task.id} "..."`;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// The result payload for `get`/`watch`. The plan-chain fields ride along only on
|
|
395
|
+
// a plan row, so a model run's --json shape is exactly what it always was.
|
|
396
|
+
function resultPayload(task, status, result, citations, images) {
|
|
397
|
+
const p = { id: task.id, status, result, citations: citations || null };
|
|
398
|
+
if (images && images.length) p.images = images;
|
|
399
|
+
if (task.kind === 'plan') {
|
|
400
|
+
p.kind = 'plan';
|
|
401
|
+
p.approve = `gemcatch approve ${task.id}`;
|
|
402
|
+
p.refine = `gemcatch refine ${task.id} "<instruction>"`;
|
|
403
|
+
}
|
|
404
|
+
return p;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function cachedResult(task) {
|
|
408
|
+
return { status: task.status, text: task.result, citations: parseList(task.citations), imagePaths: parseList(task.images_json) };
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function printResult(task, r, json) {
|
|
412
|
+
emit(json, resultPayload(task, r.status, r.text, r.citations, r.imagePaths), () => {
|
|
413
|
+
console.log(withSources(r.text, r.citations, r.imagePaths));
|
|
414
|
+
if (task.kind === 'plan') console.error(planFooter(task));
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// Both continuation commands need the same thing: a plan row that completed and
|
|
419
|
+
// whose interaction the server can still resolve. Anything else exits here,
|
|
420
|
+
// before a row is written or a request is sent.
|
|
421
|
+
function needPlan(id, verb) {
|
|
422
|
+
const task = needTask(id);
|
|
423
|
+
if (task.kind !== 'plan') {
|
|
424
|
+
die(
|
|
425
|
+
new Error(
|
|
426
|
+
`Task ${task.id} is not a plan (kind: ${task.kind || 'task'}), so there is nothing to ${verb}.\n` +
|
|
427
|
+
' Plans come from: gemcatch research "<prompt>" --agent deep-research --plan'
|
|
428
|
+
)
|
|
429
|
+
);
|
|
430
|
+
}
|
|
431
|
+
if (task.status === 'incomplete' && task.error === EXPIRED_ERROR) {
|
|
432
|
+
die(
|
|
433
|
+
new Error(
|
|
434
|
+
`Plan ${task.id}'s interaction was dropped server-side, so it can no longer be continued.\n` +
|
|
435
|
+
` ${RETENTION_NOTE}\n` +
|
|
436
|
+
` Submit a fresh plan: gemcatch research "<prompt>" --agent ${task.agent || '<agent>'} --plan`
|
|
437
|
+
)
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
// A plan that finished any way other than `completed` is never coming back, so
|
|
441
|
+
// "wait for it" would be wrong advice. Only a plan still in flight gets
|
|
442
|
+
// pointed at `watch`.
|
|
443
|
+
if (isDone(task.status) && !isSuccess(task.status)) {
|
|
444
|
+
die(
|
|
445
|
+
new Error(
|
|
446
|
+
`Plan ${task.id} ended ${task.status}, so there is nothing to ${verb}.\n` +
|
|
447
|
+
` Submit a fresh plan: gemcatch research "<prompt>" --agent ${task.agent || '<agent>'} --plan`
|
|
448
|
+
)
|
|
449
|
+
);
|
|
450
|
+
}
|
|
451
|
+
if (!isSuccess(task.status)) {
|
|
452
|
+
die(
|
|
453
|
+
new Error(
|
|
454
|
+
`Plan ${task.id} has not completed yet (status: ${task.status}), so there is nothing to ${verb}.\n` +
|
|
455
|
+
` Wait for it: gemcatch watch ${task.id}`
|
|
456
|
+
)
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
if (!task.interaction_id) die(new Error(`Plan ${task.id} was never submitted.`));
|
|
460
|
+
return task;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// A plan can complete locally and still expire server-side before you approve
|
|
464
|
+
// it, in which case the continuation 404s on an id the user never typed. Name
|
|
465
|
+
// the retention window instead of passing that through raw.
|
|
466
|
+
function expiredHint(err, plan) {
|
|
467
|
+
if (!err || err.httpStatus !== 404) return err;
|
|
468
|
+
const e = new Error(
|
|
469
|
+
`the plan's interaction (${plan.interaction_id}) is gone server-side, so it cannot be continued.\n` +
|
|
470
|
+
` ${RETENTION_NOTE}\n` +
|
|
471
|
+
` Submit a fresh plan: gemcatch research "<prompt>" --agent ${plan.agent || '<agent>'} --plan`
|
|
472
|
+
);
|
|
473
|
+
e.code = 'API_ERROR';
|
|
474
|
+
e.httpStatus = 404;
|
|
475
|
+
return e;
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// The first turn of a plan chain. A report row is displayed under the question
|
|
479
|
+
// that started the chain rather than the "plan looks good" line actually sent --
|
|
480
|
+
// that is what keeps `list` and `export` reading as research instead of as
|
|
481
|
+
// protocol chatter.
|
|
482
|
+
function chainRoot(task) {
|
|
483
|
+
let cur = task;
|
|
484
|
+
const seen = new Set([task.id]);
|
|
485
|
+
while (cur.parent_id && !seen.has(cur.parent_id)) {
|
|
486
|
+
seen.add(cur.parent_id);
|
|
487
|
+
const parent = store.getTask(cur.parent_id);
|
|
488
|
+
if (!parent) break;
|
|
489
|
+
cur = parent;
|
|
490
|
+
}
|
|
491
|
+
return cur;
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// `refine` (another plan) and `approve` (the report) are the same submission --
|
|
495
|
+
// same agent, same tag, linked to the plan by previous_interaction_id --
|
|
496
|
+
// differing only in the collaborative_planning flag they send, the kind they
|
|
497
|
+
// store and the line they print. So they share one path, and the spend guard is
|
|
498
|
+
// on it exactly once.
|
|
499
|
+
async function continuePlan(plan, opts, turn) {
|
|
500
|
+
let id;
|
|
501
|
+
try {
|
|
502
|
+
const src = storedSources(plan);
|
|
503
|
+
if (opts.dryRun) {
|
|
504
|
+
emit(
|
|
505
|
+
opts.json,
|
|
506
|
+
{
|
|
507
|
+
dry_run: true,
|
|
508
|
+
agent: plan.agent,
|
|
509
|
+
kind: turn.kind,
|
|
510
|
+
parent_id: plan.id,
|
|
511
|
+
previous_interaction_id: plan.interaction_id,
|
|
512
|
+
input: turn.input,
|
|
513
|
+
...sources.preview(src),
|
|
514
|
+
},
|
|
515
|
+
() => console.log(dryRunSpend(plan.agent, 1, turn.planning, src))
|
|
516
|
+
);
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
// Warned, not refused: whether the server still needs the files once the
|
|
520
|
+
// conversation has read them is not documented.
|
|
521
|
+
const expired = (parseList(chainRoot(plan).attachments_json) || []).filter(
|
|
522
|
+
(a) => a.expires_at && a.expires_at < Date.now()
|
|
523
|
+
);
|
|
524
|
+
if (expired.length) {
|
|
525
|
+
console.error(
|
|
526
|
+
`Note: ${expired.map((a) => a.source).join(', ')} expired from the Files API, so this turn will probably fail.`
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
await confirmSpend(plan.agent, 1, opts, turn.planning, src);
|
|
530
|
+
id = store.createTask({
|
|
531
|
+
prompt: turn.prompt,
|
|
532
|
+
agent: plan.agent,
|
|
533
|
+
tag: plan.tag,
|
|
534
|
+
kind: turn.kind,
|
|
535
|
+
parentId: plan.id,
|
|
536
|
+
collaborativePlanning: turn.planning,
|
|
537
|
+
previousInteractionId: plan.interaction_id,
|
|
538
|
+
...sourceColumns(src),
|
|
539
|
+
});
|
|
540
|
+
const r = await gemini.submit(turn.input, {
|
|
541
|
+
agent: plan.agent,
|
|
542
|
+
collaborativePlanning: turn.planning,
|
|
543
|
+
previousInteractionId: plan.interaction_id,
|
|
544
|
+
...sourceArgs(src),
|
|
545
|
+
});
|
|
546
|
+
store.setInteraction(id, r.interactionId, r.status);
|
|
547
|
+
emit(
|
|
548
|
+
opts.json,
|
|
549
|
+
{ id, interaction_id: r.interactionId, status: r.status, kind: turn.kind, parent_id: plan.id },
|
|
550
|
+
() => console.log(turn.line(id))
|
|
551
|
+
);
|
|
552
|
+
} catch (err) {
|
|
553
|
+
markSubmitFailure(id, err);
|
|
554
|
+
die(expiredHint(err, plan));
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
190
558
|
// --- input ----------------------------------------------------------------
|
|
191
559
|
|
|
192
560
|
function readStdin() {
|
|
@@ -230,25 +598,49 @@ async function refresh(task) {
|
|
|
230
598
|
// the end of time. Any other error (5xx, network) is transient and is
|
|
231
599
|
// re-thrown for the caller to retry on its next pass.
|
|
232
600
|
if (err && err.httpStatus === 404) {
|
|
233
|
-
store.setStatus(task.id, 'incomplete', { error:
|
|
601
|
+
store.setStatus(task.id, 'incomplete', { error: EXPIRED_ERROR });
|
|
234
602
|
return { status: 'incomplete', text: null, usage: null, raw: null };
|
|
235
603
|
}
|
|
236
604
|
throw err;
|
|
237
605
|
}
|
|
238
606
|
const extra = {};
|
|
239
607
|
if (isDone(r.status)) {
|
|
608
|
+
if (r.text) r.text = redactFor(task, r.text);
|
|
240
609
|
if (isSuccess(r.status)) {
|
|
241
610
|
extra.result = r.text;
|
|
242
611
|
// Agent runs return citations with the report; the docs tell users to
|
|
243
612
|
// review them to verify the sources, so they are persisted, not dropped.
|
|
244
|
-
if (r.citations && r.citations.length)
|
|
245
|
-
|
|
613
|
+
if (r.citations && r.citations.length) {
|
|
614
|
+
r.citations = JSON.parse(redactFor(task, JSON.stringify(r.citations)));
|
|
615
|
+
extra.citations = JSON.stringify(r.citations);
|
|
616
|
+
}
|
|
617
|
+
if (r.images.length) {
|
|
618
|
+
r.imagePaths = saveImages(task, r.images);
|
|
619
|
+
extra.images_json = JSON.stringify(r.imagePaths);
|
|
620
|
+
}
|
|
621
|
+
} else if (r.text) {
|
|
622
|
+
extra.error = r.text;
|
|
623
|
+
}
|
|
246
624
|
}
|
|
247
625
|
if (r.usage) extra.usage = JSON.stringify(r.usage);
|
|
248
626
|
store.setStatus(task.id, r.status, extra);
|
|
627
|
+
// A digest has no images of its own but lists its sources' (see digest).
|
|
628
|
+
if (!r.imagePaths) r.imagePaths = parseList(task.images_json);
|
|
249
629
|
return r;
|
|
250
630
|
}
|
|
251
631
|
|
|
632
|
+
// Only a failed *submit* should mark a task failed. Once it has an
|
|
633
|
+
// interaction_id it is live on the server, and a later watch/poll error must
|
|
634
|
+
// never overwrite it to failed -- that would drop it from the active set and the
|
|
635
|
+
// daemon would abandon a task whose result is still coming. Leave it active; the
|
|
636
|
+
// daemon (or a later `get`) collects it. Every command that submits calls this
|
|
637
|
+
// on its way out, so the rule cannot drift between them.
|
|
638
|
+
function markSubmitFailure(id, err) {
|
|
639
|
+
if (!id) return;
|
|
640
|
+
const t = store.getTask(id);
|
|
641
|
+
if (!t || !t.interaction_id) store.setStatus(id, 'failed', { error: err.message });
|
|
642
|
+
}
|
|
643
|
+
|
|
252
644
|
// Bounds how many polls are open at once. The *rate* limit is enforced in
|
|
253
645
|
// gemini.js (GEMCATCH_RPM), which is the part that keeps a wide fan-out inside the
|
|
254
646
|
// free tier's requests-per-minute allowance.
|
|
@@ -270,66 +662,88 @@ const program = new Command();
|
|
|
270
662
|
program
|
|
271
663
|
.name('gemcatch')
|
|
272
664
|
.description("Fire-and-forget research tasks on Gemini's Interactions API (background execution).")
|
|
273
|
-
.version(require('./package.json').version)
|
|
665
|
+
.version(require('./package.json').version)
|
|
666
|
+
// An unknown `--flag=value` is echoed whole, and the value may be a token.
|
|
667
|
+
.configureOutput({ outputError: (str, write) => write(str.replace(/(unknown option '[^'=]*)=[^']*'/g, "$1=...'")) });
|
|
274
668
|
|
|
275
669
|
// --- research -------------------------------------------------------------
|
|
276
670
|
|
|
277
|
-
program
|
|
671
|
+
const research = program
|
|
278
672
|
.command('research')
|
|
279
673
|
.argument('[prompt]', 'what you want researched; "-" reads stdin')
|
|
280
674
|
.option('-f, --file <path>', 'read the prompt from a file')
|
|
281
675
|
.option('-m, --model <id>', 'model to use', gemini.DEFAULT_MODEL)
|
|
282
676
|
.option('-a, --agent <id>', 'submit to a research agent instead of a model (e.g. deep-research)')
|
|
677
|
+
.option('--plan', 'ask the agent for a research plan first, to refine and approve (needs --agent)')
|
|
283
678
|
.option('-s, --system <text>', 'system instruction')
|
|
284
679
|
.option('-t, --tag <tag>', 'label for filtering with `gemcatch list --tag`')
|
|
285
680
|
.option('-w, --watch', 'wait for the result instead of exiting')
|
|
286
681
|
.option('--yes', 'confirm the agent cost without asking (required when stdin is not a TTY)')
|
|
287
682
|
.option('--dry-run', 'show what would be submitted (and what it would cost); submit nothing')
|
|
288
|
-
.option('--json', 'machine-readable output')
|
|
683
|
+
.option('--json', 'machine-readable output');
|
|
684
|
+
addSourceOptions(research)
|
|
289
685
|
.description('submit a background task and exit immediately')
|
|
290
686
|
.action(async (promptArg, opts, cmd) => {
|
|
291
687
|
let id;
|
|
292
688
|
try {
|
|
293
689
|
const agent = resolveAgentOpts(opts, cmd);
|
|
690
|
+
const src = await sources.resolve(opts, agent);
|
|
691
|
+
for (const w of src.warnings) console.error(`Note: ${w}`);
|
|
294
692
|
const prompt = await resolvePrompt(promptArg, opts);
|
|
693
|
+
// undefined, not false: an ordinary run must keep sending no agent_config
|
|
694
|
+
// at all, exactly as it did before collaborative planning existed.
|
|
695
|
+
const planning = opts.plan ? true : undefined;
|
|
295
696
|
if (opts.dryRun) {
|
|
296
|
-
emit(
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
697
|
+
emit(
|
|
698
|
+
opts.json,
|
|
699
|
+
{
|
|
700
|
+
dry_run: true,
|
|
701
|
+
agent: agent || null,
|
|
702
|
+
model: agent ? null : opts.model,
|
|
703
|
+
plan: !!opts.plan,
|
|
704
|
+
prompt,
|
|
705
|
+
...sources.preview(src),
|
|
706
|
+
},
|
|
707
|
+
() => {
|
|
708
|
+
if (agent) console.log(dryRunSpend(agent, 1, opts.plan, src));
|
|
709
|
+
else console.log(`Would submit to ${opts.model}: ${snippet(prompt)}. Nothing submitted (--dry-run).`);
|
|
710
|
+
}
|
|
711
|
+
);
|
|
300
712
|
return;
|
|
301
713
|
}
|
|
302
|
-
if (agent) await confirmSpend(agent, 1, opts);
|
|
714
|
+
if (agent) await confirmSpend(agent, 1, opts, opts.plan, src);
|
|
715
|
+
const attached = await attachFiles(src.files, opts.plan);
|
|
303
716
|
id = store.createTask({
|
|
304
717
|
prompt,
|
|
305
718
|
model: agent ? null : opts.model,
|
|
306
719
|
agent,
|
|
307
720
|
systemInstruction: opts.system,
|
|
308
721
|
tag: opts.tag,
|
|
722
|
+
kind: opts.plan ? 'plan' : 'task',
|
|
723
|
+
collaborativePlanning: planning,
|
|
724
|
+
...sourceColumns(src, attached.record),
|
|
725
|
+
});
|
|
726
|
+
const r = await gemini.submit(prompt, {
|
|
727
|
+
model: opts.model,
|
|
728
|
+
agent,
|
|
729
|
+
systemInstruction: opts.system,
|
|
730
|
+
collaborativePlanning: planning,
|
|
731
|
+
...sourceArgs(src, attached.items),
|
|
309
732
|
});
|
|
310
|
-
const r = await gemini.submit(prompt, { model: opts.model, agent, systemInstruction: opts.system });
|
|
311
733
|
store.setInteraction(id, r.interactionId, r.status);
|
|
312
734
|
if (opts.watch) {
|
|
313
735
|
// Under --watch the submit line is progress, not the answer, so it
|
|
314
736
|
// goes to stderr -- `gemcatch research -w "..." > out.txt` then captures
|
|
315
737
|
// only the result.
|
|
316
|
-
if (!opts.json) console.error(edim(
|
|
738
|
+
if (!opts.json) console.error(edim(`${opts.plan ? 'Plan task' : 'Task'} ${id} submitted.`));
|
|
317
739
|
await watchTask(store.getTask(id), DEFAULT_POLL_MS, opts.json);
|
|
318
740
|
return;
|
|
319
741
|
}
|
|
320
|
-
emit(opts.json, { id, interaction_id: r.interactionId, status: r.status }, () =>
|
|
321
|
-
console.log(
|
|
742
|
+
emit(opts.json, { id, interaction_id: r.interactionId, status: r.status, kind: opts.plan ? 'plan' : 'task' }, () =>
|
|
743
|
+
console.log(`${opts.plan ? 'Plan task' : 'Task'} ${id} submitted. Run: gemcatch get ${id} when ready.`)
|
|
322
744
|
);
|
|
323
745
|
} catch (err) {
|
|
324
|
-
|
|
325
|
-
// interaction_id it is live on the server, and a later watch/poll error
|
|
326
|
-
// must never overwrite it to failed -- that would drop it from the active
|
|
327
|
-
// set and the daemon would abandon a task whose result is still coming.
|
|
328
|
-
// Leave it active; the daemon (or a later `get`) collects it.
|
|
329
|
-
if (id) {
|
|
330
|
-
const t = store.getTask(id);
|
|
331
|
-
if (!t || !t.interaction_id) store.setStatus(id, 'failed', { error: err.message });
|
|
332
|
-
}
|
|
746
|
+
markSubmitFailure(id, err);
|
|
333
747
|
die(err);
|
|
334
748
|
}
|
|
335
749
|
});
|
|
@@ -407,22 +821,26 @@ async function watchBatch(tag, intervalMs, json) {
|
|
|
407
821
|
);
|
|
408
822
|
}
|
|
409
823
|
|
|
410
|
-
program
|
|
824
|
+
const batch = program
|
|
411
825
|
.command('batch')
|
|
412
826
|
.argument('<file>', 'prompts file — one per line, or "-" to read stdin')
|
|
413
827
|
.option('-m, --model <id>', 'model to use', gemini.DEFAULT_MODEL)
|
|
414
828
|
.option('-a, --agent <id>', 'submit every prompt to a research agent instead of a model')
|
|
829
|
+
.option('--plan', 'ask the agent for a research plan per prompt, to refine and approve (needs --agent)')
|
|
415
830
|
.option('-s, --system <text>', 'system instruction')
|
|
416
831
|
.option('-t, --tag <tag>', 'tag the whole batch (default: batch-<hex>)')
|
|
417
832
|
.option('--separator <str>', 'split the file on this delimiter line for multi-line prompts')
|
|
418
833
|
.option('-w, --watch', 'submit all, then poll until the whole batch finishes')
|
|
419
834
|
.option('--yes', 'confirm the agent cost without asking (required when stdin is not a TTY)')
|
|
420
835
|
.option('--dry-run', 'parse and list what would be submitted; submit nothing')
|
|
421
|
-
.option('--json', 'machine-readable output')
|
|
836
|
+
.option('--json', 'machine-readable output');
|
|
837
|
+
addSourceOptions(batch)
|
|
422
838
|
.description('submit many background tasks from a file, tagged as one batch')
|
|
423
839
|
.action(async (file, opts, cmd) => {
|
|
424
840
|
try {
|
|
425
841
|
const agent = resolveAgentOpts(opts, cmd);
|
|
842
|
+
const src = await sources.resolve(opts, agent);
|
|
843
|
+
for (const w of src.warnings) console.error(`Note: ${w}`);
|
|
426
844
|
const text = file === '-' ? await readStdin() : fs.readFileSync(file, 'utf8');
|
|
427
845
|
const { prompts, skipped } = parsePrompts(text, opts.separator);
|
|
428
846
|
if (!prompts.length) throw new Error(`no prompts found in ${file === '-' ? 'stdin' : file}`);
|
|
@@ -431,14 +849,19 @@ program
|
|
|
431
849
|
if (skipped) {
|
|
432
850
|
console.error(edim(`(skipped ${skipped} blank/comment line${skipped === 1 ? '' : 's'})`));
|
|
433
851
|
}
|
|
852
|
+
if (prompts.length > 1) src.files = sources.uploadAll(src.files);
|
|
434
853
|
// Auto-tag so the batch is collectable as a unit; a user tag wins.
|
|
435
854
|
const tag = opts.tag || `batch-${crypto.randomUUID().slice(0, 6)}`;
|
|
436
855
|
|
|
856
|
+
// undefined, not false: an ordinary run must keep sending no agent_config.
|
|
857
|
+
const planning = opts.plan ? true : undefined;
|
|
858
|
+
|
|
437
859
|
if (opts.dryRun) {
|
|
438
|
-
|
|
860
|
+
const payload = { tag, dry_run: true, agent: agent || null, plan: !!opts.plan, prompts, ...sources.preview(src) };
|
|
861
|
+
emit(opts.json, payload, () => {
|
|
439
862
|
if (agent) {
|
|
440
863
|
// The whole point of the guard: N × the per-task band, up front.
|
|
441
|
-
console.log(
|
|
864
|
+
console.log(dryRunSpend(agent, prompts.length, opts.plan, src));
|
|
442
865
|
} else {
|
|
443
866
|
console.log(`Batch ${tag}: ${prompts.length} prompt(s) would be submitted:`);
|
|
444
867
|
for (const p of prompts) console.log(` ${snippet(p)}`);
|
|
@@ -449,14 +872,31 @@ program
|
|
|
449
872
|
|
|
450
873
|
// An agent batch multiplies a per-task dollar band by the whole file, so
|
|
451
874
|
// it is confirmed as one total before a single row is written.
|
|
452
|
-
if (agent) await confirmSpend(agent, prompts.length, opts);
|
|
875
|
+
if (agent) await confirmSpend(agent, prompts.length, opts, opts.plan, src);
|
|
876
|
+
// Uploaded once (see uploadAll) and attached to every prompt in the file.
|
|
877
|
+
const attached = await attachFiles(src.files, opts.plan);
|
|
453
878
|
|
|
454
879
|
// One failed submit must not sink the batch: mark that task failed and
|
|
455
880
|
// keep going. mapLimit preserves input order, so the report is stable.
|
|
456
881
|
const results = await mapLimit(prompts, 4, async (prompt) => {
|
|
457
|
-
const id = store.createTask({
|
|
882
|
+
const id = store.createTask({
|
|
883
|
+
prompt,
|
|
884
|
+
model: agent ? null : opts.model,
|
|
885
|
+
agent,
|
|
886
|
+
systemInstruction: opts.system,
|
|
887
|
+
tag,
|
|
888
|
+
kind: opts.plan ? 'plan' : 'task',
|
|
889
|
+
collaborativePlanning: planning,
|
|
890
|
+
...sourceColumns(src, attached.record),
|
|
891
|
+
});
|
|
458
892
|
try {
|
|
459
|
-
const r = await gemini.submit(prompt, {
|
|
893
|
+
const r = await gemini.submit(prompt, {
|
|
894
|
+
model: opts.model,
|
|
895
|
+
agent,
|
|
896
|
+
systemInstruction: opts.system,
|
|
897
|
+
collaborativePlanning: planning,
|
|
898
|
+
...sourceArgs(src, attached.items),
|
|
899
|
+
});
|
|
460
900
|
store.setInteraction(id, r.interactionId, r.status);
|
|
461
901
|
return { id, interaction_id: r.interactionId, status: r.status, prompt };
|
|
462
902
|
} catch (err) {
|
|
@@ -484,9 +924,16 @@ program
|
|
|
484
924
|
const pad = ' '.repeat(Math.max(0, 16 - status.length));
|
|
485
925
|
console.log(`${r.id} ${colorStatus(status)}${pad} ${snippet(r.prompt)}`);
|
|
486
926
|
}
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
927
|
+
if (opts.plan) {
|
|
928
|
+
console.log(dim('\nCollect the plans, then approve the ones worth running:'));
|
|
929
|
+
console.log(dim(' gemcatch daemon --exit-when-idle'));
|
|
930
|
+
console.log(dim(` gemcatch list --tag ${tag}`));
|
|
931
|
+
console.log(dim(' gemcatch get <id> # prints the plan and the approve command'));
|
|
932
|
+
} else {
|
|
933
|
+
console.log(dim('\nCollect them:'));
|
|
934
|
+
console.log(dim(' gemcatch daemon --exit-when-idle'));
|
|
935
|
+
console.log(dim(` gemcatch list --tag ${tag} --status completed`));
|
|
936
|
+
}
|
|
490
937
|
});
|
|
491
938
|
} catch (err) {
|
|
492
939
|
die(err);
|
|
@@ -528,19 +975,11 @@ program
|
|
|
528
975
|
// result being *present*, not truthy: a task that completes with empty
|
|
529
976
|
// text stores `''`, which is exactly the case the cache must still serve
|
|
530
977
|
// -- re-polling it would 404 after 24h, the very thing we cache to avoid.
|
|
531
|
-
if (isSuccess(task.status) && task.result != null && !opts.raw)
|
|
532
|
-
const cits = parseCitations(task.citations);
|
|
533
|
-
emit(opts.json, { id: task.id, status: task.status, result: task.result, citations: cits }, () =>
|
|
534
|
-
console.log(withSources(task.result, cits))
|
|
535
|
-
);
|
|
536
|
-
return;
|
|
537
|
-
}
|
|
978
|
+
if (isSuccess(task.status) && task.result != null && !opts.raw) return printResult(task, cachedResult(task), opts.json);
|
|
538
979
|
const r = await refresh(task);
|
|
539
|
-
if (opts.raw) return console.log(
|
|
980
|
+
if (opts.raw) return console.log(maskRaw(r.raw, task));
|
|
540
981
|
if (isSuccess(r.status)) {
|
|
541
|
-
|
|
542
|
-
console.log(withSources(r.text, r.citations))
|
|
543
|
-
);
|
|
982
|
+
printResult(task, r, opts.json);
|
|
544
983
|
} else if (isDone(r.status)) {
|
|
545
984
|
emit(opts.json, { id: task.id, status: r.status, error: r.text || null }, () =>
|
|
546
985
|
console.log(`Task ${task.id}: ${colorStatus(r.status)}${r.text ? `\n${r.text}` : ''}`)
|
|
@@ -556,8 +995,94 @@ program
|
|
|
556
995
|
}
|
|
557
996
|
});
|
|
558
997
|
|
|
998
|
+
// --- refine / approve -----------------------------------------------------
|
|
999
|
+
|
|
1000
|
+
program
|
|
1001
|
+
.command('refine')
|
|
1002
|
+
.argument('<id>', 'plan task id')
|
|
1003
|
+
.argument('<instruction>', 'what the plan should do differently')
|
|
1004
|
+
.option('--yes', 'confirm the agent cost without asking (required when stdin is not a TTY)')
|
|
1005
|
+
.option('--dry-run', 'show what it would cost; submit nothing')
|
|
1006
|
+
.option('--json', 'machine-readable output')
|
|
1007
|
+
.description('send an instruction back to a plan and get a revised plan')
|
|
1008
|
+
.action(async (id, instruction, opts) => {
|
|
1009
|
+
const text = (instruction || '').trim();
|
|
1010
|
+
if (!text) {
|
|
1011
|
+
return die(new Error('provide an instruction, e.g. gemcatch refine 8f3a1c04 "focus on enforcement dates"'));
|
|
1012
|
+
}
|
|
1013
|
+
const plan = needPlan(id, 'refine');
|
|
1014
|
+
await continuePlan(plan, opts, {
|
|
1015
|
+
planning: true,
|
|
1016
|
+
kind: 'plan',
|
|
1017
|
+
input: text,
|
|
1018
|
+
prompt: text,
|
|
1019
|
+
line: (newId) => `Plan task ${newId} submitted (refines ${plan.id}). Run: gemcatch get ${newId} when ready.`,
|
|
1020
|
+
});
|
|
1021
|
+
});
|
|
1022
|
+
|
|
1023
|
+
program
|
|
1024
|
+
.command('approve')
|
|
1025
|
+
.argument('<id>', 'plan task id')
|
|
1026
|
+
.option('--yes', 'confirm the agent cost without asking (required when stdin is not a TTY)')
|
|
1027
|
+
.option('--dry-run', 'show what it would cost; submit nothing')
|
|
1028
|
+
.option('--json', 'machine-readable output')
|
|
1029
|
+
.description('approve a plan and submit the research run it describes')
|
|
1030
|
+
.action(async (id, opts) => {
|
|
1031
|
+
const plan = needPlan(id, 'approve');
|
|
1032
|
+
await continuePlan(plan, opts, {
|
|
1033
|
+
planning: false,
|
|
1034
|
+
kind: 'report',
|
|
1035
|
+
input: APPROVE_INPUT,
|
|
1036
|
+
prompt: chainRoot(plan).prompt,
|
|
1037
|
+
line: (newId) => `Task ${newId} submitted (approves plan ${plan.id}).`,
|
|
1038
|
+
});
|
|
1039
|
+
});
|
|
1040
|
+
|
|
559
1041
|
// --- list -----------------------------------------------------------------
|
|
560
1042
|
|
|
1043
|
+
// A plan chain is one piece of work, so it is listed as one: the root in its
|
|
1044
|
+
// normal newest-first position, its continuations indented under it in the order
|
|
1045
|
+
// they were submitted. A row whose parent is filtered out of this listing (by
|
|
1046
|
+
// --status, --tag or -n) is rendered as its own root rather than dropped.
|
|
1047
|
+
function chainOrder(tasks) {
|
|
1048
|
+
const present = new Set(tasks.map((t) => t.id));
|
|
1049
|
+
const kids = new Map();
|
|
1050
|
+
for (const t of tasks) {
|
|
1051
|
+
if (!t.parent_id || !present.has(t.parent_id)) continue;
|
|
1052
|
+
if (!kids.has(t.parent_id)) kids.set(t.parent_id, []);
|
|
1053
|
+
kids.get(t.parent_id).push(t);
|
|
1054
|
+
}
|
|
1055
|
+
for (const list of kids.values()) list.sort((a, b) => a.created_at - b.created_at);
|
|
1056
|
+
const out = [];
|
|
1057
|
+
const seen = new Set();
|
|
1058
|
+
const walk = (t, depth) => {
|
|
1059
|
+
if (seen.has(t.id)) return;
|
|
1060
|
+
seen.add(t.id);
|
|
1061
|
+
out.push({ task: t, depth });
|
|
1062
|
+
for (const c of kids.get(t.id) || []) walk(c, depth + 1);
|
|
1063
|
+
};
|
|
1064
|
+
for (const t of tasks) if (!t.parent_id || !present.has(t.parent_id)) walk(t, 0);
|
|
1065
|
+
// A listing must never lose a row. Nothing the CLI writes can put a cycle in
|
|
1066
|
+
// parent_id, but a row that is unreachable from any root would otherwise
|
|
1067
|
+
// vanish silently, so anything left over is rendered flat.
|
|
1068
|
+
for (const t of tasks) walk(t, 0);
|
|
1069
|
+
return out;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
// A word per kind of source a row used, for the list's SOURCES column.
|
|
1073
|
+
function sourcesUsed(t) {
|
|
1074
|
+
const tools = parseList(t.tools_json) || [];
|
|
1075
|
+
const files = parseList(t.attachments_json) || [];
|
|
1076
|
+
const mcp = tools.filter((x) => x && x.type === 'mcp_server').length;
|
|
1077
|
+
const words = [];
|
|
1078
|
+
if (mcp) words.push(mcp > 1 ? `${mcp} mcp` : 'mcp');
|
|
1079
|
+
if (tools.some((x) => x && x.type === 'file_search')) words.push('store');
|
|
1080
|
+
if (tools.length && !tools.some((x) => x && x.type === 'google_search')) words.push('no-web');
|
|
1081
|
+
if (files.length) words.push(files.length > 1 ? `${files.length} files` : 'file');
|
|
1082
|
+
if (t.visualization) words.push('charts');
|
|
1083
|
+
return words.join(',');
|
|
1084
|
+
}
|
|
1085
|
+
|
|
561
1086
|
program
|
|
562
1087
|
.command('list')
|
|
563
1088
|
.alias('ls')
|
|
@@ -573,26 +1098,39 @@ program
|
|
|
573
1098
|
return die(new Error(`--limit must be a non-negative integer (got ${opts.limit})`));
|
|
574
1099
|
}
|
|
575
1100
|
const tasks = store.listTasks({ status: opts.status, tag: opts.tag, limit: opts.limit });
|
|
576
|
-
if (opts.json)
|
|
1101
|
+
if (opts.json) {
|
|
1102
|
+
const masked = tasks.map((t) => (t.tools_json ? { ...t, tools_json: sources.redactToolsJson(t.tools_json) } : t));
|
|
1103
|
+
return console.log(JSON.stringify(masked, null, 2));
|
|
1104
|
+
}
|
|
577
1105
|
if (!tasks.length) {
|
|
578
1106
|
console.log('No tasks yet. Submit one: gemcatch research "your question"');
|
|
579
1107
|
return;
|
|
580
1108
|
}
|
|
581
|
-
// The AGENT
|
|
582
|
-
// a pure-model store keeps the compact four-column
|
|
583
|
-
// Agent ids are shown compact -- the "-preview-MM-YYYY"
|
|
584
|
-
// noise in a table (the full id is in --json and in stats).
|
|
1109
|
+
// The AGENT, KIND and SOURCES columns only appear when something in the
|
|
1110
|
+
// listing uses them, so a pure-model store keeps the compact four-column
|
|
1111
|
+
// layout it always had. Agent ids are shown compact -- the "-preview-MM-YYYY"
|
|
1112
|
+
// suffix is version noise in a table (the full id is in --json and in stats).
|
|
585
1113
|
const showAgent = tasks.some((t) => t.agent);
|
|
1114
|
+
const showKind = tasks.some((t) => t.kind && t.kind !== 'task');
|
|
586
1115
|
const shortAgent = (a) => (a ? a.replace(/-preview-\d{2}-\d{4}$/, '') : '-');
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
1116
|
+
const used = new Map(tasks.map((t) => [t.id, sourcesUsed(t)]));
|
|
1117
|
+
const srcWidth = Math.max(...[...used.values()].map((u) => u.length));
|
|
1118
|
+
const srcHead = srcWidth ? `${'SOURCES'.padEnd(Math.max(7, srcWidth))} ` : '';
|
|
1119
|
+
console.log(
|
|
1120
|
+
dim(`ID AGE STATUS ${showKind ? 'KIND ' : ''}${showAgent ? 'AGENT ' : ''}${srcHead}PROMPT`)
|
|
1121
|
+
);
|
|
1122
|
+
for (const { task: t, depth } of chainOrder(tasks)) {
|
|
590
1123
|
const status = t.status || PENDING;
|
|
591
1124
|
// Pad before colouring: ANSI codes would break the column width.
|
|
592
1125
|
const pad = ' '.repeat(Math.max(0, 16 - status.length));
|
|
1126
|
+
const kindCol = showKind ? `${(t.kind || 'task').padEnd(7)} ` : '';
|
|
593
1127
|
const agentCol = showAgent ? `${shortAgent(t.agent).padEnd(18)} ` : '';
|
|
1128
|
+
const srcCol = srcWidth ? `${(used.get(t.id) || '-').padEnd(Math.max(7, srcWidth))} ` : '';
|
|
1129
|
+
// Indent the prompt, not the id: the fixed-width columns stay aligned and
|
|
1130
|
+
// the chain still reads as one thing.
|
|
1131
|
+
const branch = depth ? `${' '.repeat(depth - 1)}└─ ` : '';
|
|
594
1132
|
console.log(
|
|
595
|
-
`${t.id} ${age(t.created_at).padEnd(4)} ${colorStatus(status)}${pad} ${agentCol}${
|
|
1133
|
+
`${t.id} ${age(t.created_at).padEnd(4)} ${colorStatus(status)}${pad} ${kindCol}${agentCol}${srcCol}${branch}${snippet(t.prompt)}`
|
|
596
1134
|
);
|
|
597
1135
|
}
|
|
598
1136
|
});
|
|
@@ -609,6 +1147,7 @@ program
|
|
|
609
1147
|
.addOption(new Option('--status <status>', 'only this status').choices(ALL_STATUSES).default('completed'))
|
|
610
1148
|
.addOption(new Option('--format <fmt>', 'output format').choices(['md', 'json']).default('md'))
|
|
611
1149
|
.option('-o, --out <file>', 'write to a file instead of stdout')
|
|
1150
|
+
.option('--include-plans', 'also export the plan turns of a chain, not just its report')
|
|
612
1151
|
.description('concatenate finished results, each under its prompt, to stdout or a file')
|
|
613
1152
|
.action((opts) => {
|
|
614
1153
|
const tasks = store.listTasks({ tag: opts.tag, status: opts.status });
|
|
@@ -617,25 +1156,54 @@ program
|
|
|
617
1156
|
tasks.reverse();
|
|
618
1157
|
// Only rows that actually carry a result are worth exporting: a status
|
|
619
1158
|
// filter other than `completed` can match tasks that never stored text.
|
|
620
|
-
|
|
1159
|
+
// A chain's plan turns are working notes on the way to its report, so an
|
|
1160
|
+
// export of a tag follows the chain to the report and leaves them out
|
|
1161
|
+
// unless they were asked for.
|
|
1162
|
+
const rows = tasks.filter((t) => t.result != null && (opts.includePlans || t.kind !== 'plan'));
|
|
621
1163
|
if (!rows.length) {
|
|
622
1164
|
// Nothing to write isn't an error, but say why so an empty -o file (or an
|
|
623
1165
|
// empty pipe) isn't a mystery. The note goes to stderr, never the output.
|
|
624
1166
|
console.error(`No ${opts.status} results to export${opts.tag ? ` for tag '${opts.tag}'` : ''}.`);
|
|
1167
|
+
// A chain with no approved run yet has plans and nothing else, which would
|
|
1168
|
+
// otherwise read as "there is nothing here".
|
|
1169
|
+
if (!opts.includePlans && tasks.some((t) => t.result != null && t.kind === 'plan')) {
|
|
1170
|
+
console.error(' Only plan turns matched. Approve one (gemcatch approve <id>), or pass --include-plans.');
|
|
1171
|
+
}
|
|
625
1172
|
return;
|
|
626
1173
|
}
|
|
627
1174
|
|
|
1175
|
+
// Images go next to the export file so its links still resolve when the
|
|
1176
|
+
// file is moved with them; on stdout they point into the data dir.
|
|
1177
|
+
const exportImages = (t) =>
|
|
1178
|
+
(parseList(t.images_json) || []).map((p) => {
|
|
1179
|
+
if (!opts.out) return p;
|
|
1180
|
+
const name = path.basename(p);
|
|
1181
|
+
try {
|
|
1182
|
+
fs.copyFileSync(p, path.join(path.dirname(path.resolve(opts.out)), name));
|
|
1183
|
+
return name;
|
|
1184
|
+
} catch (err) {
|
|
1185
|
+
console.error(`Note: could not copy ${p} next to the export (${err.message}).`);
|
|
1186
|
+
return p;
|
|
1187
|
+
}
|
|
1188
|
+
});
|
|
1189
|
+
|
|
628
1190
|
let output;
|
|
629
1191
|
if (opts.format === 'json') {
|
|
630
1192
|
output = JSON.stringify(
|
|
631
|
-
rows.map((t) =>
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
1193
|
+
rows.map((t) => {
|
|
1194
|
+
const row = {
|
|
1195
|
+
id: t.id,
|
|
1196
|
+
tag: t.tag,
|
|
1197
|
+
status: t.status,
|
|
1198
|
+
kind: t.kind || 'task',
|
|
1199
|
+
prompt: t.prompt,
|
|
1200
|
+
result: t.result,
|
|
1201
|
+
created_at: t.created_at,
|
|
1202
|
+
};
|
|
1203
|
+
const images = exportImages(t);
|
|
1204
|
+
if (images.length) row.images = images;
|
|
1205
|
+
return row;
|
|
1206
|
+
}),
|
|
639
1207
|
null,
|
|
640
1208
|
2
|
|
641
1209
|
);
|
|
@@ -645,7 +1213,10 @@ program
|
|
|
645
1213
|
const when = new Date(t.created_at).toISOString().replace('T', ' ').slice(0, 16);
|
|
646
1214
|
const head = (t.prompt || '(no prompt)').replace(/\s+/g, ' ').trim();
|
|
647
1215
|
const body = t.result && t.result.trim() ? t.result : '_(empty result)_';
|
|
648
|
-
|
|
1216
|
+
const kind = t.kind && t.kind !== 'task' ? ` · ${t.kind}` : '';
|
|
1217
|
+
const link = (p) => (path.isAbsolute(p) ? pathToFileURL(p).href : p);
|
|
1218
|
+
const images = exportImages(t).map((p, i) => `\n\n})`);
|
|
1219
|
+
return `## ${head}\n\n\`${t.id}\` · ${t.status}${kind} · ${when} UTC\n\n${body}${images.join('')}`;
|
|
649
1220
|
})
|
|
650
1221
|
.join('\n\n---\n\n');
|
|
651
1222
|
}
|
|
@@ -684,26 +1255,30 @@ program
|
|
|
684
1255
|
);
|
|
685
1256
|
}
|
|
686
1257
|
done.reverse(); // oldest first, so the sources read in submission order
|
|
687
|
-
const
|
|
1258
|
+
const results = done
|
|
688
1259
|
.map((t, i) => `## Source ${i + 1}: ${(t.prompt || '').replace(/\s+/g, ' ').trim()}\n\n${t.result}`)
|
|
689
1260
|
.join('\n\n');
|
|
690
1261
|
const prompt =
|
|
691
1262
|
`Synthesize the following ${done.length} research result(s) into one coherent summary.` +
|
|
692
1263
|
' Note where they agree and disagree, and do not simply repeat each verbatim.\n\n' +
|
|
693
|
-
|
|
1264
|
+
results;
|
|
694
1265
|
// The digest is itself a task, tagged so it is findable but kept out of
|
|
695
1266
|
// the source tag so a later digest never digests its own output.
|
|
696
|
-
|
|
1267
|
+
// A model can't carry the sources' charts forward, so the digest lists them.
|
|
1268
|
+
const images = done.flatMap((t) => parseList(t.images_json) || []);
|
|
1269
|
+
id = store.createTask({
|
|
1270
|
+
prompt,
|
|
1271
|
+
model: opts.model,
|
|
1272
|
+
systemInstruction: opts.system,
|
|
1273
|
+
tag: `${opts.tag}-digest`,
|
|
1274
|
+
imagesJson: images.length ? JSON.stringify(images) : null,
|
|
1275
|
+
});
|
|
697
1276
|
const r = await gemini.submit(prompt, { model: opts.model, systemInstruction: opts.system });
|
|
698
1277
|
store.setInteraction(id, r.interactionId, r.status);
|
|
699
1278
|
if (!opts.json) console.error(edim(`Digesting ${done.length} result(s) tagged ${opts.tag} -> task ${id}.`));
|
|
700
1279
|
await watchTask(store.getTask(id), DEFAULT_POLL_MS, opts.json);
|
|
701
1280
|
} catch (err) {
|
|
702
|
-
|
|
703
|
-
if (id) {
|
|
704
|
-
const t = store.getTask(id);
|
|
705
|
-
if (!t || !t.interaction_id) store.setStatus(id, 'failed', { error: err.message });
|
|
706
|
-
}
|
|
1281
|
+
markSubmitFailure(id, err);
|
|
707
1282
|
die(err);
|
|
708
1283
|
}
|
|
709
1284
|
});
|
|
@@ -860,12 +1435,7 @@ async function watchTask(task, intervalMs, json) {
|
|
|
860
1435
|
console.error(edim(`[${new Date().toISOString().slice(11, 19)}] ${task.id}: `) + ecolorStatus(r.status));
|
|
861
1436
|
last = r.status;
|
|
862
1437
|
}
|
|
863
|
-
if (isSuccess(r.status))
|
|
864
|
-
emit(json, { id: task.id, status: r.status, result: r.text, citations: r.citations || null }, () =>
|
|
865
|
-
console.log(withSources(r.text, r.citations))
|
|
866
|
-
);
|
|
867
|
-
return;
|
|
868
|
-
}
|
|
1438
|
+
if (isSuccess(r.status)) return printResult(task, r, json);
|
|
869
1439
|
if (isDone(r.status)) {
|
|
870
1440
|
emit(json, { id: task.id, status: r.status, error: r.text || null }, () => {
|
|
871
1441
|
console.error(`Task ${task.id} ended: ${ecolorStatus(r.status)}`);
|
|
@@ -889,13 +1459,7 @@ program
|
|
|
889
1459
|
try {
|
|
890
1460
|
// Serve a completed result from cache -- present, not merely truthy, so an
|
|
891
1461
|
// empty-text completion is served instead of re-polled (and lost at 24h).
|
|
892
|
-
if (isSuccess(task.status) && task.result != null)
|
|
893
|
-
const cits = parseCitations(task.citations);
|
|
894
|
-
emit(opts.json, { id: task.id, status: task.status, result: task.result, citations: cits }, () =>
|
|
895
|
-
console.log(withSources(task.result, cits))
|
|
896
|
-
);
|
|
897
|
-
return;
|
|
898
|
-
}
|
|
1462
|
+
if (isSuccess(task.status) && task.result != null) return printResult(task, cachedResult(task), opts.json);
|
|
899
1463
|
if (opts.interval != null && (!Number.isFinite(opts.interval) || opts.interval <= 0)) {
|
|
900
1464
|
return die(new Error(`--interval must be a positive number of seconds (got ${opts.interval})`));
|
|
901
1465
|
}
|
|
@@ -944,7 +1508,10 @@ program
|
|
|
944
1508
|
console.error(edim(` (remote delete failed for ${task.id}: ${err.message})`));
|
|
945
1509
|
}
|
|
946
1510
|
}
|
|
947
|
-
if (store.removeTask(task.id))
|
|
1511
|
+
if (store.removeTask(task.id)) {
|
|
1512
|
+
removeImages(task);
|
|
1513
|
+
removed += 1;
|
|
1514
|
+
}
|
|
948
1515
|
}
|
|
949
1516
|
console.log(`Removed ${removed} task${removed === 1 ? '' : 's'}.`);
|
|
950
1517
|
});
|
|
@@ -975,6 +1542,7 @@ program
|
|
|
975
1542
|
return;
|
|
976
1543
|
}
|
|
977
1544
|
const n = store.removeMany(doomed.map((t) => t.id));
|
|
1545
|
+
for (const t of doomed) removeImages(t);
|
|
978
1546
|
console.log(`Pruned ${n} task${n === 1 ? '' : 's'}.`);
|
|
979
1547
|
});
|
|
980
1548
|
|
|
@@ -983,20 +1551,44 @@ program
|
|
|
983
1551
|
program
|
|
984
1552
|
.command('stats')
|
|
985
1553
|
.option('--json', 'machine-readable output')
|
|
986
|
-
.description('where the store lives
|
|
1554
|
+
.description('where the store lives, what is in it, and what the agent runs have plausibly cost')
|
|
987
1555
|
.action((opts) => {
|
|
988
1556
|
const rows = store.counts();
|
|
989
1557
|
const agents = store.agentCounts();
|
|
1558
|
+
const kinds = store.kindCounts();
|
|
990
1559
|
const total = rows.reduce((n, r) => n + r.n, 0);
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
1560
|
+
// Priced from the runs that actually reached the server, not from every
|
|
1561
|
+
// attempt: a submit that failed before it left the machine cost nothing.
|
|
1562
|
+
const spend = estimatedSpend(store.billedAgentCounts());
|
|
1563
|
+
emit(
|
|
1564
|
+
opts.json,
|
|
1565
|
+
{ db: store.DB_PATH, total, by_status: rows, by_agent: agents, by_kind: kinds, estimated_spend: spend },
|
|
1566
|
+
() => {
|
|
1567
|
+
console.log(`Store: ${store.DB_PATH}`);
|
|
1568
|
+
console.log(`Tasks: ${total}`);
|
|
1569
|
+
for (const r of rows) console.log(` ${colorStatus(r.status).padEnd(useColor ? 26 : 17)} ${r.n}`);
|
|
1570
|
+
if (agents.length) {
|
|
1571
|
+
console.log('Agent runs:');
|
|
1572
|
+
for (const a of agents) console.log(` ${a.agent.padEnd(34)} ${a.n}`);
|
|
1573
|
+
}
|
|
1574
|
+
if (kinds.length) {
|
|
1575
|
+
console.log(`Plan chains: ${kinds.map((k) => `${k.n} ${k.kind}`).join(', ')}`);
|
|
1576
|
+
}
|
|
1577
|
+
if (spend && !spend.tasks) {
|
|
1578
|
+
console.log(
|
|
1579
|
+
`Estimated spend: unknown for ${spend.unpriced} agent task(s) on an agent with no published price band.`
|
|
1580
|
+
);
|
|
1581
|
+
} else if (spend) {
|
|
1582
|
+
const rest = spend.unpriced
|
|
1583
|
+
? `, plus ${spend.unpriced} on an agent with no published band`
|
|
1584
|
+
: '';
|
|
1585
|
+
console.log(
|
|
1586
|
+
`Estimated spend: $${spend.low.toFixed(2)}–$${spend.high.toFixed(2)} across ${spend.tasks} billed task(s)` +
|
|
1587
|
+
` (preview rates, subject to change)${rest}.`
|
|
1588
|
+
);
|
|
1589
|
+
}
|
|
998
1590
|
}
|
|
999
|
-
|
|
1591
|
+
);
|
|
1000
1592
|
});
|
|
1001
1593
|
|
|
1002
1594
|
// Close the store on the way out so a one-shot command doesn't leave the
|