@trawlme/cli 1.13.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/doctor.d.ts +94 -0
- package/dist/commands/doctor.js +140 -0
- package/dist/commands/scraps.js +242 -11
- package/dist/commands/token.d.ts +2 -0
- package/dist/commands/token.js +57 -0
- package/dist/index.js +2 -0
- package/dist/lib/api.d.ts +1 -0
- package/dist/lib/api.js +15 -0
- package/package.json +1 -1
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run diagnostics shape — mirrors the owner-safe REST shape returned by
|
|
3
|
+
* GET /api/historys/:id (Phase 1 projection, 2026-05-27).
|
|
4
|
+
* Cost and proxy-nature fields are intentionally absent by design.
|
|
5
|
+
* Abstract proxyTier (Tier 0–4) is kept.
|
|
6
|
+
*/
|
|
7
|
+
export interface Run {
|
|
8
|
+
_id: string;
|
|
9
|
+
status: boolean | null;
|
|
10
|
+
statusDetail: 'success' | 'error' | 'empty' | 'regression' | null;
|
|
11
|
+
length?: number | null;
|
|
12
|
+
errorMessage?: string | null;
|
|
13
|
+
errorSnapshot?: {
|
|
14
|
+
selector?: string | null;
|
|
15
|
+
errorMessage?: string | null;
|
|
16
|
+
html?: string | null;
|
|
17
|
+
capturedAt?: string | null;
|
|
18
|
+
} | null;
|
|
19
|
+
emptyContext?: {
|
|
20
|
+
page?: {
|
|
21
|
+
url?: string;
|
|
22
|
+
title?: string;
|
|
23
|
+
totalAnchors?: number;
|
|
24
|
+
};
|
|
25
|
+
selectors?: Record<string, number>;
|
|
26
|
+
} | null;
|
|
27
|
+
blocked?: boolean;
|
|
28
|
+
proxyTier?: string | null;
|
|
29
|
+
regressionDetected?: boolean;
|
|
30
|
+
baselineLength?: number | null;
|
|
31
|
+
fixVersionId?: string | null;
|
|
32
|
+
createdAt?: string;
|
|
33
|
+
time?: number | null;
|
|
34
|
+
triggeredBy?: string | null;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Autofix activity metadata — from the persisted ai_fix_end activity.
|
|
38
|
+
* aiUsage (cost) is stripped server-side; all diagnostics are kept.
|
|
39
|
+
*/
|
|
40
|
+
export interface FixActivity {
|
|
41
|
+
outcome: 'applied' | 'failed' | 'skipped' | 'breaker_tripped' | 'timeout' | null;
|
|
42
|
+
classification?: string | null;
|
|
43
|
+
reason?: string | null;
|
|
44
|
+
fixDiff?: string | null;
|
|
45
|
+
dryRunResults?: Array<{
|
|
46
|
+
status: 'success' | 'error';
|
|
47
|
+
error?: string;
|
|
48
|
+
}> | null;
|
|
49
|
+
knowledgeUsed?: Array<{
|
|
50
|
+
fingerprint: string;
|
|
51
|
+
confidence?: number | null;
|
|
52
|
+
}> | null;
|
|
53
|
+
attempt?: number;
|
|
54
|
+
maxAttempts?: number;
|
|
55
|
+
versionId?: string | null;
|
|
56
|
+
createdAt?: string;
|
|
57
|
+
}
|
|
58
|
+
interface ScrapHead {
|
|
59
|
+
_id: string;
|
|
60
|
+
title: string;
|
|
61
|
+
history?: Array<{
|
|
62
|
+
_id?: string;
|
|
63
|
+
status?: boolean | null;
|
|
64
|
+
createdAt?: string;
|
|
65
|
+
}>;
|
|
66
|
+
}
|
|
67
|
+
export declare function pickRun(run: Run): Partial<Run>;
|
|
68
|
+
export declare function pickFix(fix: FixActivity | null): Partial<FixActivity> | null;
|
|
69
|
+
/**
|
|
70
|
+
* Pure formatter — returns a human-readable diagnosis string for a run.
|
|
71
|
+
* Used by `doctor`, `doctor --autofix`, and `data --errors`.
|
|
72
|
+
*
|
|
73
|
+
* @param scrapTitle - Display name of the scrap
|
|
74
|
+
* @param run - Owner-safe history run object
|
|
75
|
+
* @param fix - Optional autofix activity (null = no fix attempted)
|
|
76
|
+
* @param scrapId - Scrap document id (used in hint lines — autofix/snapshot take scrap id, not run id)
|
|
77
|
+
* @returns Multi-line string ready for console.log
|
|
78
|
+
*/
|
|
79
|
+
export declare function formatDoctor(scrapTitle: string, run: Run, fix?: FixActivity | null, scrapId?: string): string;
|
|
80
|
+
/**
|
|
81
|
+
* Pure formatter — returns a human-readable autofix detail block.
|
|
82
|
+
* Shows diff, dry-run results, knowledge consulted.
|
|
83
|
+
*/
|
|
84
|
+
export declare function formatAutofix(fix: FixActivity): string;
|
|
85
|
+
/**
|
|
86
|
+
* Fetch the latest run + its ai_fix_end activity for a scrap.
|
|
87
|
+
* Returns null if the scrap has no runs yet.
|
|
88
|
+
*/
|
|
89
|
+
export declare function fetchRunAndFix(scrapId: string): Promise<{
|
|
90
|
+
scrap: ScrapHead;
|
|
91
|
+
run: Run;
|
|
92
|
+
fix: FixActivity | null;
|
|
93
|
+
} | null>;
|
|
94
|
+
export {};
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { api } from '../lib/api.js';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
const TIER_LABELS = {
|
|
4
|
+
tier0: 'Tier 0',
|
|
5
|
+
tier1: 'Tier 1',
|
|
6
|
+
tier2: 'Tier 2',
|
|
7
|
+
tier3: 'Tier 3',
|
|
8
|
+
tier4: 'Tier 4',
|
|
9
|
+
};
|
|
10
|
+
const RUN_ALLOWLIST = [
|
|
11
|
+
'_id', 'status', 'statusDetail', 'length', 'errorMessage', 'errorSnapshot',
|
|
12
|
+
'emptyContext', 'blocked', 'proxyTier', 'regressionDetected', 'baselineLength',
|
|
13
|
+
'fixVersionId', 'createdAt', 'time', 'triggeredBy',
|
|
14
|
+
];
|
|
15
|
+
const FIX_ALLOWLIST = [
|
|
16
|
+
'outcome', 'classification', 'reason', 'fixDiff', 'dryRunResults',
|
|
17
|
+
'knowledgeUsed', 'attempt', 'maxAttempts', 'versionId', 'createdAt',
|
|
18
|
+
];
|
|
19
|
+
export function pickRun(run) {
|
|
20
|
+
return Object.fromEntries(RUN_ALLOWLIST
|
|
21
|
+
.filter((k) => k in run)
|
|
22
|
+
.map((k) => [k, run[k]]));
|
|
23
|
+
}
|
|
24
|
+
export function pickFix(fix) {
|
|
25
|
+
if (!fix)
|
|
26
|
+
return null;
|
|
27
|
+
return Object.fromEntries(FIX_ALLOWLIST
|
|
28
|
+
.filter((k) => k in fix)
|
|
29
|
+
.map((k) => [k, fix[k]]));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Pure formatter — returns a human-readable diagnosis string for a run.
|
|
33
|
+
* Used by `doctor`, `doctor --autofix`, and `data --errors`.
|
|
34
|
+
*
|
|
35
|
+
* @param scrapTitle - Display name of the scrap
|
|
36
|
+
* @param run - Owner-safe history run object
|
|
37
|
+
* @param fix - Optional autofix activity (null = no fix attempted)
|
|
38
|
+
* @param scrapId - Scrap document id (used in hint lines — autofix/snapshot take scrap id, not run id)
|
|
39
|
+
* @returns Multi-line string ready for console.log
|
|
40
|
+
*/
|
|
41
|
+
export function formatDoctor(scrapTitle, run, fix = null, scrapId) {
|
|
42
|
+
const lines = [];
|
|
43
|
+
// Header + status badge
|
|
44
|
+
const ok = run.status === true;
|
|
45
|
+
const badge = ok
|
|
46
|
+
? chalk.green('● success')
|
|
47
|
+
: run.statusDetail === 'empty'
|
|
48
|
+
? chalk.yellow('● empty')
|
|
49
|
+
: chalk.red('● failed');
|
|
50
|
+
lines.push(`${chalk.bold(scrapTitle)} ${badge}${run.statusDetail ? ` (${run.statusDetail})` : ''}`);
|
|
51
|
+
lines.push(chalk.dim(` Run ID: ${run._id}`));
|
|
52
|
+
// Error message
|
|
53
|
+
const errMsg = run.errorMessage ?? run.errorSnapshot?.errorMessage;
|
|
54
|
+
if (errMsg) {
|
|
55
|
+
lines.push(chalk.dim(' Error: ') + chalk.red(errMsg));
|
|
56
|
+
}
|
|
57
|
+
// Failed selector
|
|
58
|
+
if (run.errorSnapshot?.selector) {
|
|
59
|
+
lines.push(chalk.dim(' Failed selector: ') + chalk.cyan(run.errorSnapshot.selector));
|
|
60
|
+
}
|
|
61
|
+
// Blocked
|
|
62
|
+
lines.push(chalk.dim(' Blocked: ') + (run.blocked ? chalk.red('yes') : 'no'));
|
|
63
|
+
// Proxy tier (abstract, kept for owner)
|
|
64
|
+
if (run.proxyTier && TIER_LABELS[run.proxyTier]) {
|
|
65
|
+
lines.push(chalk.dim(' Proxy: ') + TIER_LABELS[run.proxyTier]);
|
|
66
|
+
}
|
|
67
|
+
// Empty context detail
|
|
68
|
+
if (run.statusDetail === 'empty' && run.emptyContext?.page) {
|
|
69
|
+
const page = run.emptyContext.page;
|
|
70
|
+
lines.push(chalk.dim(' Empty context: ') + `url=${page.url ?? '?'} anchors=${page.totalAnchors ?? '?'}`);
|
|
71
|
+
}
|
|
72
|
+
// Regression detail
|
|
73
|
+
if (run.regressionDetected) {
|
|
74
|
+
lines.push(chalk.dim(' Regression: ') + `length ${run.length ?? '?'} vs baseline ${run.baselineLength ?? '?'}`);
|
|
75
|
+
}
|
|
76
|
+
// Autofix summary (when fix exists)
|
|
77
|
+
if (fix) {
|
|
78
|
+
lines.push(` ${chalk.magenta('Autofix:')} ${fix.outcome ?? '?'}${fix.classification ? ` — ${fix.classification}` : ''}${fix.reason ? ` (${fix.reason})` : ''}`);
|
|
79
|
+
lines.push(chalk.dim(` → full diff/dry-run/knowledge: trawl scraps autofix ${scrapId ?? run._id} (or doctor --autofix)`));
|
|
80
|
+
}
|
|
81
|
+
// Timestamp
|
|
82
|
+
if (run.createdAt) {
|
|
83
|
+
lines.push(chalk.dim(' Run at: ') + new Date(run.createdAt).toLocaleString());
|
|
84
|
+
}
|
|
85
|
+
// Snapshot hint
|
|
86
|
+
if (run.errorSnapshot?.html || run.statusDetail === 'empty' || run.statusDetail === 'error') {
|
|
87
|
+
lines.push('');
|
|
88
|
+
lines.push(chalk.dim(` → trawl scraps snapshot ${scrapId ?? run._id} --error (download error-path HTML)`));
|
|
89
|
+
}
|
|
90
|
+
return lines.join('\n');
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Pure formatter — returns a human-readable autofix detail block.
|
|
94
|
+
* Shows diff, dry-run results, knowledge consulted.
|
|
95
|
+
*/
|
|
96
|
+
export function formatAutofix(fix) {
|
|
97
|
+
const lines = [];
|
|
98
|
+
lines.push(chalk.magenta.bold('Autofix attempt'));
|
|
99
|
+
lines.push(` Decision: ${fix.outcome ?? '?'}${fix.classification ? ` — ${fix.classification}` : ''}${fix.reason ? ` (${fix.reason})` : ''}`);
|
|
100
|
+
if (fix.versionId)
|
|
101
|
+
lines.push(` Version: ${fix.versionId}`);
|
|
102
|
+
lines.push(' Fix diff:');
|
|
103
|
+
if (fix.fixDiff && fix.fixDiff.trim()) {
|
|
104
|
+
for (const l of fix.fixDiff.split('\n')) {
|
|
105
|
+
const c = l.startsWith('+') ? chalk.green(l) : l.startsWith('-') ? chalk.red(l) : chalk.dim(l);
|
|
106
|
+
lines.push(` ${c}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
lines.push(chalk.dim(' No code change — no-op recovery (site recovered between fail and dry-run).'));
|
|
111
|
+
}
|
|
112
|
+
if (fix.dryRunResults?.length) {
|
|
113
|
+
const ok = fix.dryRunResults.filter((d) => d.status === 'success').length;
|
|
114
|
+
lines.push(` Dry-run: ${ok}/${fix.dryRunResults.length} succeeded`);
|
|
115
|
+
fix.dryRunResults
|
|
116
|
+
.filter((d) => d.status === 'error' && d.error)
|
|
117
|
+
.forEach((d) => lines.push(chalk.red(` ✗ ${d.error}`)));
|
|
118
|
+
}
|
|
119
|
+
if (fix.knowledgeUsed?.length) {
|
|
120
|
+
lines.push(' Knowledge consulted:');
|
|
121
|
+
fix.knowledgeUsed.forEach((k) => lines.push(` • ${k.fingerprint}${k.confidence != null ? ` (${Math.round(k.confidence * 100)}%)` : ''}`));
|
|
122
|
+
}
|
|
123
|
+
return lines.join('\n');
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Fetch the latest run + its ai_fix_end activity for a scrap.
|
|
127
|
+
* Returns null if the scrap has no runs yet.
|
|
128
|
+
*/
|
|
129
|
+
export async function fetchRunAndFix(scrapId) {
|
|
130
|
+
const scrap = await api.get(`/api/scraps/${scrapId}`);
|
|
131
|
+
const hid = scrap.history?.[0]?._id;
|
|
132
|
+
if (!hid)
|
|
133
|
+
return null;
|
|
134
|
+
const run = await api.get(`/api/historys/${hid}`);
|
|
135
|
+
// api.get unwraps the { data: T } envelope — the response is the array directly
|
|
136
|
+
const acts = await api.get(`/api/scraps/${scrapId}/activities?history=${hid}&limit=5`);
|
|
137
|
+
const fixAct = Array.isArray(acts) ? acts.find((a) => a.type === 'ai_fix_end') : undefined;
|
|
138
|
+
const fix = fixAct ? { ...fixAct.metadata, createdAt: fixAct.createdAt } : null;
|
|
139
|
+
return { scrap, run, fix };
|
|
140
|
+
}
|
package/dist/commands/scraps.js
CHANGED
|
@@ -5,6 +5,7 @@ import { api } from '../lib/api.js';
|
|
|
5
5
|
import { table, json } from '../lib/format.js';
|
|
6
6
|
import { promptPassword } from '../lib/prompt.js';
|
|
7
7
|
import { validateObjectId } from '../lib/validate.js';
|
|
8
|
+
import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix } from './doctor.js';
|
|
8
9
|
function lastStatus(scrap) {
|
|
9
10
|
const last = scrap.history?.[0];
|
|
10
11
|
if (!last || last.status === null || last.status === undefined)
|
|
@@ -55,13 +56,51 @@ scraps
|
|
|
55
56
|
.description('List all scraps')
|
|
56
57
|
.option('--json', 'Output as JSON')
|
|
57
58
|
.option('--status <status>', 'Filter by last run status (success|failure|never)')
|
|
59
|
+
.option('--limit <n>', 'Show only the first N results', (v) => parseInt(v, 10))
|
|
60
|
+
.option('--page <n>', 'Fetch a specific page only (50 per page, no auto-pagination)', (v) => parseInt(v, 10))
|
|
58
61
|
.action(async (opts) => {
|
|
62
|
+
// Guard: --limit and --page are mutually exclusive
|
|
63
|
+
if (opts.limit !== undefined && opts.page !== undefined) {
|
|
64
|
+
console.log(chalk.red('✗ --limit and --page are mutually exclusive. Use one or the other.'));
|
|
65
|
+
process.exitCode = 1;
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
59
68
|
const spinner = ora('Fetching scraps…').start();
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
69
|
+
let data;
|
|
70
|
+
try {
|
|
71
|
+
if (opts.page !== undefined) {
|
|
72
|
+
// Single-page mode: explicit page requested, no loop
|
|
73
|
+
const pageNum = opts.page;
|
|
74
|
+
data = await api.get(`/api/scraps?perPage=50&page=${pageNum}`);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
// Fetch-all mode: paginate until a page returns < 200 items
|
|
78
|
+
const perPage = 200;
|
|
79
|
+
data = [];
|
|
80
|
+
let page = 1;
|
|
81
|
+
while (true) {
|
|
82
|
+
const batch = await api.get(`/api/scraps?perPage=${perPage}&page=${page}`);
|
|
83
|
+
data = data.concat(batch);
|
|
84
|
+
if (batch.length < perPage)
|
|
85
|
+
break;
|
|
86
|
+
page++;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
spinner.stop();
|
|
92
|
+
console.log(chalk.red(`✗ Failed to fetch scraps: ${err.message}`));
|
|
93
|
+
process.exitCode = 1;
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
spinner.stop();
|
|
98
|
+
}
|
|
63
99
|
if (opts.status)
|
|
64
|
-
|
|
100
|
+
data = data.filter((s) => lastStatus(s) === opts.status);
|
|
101
|
+
const totalMatched = data.length;
|
|
102
|
+
const limit = opts.limit;
|
|
103
|
+
const rows = limit !== undefined ? data.slice(0, limit) : data;
|
|
65
104
|
if (opts.json)
|
|
66
105
|
return json(rows);
|
|
67
106
|
const tableRows = rows.map((s) => ({
|
|
@@ -73,6 +112,10 @@ scraps
|
|
|
73
112
|
updated: new Date(s.updatedAt).toLocaleDateString(),
|
|
74
113
|
}));
|
|
75
114
|
table(tableRows, ['id', 'title', 'cron', 'status', 'last run', 'updated']);
|
|
115
|
+
// Print footer when --limit truncates
|
|
116
|
+
if (limit !== undefined && rows.length < totalMatched) {
|
|
117
|
+
console.log(chalk.dim(`Showing ${rows.length} of ${totalMatched} — omit --limit to see all`));
|
|
118
|
+
}
|
|
76
119
|
});
|
|
77
120
|
// get
|
|
78
121
|
scraps
|
|
@@ -220,8 +263,25 @@ scraps
|
|
|
220
263
|
.command('data <id>')
|
|
221
264
|
.description('Get scrap data (latest run payload)')
|
|
222
265
|
.option('--json', 'Output as JSON')
|
|
266
|
+
.option('--errors', 'Show failure diagnostics when the last run failed')
|
|
223
267
|
.action(async (id, opts) => {
|
|
224
268
|
validateObjectId(id);
|
|
269
|
+
// --errors: fetch full run + fix detail via fetchRunAndFix (DRY with doctor)
|
|
270
|
+
if (opts.errors) {
|
|
271
|
+
const result = await fetchRunAndFix(id);
|
|
272
|
+
if (!result) {
|
|
273
|
+
console.log(chalk.dim('No runs yet.'));
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
if (result.run.status === true) {
|
|
277
|
+
console.log(chalk.green('✓ Last run succeeded. No errors to show.'));
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (opts.json)
|
|
281
|
+
return json(pickRun(result.run));
|
|
282
|
+
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
225
285
|
const loaded = await api.get(`/api/scraps/load/${id}`);
|
|
226
286
|
const items = loaded?.result?.data;
|
|
227
287
|
if (!Array.isArray(items)) {
|
|
@@ -232,14 +292,67 @@ scraps
|
|
|
232
292
|
return json(items);
|
|
233
293
|
console.log(chalk.bold('Last run data:'));
|
|
234
294
|
console.log(chalk.dim(` Items: ${items.length}`));
|
|
235
|
-
if (loaded?.result?._proxyTier) {
|
|
236
|
-
console.log(chalk.dim(` Proxy tier: ${loaded.result._proxyTier}`));
|
|
237
|
-
}
|
|
238
295
|
if (items.length > 0 && typeof items[0] === 'object' && items[0] !== null) {
|
|
239
296
|
console.log(chalk.dim(` First item keys: ${Object.keys(items[0]).join(', ')}`));
|
|
240
297
|
}
|
|
241
298
|
console.log(chalk.dim(' Use --json for full output.'));
|
|
242
299
|
});
|
|
300
|
+
// history — list past runs for a scrap
|
|
301
|
+
scraps
|
|
302
|
+
.command('history <id>')
|
|
303
|
+
.description('List past runs for a scrap (newest first)')
|
|
304
|
+
.option('--json', 'Output as JSON')
|
|
305
|
+
.option('-n, --limit <n>', 'Max runs to show', '20')
|
|
306
|
+
.action(async (id, opts) => {
|
|
307
|
+
validateObjectId(id);
|
|
308
|
+
const limit = Number(opts.limit);
|
|
309
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
310
|
+
console.log(chalk.red(`✗ Invalid --limit "${opts.limit}" (expected a positive integer)`));
|
|
311
|
+
process.exitCode = 1;
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
const scrap = await api.get(`/api/scraps/${id}`);
|
|
315
|
+
const runs = (scrap.history ?? []).slice(0, limit).map((h) => ({
|
|
316
|
+
hid: h._id,
|
|
317
|
+
status: h.statusDetail ?? null,
|
|
318
|
+
time: h.time ?? null,
|
|
319
|
+
tier: h.proxyTier ?? null,
|
|
320
|
+
failureKind: h.failureKind ?? null,
|
|
321
|
+
blockType: h.blockType ?? null,
|
|
322
|
+
createdAt: h.createdAt ?? null,
|
|
323
|
+
}));
|
|
324
|
+
if (opts.json) {
|
|
325
|
+
json(runs);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
table(runs, ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'createdAt']);
|
|
329
|
+
});
|
|
330
|
+
// run-info — single-run detail by history id
|
|
331
|
+
scraps
|
|
332
|
+
.command('run-info <hid>')
|
|
333
|
+
.description('Show details of a single run (status, tier, failureKind, error)')
|
|
334
|
+
.option('--json', 'Output as JSON')
|
|
335
|
+
.action(async (hid, opts) => {
|
|
336
|
+
validateObjectId(hid);
|
|
337
|
+
const h = await api.get(`/api/historys/${hid}`);
|
|
338
|
+
const info = {
|
|
339
|
+
hid,
|
|
340
|
+
status: h.statusDetail ?? null,
|
|
341
|
+
time: h.time ?? null,
|
|
342
|
+
tier: h.proxyTier ?? null,
|
|
343
|
+
failureKind: h.failureKind ?? null,
|
|
344
|
+
blockType: h.blockType ?? null,
|
|
345
|
+
errorMessage: h.errorSnapshot?.errorMessage ?? null,
|
|
346
|
+
selector: h.errorSnapshot?.selector ?? null,
|
|
347
|
+
emptyContext: h.errorSnapshot?.emptyContext ?? null,
|
|
348
|
+
createdAt: h.createdAt ?? null,
|
|
349
|
+
};
|
|
350
|
+
if (opts.json) {
|
|
351
|
+
json(info);
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
table([info], ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'errorMessage', 'selector', 'emptyContext', 'createdAt']);
|
|
355
|
+
});
|
|
243
356
|
// delete
|
|
244
357
|
scraps
|
|
245
358
|
.command('delete <id>')
|
|
@@ -311,13 +424,18 @@ scraps
|
|
|
311
424
|
// trigger
|
|
312
425
|
scraps
|
|
313
426
|
.command('trigger <id>')
|
|
314
|
-
.description('Launch a scrap as a background worker')
|
|
427
|
+
.description('Launch a scrap as a background worker (returns immediately)')
|
|
315
428
|
.option('-w, --watch', 'Stream activities after triggering')
|
|
429
|
+
.option('--wait', 'Run synchronously and wait for the result (legacy behaviour)')
|
|
316
430
|
.action(async (id, opts) => {
|
|
317
431
|
validateObjectId(id);
|
|
318
|
-
const spinner = ora('Triggering worker…').start();
|
|
319
|
-
|
|
320
|
-
|
|
432
|
+
const spinner = ora(opts.wait ? 'Running worker…' : 'Triggering worker…').start();
|
|
433
|
+
// #50 — default async: the backend (#1313) kicks off the run and returns a
|
|
434
|
+
// 'queued' envelope immediately instead of holding the connection for the
|
|
435
|
+
// whole run. --wait restores the old synchronous round-trip.
|
|
436
|
+
const path = opts.wait ? `/api/scraps/worker/${id}` : `/api/scraps/worker/${id}?wait=false`;
|
|
437
|
+
await api.post(path);
|
|
438
|
+
spinner.succeed(opts.wait ? 'Worker run complete' : 'Worker triggered');
|
|
321
439
|
if (opts.watch)
|
|
322
440
|
await watchActivities(id);
|
|
323
441
|
});
|
|
@@ -412,6 +530,54 @@ account
|
|
|
412
530
|
await api.delete(`/api/scraps/${id}/account/session`);
|
|
413
531
|
spinner.succeed('Session cleared');
|
|
414
532
|
});
|
|
533
|
+
// account session subcommand group
|
|
534
|
+
const accountSession = account
|
|
535
|
+
.command('session')
|
|
536
|
+
.description('Manage scrap account session cookies (flavour B BYO-cookies)');
|
|
537
|
+
// account session set
|
|
538
|
+
accountSession
|
|
539
|
+
.command('set <id>')
|
|
540
|
+
.description('Upload browser session cookies for a scrap (Puppeteer cookie JSON array)')
|
|
541
|
+
.requiredOption('-c, --cookies <file>', 'Path to a Puppeteer cookie JSON array file')
|
|
542
|
+
.action(async (id, opts) => {
|
|
543
|
+
validateObjectId(id);
|
|
544
|
+
const { existsSync, readFileSync } = await import('fs');
|
|
545
|
+
if (!existsSync(opts.cookies)) {
|
|
546
|
+
console.log(chalk.red(`✗ File not found: ${opts.cookies}`));
|
|
547
|
+
process.exitCode = 1;
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
let cookies;
|
|
551
|
+
try {
|
|
552
|
+
const raw = readFileSync(opts.cookies, 'utf-8');
|
|
553
|
+
cookies = JSON.parse(raw);
|
|
554
|
+
}
|
|
555
|
+
catch (e) {
|
|
556
|
+
console.log(chalk.red(`✗ Failed to parse cookies file: ${e.message}`));
|
|
557
|
+
process.exitCode = 1;
|
|
558
|
+
return;
|
|
559
|
+
}
|
|
560
|
+
if (!Array.isArray(cookies)) {
|
|
561
|
+
console.log(chalk.red('✗ Cookies file must contain a JSON array'));
|
|
562
|
+
process.exitCode = 1;
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
565
|
+
if (cookies.length === 0) {
|
|
566
|
+
console.log(chalk.red('✗ Cookies array must not be empty'));
|
|
567
|
+
process.exitCode = 1;
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
if (!cookies.every((c) => c && typeof c.name === 'string' && typeof c.value === 'string')) {
|
|
571
|
+
console.log(chalk.red('✗ Each cookie must have a name (string) and value (string)'));
|
|
572
|
+
process.exitCode = 1;
|
|
573
|
+
return;
|
|
574
|
+
}
|
|
575
|
+
const spinner = ora('Uploading session cookies…').start();
|
|
576
|
+
const data = await api.put(`/api/scraps/${id}/account/session`, { cookies });
|
|
577
|
+
spinner.succeed(`Session cookies saved for scrap ${chalk.bold(id)}`);
|
|
578
|
+
const acc = data.account;
|
|
579
|
+
console.log(chalk.dim(' Session: ') + (acc.hasSession ? chalk.green('✓ active') : chalk.dim('none')));
|
|
580
|
+
});
|
|
415
581
|
// account status
|
|
416
582
|
account
|
|
417
583
|
.command('status <id>')
|
|
@@ -453,3 +619,68 @@ account
|
|
|
453
619
|
}
|
|
454
620
|
console.log(`${credLine} | ${sessionLine}`);
|
|
455
621
|
});
|
|
622
|
+
// doctor — diagnose last run of a scrap (error, failed selector, block status, page state, autofix)
|
|
623
|
+
scraps
|
|
624
|
+
.command('doctor <id>')
|
|
625
|
+
.description('Diagnose the last run (error, failed selector, block status, page state, autofix outcome)')
|
|
626
|
+
.option('--json', 'Output raw run + autofix JSON')
|
|
627
|
+
.option('--autofix', 'Include the full autofix diff / dry-run / knowledge')
|
|
628
|
+
.action(async (id, opts) => {
|
|
629
|
+
validateObjectId(id);
|
|
630
|
+
const result = await fetchRunAndFix(id);
|
|
631
|
+
if (!result) {
|
|
632
|
+
console.log(chalk.dim('No runs yet.'));
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
if (opts.json)
|
|
636
|
+
return json({ run: pickRun(result.run), fix: pickFix(result.fix) });
|
|
637
|
+
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
638
|
+
if (opts.autofix && result.fix) {
|
|
639
|
+
console.log('\n' + formatAutofix(result.fix));
|
|
640
|
+
}
|
|
641
|
+
});
|
|
642
|
+
// autofix — show full auto-fix attempt detail (diff, dry-run, knowledge)
|
|
643
|
+
scraps
|
|
644
|
+
.command('autofix <id>')
|
|
645
|
+
.description('Show the last auto-fix attempt for a scrap (decision, diff, dry-run, knowledge)')
|
|
646
|
+
.option('--json', 'Output raw autofix JSON')
|
|
647
|
+
.action(async (id, opts) => {
|
|
648
|
+
validateObjectId(id);
|
|
649
|
+
const result = await fetchRunAndFix(id);
|
|
650
|
+
if (!result) {
|
|
651
|
+
console.log(chalk.dim('No runs yet.'));
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
if (!result.fix) {
|
|
655
|
+
console.log(chalk.dim('No auto-fix attempt on the last run.'));
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
if (opts.json)
|
|
659
|
+
return json(result.fix);
|
|
660
|
+
console.log(formatAutofix(result.fix));
|
|
661
|
+
});
|
|
662
|
+
// snapshot — download the captured page HTML (or error-path HTML) for the last run
|
|
663
|
+
scraps
|
|
664
|
+
.command('snapshot <id>')
|
|
665
|
+
.description('Download captured page HTML for the last run of a scrap')
|
|
666
|
+
.option('--error', 'Fetch the error-path snapshot (errorSnapshot.html)')
|
|
667
|
+
.option('-o, --out <file>', 'Write HTML to a file instead of stdout')
|
|
668
|
+
.action(async (id, opts) => {
|
|
669
|
+
validateObjectId(id);
|
|
670
|
+
const scrap = await api.get(`/api/scraps/${id}`);
|
|
671
|
+
const hid = scrap.history?.[0]?._id;
|
|
672
|
+
if (!hid) {
|
|
673
|
+
console.log(chalk.dim('No runs yet.'));
|
|
674
|
+
return;
|
|
675
|
+
}
|
|
676
|
+
const kindParam = opts.error ? '?kind=error' : '';
|
|
677
|
+
const html = await api.getText(`/api/historys/${hid}/html-snapshot${kindParam}`);
|
|
678
|
+
if (opts.out) {
|
|
679
|
+
const { writeFile } = await import('fs/promises');
|
|
680
|
+
await writeFile(opts.out, html, 'utf8');
|
|
681
|
+
console.log(chalk.green(`✓ Snapshot written to ${opts.out}`));
|
|
682
|
+
}
|
|
683
|
+
else {
|
|
684
|
+
process.stdout.write(html);
|
|
685
|
+
}
|
|
686
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
import config from '../lib/config.js';
|
|
4
|
+
/**
|
|
5
|
+
* Decode the exp claim from a JWT (middle segment, base64url encoded JSON).
|
|
6
|
+
* Returns null if the payload cannot be decoded or has no exp field.
|
|
7
|
+
*/
|
|
8
|
+
function decodeExp(jwt) {
|
|
9
|
+
try {
|
|
10
|
+
const parts = jwt.split('.');
|
|
11
|
+
if (parts.length !== 3)
|
|
12
|
+
return null;
|
|
13
|
+
const payload = Buffer.from(parts[1], 'base64url').toString('utf8');
|
|
14
|
+
const parsed = JSON.parse(payload);
|
|
15
|
+
if (typeof parsed.exp !== 'number')
|
|
16
|
+
return null;
|
|
17
|
+
return parsed.exp;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export const token = new Command('token')
|
|
24
|
+
.description('Print the stored session JWT (for MCP Bearer auth)')
|
|
25
|
+
.action(() => {
|
|
26
|
+
const stored = config.get('token');
|
|
27
|
+
if (!stored) {
|
|
28
|
+
console.error(chalk.red('✗ Not logged in. Run: trawl login'));
|
|
29
|
+
process.exitCode = 1;
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const exp = decodeExp(stored);
|
|
33
|
+
const nowSeconds = Math.floor(Date.now() / 1000);
|
|
34
|
+
if (exp !== null && exp < nowSeconds) {
|
|
35
|
+
console.error(chalk.red('✗ Session token expired. Run: trawl login to refresh.'));
|
|
36
|
+
process.exitCode = 1;
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
// Print the raw token first (so it can be piped / copied)
|
|
40
|
+
console.log(stored);
|
|
41
|
+
if (exp === null) {
|
|
42
|
+
// Could not decode expiry (malformed/opaque JWT) — advisory only, do not block piping
|
|
43
|
+
console.error(chalk.dim(' (could not decode expiry — verify the token manually)'));
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
const secsLeft = exp - nowSeconds;
|
|
47
|
+
const daysLeft = secsLeft / 86400;
|
|
48
|
+
if (daysLeft < 1) {
|
|
49
|
+
const hoursLeft = Math.floor(secsLeft / 3600);
|
|
50
|
+
console.log(chalk.yellow(`⚠ Token expiring in ${hoursLeft}h. Run: trawl login to refresh.`));
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
const daysRounded = Math.floor(daysLeft);
|
|
54
|
+
console.log(chalk.dim(` Expires in ${daysRounded}d. Renew with: trawl login`));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import { login, logout } from './commands/login.js';
|
|
|
8
8
|
import { scraps } from './commands/scraps.js';
|
|
9
9
|
import { skills } from './commands/skills.js';
|
|
10
10
|
import { telemetry } from './commands/telemetry.js';
|
|
11
|
+
import { token } from './commands/token.js';
|
|
11
12
|
import { autoUpdateInstalledSkills } from './lib/skills.js';
|
|
12
13
|
import { initPostHog, captureCommand, shutdown } from './lib/posthog.js';
|
|
13
14
|
autoUpdateInstalledSkills();
|
|
@@ -38,6 +39,7 @@ program.addCommand(logout);
|
|
|
38
39
|
program.addCommand(scraps);
|
|
39
40
|
program.addCommand(skills);
|
|
40
41
|
program.addCommand(telemetry);
|
|
42
|
+
program.addCommand(token);
|
|
41
43
|
process.on('exit', () => {
|
|
42
44
|
void shutdown();
|
|
43
45
|
});
|
package/dist/lib/api.d.ts
CHANGED
package/dist/lib/api.js
CHANGED
|
@@ -133,8 +133,23 @@ async function publicPost(path, body) {
|
|
|
133
133
|
throw new Error('Invalid JSON in server response');
|
|
134
134
|
}
|
|
135
135
|
}
|
|
136
|
+
async function getText(path) {
|
|
137
|
+
const token = config.get('token');
|
|
138
|
+
if (!token)
|
|
139
|
+
throw new Error('Not logged in. Run: trawl login');
|
|
140
|
+
const url = `${config.get('apiUrl')}${path}`;
|
|
141
|
+
const res = await fetch(url, {
|
|
142
|
+
headers: {
|
|
143
|
+
'User-Agent': USER_AGENT,
|
|
144
|
+
Cookie: `TOKEN=${token}`,
|
|
145
|
+
},
|
|
146
|
+
});
|
|
147
|
+
await throwIfError(res);
|
|
148
|
+
return res.text();
|
|
149
|
+
}
|
|
136
150
|
export const api = {
|
|
137
151
|
get: (path) => request(path),
|
|
152
|
+
getText: (path) => getText(path),
|
|
138
153
|
post: (path, body) => request(path, {
|
|
139
154
|
method: 'POST',
|
|
140
155
|
body: body ? JSON.stringify(body) : undefined,
|