@trawlme/cli 1.14.0 → 1.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/commands/doctor.d.ts +94 -0
- package/dist/commands/doctor.js +140 -0
- package/dist/commands/login.js +4 -4
- package/dist/commands/scraps.js +194 -11
- package/dist/commands/token.d.ts +2 -0
- package/dist/commands/token.js +57 -0
- package/dist/index.js +2 -0
- package/dist/lib/api.d.ts +1 -0
- package/dist/lib/api.js +20 -5
- package/dist/lib/config.d.ts +8 -0
- package/dist/lib/config.js +11 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -117,9 +117,15 @@ TRAWL_TELEMETRY=0 trawl scraps list
|
|
|
117
117
|
| Variable | Description |
|
|
118
118
|
|--------------------|-------------------------------------------------------------------|
|
|
119
119
|
| `TRAWL_TOKEN` | JWT token — bypasses login prompt, useful for CI/CD |
|
|
120
|
-
| `TRAWL_API_URL` | Override the
|
|
120
|
+
| `TRAWL_API_URL` | Override the API base URL for the session (takes precedence over `trawl login --url`) |
|
|
121
121
|
| `TRAWL_TELEMETRY` | Set to `0` to disable telemetry for the current session |
|
|
122
122
|
|
|
123
|
+
Session override (no `trawl login` mutation, ideal for CI/QA against another env):
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
TRAWL_API_URL=https://dev.trawl.me TRAWL_TOKEN=<jwt> trawl scraps list --json
|
|
127
|
+
```
|
|
128
|
+
|
|
123
129
|
## API documentation
|
|
124
130
|
|
|
125
131
|
The Trawl API is documented at [trawl.me/api/spec.json](https://trawl.me/api/spec.json) (OpenAPI 3).
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run diagnostics shape — mirrors the owner-safe REST shape returned by
|
|
3
|
+
* GET /api/historys/:id (Phase 1 projection, 2026-05-27).
|
|
4
|
+
* Cost and proxy-nature fields are intentionally absent by design.
|
|
5
|
+
* Abstract proxyTier (Tier 0–4) is kept.
|
|
6
|
+
*/
|
|
7
|
+
export interface Run {
|
|
8
|
+
_id: string;
|
|
9
|
+
status: boolean | null;
|
|
10
|
+
statusDetail: 'success' | 'error' | 'empty' | 'regression' | null;
|
|
11
|
+
length?: number | null;
|
|
12
|
+
errorMessage?: string | null;
|
|
13
|
+
errorSnapshot?: {
|
|
14
|
+
selector?: string | null;
|
|
15
|
+
errorMessage?: string | null;
|
|
16
|
+
html?: string | null;
|
|
17
|
+
capturedAt?: string | null;
|
|
18
|
+
} | null;
|
|
19
|
+
emptyContext?: {
|
|
20
|
+
page?: {
|
|
21
|
+
url?: string;
|
|
22
|
+
title?: string;
|
|
23
|
+
totalAnchors?: number;
|
|
24
|
+
};
|
|
25
|
+
selectors?: Record<string, number>;
|
|
26
|
+
} | null;
|
|
27
|
+
blocked?: boolean;
|
|
28
|
+
proxyTier?: string | null;
|
|
29
|
+
regressionDetected?: boolean;
|
|
30
|
+
baselineLength?: number | null;
|
|
31
|
+
fixVersionId?: string | null;
|
|
32
|
+
createdAt?: string;
|
|
33
|
+
time?: number | null;
|
|
34
|
+
triggeredBy?: string | null;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Autofix activity metadata — from the persisted ai_fix_end activity.
|
|
38
|
+
* aiUsage (cost) is stripped server-side; all diagnostics are kept.
|
|
39
|
+
*/
|
|
40
|
+
export interface FixActivity {
|
|
41
|
+
outcome: 'applied' | 'failed' | 'skipped' | 'breaker_tripped' | 'timeout' | null;
|
|
42
|
+
classification?: string | null;
|
|
43
|
+
reason?: string | null;
|
|
44
|
+
fixDiff?: string | null;
|
|
45
|
+
dryRunResults?: Array<{
|
|
46
|
+
status: 'success' | 'error';
|
|
47
|
+
error?: string;
|
|
48
|
+
}> | null;
|
|
49
|
+
knowledgeUsed?: Array<{
|
|
50
|
+
fingerprint: string;
|
|
51
|
+
confidence?: number | null;
|
|
52
|
+
}> | null;
|
|
53
|
+
attempt?: number;
|
|
54
|
+
maxAttempts?: number;
|
|
55
|
+
versionId?: string | null;
|
|
56
|
+
createdAt?: string;
|
|
57
|
+
}
|
|
58
|
+
interface ScrapHead {
|
|
59
|
+
_id: string;
|
|
60
|
+
title: string;
|
|
61
|
+
history?: Array<{
|
|
62
|
+
_id?: string;
|
|
63
|
+
status?: boolean | null;
|
|
64
|
+
createdAt?: string;
|
|
65
|
+
}>;
|
|
66
|
+
}
|
|
67
|
+
export declare function pickRun(run: Run): Partial<Run>;
|
|
68
|
+
export declare function pickFix(fix: FixActivity | null): Partial<FixActivity> | null;
|
|
69
|
+
/**
|
|
70
|
+
* Pure formatter — returns a human-readable diagnosis string for a run.
|
|
71
|
+
* Used by `doctor`, `doctor --autofix`, and `data --errors`.
|
|
72
|
+
*
|
|
73
|
+
* @param scrapTitle - Display name of the scrap
|
|
74
|
+
* @param run - Owner-safe history run object
|
|
75
|
+
* @param fix - Optional autofix activity (null = no fix attempted)
|
|
76
|
+
* @param scrapId - Scrap document id (used in hint lines — autofix/snapshot take scrap id, not run id)
|
|
77
|
+
* @returns Multi-line string ready for console.log
|
|
78
|
+
*/
|
|
79
|
+
export declare function formatDoctor(scrapTitle: string, run: Run, fix?: FixActivity | null, scrapId?: string): string;
|
|
80
|
+
/**
|
|
81
|
+
* Pure formatter — returns a human-readable autofix detail block.
|
|
82
|
+
* Shows diff, dry-run results, knowledge consulted.
|
|
83
|
+
*/
|
|
84
|
+
export declare function formatAutofix(fix: FixActivity): string;
|
|
85
|
+
/**
|
|
86
|
+
* Fetch the latest run + its ai_fix_end activity for a scrap.
|
|
87
|
+
* Returns null if the scrap has no runs yet.
|
|
88
|
+
*/
|
|
89
|
+
export declare function fetchRunAndFix(scrapId: string): Promise<{
|
|
90
|
+
scrap: ScrapHead;
|
|
91
|
+
run: Run;
|
|
92
|
+
fix: FixActivity | null;
|
|
93
|
+
} | null>;
|
|
94
|
+
export {};
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { api } from '../lib/api.js';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
const TIER_LABELS = {
|
|
4
|
+
tier0: 'Tier 0',
|
|
5
|
+
tier1: 'Tier 1',
|
|
6
|
+
tier2: 'Tier 2',
|
|
7
|
+
tier3: 'Tier 3',
|
|
8
|
+
tier4: 'Tier 4',
|
|
9
|
+
};
|
|
10
|
+
const RUN_ALLOWLIST = [
|
|
11
|
+
'_id', 'status', 'statusDetail', 'length', 'errorMessage', 'errorSnapshot',
|
|
12
|
+
'emptyContext', 'blocked', 'proxyTier', 'regressionDetected', 'baselineLength',
|
|
13
|
+
'fixVersionId', 'createdAt', 'time', 'triggeredBy',
|
|
14
|
+
];
|
|
15
|
+
const FIX_ALLOWLIST = [
|
|
16
|
+
'outcome', 'classification', 'reason', 'fixDiff', 'dryRunResults',
|
|
17
|
+
'knowledgeUsed', 'attempt', 'maxAttempts', 'versionId', 'createdAt',
|
|
18
|
+
];
|
|
19
|
+
export function pickRun(run) {
|
|
20
|
+
return Object.fromEntries(RUN_ALLOWLIST
|
|
21
|
+
.filter((k) => k in run)
|
|
22
|
+
.map((k) => [k, run[k]]));
|
|
23
|
+
}
|
|
24
|
+
export function pickFix(fix) {
|
|
25
|
+
if (!fix)
|
|
26
|
+
return null;
|
|
27
|
+
return Object.fromEntries(FIX_ALLOWLIST
|
|
28
|
+
.filter((k) => k in fix)
|
|
29
|
+
.map((k) => [k, fix[k]]));
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Pure formatter — returns a human-readable diagnosis string for a run.
|
|
33
|
+
* Used by `doctor`, `doctor --autofix`, and `data --errors`.
|
|
34
|
+
*
|
|
35
|
+
* @param scrapTitle - Display name of the scrap
|
|
36
|
+
* @param run - Owner-safe history run object
|
|
37
|
+
* @param fix - Optional autofix activity (null = no fix attempted)
|
|
38
|
+
* @param scrapId - Scrap document id (used in hint lines — autofix/snapshot take scrap id, not run id)
|
|
39
|
+
* @returns Multi-line string ready for console.log
|
|
40
|
+
*/
|
|
41
|
+
export function formatDoctor(scrapTitle, run, fix = null, scrapId) {
|
|
42
|
+
const lines = [];
|
|
43
|
+
// Header + status badge
|
|
44
|
+
const ok = run.status === true;
|
|
45
|
+
const badge = ok
|
|
46
|
+
? chalk.green('● success')
|
|
47
|
+
: run.statusDetail === 'empty'
|
|
48
|
+
? chalk.yellow('● empty')
|
|
49
|
+
: chalk.red('● failed');
|
|
50
|
+
lines.push(`${chalk.bold(scrapTitle)} ${badge}${run.statusDetail ? ` (${run.statusDetail})` : ''}`);
|
|
51
|
+
lines.push(chalk.dim(` Run ID: ${run._id}`));
|
|
52
|
+
// Error message
|
|
53
|
+
const errMsg = run.errorMessage ?? run.errorSnapshot?.errorMessage;
|
|
54
|
+
if (errMsg) {
|
|
55
|
+
lines.push(chalk.dim(' Error: ') + chalk.red(errMsg));
|
|
56
|
+
}
|
|
57
|
+
// Failed selector
|
|
58
|
+
if (run.errorSnapshot?.selector) {
|
|
59
|
+
lines.push(chalk.dim(' Failed selector: ') + chalk.cyan(run.errorSnapshot.selector));
|
|
60
|
+
}
|
|
61
|
+
// Blocked
|
|
62
|
+
lines.push(chalk.dim(' Blocked: ') + (run.blocked ? chalk.red('yes') : 'no'));
|
|
63
|
+
// Proxy tier (abstract, kept for owner)
|
|
64
|
+
if (run.proxyTier && TIER_LABELS[run.proxyTier]) {
|
|
65
|
+
lines.push(chalk.dim(' Proxy: ') + TIER_LABELS[run.proxyTier]);
|
|
66
|
+
}
|
|
67
|
+
// Empty context detail
|
|
68
|
+
if (run.statusDetail === 'empty' && run.emptyContext?.page) {
|
|
69
|
+
const page = run.emptyContext.page;
|
|
70
|
+
lines.push(chalk.dim(' Empty context: ') + `url=${page.url ?? '?'} anchors=${page.totalAnchors ?? '?'}`);
|
|
71
|
+
}
|
|
72
|
+
// Regression detail
|
|
73
|
+
if (run.regressionDetected) {
|
|
74
|
+
lines.push(chalk.dim(' Regression: ') + `length ${run.length ?? '?'} vs baseline ${run.baselineLength ?? '?'}`);
|
|
75
|
+
}
|
|
76
|
+
// Autofix summary (when fix exists)
|
|
77
|
+
if (fix) {
|
|
78
|
+
lines.push(` ${chalk.magenta('Autofix:')} ${fix.outcome ?? '?'}${fix.classification ? ` — ${fix.classification}` : ''}${fix.reason ? ` (${fix.reason})` : ''}`);
|
|
79
|
+
lines.push(chalk.dim(` → full diff/dry-run/knowledge: trawl scraps autofix ${scrapId ?? run._id} (or doctor --autofix)`));
|
|
80
|
+
}
|
|
81
|
+
// Timestamp
|
|
82
|
+
if (run.createdAt) {
|
|
83
|
+
lines.push(chalk.dim(' Run at: ') + new Date(run.createdAt).toLocaleString());
|
|
84
|
+
}
|
|
85
|
+
// Snapshot hint
|
|
86
|
+
if (run.errorSnapshot?.html || run.statusDetail === 'empty' || run.statusDetail === 'error') {
|
|
87
|
+
lines.push('');
|
|
88
|
+
lines.push(chalk.dim(` → trawl scraps snapshot ${scrapId ?? run._id} --error (download error-path HTML)`));
|
|
89
|
+
}
|
|
90
|
+
return lines.join('\n');
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Pure formatter — returns a human-readable autofix detail block.
|
|
94
|
+
* Shows diff, dry-run results, knowledge consulted.
|
|
95
|
+
*/
|
|
96
|
+
export function formatAutofix(fix) {
|
|
97
|
+
const lines = [];
|
|
98
|
+
lines.push(chalk.magenta.bold('Autofix attempt'));
|
|
99
|
+
lines.push(` Decision: ${fix.outcome ?? '?'}${fix.classification ? ` — ${fix.classification}` : ''}${fix.reason ? ` (${fix.reason})` : ''}`);
|
|
100
|
+
if (fix.versionId)
|
|
101
|
+
lines.push(` Version: ${fix.versionId}`);
|
|
102
|
+
lines.push(' Fix diff:');
|
|
103
|
+
if (fix.fixDiff && fix.fixDiff.trim()) {
|
|
104
|
+
for (const l of fix.fixDiff.split('\n')) {
|
|
105
|
+
const c = l.startsWith('+') ? chalk.green(l) : l.startsWith('-') ? chalk.red(l) : chalk.dim(l);
|
|
106
|
+
lines.push(` ${c}`);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
lines.push(chalk.dim(' No code change — no-op recovery (site recovered between fail and dry-run).'));
|
|
111
|
+
}
|
|
112
|
+
if (fix.dryRunResults?.length) {
|
|
113
|
+
const ok = fix.dryRunResults.filter((d) => d.status === 'success').length;
|
|
114
|
+
lines.push(` Dry-run: ${ok}/${fix.dryRunResults.length} succeeded`);
|
|
115
|
+
fix.dryRunResults
|
|
116
|
+
.filter((d) => d.status === 'error' && d.error)
|
|
117
|
+
.forEach((d) => lines.push(chalk.red(` ✗ ${d.error}`)));
|
|
118
|
+
}
|
|
119
|
+
if (fix.knowledgeUsed?.length) {
|
|
120
|
+
lines.push(' Knowledge consulted:');
|
|
121
|
+
fix.knowledgeUsed.forEach((k) => lines.push(` • ${k.fingerprint}${k.confidence != null ? ` (${Math.round(k.confidence * 100)}%)` : ''}`));
|
|
122
|
+
}
|
|
123
|
+
return lines.join('\n');
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Fetch the latest run + its ai_fix_end activity for a scrap.
|
|
127
|
+
* Returns null if the scrap has no runs yet.
|
|
128
|
+
*/
|
|
129
|
+
export async function fetchRunAndFix(scrapId) {
|
|
130
|
+
const scrap = await api.get(`/api/scraps/${scrapId}`);
|
|
131
|
+
const hid = scrap.history?.[0]?._id;
|
|
132
|
+
if (!hid)
|
|
133
|
+
return null;
|
|
134
|
+
const run = await api.get(`/api/historys/${hid}`);
|
|
135
|
+
// api.get unwraps the { data: T } envelope — the response is the array directly
|
|
136
|
+
const acts = await api.get(`/api/scraps/${scrapId}/activities?history=${hid}&limit=5`);
|
|
137
|
+
const fixAct = Array.isArray(acts) ? acts.find((a) => a.type === 'ai_fix_end') : undefined;
|
|
138
|
+
const fix = fixAct ? { ...fixAct.metadata, createdAt: fixAct.createdAt } : null;
|
|
139
|
+
return { scrap, run, fix };
|
|
140
|
+
}
|
package/dist/commands/login.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import chalk from 'chalk';
|
|
3
|
-
import config from '../lib/config.js';
|
|
3
|
+
import config, { getApiUrl } from '../lib/config.js';
|
|
4
4
|
import { api } from '../lib/api.js';
|
|
5
5
|
import { requireJwt, requireUrl } from '../lib/validate.js';
|
|
6
6
|
import { promptPassword } from '../lib/prompt.js';
|
|
@@ -30,7 +30,7 @@ export const login = new Command('login')
|
|
|
30
30
|
if (envToken) {
|
|
31
31
|
config.set('token', requireJwt(envToken, 'TRAWL_TOKEN'));
|
|
32
32
|
console.log(chalk.green('✓ Logged in'));
|
|
33
|
-
console.log(chalk.dim(` API: ${
|
|
33
|
+
console.log(chalk.dim(` API: ${getApiUrl()}`));
|
|
34
34
|
console.log(chalk.dim(` Config: ${config.path}`));
|
|
35
35
|
return;
|
|
36
36
|
}
|
|
@@ -38,7 +38,7 @@ export const login = new Command('login')
|
|
|
38
38
|
if (opts.token) {
|
|
39
39
|
config.set('token', requireJwt(opts.token, '--token'));
|
|
40
40
|
console.log(chalk.green('✓ Logged in'));
|
|
41
|
-
console.log(chalk.dim(` API: ${
|
|
41
|
+
console.log(chalk.dim(` API: ${getApiUrl()}`));
|
|
42
42
|
console.log(chalk.dim(` Config: ${config.path}`));
|
|
43
43
|
return;
|
|
44
44
|
}
|
|
@@ -63,7 +63,7 @@ export const login = new Command('login')
|
|
|
63
63
|
const token = requireJwt(raw, 'token');
|
|
64
64
|
config.set('token', token);
|
|
65
65
|
console.log(chalk.green(`✓ Logged in as ${email}`));
|
|
66
|
-
console.log(chalk.dim(` API: ${
|
|
66
|
+
console.log(chalk.dim(` API: ${getApiUrl()}`));
|
|
67
67
|
console.log(chalk.dim(` Config: ${config.path}`));
|
|
68
68
|
});
|
|
69
69
|
export const logout = new Command('logout')
|
package/dist/commands/scraps.js
CHANGED
|
@@ -5,6 +5,7 @@ import { api } from '../lib/api.js';
|
|
|
5
5
|
import { table, json } from '../lib/format.js';
|
|
6
6
|
import { promptPassword } from '../lib/prompt.js';
|
|
7
7
|
import { validateObjectId } from '../lib/validate.js';
|
|
8
|
+
import { formatDoctor, formatAutofix, fetchRunAndFix, pickRun, pickFix } from './doctor.js';
|
|
8
9
|
function lastStatus(scrap) {
|
|
9
10
|
const last = scrap.history?.[0];
|
|
10
11
|
if (!last || last.status === null || last.status === undefined)
|
|
@@ -55,13 +56,51 @@ scraps
|
|
|
55
56
|
.description('List all scraps')
|
|
56
57
|
.option('--json', 'Output as JSON')
|
|
57
58
|
.option('--status <status>', 'Filter by last run status (success|failure|never)')
|
|
59
|
+
.option('--limit <n>', 'Show only the first N results', (v) => parseInt(v, 10))
|
|
60
|
+
.option('--page <n>', 'Fetch a specific page only (50 per page, no auto-pagination)', (v) => parseInt(v, 10))
|
|
58
61
|
.action(async (opts) => {
|
|
62
|
+
// Guard: --limit and --page are mutually exclusive
|
|
63
|
+
if (opts.limit !== undefined && opts.page !== undefined) {
|
|
64
|
+
console.log(chalk.red('✗ --limit and --page are mutually exclusive. Use one or the other.'));
|
|
65
|
+
process.exitCode = 1;
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
59
68
|
const spinner = ora('Fetching scraps…').start();
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
69
|
+
let data;
|
|
70
|
+
try {
|
|
71
|
+
if (opts.page !== undefined) {
|
|
72
|
+
// Single-page mode: explicit page requested, no loop
|
|
73
|
+
const pageNum = opts.page;
|
|
74
|
+
data = await api.get(`/api/scraps?perPage=50&page=${pageNum}`);
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
// Fetch-all mode: paginate until a page returns < 200 items
|
|
78
|
+
const perPage = 200;
|
|
79
|
+
data = [];
|
|
80
|
+
let page = 1;
|
|
81
|
+
while (true) {
|
|
82
|
+
const batch = await api.get(`/api/scraps?perPage=${perPage}&page=${page}`);
|
|
83
|
+
data = data.concat(batch);
|
|
84
|
+
if (batch.length < perPage)
|
|
85
|
+
break;
|
|
86
|
+
page++;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
spinner.stop();
|
|
92
|
+
console.log(chalk.red(`✗ Failed to fetch scraps: ${err.message}`));
|
|
93
|
+
process.exitCode = 1;
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
spinner.stop();
|
|
98
|
+
}
|
|
63
99
|
if (opts.status)
|
|
64
|
-
|
|
100
|
+
data = data.filter((s) => lastStatus(s) === opts.status);
|
|
101
|
+
const totalMatched = data.length;
|
|
102
|
+
const limit = opts.limit;
|
|
103
|
+
const rows = limit !== undefined ? data.slice(0, limit) : data;
|
|
65
104
|
if (opts.json)
|
|
66
105
|
return json(rows);
|
|
67
106
|
const tableRows = rows.map((s) => ({
|
|
@@ -73,6 +112,10 @@ scraps
|
|
|
73
112
|
updated: new Date(s.updatedAt).toLocaleDateString(),
|
|
74
113
|
}));
|
|
75
114
|
table(tableRows, ['id', 'title', 'cron', 'status', 'last run', 'updated']);
|
|
115
|
+
// Print footer when --limit truncates
|
|
116
|
+
if (limit !== undefined && rows.length < totalMatched) {
|
|
117
|
+
console.log(chalk.dim(`Showing ${rows.length} of ${totalMatched} — omit --limit to see all`));
|
|
118
|
+
}
|
|
76
119
|
});
|
|
77
120
|
// get
|
|
78
121
|
scraps
|
|
@@ -220,8 +263,25 @@ scraps
|
|
|
220
263
|
.command('data <id>')
|
|
221
264
|
.description('Get scrap data (latest run payload)')
|
|
222
265
|
.option('--json', 'Output as JSON')
|
|
266
|
+
.option('--errors', 'Show failure diagnostics when the last run failed')
|
|
223
267
|
.action(async (id, opts) => {
|
|
224
268
|
validateObjectId(id);
|
|
269
|
+
// --errors: fetch full run + fix detail via fetchRunAndFix (DRY with doctor)
|
|
270
|
+
if (opts.errors) {
|
|
271
|
+
const result = await fetchRunAndFix(id);
|
|
272
|
+
if (!result) {
|
|
273
|
+
console.log(chalk.dim('No runs yet.'));
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
if (result.run.status === true) {
|
|
277
|
+
console.log(chalk.green('✓ Last run succeeded. No errors to show.'));
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (opts.json)
|
|
281
|
+
return json(pickRun(result.run));
|
|
282
|
+
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
225
285
|
const loaded = await api.get(`/api/scraps/load/${id}`);
|
|
226
286
|
const items = loaded?.result?.data;
|
|
227
287
|
if (!Array.isArray(items)) {
|
|
@@ -232,14 +292,67 @@ scraps
|
|
|
232
292
|
return json(items);
|
|
233
293
|
console.log(chalk.bold('Last run data:'));
|
|
234
294
|
console.log(chalk.dim(` Items: ${items.length}`));
|
|
235
|
-
if (loaded?.result?._proxyTier) {
|
|
236
|
-
console.log(chalk.dim(` Proxy tier: ${loaded.result._proxyTier}`));
|
|
237
|
-
}
|
|
238
295
|
if (items.length > 0 && typeof items[0] === 'object' && items[0] !== null) {
|
|
239
296
|
console.log(chalk.dim(` First item keys: ${Object.keys(items[0]).join(', ')}`));
|
|
240
297
|
}
|
|
241
298
|
console.log(chalk.dim(' Use --json for full output.'));
|
|
242
299
|
});
|
|
300
|
+
// history — list past runs for a scrap
|
|
301
|
+
scraps
|
|
302
|
+
.command('history <id>')
|
|
303
|
+
.description('List past runs for a scrap (newest first)')
|
|
304
|
+
.option('--json', 'Output as JSON')
|
|
305
|
+
.option('-n, --limit <n>', 'Max runs to show', '20')
|
|
306
|
+
.action(async (id, opts) => {
|
|
307
|
+
validateObjectId(id);
|
|
308
|
+
const limit = Number(opts.limit);
|
|
309
|
+
if (!Number.isInteger(limit) || limit < 1) {
|
|
310
|
+
console.log(chalk.red(`✗ Invalid --limit "${opts.limit}" (expected a positive integer)`));
|
|
311
|
+
process.exitCode = 1;
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
const scrap = await api.get(`/api/scraps/${id}`);
|
|
315
|
+
const runs = (scrap.history ?? []).slice(0, limit).map((h) => ({
|
|
316
|
+
hid: h._id,
|
|
317
|
+
status: h.statusDetail ?? null,
|
|
318
|
+
time: h.time ?? null,
|
|
319
|
+
tier: h.proxyTier ?? null,
|
|
320
|
+
failureKind: h.failureKind ?? null,
|
|
321
|
+
blockType: h.blockType ?? null,
|
|
322
|
+
createdAt: h.createdAt ?? null,
|
|
323
|
+
}));
|
|
324
|
+
if (opts.json) {
|
|
325
|
+
json(runs);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
table(runs, ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'createdAt']);
|
|
329
|
+
});
|
|
330
|
+
// run-info — single-run detail by history id
|
|
331
|
+
scraps
|
|
332
|
+
.command('run-info <hid>')
|
|
333
|
+
.description('Show details of a single run (status, tier, failureKind, error)')
|
|
334
|
+
.option('--json', 'Output as JSON')
|
|
335
|
+
.action(async (hid, opts) => {
|
|
336
|
+
validateObjectId(hid);
|
|
337
|
+
const h = await api.get(`/api/historys/${hid}`);
|
|
338
|
+
const info = {
|
|
339
|
+
hid,
|
|
340
|
+
status: h.statusDetail ?? null,
|
|
341
|
+
time: h.time ?? null,
|
|
342
|
+
tier: h.proxyTier ?? null,
|
|
343
|
+
failureKind: h.failureKind ?? null,
|
|
344
|
+
blockType: h.blockType ?? null,
|
|
345
|
+
errorMessage: h.errorSnapshot?.errorMessage ?? null,
|
|
346
|
+
selector: h.errorSnapshot?.selector ?? null,
|
|
347
|
+
emptyContext: h.errorSnapshot?.emptyContext ?? null,
|
|
348
|
+
createdAt: h.createdAt ?? null,
|
|
349
|
+
};
|
|
350
|
+
if (opts.json) {
|
|
351
|
+
json(info);
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
table([info], ['hid', 'status', 'time', 'tier', 'failureKind', 'blockType', 'errorMessage', 'selector', 'emptyContext', 'createdAt']);
|
|
355
|
+
});
|
|
243
356
|
// delete
|
|
244
357
|
scraps
|
|
245
358
|
.command('delete <id>')
|
|
@@ -311,13 +424,18 @@ scraps
|
|
|
311
424
|
// trigger
|
|
312
425
|
scraps
|
|
313
426
|
.command('trigger <id>')
|
|
314
|
-
.description('Launch a scrap as a background worker')
|
|
427
|
+
.description('Launch a scrap as a background worker (returns immediately)')
|
|
315
428
|
.option('-w, --watch', 'Stream activities after triggering')
|
|
429
|
+
.option('--wait', 'Run synchronously and wait for the result (legacy behaviour)')
|
|
316
430
|
.action(async (id, opts) => {
|
|
317
431
|
validateObjectId(id);
|
|
318
|
-
const spinner = ora('Triggering worker…').start();
|
|
319
|
-
|
|
320
|
-
|
|
432
|
+
const spinner = ora(opts.wait ? 'Running worker…' : 'Triggering worker…').start();
|
|
433
|
+
// #50 — default async: the backend (#1313) kicks off the run and returns a
|
|
434
|
+
// 'queued' envelope immediately instead of holding the connection for the
|
|
435
|
+
// whole run. --wait restores the old synchronous round-trip.
|
|
436
|
+
const path = opts.wait ? `/api/scraps/worker/${id}` : `/api/scraps/worker/${id}?wait=false`;
|
|
437
|
+
await api.post(path);
|
|
438
|
+
spinner.succeed(opts.wait ? 'Worker run complete' : 'Worker triggered');
|
|
321
439
|
if (opts.watch)
|
|
322
440
|
await watchActivities(id);
|
|
323
441
|
});
|
|
@@ -501,3 +619,68 @@ account
|
|
|
501
619
|
}
|
|
502
620
|
console.log(`${credLine} | ${sessionLine}`);
|
|
503
621
|
});
|
|
622
|
+
// doctor — diagnose last run of a scrap (error, failed selector, block status, page state, autofix)
|
|
623
|
+
scraps
|
|
624
|
+
.command('doctor <id>')
|
|
625
|
+
.description('Diagnose the last run (error, failed selector, block status, page state, autofix outcome)')
|
|
626
|
+
.option('--json', 'Output raw run + autofix JSON')
|
|
627
|
+
.option('--autofix', 'Include the full autofix diff / dry-run / knowledge')
|
|
628
|
+
.action(async (id, opts) => {
|
|
629
|
+
validateObjectId(id);
|
|
630
|
+
const result = await fetchRunAndFix(id);
|
|
631
|
+
if (!result) {
|
|
632
|
+
console.log(chalk.dim('No runs yet.'));
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
if (opts.json)
|
|
636
|
+
return json({ run: pickRun(result.run), fix: pickFix(result.fix) });
|
|
637
|
+
console.log(formatDoctor(result.scrap.title, result.run, result.fix, id));
|
|
638
|
+
if (opts.autofix && result.fix) {
|
|
639
|
+
console.log('\n' + formatAutofix(result.fix));
|
|
640
|
+
}
|
|
641
|
+
});
|
|
642
|
+
// autofix — show full auto-fix attempt detail (diff, dry-run, knowledge)
|
|
643
|
+
scraps
|
|
644
|
+
.command('autofix <id>')
|
|
645
|
+
.description('Show the last auto-fix attempt for a scrap (decision, diff, dry-run, knowledge)')
|
|
646
|
+
.option('--json', 'Output raw autofix JSON')
|
|
647
|
+
.action(async (id, opts) => {
|
|
648
|
+
validateObjectId(id);
|
|
649
|
+
const result = await fetchRunAndFix(id);
|
|
650
|
+
if (!result) {
|
|
651
|
+
console.log(chalk.dim('No runs yet.'));
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
if (!result.fix) {
|
|
655
|
+
console.log(chalk.dim('No auto-fix attempt on the last run.'));
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
if (opts.json)
|
|
659
|
+
return json(result.fix);
|
|
660
|
+
console.log(formatAutofix(result.fix));
|
|
661
|
+
});
|
|
662
|
+
// snapshot — download the captured page HTML (or error-path HTML) for the last run
|
|
663
|
+
scraps
|
|
664
|
+
.command('snapshot <id>')
|
|
665
|
+
.description('Download captured page HTML for the last run of a scrap')
|
|
666
|
+
.option('--error', 'Fetch the error-path snapshot (errorSnapshot.html)')
|
|
667
|
+
.option('-o, --out <file>', 'Write HTML to a file instead of stdout')
|
|
668
|
+
.action(async (id, opts) => {
|
|
669
|
+
validateObjectId(id);
|
|
670
|
+
const scrap = await api.get(`/api/scraps/${id}`);
|
|
671
|
+
const hid = scrap.history?.[0]?._id;
|
|
672
|
+
if (!hid) {
|
|
673
|
+
console.log(chalk.dim('No runs yet.'));
|
|
674
|
+
return;
|
|
675
|
+
}
|
|
676
|
+
const kindParam = opts.error ? '?kind=error' : '';
|
|
677
|
+
const html = await api.getText(`/api/historys/${hid}/html-snapshot${kindParam}`);
|
|
678
|
+
if (opts.out) {
|
|
679
|
+
const { writeFile } = await import('fs/promises');
|
|
680
|
+
await writeFile(opts.out, html, 'utf8');
|
|
681
|
+
console.log(chalk.green(`✓ Snapshot written to ${opts.out}`));
|
|
682
|
+
}
|
|
683
|
+
else {
|
|
684
|
+
process.stdout.write(html);
|
|
685
|
+
}
|
|
686
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import chalk from 'chalk';
|
|
3
|
+
import config from '../lib/config.js';
|
|
4
|
+
/**
|
|
5
|
+
* Decode the exp claim from a JWT (middle segment, base64url encoded JSON).
|
|
6
|
+
* Returns null if the payload cannot be decoded or has no exp field.
|
|
7
|
+
*/
|
|
8
|
+
function decodeExp(jwt) {
|
|
9
|
+
try {
|
|
10
|
+
const parts = jwt.split('.');
|
|
11
|
+
if (parts.length !== 3)
|
|
12
|
+
return null;
|
|
13
|
+
const payload = Buffer.from(parts[1], 'base64url').toString('utf8');
|
|
14
|
+
const parsed = JSON.parse(payload);
|
|
15
|
+
if (typeof parsed.exp !== 'number')
|
|
16
|
+
return null;
|
|
17
|
+
return parsed.exp;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export const token = new Command('token')
|
|
24
|
+
.description('Print the stored session JWT (for MCP Bearer auth)')
|
|
25
|
+
.action(() => {
|
|
26
|
+
const stored = config.get('token');
|
|
27
|
+
if (!stored) {
|
|
28
|
+
console.error(chalk.red('✗ Not logged in. Run: trawl login'));
|
|
29
|
+
process.exitCode = 1;
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const exp = decodeExp(stored);
|
|
33
|
+
const nowSeconds = Math.floor(Date.now() / 1000);
|
|
34
|
+
if (exp !== null && exp < nowSeconds) {
|
|
35
|
+
console.error(chalk.red('✗ Session token expired. Run: trawl login to refresh.'));
|
|
36
|
+
process.exitCode = 1;
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
// Print the raw token first (so it can be piped / copied)
|
|
40
|
+
console.log(stored);
|
|
41
|
+
if (exp === null) {
|
|
42
|
+
// Could not decode expiry (malformed/opaque JWT) — advisory only, do not block piping
|
|
43
|
+
console.error(chalk.dim(' (could not decode expiry — verify the token manually)'));
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
const secsLeft = exp - nowSeconds;
|
|
47
|
+
const daysLeft = secsLeft / 86400;
|
|
48
|
+
if (daysLeft < 1) {
|
|
49
|
+
const hoursLeft = Math.floor(secsLeft / 3600);
|
|
50
|
+
console.log(chalk.yellow(`⚠ Token expiring in ${hoursLeft}h. Run: trawl login to refresh.`));
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
const daysRounded = Math.floor(daysLeft);
|
|
54
|
+
console.log(chalk.dim(` Expires in ${daysRounded}d. Renew with: trawl login`));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ import { login, logout } from './commands/login.js';
|
|
|
8
8
|
import { scraps } from './commands/scraps.js';
|
|
9
9
|
import { skills } from './commands/skills.js';
|
|
10
10
|
import { telemetry } from './commands/telemetry.js';
|
|
11
|
+
import { token } from './commands/token.js';
|
|
11
12
|
import { autoUpdateInstalledSkills } from './lib/skills.js';
|
|
12
13
|
import { initPostHog, captureCommand, shutdown } from './lib/posthog.js';
|
|
13
14
|
autoUpdateInstalledSkills();
|
|
@@ -38,6 +39,7 @@ program.addCommand(logout);
|
|
|
38
39
|
program.addCommand(scraps);
|
|
39
40
|
program.addCommand(skills);
|
|
40
41
|
program.addCommand(telemetry);
|
|
42
|
+
program.addCommand(token);
|
|
41
43
|
process.on('exit', () => {
|
|
42
44
|
void shutdown();
|
|
43
45
|
});
|
package/dist/lib/api.d.ts
CHANGED
package/dist/lib/api.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { dirname, resolve } from 'node:path';
|
|
4
|
-
import config from './config.js';
|
|
4
|
+
import config, { getApiUrl } from './config.js';
|
|
5
5
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
6
6
|
const pkg = JSON.parse(readFileSync(resolve(__dirname, '../../package.json'), 'utf8'));
|
|
7
7
|
const USER_AGENT = `@trawlme/cli/${pkg.version}`;
|
|
@@ -60,7 +60,7 @@ async function request(path, options = {}) {
|
|
|
60
60
|
const token = config.get('token');
|
|
61
61
|
if (!token)
|
|
62
62
|
throw new Error('Not logged in. Run: trawl login');
|
|
63
|
-
const url = `${
|
|
63
|
+
const url = `${getApiUrl()}${path}`;
|
|
64
64
|
const res = await fetch(url, {
|
|
65
65
|
...options,
|
|
66
66
|
headers: {
|
|
@@ -90,7 +90,7 @@ async function upload(path, formData) {
|
|
|
90
90
|
const token = config.get('token');
|
|
91
91
|
if (!token)
|
|
92
92
|
throw new Error('Not logged in. Run: trawl login');
|
|
93
|
-
const url = `${
|
|
93
|
+
const url = `${getApiUrl()}${path}`;
|
|
94
94
|
// Do NOT set Content-Type — fetch sets it automatically with the correct multipart boundary
|
|
95
95
|
const res = await fetch(url, {
|
|
96
96
|
method: 'POST',
|
|
@@ -117,7 +117,7 @@ async function upload(path, formData) {
|
|
|
117
117
|
}
|
|
118
118
|
}
|
|
119
119
|
async function publicPost(path, body) {
|
|
120
|
-
const url = `${
|
|
120
|
+
const url = `${getApiUrl()}${path}`;
|
|
121
121
|
const res = await fetch(url, {
|
|
122
122
|
method: 'POST',
|
|
123
123
|
headers: { 'Content-Type': 'application/json', 'User-Agent': USER_AGENT },
|
|
@@ -133,8 +133,23 @@ async function publicPost(path, body) {
|
|
|
133
133
|
throw new Error('Invalid JSON in server response');
|
|
134
134
|
}
|
|
135
135
|
}
|
|
136
|
+
async function getText(path) {
|
|
137
|
+
const token = config.get('token');
|
|
138
|
+
if (!token)
|
|
139
|
+
throw new Error('Not logged in. Run: trawl login');
|
|
140
|
+
const url = `${getApiUrl()}${path}`;
|
|
141
|
+
const res = await fetch(url, {
|
|
142
|
+
headers: {
|
|
143
|
+
'User-Agent': USER_AGENT,
|
|
144
|
+
Cookie: `TOKEN=${token}`,
|
|
145
|
+
},
|
|
146
|
+
});
|
|
147
|
+
await throwIfError(res);
|
|
148
|
+
return res.text();
|
|
149
|
+
}
|
|
136
150
|
export const api = {
|
|
137
151
|
get: (path) => request(path),
|
|
152
|
+
getText: (path) => getText(path),
|
|
138
153
|
post: (path, body) => request(path, {
|
|
139
154
|
method: 'POST',
|
|
140
155
|
body: body ? JSON.stringify(body) : undefined,
|
|
@@ -150,7 +165,7 @@ export const api = {
|
|
|
150
165
|
const token = config.get('token');
|
|
151
166
|
if (!token)
|
|
152
167
|
throw new Error('Not logged in. Run: trawl login');
|
|
153
|
-
const url = `${
|
|
168
|
+
const url = `${getApiUrl()}${path}`;
|
|
154
169
|
const res = await fetch(url, {
|
|
155
170
|
headers: {
|
|
156
171
|
Accept: 'text/event-stream',
|
package/dist/lib/config.d.ts
CHANGED
|
@@ -6,4 +6,12 @@ interface TrawlConfig {
|
|
|
6
6
|
telemetryUserId: string;
|
|
7
7
|
}
|
|
8
8
|
declare const config: Conf<TrawlConfig>;
|
|
9
|
+
/**
|
|
10
|
+
* Resolve the effective API base URL.
|
|
11
|
+
* Precedence: TRAWL_API_URL env > stored `trawl login --url` > default.
|
|
12
|
+
* Mirrors the TRAWL_TOKEN / TRAWL_TELEMETRY session-override pattern so scripts
|
|
13
|
+
* (e.g. infra /trawl-prod-qa against https://dev.trawl.me) can retarget the CLI
|
|
14
|
+
* without mutating the operator's persisted config. (#56)
|
|
15
|
+
*/
|
|
16
|
+
export declare function getApiUrl(): string;
|
|
9
17
|
export default config;
|
package/dist/lib/config.js
CHANGED
|
@@ -8,4 +8,15 @@ const config = new Conf({
|
|
|
8
8
|
telemetryUserId: '',
|
|
9
9
|
},
|
|
10
10
|
});
|
|
11
|
+
/**
|
|
12
|
+
* Resolve the effective API base URL.
|
|
13
|
+
* Precedence: TRAWL_API_URL env > stored `trawl login --url` > default.
|
|
14
|
+
* Mirrors the TRAWL_TOKEN / TRAWL_TELEMETRY session-override pattern so scripts
|
|
15
|
+
* (e.g. infra /trawl-prod-qa against https://dev.trawl.me) can retarget the CLI
|
|
16
|
+
* without mutating the operator's persisted config. (#56)
|
|
17
|
+
*/
|
|
18
|
+
export function getApiUrl() {
|
|
19
|
+
const override = process.env['TRAWL_API_URL']?.trim();
|
|
20
|
+
return override ? override : config.get('apiUrl');
|
|
21
|
+
}
|
|
11
22
|
export default config;
|