@trawlme/cli 3.9.1 → 3.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/commands/doctor.d.ts +25 -0
- package/dist/commands/doctor.js +24 -1
- package/dist/commands/login.js +58 -1
- package/dist/commands/scraps.js +58 -3
- package/dist/commands/spec.d.ts +32 -1
- package/dist/commands/spec.js +83 -11
- package/dist/index.d.ts +17 -0
- package/dist/index.js +61 -0
- package/dist/lib/api.d.ts +5 -0
- package/dist/lib/api.js +118 -0
- package/dist/lib/config.d.ts +1 -0
- package/dist/lib/config.js +1 -0
- package/dist/lib/docs.d.ts +140 -0
- package/dist/lib/docs.js +238 -0
- package/dist/lib/skills.d.ts +156 -0
- package/dist/lib/skills.js +227 -2
- package/dist/lib/skillsNudge.d.ts +33 -0
- package/dist/lib/skillsNudge.js +137 -0
- package/docs/agent-quickstart.md +9 -1
- package/package.json +1 -1
package/dist/lib/api.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
2
|
import { fileURLToPath } from 'node:url';
|
|
3
3
|
import { dirname, resolve } from 'node:path';
|
|
4
|
+
import { request as httpRequest } from 'node:http';
|
|
5
|
+
import { request as httpsRequest } from 'node:https';
|
|
4
6
|
import { getApiUrl, getToken, getAuthMode, getLiveAuthEnvVar } from './config.js';
|
|
5
7
|
import { parseServerJson } from './json.js';
|
|
6
8
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
@@ -518,6 +520,119 @@ async function publicGet(path, reqOpts = {}) {
|
|
|
518
520
|
throw new Error('Invalid JSON in server response');
|
|
519
521
|
}
|
|
520
522
|
}
|
|
523
|
+
/**
|
|
524
|
+
* A hard-bounded, PROBE-ONLY GET — never use this for a command's real work
|
|
525
|
+
* (no auth headers, no retry, no NetworkError-with-cause classification).
|
|
526
|
+
*
|
|
527
|
+
* trawl_cli#185 defect: every OTHER fetch in this file rides Node's global
|
|
528
|
+
* `fetch()` + `AbortSignal.timeout()`, which bounds the PROMISE, not the
|
|
529
|
+
* process. Aborting a fetch mid-connect does not necessarily tear down an
|
|
530
|
+
* in-flight TCP/TLS handshake — a documented Node/undici characteristic —
|
|
531
|
+
* so against a routable-but-silently-dropping host (a dropped SYN, never a
|
|
532
|
+
* RST — a realistic firewalled/air-gapped shape) the dangling socket keeps
|
|
533
|
+
* the event loop alive until Node's OWN internal connect-timeout eventually
|
|
534
|
+
* fires. Measured on this host: a bare `fetch()` + `AbortSignal.timeout(2000)`
|
|
535
|
+
* against such a host rejects its promise at ~2005ms, but the PROCESS does
|
|
536
|
+
* not exit until ~10.5s later — not configurable from here, and not paid by
|
|
537
|
+
* `ECONNREFUSED` (~0.13s) or a failed DNS lookup (~0.29s), only by the
|
|
538
|
+
* silent-drop shape. Every other command in this file accepts that risk
|
|
539
|
+
* because the user explicitly asked for that network call (`trawl ping`
|
|
540
|
+
* hanging on an unreachable host is the command doing its job — nothing to
|
|
541
|
+
* fix there). This primitive exists for the one case where the network call
|
|
542
|
+
* ITSELF is optional: `spec --json`'s own doc-discovery probe (the one
|
|
543
|
+
* command an agent runs to orient itself) has no business costing 10.5s to
|
|
544
|
+
* learn a host is unreachable, so this manages the raw socket directly.
|
|
545
|
+
*
|
|
546
|
+
* The timeout here is an INDEPENDENT `setTimeout` that calls `req.destroy()`
|
|
547
|
+
* — deliberately NOT `http.request`'s own `timeout` option (idle-based via
|
|
548
|
+
* `socket.setTimeout`, so it only fires when zero bytes arrive for that
|
|
549
|
+
* long — a coincidentally-same condition against a silent black hole, but a
|
|
550
|
+
* dishonest bound in general: it would never fire against a host that
|
|
551
|
+
* connects instantly then trickles bytes just often enough to reset the
|
|
552
|
+
* idle clock). Destroying the request/socket on this timer is what actually
|
|
553
|
+
* frees the OS-level handle so the process can exit promptly instead of
|
|
554
|
+
* waiting out Node's own multi-second connect-timeout — verified empirically
|
|
555
|
+
* (same black-holed host): total process time dropped from ~10.5s to ~2.0s
|
|
556
|
+
* with this change.
|
|
557
|
+
*
|
|
558
|
+
* Resolves to `null` on ANY failure (timeout, refusal, non-2xx, malformed
|
|
559
|
+
* JSON, an unsupported protocol) — never throws — so the one caller
|
|
560
|
+
* (spec.ts's fetchExternalDocsUrl) needs no try/catch of its own. This is
|
|
561
|
+
* NOT automatic: `node:http`/`node:https`' own `request()` throws
|
|
562
|
+
* SYNCHRONOUSLY (`ERR_INVALID_PROTOCOL`) for a URL whose protocol doesn't
|
|
563
|
+
* match the module (e.g. a stray `ftp://` in a misconfigured `TRAWL_API_URL`)
|
|
564
|
+
* — inside a Promise executor a synchronous throw becomes a REJECTED
|
|
565
|
+
* promise, which would have propagated straight through `fetchExternalDocsUrl`
|
|
566
|
+
* and turned `spec --json` into an error envelope instead of the spec.
|
|
567
|
+
* Reproduced: `TRAWL_API_URL=ftp://x node dist/index.js spec --json` failed
|
|
568
|
+
* entirely before this guard was added. The explicit protocol check below
|
|
569
|
+
* closes that.
|
|
570
|
+
*
|
|
571
|
+
* Two more deliberate departures from every other call in this file:
|
|
572
|
+
* - `TRAWL_TIMEOUT` (getTimeoutMs's env override, #91) is NOT consulted
|
|
573
|
+
* here — `opts.timeoutMs` is absolute. An operator raising that ceiling
|
|
574
|
+
* for a genuinely long-running command (e.g. LONG_RUN_TIMEOUT_MS's 300s)
|
|
575
|
+
* must never make an OPTIONAL probe wait 300s too; the two knobs measure
|
|
576
|
+
* different things and must not share a dial.
|
|
577
|
+
* - Redirects are NOT followed (global `fetch()`, used everywhere else in
|
|
578
|
+
* this file, follows them by default; raw `http`/`https` `request()`
|
|
579
|
+
* does not). For this probe a 3xx is just another non-2xx -> `null` ->
|
|
580
|
+
* fall through to rung 2 — an acceptable degradation, not a bug.
|
|
581
|
+
*/
|
|
582
|
+
function probeJson(path, opts) {
|
|
583
|
+
return new Promise((settle) => {
|
|
584
|
+
let url;
|
|
585
|
+
try {
|
|
586
|
+
url = new URL(`${getApiUrl()}${path}`);
|
|
587
|
+
}
|
|
588
|
+
catch {
|
|
589
|
+
settle(null);
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
593
|
+
settle(null);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
let settled = false;
|
|
597
|
+
const finish = (value) => {
|
|
598
|
+
if (settled)
|
|
599
|
+
return;
|
|
600
|
+
settled = true;
|
|
601
|
+
clearTimeout(timer);
|
|
602
|
+
settle(value);
|
|
603
|
+
};
|
|
604
|
+
const transport = url.protocol === 'http:' ? httpRequest : httpsRequest;
|
|
605
|
+
const req = transport(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
|
606
|
+
if (!res.statusCode || res.statusCode < 200 || res.statusCode >= 300) {
|
|
607
|
+
res.resume(); // drain so the socket can be released
|
|
608
|
+
finish(null);
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
let body = '';
|
|
612
|
+
res.setEncoding('utf8');
|
|
613
|
+
res.on('data', (chunk) => {
|
|
614
|
+
body += chunk;
|
|
615
|
+
});
|
|
616
|
+
res.on('end', () => {
|
|
617
|
+
try {
|
|
618
|
+
finish(JSON.parse(body));
|
|
619
|
+
}
|
|
620
|
+
catch {
|
|
621
|
+
finish(null);
|
|
622
|
+
}
|
|
623
|
+
});
|
|
624
|
+
res.on('error', () => finish(null));
|
|
625
|
+
});
|
|
626
|
+
// The independent, non-idle-based ceiling this function exists for —
|
|
627
|
+
// fires regardless of connect/response state and forcibly destroys the
|
|
628
|
+
// socket, unlike AbortSignal.timeout() everywhere else in this file.
|
|
629
|
+
const timer = setTimeout(() => {
|
|
630
|
+
req.destroy(new Error(`probe to ${url.href} timed out after ${opts.timeoutMs}ms`));
|
|
631
|
+
}, opts.timeoutMs);
|
|
632
|
+
req.on('error', () => finish(null));
|
|
633
|
+
req.end();
|
|
634
|
+
});
|
|
635
|
+
}
|
|
521
636
|
async function getText(path, reqOpts = {}) {
|
|
522
637
|
const token = getToken();
|
|
523
638
|
if (!token)
|
|
@@ -537,6 +652,9 @@ async function getText(path, reqOpts = {}) {
|
|
|
537
652
|
export const api = {
|
|
538
653
|
get: (path, opts) => request(path, {}, opts),
|
|
539
654
|
publicGet: (path, opts) => publicGet(path, opts),
|
|
655
|
+
/** See `probeJson`'s own doc comment — hard-bounded, probe-only, never for
|
|
656
|
+
* a command's real work. Today's one caller: spec.ts's fetchExternalDocsUrl. */
|
|
657
|
+
probeJson: (path, opts) => probeJson(path, opts),
|
|
540
658
|
getText: (path, opts) => getText(path, opts),
|
|
541
659
|
post: (path, body, opts) => request(path, {
|
|
542
660
|
method: 'POST',
|
package/dist/lib/config.d.ts
CHANGED
package/dist/lib/config.js
CHANGED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trawl_cli#185 — ONE source of truth for the docs URL(s) an agent (or
|
|
3
|
+
* human) can be pointed at from three surfaces: `spec --json` (top-level
|
|
4
|
+
* `docsUrl`/`llmsUrl` + a per-command `docs` deep link), a run's JSON
|
|
5
|
+
* payload (`doctor`/`data --errors`/`run-info`, keyed on `failureKind` —
|
|
6
|
+
* NOT errors.ts's unrelated `ErrorEnvelope.kind`, see the doc comment on
|
|
7
|
+
* `FAILURE_KIND_DOC_PATHS` below), and the `--help` footer. Three hardcoded
|
|
8
|
+
* lists here would diverge within months — that exact "same fact computed
|
|
9
|
+
* in two places" defect has recurred repeatedly across this epic — so every
|
|
10
|
+
* surface reads these same tables/functions, never its own copy.
|
|
11
|
+
*
|
|
12
|
+
* Resolution ladder (issue #185 — all three rungs REQUIRED, in this order,
|
|
13
|
+
* evaluated PER FIELD, never as one bundled decision — see `resolveDocsUrls`):
|
|
14
|
+
*
|
|
15
|
+
* 1. Prefer the server. `externalDocs.url` on the OpenAPI document at
|
|
16
|
+
* `<apiBase>/api/spec.json` is the standard OpenAPI field for exactly
|
|
17
|
+
* this, and reading it makes a self-hosted install work with ZERO CLI
|
|
18
|
+
* change. It is unset (null) server-side today, so this is
|
|
19
|
+
* forward-looking — build it anyway, and it must win outright over rung
|
|
20
|
+
* 2 when present. The one caller allowed to fetch it is `spec.ts`'s own
|
|
21
|
+
* action, bounded and swallowed-on-failure (mirrors lib/tips.ts's
|
|
22
|
+
* `isReferralProgramUserFacing`) — a deliberate, once-per-invocation
|
|
23
|
+
* agent probe, not "every command". Every other surface (an error
|
|
24
|
+
* payload on an arbitrary failing command, the `--help` footer) must
|
|
25
|
+
* never add a network call or a failure mode to a command nobody asked
|
|
26
|
+
* to hit the docs host for — see `resolveDocsUrls`'s `externalDocsUrl`
|
|
27
|
+
* parameter, which only ever arrives pre-fetched.
|
|
28
|
+
* 2. Else derive, by stripping a leading `api.` host label from the
|
|
29
|
+
* configured API base — but ONLY for a KNOWN first-party `trawl.me`
|
|
30
|
+
* host. `api.trawl.me` -> `trawl.me` (prod: API and docs are genuinely
|
|
31
|
+
* on different hosts); `dev.trawl.me` (no `api.` prefix) -> unchanged
|
|
32
|
+
* (dev serves docs on the SAME host as its API). A generic, unscoped
|
|
33
|
+
* `api.`-strip applied to ANY host is itself a guess — it assumes a
|
|
34
|
+
* self-hosted `api.acme.internal` serves docs at `acme.internal`, which
|
|
35
|
+
* nothing here can know. Scoping the strip to `trawl.me` is what makes
|
|
36
|
+
* rung 3 (below) ever actually fire for a self-hosted base.
|
|
37
|
+
* 3. Else OMIT the field entirely. Never emit a guessed URL (issue's Rule
|
|
38
|
+
* 3): a wrong URL sends an agent to a 404 WITH CONFIDENCE, worse than no
|
|
39
|
+
* URL at all — a self-hosted install with no server-declared
|
|
40
|
+
* `externalDocs` and a base outside `trawl.me` gets no `docsUrl`/
|
|
41
|
+
* `llmsUrl`, not a hopeful default pointed at OUR docs host.
|
|
42
|
+
*
|
|
43
|
+
* `llmsUrl` has no OpenAPI-standard field to read (rung 1 contributes
|
|
44
|
+
* nothing to it) — deriving it from the ORIGIN of a server-declared
|
|
45
|
+
* `externalDocs.url` would itself be a guess for a self-hosted install
|
|
46
|
+
* (rule 3 again), so `llmsUrl` resolves ONLY via rung 2 (the known-host
|
|
47
|
+
* derivation) or omission. It never rides along with a rung-1 `docsUrl`.
|
|
48
|
+
*
|
|
49
|
+
* PROVENANCE (defect fix, reviewer repro against a live mock server): the
|
|
50
|
+
* top-level `docsUrl` above is deliberately the FLATTENED "whichever rung
|
|
51
|
+
* won" value — right for a human reading `spec --json`'s top-level field,
|
|
52
|
+
* WRONG as an input to `resolveCommandDocsUrl`/`resolveFailureKindDocsUrl`
|
|
53
|
+
* below. Those two append one of THIS CLI's own hardcoded guide slugs
|
|
54
|
+
* (`COMMAND_DOC_PATHS`/`FAILURE_KIND_DOC_PATHS`) onto whatever `docsUrl`
|
|
55
|
+
* resolved — safe onto a rung-2 root we derived ourselves (we know
|
|
56
|
+
* trawl.me's guide tree), never safe onto a rung-1 root (an arbitrary
|
|
57
|
+
* third party's own docs site, self-hosted, with no reason to carry our
|
|
58
|
+
* slugs). A mock server declaring `externalDocs.url: 'http://h/guide'`
|
|
59
|
+
* used to get `http://h/guide/build-your-scrap/account-sessions` appended
|
|
60
|
+
* — a confidently-wrong 404. So `DocsUrls.docsUrlIsDerived` carries rung
|
|
61
|
+
* provenance ALONGSIDE the string (never flattened to a bare string again)
|
|
62
|
+
* and both deep-link functions now take the whole `DocsUrls`-shaped object
|
|
63
|
+
* and gate construction on that flag — see its own doc comment below.
|
|
64
|
+
*/
|
|
65
|
+
export interface DocsUrls {
|
|
66
|
+
docsUrl?: string;
|
|
67
|
+
llmsUrl?: string;
|
|
68
|
+
/**
|
|
69
|
+
* True iff `docsUrl` came from rung 2 (THIS CLI's own derivation from a
|
|
70
|
+
* KNOWN `trawl.me` host) — the only case where it is safe to append one of
|
|
71
|
+
* this CLI's own guide slugs onto it (see `resolveCommandDocsUrl`/
|
|
72
|
+
* `resolveFailureKindDocsUrl`). False when `docsUrl` is a rung-1
|
|
73
|
+
* SERVER-declared root instead — an arbitrary third party's own docs site,
|
|
74
|
+
* which has no reason to host our guide slugs, even when that server
|
|
75
|
+
* happens to run on a `trawl.me` host. Always a defined boolean whenever
|
|
76
|
+
* `docsUrl` itself is present; undefined only when `docsUrl` is undefined
|
|
77
|
+
* too (rung 3 — nothing resolved at all).
|
|
78
|
+
*/
|
|
79
|
+
docsUrlIsDerived?: boolean;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Rung 2 — see the module doc comment. Returns `{ protocol, host }` for a
|
|
83
|
+
* KNOWN first-party API base (preserving the base's own protocol rather
|
|
84
|
+
* than assuming `https:`), or `null` when the base isn't recognized
|
|
85
|
+
* (self-hosted, a custom domain, `localhost`, an unparseable string, …) —
|
|
86
|
+
* `null` must propagate to omission (rung 3), never to a guessed host.
|
|
87
|
+
*/
|
|
88
|
+
export declare function deriveDocsOrigin(apiBaseUrl: string): {
|
|
89
|
+
protocol: string;
|
|
90
|
+
host: string;
|
|
91
|
+
} | null;
|
|
92
|
+
/** Same rung as `deriveDocsOrigin`, collapsed to just the host string —
|
|
93
|
+
* convenience for a caller that only needs the derived host, not the
|
|
94
|
+
* protocol (kept as its own export since `docs.test.ts` exercises the host
|
|
95
|
+
* derivation independently of protocol handling). */
|
|
96
|
+
export declare function deriveDocsHost(apiBaseUrl: string): string | null;
|
|
97
|
+
/**
|
|
98
|
+
* The full ladder, PER FIELD (see module doc comment for why `llmsUrl`
|
|
99
|
+
* cannot ride along with a rung-1 `docsUrl`). Pure and synchronous — no
|
|
100
|
+
* network, safe to call from any surface (an error payload, the `--help`
|
|
101
|
+
* footer) without adding latency or a new failure mode. `externalDocsUrl`
|
|
102
|
+
* is rung 1's input: only ever supplied by `spec.ts`'s action, after its
|
|
103
|
+
* own bounded, swallowed-on-failure fetch — every other caller omits it and
|
|
104
|
+
* gets rungs 2/3 only.
|
|
105
|
+
*/
|
|
106
|
+
export declare function resolveDocsUrls(opts: {
|
|
107
|
+
apiBaseUrl: string;
|
|
108
|
+
externalDocsUrl?: string | null;
|
|
109
|
+
}): DocsUrls;
|
|
110
|
+
/**
|
|
111
|
+
* `resolveCommandDocsUrl` returns `undefined` (never a bare path or a
|
|
112
|
+
* relative link) whenever any of three things is missing — `docsUrl`
|
|
113
|
+
* unresolved (rung 3 already fired), `docsUrl` resolved but NOT derived
|
|
114
|
+
* (rung 1 — a server-declared root; see `DocsUrls.docsUrlIsDerived`'s doc
|
|
115
|
+
* comment for why appending our own guide slug onto a third party's root is
|
|
116
|
+
* exactly the confidently-wrong-404 defect this gate closes), or no guide is
|
|
117
|
+
* mapped for this command — so a spec consumer never has to special-case a
|
|
118
|
+
* partial value. Takes the whole resolved `DocsUrls` object (never a bare
|
|
119
|
+
* string) specifically so this provenance can never again be flattened away
|
|
120
|
+
* before it reaches here.
|
|
121
|
+
*/
|
|
122
|
+
export declare function resolveCommandDocsUrl(commandName: string, docs: Pick<DocsUrls, 'docsUrl' | 'docsUrlIsDerived'>): string | undefined;
|
|
123
|
+
/** Same "undefined unless docsUrl is resolved AND derived (rung 2)" gate as
|
|
124
|
+
* `resolveCommandDocsUrl` above, same reason — never append this CLI's own
|
|
125
|
+
* guide slug onto a rung-1 server-declared root. Callers MUST gate the call
|
|
126
|
+
* on their own staleness rule first (e.g. `doctor.ts`'s `isAuthWall`) — this
|
|
127
|
+
* function only knows the string-to-path mapping, not whether a stamped
|
|
128
|
+
* `failureKind` is still live on a run patched afterward. */
|
|
129
|
+
export declare function resolveFailureKindDocsUrl(kind: string | null | undefined, docs: Pick<DocsUrls, 'docsUrl' | 'docsUrlIsDerived'>): string | undefined;
|
|
130
|
+
/**
|
|
131
|
+
* `--help` footer text (issue #185 scope item 3) — a single dim line, for
|
|
132
|
+
* humans, e.g. "Docs: https://trawl.me/docs". Returns `undefined` (never an
|
|
133
|
+
* empty or guessed line) when no `docsUrl` resolved — mirrors
|
|
134
|
+
* lib/tips.ts's shape (a pure function separated from IO, so it's testable
|
|
135
|
+
* without mocking chalk/console) but NOT its TTY gate: tips.ts suppresses a
|
|
136
|
+
* promotional nudge from piped output, but a docs line inside `--help |
|
|
137
|
+
* less` is exactly the kind of thing worth keeping. Un-colored here — the
|
|
138
|
+
* caller (index.ts) applies `chalk.dim` so this stays trivially testable.
|
|
139
|
+
*/
|
|
140
|
+
export declare function docsFooterLine(docsUrl?: string): string | undefined;
|
package/dist/lib/docs.js
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* trawl_cli#185 — ONE source of truth for the docs URL(s) an agent (or
|
|
3
|
+
* human) can be pointed at from three surfaces: `spec --json` (top-level
|
|
4
|
+
* `docsUrl`/`llmsUrl` + a per-command `docs` deep link), a run's JSON
|
|
5
|
+
* payload (`doctor`/`data --errors`/`run-info`, keyed on `failureKind` —
|
|
6
|
+
* NOT errors.ts's unrelated `ErrorEnvelope.kind`, see the doc comment on
|
|
7
|
+
* `FAILURE_KIND_DOC_PATHS` below), and the `--help` footer. Three hardcoded
|
|
8
|
+
* lists here would diverge within months — that exact "same fact computed
|
|
9
|
+
* in two places" defect has recurred repeatedly across this epic — so every
|
|
10
|
+
* surface reads these same tables/functions, never its own copy.
|
|
11
|
+
*
|
|
12
|
+
* Resolution ladder (issue #185 — all three rungs REQUIRED, in this order,
|
|
13
|
+
* evaluated PER FIELD, never as one bundled decision — see `resolveDocsUrls`):
|
|
14
|
+
*
|
|
15
|
+
* 1. Prefer the server. `externalDocs.url` on the OpenAPI document at
|
|
16
|
+
* `<apiBase>/api/spec.json` is the standard OpenAPI field for exactly
|
|
17
|
+
* this, and reading it makes a self-hosted install work with ZERO CLI
|
|
18
|
+
* change. It is unset (null) server-side today, so this is
|
|
19
|
+
* forward-looking — build it anyway, and it must win outright over rung
|
|
20
|
+
* 2 when present. The one caller allowed to fetch it is `spec.ts`'s own
|
|
21
|
+
* action, bounded and swallowed-on-failure (mirrors lib/tips.ts's
|
|
22
|
+
* `isReferralProgramUserFacing`) — a deliberate, once-per-invocation
|
|
23
|
+
* agent probe, not "every command". Every other surface (an error
|
|
24
|
+
* payload on an arbitrary failing command, the `--help` footer) must
|
|
25
|
+
* never add a network call or a failure mode to a command nobody asked
|
|
26
|
+
* to hit the docs host for — see `resolveDocsUrls`'s `externalDocsUrl`
|
|
27
|
+
* parameter, which only ever arrives pre-fetched.
|
|
28
|
+
* 2. Else derive, by stripping a leading `api.` host label from the
|
|
29
|
+
* configured API base — but ONLY for a KNOWN first-party `trawl.me`
|
|
30
|
+
* host. `api.trawl.me` -> `trawl.me` (prod: API and docs are genuinely
|
|
31
|
+
* on different hosts); `dev.trawl.me` (no `api.` prefix) -> unchanged
|
|
32
|
+
* (dev serves docs on the SAME host as its API). A generic, unscoped
|
|
33
|
+
* `api.`-strip applied to ANY host is itself a guess — it assumes a
|
|
34
|
+
* self-hosted `api.acme.internal` serves docs at `acme.internal`, which
|
|
35
|
+
* nothing here can know. Scoping the strip to `trawl.me` is what makes
|
|
36
|
+
* rung 3 (below) ever actually fire for a self-hosted base.
|
|
37
|
+
* 3. Else OMIT the field entirely. Never emit a guessed URL (issue's Rule
|
|
38
|
+
* 3): a wrong URL sends an agent to a 404 WITH CONFIDENCE, worse than no
|
|
39
|
+
* URL at all — a self-hosted install with no server-declared
|
|
40
|
+
* `externalDocs` and a base outside `trawl.me` gets no `docsUrl`/
|
|
41
|
+
* `llmsUrl`, not a hopeful default pointed at OUR docs host.
|
|
42
|
+
*
|
|
43
|
+
* `llmsUrl` has no OpenAPI-standard field to read (rung 1 contributes
|
|
44
|
+
* nothing to it) — deriving it from the ORIGIN of a server-declared
|
|
45
|
+
* `externalDocs.url` would itself be a guess for a self-hosted install
|
|
46
|
+
* (rule 3 again), so `llmsUrl` resolves ONLY via rung 2 (the known-host
|
|
47
|
+
* derivation) or omission. It never rides along with a rung-1 `docsUrl`.
|
|
48
|
+
*
|
|
49
|
+
* PROVENANCE (defect fix, reviewer repro against a live mock server): the
|
|
50
|
+
* top-level `docsUrl` above is deliberately the FLATTENED "whichever rung
|
|
51
|
+
* won" value — right for a human reading `spec --json`'s top-level field,
|
|
52
|
+
* WRONG as an input to `resolveCommandDocsUrl`/`resolveFailureKindDocsUrl`
|
|
53
|
+
* below. Those two append one of THIS CLI's own hardcoded guide slugs
|
|
54
|
+
* (`COMMAND_DOC_PATHS`/`FAILURE_KIND_DOC_PATHS`) onto whatever `docsUrl`
|
|
55
|
+
* resolved — safe onto a rung-2 root we derived ourselves (we know
|
|
56
|
+
* trawl.me's guide tree), never safe onto a rung-1 root (an arbitrary
|
|
57
|
+
* third party's own docs site, self-hosted, with no reason to carry our
|
|
58
|
+
* slugs). A mock server declaring `externalDocs.url: 'http://h/guide'`
|
|
59
|
+
* used to get `http://h/guide/build-your-scrap/account-sessions` appended
|
|
60
|
+
* — a confidently-wrong 404. So `DocsUrls.docsUrlIsDerived` carries rung
|
|
61
|
+
* provenance ALONGSIDE the string (never flattened to a bare string again)
|
|
62
|
+
* and both deep-link functions now take the whole `DocsUrls`-shaped object
|
|
63
|
+
* and gate construction on that flag — see its own doc comment below.
|
|
64
|
+
*/
|
|
65
|
+
/** The one first-party domain this CLI knows to serve docs — see rung 2 in
|
|
66
|
+
* the module doc comment above. Deliberately not "any host", so an
|
|
67
|
+
* unrecognized (self-hosted/custom) base falls through to omission. */
|
|
68
|
+
const KNOWN_DOCS_DOMAIN = 'trawl.me';
|
|
69
|
+
/**
|
|
70
|
+
* Rung 2 — see the module doc comment. Returns `{ protocol, host }` for a
|
|
71
|
+
* KNOWN first-party API base (preserving the base's own protocol rather
|
|
72
|
+
* than assuming `https:`), or `null` when the base isn't recognized
|
|
73
|
+
* (self-hosted, a custom domain, `localhost`, an unparseable string, …) —
|
|
74
|
+
* `null` must propagate to omission (rung 3), never to a guessed host.
|
|
75
|
+
*/
|
|
76
|
+
export function deriveDocsOrigin(apiBaseUrl) {
|
|
77
|
+
let url;
|
|
78
|
+
try {
|
|
79
|
+
url = new URL(apiBaseUrl);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const hostname = url.hostname.toLowerCase();
|
|
85
|
+
const stripped = hostname.startsWith('api.') ? hostname.slice('api.'.length) : hostname;
|
|
86
|
+
if (stripped === KNOWN_DOCS_DOMAIN || stripped.endsWith(`.${KNOWN_DOCS_DOMAIN}`)) {
|
|
87
|
+
return { protocol: url.protocol, host: stripped };
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
/** Same rung as `deriveDocsOrigin`, collapsed to just the host string —
|
|
92
|
+
* convenience for a caller that only needs the derived host, not the
|
|
93
|
+
* protocol (kept as its own export since `docs.test.ts` exercises the host
|
|
94
|
+
* derivation independently of protocol handling). */
|
|
95
|
+
export function deriveDocsHost(apiBaseUrl) {
|
|
96
|
+
return deriveDocsOrigin(apiBaseUrl)?.host ?? null;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* The full ladder, PER FIELD (see module doc comment for why `llmsUrl`
|
|
100
|
+
* cannot ride along with a rung-1 `docsUrl`). Pure and synchronous — no
|
|
101
|
+
* network, safe to call from any surface (an error payload, the `--help`
|
|
102
|
+
* footer) without adding latency or a new failure mode. `externalDocsUrl`
|
|
103
|
+
* is rung 1's input: only ever supplied by `spec.ts`'s action, after its
|
|
104
|
+
* own bounded, swallowed-on-failure fetch — every other caller omits it and
|
|
105
|
+
* gets rungs 2/3 only.
|
|
106
|
+
*/
|
|
107
|
+
export function resolveDocsUrls(opts) {
|
|
108
|
+
const origin = deriveDocsOrigin(opts.apiBaseUrl);
|
|
109
|
+
const derived = origin
|
|
110
|
+
? {
|
|
111
|
+
docsUrl: `${origin.protocol}//${origin.host}/docs`,
|
|
112
|
+
llmsUrl: `${origin.protocol}//${origin.host}/llms.txt`,
|
|
113
|
+
docsUrlIsDerived: true,
|
|
114
|
+
}
|
|
115
|
+
: {};
|
|
116
|
+
if (opts.externalDocsUrl) {
|
|
117
|
+
// Rung 1 wins for docsUrl outright; llmsUrl stays rung-2-only (see doc
|
|
118
|
+
// comment) — a self-hosted `externalDocsUrl` outside `trawl.me` yields
|
|
119
|
+
// `{ docsUrl: <server's own> }` with `llmsUrl` omitted, never guessed.
|
|
120
|
+
// `docsUrlIsDerived: false` regardless of whether rung 2 ALSO resolved
|
|
121
|
+
// (e.g. a `trawl.me`-hosted server declaring its own externalDocs) — a
|
|
122
|
+
// server-declared root is never "derived" by THIS CLI, and never a safe
|
|
123
|
+
// base for our own guide slugs (see DocsUrls.docsUrlIsDerived).
|
|
124
|
+
return derived.llmsUrl
|
|
125
|
+
? { docsUrl: opts.externalDocsUrl, llmsUrl: derived.llmsUrl, docsUrlIsDerived: false }
|
|
126
|
+
: { docsUrl: opts.externalDocsUrl, docsUrlIsDerived: false };
|
|
127
|
+
}
|
|
128
|
+
return derived;
|
|
129
|
+
}
|
|
130
|
+
/** Join a docs root (e.g. `https://trawl.me/docs`) with a guide path
|
|
131
|
+
* suffix (e.g. `build-your-scrap/account-sessions`) — plain string
|
|
132
|
+
* concatenation, deliberately NOT `new URL(suffix, docsUrl)`: a
|
|
133
|
+
* leading-slash suffix resolved against a URL with its own path component
|
|
134
|
+
* (`/docs`) would resolve relative to the ORIGIN, silently dropping
|
|
135
|
+
* `/docs` from the result (`new URL('/x', 'https://h/docs')` ==
|
|
136
|
+
* `https://h/x`, not `https://h/docs/x`). */
|
|
137
|
+
function joinDocsPath(docsUrl, suffix) {
|
|
138
|
+
return `${docsUrl.replace(/\/+$/, '')}/${suffix.replace(/^\/+/, '')}`;
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* The single source for the account-sessions guide slug — referenced by
|
|
142
|
+
* BOTH `COMMAND_DOC_PATHS` and `FAILURE_KIND_DOC_PATHS` below. Those two
|
|
143
|
+
* tables key on different fields (a CLI command's full path name vs a run's
|
|
144
|
+
* `failureKind`) and legitimately stay separate, but they name the SAME
|
|
145
|
+
* guide, so the path literal itself must exist exactly once: this is the
|
|
146
|
+
* "same fact computed in two places" defect class this module's own doc
|
|
147
|
+
* comment says it exists to prevent, and it has recurred repeatedly across
|
|
148
|
+
* this epic. Renaming the guide used to require editing two literals in
|
|
149
|
+
* lockstep — miss one and one surface (a run's `docs` field, or `spec
|
|
150
|
+
* --json`'s per-command deep link) silently 404s while the other still
|
|
151
|
+
* works (trawl_cli#185 review).
|
|
152
|
+
*/
|
|
153
|
+
const ACCOUNT_SESSIONS_GUIDE_PATH = 'build-your-scrap/account-sessions';
|
|
154
|
+
/**
|
|
155
|
+
* Per-command deep links (issue #185 scope item 1) — a command's FULL path
|
|
156
|
+
* name (matches `CliSpecCommand.name` in spec.ts, e.g.
|
|
157
|
+
* `"scraps account session set"`) is matched against `prefix` either
|
|
158
|
+
* exactly or as a whole path SEGMENT prefix (`startsWith(prefix + ' ')`) —
|
|
159
|
+
* never a bare substring, so a hypothetical future `scraps accounting`
|
|
160
|
+
* command could never false-match the `scraps account` entry below.
|
|
161
|
+
*/
|
|
162
|
+
const COMMAND_DOC_PATHS = Object.freeze([
|
|
163
|
+
// Every account/session-management command — set, delete, clear-session,
|
|
164
|
+
// status, and the nested `session set` — is covered by one prefix entry
|
|
165
|
+
// rather than one row per leaf, so a new leaf added under `scraps account`
|
|
166
|
+
// later (e.g. a future `session capture`, trawl_cli#183) inherits the
|
|
167
|
+
// link for free instead of needing its own row.
|
|
168
|
+
{ prefix: 'scraps account', path: ACCOUNT_SESSIONS_GUIDE_PATH },
|
|
169
|
+
]);
|
|
170
|
+
function docsPathForCommand(commandName) {
|
|
171
|
+
const match = COMMAND_DOC_PATHS.find((e) => commandName === e.prefix || commandName.startsWith(`${e.prefix} `));
|
|
172
|
+
return match ? match.path : null;
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* `resolveCommandDocsUrl` returns `undefined` (never a bare path or a
|
|
176
|
+
* relative link) whenever any of three things is missing — `docsUrl`
|
|
177
|
+
* unresolved (rung 3 already fired), `docsUrl` resolved but NOT derived
|
|
178
|
+
* (rung 1 — a server-declared root; see `DocsUrls.docsUrlIsDerived`'s doc
|
|
179
|
+
* comment for why appending our own guide slug onto a third party's root is
|
|
180
|
+
* exactly the confidently-wrong-404 defect this gate closes), or no guide is
|
|
181
|
+
* mapped for this command — so a spec consumer never has to special-case a
|
|
182
|
+
* partial value. Takes the whole resolved `DocsUrls` object (never a bare
|
|
183
|
+
* string) specifically so this provenance can never again be flattened away
|
|
184
|
+
* before it reaches here.
|
|
185
|
+
*/
|
|
186
|
+
export function resolveCommandDocsUrl(commandName, docs) {
|
|
187
|
+
if (!docs.docsUrl || !docs.docsUrlIsDerived)
|
|
188
|
+
return undefined;
|
|
189
|
+
const path = docsPathForCommand(commandName);
|
|
190
|
+
return path ? joinDocsPath(docs.docsUrl, path) : undefined;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* failureKind -> guide path (issue #185 scope item 2). Keyed on trawl_node's
|
|
194
|
+
* run-level `failureKind` field (trawl_node#1975, surfaced by this CLI's
|
|
195
|
+
* own `doctor`/`data --errors`/`run-info` JSON payloads since cli#182) —
|
|
196
|
+
* this is a DIFFERENT axis from errors.ts's `ErrorEnvelope.kind` (a CLI
|
|
197
|
+
* transport/execution outcome like `"auth"` meaning "you're not logged
|
|
198
|
+
* into trawl", `"network"`, `"usage"`, …). Both happen to use the string
|
|
199
|
+
* `"auth"` for unrelated things: this table's `'auth'` means "the SCRAPED
|
|
200
|
+
* TARGET site showed a login wall", errors.ts's `'auth'` means "the TRAWL
|
|
201
|
+
* API rejected your OWN credentials". Never merge these two tables — they
|
|
202
|
+
* are keyed off different fields on different objects, and conflating them
|
|
203
|
+
* would either miss the run-level guide or wrongly attach it to an
|
|
204
|
+
* unrelated CLI auth failure.
|
|
205
|
+
*/
|
|
206
|
+
const FAILURE_KIND_DOC_PATHS = Object.freeze({
|
|
207
|
+
auth: ACCOUNT_SESSIONS_GUIDE_PATH,
|
|
208
|
+
});
|
|
209
|
+
function docsPathForFailureKind(kind) {
|
|
210
|
+
if (!kind)
|
|
211
|
+
return null;
|
|
212
|
+
return FAILURE_KIND_DOC_PATHS[kind] ?? null;
|
|
213
|
+
}
|
|
214
|
+
/** Same "undefined unless docsUrl is resolved AND derived (rung 2)" gate as
|
|
215
|
+
* `resolveCommandDocsUrl` above, same reason — never append this CLI's own
|
|
216
|
+
* guide slug onto a rung-1 server-declared root. Callers MUST gate the call
|
|
217
|
+
* on their own staleness rule first (e.g. `doctor.ts`'s `isAuthWall`) — this
|
|
218
|
+
* function only knows the string-to-path mapping, not whether a stamped
|
|
219
|
+
* `failureKind` is still live on a run patched afterward. */
|
|
220
|
+
export function resolveFailureKindDocsUrl(kind, docs) {
|
|
221
|
+
if (!docs.docsUrl || !docs.docsUrlIsDerived)
|
|
222
|
+
return undefined;
|
|
223
|
+
const path = docsPathForFailureKind(kind);
|
|
224
|
+
return path ? joinDocsPath(docs.docsUrl, path) : undefined;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* `--help` footer text (issue #185 scope item 3) — a single dim line, for
|
|
228
|
+
* humans, e.g. "Docs: https://trawl.me/docs". Returns `undefined` (never an
|
|
229
|
+
* empty or guessed line) when no `docsUrl` resolved — mirrors
|
|
230
|
+
* lib/tips.ts's shape (a pure function separated from IO, so it's testable
|
|
231
|
+
* without mocking chalk/console) but NOT its TTY gate: tips.ts suppresses a
|
|
232
|
+
* promotional nudge from piped output, but a docs line inside `--help |
|
|
233
|
+
* less` is exactly the kind of thing worth keeping. Un-colored here — the
|
|
234
|
+
* caller (index.ts) applies `chalk.dim` so this stays trivially testable.
|
|
235
|
+
*/
|
|
236
|
+
export function docsFooterLine(docsUrl) {
|
|
237
|
+
return docsUrl ? `Docs: ${docsUrl}` : undefined;
|
|
238
|
+
}
|