@agentproto/corpus-cli 0.1.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +37 -0
- package/dist/cli.mjs +2607 -0
- package/dist/cli.mjs.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.mjs +172 -0
- package/dist/index.mjs.map +1 -0
- package/dist/os-identity.adapter-CLBmnRoD.d.ts +83 -0
- package/dist/ports/index.d.ts +531 -0
- package/dist/ports/index.mjs +1129 -0
- package/dist/ports/index.mjs.map +1 -0
- package/package.json +76 -0
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
export { N as NodeFsAdapter, a as NodeFsAdapterOptions, O as OsIdentityAdapter, b as OsIdentityAdapterOptions } from '../os-identity.adapter-CLBmnRoD.js';
|
|
2
|
+
import { FetcherPort, FetchedSource, DistillPort, DistillInput, DistilledItem, SinkPort, SinkItem, SinkPushResult } from '@agentproto/corpus';
|
|
3
|
+
export { buildDistillPrompt, parseItems } from '@agentproto/corpus';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* BrowserMcpFetcher — a FetcherPort that extracts ARTICLE text through
|
|
8
|
+
* the user's authenticated browser (chrome-devtools-mcp). Use it for
|
|
9
|
+
* pages that a plain HTTP fetch can't read: login-walled, consent-walled,
|
|
10
|
+
* or JS-rendered (SPA) content. For public static articles the cheaper
|
|
11
|
+
* HttpReadabilityFetcher suffices; for video, YtDlpWhisperFetcher handles
|
|
12
|
+
* transcription — this fetcher does NOT touch video.
|
|
13
|
+
*
|
|
14
|
+
* navigate_page(url) → evaluate_script(readability) → { title, text }
|
|
15
|
+
*
|
|
16
|
+
* (It used to also scrape YouTube captions, but YouTube's caption APIs
|
|
17
|
+
* are pot/SABR-gated and unreliable; transcription via YtDlpWhisperFetcher
|
|
18
|
+
* replaced that path entirely.)
|
|
19
|
+
*
|
|
20
|
+
* The browser is injected as a minimal structural `BrowserMcpLike` (just
|
|
21
|
+
* `callTool`, matching `@agentproto/driver-mcp`'s `McpClient`), so this
|
|
22
|
+
* adapter has no hard MCP-SDK dependency and is unit-testable with a fake
|
|
23
|
+
* client. The CLI wires the real chrome-devtools-mcp client.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/** Minimal MCP client surface — matches `@agentproto/driver-mcp`'s `McpClient`. */
|
|
27
|
+
interface BrowserMcpLike {
|
|
28
|
+
callTool(args: {
|
|
29
|
+
name: string;
|
|
30
|
+
arguments: Record<string, unknown>;
|
|
31
|
+
}): Promise<{
|
|
32
|
+
content?: unknown;
|
|
33
|
+
structuredContent?: unknown;
|
|
34
|
+
isError?: boolean;
|
|
35
|
+
}>;
|
|
36
|
+
}
|
|
37
|
+
interface BrowserMcpFetcherOptions {
|
|
38
|
+
readonly browser: BrowserMcpLike;
|
|
39
|
+
/** chrome-devtools-mcp tool names — overridable if a host renames them. */
|
|
40
|
+
readonly navigateToolName?: string;
|
|
41
|
+
readonly evaluateToolName?: string;
|
|
42
|
+
}
|
|
43
|
+
declare class BrowserMcpFetcher implements FetcherPort {
|
|
44
|
+
private readonly browser;
|
|
45
|
+
private readonly navigateTool;
|
|
46
|
+
private readonly evaluateTool;
|
|
47
|
+
constructor(opts: BrowserMcpFetcherOptions);
|
|
48
|
+
fetch(url: string): Promise<FetchedSource | null>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* ScrapeMcpFetcher — a FetcherPort that delegates article/page fetching to
|
|
53
|
+
* any MCP server exposing a `scrape(url)` tool (e.g. the browser project's
|
|
54
|
+
* tiered router: HTTP → Chromium → Camofox stealth → agent, with cookie
|
|
55
|
+
* injection and auto-escalation on bot-walls). The browser stack is reached
|
|
56
|
+
* over MCP, so this stays vendor-neutral — corpus-cli never imports it.
|
|
57
|
+
*
|
|
58
|
+
* scrape(url) → { title, markdown } → FetchedSource
|
|
59
|
+
*
|
|
60
|
+
* It does NOT touch video URLs (YtDlpWhisperFetcher owns those) and returns
|
|
61
|
+
* `null` for a blocked / empty page so the importer skips-with-warning
|
|
62
|
+
* rather than aborting the batch.
|
|
63
|
+
*
|
|
64
|
+
* The MCP client is injected as the minimal structural `BrowserMcpLike`
|
|
65
|
+
* (just `callTool`), the same surface `BrowserMcpFetcher` consumes — so this
|
|
66
|
+
* adapter has no MCP-SDK dependency and is unit-testable with a fake client.
|
|
67
|
+
*/
|
|
68
|
+
|
|
69
|
+
interface ScrapeMcpFetcherOptions {
|
|
70
|
+
readonly client: BrowserMcpLike;
|
|
71
|
+
/** Tool name on the MCP server — overridable if a host renames it. */
|
|
72
|
+
readonly scrapeToolName?: string;
|
|
73
|
+
}
|
|
74
|
+
declare class ScrapeMcpFetcher implements FetcherPort {
|
|
75
|
+
private readonly client;
|
|
76
|
+
private readonly toolName;
|
|
77
|
+
constructor(opts: ScrapeMcpFetcherOptions);
|
|
78
|
+
fetch(url: string): Promise<FetchedSource | null>;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* SttPort — speech-to-text boundary the YtDlpWhisperFetcher consumes.
|
|
83
|
+
*
|
|
84
|
+
* Keeps the fetcher pure of any STT-vendor specifics: it hands a local
|
|
85
|
+
* audio file path to `transcribe()` and gets back text. The CLI wires a
|
|
86
|
+
* concrete impl (OpenAI Whisper here; swap for a self-hosted whisper.cpp
|
|
87
|
+
* or Deepgram impl without touching the fetcher).
|
|
88
|
+
*/
|
|
89
|
+
interface Transcript {
|
|
90
|
+
readonly text: string;
|
|
91
|
+
/** BCP-47 language, when the engine detects it. */
|
|
92
|
+
readonly language?: string;
|
|
93
|
+
}
|
|
94
|
+
interface SttPort {
|
|
95
|
+
transcribe(audioPath: string): Promise<Transcript>;
|
|
96
|
+
}
|
|
97
|
+
interface OpenAiWhisperSttOptions {
|
|
98
|
+
readonly apiKey: string;
|
|
99
|
+
/** Default "whisper-1". */
|
|
100
|
+
readonly model?: string;
|
|
101
|
+
/** Override base URL (proxy / Azure). Default OpenAI. */
|
|
102
|
+
readonly baseUrl?: string;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* OpenAI Whisper STT. Posts the audio file to /audio/transcriptions with
|
|
106
|
+
* `response_format=verbose_json` so we recover the detected language.
|
|
107
|
+
*
|
|
108
|
+
* Files above OpenAI's 25 MB cap throw a clear error — the caller should
|
|
109
|
+
* have downloaded at a lower bitrate (the fetcher uses 64 kbps mono) or
|
|
110
|
+
* chunk long media. We surface the size rather than letting the API 413.
|
|
111
|
+
*/
|
|
112
|
+
declare class OpenAiWhisperStt implements SttPort {
|
|
113
|
+
private readonly apiKey;
|
|
114
|
+
private readonly model;
|
|
115
|
+
private readonly baseUrl;
|
|
116
|
+
constructor(opts: OpenAiWhisperSttOptions);
|
|
117
|
+
transcribe(audioPath: string): Promise<Transcript>;
|
|
118
|
+
/**
|
|
119
|
+
* POST the audio with bounded retry. Transcribing a long talk fans out
|
|
120
|
+
* into many segment uploads (see ChunkedStt), so a single transient
|
|
121
|
+
* network blip ("fetch failed") or a 429 / 5xx must not abort the whole
|
|
122
|
+
* batch. Network errors and retryable statuses back off and retry;
|
|
123
|
+
* client errors (401 auth, 400 bad request) return immediately so a real
|
|
124
|
+
* misconfiguration still surfaces loudly.
|
|
125
|
+
*/
|
|
126
|
+
private postWithRetry;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* YtDlpWhisperFetcher — a FetcherPort for video URLs that does NOT rely
|
|
131
|
+
* on captions (YouTube's caption/transcript APIs are pot/SABR-gated and
|
|
132
|
+
* frequently unavailable). Instead it pulls the audio with yt-dlp and
|
|
133
|
+
* transcribes it with an injected SttPort (Whisper). Captions-independent
|
|
134
|
+
* and robust: works whether or not the video has subtitles.
|
|
135
|
+
*
|
|
136
|
+
* video URL → yt-dlp -f bestaudio -x (mp3) → SttPort.transcribe → text
|
|
137
|
+
*
|
|
138
|
+
* yt-dlp handles the pot token / SABR negotiation / signature itself, so
|
|
139
|
+
* no browser or cookies are needed for public videos. Non-video URLs
|
|
140
|
+
* resolve to `null` so this fetcher composes ahead of a readability
|
|
141
|
+
* fetcher (see CompositeFetcher).
|
|
142
|
+
*
|
|
143
|
+
* The yt-dlp invocation is injected as `download` so the adapter is unit-
|
|
144
|
+
* testable with a fake downloader + fake SttPort — no binary, no network.
|
|
145
|
+
*/
|
|
146
|
+
|
|
147
|
+
interface AudioDownload {
|
|
148
|
+
readonly audioPath: string;
|
|
149
|
+
readonly title: string;
|
|
150
|
+
readonly language?: string;
|
|
151
|
+
/** Remove the temp working dir. Always called by the fetcher. */
|
|
152
|
+
cleanup(): Promise<void>;
|
|
153
|
+
}
|
|
154
|
+
/** Pull a URL's audio to a local file. Injectable for tests. */
|
|
155
|
+
type AudioDownloader = (url: string) => Promise<AudioDownload>;
|
|
156
|
+
interface YtDlpWhisperFetcherOptions {
|
|
157
|
+
readonly stt: SttPort;
|
|
158
|
+
/** Defaults to a yt-dlp subprocess downloader. */
|
|
159
|
+
readonly download?: AudioDownloader;
|
|
160
|
+
/** Path to the yt-dlp binary (default: "yt-dlp" on PATH). */
|
|
161
|
+
readonly ytDlpBin?: string;
|
|
162
|
+
/** Treat these hostnames as video (in addition to the YouTube/Vimeo set). */
|
|
163
|
+
readonly extraVideoHosts?: readonly string[];
|
|
164
|
+
/**
|
|
165
|
+
* Skip videos longer than this many seconds, BEFORE downloading
|
|
166
|
+
* (yt-dlp `--match-filter "duration <= N"`). A filtered video pulls no
|
|
167
|
+
* bytes and resolves to `null` (skipped), so a stray 40-hour stream
|
|
168
|
+
* can't blow up the batch. Omit for no cap — long media is segmented by
|
|
169
|
+
* `ChunkedStt`, so length alone is not a blocker.
|
|
170
|
+
*/
|
|
171
|
+
readonly maxDurationSec?: number;
|
|
172
|
+
/**
|
|
173
|
+
* Read cookies from a local browser (`chrome`, `firefox`, `chrome:Profile 1`)
|
|
174
|
+
* — passed to yt-dlp as `--cookies-from-browser`. Authenticated requests
|
|
175
|
+
* sidestep YouTube's "confirm you're not a bot" rate-limit that public
|
|
176
|
+
* fetches hit after a burst of downloads.
|
|
177
|
+
*/
|
|
178
|
+
readonly cookiesFromBrowser?: string;
|
|
179
|
+
/** Path to a Netscape cookies.txt — passed to yt-dlp as `--cookies`. */
|
|
180
|
+
readonly cookiesFile?: string;
|
|
181
|
+
}
|
|
182
|
+
declare class YtDlpWhisperFetcher implements FetcherPort {
|
|
183
|
+
private readonly stt;
|
|
184
|
+
private readonly download;
|
|
185
|
+
private readonly extraHosts;
|
|
186
|
+
constructor(opts: YtDlpWhisperFetcherOptions);
|
|
187
|
+
fetch(url: string): Promise<FetchedSource | null>;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* AssemblyAiStt — a diarizing SttPort. Where OpenAiWhisperStt returns a
|
|
192
|
+
* flat transcript, this returns SPEAKER-LABELLED text (`Speaker A: … /
|
|
193
|
+
* Speaker B: …`) via AssemblyAI's `speaker_labels`. Use it ad-hoc for
|
|
194
|
+
* multi-speaker videos (interviews, mock interviews, panels, "day in the
|
|
195
|
+
* life") where the content only makes sense if you know who said what.
|
|
196
|
+
*
|
|
197
|
+
* Same `SttPort` contract — the diarized layout lives in `text`, so it
|
|
198
|
+
* drops straight into YtDlpWhisperFetcher without any other change.
|
|
199
|
+
*
|
|
200
|
+
* Flow: upload audio → create transcript(speaker_labels) → poll → format
|
|
201
|
+
* utterances. No 25 MB cap (unlike Whisper), so long videos are fine; the
|
|
202
|
+
* trade-off is latency (~a third of audio duration) — slower than Whisper.
|
|
203
|
+
*/
|
|
204
|
+
|
|
205
|
+
interface AssemblyAiSttOptions {
|
|
206
|
+
readonly apiKey: string;
|
|
207
|
+
readonly baseUrl?: string;
|
|
208
|
+
readonly pollIntervalMs?: number;
|
|
209
|
+
readonly maxWaitMs?: number;
|
|
210
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
211
|
+
}
|
|
212
|
+
declare class AssemblyAiStt implements SttPort {
|
|
213
|
+
private readonly apiKey;
|
|
214
|
+
private readonly baseUrl;
|
|
215
|
+
private readonly pollIntervalMs;
|
|
216
|
+
private readonly maxWaitMs;
|
|
217
|
+
private readonly sleep;
|
|
218
|
+
constructor(opts: AssemblyAiSttOptions);
|
|
219
|
+
transcribe(audioPath: string): Promise<Transcript>;
|
|
220
|
+
private post;
|
|
221
|
+
private get;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* ChunkedStt — a SttPort decorator that makes a capped engine handle
|
|
226
|
+
* arbitrarily long audio. Whisper rejects files over 25 MB, so a
|
|
227
|
+
* multi-hour talk (full course, masterclass, panel) can't be sent in one
|
|
228
|
+
* request. This wrapper splits oversized audio into time segments with
|
|
229
|
+
* ffmpeg, transcribes each through the wrapped engine, and concatenates
|
|
230
|
+
* the text back into a single transcript.
|
|
231
|
+
*
|
|
232
|
+
* transcribe(path) → ≤cap ? base.transcribe(path)
|
|
233
|
+
* : ffmpeg segment → base.transcribe(eachPart) → join
|
|
234
|
+
*
|
|
235
|
+
* The cap is a vendor concern (Whisper's limit), so handling it belongs in
|
|
236
|
+
* the STT layer rather than the fetcher — the fetcher stays pure and the
|
|
237
|
+
* concrete `OpenAiWhisperStt` stays a single-request transcriber. Splitting
|
|
238
|
+
* is injected as `split` so the decorator is unit-testable with a fake
|
|
239
|
+
* splitter + fake base engine — no ffmpeg, no network.
|
|
240
|
+
*
|
|
241
|
+
* Segmenting uses `-c copy` (stream copy, no re-encode): fast, and each
|
|
242
|
+
* segment is a self-contained valid file. At the fetcher's 64 kbps a 20-min
|
|
243
|
+
* segment is ~9.6 MB — comfortably under the 25 MB cap with margin for
|
|
244
|
+
* VBR drift.
|
|
245
|
+
*/
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Split a local audio file into ≤`segmentSeconds` parts inside `outDir`,
|
|
249
|
+
* returning the segment paths in playback order. Injectable for tests.
|
|
250
|
+
*/
|
|
251
|
+
type AudioSplitter = (audioPath: string, segmentSeconds: number, outDir: string) => Promise<string[]>;
|
|
252
|
+
interface ChunkedSttOptions {
|
|
253
|
+
/** The engine each (sub-cap) part is transcribed through. */
|
|
254
|
+
readonly base: SttPort;
|
|
255
|
+
/**
|
|
256
|
+
* Split anything strictly larger than this. Default 24 MB — just under
|
|
257
|
+
* Whisper's 25 MB hard cap so VBR drift never pushes a part over.
|
|
258
|
+
*/
|
|
259
|
+
readonly maxBytes?: number;
|
|
260
|
+
/** Segment length when splitting. Default 1200 s (20 min). */
|
|
261
|
+
readonly segmentSeconds?: number;
|
|
262
|
+
/** Defaults to an ffmpeg segment-muxer splitter. */
|
|
263
|
+
readonly split?: AudioSplitter;
|
|
264
|
+
/** Path to the ffmpeg binary (default: "ffmpeg" on PATH). */
|
|
265
|
+
readonly ffmpegBin?: string;
|
|
266
|
+
}
|
|
267
|
+
declare class ChunkedStt implements SttPort {
|
|
268
|
+
private readonly base;
|
|
269
|
+
private readonly maxBytes;
|
|
270
|
+
private readonly segmentSeconds;
|
|
271
|
+
private readonly split;
|
|
272
|
+
constructor(opts: ChunkedSttOptions);
|
|
273
|
+
transcribe(audioPath: string): Promise<Transcript>;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* CompositeFetcher — tries child fetchers in order, first non-null wins.
|
|
278
|
+
*
|
|
279
|
+
* Lets the importer route by URL without itself knowing the routing:
|
|
280
|
+
* new CompositeFetcher([ ytDlpWhisper, browserReadability ])
|
|
281
|
+
* sends videos to whisper (the readability one returns null for them is
|
|
282
|
+
* never reached because whisper handled it) and everything else falls
|
|
283
|
+
* through to readability. A child that THROWS aborts (hard failure);
|
|
284
|
+
* a child that returns null means "not mine, try the next".
|
|
285
|
+
*/
|
|
286
|
+
|
|
287
|
+
declare class CompositeFetcher implements FetcherPort {
|
|
288
|
+
private readonly children;
|
|
289
|
+
constructor(children: readonly FetcherPort[]);
|
|
290
|
+
fetch(url: string): Promise<FetchedSource | null>;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* ThrottleFetcher — wraps a FetcherPort to enforce a minimum interval
|
|
295
|
+
* between fetches (rate limiting). The importer already calls fetch
|
|
296
|
+
* serially (one URL at a time), so this adds the *pacing* that protects
|
|
297
|
+
* against YouTube anti-abuse blocks and Whisper RPM limits when running
|
|
298
|
+
* a large batch.
|
|
299
|
+
*
|
|
300
|
+
* `now` + `sleep` are injectable so the throttle is unit-testable without
|
|
301
|
+
* real timers.
|
|
302
|
+
*/
|
|
303
|
+
|
|
304
|
+
interface ThrottleFetcherOptions {
|
|
305
|
+
/** Minimum milliseconds between the START of consecutive fetches. */
|
|
306
|
+
readonly minIntervalMs: number;
|
|
307
|
+
readonly now?: () => number;
|
|
308
|
+
readonly sleep?: (ms: number) => Promise<void>;
|
|
309
|
+
}
|
|
310
|
+
declare class ThrottleFetcher implements FetcherPort {
|
|
311
|
+
private readonly inner;
|
|
312
|
+
private readonly minIntervalMs;
|
|
313
|
+
private readonly now;
|
|
314
|
+
private readonly sleep;
|
|
315
|
+
private lastStart;
|
|
316
|
+
private started;
|
|
317
|
+
constructor(inner: FetcherPort, opts: ThrottleFetcherOptions);
|
|
318
|
+
fetch(url: string): Promise<FetchedSource | null>;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* HttpReadabilityFetcher — a browser-free FetcherPort for articles:
|
|
323
|
+
* plain `fetch` + a lightweight readability extraction (prefer
|
|
324
|
+
* <article>/<main>, strip chrome). Validated to pull ~35k chars from a
|
|
325
|
+
* real corpus article. Returns `null` for non-HTML or empty bodies so a
|
|
326
|
+
* video or authed-browser fetcher can take those.
|
|
327
|
+
*
|
|
328
|
+
* No JS execution — so it won't get SPA-only content; for login-walled
|
|
329
|
+
* or JS-rendered pages, compose a BrowserMcpFetcher ahead of/after this.
|
|
330
|
+
*/
|
|
331
|
+
|
|
332
|
+
interface HttpReadabilityFetcherOptions {
|
|
333
|
+
readonly userAgent?: string;
|
|
334
|
+
readonly maxChars?: number;
|
|
335
|
+
}
|
|
336
|
+
declare class HttpReadabilityFetcher implements FetcherPort {
|
|
337
|
+
private readonly ua;
|
|
338
|
+
private readonly maxChars;
|
|
339
|
+
constructor(opts?: HttpReadabilityFetcherOptions);
|
|
340
|
+
fetch(url: string): Promise<FetchedSource | null>;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Usage telemetry for the metered distill path — cost visibility + optional
|
|
345
|
+
* Langfuse export. SDK-free (raw fetch to Langfuse's ingestion API), mirroring
|
|
346
|
+
* the distiller's own no-client-lib ethos and keeping corpus-cli lean.
|
|
347
|
+
*
|
|
348
|
+
* Two outputs, both best-effort (telemetry must never break a distill run):
|
|
349
|
+
* 1. a per-run cost summary printed to stderr (always);
|
|
350
|
+
* 2. one Langfuse trace with a GENERATION observation per LLM call, when
|
|
351
|
+
* LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY (+ LANGFUSE_BASE_URL) are set.
|
|
352
|
+
*
|
|
353
|
+
* Langfuse runs (test/prod/local) share one project, filtered by the
|
|
354
|
+
* `environment` tag (LANGFUSE_TRACING_ENVIRONMENT) — same convention as the
|
|
355
|
+
* Mastra agent tracing in @agstudio/agent-framework.
|
|
356
|
+
*/
|
|
357
|
+
/** One metered LLM call's usage, reported by a distiller. */
|
|
358
|
+
interface DistillUsage {
|
|
359
|
+
/** Model id as returned by the API (falls back to the requested model). */
|
|
360
|
+
readonly model: string;
|
|
361
|
+
readonly inputTokens: number;
|
|
362
|
+
readonly outputTokens: number;
|
|
363
|
+
/** Human label for the call — the source title. */
|
|
364
|
+
readonly label: string;
|
|
365
|
+
readonly startedAt: string;
|
|
366
|
+
readonly endedAt: string;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* AnthropicDistiller — a DistillPort backed by Claude over the Messages API.
|
|
371
|
+
* Extracts refined AIP-10 items (principle/pattern/critique/summary/example)
|
|
372
|
+
* from a raw source. Returns self-contained insights, not quote dumps.
|
|
373
|
+
*
|
|
374
|
+
* Hand-rolled over the HTTP API (no SDK dep), mirroring the other corpus-cli
|
|
375
|
+
* port adapters. Prompt + parse are the shared `distill-prompt` core; this
|
|
376
|
+
* adapter owns only the metered-API transport. For the subscription-billed
|
|
377
|
+
* alternative see CliAgentDistiller + the `claude-code` engine.
|
|
378
|
+
*/
|
|
379
|
+
|
|
380
|
+
interface AnthropicDistillerOptions {
|
|
381
|
+
readonly apiKey: string;
|
|
382
|
+
/** Default a current Claude model id. */
|
|
383
|
+
readonly model?: string;
|
|
384
|
+
readonly baseUrl?: string;
|
|
385
|
+
/** Max refined items to extract per source. */
|
|
386
|
+
readonly maxItems?: number;
|
|
387
|
+
/** Optional sink for per-call token usage (cost + Langfuse export). */
|
|
388
|
+
readonly onUsage?: (usage: DistillUsage) => void;
|
|
389
|
+
}
|
|
390
|
+
declare class AnthropicDistiller implements DistillPort {
|
|
391
|
+
private readonly apiKey;
|
|
392
|
+
private readonly model;
|
|
393
|
+
private readonly baseUrl;
|
|
394
|
+
private readonly maxItems;
|
|
395
|
+
private readonly onUsage;
|
|
396
|
+
constructor(opts: AnthropicDistillerOptions);
|
|
397
|
+
distill(input: DistillInput): Promise<readonly DistilledItem[]>;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* CLI agent engines — descriptors for distilling through a local agent CLI in
|
|
402
|
+
* headless one-shot mode (prompt on stdin, response on stdout), instead of a
|
|
403
|
+
* metered API. Each engine declares its `command`, how to `buildArgs` (model
|
|
404
|
+
* injection, output format, tool/MCP lockdown), and how to `parseOutput` of the
|
|
405
|
+
* captured stdout into the model's text.
|
|
406
|
+
*
|
|
407
|
+
* WHY: distillation is a high-volume batch (hundreds of sources × ~8 items). On
|
|
408
|
+
* the API that bills per token; routed through an authenticated agent-CLI
|
|
409
|
+
* subscription (e.g. Claude Max) it draws on a flat monthly plan — ~$0 marginal
|
|
410
|
+
* cost. Same DistillPort contract + same prompt/parse, swappable transport.
|
|
411
|
+
*
|
|
412
|
+
* Engines register full descriptors and are dispatched by id (no `engineId ===`
|
|
413
|
+
* branching at the call site). Add a CLI by adding a descriptor below; gemini /
|
|
414
|
+
* goose / codex etc. each become one entry.
|
|
415
|
+
*/
|
|
416
|
+
/** What a CLI engine needs to drive one stdin→stdout completion. */
|
|
417
|
+
interface CliEngine {
|
|
418
|
+
/** Stable id, used by `corpus distill --engine <id>`. */
|
|
419
|
+
readonly id: string;
|
|
420
|
+
/** Whether this engine bills a subscription (no API key needed). Surfaced in help. */
|
|
421
|
+
readonly subscriptionBilled: boolean;
|
|
422
|
+
/** Executable to invoke — must be on PATH and logged in. */
|
|
423
|
+
readonly command: string;
|
|
424
|
+
/** Build the argv for one completion. The prompt is fed over stdin, not argv. */
|
|
425
|
+
buildArgs(opts: {
|
|
426
|
+
readonly model?: string;
|
|
427
|
+
}): string[];
|
|
428
|
+
/** Extract the model's text from captured stdout; null → fall back to raw stdout. */
|
|
429
|
+
parseOutput(stdout: string): string | null;
|
|
430
|
+
}
|
|
431
|
+
/** Registry of CLI engines, keyed by id. */
|
|
432
|
+
declare const CLI_ENGINES: Readonly<Record<string, CliEngine>>;
|
|
433
|
+
|
|
434
|
+
/**
|
|
435
|
+
* CliAgentDistiller — a DistillPort backed by any local agent CLI running in
|
|
436
|
+
* one-shot mode (prompt on stdin, response on stdout). The transport is generic;
|
|
437
|
+
* the per-CLI specifics (command, argv, output parse) come from a CliEngine
|
|
438
|
+
* descriptor, so claude / gemini / goose / … are one descriptor each.
|
|
439
|
+
*
|
|
440
|
+
* Same DistillPort contract + same prompt/parse as AnthropicDistiller (both use
|
|
441
|
+
* the shared `distill-prompt` core), so the two are drop-in swappable behind the
|
|
442
|
+
* `corpus distill --engine` flag.
|
|
443
|
+
*
|
|
444
|
+
* TRADE-OFFS vs the metered API:
|
|
445
|
+
* - Subscription rate caps — slower for big batches; pace with --throttle and
|
|
446
|
+
* let the runner skip+retry on non-zero exits.
|
|
447
|
+
* - Output is CLI text, not API structured output — same tolerant parse.
|
|
448
|
+
* - One process spawn per source — higher per-call overhead than fetch().
|
|
449
|
+
*/
|
|
450
|
+
|
|
451
|
+
interface CliAgentDistillerOptions {
|
|
452
|
+
/** The CLI engine descriptor (command + argv + output parse). */
|
|
453
|
+
readonly engine: CliEngine;
|
|
454
|
+
/** Optional model override passed to the engine, e.g. "haiku" to spend less quota. */
|
|
455
|
+
readonly model?: string;
|
|
456
|
+
/** Max refined items per source (feeds the shared prompt). */
|
|
457
|
+
readonly maxItems?: number;
|
|
458
|
+
/** Hard per-source timeout; kills a wedged CLI call. Default 120s. */
|
|
459
|
+
readonly timeoutMs?: number;
|
|
460
|
+
/** Working dir. Default os.tmpdir() — neutral, no project CLAUDE.md / repo. */
|
|
461
|
+
readonly cwd?: string;
|
|
462
|
+
}
|
|
463
|
+
declare class CliAgentDistiller implements DistillPort {
|
|
464
|
+
private readonly engine;
|
|
465
|
+
private readonly model?;
|
|
466
|
+
private readonly maxItems;
|
|
467
|
+
private readonly timeoutMs;
|
|
468
|
+
private readonly cwd;
|
|
469
|
+
constructor(opts: CliAgentDistillerOptions);
|
|
470
|
+
distill(input: DistillInput): Promise<readonly DistilledItem[]>;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* Minimal streamable-HTTP MCP client (host-agnostic). Used by config-driven
|
|
475
|
+
* sinks/sources to call ANY MCP server's tools — the corpus never names a
|
|
476
|
+
* specific host; the endpoint + tool come from a manifest.
|
|
477
|
+
*
|
|
478
|
+
* Handles both JSON and SSE (`text/event-stream`) responses, optional
|
|
479
|
+
* `Mcp-Session-Id`, and the initialize handshake.
|
|
480
|
+
*/
|
|
481
|
+
interface McpClientLike {
|
|
482
|
+
callTool(name: string, args: Record<string, unknown>): Promise<{
|
|
483
|
+
content?: unknown;
|
|
484
|
+
structuredContent?: unknown;
|
|
485
|
+
isError?: boolean;
|
|
486
|
+
}>;
|
|
487
|
+
}
|
|
488
|
+
interface ConnectMcpHttpOptions {
|
|
489
|
+
readonly endpoint: string;
|
|
490
|
+
readonly headers?: Record<string, string>;
|
|
491
|
+
readonly protocolVersion?: string;
|
|
492
|
+
/** "streamable-http" (POST, default) or "sse" (legacy GET stream + POST messages). */
|
|
493
|
+
readonly transport?: "streamable-http" | "sse";
|
|
494
|
+
}
|
|
495
|
+
declare function connectMcpHttp(opts: ConnectMcpHttpOptions): Promise<McpClientLike>;
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* McpSink — a config-driven SinkPort that pushes each refined entry by calling
|
|
499
|
+
* a configured MCP tool. Host-agnostic: the endpoint, tool name, and the
|
|
500
|
+
* entry→args mapping all come from a sink manifest, so the corpus links to ANY
|
|
501
|
+
* host (a Guilde guild KB, a Notion db, a vector store) without naming it.
|
|
502
|
+
*
|
|
503
|
+
* The manifest's `args` is a template: string values containing `${slug}`,
|
|
504
|
+
* `${title}`, `${body}`, `${sources}`, `${tags}`, `${access}`, `${uri}`,
|
|
505
|
+
* `${kind}`, `${confidence}` are substituted per entry. A value that is exactly
|
|
506
|
+
* one placeholder (e.g. `"${sources}"`) is replaced with the typed value (array
|
|
507
|
+
* / number), not a string.
|
|
508
|
+
*/
|
|
509
|
+
|
|
510
|
+
/** Validates a sink manifest (the `--config` JSON) — single source for the type. */
|
|
511
|
+
declare const SINK_CONFIG_SCHEMA: z.ZodObject<{
|
|
512
|
+
endpoint: z.ZodString;
|
|
513
|
+
tool: z.ZodString;
|
|
514
|
+
args: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
515
|
+
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
516
|
+
transport: z.ZodOptional<z.ZodEnum<{
|
|
517
|
+
"streamable-http": "streamable-http";
|
|
518
|
+
sse: "sse";
|
|
519
|
+
}>>;
|
|
520
|
+
importedAlias: z.ZodOptional<z.ZodString>;
|
|
521
|
+
}, z.core.$strip>;
|
|
522
|
+
type McpSinkConfig = z.infer<typeof SINK_CONFIG_SCHEMA>;
|
|
523
|
+
declare class McpSink implements SinkPort {
|
|
524
|
+
private readonly config;
|
|
525
|
+
private client;
|
|
526
|
+
constructor(config: McpSinkConfig, client?: McpClientLike);
|
|
527
|
+
private ensure;
|
|
528
|
+
push(item: SinkItem): Promise<SinkPushResult>;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
export { AnthropicDistiller, type AnthropicDistillerOptions, AssemblyAiStt, type AssemblyAiSttOptions, type AudioDownload, type AudioDownloader, type AudioSplitter, BrowserMcpFetcher, type BrowserMcpFetcherOptions, type BrowserMcpLike, CLI_ENGINES, ChunkedStt, type ChunkedSttOptions, CliAgentDistiller, type CliAgentDistillerOptions, type CliEngine, CompositeFetcher, type ConnectMcpHttpOptions, HttpReadabilityFetcher, type HttpReadabilityFetcherOptions, type McpClientLike, McpSink, type McpSinkConfig, OpenAiWhisperStt, type OpenAiWhisperSttOptions, ScrapeMcpFetcher, type ScrapeMcpFetcherOptions, type SttPort, ThrottleFetcher, type ThrottleFetcherOptions, type Transcript, YtDlpWhisperFetcher, type YtDlpWhisperFetcherOptions, connectMcpHttp };
|