@steipete/summarize 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -3
- package/README.md +40 -6
- package/dist/cli.cjs +6502 -634
- package/dist/cli.cjs.map +4 -4
- package/dist/esm/content/asset.js +18 -0
- package/dist/esm/content/asset.js.map +1 -1
- package/dist/esm/content/link-preview/client.js +8 -0
- package/dist/esm/content/link-preview/client.js.map +1 -1
- package/dist/esm/content/link-preview/content/article.js +15 -1
- package/dist/esm/content/link-preview/content/article.js.map +1 -1
- package/dist/esm/content/link-preview/content/index.js +151 -4
- package/dist/esm/content/link-preview/content/index.js.map +1 -1
- package/dist/esm/content/link-preview/transcript/index.js +6 -0
- package/dist/esm/content/link-preview/transcript/index.js.map +1 -1
- package/dist/esm/content/link-preview/transcript/providers/youtube/yt-dlp.js +213 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/yt-dlp.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube.js +40 -2
- package/dist/esm/content/link-preview/transcript/providers/youtube.js.map +1 -1
- package/dist/esm/flags.js +14 -2
- package/dist/esm/flags.js.map +1 -1
- package/dist/esm/llm/generate-text.js +125 -21
- package/dist/esm/llm/generate-text.js.map +1 -1
- package/dist/esm/llm/html-to-markdown.js +3 -2
- package/dist/esm/llm/html-to-markdown.js.map +1 -1
- package/dist/esm/pricing/litellm.js +4 -1
- package/dist/esm/pricing/litellm.js.map +1 -1
- package/dist/esm/prompts/file.js +15 -4
- package/dist/esm/prompts/file.js.map +1 -1
- package/dist/esm/prompts/link-summary.js +20 -6
- package/dist/esm/prompts/link-summary.js.map +1 -1
- package/dist/esm/run.js +545 -407
- package/dist/esm/run.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/dist/types/content/link-preview/client.d.ts +5 -1
- package/dist/types/content/link-preview/content/types.d.ts +1 -1
- package/dist/types/content/link-preview/deps.d.ts +33 -0
- package/dist/types/content/link-preview/transcript/providers/youtube/yt-dlp.d.ts +15 -0
- package/dist/types/content/link-preview/transcript/types.d.ts +4 -0
- package/dist/types/content/link-preview/types.d.ts +1 -1
- package/dist/types/costs.d.ts +1 -1
- package/dist/types/flags.d.ts +1 -1
- package/dist/types/llm/generate-text.d.ts +8 -2
- package/dist/types/llm/html-to-markdown.d.ts +4 -1
- package/dist/types/pricing/litellm.d.ts +1 -0
- package/dist/types/prompts/file.d.ts +2 -1
- package/dist/types/version.d.ts +1 -1
- package/docs/extract-only.md +1 -1
- package/docs/firecrawl.md +2 -2
- package/docs/llm.md +7 -0
- package/docs/site/docs/config.html +1 -1
- package/docs/site/docs/firecrawl.html +1 -1
- package/docs/website.md +3 -3
- package/docs/youtube.md +5 -2
- package/package.json +7 -2
package/dist/esm/run.js
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants as fsConstants } from 'node:fs';
|
|
1
3
|
import fs from 'node:fs/promises';
|
|
4
|
+
import path from 'node:path';
|
|
2
5
|
import { Command, CommanderError, Option } from 'commander';
|
|
6
|
+
import { countTokens } from 'gpt-tokenizer';
|
|
3
7
|
import { createLiveRenderer, render as renderMarkdownAnsi } from 'markdansi';
|
|
8
|
+
import { normalizeTokenUsage, tallyCosts } from 'tokentally';
|
|
4
9
|
import { loadSummarizeConfig } from './config.js';
|
|
5
10
|
import { buildAssetPromptMessages, classifyUrl, loadLocalAsset, loadRemoteAsset, resolveInputTarget, } from './content/asset.js';
|
|
6
11
|
import { createLinkPreviewClient } from './content/index.js';
|
|
@@ -11,21 +16,107 @@ import { generateTextWithModelId, streamTextWithModelId } from './llm/generate-t
|
|
|
11
16
|
import { resolveGoogleModelForUsage } from './llm/google-models.js';
|
|
12
17
|
import { createHtmlToMarkdownConverter } from './llm/html-to-markdown.js';
|
|
13
18
|
import { normalizeGatewayStyleModelId, parseGatewayStyleModelId } from './llm/model-id.js';
|
|
14
|
-
import { loadLiteLlmCatalog, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
|
|
15
|
-
import { buildFileSummaryPrompt, buildLinkSummaryPrompt
|
|
19
|
+
import { loadLiteLlmCatalog, resolveLiteLlmMaxInputTokensForModelId, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
|
|
20
|
+
import { buildFileSummaryPrompt, buildLinkSummaryPrompt } from './prompts/index.js';
|
|
16
21
|
import { startOscProgress } from './tty/osc-progress.js';
|
|
17
22
|
import { startSpinner } from './tty/spinner.js';
|
|
18
23
|
import { resolvePackageVersion } from './version.js';
|
|
19
|
-
const
|
|
20
|
-
const
|
|
24
|
+
const BIRD_TIP = 'Tip: Install bird🐦 for better Twitter support: https://github.com/steipete/bird';
|
|
25
|
+
const TWITTER_HOSTS = new Set(['x.com', 'twitter.com', 'mobile.twitter.com']);
|
|
26
|
+
const SUMMARY_LENGTH_MAX_CHARACTERS = {
|
|
27
|
+
short: 1200,
|
|
28
|
+
medium: 2500,
|
|
29
|
+
long: 6000,
|
|
30
|
+
xl: 14000,
|
|
31
|
+
xxl: Number.POSITIVE_INFINITY,
|
|
32
|
+
};
|
|
33
|
+
function resolveTargetCharacters(lengthArg) {
|
|
34
|
+
return lengthArg.kind === 'chars'
|
|
35
|
+
? lengthArg.maxCharacters
|
|
36
|
+
: SUMMARY_LENGTH_MAX_CHARACTERS[lengthArg.preset];
|
|
37
|
+
}
|
|
38
|
+
function isTwitterStatusUrl(raw) {
|
|
39
|
+
try {
|
|
40
|
+
const parsed = new URL(raw);
|
|
41
|
+
const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
|
|
42
|
+
if (!TWITTER_HOSTS.has(host))
|
|
43
|
+
return false;
|
|
44
|
+
return /\/status\/\d+/.test(parsed.pathname);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function isExecutable(filePath) {
|
|
51
|
+
try {
|
|
52
|
+
accessSync(filePath, fsConstants.X_OK);
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function hasBirdCli(env) {
|
|
60
|
+
const candidates = [];
|
|
61
|
+
const pathEnv = env.PATH ?? process.env.PATH ?? '';
|
|
62
|
+
for (const entry of pathEnv.split(path.delimiter)) {
|
|
63
|
+
if (!entry)
|
|
64
|
+
continue;
|
|
65
|
+
candidates.push(path.join(entry, 'bird'));
|
|
66
|
+
}
|
|
67
|
+
return candidates.some((candidate) => isExecutable(candidate));
|
|
68
|
+
}
|
|
69
|
+
async function readTweetWithBird(args) {
|
|
70
|
+
return await new Promise((resolve, reject) => {
|
|
71
|
+
execFile('bird', ['read', args.url, '--json'], {
|
|
72
|
+
timeout: args.timeoutMs,
|
|
73
|
+
env: { ...process.env, ...args.env },
|
|
74
|
+
maxBuffer: 1024 * 1024,
|
|
75
|
+
}, (error, stdout, stderr) => {
|
|
76
|
+
if (error) {
|
|
77
|
+
const detail = stderr?.trim();
|
|
78
|
+
const suffix = detail ? `: ${detail}` : '';
|
|
79
|
+
reject(new Error(`bird read failed${suffix}`));
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const trimmed = stdout.trim();
|
|
83
|
+
if (!trimmed) {
|
|
84
|
+
reject(new Error('bird read returned empty output'));
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
try {
|
|
88
|
+
const parsed = JSON.parse(trimmed);
|
|
89
|
+
const tweet = Array.isArray(parsed) ? parsed[0] : parsed;
|
|
90
|
+
if (!tweet || typeof tweet.text !== 'string') {
|
|
91
|
+
reject(new Error('bird read returned invalid payload'));
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
resolve(tweet);
|
|
95
|
+
}
|
|
96
|
+
catch (parseError) {
|
|
97
|
+
const message = parseError instanceof Error ? parseError.message : String(parseError);
|
|
98
|
+
reject(new Error(`bird read returned invalid JSON: ${message}`));
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
function withBirdTip(error, url, env) {
|
|
104
|
+
if (!url || !isTwitterStatusUrl(url) || hasBirdCli(env)) {
|
|
105
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
106
|
+
}
|
|
107
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
108
|
+
const combined = `${message}\n${BIRD_TIP}`;
|
|
109
|
+
return error instanceof Error ? new Error(combined, { cause: error }) : new Error(combined);
|
|
110
|
+
}
|
|
111
|
+
const MAX_TEXT_BYTES_DEFAULT = 10 * 1024 * 1024;
|
|
21
112
|
function buildProgram() {
|
|
22
113
|
return new Command()
|
|
23
114
|
.name('summarize')
|
|
24
115
|
.description('Summarize web pages and YouTube links (uses direct provider API keys).')
|
|
25
116
|
.argument('[input]', 'URL or local file path to summarize')
|
|
26
|
-
.option('--youtube <mode>', 'YouTube transcript source: auto
|
|
27
|
-
.option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first).
|
|
28
|
-
.option('--markdown <mode>', 'Website Markdown output: off, auto (
|
|
117
|
+
.option('--youtube <mode>', 'YouTube transcript source: auto, web (youtubei/captionTracks), yt-dlp (audio+whisper), apify', 'auto')
|
|
118
|
+
.option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first).', 'auto')
|
|
119
|
+
.option('--markdown <mode>', 'Website Markdown output: off, auto (use LLM when configured), llm (force LLM). Only affects --extract-only for non-YouTube URLs.', 'auto')
|
|
29
120
|
.option('--length <length>', 'Summary length: short|medium|long|xl|xxl or a character limit like 20000, 20k', 'medium')
|
|
30
121
|
.option('--max-output-tokens <count>', 'Hard cap for LLM output tokens (e.g. 2000, 2k). Overrides provider defaults.', undefined)
|
|
31
122
|
.option('--timeout <duration>', 'Timeout for content fetching and LLM request: 30 (seconds), 30s, 2m, 5000ms', '2m')
|
|
@@ -139,19 +230,26 @@ function assertAssetMediaTypeSupported({ attachment, sizeLabel, }) {
|
|
|
139
230
|
`Archive formats (zip/tar/7z/rar) can’t be sent to the model.\n` +
|
|
140
231
|
`Unzip and summarize a specific file instead (e.g. README.md).`);
|
|
141
232
|
}
|
|
142
|
-
function buildAssetPromptPayload({ promptText, attachment, }) {
|
|
143
|
-
if (attachment.part.type === 'file' && isTextLikeMediaType(attachment.mediaType)) {
|
|
144
|
-
const data = attachment.part.data;
|
|
145
|
-
const content = typeof data === 'string'
|
|
146
|
-
? data
|
|
147
|
-
: data instanceof Uint8Array
|
|
148
|
-
? new TextDecoder().decode(data)
|
|
149
|
-
: '';
|
|
233
|
+
function buildAssetPromptPayload({ promptText, attachment, textContent, }) {
|
|
234
|
+
if (textContent && attachment.part.type === 'file' && isTextLikeMediaType(attachment.mediaType)) {
|
|
150
235
|
const header = `File: ${attachment.filename ?? 'unknown'} (${attachment.mediaType})`;
|
|
151
|
-
return `${promptText}\n\n---\n${header}\n\n${content}`.trim();
|
|
236
|
+
return `${promptText}\n\n---\n${header}\n\n${textContent.content}`.trim();
|
|
152
237
|
}
|
|
153
238
|
return buildAssetPromptMessages({ promptText, attachment });
|
|
154
239
|
}
|
|
240
|
+
function getTextContentFromAttachment(attachment) {
|
|
241
|
+
if (attachment.part.type !== 'file' || !isTextLikeMediaType(attachment.mediaType)) {
|
|
242
|
+
return null;
|
|
243
|
+
}
|
|
244
|
+
const data = attachment.part.data;
|
|
245
|
+
if (typeof data === 'string') {
|
|
246
|
+
return { content: data, bytes: Buffer.byteLength(data, 'utf8') };
|
|
247
|
+
}
|
|
248
|
+
if (data instanceof Uint8Array) {
|
|
249
|
+
return { content: new TextDecoder().decode(data), bytes: data.byteLength };
|
|
250
|
+
}
|
|
251
|
+
return { content: '', bytes: 0 };
|
|
252
|
+
}
|
|
155
253
|
function assertProviderSupportsAttachment({ provider, modelId, attachment, }) {
|
|
156
254
|
// xAI via AI SDK currently supports image parts, but not generic file parts (e.g. PDFs).
|
|
157
255
|
if (provider === 'xai' &&
|
|
@@ -196,6 +294,18 @@ function isGoogleStreamingUnsupportedError(error) {
|
|
|
196
294
|
/Call ListModels/i.test(errorText) ||
|
|
197
295
|
/supported methods/i.test(errorText));
|
|
198
296
|
}
|
|
297
|
+
function isStreamingTimeoutError(error) {
|
|
298
|
+
if (!error)
|
|
299
|
+
return false;
|
|
300
|
+
const message = typeof error === 'string'
|
|
301
|
+
? error
|
|
302
|
+
: error instanceof Error
|
|
303
|
+
? error.message
|
|
304
|
+
: typeof error.message === 'string'
|
|
305
|
+
? String(error.message)
|
|
306
|
+
: '';
|
|
307
|
+
return /timed out/i.test(message);
|
|
308
|
+
}
|
|
199
309
|
function attachRichHelp(program, env, stdout) {
|
|
200
310
|
const color = supportsColor(stdout, env);
|
|
201
311
|
const heading = (text) => ansi('1;36', text, color);
|
|
@@ -204,29 +314,33 @@ function attachRichHelp(program, env, stdout) {
|
|
|
204
314
|
program.addHelpText('after', () => `
|
|
205
315
|
${heading('Examples')}
|
|
206
316
|
${cmd('summarize "https://example.com"')}
|
|
207
|
-
${cmd('summarize "https://example.com" --extract-only')} ${dim('# website markdown (
|
|
317
|
+
${cmd('summarize "https://example.com" --extract-only')} ${dim('# website markdown (LLM if configured)')}
|
|
208
318
|
${cmd('summarize "https://example.com" --extract-only --markdown llm')} ${dim('# website markdown via LLM')}
|
|
209
319
|
${cmd('summarize "https://www.youtube.com/watch?v=I845O57ZSy4&t=11s" --extract-only --youtube web')}
|
|
210
320
|
${cmd('summarize "https://example.com" --length 20k --max-output-tokens 2k --timeout 2m --model openai/gpt-5.2')}
|
|
211
|
-
${cmd('
|
|
321
|
+
${cmd('OPENROUTER_API_KEY=... summarize "https://example.com" --model openai/openai/gpt-oss-20b')}
|
|
212
322
|
${cmd('summarize "https://example.com" --json --verbose')}
|
|
213
323
|
|
|
214
324
|
${heading('Env Vars')}
|
|
215
325
|
XAI_API_KEY optional (required for xai/... models)
|
|
216
326
|
OPENAI_API_KEY optional (required for openai/... models)
|
|
217
327
|
OPENAI_BASE_URL optional (OpenAI-compatible API endpoint; e.g. OpenRouter)
|
|
218
|
-
OPENROUTER_API_KEY optional (
|
|
328
|
+
OPENROUTER_API_KEY optional (routes openai/... models through OpenRouter)
|
|
329
|
+
OPENROUTER_PROVIDERS optional (provider fallback order, e.g. "groq,google-vertex")
|
|
219
330
|
GEMINI_API_KEY optional (required for google/... models)
|
|
220
331
|
ANTHROPIC_API_KEY optional (required for anthropic/... models)
|
|
221
332
|
SUMMARIZE_MODEL optional (overrides default model selection)
|
|
222
333
|
FIRECRAWL_API_KEY optional website extraction fallback (Markdown)
|
|
223
334
|
APIFY_API_TOKEN optional YouTube transcript fallback
|
|
335
|
+
YT_DLP_PATH optional path to yt-dlp binary for audio extraction
|
|
336
|
+
FAL_KEY optional FAL AI API key for audio transcription
|
|
224
337
|
`);
|
|
225
338
|
}
|
|
226
|
-
async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutMs, fetchImpl, apiKeys, }) {
|
|
339
|
+
async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutMs, fetchImpl, apiKeys, openrouter, }) {
|
|
227
340
|
const result = await generateTextWithModelId({
|
|
228
341
|
modelId,
|
|
229
342
|
apiKeys,
|
|
343
|
+
openrouter,
|
|
230
344
|
prompt,
|
|
231
345
|
temperature: 0,
|
|
232
346
|
maxOutputTokens,
|
|
@@ -240,38 +354,6 @@ async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutM
|
|
|
240
354
|
usage: result.usage,
|
|
241
355
|
};
|
|
242
356
|
}
|
|
243
|
-
function splitTextIntoChunks(input, maxCharacters) {
|
|
244
|
-
if (maxCharacters <= 0) {
|
|
245
|
-
return [input];
|
|
246
|
-
}
|
|
247
|
-
const text = input.trim();
|
|
248
|
-
if (text.length <= maxCharacters) {
|
|
249
|
-
return [text];
|
|
250
|
-
}
|
|
251
|
-
const chunks = [];
|
|
252
|
-
let offset = 0;
|
|
253
|
-
while (offset < text.length) {
|
|
254
|
-
const end = Math.min(offset + maxCharacters, text.length);
|
|
255
|
-
const slice = text.slice(offset, end);
|
|
256
|
-
if (end === text.length) {
|
|
257
|
-
chunks.push(slice.trim());
|
|
258
|
-
break;
|
|
259
|
-
}
|
|
260
|
-
const candidateBreaks = [
|
|
261
|
-
slice.lastIndexOf('\n\n'),
|
|
262
|
-
slice.lastIndexOf('\n'),
|
|
263
|
-
slice.lastIndexOf('. '),
|
|
264
|
-
];
|
|
265
|
-
const lastBreak = Math.max(...candidateBreaks);
|
|
266
|
-
const splitAt = lastBreak > Math.floor(maxCharacters * 0.5) ? lastBreak + 1 : slice.length;
|
|
267
|
-
const chunk = slice.slice(0, splitAt).trim();
|
|
268
|
-
if (chunk.length > 0) {
|
|
269
|
-
chunks.push(chunk);
|
|
270
|
-
}
|
|
271
|
-
offset += splitAt;
|
|
272
|
-
}
|
|
273
|
-
return chunks.filter((chunk) => chunk.length > 0);
|
|
274
|
-
}
|
|
275
357
|
const VERBOSE_PREFIX = '[summarize]';
|
|
276
358
|
function writeVerbose(stderr, verbose, message, color) {
|
|
277
359
|
if (!verbose) {
|
|
@@ -315,6 +397,11 @@ function formatBytes(bytes) {
|
|
|
315
397
|
}
|
|
316
398
|
return `${value.toFixed(value >= 10 ? 0 : 1)} ${units[unitIndex]}`;
|
|
317
399
|
}
|
|
400
|
+
function formatCount(value) {
|
|
401
|
+
if (!Number.isFinite(value))
|
|
402
|
+
return 'unknown';
|
|
403
|
+
return value.toLocaleString('en-US');
|
|
404
|
+
}
|
|
318
405
|
function sumNumbersOrNull(values) {
|
|
319
406
|
let sum = 0;
|
|
320
407
|
let any = false;
|
|
@@ -339,7 +426,7 @@ function mergeStreamingChunk(previous, chunk) {
|
|
|
339
426
|
}
|
|
340
427
|
return { next: previous + chunk, appended: chunk };
|
|
341
428
|
}
|
|
342
|
-
function writeFinishLine({ stderr, elapsedMs, model,
|
|
429
|
+
function writeFinishLine({ stderr, elapsedMs, model, report, costUsd, color, }) {
|
|
343
430
|
const promptTokens = sumNumbersOrNull(report.llm.map((row) => row.promptTokens));
|
|
344
431
|
const completionTokens = sumNumbersOrNull(report.llm.map((row) => row.completionTokens));
|
|
345
432
|
const totalTokens = sumNumbersOrNull(report.llm.map((row) => row.totalTokens));
|
|
@@ -357,25 +444,10 @@ function writeFinishLine({ stderr, elapsedMs, model, strategy, chunkCount, repor
|
|
|
357
444
|
if (report.services.apify.requests > 0) {
|
|
358
445
|
parts.push(`apify=${report.services.apify.requests}`);
|
|
359
446
|
}
|
|
360
|
-
if (strategy === 'map-reduce') {
|
|
361
|
-
parts.push('strategy=map-reduce');
|
|
362
|
-
if (typeof chunkCount === 'number' && Number.isFinite(chunkCount) && chunkCount > 0) {
|
|
363
|
-
parts.push(`chunks=${chunkCount}`);
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
447
|
const line = `Finished in ${formatElapsedMs(elapsedMs)} (${parts.join(' | ')})`;
|
|
367
448
|
stderr.write('\n');
|
|
368
449
|
stderr.write(`${ansi('1;32', line, color)}\n`);
|
|
369
450
|
}
|
|
370
|
-
function buildChunkNotesPrompt({ content }) {
|
|
371
|
-
return `Return 10 bullet points summarizing the content below (Markdown).
|
|
372
|
-
|
|
373
|
-
CONTENT:
|
|
374
|
-
"""
|
|
375
|
-
${content}
|
|
376
|
-
"""
|
|
377
|
-
`;
|
|
378
|
-
}
|
|
379
451
|
export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
380
452
|
;
|
|
381
453
|
globalThis.AI_SDK_LOG_WARNINGS = false;
|
|
@@ -427,18 +499,26 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
427
499
|
const markdownMode = parseMarkdownMode(program.opts().markdown);
|
|
428
500
|
const shouldComputeReport = metricsEnabled;
|
|
429
501
|
const isYoutubeUrl = typeof url === 'string' ? /youtube\.com|youtu\.be/i.test(url) : false;
|
|
430
|
-
const firecrawlExplicitlySet = normalizedArgv.some((arg) => arg === '--firecrawl' || arg.startsWith('--firecrawl='));
|
|
431
502
|
const requestedFirecrawlMode = parseFirecrawlMode(program.opts().firecrawl);
|
|
432
503
|
const modelArg = typeof program.opts().model === 'string' ? program.opts().model : null;
|
|
433
504
|
const { config, path: configPath } = loadSummarizeConfig({ env });
|
|
434
505
|
const xaiKeyRaw = typeof env.XAI_API_KEY === 'string' ? env.XAI_API_KEY : null;
|
|
435
506
|
const openaiBaseUrl = typeof env.OPENAI_BASE_URL === 'string' ? env.OPENAI_BASE_URL : null;
|
|
436
507
|
const openRouterKeyRaw = typeof env.OPENROUTER_API_KEY === 'string' ? env.OPENROUTER_API_KEY : null;
|
|
508
|
+
const openRouterProvidersRaw = typeof env.OPENROUTER_PROVIDERS === 'string' ? env.OPENROUTER_PROVIDERS : null;
|
|
509
|
+
const openRouterProviders = openRouterProvidersRaw
|
|
510
|
+
? openRouterProvidersRaw
|
|
511
|
+
.split(',')
|
|
512
|
+
.map((p) => p.trim())
|
|
513
|
+
.filter(Boolean)
|
|
514
|
+
: null;
|
|
437
515
|
const openaiKeyRaw = typeof env.OPENAI_API_KEY === 'string' ? env.OPENAI_API_KEY : null;
|
|
438
516
|
const apiKey = typeof openaiBaseUrl === 'string' && /openrouter\.ai/i.test(openaiBaseUrl)
|
|
439
517
|
? (openRouterKeyRaw ?? openaiKeyRaw)
|
|
440
518
|
: openaiKeyRaw;
|
|
441
519
|
const apifyToken = typeof env.APIFY_API_TOKEN === 'string' ? env.APIFY_API_TOKEN : null;
|
|
520
|
+
const ytDlpPath = typeof env.YT_DLP_PATH === 'string' ? env.YT_DLP_PATH : null;
|
|
521
|
+
const falApiKey = typeof env.FAL_KEY === 'string' ? env.FAL_KEY : null;
|
|
442
522
|
const firecrawlKey = typeof env.FIRECRAWL_API_KEY === 'string' ? env.FIRECRAWL_API_KEY : null;
|
|
443
523
|
const anthropicKeyRaw = typeof env.ANTHROPIC_API_KEY === 'string' ? env.ANTHROPIC_API_KEY : null;
|
|
444
524
|
const googleKeyRaw = typeof env.GEMINI_API_KEY === 'string'
|
|
@@ -453,9 +533,13 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
453
533
|
const xaiApiKey = xaiKeyRaw?.trim() ?? null;
|
|
454
534
|
const googleApiKey = googleKeyRaw?.trim() ?? null;
|
|
455
535
|
const anthropicApiKey = anthropicKeyRaw?.trim() ?? null;
|
|
536
|
+
const openrouterApiKey = openRouterKeyRaw?.trim() ?? null;
|
|
537
|
+
const openaiTranscriptionKey = openaiKeyRaw?.trim() ?? null;
|
|
456
538
|
const googleConfigured = typeof googleApiKey === 'string' && googleApiKey.length > 0;
|
|
457
539
|
const xaiConfigured = typeof xaiApiKey === 'string' && xaiApiKey.length > 0;
|
|
458
540
|
const anthropicConfigured = typeof anthropicApiKey === 'string' && anthropicApiKey.length > 0;
|
|
541
|
+
const openrouterConfigured = typeof openrouterApiKey === 'string' && openrouterApiKey.length > 0;
|
|
542
|
+
const openrouterOptions = openRouterProviders ? { providers: openRouterProviders } : undefined;
|
|
459
543
|
const llmCalls = [];
|
|
460
544
|
let firecrawlRequests = 0;
|
|
461
545
|
let apifyRequests = 0;
|
|
@@ -485,29 +569,41 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
485
569
|
return null;
|
|
486
570
|
return capMaxOutputTokensForModel({ modelId, requested: maxOutputTokensArg });
|
|
487
571
|
};
|
|
572
|
+
const resolveMaxInputTokensForCall = async (modelId) => {
|
|
573
|
+
const catalog = await getLiteLlmCatalog();
|
|
574
|
+
if (!catalog)
|
|
575
|
+
return null;
|
|
576
|
+
const limit = resolveLiteLlmMaxInputTokensForModelId(catalog, modelId);
|
|
577
|
+
if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0) {
|
|
578
|
+
return limit;
|
|
579
|
+
}
|
|
580
|
+
return null;
|
|
581
|
+
};
|
|
488
582
|
const estimateCostUsd = async () => {
|
|
489
583
|
const catalog = await getLiteLlmCatalog();
|
|
490
584
|
if (!catalog)
|
|
491
585
|
return null;
|
|
492
|
-
|
|
493
|
-
let any = false;
|
|
494
|
-
for (const call of llmCalls) {
|
|
586
|
+
const calls = llmCalls.map((call) => {
|
|
495
587
|
const promptTokens = call.usage?.promptTokens ?? null;
|
|
496
588
|
const completionTokens = call.usage?.completionTokens ?? null;
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
typeof completionTokens
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
}
|
|
510
|
-
|
|
589
|
+
const hasTokens = typeof promptTokens === 'number' &&
|
|
590
|
+
Number.isFinite(promptTokens) &&
|
|
591
|
+
typeof completionTokens === 'number' &&
|
|
592
|
+
Number.isFinite(completionTokens);
|
|
593
|
+
const usage = hasTokens
|
|
594
|
+
? normalizeTokenUsage({
|
|
595
|
+
inputTokens: promptTokens,
|
|
596
|
+
outputTokens: completionTokens,
|
|
597
|
+
totalTokens: call.usage?.totalTokens ?? undefined,
|
|
598
|
+
})
|
|
599
|
+
: null;
|
|
600
|
+
return { model: call.model, usage };
|
|
601
|
+
});
|
|
602
|
+
const result = await tallyCosts({
|
|
603
|
+
calls,
|
|
604
|
+
resolvePricing: (modelId) => resolveLiteLlmPricingForModelId(catalog, modelId),
|
|
605
|
+
});
|
|
606
|
+
return result.total?.totalUsd ?? null;
|
|
511
607
|
};
|
|
512
608
|
const buildReport = async () => {
|
|
513
609
|
return buildRunMetricsReport({ llmCalls, firecrawlRequests, apifyRequests });
|
|
@@ -582,6 +678,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
582
678
|
openaiApiKey: apiKey,
|
|
583
679
|
googleApiKey: googleConfigured ? googleApiKey : null,
|
|
584
680
|
anthropicApiKey: anthropicConfigured ? anthropicApiKey : null,
|
|
681
|
+
openrouterApiKey: openrouterConfigured ? openrouterApiKey : null,
|
|
585
682
|
};
|
|
586
683
|
const requiredKeyEnv = parsedModel.provider === 'xai'
|
|
587
684
|
? 'XAI_API_KEY'
|
|
@@ -589,14 +686,14 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
589
686
|
? 'GEMINI_API_KEY (or GOOGLE_GENERATIVE_AI_API_KEY / GOOGLE_API_KEY)'
|
|
590
687
|
: parsedModel.provider === 'anthropic'
|
|
591
688
|
? 'ANTHROPIC_API_KEY'
|
|
592
|
-
: 'OPENAI_API_KEY';
|
|
689
|
+
: 'OPENAI_API_KEY (or OPENROUTER_API_KEY)';
|
|
593
690
|
const hasRequiredKey = parsedModel.provider === 'xai'
|
|
594
691
|
? Boolean(xaiApiKey)
|
|
595
692
|
: parsedModel.provider === 'google'
|
|
596
693
|
? googleConfigured
|
|
597
694
|
: parsedModel.provider === 'anthropic'
|
|
598
695
|
? anthropicConfigured
|
|
599
|
-
: Boolean(apiKey);
|
|
696
|
+
: Boolean(apiKey) || openrouterConfigured;
|
|
600
697
|
if (!hasRequiredKey) {
|
|
601
698
|
throw new Error(`Missing ${requiredKeyEnv} for model ${parsedModel.canonical}. Set the env var or choose a different --model.`);
|
|
602
699
|
}
|
|
@@ -617,14 +714,29 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
617
714
|
const effectiveModelId = modelResolution.modelId;
|
|
618
715
|
const parsedModelEffective = parseGatewayStyleModelId(effectiveModelId);
|
|
619
716
|
const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
|
|
717
|
+
const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
|
|
718
|
+
const textContent = getTextContentFromAttachment(attachment);
|
|
719
|
+
if (textContent && textContent.bytes > MAX_TEXT_BYTES_DEFAULT) {
|
|
720
|
+
throw new Error(`Text file too large (${formatBytes(textContent.bytes)}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
|
|
721
|
+
}
|
|
620
722
|
const summaryLengthTarget = lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters };
|
|
621
723
|
const promptText = buildFileSummaryPrompt({
|
|
622
724
|
filename: attachment.filename,
|
|
623
725
|
mediaType: attachment.mediaType,
|
|
624
726
|
summaryLength: summaryLengthTarget,
|
|
727
|
+
contentLength: textContent?.content.length ?? null,
|
|
625
728
|
});
|
|
626
|
-
const
|
|
627
|
-
const
|
|
729
|
+
const promptPayload = buildAssetPromptPayload({ promptText, attachment, textContent });
|
|
730
|
+
const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
|
|
731
|
+
if (typeof maxInputTokensForCall === 'number' &&
|
|
732
|
+
Number.isFinite(maxInputTokensForCall) &&
|
|
733
|
+
maxInputTokensForCall > 0 &&
|
|
734
|
+
typeof promptPayload === 'string') {
|
|
735
|
+
const tokenCount = countTokens(promptPayload);
|
|
736
|
+
if (tokenCount > maxInputTokensForCall) {
|
|
737
|
+
throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
628
740
|
const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
|
|
629
741
|
const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
|
|
630
742
|
const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
|
|
@@ -637,6 +749,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
637
749
|
streamResult = await streamTextWithModelId({
|
|
638
750
|
modelId: parsedModelEffective.canonical,
|
|
639
751
|
apiKeys: apiKeysForLlm,
|
|
752
|
+
openrouter: openrouterOptions,
|
|
640
753
|
prompt: promptPayload,
|
|
641
754
|
temperature: 0,
|
|
642
755
|
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
@@ -645,7 +758,27 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
645
758
|
});
|
|
646
759
|
}
|
|
647
760
|
catch (error) {
|
|
648
|
-
if (
|
|
761
|
+
if (isStreamingTimeoutError(error)) {
|
|
762
|
+
writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
|
|
763
|
+
const result = await summarizeWithModelId({
|
|
764
|
+
modelId: parsedModelEffective.canonical,
|
|
765
|
+
prompt: promptPayload,
|
|
766
|
+
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
767
|
+
timeoutMs,
|
|
768
|
+
fetchImpl: trackedFetch,
|
|
769
|
+
apiKeys: apiKeysForLlm,
|
|
770
|
+
openrouter: openrouterOptions,
|
|
771
|
+
});
|
|
772
|
+
llmCalls.push({
|
|
773
|
+
provider: result.provider,
|
|
774
|
+
model: result.canonicalModelId,
|
|
775
|
+
usage: result.usage,
|
|
776
|
+
purpose: 'summary',
|
|
777
|
+
});
|
|
778
|
+
summary = result.text;
|
|
779
|
+
streamResult = null;
|
|
780
|
+
}
|
|
781
|
+
else if (parsedModelEffective.provider === 'google' &&
|
|
649
782
|
isGoogleStreamingUnsupportedError(error)) {
|
|
650
783
|
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
651
784
|
const result = await summarizeWithModelId({
|
|
@@ -655,6 +788,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
655
788
|
timeoutMs,
|
|
656
789
|
fetchImpl: trackedFetch,
|
|
657
790
|
apiKeys: apiKeysForLlm,
|
|
791
|
+
openrouter: openrouterOptions,
|
|
658
792
|
});
|
|
659
793
|
llmCalls.push({
|
|
660
794
|
provider: result.provider,
|
|
@@ -758,6 +892,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
758
892
|
timeoutMs,
|
|
759
893
|
fetchImpl: trackedFetch,
|
|
760
894
|
apiKeys: apiKeysForLlm,
|
|
895
|
+
openrouter: openrouterOptions,
|
|
761
896
|
});
|
|
762
897
|
}
|
|
763
898
|
catch (error) {
|
|
@@ -829,7 +964,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
829
964
|
model: parsedModelEffective.canonical,
|
|
830
965
|
maxCompletionTokens: maxOutputTokensForCall,
|
|
831
966
|
strategy: 'single',
|
|
832
|
-
chunkCount: 1,
|
|
833
967
|
},
|
|
834
968
|
metrics: metricsEnabled ? finishReport : null,
|
|
835
969
|
summary,
|
|
@@ -844,8 +978,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
844
978
|
stderr,
|
|
845
979
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
846
980
|
model: parsedModelEffective.canonical,
|
|
847
|
-
strategy: 'single',
|
|
848
|
-
chunkCount: 1,
|
|
849
981
|
report: finishReport,
|
|
850
982
|
costUsd,
|
|
851
983
|
color: verboseColor,
|
|
@@ -876,8 +1008,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
876
1008
|
stderr,
|
|
877
1009
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
878
1010
|
model: parsedModelEffective.canonical,
|
|
879
|
-
strategy: 'single',
|
|
880
|
-
chunkCount: 1,
|
|
881
1011
|
report,
|
|
882
1012
|
costUsd,
|
|
883
1013
|
color: verboseColor,
|
|
@@ -998,12 +1128,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
998
1128
|
if (!url) {
|
|
999
1129
|
throw new Error('Only HTTP and HTTPS URLs can be summarized');
|
|
1000
1130
|
}
|
|
1001
|
-
const firecrawlMode =
|
|
1002
|
-
if (extractOnly && !isYoutubeUrl && !firecrawlExplicitlySet && firecrawlConfigured) {
|
|
1003
|
-
return 'always';
|
|
1004
|
-
}
|
|
1005
|
-
return requestedFirecrawlMode;
|
|
1006
|
-
})();
|
|
1131
|
+
const firecrawlMode = requestedFirecrawlMode;
|
|
1007
1132
|
if (firecrawlMode === 'always' && !firecrawlConfigured) {
|
|
1008
1133
|
throw new Error('--firecrawl always requires FIRECRAWL_API_KEY');
|
|
1009
1134
|
}
|
|
@@ -1029,7 +1154,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1029
1154
|
}
|
|
1030
1155
|
writeVerbose(stderr, verbose, `config url=${url} timeoutMs=${timeoutMs} youtube=${youtubeMode} firecrawl=${firecrawlMode} length=${lengthArg.kind === 'preset' ? lengthArg.preset : `${lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(maxOutputTokensArg)} json=${json} extractOnly=${extractOnly} markdown=${effectiveMarkdownMode} model=${model} stream=${effectiveStreamMode} render=${effectiveRenderMode}`, verboseColor);
|
|
1031
1156
|
writeVerbose(stderr, verbose, `configFile path=${formatOptionalString(configPath)} model=${formatOptionalString(config?.model ?? null)}`, verboseColor);
|
|
1032
|
-
writeVerbose(stderr, verbose, `env xaiKey=${xaiConfigured} openaiKey=${Boolean(apiKey)} googleKey=${googleConfigured} anthropicKey=${anthropicConfigured} apifyToken=${Boolean(apifyToken)} firecrawlKey=${firecrawlConfigured}`, verboseColor);
|
|
1157
|
+
writeVerbose(stderr, verbose, `env xaiKey=${xaiConfigured} openaiKey=${Boolean(apiKey)} googleKey=${googleConfigured} anthropicKey=${anthropicConfigured} openrouterKey=${openrouterConfigured} apifyToken=${Boolean(apifyToken)} firecrawlKey=${firecrawlConfigured}`, verboseColor);
|
|
1033
1158
|
writeVerbose(stderr, verbose, `markdown requested=${markdownRequested} provider=${markdownProvider}`, verboseColor);
|
|
1034
1159
|
const scrapeWithFirecrawl = firecrawlConfigured && firecrawlMode !== 'off'
|
|
1035
1160
|
? createFirecrawlScraper({ apiKey: firecrawlApiKey, fetchImpl: trackedFetch })
|
|
@@ -1041,12 +1166,17 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1041
1166
|
googleApiKey: googleConfigured ? googleApiKey : null,
|
|
1042
1167
|
openaiApiKey: apiKey,
|
|
1043
1168
|
anthropicApiKey: anthropicConfigured ? anthropicApiKey : null,
|
|
1169
|
+
openrouterApiKey: openrouterConfigured ? openrouterApiKey : null,
|
|
1170
|
+
openrouter: openrouterOptions,
|
|
1044
1171
|
fetchImpl: trackedFetch,
|
|
1045
1172
|
onUsage: ({ model: usedModel, provider, usage }) => {
|
|
1046
1173
|
llmCalls.push({ provider, model: usedModel, usage, purpose: 'markdown' });
|
|
1047
1174
|
},
|
|
1048
1175
|
})
|
|
1049
1176
|
: null;
|
|
1177
|
+
const readTweetWithBirdClient = hasBirdCli(env)
|
|
1178
|
+
? ({ url, timeoutMs }) => readTweetWithBird({ url, timeoutMs, env })
|
|
1179
|
+
: null;
|
|
1050
1180
|
writeVerbose(stderr, verbose, 'extract start', verboseColor);
|
|
1051
1181
|
const stopOscProgress = startOscProgress({
|
|
1052
1182
|
label: 'Fetching website',
|
|
@@ -1067,22 +1197,65 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1067
1197
|
phase: 'idle',
|
|
1068
1198
|
htmlDownloadedBytes: 0,
|
|
1069
1199
|
htmlTotalBytes: null,
|
|
1200
|
+
fetchStartedAtMs: null,
|
|
1070
1201
|
lastSpinnerUpdateAtMs: 0,
|
|
1071
1202
|
};
|
|
1072
|
-
|
|
1203
|
+
let ticker = null;
|
|
1204
|
+
const updateSpinner = (text, options) => {
|
|
1073
1205
|
const now = Date.now();
|
|
1074
|
-
if (now - state.lastSpinnerUpdateAtMs < 100)
|
|
1206
|
+
if (!options?.force && now - state.lastSpinnerUpdateAtMs < 100)
|
|
1075
1207
|
return;
|
|
1076
1208
|
state.lastSpinnerUpdateAtMs = now;
|
|
1077
1209
|
spinner.setText(text);
|
|
1078
1210
|
};
|
|
1211
|
+
const formatFirecrawlReason = (reason) => {
|
|
1212
|
+
const lower = reason.toLowerCase();
|
|
1213
|
+
if (lower.includes('forced'))
|
|
1214
|
+
return 'forced';
|
|
1215
|
+
if (lower.includes('html fetch failed'))
|
|
1216
|
+
return 'fallback: HTML fetch failed';
|
|
1217
|
+
if (lower.includes('blocked') || lower.includes('thin'))
|
|
1218
|
+
return 'fallback: blocked/thin HTML';
|
|
1219
|
+
return reason;
|
|
1220
|
+
};
|
|
1221
|
+
const renderFetchLine = () => {
|
|
1222
|
+
const downloaded = formatBytes(state.htmlDownloadedBytes);
|
|
1223
|
+
const total = typeof state.htmlTotalBytes === 'number' ? `/${formatBytes(state.htmlTotalBytes)}` : '';
|
|
1224
|
+
const elapsedMs = typeof state.fetchStartedAtMs === 'number' ? Date.now() - state.fetchStartedAtMs : 0;
|
|
1225
|
+
const elapsed = formatElapsedMs(elapsedMs);
|
|
1226
|
+
if (state.htmlDownloadedBytes === 0 && !state.htmlTotalBytes) {
|
|
1227
|
+
return `Fetching website (connecting, ${elapsed})…`;
|
|
1228
|
+
}
|
|
1229
|
+
const rate = elapsedMs > 0 && state.htmlDownloadedBytes > 0
|
|
1230
|
+
? `, ${formatBytes(state.htmlDownloadedBytes / (elapsedMs / 1000))}/s`
|
|
1231
|
+
: '';
|
|
1232
|
+
return `Fetching website (${downloaded}${total}, ${elapsed}${rate})…`;
|
|
1233
|
+
};
|
|
1234
|
+
const startTicker = () => {
|
|
1235
|
+
if (ticker)
|
|
1236
|
+
return;
|
|
1237
|
+
ticker = setInterval(() => {
|
|
1238
|
+
if (state.phase !== 'fetching')
|
|
1239
|
+
return;
|
|
1240
|
+
updateSpinner(renderFetchLine());
|
|
1241
|
+
}, 1000);
|
|
1242
|
+
};
|
|
1243
|
+
const stopTicker = () => {
|
|
1244
|
+
if (!ticker)
|
|
1245
|
+
return;
|
|
1246
|
+
clearInterval(ticker);
|
|
1247
|
+
ticker = null;
|
|
1248
|
+
};
|
|
1079
1249
|
return {
|
|
1080
1250
|
getHtmlDownloadedBytes: () => state.htmlDownloadedBytes,
|
|
1251
|
+
stop: stopTicker,
|
|
1081
1252
|
onProgress: (event) => {
|
|
1082
1253
|
if (event.kind === 'fetch-html-start') {
|
|
1083
1254
|
state.phase = 'fetching';
|
|
1084
1255
|
state.htmlDownloadedBytes = 0;
|
|
1085
1256
|
state.htmlTotalBytes = null;
|
|
1257
|
+
state.fetchStartedAtMs = Date.now();
|
|
1258
|
+
startTicker();
|
|
1086
1259
|
updateSpinner('Fetching website (connecting)…');
|
|
1087
1260
|
return;
|
|
1088
1261
|
}
|
|
@@ -1090,31 +1263,69 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1090
1263
|
state.phase = 'fetching';
|
|
1091
1264
|
state.htmlDownloadedBytes = event.downloadedBytes;
|
|
1092
1265
|
state.htmlTotalBytes = event.totalBytes;
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1266
|
+
updateSpinner(renderFetchLine());
|
|
1267
|
+
return;
|
|
1268
|
+
}
|
|
1269
|
+
if (event.kind === 'bird-start') {
|
|
1270
|
+
state.phase = 'bird';
|
|
1271
|
+
stopTicker();
|
|
1272
|
+
updateSpinner('Bird: reading tweet…', { force: true });
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
1275
|
+
if (event.kind === 'bird-done') {
|
|
1276
|
+
state.phase = 'bird';
|
|
1277
|
+
stopTicker();
|
|
1278
|
+
if (event.ok && typeof event.textBytes === 'number') {
|
|
1279
|
+
updateSpinner(`Bird: got ${formatBytes(event.textBytes)}…`, { force: true });
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
updateSpinner('Bird: failed; fallback…', { force: true });
|
|
1283
|
+
return;
|
|
1284
|
+
}
|
|
1285
|
+
if (event.kind === 'nitter-start') {
|
|
1286
|
+
state.phase = 'nitter';
|
|
1287
|
+
stopTicker();
|
|
1288
|
+
updateSpinner('Nitter: fetching…', { force: true });
|
|
1289
|
+
return;
|
|
1290
|
+
}
|
|
1291
|
+
if (event.kind === 'nitter-done') {
|
|
1292
|
+
state.phase = 'nitter';
|
|
1293
|
+
stopTicker();
|
|
1294
|
+
if (event.ok && typeof event.textBytes === 'number') {
|
|
1295
|
+
updateSpinner(`Nitter: got ${formatBytes(event.textBytes)}…`, { force: true });
|
|
1296
|
+
return;
|
|
1297
|
+
}
|
|
1298
|
+
updateSpinner('Nitter: failed; fallback…', { force: true });
|
|
1096
1299
|
return;
|
|
1097
1300
|
}
|
|
1098
1301
|
if (event.kind === 'firecrawl-start') {
|
|
1099
1302
|
state.phase = 'firecrawl';
|
|
1100
|
-
|
|
1303
|
+
stopTicker();
|
|
1304
|
+
const reason = event.reason ? formatFirecrawlReason(event.reason) : '';
|
|
1305
|
+
const suffix = reason ? ` (${reason})` : '';
|
|
1306
|
+
updateSpinner(`Firecrawl: scraping${suffix}…`, { force: true });
|
|
1101
1307
|
return;
|
|
1102
1308
|
}
|
|
1103
1309
|
if (event.kind === 'firecrawl-done') {
|
|
1104
1310
|
state.phase = 'firecrawl';
|
|
1311
|
+
stopTicker();
|
|
1105
1312
|
if (event.ok && typeof event.markdownBytes === 'number') {
|
|
1106
|
-
updateSpinner(`Firecrawl: got ${formatBytes(event.markdownBytes)}
|
|
1313
|
+
updateSpinner(`Firecrawl: got ${formatBytes(event.markdownBytes)}…`, { force: true });
|
|
1107
1314
|
return;
|
|
1108
1315
|
}
|
|
1109
|
-
updateSpinner('Firecrawl: no content; fallback…');
|
|
1316
|
+
updateSpinner('Firecrawl: no content; fallback…', { force: true });
|
|
1110
1317
|
}
|
|
1111
1318
|
},
|
|
1112
1319
|
};
|
|
1113
1320
|
})();
|
|
1114
1321
|
const client = createLinkPreviewClient({
|
|
1115
1322
|
apifyApiToken: apifyToken,
|
|
1323
|
+
ytDlpPath,
|
|
1324
|
+
falApiKey,
|
|
1325
|
+
openaiApiKey: openaiTranscriptionKey,
|
|
1116
1326
|
scrapeWithFirecrawl,
|
|
1117
1327
|
convertHtmlToMarkdown,
|
|
1328
|
+
readTweetWithBird: readTweetWithBirdClient,
|
|
1118
1329
|
fetch: trackedFetch,
|
|
1119
1330
|
onProgress: websiteProgress?.onProgress ?? null,
|
|
1120
1331
|
});
|
|
@@ -1123,24 +1334,42 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1123
1334
|
if (stopped)
|
|
1124
1335
|
return;
|
|
1125
1336
|
stopped = true;
|
|
1337
|
+
websiteProgress?.stop?.();
|
|
1126
1338
|
spinner.stopAndClear();
|
|
1127
1339
|
stopOscProgress();
|
|
1128
1340
|
};
|
|
1129
1341
|
clearProgressBeforeStdout = stopProgress;
|
|
1130
1342
|
try {
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1343
|
+
let extracted;
|
|
1344
|
+
try {
|
|
1345
|
+
extracted = await client.fetchLinkContent(url, {
|
|
1346
|
+
timeoutMs,
|
|
1347
|
+
youtubeTranscript: youtubeMode,
|
|
1348
|
+
firecrawl: firecrawlMode,
|
|
1349
|
+
format: markdownRequested ? 'markdown' : 'text',
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
1352
|
+
catch (error) {
|
|
1353
|
+
throw withBirdTip(error, url, env);
|
|
1354
|
+
}
|
|
1137
1355
|
const extractedContentBytes = Buffer.byteLength(extracted.content, 'utf8');
|
|
1138
1356
|
const extractedContentSize = formatBytes(extractedContentBytes);
|
|
1139
|
-
const
|
|
1357
|
+
const viaSources = [];
|
|
1358
|
+
if (extracted.diagnostics.strategy === 'bird') {
|
|
1359
|
+
viaSources.push('bird');
|
|
1360
|
+
}
|
|
1361
|
+
if (extracted.diagnostics.strategy === 'nitter') {
|
|
1362
|
+
viaSources.push('Nitter');
|
|
1363
|
+
}
|
|
1364
|
+
if (extracted.diagnostics.firecrawl.used) {
|
|
1365
|
+
viaSources.push('Firecrawl');
|
|
1366
|
+
}
|
|
1367
|
+
const viaSourceLabel = viaSources.length > 0 ? `, ${viaSources.join('+')}` : '';
|
|
1140
1368
|
if (progressEnabled) {
|
|
1369
|
+
websiteProgress?.stop?.();
|
|
1141
1370
|
spinner.setText(extractOnly
|
|
1142
|
-
? `Extracted (${extractedContentSize})`
|
|
1143
|
-
: `Summarizing (sent ${extractedContentSize}${
|
|
1371
|
+
? `Extracted (${extractedContentSize}${viaSourceLabel})`
|
|
1372
|
+
: `Summarizing (sent ${extractedContentSize}${viaSourceLabel})…`);
|
|
1144
1373
|
}
|
|
1145
1374
|
writeVerbose(stderr, verbose, `extract done strategy=${extracted.diagnostics.strategy} siteName=${formatOptionalString(extracted.siteName)} title=${formatOptionalString(extracted.title)} transcriptSource=${formatOptionalString(extracted.transcriptSource)}`, verboseColor);
|
|
1146
1375
|
writeVerbose(stderr, verbose, `extract stats characters=${extracted.totalCharacters} words=${extracted.wordCount} transcriptCharacters=${formatOptionalNumber(extracted.transcriptCharacters)} transcriptLines=${formatOptionalNumber(extracted.transcriptLines)}`, verboseColor);
|
|
@@ -1204,8 +1433,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1204
1433
|
stderr,
|
|
1205
1434
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1206
1435
|
model,
|
|
1207
|
-
strategy: 'none',
|
|
1208
|
-
chunkCount: null,
|
|
1209
1436
|
report: finishReport,
|
|
1210
1437
|
costUsd,
|
|
1211
1438
|
color: verboseColor,
|
|
@@ -1223,8 +1450,76 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1223
1450
|
stderr,
|
|
1224
1451
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1225
1452
|
model,
|
|
1226
|
-
|
|
1227
|
-
|
|
1453
|
+
report,
|
|
1454
|
+
costUsd,
|
|
1455
|
+
color: verboseColor,
|
|
1456
|
+
});
|
|
1457
|
+
}
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
const shouldSkipTweetSummary = isTwitterStatusUrl(url) &&
|
|
1461
|
+
extracted.content.length > 0 &&
|
|
1462
|
+
extracted.content.length <= resolveTargetCharacters(lengthArg);
|
|
1463
|
+
if (shouldSkipTweetSummary) {
|
|
1464
|
+
clearProgressForStdout();
|
|
1465
|
+
writeVerbose(stderr, verbose, `skip summary: tweet content length=${extracted.content.length} target=${resolveTargetCharacters(lengthArg)}`, verboseColor);
|
|
1466
|
+
if (json) {
|
|
1467
|
+
const finishReport = shouldComputeReport ? await buildReport() : null;
|
|
1468
|
+
const payload = {
|
|
1469
|
+
input: {
|
|
1470
|
+
kind: 'url',
|
|
1471
|
+
url,
|
|
1472
|
+
timeoutMs,
|
|
1473
|
+
youtube: youtubeMode,
|
|
1474
|
+
firecrawl: firecrawlMode,
|
|
1475
|
+
markdown: effectiveMarkdownMode,
|
|
1476
|
+
length: lengthArg.kind === 'preset'
|
|
1477
|
+
? { kind: 'preset', preset: lengthArg.preset }
|
|
1478
|
+
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
1479
|
+
maxOutputTokens: maxOutputTokensArg,
|
|
1480
|
+
model,
|
|
1481
|
+
},
|
|
1482
|
+
env: {
|
|
1483
|
+
hasXaiKey: Boolean(xaiApiKey),
|
|
1484
|
+
hasOpenAIKey: Boolean(apiKey),
|
|
1485
|
+
hasApifyToken: Boolean(apifyToken),
|
|
1486
|
+
hasFirecrawlKey: firecrawlConfigured,
|
|
1487
|
+
hasGoogleKey: googleConfigured,
|
|
1488
|
+
hasAnthropicKey: anthropicConfigured,
|
|
1489
|
+
},
|
|
1490
|
+
extracted,
|
|
1491
|
+
prompt,
|
|
1492
|
+
llm: null,
|
|
1493
|
+
metrics: metricsEnabled ? finishReport : null,
|
|
1494
|
+
summary: extracted.content,
|
|
1495
|
+
};
|
|
1496
|
+
if (metricsDetailed && finishReport) {
|
|
1497
|
+
writeMetricsReport(finishReport);
|
|
1498
|
+
}
|
|
1499
|
+
stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
1500
|
+
if (metricsEnabled && finishReport) {
|
|
1501
|
+
const costUsd = await estimateCostUsd();
|
|
1502
|
+
writeFinishLine({
|
|
1503
|
+
stderr,
|
|
1504
|
+
elapsedMs: Date.now() - runStartedAtMs,
|
|
1505
|
+
model,
|
|
1506
|
+
report: finishReport,
|
|
1507
|
+
costUsd,
|
|
1508
|
+
color: verboseColor,
|
|
1509
|
+
});
|
|
1510
|
+
}
|
|
1511
|
+
return;
|
|
1512
|
+
}
|
|
1513
|
+
stdout.write(`${extracted.content}\n`);
|
|
1514
|
+
const report = shouldComputeReport ? await buildReport() : null;
|
|
1515
|
+
if (metricsDetailed && report)
|
|
1516
|
+
writeMetricsReport(report);
|
|
1517
|
+
if (metricsEnabled && report) {
|
|
1518
|
+
const costUsd = await estimateCostUsd();
|
|
1519
|
+
writeFinishLine({
|
|
1520
|
+
stderr,
|
|
1521
|
+
elapsedMs: Date.now() - runStartedAtMs,
|
|
1522
|
+
model,
|
|
1228
1523
|
report,
|
|
1229
1524
|
costUsd,
|
|
1230
1525
|
color: verboseColor,
|
|
@@ -1238,6 +1533,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1238
1533
|
openaiApiKey: apiKey,
|
|
1239
1534
|
googleApiKey: googleConfigured ? googleApiKey : null,
|
|
1240
1535
|
anthropicApiKey: anthropicConfigured ? anthropicApiKey : null,
|
|
1536
|
+
openrouterApiKey: openrouterConfigured ? openrouterApiKey : null,
|
|
1241
1537
|
};
|
|
1242
1538
|
const requiredKeyEnv = parsedModel.provider === 'xai'
|
|
1243
1539
|
? 'XAI_API_KEY'
|
|
@@ -1245,14 +1541,14 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1245
1541
|
? 'GEMINI_API_KEY (or GOOGLE_GENERATIVE_AI_API_KEY / GOOGLE_API_KEY)'
|
|
1246
1542
|
: parsedModel.provider === 'anthropic'
|
|
1247
1543
|
? 'ANTHROPIC_API_KEY'
|
|
1248
|
-
: 'OPENAI_API_KEY';
|
|
1544
|
+
: 'OPENAI_API_KEY (or OPENROUTER_API_KEY)';
|
|
1249
1545
|
const hasRequiredKey = parsedModel.provider === 'xai'
|
|
1250
1546
|
? Boolean(xaiApiKey)
|
|
1251
1547
|
: parsedModel.provider === 'google'
|
|
1252
1548
|
? googleConfigured
|
|
1253
1549
|
: parsedModel.provider === 'anthropic'
|
|
1254
1550
|
? anthropicConfigured
|
|
1255
|
-
: Boolean(apiKey);
|
|
1551
|
+
: Boolean(apiKey) || openrouterConfigured;
|
|
1256
1552
|
if (!hasRequiredKey) {
|
|
1257
1553
|
throw new Error(`Missing ${requiredKeyEnv} for model ${parsedModel.canonical}. Set the env var or choose a different --model.`);
|
|
1258
1554
|
}
|
|
@@ -1269,321 +1565,168 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1269
1565
|
const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
|
|
1270
1566
|
writeVerbose(stderr, verbose, `mode summarize provider=${parsedModelEffective.provider} model=${parsedModelEffective.canonical}`, verboseColor);
|
|
1271
1567
|
const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
|
|
1272
|
-
const
|
|
1273
|
-
|
|
1274
|
-
|
|
1568
|
+
const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
|
|
1569
|
+
if (typeof maxInputTokensForCall === 'number' &&
|
|
1570
|
+
Number.isFinite(maxInputTokensForCall) &&
|
|
1571
|
+
maxInputTokensForCall > 0) {
|
|
1572
|
+
const tokenCount = countTokens(prompt);
|
|
1573
|
+
if (tokenCount > maxInputTokensForCall) {
|
|
1574
|
+
throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1275
1577
|
const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
|
|
1276
1578
|
const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
|
|
1277
1579
|
const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
|
|
1278
1580
|
let summaryAlreadyPrinted = false;
|
|
1279
1581
|
let summary = '';
|
|
1280
1582
|
let getLastStreamError = null;
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1583
|
+
writeVerbose(stderr, verbose, 'summarize strategy=single', verboseColor);
|
|
1584
|
+
if (streamingEnabledForCall) {
|
|
1585
|
+
writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
|
|
1586
|
+
let streamResult = null;
|
|
1587
|
+
try {
|
|
1588
|
+
streamResult = await streamTextWithModelId({
|
|
1589
|
+
modelId: parsedModelEffective.canonical,
|
|
1590
|
+
apiKeys: apiKeysForLlm,
|
|
1591
|
+
prompt,
|
|
1592
|
+
temperature: 0,
|
|
1593
|
+
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1594
|
+
timeoutMs,
|
|
1595
|
+
fetchImpl: trackedFetch,
|
|
1596
|
+
});
|
|
1597
|
+
}
|
|
1598
|
+
catch (error) {
|
|
1599
|
+
if (isStreamingTimeoutError(error)) {
|
|
1600
|
+
writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
|
|
1601
|
+
const result = await summarizeWithModelId({
|
|
1288
1602
|
modelId: parsedModelEffective.canonical,
|
|
1289
|
-
apiKeys: apiKeysForLlm,
|
|
1290
1603
|
prompt,
|
|
1291
|
-
temperature: 0,
|
|
1292
1604
|
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1293
1605
|
timeoutMs,
|
|
1294
1606
|
fetchImpl: trackedFetch,
|
|
1607
|
+
apiKeys: apiKeysForLlm,
|
|
1608
|
+
openrouter: openrouterOptions,
|
|
1295
1609
|
});
|
|
1296
|
-
}
|
|
1297
|
-
catch (error) {
|
|
1298
|
-
if (parsedModelEffective.provider === 'google' &&
|
|
1299
|
-
isGoogleStreamingUnsupportedError(error)) {
|
|
1300
|
-
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
1301
|
-
const result = await summarizeWithModelId({
|
|
1302
|
-
modelId: parsedModelEffective.canonical,
|
|
1303
|
-
prompt,
|
|
1304
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1305
|
-
timeoutMs,
|
|
1306
|
-
fetchImpl: trackedFetch,
|
|
1307
|
-
apiKeys: apiKeysForLlm,
|
|
1308
|
-
});
|
|
1309
|
-
llmCalls.push({
|
|
1310
|
-
provider: result.provider,
|
|
1311
|
-
model: result.canonicalModelId,
|
|
1312
|
-
usage: result.usage,
|
|
1313
|
-
purpose: 'summary',
|
|
1314
|
-
});
|
|
1315
|
-
summary = result.text;
|
|
1316
|
-
streamResult = null;
|
|
1317
|
-
}
|
|
1318
|
-
else {
|
|
1319
|
-
throw error;
|
|
1320
|
-
}
|
|
1321
|
-
}
|
|
1322
|
-
if (streamResult) {
|
|
1323
|
-
getLastStreamError = streamResult.lastError;
|
|
1324
|
-
let streamed = '';
|
|
1325
|
-
const liveRenderer = shouldLiveRenderSummary
|
|
1326
|
-
? createLiveRenderer({
|
|
1327
|
-
write: (chunk) => {
|
|
1328
|
-
clearProgressForStdout();
|
|
1329
|
-
stdout.write(chunk);
|
|
1330
|
-
},
|
|
1331
|
-
width: markdownRenderWidth(stdout, env),
|
|
1332
|
-
renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
|
|
1333
|
-
width: markdownRenderWidth(stdout, env),
|
|
1334
|
-
wrap: true,
|
|
1335
|
-
color: supportsColor(stdout, env),
|
|
1336
|
-
}),
|
|
1337
|
-
})
|
|
1338
|
-
: null;
|
|
1339
|
-
let lastFrameAtMs = 0;
|
|
1340
|
-
try {
|
|
1341
|
-
let cleared = false;
|
|
1342
|
-
for await (const delta of streamResult.textStream) {
|
|
1343
|
-
const merged = mergeStreamingChunk(streamed, delta);
|
|
1344
|
-
streamed = merged.next;
|
|
1345
|
-
if (shouldStreamSummaryToStdout) {
|
|
1346
|
-
if (!cleared) {
|
|
1347
|
-
clearProgressForStdout();
|
|
1348
|
-
cleared = true;
|
|
1349
|
-
}
|
|
1350
|
-
if (merged.appended)
|
|
1351
|
-
stdout.write(merged.appended);
|
|
1352
|
-
continue;
|
|
1353
|
-
}
|
|
1354
|
-
if (liveRenderer) {
|
|
1355
|
-
const now = Date.now();
|
|
1356
|
-
const due = now - lastFrameAtMs >= 120;
|
|
1357
|
-
const hasNewline = delta.includes('\n');
|
|
1358
|
-
if (hasNewline || due) {
|
|
1359
|
-
liveRenderer.render(streamed);
|
|
1360
|
-
lastFrameAtMs = now;
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
}
|
|
1364
|
-
const trimmed = streamed.trim();
|
|
1365
|
-
streamed = trimmed;
|
|
1366
|
-
if (liveRenderer) {
|
|
1367
|
-
liveRenderer.render(trimmed);
|
|
1368
|
-
summaryAlreadyPrinted = true;
|
|
1369
|
-
}
|
|
1370
|
-
}
|
|
1371
|
-
finally {
|
|
1372
|
-
liveRenderer?.finish();
|
|
1373
|
-
}
|
|
1374
|
-
const usage = await streamResult.usage;
|
|
1375
1610
|
llmCalls.push({
|
|
1376
|
-
provider:
|
|
1377
|
-
model:
|
|
1378
|
-
usage,
|
|
1611
|
+
provider: result.provider,
|
|
1612
|
+
model: result.canonicalModelId,
|
|
1613
|
+
usage: result.usage,
|
|
1379
1614
|
purpose: 'summary',
|
|
1380
1615
|
});
|
|
1381
|
-
summary =
|
|
1382
|
-
|
|
1383
|
-
if (!streamed.endsWith('\n')) {
|
|
1384
|
-
stdout.write('\n');
|
|
1385
|
-
}
|
|
1386
|
-
summaryAlreadyPrinted = true;
|
|
1387
|
-
}
|
|
1616
|
+
summary = result.text;
|
|
1617
|
+
streamResult = null;
|
|
1388
1618
|
}
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
prompt,
|
|
1394
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1395
|
-
timeoutMs,
|
|
1396
|
-
fetchImpl: trackedFetch,
|
|
1397
|
-
apiKeys: apiKeysForLlm,
|
|
1398
|
-
});
|
|
1399
|
-
llmCalls.push({
|
|
1400
|
-
provider: result.provider,
|
|
1401
|
-
model: result.canonicalModelId,
|
|
1402
|
-
usage: result.usage,
|
|
1403
|
-
purpose: 'summary',
|
|
1404
|
-
});
|
|
1405
|
-
summary = result.text;
|
|
1406
|
-
}
|
|
1407
|
-
}
|
|
1408
|
-
else {
|
|
1409
|
-
strategy = 'map-reduce';
|
|
1410
|
-
const chunks = splitTextIntoChunks(extracted.content, MAP_REDUCE_CHUNK_CHARACTERS);
|
|
1411
|
-
chunkCount = chunks.length;
|
|
1412
|
-
stderr.write(`Large input (${extracted.content.length} chars); summarizing in ${chunks.length} chunks.\n`);
|
|
1413
|
-
writeVerbose(stderr, verbose, `summarize strategy=map-reduce chunks=${chunks.length}`, verboseColor);
|
|
1414
|
-
const chunkNotes = [];
|
|
1415
|
-
for (let i = 0; i < chunks.length; i += 1) {
|
|
1416
|
-
writeVerbose(stderr, verbose, `summarize chunk ${i + 1}/${chunks.length} notes start`, verboseColor);
|
|
1417
|
-
const chunkPrompt = buildChunkNotesPrompt({
|
|
1418
|
-
content: chunks[i] ?? '',
|
|
1419
|
-
});
|
|
1420
|
-
const chunkNoteTokensRequested = typeof maxOutputTokensArg === 'number'
|
|
1421
|
-
? Math.min(SUMMARY_LENGTH_TO_TOKENS.medium, maxOutputTokensArg)
|
|
1422
|
-
: SUMMARY_LENGTH_TO_TOKENS.medium;
|
|
1423
|
-
const chunkNoteTokens = await capMaxOutputTokensForModel({
|
|
1424
|
-
modelId: parsedModelEffective.canonical,
|
|
1425
|
-
requested: chunkNoteTokensRequested,
|
|
1426
|
-
});
|
|
1427
|
-
const notesResult = await summarizeWithModelId({
|
|
1428
|
-
modelId: parsedModelEffective.canonical,
|
|
1429
|
-
prompt: chunkPrompt,
|
|
1430
|
-
maxOutputTokens: chunkNoteTokens,
|
|
1431
|
-
timeoutMs,
|
|
1432
|
-
fetchImpl: trackedFetch,
|
|
1433
|
-
apiKeys: apiKeysForLlm,
|
|
1434
|
-
});
|
|
1435
|
-
const notes = notesResult.text;
|
|
1436
|
-
llmCalls.push({
|
|
1437
|
-
provider: notesResult.provider,
|
|
1438
|
-
model: notesResult.canonicalModelId,
|
|
1439
|
-
usage: notesResult.usage,
|
|
1440
|
-
purpose: 'chunk-notes',
|
|
1441
|
-
});
|
|
1442
|
-
chunkNotes.push(notes.trim());
|
|
1443
|
-
}
|
|
1444
|
-
writeVerbose(stderr, verbose, 'summarize merge chunk notes', verboseColor);
|
|
1445
|
-
const mergedContent = `Chunk notes (generated from the full input):\n\n${chunkNotes
|
|
1446
|
-
.filter((value) => value.length > 0)
|
|
1447
|
-
.join('\n\n')}`;
|
|
1448
|
-
const mergedPrompt = buildLinkSummaryPrompt({
|
|
1449
|
-
url: extracted.url,
|
|
1450
|
-
title: extracted.title,
|
|
1451
|
-
siteName: extracted.siteName,
|
|
1452
|
-
description: extracted.description,
|
|
1453
|
-
content: mergedContent,
|
|
1454
|
-
truncated: false,
|
|
1455
|
-
hasTranscript: isYouTube ||
|
|
1456
|
-
(extracted.transcriptSource !== null && extracted.transcriptSource !== 'unavailable'),
|
|
1457
|
-
summaryLength: lengthArg.kind === 'preset'
|
|
1458
|
-
? lengthArg.preset
|
|
1459
|
-
: { maxCharacters: lengthArg.maxCharacters },
|
|
1460
|
-
shares: [],
|
|
1461
|
-
});
|
|
1462
|
-
if (streamingEnabledForCall) {
|
|
1463
|
-
writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
|
|
1464
|
-
let streamResult = null;
|
|
1465
|
-
try {
|
|
1466
|
-
streamResult = await streamTextWithModelId({
|
|
1619
|
+
else if (parsedModelEffective.provider === 'google' &&
|
|
1620
|
+
isGoogleStreamingUnsupportedError(error)) {
|
|
1621
|
+
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
1622
|
+
const result = await summarizeWithModelId({
|
|
1467
1623
|
modelId: parsedModelEffective.canonical,
|
|
1468
|
-
|
|
1469
|
-
prompt: mergedPrompt,
|
|
1470
|
-
temperature: 0,
|
|
1624
|
+
prompt,
|
|
1471
1625
|
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1472
1626
|
timeoutMs,
|
|
1473
1627
|
fetchImpl: trackedFetch,
|
|
1628
|
+
apiKeys: apiKeysForLlm,
|
|
1629
|
+
openrouter: openrouterOptions,
|
|
1630
|
+
});
|
|
1631
|
+
llmCalls.push({
|
|
1632
|
+
provider: result.provider,
|
|
1633
|
+
model: result.canonicalModelId,
|
|
1634
|
+
usage: result.usage,
|
|
1635
|
+
purpose: 'summary',
|
|
1474
1636
|
});
|
|
1637
|
+
summary = result.text;
|
|
1638
|
+
streamResult = null;
|
|
1475
1639
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
isGoogleStreamingUnsupportedError(error)) {
|
|
1479
|
-
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
1480
|
-
const mergedResult = await summarizeWithModelId({
|
|
1481
|
-
modelId: parsedModelEffective.canonical,
|
|
1482
|
-
prompt: mergedPrompt,
|
|
1483
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1484
|
-
timeoutMs,
|
|
1485
|
-
fetchImpl: trackedFetch,
|
|
1486
|
-
apiKeys: apiKeysForLlm,
|
|
1487
|
-
});
|
|
1488
|
-
llmCalls.push({
|
|
1489
|
-
provider: mergedResult.provider,
|
|
1490
|
-
model: mergedResult.canonicalModelId,
|
|
1491
|
-
usage: mergedResult.usage,
|
|
1492
|
-
purpose: 'summary',
|
|
1493
|
-
});
|
|
1494
|
-
summary = mergedResult.text;
|
|
1495
|
-
streamResult = null;
|
|
1496
|
-
}
|
|
1497
|
-
else {
|
|
1498
|
-
throw error;
|
|
1499
|
-
}
|
|
1640
|
+
else {
|
|
1641
|
+
throw error;
|
|
1500
1642
|
}
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1643
|
+
}
|
|
1644
|
+
if (streamResult) {
|
|
1645
|
+
getLastStreamError = streamResult.lastError;
|
|
1646
|
+
let streamed = '';
|
|
1647
|
+
const liveRenderer = shouldLiveRenderSummary
|
|
1648
|
+
? createLiveRenderer({
|
|
1649
|
+
write: (chunk) => {
|
|
1650
|
+
clearProgressForStdout();
|
|
1651
|
+
stdout.write(chunk);
|
|
1652
|
+
},
|
|
1653
|
+
width: markdownRenderWidth(stdout, env),
|
|
1654
|
+
renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
|
|
1510
1655
|
width: markdownRenderWidth(stdout, env),
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
let
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1656
|
+
wrap: true,
|
|
1657
|
+
color: supportsColor(stdout, env),
|
|
1658
|
+
}),
|
|
1659
|
+
})
|
|
1660
|
+
: null;
|
|
1661
|
+
let lastFrameAtMs = 0;
|
|
1662
|
+
try {
|
|
1663
|
+
let cleared = false;
|
|
1664
|
+
for await (const delta of streamResult.textStream) {
|
|
1665
|
+
const merged = mergeStreamingChunk(streamed, delta);
|
|
1666
|
+
streamed = merged.next;
|
|
1667
|
+
if (shouldStreamSummaryToStdout) {
|
|
1522
1668
|
if (!cleared) {
|
|
1523
1669
|
clearProgressForStdout();
|
|
1524
1670
|
cleared = true;
|
|
1525
1671
|
}
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
if (merged.appended)
|
|
1530
|
-
stdout.write(merged.appended);
|
|
1531
|
-
continue;
|
|
1532
|
-
}
|
|
1533
|
-
if (liveRenderer) {
|
|
1534
|
-
const now = Date.now();
|
|
1535
|
-
const due = now - lastFrameAtMs >= 120;
|
|
1536
|
-
const hasNewline = delta.includes('\n');
|
|
1537
|
-
if (hasNewline || due) {
|
|
1538
|
-
liveRenderer.render(streamed);
|
|
1539
|
-
lastFrameAtMs = now;
|
|
1540
|
-
}
|
|
1541
|
-
}
|
|
1672
|
+
if (merged.appended)
|
|
1673
|
+
stdout.write(merged.appended);
|
|
1674
|
+
continue;
|
|
1542
1675
|
}
|
|
1543
|
-
const trimmed = streamed.trim();
|
|
1544
|
-
streamed = trimmed;
|
|
1545
1676
|
if (liveRenderer) {
|
|
1546
|
-
|
|
1547
|
-
|
|
1677
|
+
const now = Date.now();
|
|
1678
|
+
const due = now - lastFrameAtMs >= 120;
|
|
1679
|
+
const hasNewline = delta.includes('\n');
|
|
1680
|
+
if (hasNewline || due) {
|
|
1681
|
+
liveRenderer.render(streamed);
|
|
1682
|
+
lastFrameAtMs = now;
|
|
1683
|
+
}
|
|
1548
1684
|
}
|
|
1549
1685
|
}
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
llmCalls.push({
|
|
1555
|
-
provider: streamResult.provider,
|
|
1556
|
-
model: streamResult.canonicalModelId,
|
|
1557
|
-
usage,
|
|
1558
|
-
purpose: 'summary',
|
|
1559
|
-
});
|
|
1560
|
-
summary = streamed;
|
|
1561
|
-
if (shouldStreamSummaryToStdout) {
|
|
1562
|
-
if (!streamed.endsWith('\n')) {
|
|
1563
|
-
stdout.write('\n');
|
|
1564
|
-
}
|
|
1686
|
+
const trimmed = streamed.trim();
|
|
1687
|
+
streamed = trimmed;
|
|
1688
|
+
if (liveRenderer) {
|
|
1689
|
+
liveRenderer.render(trimmed);
|
|
1565
1690
|
summaryAlreadyPrinted = true;
|
|
1566
1691
|
}
|
|
1567
1692
|
}
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
prompt: mergedPrompt,
|
|
1573
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1574
|
-
timeoutMs,
|
|
1575
|
-
fetchImpl: trackedFetch,
|
|
1576
|
-
apiKeys: apiKeysForLlm,
|
|
1577
|
-
});
|
|
1693
|
+
finally {
|
|
1694
|
+
liveRenderer?.finish();
|
|
1695
|
+
}
|
|
1696
|
+
const usage = await streamResult.usage;
|
|
1578
1697
|
llmCalls.push({
|
|
1579
|
-
provider:
|
|
1580
|
-
model:
|
|
1581
|
-
usage
|
|
1698
|
+
provider: streamResult.provider,
|
|
1699
|
+
model: streamResult.canonicalModelId,
|
|
1700
|
+
usage,
|
|
1582
1701
|
purpose: 'summary',
|
|
1583
1702
|
});
|
|
1584
|
-
summary =
|
|
1703
|
+
summary = streamed;
|
|
1704
|
+
if (shouldStreamSummaryToStdout) {
|
|
1705
|
+
if (!streamed.endsWith('\n')) {
|
|
1706
|
+
stdout.write('\n');
|
|
1707
|
+
}
|
|
1708
|
+
summaryAlreadyPrinted = true;
|
|
1709
|
+
}
|
|
1585
1710
|
}
|
|
1586
1711
|
}
|
|
1712
|
+
else {
|
|
1713
|
+
const result = await summarizeWithModelId({
|
|
1714
|
+
modelId: parsedModelEffective.canonical,
|
|
1715
|
+
prompt,
|
|
1716
|
+
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1717
|
+
timeoutMs,
|
|
1718
|
+
fetchImpl: trackedFetch,
|
|
1719
|
+
apiKeys: apiKeysForLlm,
|
|
1720
|
+
openrouter: openrouterOptions,
|
|
1721
|
+
});
|
|
1722
|
+
llmCalls.push({
|
|
1723
|
+
provider: result.provider,
|
|
1724
|
+
model: result.canonicalModelId,
|
|
1725
|
+
usage: result.usage,
|
|
1726
|
+
purpose: 'summary',
|
|
1727
|
+
});
|
|
1728
|
+
summary = result.text;
|
|
1729
|
+
}
|
|
1587
1730
|
summary = summary.trim();
|
|
1588
1731
|
if (summary.length === 0) {
|
|
1589
1732
|
const last = getLastStreamError?.();
|
|
@@ -1622,8 +1765,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1622
1765
|
provider: parsedModelEffective.provider,
|
|
1623
1766
|
model: parsedModelEffective.canonical,
|
|
1624
1767
|
maxCompletionTokens: maxOutputTokensForCall,
|
|
1625
|
-
strategy,
|
|
1626
|
-
chunkCount,
|
|
1768
|
+
strategy: 'single',
|
|
1627
1769
|
},
|
|
1628
1770
|
metrics: metricsEnabled ? finishReport : null,
|
|
1629
1771
|
summary,
|
|
@@ -1638,8 +1780,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1638
1780
|
stderr,
|
|
1639
1781
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1640
1782
|
model: parsedModelEffective.canonical,
|
|
1641
|
-
strategy,
|
|
1642
|
-
chunkCount,
|
|
1643
1783
|
report: finishReport,
|
|
1644
1784
|
costUsd,
|
|
1645
1785
|
color: verboseColor,
|
|
@@ -1670,8 +1810,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1670
1810
|
stderr,
|
|
1671
1811
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1672
1812
|
model: parsedModelEffective.canonical,
|
|
1673
|
-
strategy,
|
|
1674
|
-
chunkCount,
|
|
1675
1813
|
report,
|
|
1676
1814
|
costUsd,
|
|
1677
1815
|
color: verboseColor,
|