@steipete/summarize 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +44 -4
  2. package/README.md +6 -0
  3. package/dist/cli.cjs +2727 -1010
  4. package/dist/cli.cjs.map +4 -4
  5. package/dist/esm/content/asset.js +18 -0
  6. package/dist/esm/content/asset.js.map +1 -1
  7. package/dist/esm/content/link-preview/client.js +2 -0
  8. package/dist/esm/content/link-preview/client.js.map +1 -1
  9. package/dist/esm/content/link-preview/content/article.js +15 -1
  10. package/dist/esm/content/link-preview/content/article.js.map +1 -1
  11. package/dist/esm/content/link-preview/content/index.js +151 -4
  12. package/dist/esm/content/link-preview/content/index.js.map +1 -1
  13. package/dist/esm/flags.js +12 -2
  14. package/dist/esm/flags.js.map +1 -1
  15. package/dist/esm/llm/generate-text.js +74 -7
  16. package/dist/esm/llm/generate-text.js.map +1 -1
  17. package/dist/esm/pricing/litellm.js +4 -1
  18. package/dist/esm/pricing/litellm.js.map +1 -1
  19. package/dist/esm/prompts/file.js +15 -4
  20. package/dist/esm/prompts/file.js.map +1 -1
  21. package/dist/esm/prompts/link-summary.js +20 -6
  22. package/dist/esm/prompts/link-summary.js.map +1 -1
  23. package/dist/esm/run.js +505 -398
  24. package/dist/esm/run.js.map +1 -1
  25. package/dist/esm/version.js +1 -1
  26. package/dist/types/content/link-preview/client.d.ts +2 -1
  27. package/dist/types/content/link-preview/deps.d.ts +30 -0
  28. package/dist/types/content/link-preview/types.d.ts +1 -1
  29. package/dist/types/costs.d.ts +1 -1
  30. package/dist/types/pricing/litellm.d.ts +1 -0
  31. package/dist/types/prompts/file.d.ts +2 -1
  32. package/dist/types/version.d.ts +1 -1
  33. package/docs/extract-only.md +1 -1
  34. package/docs/firecrawl.md +2 -2
  35. package/docs/llm.md +7 -0
  36. package/docs/site/docs/config.html +1 -1
  37. package/docs/site/docs/firecrawl.html +1 -1
  38. package/docs/website.md +3 -3
  39. package/package.json +3 -2
package/dist/esm/run.js CHANGED
@@ -1,6 +1,11 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { accessSync, constants as fsConstants } from 'node:fs';
1
3
  import fs from 'node:fs/promises';
4
+ import path from 'node:path';
2
5
  import { Command, CommanderError, Option } from 'commander';
6
+ import { countTokens } from 'gpt-tokenizer';
3
7
  import { createLiveRenderer, render as renderMarkdownAnsi } from 'markdansi';
8
+ import { normalizeTokenUsage, tallyCosts } from 'tokentally';
4
9
  import { loadSummarizeConfig } from './config.js';
5
10
  import { buildAssetPromptMessages, classifyUrl, loadLocalAsset, loadRemoteAsset, resolveInputTarget, } from './content/asset.js';
6
11
  import { createLinkPreviewClient } from './content/index.js';
@@ -11,21 +16,107 @@ import { generateTextWithModelId, streamTextWithModelId } from './llm/generate-t
11
16
  import { resolveGoogleModelForUsage } from './llm/google-models.js';
12
17
  import { createHtmlToMarkdownConverter } from './llm/html-to-markdown.js';
13
18
  import { normalizeGatewayStyleModelId, parseGatewayStyleModelId } from './llm/model-id.js';
14
- import { loadLiteLlmCatalog, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
15
- import { buildFileSummaryPrompt, buildLinkSummaryPrompt, SUMMARY_LENGTH_TO_TOKENS, } from './prompts/index.js';
19
+ import { loadLiteLlmCatalog, resolveLiteLlmMaxInputTokensForModelId, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
20
+ import { buildFileSummaryPrompt, buildLinkSummaryPrompt } from './prompts/index.js';
16
21
  import { startOscProgress } from './tty/osc-progress.js';
17
22
  import { startSpinner } from './tty/spinner.js';
18
23
  import { resolvePackageVersion } from './version.js';
19
- const MAP_REDUCE_TRIGGER_CHARACTERS = 120_000;
20
- const MAP_REDUCE_CHUNK_CHARACTERS = 60_000;
24
+ const BIRD_TIP = 'Tip: Install bird🐦 for better Twitter support: https://github.com/steipete/bird';
25
+ const TWITTER_HOSTS = new Set(['x.com', 'twitter.com', 'mobile.twitter.com']);
26
+ const SUMMARY_LENGTH_MAX_CHARACTERS = {
27
+ short: 1200,
28
+ medium: 2500,
29
+ long: 6000,
30
+ xl: 14000,
31
+ xxl: Number.POSITIVE_INFINITY,
32
+ };
33
+ function resolveTargetCharacters(lengthArg) {
34
+ return lengthArg.kind === 'chars'
35
+ ? lengthArg.maxCharacters
36
+ : SUMMARY_LENGTH_MAX_CHARACTERS[lengthArg.preset];
37
+ }
38
+ function isTwitterStatusUrl(raw) {
39
+ try {
40
+ const parsed = new URL(raw);
41
+ const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
42
+ if (!TWITTER_HOSTS.has(host))
43
+ return false;
44
+ return /\/status\/\d+/.test(parsed.pathname);
45
+ }
46
+ catch {
47
+ return false;
48
+ }
49
+ }
50
+ function isExecutable(filePath) {
51
+ try {
52
+ accessSync(filePath, fsConstants.X_OK);
53
+ return true;
54
+ }
55
+ catch {
56
+ return false;
57
+ }
58
+ }
59
+ function hasBirdCli(env) {
60
+ const candidates = [];
61
+ const pathEnv = env.PATH ?? process.env.PATH ?? '';
62
+ for (const entry of pathEnv.split(path.delimiter)) {
63
+ if (!entry)
64
+ continue;
65
+ candidates.push(path.join(entry, 'bird'));
66
+ }
67
+ return candidates.some((candidate) => isExecutable(candidate));
68
+ }
69
+ async function readTweetWithBird(args) {
70
+ return await new Promise((resolve, reject) => {
71
+ execFile('bird', ['read', args.url, '--json'], {
72
+ timeout: args.timeoutMs,
73
+ env: { ...process.env, ...args.env },
74
+ maxBuffer: 1024 * 1024,
75
+ }, (error, stdout, stderr) => {
76
+ if (error) {
77
+ const detail = stderr?.trim();
78
+ const suffix = detail ? `: ${detail}` : '';
79
+ reject(new Error(`bird read failed${suffix}`));
80
+ return;
81
+ }
82
+ const trimmed = stdout.trim();
83
+ if (!trimmed) {
84
+ reject(new Error('bird read returned empty output'));
85
+ return;
86
+ }
87
+ try {
88
+ const parsed = JSON.parse(trimmed);
89
+ const tweet = Array.isArray(parsed) ? parsed[0] : parsed;
90
+ if (!tweet || typeof tweet.text !== 'string') {
91
+ reject(new Error('bird read returned invalid payload'));
92
+ return;
93
+ }
94
+ resolve(tweet);
95
+ }
96
+ catch (parseError) {
97
+ const message = parseError instanceof Error ? parseError.message : String(parseError);
98
+ reject(new Error(`bird read returned invalid JSON: ${message}`));
99
+ }
100
+ });
101
+ });
102
+ }
103
+ function withBirdTip(error, url, env) {
104
+ if (!url || !isTwitterStatusUrl(url) || hasBirdCli(env)) {
105
+ return error instanceof Error ? error : new Error(String(error));
106
+ }
107
+ const message = error instanceof Error ? error.message : String(error);
108
+ const combined = `${message}\n${BIRD_TIP}`;
109
+ return error instanceof Error ? new Error(combined, { cause: error }) : new Error(combined);
110
+ }
111
+ const MAX_TEXT_BYTES_DEFAULT = 10 * 1024 * 1024;
21
112
  function buildProgram() {
22
113
  return new Command()
23
114
  .name('summarize')
24
115
  .description('Summarize web pages and YouTube links (uses direct provider API keys).')
25
116
  .argument('[input]', 'URL or local file path to summarize')
26
117
  .option('--youtube <mode>', 'YouTube transcript source: auto (web then apify), web (youtubei/captionTracks), apify', 'auto')
27
- .option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first). Note: in --extract-only website mode, defaults to always when FIRECRAWL_API_KEY is set.', 'auto')
28
- .option('--markdown <mode>', 'Website Markdown output: off, auto (prefer Firecrawl, then LLM when configured), llm (force LLM). Only affects --extract-only for non-YouTube URLs.', 'auto')
118
+ .option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first).', 'auto')
119
+ .option('--markdown <mode>', 'Website Markdown output: off, auto (use LLM when configured), llm (force LLM). Only affects --extract-only for non-YouTube URLs.', 'auto')
29
120
  .option('--length <length>', 'Summary length: short|medium|long|xl|xxl or a character limit like 20000, 20k', 'medium')
30
121
  .option('--max-output-tokens <count>', 'Hard cap for LLM output tokens (e.g. 2000, 2k). Overrides provider defaults.', undefined)
31
122
  .option('--timeout <duration>', 'Timeout for content fetching and LLM request: 30 (seconds), 30s, 2m, 5000ms', '2m')
@@ -139,19 +230,26 @@ function assertAssetMediaTypeSupported({ attachment, sizeLabel, }) {
139
230
  `Archive formats (zip/tar/7z/rar) can’t be sent to the model.\n` +
140
231
  `Unzip and summarize a specific file instead (e.g. README.md).`);
141
232
  }
142
- function buildAssetPromptPayload({ promptText, attachment, }) {
143
- if (attachment.part.type === 'file' && isTextLikeMediaType(attachment.mediaType)) {
144
- const data = attachment.part.data;
145
- const content = typeof data === 'string'
146
- ? data
147
- : data instanceof Uint8Array
148
- ? new TextDecoder().decode(data)
149
- : '';
233
+ function buildAssetPromptPayload({ promptText, attachment, textContent, }) {
234
+ if (textContent && attachment.part.type === 'file' && isTextLikeMediaType(attachment.mediaType)) {
150
235
  const header = `File: ${attachment.filename ?? 'unknown'} (${attachment.mediaType})`;
151
- return `${promptText}\n\n---\n${header}\n\n${content}`.trim();
236
+ return `${promptText}\n\n---\n${header}\n\n${textContent.content}`.trim();
152
237
  }
153
238
  return buildAssetPromptMessages({ promptText, attachment });
154
239
  }
240
+ function getTextContentFromAttachment(attachment) {
241
+ if (attachment.part.type !== 'file' || !isTextLikeMediaType(attachment.mediaType)) {
242
+ return null;
243
+ }
244
+ const data = attachment.part.data;
245
+ if (typeof data === 'string') {
246
+ return { content: data, bytes: Buffer.byteLength(data, 'utf8') };
247
+ }
248
+ if (data instanceof Uint8Array) {
249
+ return { content: new TextDecoder().decode(data), bytes: data.byteLength };
250
+ }
251
+ return { content: '', bytes: 0 };
252
+ }
155
253
  function assertProviderSupportsAttachment({ provider, modelId, attachment, }) {
156
254
  // xAI via AI SDK currently supports image parts, but not generic file parts (e.g. PDFs).
157
255
  if (provider === 'xai' &&
@@ -196,6 +294,18 @@ function isGoogleStreamingUnsupportedError(error) {
196
294
  /Call ListModels/i.test(errorText) ||
197
295
  /supported methods/i.test(errorText));
198
296
  }
297
+ function isStreamingTimeoutError(error) {
298
+ if (!error)
299
+ return false;
300
+ const message = typeof error === 'string'
301
+ ? error
302
+ : error instanceof Error
303
+ ? error.message
304
+ : typeof error.message === 'string'
305
+ ? String(error.message)
306
+ : '';
307
+ return /timed out/i.test(message);
308
+ }
199
309
  function attachRichHelp(program, env, stdout) {
200
310
  const color = supportsColor(stdout, env);
201
311
  const heading = (text) => ansi('1;36', text, color);
@@ -204,7 +314,7 @@ function attachRichHelp(program, env, stdout) {
204
314
  program.addHelpText('after', () => `
205
315
  ${heading('Examples')}
206
316
  ${cmd('summarize "https://example.com"')}
207
- ${cmd('summarize "https://example.com" --extract-only')} ${dim('# website markdown (prefers Firecrawl when configured)')}
317
+ ${cmd('summarize "https://example.com" --extract-only')} ${dim('# website markdown (LLM if configured)')}
208
318
  ${cmd('summarize "https://example.com" --extract-only --markdown llm')} ${dim('# website markdown via LLM')}
209
319
  ${cmd('summarize "https://www.youtube.com/watch?v=I845O57ZSy4&t=11s" --extract-only --youtube web')}
210
320
  ${cmd('summarize "https://example.com" --length 20k --max-output-tokens 2k --timeout 2m --model openai/gpt-5.2')}
@@ -240,38 +350,6 @@ async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutM
240
350
  usage: result.usage,
241
351
  };
242
352
  }
243
- function splitTextIntoChunks(input, maxCharacters) {
244
- if (maxCharacters <= 0) {
245
- return [input];
246
- }
247
- const text = input.trim();
248
- if (text.length <= maxCharacters) {
249
- return [text];
250
- }
251
- const chunks = [];
252
- let offset = 0;
253
- while (offset < text.length) {
254
- const end = Math.min(offset + maxCharacters, text.length);
255
- const slice = text.slice(offset, end);
256
- if (end === text.length) {
257
- chunks.push(slice.trim());
258
- break;
259
- }
260
- const candidateBreaks = [
261
- slice.lastIndexOf('\n\n'),
262
- slice.lastIndexOf('\n'),
263
- slice.lastIndexOf('. '),
264
- ];
265
- const lastBreak = Math.max(...candidateBreaks);
266
- const splitAt = lastBreak > Math.floor(maxCharacters * 0.5) ? lastBreak + 1 : slice.length;
267
- const chunk = slice.slice(0, splitAt).trim();
268
- if (chunk.length > 0) {
269
- chunks.push(chunk);
270
- }
271
- offset += splitAt;
272
- }
273
- return chunks.filter((chunk) => chunk.length > 0);
274
- }
275
353
  const VERBOSE_PREFIX = '[summarize]';
276
354
  function writeVerbose(stderr, verbose, message, color) {
277
355
  if (!verbose) {
@@ -315,6 +393,11 @@ function formatBytes(bytes) {
315
393
  }
316
394
  return `${value.toFixed(value >= 10 ? 0 : 1)} ${units[unitIndex]}`;
317
395
  }
396
+ function formatCount(value) {
397
+ if (!Number.isFinite(value))
398
+ return 'unknown';
399
+ return value.toLocaleString('en-US');
400
+ }
318
401
  function sumNumbersOrNull(values) {
319
402
  let sum = 0;
320
403
  let any = false;
@@ -339,7 +422,7 @@ function mergeStreamingChunk(previous, chunk) {
339
422
  }
340
423
  return { next: previous + chunk, appended: chunk };
341
424
  }
342
- function writeFinishLine({ stderr, elapsedMs, model, strategy, chunkCount, report, costUsd, color, }) {
425
+ function writeFinishLine({ stderr, elapsedMs, model, report, costUsd, color, }) {
343
426
  const promptTokens = sumNumbersOrNull(report.llm.map((row) => row.promptTokens));
344
427
  const completionTokens = sumNumbersOrNull(report.llm.map((row) => row.completionTokens));
345
428
  const totalTokens = sumNumbersOrNull(report.llm.map((row) => row.totalTokens));
@@ -357,25 +440,10 @@ function writeFinishLine({ stderr, elapsedMs, model, strategy, chunkCount, repor
357
440
  if (report.services.apify.requests > 0) {
358
441
  parts.push(`apify=${report.services.apify.requests}`);
359
442
  }
360
- if (strategy === 'map-reduce') {
361
- parts.push('strategy=map-reduce');
362
- if (typeof chunkCount === 'number' && Number.isFinite(chunkCount) && chunkCount > 0) {
363
- parts.push(`chunks=${chunkCount}`);
364
- }
365
- }
366
443
  const line = `Finished in ${formatElapsedMs(elapsedMs)} (${parts.join(' | ')})`;
367
444
  stderr.write('\n');
368
445
  stderr.write(`${ansi('1;32', line, color)}\n`);
369
446
  }
370
- function buildChunkNotesPrompt({ content }) {
371
- return `Return 10 bullet points summarizing the content below (Markdown).
372
-
373
- CONTENT:
374
- """
375
- ${content}
376
- """
377
- `;
378
- }
379
447
  export async function runCli(argv, { env, fetch, stdout, stderr }) {
380
448
  ;
381
449
  globalThis.AI_SDK_LOG_WARNINGS = false;
@@ -427,7 +495,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
427
495
  const markdownMode = parseMarkdownMode(program.opts().markdown);
428
496
  const shouldComputeReport = metricsEnabled;
429
497
  const isYoutubeUrl = typeof url === 'string' ? /youtube\.com|youtu\.be/i.test(url) : false;
430
- const firecrawlExplicitlySet = normalizedArgv.some((arg) => arg === '--firecrawl' || arg.startsWith('--firecrawl='));
431
498
  const requestedFirecrawlMode = parseFirecrawlMode(program.opts().firecrawl);
432
499
  const modelArg = typeof program.opts().model === 'string' ? program.opts().model : null;
433
500
  const { config, path: configPath } = loadSummarizeConfig({ env });
@@ -485,29 +552,41 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
485
552
  return null;
486
553
  return capMaxOutputTokensForModel({ modelId, requested: maxOutputTokensArg });
487
554
  };
555
+ const resolveMaxInputTokensForCall = async (modelId) => {
556
+ const catalog = await getLiteLlmCatalog();
557
+ if (!catalog)
558
+ return null;
559
+ const limit = resolveLiteLlmMaxInputTokensForModelId(catalog, modelId);
560
+ if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0) {
561
+ return limit;
562
+ }
563
+ return null;
564
+ };
488
565
  const estimateCostUsd = async () => {
489
566
  const catalog = await getLiteLlmCatalog();
490
567
  if (!catalog)
491
568
  return null;
492
- let total = 0;
493
- let any = false;
494
- for (const call of llmCalls) {
569
+ const calls = llmCalls.map((call) => {
495
570
  const promptTokens = call.usage?.promptTokens ?? null;
496
571
  const completionTokens = call.usage?.completionTokens ?? null;
497
- if (typeof promptTokens !== 'number' ||
498
- !Number.isFinite(promptTokens) ||
499
- typeof completionTokens !== 'number' ||
500
- !Number.isFinite(completionTokens)) {
501
- continue;
502
- }
503
- const pricing = resolveLiteLlmPricingForModelId(catalog, call.model);
504
- if (!pricing)
505
- continue;
506
- total +=
507
- promptTokens * pricing.inputUsdPerToken + completionTokens * pricing.outputUsdPerToken;
508
- any = true;
509
- }
510
- return any ? total : null;
572
+ const hasTokens = typeof promptTokens === 'number' &&
573
+ Number.isFinite(promptTokens) &&
574
+ typeof completionTokens === 'number' &&
575
+ Number.isFinite(completionTokens);
576
+ const usage = hasTokens
577
+ ? normalizeTokenUsage({
578
+ inputTokens: promptTokens,
579
+ outputTokens: completionTokens,
580
+ totalTokens: call.usage?.totalTokens ?? undefined,
581
+ })
582
+ : null;
583
+ return { model: call.model, usage };
584
+ });
585
+ const result = await tallyCosts({
586
+ calls,
587
+ resolvePricing: (modelId) => resolveLiteLlmPricingForModelId(catalog, modelId),
588
+ });
589
+ return result.total?.totalUsd ?? null;
511
590
  };
512
591
  const buildReport = async () => {
513
592
  return buildRunMetricsReport({ llmCalls, firecrawlRequests, apifyRequests });
@@ -617,14 +696,29 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
617
696
  const effectiveModelId = modelResolution.modelId;
618
697
  const parsedModelEffective = parseGatewayStyleModelId(effectiveModelId);
619
698
  const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
699
+ const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
700
+ const textContent = getTextContentFromAttachment(attachment);
701
+ if (textContent && textContent.bytes > MAX_TEXT_BYTES_DEFAULT) {
702
+ throw new Error(`Text file too large (${formatBytes(textContent.bytes)}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
703
+ }
620
704
  const summaryLengthTarget = lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters };
621
705
  const promptText = buildFileSummaryPrompt({
622
706
  filename: attachment.filename,
623
707
  mediaType: attachment.mediaType,
624
708
  summaryLength: summaryLengthTarget,
709
+ contentLength: textContent?.content.length ?? null,
625
710
  });
626
- const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
627
- const promptPayload = buildAssetPromptPayload({ promptText, attachment });
711
+ const promptPayload = buildAssetPromptPayload({ promptText, attachment, textContent });
712
+ const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
713
+ if (typeof maxInputTokensForCall === 'number' &&
714
+ Number.isFinite(maxInputTokensForCall) &&
715
+ maxInputTokensForCall > 0 &&
716
+ typeof promptPayload === 'string') {
717
+ const tokenCount = countTokens(promptPayload);
718
+ if (tokenCount > maxInputTokensForCall) {
719
+ throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
720
+ }
721
+ }
628
722
  const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
629
723
  const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
630
724
  const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
@@ -645,7 +739,26 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
645
739
  });
646
740
  }
647
741
  catch (error) {
648
- if (parsedModelEffective.provider === 'google' &&
742
+ if (isStreamingTimeoutError(error)) {
743
+ writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
744
+ const result = await summarizeWithModelId({
745
+ modelId: parsedModelEffective.canonical,
746
+ prompt: promptPayload,
747
+ maxOutputTokens: maxOutputTokensForCall ?? undefined,
748
+ timeoutMs,
749
+ fetchImpl: trackedFetch,
750
+ apiKeys: apiKeysForLlm,
751
+ });
752
+ llmCalls.push({
753
+ provider: result.provider,
754
+ model: result.canonicalModelId,
755
+ usage: result.usage,
756
+ purpose: 'summary',
757
+ });
758
+ summary = result.text;
759
+ streamResult = null;
760
+ }
761
+ else if (parsedModelEffective.provider === 'google' &&
649
762
  isGoogleStreamingUnsupportedError(error)) {
650
763
  writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
651
764
  const result = await summarizeWithModelId({
@@ -829,7 +942,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
829
942
  model: parsedModelEffective.canonical,
830
943
  maxCompletionTokens: maxOutputTokensForCall,
831
944
  strategy: 'single',
832
- chunkCount: 1,
833
945
  },
834
946
  metrics: metricsEnabled ? finishReport : null,
835
947
  summary,
@@ -844,8 +956,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
844
956
  stderr,
845
957
  elapsedMs: Date.now() - runStartedAtMs,
846
958
  model: parsedModelEffective.canonical,
847
- strategy: 'single',
848
- chunkCount: 1,
849
959
  report: finishReport,
850
960
  costUsd,
851
961
  color: verboseColor,
@@ -876,8 +986,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
876
986
  stderr,
877
987
  elapsedMs: Date.now() - runStartedAtMs,
878
988
  model: parsedModelEffective.canonical,
879
- strategy: 'single',
880
- chunkCount: 1,
881
989
  report,
882
990
  costUsd,
883
991
  color: verboseColor,
@@ -998,12 +1106,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
998
1106
  if (!url) {
999
1107
  throw new Error('Only HTTP and HTTPS URLs can be summarized');
1000
1108
  }
1001
- const firecrawlMode = (() => {
1002
- if (extractOnly && !isYoutubeUrl && !firecrawlExplicitlySet && firecrawlConfigured) {
1003
- return 'always';
1004
- }
1005
- return requestedFirecrawlMode;
1006
- })();
1109
+ const firecrawlMode = requestedFirecrawlMode;
1007
1110
  if (firecrawlMode === 'always' && !firecrawlConfigured) {
1008
1111
  throw new Error('--firecrawl always requires FIRECRAWL_API_KEY');
1009
1112
  }
@@ -1047,6 +1150,9 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1047
1150
  },
1048
1151
  })
1049
1152
  : null;
1153
+ const readTweetWithBirdClient = hasBirdCli(env)
1154
+ ? ({ url, timeoutMs }) => readTweetWithBird({ url, timeoutMs, env })
1155
+ : null;
1050
1156
  writeVerbose(stderr, verbose, 'extract start', verboseColor);
1051
1157
  const stopOscProgress = startOscProgress({
1052
1158
  label: 'Fetching website',
@@ -1067,22 +1173,65 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1067
1173
  phase: 'idle',
1068
1174
  htmlDownloadedBytes: 0,
1069
1175
  htmlTotalBytes: null,
1176
+ fetchStartedAtMs: null,
1070
1177
  lastSpinnerUpdateAtMs: 0,
1071
1178
  };
1072
- const updateSpinner = (text) => {
1179
+ let ticker = null;
1180
+ const updateSpinner = (text, options) => {
1073
1181
  const now = Date.now();
1074
- if (now - state.lastSpinnerUpdateAtMs < 100)
1182
+ if (!options?.force && now - state.lastSpinnerUpdateAtMs < 100)
1075
1183
  return;
1076
1184
  state.lastSpinnerUpdateAtMs = now;
1077
1185
  spinner.setText(text);
1078
1186
  };
1187
+ const formatFirecrawlReason = (reason) => {
1188
+ const lower = reason.toLowerCase();
1189
+ if (lower.includes('forced'))
1190
+ return 'forced';
1191
+ if (lower.includes('html fetch failed'))
1192
+ return 'fallback: HTML fetch failed';
1193
+ if (lower.includes('blocked') || lower.includes('thin'))
1194
+ return 'fallback: blocked/thin HTML';
1195
+ return reason;
1196
+ };
1197
+ const renderFetchLine = () => {
1198
+ const downloaded = formatBytes(state.htmlDownloadedBytes);
1199
+ const total = typeof state.htmlTotalBytes === 'number' ? `/${formatBytes(state.htmlTotalBytes)}` : '';
1200
+ const elapsedMs = typeof state.fetchStartedAtMs === 'number' ? Date.now() - state.fetchStartedAtMs : 0;
1201
+ const elapsed = formatElapsedMs(elapsedMs);
1202
+ if (state.htmlDownloadedBytes === 0 && !state.htmlTotalBytes) {
1203
+ return `Fetching website (connecting, ${elapsed})…`;
1204
+ }
1205
+ const rate = elapsedMs > 0 && state.htmlDownloadedBytes > 0
1206
+ ? `, ${formatBytes(state.htmlDownloadedBytes / (elapsedMs / 1000))}/s`
1207
+ : '';
1208
+ return `Fetching website (${downloaded}${total}, ${elapsed}${rate})…`;
1209
+ };
1210
+ const startTicker = () => {
1211
+ if (ticker)
1212
+ return;
1213
+ ticker = setInterval(() => {
1214
+ if (state.phase !== 'fetching')
1215
+ return;
1216
+ updateSpinner(renderFetchLine());
1217
+ }, 1000);
1218
+ };
1219
+ const stopTicker = () => {
1220
+ if (!ticker)
1221
+ return;
1222
+ clearInterval(ticker);
1223
+ ticker = null;
1224
+ };
1079
1225
  return {
1080
1226
  getHtmlDownloadedBytes: () => state.htmlDownloadedBytes,
1227
+ stop: stopTicker,
1081
1228
  onProgress: (event) => {
1082
1229
  if (event.kind === 'fetch-html-start') {
1083
1230
  state.phase = 'fetching';
1084
1231
  state.htmlDownloadedBytes = 0;
1085
1232
  state.htmlTotalBytes = null;
1233
+ state.fetchStartedAtMs = Date.now();
1234
+ startTicker();
1086
1235
  updateSpinner('Fetching website (connecting)…');
1087
1236
  return;
1088
1237
  }
@@ -1090,23 +1239,57 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1090
1239
  state.phase = 'fetching';
1091
1240
  state.htmlDownloadedBytes = event.downloadedBytes;
1092
1241
  state.htmlTotalBytes = event.totalBytes;
1093
- const downloaded = formatBytes(event.downloadedBytes);
1094
- const total = typeof event.totalBytes === 'number' ? `/${formatBytes(event.totalBytes)}` : '';
1095
- updateSpinner(`Fetching website (${downloaded}${total})…`);
1242
+ updateSpinner(renderFetchLine());
1243
+ return;
1244
+ }
1245
+ if (event.kind === 'bird-start') {
1246
+ state.phase = 'bird';
1247
+ stopTicker();
1248
+ updateSpinner('Bird: reading tweet…', { force: true });
1249
+ return;
1250
+ }
1251
+ if (event.kind === 'bird-done') {
1252
+ state.phase = 'bird';
1253
+ stopTicker();
1254
+ if (event.ok && typeof event.textBytes === 'number') {
1255
+ updateSpinner(`Bird: got ${formatBytes(event.textBytes)}…`, { force: true });
1256
+ return;
1257
+ }
1258
+ updateSpinner('Bird: failed; fallback…', { force: true });
1259
+ return;
1260
+ }
1261
+ if (event.kind === 'nitter-start') {
1262
+ state.phase = 'nitter';
1263
+ stopTicker();
1264
+ updateSpinner('Nitter: fetching…', { force: true });
1265
+ return;
1266
+ }
1267
+ if (event.kind === 'nitter-done') {
1268
+ state.phase = 'nitter';
1269
+ stopTicker();
1270
+ if (event.ok && typeof event.textBytes === 'number') {
1271
+ updateSpinner(`Nitter: got ${formatBytes(event.textBytes)}…`, { force: true });
1272
+ return;
1273
+ }
1274
+ updateSpinner('Nitter: failed; fallback…', { force: true });
1096
1275
  return;
1097
1276
  }
1098
1277
  if (event.kind === 'firecrawl-start') {
1099
1278
  state.phase = 'firecrawl';
1100
- updateSpinner('Firecrawl: scraping…');
1279
+ stopTicker();
1280
+ const reason = event.reason ? formatFirecrawlReason(event.reason) : '';
1281
+ const suffix = reason ? ` (${reason})` : '';
1282
+ updateSpinner(`Firecrawl: scraping${suffix}…`, { force: true });
1101
1283
  return;
1102
1284
  }
1103
1285
  if (event.kind === 'firecrawl-done') {
1104
1286
  state.phase = 'firecrawl';
1287
+ stopTicker();
1105
1288
  if (event.ok && typeof event.markdownBytes === 'number') {
1106
- updateSpinner(`Firecrawl: got ${formatBytes(event.markdownBytes)}…`);
1289
+ updateSpinner(`Firecrawl: got ${formatBytes(event.markdownBytes)}…`, { force: true });
1107
1290
  return;
1108
1291
  }
1109
- updateSpinner('Firecrawl: no content; fallback…');
1292
+ updateSpinner('Firecrawl: no content; fallback…', { force: true });
1110
1293
  }
1111
1294
  },
1112
1295
  };
@@ -1115,6 +1298,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1115
1298
  apifyApiToken: apifyToken,
1116
1299
  scrapeWithFirecrawl,
1117
1300
  convertHtmlToMarkdown,
1301
+ readTweetWithBird: readTweetWithBirdClient,
1118
1302
  fetch: trackedFetch,
1119
1303
  onProgress: websiteProgress?.onProgress ?? null,
1120
1304
  });
@@ -1123,24 +1307,42 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1123
1307
  if (stopped)
1124
1308
  return;
1125
1309
  stopped = true;
1310
+ websiteProgress?.stop?.();
1126
1311
  spinner.stopAndClear();
1127
1312
  stopOscProgress();
1128
1313
  };
1129
1314
  clearProgressBeforeStdout = stopProgress;
1130
1315
  try {
1131
- const extracted = await client.fetchLinkContent(url, {
1132
- timeoutMs,
1133
- youtubeTranscript: youtubeMode,
1134
- firecrawl: firecrawlMode,
1135
- format: markdownRequested ? 'markdown' : 'text',
1136
- });
1316
+ let extracted;
1317
+ try {
1318
+ extracted = await client.fetchLinkContent(url, {
1319
+ timeoutMs,
1320
+ youtubeTranscript: youtubeMode,
1321
+ firecrawl: firecrawlMode,
1322
+ format: markdownRequested ? 'markdown' : 'text',
1323
+ });
1324
+ }
1325
+ catch (error) {
1326
+ throw withBirdTip(error, url, env);
1327
+ }
1137
1328
  const extractedContentBytes = Buffer.byteLength(extracted.content, 'utf8');
1138
1329
  const extractedContentSize = formatBytes(extractedContentBytes);
1139
- const viaFirecrawl = extracted.diagnostics.firecrawl.used ? ', Firecrawl' : '';
1330
+ const viaSources = [];
1331
+ if (extracted.diagnostics.strategy === 'bird') {
1332
+ viaSources.push('bird');
1333
+ }
1334
+ if (extracted.diagnostics.strategy === 'nitter') {
1335
+ viaSources.push('Nitter');
1336
+ }
1337
+ if (extracted.diagnostics.firecrawl.used) {
1338
+ viaSources.push('Firecrawl');
1339
+ }
1340
+ const viaSourceLabel = viaSources.length > 0 ? `, ${viaSources.join('+')}` : '';
1140
1341
  if (progressEnabled) {
1342
+ websiteProgress?.stop?.();
1141
1343
  spinner.setText(extractOnly
1142
- ? `Extracted (${extractedContentSize})`
1143
- : `Summarizing (sent ${extractedContentSize}${viaFirecrawl})…`);
1344
+ ? `Extracted (${extractedContentSize}${viaSourceLabel})`
1345
+ : `Summarizing (sent ${extractedContentSize}${viaSourceLabel})…`);
1144
1346
  }
1145
1347
  writeVerbose(stderr, verbose, `extract done strategy=${extracted.diagnostics.strategy} siteName=${formatOptionalString(extracted.siteName)} title=${formatOptionalString(extracted.title)} transcriptSource=${formatOptionalString(extracted.transcriptSource)}`, verboseColor);
1146
1348
  writeVerbose(stderr, verbose, `extract stats characters=${extracted.totalCharacters} words=${extracted.wordCount} transcriptCharacters=${formatOptionalNumber(extracted.transcriptCharacters)} transcriptLines=${formatOptionalNumber(extracted.transcriptLines)}`, verboseColor);
@@ -1204,8 +1406,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1204
1406
  stderr,
1205
1407
  elapsedMs: Date.now() - runStartedAtMs,
1206
1408
  model,
1207
- strategy: 'none',
1208
- chunkCount: null,
1209
1409
  report: finishReport,
1210
1410
  costUsd,
1211
1411
  color: verboseColor,
@@ -1223,8 +1423,76 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1223
1423
  stderr,
1224
1424
  elapsedMs: Date.now() - runStartedAtMs,
1225
1425
  model,
1226
- strategy: 'none',
1227
- chunkCount: null,
1426
+ report,
1427
+ costUsd,
1428
+ color: verboseColor,
1429
+ });
1430
+ }
1431
+ return;
1432
+ }
1433
+ const shouldSkipTweetSummary = isTwitterStatusUrl(url) &&
1434
+ extracted.content.length > 0 &&
1435
+ extracted.content.length <= resolveTargetCharacters(lengthArg);
1436
+ if (shouldSkipTweetSummary) {
1437
+ clearProgressForStdout();
1438
+ writeVerbose(stderr, verbose, `skip summary: tweet content length=${extracted.content.length} target=${resolveTargetCharacters(lengthArg)}`, verboseColor);
1439
+ if (json) {
1440
+ const finishReport = shouldComputeReport ? await buildReport() : null;
1441
+ const payload = {
1442
+ input: {
1443
+ kind: 'url',
1444
+ url,
1445
+ timeoutMs,
1446
+ youtube: youtubeMode,
1447
+ firecrawl: firecrawlMode,
1448
+ markdown: effectiveMarkdownMode,
1449
+ length: lengthArg.kind === 'preset'
1450
+ ? { kind: 'preset', preset: lengthArg.preset }
1451
+ : { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
1452
+ maxOutputTokens: maxOutputTokensArg,
1453
+ model,
1454
+ },
1455
+ env: {
1456
+ hasXaiKey: Boolean(xaiApiKey),
1457
+ hasOpenAIKey: Boolean(apiKey),
1458
+ hasApifyToken: Boolean(apifyToken),
1459
+ hasFirecrawlKey: firecrawlConfigured,
1460
+ hasGoogleKey: googleConfigured,
1461
+ hasAnthropicKey: anthropicConfigured,
1462
+ },
1463
+ extracted,
1464
+ prompt,
1465
+ llm: null,
1466
+ metrics: metricsEnabled ? finishReport : null,
1467
+ summary: extracted.content,
1468
+ };
1469
+ if (metricsDetailed && finishReport) {
1470
+ writeMetricsReport(finishReport);
1471
+ }
1472
+ stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
1473
+ if (metricsEnabled && finishReport) {
1474
+ const costUsd = await estimateCostUsd();
1475
+ writeFinishLine({
1476
+ stderr,
1477
+ elapsedMs: Date.now() - runStartedAtMs,
1478
+ model,
1479
+ report: finishReport,
1480
+ costUsd,
1481
+ color: verboseColor,
1482
+ });
1483
+ }
1484
+ return;
1485
+ }
1486
+ stdout.write(`${extracted.content}\n`);
1487
+ const report = shouldComputeReport ? await buildReport() : null;
1488
+ if (metricsDetailed && report)
1489
+ writeMetricsReport(report);
1490
+ if (metricsEnabled && report) {
1491
+ const costUsd = await estimateCostUsd();
1492
+ writeFinishLine({
1493
+ stderr,
1494
+ elapsedMs: Date.now() - runStartedAtMs,
1495
+ model,
1228
1496
  report,
1229
1497
  costUsd,
1230
1498
  color: verboseColor,
@@ -1269,321 +1537,165 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1269
1537
  const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
1270
1538
  writeVerbose(stderr, verbose, `mode summarize provider=${parsedModelEffective.provider} model=${parsedModelEffective.canonical}`, verboseColor);
1271
1539
  const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
1272
- const isLargeContent = extracted.content.length >= MAP_REDUCE_TRIGGER_CHARACTERS;
1273
- let strategy = 'single';
1274
- let chunkCount = 1;
1540
+ const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
1541
+ if (typeof maxInputTokensForCall === 'number' &&
1542
+ Number.isFinite(maxInputTokensForCall) &&
1543
+ maxInputTokensForCall > 0) {
1544
+ const tokenCount = countTokens(prompt);
1545
+ if (tokenCount > maxInputTokensForCall) {
1546
+ throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
1547
+ }
1548
+ }
1275
1549
  const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
1276
1550
  const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
1277
1551
  const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
1278
1552
  let summaryAlreadyPrinted = false;
1279
1553
  let summary = '';
1280
1554
  let getLastStreamError = null;
1281
- if (!isLargeContent) {
1282
- writeVerbose(stderr, verbose, 'summarize strategy=single', verboseColor);
1283
- if (streamingEnabledForCall) {
1284
- writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
1285
- let streamResult = null;
1286
- try {
1287
- streamResult = await streamTextWithModelId({
1555
+ writeVerbose(stderr, verbose, 'summarize strategy=single', verboseColor);
1556
+ if (streamingEnabledForCall) {
1557
+ writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
1558
+ let streamResult = null;
1559
+ try {
1560
+ streamResult = await streamTextWithModelId({
1561
+ modelId: parsedModelEffective.canonical,
1562
+ apiKeys: apiKeysForLlm,
1563
+ prompt,
1564
+ temperature: 0,
1565
+ maxOutputTokens: maxOutputTokensForCall ?? undefined,
1566
+ timeoutMs,
1567
+ fetchImpl: trackedFetch,
1568
+ });
1569
+ }
1570
+ catch (error) {
1571
+ if (isStreamingTimeoutError(error)) {
1572
+ writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
1573
+ const result = await summarizeWithModelId({
1288
1574
  modelId: parsedModelEffective.canonical,
1289
- apiKeys: apiKeysForLlm,
1290
1575
  prompt,
1291
- temperature: 0,
1292
1576
  maxOutputTokens: maxOutputTokensForCall ?? undefined,
1293
1577
  timeoutMs,
1294
1578
  fetchImpl: trackedFetch,
1579
+ apiKeys: apiKeysForLlm,
1295
1580
  });
1296
- }
1297
- catch (error) {
1298
- if (parsedModelEffective.provider === 'google' &&
1299
- isGoogleStreamingUnsupportedError(error)) {
1300
- writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
1301
- const result = await summarizeWithModelId({
1302
- modelId: parsedModelEffective.canonical,
1303
- prompt,
1304
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1305
- timeoutMs,
1306
- fetchImpl: trackedFetch,
1307
- apiKeys: apiKeysForLlm,
1308
- });
1309
- llmCalls.push({
1310
- provider: result.provider,
1311
- model: result.canonicalModelId,
1312
- usage: result.usage,
1313
- purpose: 'summary',
1314
- });
1315
- summary = result.text;
1316
- streamResult = null;
1317
- }
1318
- else {
1319
- throw error;
1320
- }
1321
- }
1322
- if (streamResult) {
1323
- getLastStreamError = streamResult.lastError;
1324
- let streamed = '';
1325
- const liveRenderer = shouldLiveRenderSummary
1326
- ? createLiveRenderer({
1327
- write: (chunk) => {
1328
- clearProgressForStdout();
1329
- stdout.write(chunk);
1330
- },
1331
- width: markdownRenderWidth(stdout, env),
1332
- renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
1333
- width: markdownRenderWidth(stdout, env),
1334
- wrap: true,
1335
- color: supportsColor(stdout, env),
1336
- }),
1337
- })
1338
- : null;
1339
- let lastFrameAtMs = 0;
1340
- try {
1341
- let cleared = false;
1342
- for await (const delta of streamResult.textStream) {
1343
- const merged = mergeStreamingChunk(streamed, delta);
1344
- streamed = merged.next;
1345
- if (shouldStreamSummaryToStdout) {
1346
- if (!cleared) {
1347
- clearProgressForStdout();
1348
- cleared = true;
1349
- }
1350
- if (merged.appended)
1351
- stdout.write(merged.appended);
1352
- continue;
1353
- }
1354
- if (liveRenderer) {
1355
- const now = Date.now();
1356
- const due = now - lastFrameAtMs >= 120;
1357
- const hasNewline = delta.includes('\n');
1358
- if (hasNewline || due) {
1359
- liveRenderer.render(streamed);
1360
- lastFrameAtMs = now;
1361
- }
1362
- }
1363
- }
1364
- const trimmed = streamed.trim();
1365
- streamed = trimmed;
1366
- if (liveRenderer) {
1367
- liveRenderer.render(trimmed);
1368
- summaryAlreadyPrinted = true;
1369
- }
1370
- }
1371
- finally {
1372
- liveRenderer?.finish();
1373
- }
1374
- const usage = await streamResult.usage;
1375
1581
  llmCalls.push({
1376
- provider: streamResult.provider,
1377
- model: streamResult.canonicalModelId,
1378
- usage,
1582
+ provider: result.provider,
1583
+ model: result.canonicalModelId,
1584
+ usage: result.usage,
1379
1585
  purpose: 'summary',
1380
1586
  });
1381
- summary = streamed;
1382
- if (shouldStreamSummaryToStdout) {
1383
- if (!streamed.endsWith('\n')) {
1384
- stdout.write('\n');
1385
- }
1386
- summaryAlreadyPrinted = true;
1387
- }
1587
+ summary = result.text;
1588
+ streamResult = null;
1388
1589
  }
1389
- }
1390
- else {
1391
- const result = await summarizeWithModelId({
1392
- modelId: parsedModelEffective.canonical,
1393
- prompt,
1394
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1395
- timeoutMs,
1396
- fetchImpl: trackedFetch,
1397
- apiKeys: apiKeysForLlm,
1398
- });
1399
- llmCalls.push({
1400
- provider: result.provider,
1401
- model: result.canonicalModelId,
1402
- usage: result.usage,
1403
- purpose: 'summary',
1404
- });
1405
- summary = result.text;
1406
- }
1407
- }
1408
- else {
1409
- strategy = 'map-reduce';
1410
- const chunks = splitTextIntoChunks(extracted.content, MAP_REDUCE_CHUNK_CHARACTERS);
1411
- chunkCount = chunks.length;
1412
- stderr.write(`Large input (${extracted.content.length} chars); summarizing in ${chunks.length} chunks.\n`);
1413
- writeVerbose(stderr, verbose, `summarize strategy=map-reduce chunks=${chunks.length}`, verboseColor);
1414
- const chunkNotes = [];
1415
- for (let i = 0; i < chunks.length; i += 1) {
1416
- writeVerbose(stderr, verbose, `summarize chunk ${i + 1}/${chunks.length} notes start`, verboseColor);
1417
- const chunkPrompt = buildChunkNotesPrompt({
1418
- content: chunks[i] ?? '',
1419
- });
1420
- const chunkNoteTokensRequested = typeof maxOutputTokensArg === 'number'
1421
- ? Math.min(SUMMARY_LENGTH_TO_TOKENS.medium, maxOutputTokensArg)
1422
- : SUMMARY_LENGTH_TO_TOKENS.medium;
1423
- const chunkNoteTokens = await capMaxOutputTokensForModel({
1424
- modelId: parsedModelEffective.canonical,
1425
- requested: chunkNoteTokensRequested,
1426
- });
1427
- const notesResult = await summarizeWithModelId({
1428
- modelId: parsedModelEffective.canonical,
1429
- prompt: chunkPrompt,
1430
- maxOutputTokens: chunkNoteTokens,
1431
- timeoutMs,
1432
- fetchImpl: trackedFetch,
1433
- apiKeys: apiKeysForLlm,
1434
- });
1435
- const notes = notesResult.text;
1436
- llmCalls.push({
1437
- provider: notesResult.provider,
1438
- model: notesResult.canonicalModelId,
1439
- usage: notesResult.usage,
1440
- purpose: 'chunk-notes',
1441
- });
1442
- chunkNotes.push(notes.trim());
1443
- }
1444
- writeVerbose(stderr, verbose, 'summarize merge chunk notes', verboseColor);
1445
- const mergedContent = `Chunk notes (generated from the full input):\n\n${chunkNotes
1446
- .filter((value) => value.length > 0)
1447
- .join('\n\n')}`;
1448
- const mergedPrompt = buildLinkSummaryPrompt({
1449
- url: extracted.url,
1450
- title: extracted.title,
1451
- siteName: extracted.siteName,
1452
- description: extracted.description,
1453
- content: mergedContent,
1454
- truncated: false,
1455
- hasTranscript: isYouTube ||
1456
- (extracted.transcriptSource !== null && extracted.transcriptSource !== 'unavailable'),
1457
- summaryLength: lengthArg.kind === 'preset'
1458
- ? lengthArg.preset
1459
- : { maxCharacters: lengthArg.maxCharacters },
1460
- shares: [],
1461
- });
1462
- if (streamingEnabledForCall) {
1463
- writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
1464
- let streamResult = null;
1465
- try {
1466
- streamResult = await streamTextWithModelId({
1590
+ else if (parsedModelEffective.provider === 'google' &&
1591
+ isGoogleStreamingUnsupportedError(error)) {
1592
+ writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
1593
+ const result = await summarizeWithModelId({
1467
1594
  modelId: parsedModelEffective.canonical,
1468
- apiKeys: apiKeysForLlm,
1469
- prompt: mergedPrompt,
1470
- temperature: 0,
1595
+ prompt,
1471
1596
  maxOutputTokens: maxOutputTokensForCall ?? undefined,
1472
1597
  timeoutMs,
1473
1598
  fetchImpl: trackedFetch,
1599
+ apiKeys: apiKeysForLlm,
1600
+ });
1601
+ llmCalls.push({
1602
+ provider: result.provider,
1603
+ model: result.canonicalModelId,
1604
+ usage: result.usage,
1605
+ purpose: 'summary',
1474
1606
  });
1607
+ summary = result.text;
1608
+ streamResult = null;
1475
1609
  }
1476
- catch (error) {
1477
- if (parsedModelEffective.provider === 'google' &&
1478
- isGoogleStreamingUnsupportedError(error)) {
1479
- writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
1480
- const mergedResult = await summarizeWithModelId({
1481
- modelId: parsedModelEffective.canonical,
1482
- prompt: mergedPrompt,
1483
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1484
- timeoutMs,
1485
- fetchImpl: trackedFetch,
1486
- apiKeys: apiKeysForLlm,
1487
- });
1488
- llmCalls.push({
1489
- provider: mergedResult.provider,
1490
- model: mergedResult.canonicalModelId,
1491
- usage: mergedResult.usage,
1492
- purpose: 'summary',
1493
- });
1494
- summary = mergedResult.text;
1495
- streamResult = null;
1496
- }
1497
- else {
1498
- throw error;
1499
- }
1610
+ else {
1611
+ throw error;
1500
1612
  }
1501
- if (streamResult) {
1502
- getLastStreamError = streamResult.lastError;
1503
- let streamed = '';
1504
- const liveRenderer = shouldLiveRenderSummary
1505
- ? createLiveRenderer({
1506
- write: (chunk) => {
1507
- clearProgressForStdout();
1508
- stdout.write(chunk);
1509
- },
1613
+ }
1614
+ if (streamResult) {
1615
+ getLastStreamError = streamResult.lastError;
1616
+ let streamed = '';
1617
+ const liveRenderer = shouldLiveRenderSummary
1618
+ ? createLiveRenderer({
1619
+ write: (chunk) => {
1620
+ clearProgressForStdout();
1621
+ stdout.write(chunk);
1622
+ },
1623
+ width: markdownRenderWidth(stdout, env),
1624
+ renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
1510
1625
  width: markdownRenderWidth(stdout, env),
1511
- renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
1512
- width: markdownRenderWidth(stdout, env),
1513
- wrap: true,
1514
- color: supportsColor(stdout, env),
1515
- }),
1516
- })
1517
- : null;
1518
- let lastFrameAtMs = 0;
1519
- try {
1520
- let cleared = false;
1521
- for await (const delta of streamResult.textStream) {
1626
+ wrap: true,
1627
+ color: supportsColor(stdout, env),
1628
+ }),
1629
+ })
1630
+ : null;
1631
+ let lastFrameAtMs = 0;
1632
+ try {
1633
+ let cleared = false;
1634
+ for await (const delta of streamResult.textStream) {
1635
+ const merged = mergeStreamingChunk(streamed, delta);
1636
+ streamed = merged.next;
1637
+ if (shouldStreamSummaryToStdout) {
1522
1638
  if (!cleared) {
1523
1639
  clearProgressForStdout();
1524
1640
  cleared = true;
1525
1641
  }
1526
- const merged = mergeStreamingChunk(streamed, delta);
1527
- streamed = merged.next;
1528
- if (shouldStreamSummaryToStdout) {
1529
- if (merged.appended)
1530
- stdout.write(merged.appended);
1531
- continue;
1532
- }
1533
- if (liveRenderer) {
1534
- const now = Date.now();
1535
- const due = now - lastFrameAtMs >= 120;
1536
- const hasNewline = delta.includes('\n');
1537
- if (hasNewline || due) {
1538
- liveRenderer.render(streamed);
1539
- lastFrameAtMs = now;
1540
- }
1541
- }
1642
+ if (merged.appended)
1643
+ stdout.write(merged.appended);
1644
+ continue;
1542
1645
  }
1543
- const trimmed = streamed.trim();
1544
- streamed = trimmed;
1545
1646
  if (liveRenderer) {
1546
- liveRenderer.render(trimmed);
1547
- summaryAlreadyPrinted = true;
1647
+ const now = Date.now();
1648
+ const due = now - lastFrameAtMs >= 120;
1649
+ const hasNewline = delta.includes('\n');
1650
+ if (hasNewline || due) {
1651
+ liveRenderer.render(streamed);
1652
+ lastFrameAtMs = now;
1653
+ }
1548
1654
  }
1549
1655
  }
1550
- finally {
1551
- liveRenderer?.finish();
1552
- }
1553
- const usage = await streamResult.usage;
1554
- llmCalls.push({
1555
- provider: streamResult.provider,
1556
- model: streamResult.canonicalModelId,
1557
- usage,
1558
- purpose: 'summary',
1559
- });
1560
- summary = streamed;
1561
- if (shouldStreamSummaryToStdout) {
1562
- if (!streamed.endsWith('\n')) {
1563
- stdout.write('\n');
1564
- }
1656
+ const trimmed = streamed.trim();
1657
+ streamed = trimmed;
1658
+ if (liveRenderer) {
1659
+ liveRenderer.render(trimmed);
1565
1660
  summaryAlreadyPrinted = true;
1566
1661
  }
1567
1662
  }
1568
- }
1569
- else {
1570
- const mergedResult = await summarizeWithModelId({
1571
- modelId: parsedModelEffective.canonical,
1572
- prompt: mergedPrompt,
1573
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1574
- timeoutMs,
1575
- fetchImpl: trackedFetch,
1576
- apiKeys: apiKeysForLlm,
1577
- });
1663
+ finally {
1664
+ liveRenderer?.finish();
1665
+ }
1666
+ const usage = await streamResult.usage;
1578
1667
  llmCalls.push({
1579
- provider: mergedResult.provider,
1580
- model: mergedResult.canonicalModelId,
1581
- usage: mergedResult.usage,
1668
+ provider: streamResult.provider,
1669
+ model: streamResult.canonicalModelId,
1670
+ usage,
1582
1671
  purpose: 'summary',
1583
1672
  });
1584
- summary = mergedResult.text;
1673
+ summary = streamed;
1674
+ if (shouldStreamSummaryToStdout) {
1675
+ if (!streamed.endsWith('\n')) {
1676
+ stdout.write('\n');
1677
+ }
1678
+ summaryAlreadyPrinted = true;
1679
+ }
1585
1680
  }
1586
1681
  }
1682
+ else {
1683
+ const result = await summarizeWithModelId({
1684
+ modelId: parsedModelEffective.canonical,
1685
+ prompt,
1686
+ maxOutputTokens: maxOutputTokensForCall ?? undefined,
1687
+ timeoutMs,
1688
+ fetchImpl: trackedFetch,
1689
+ apiKeys: apiKeysForLlm,
1690
+ });
1691
+ llmCalls.push({
1692
+ provider: result.provider,
1693
+ model: result.canonicalModelId,
1694
+ usage: result.usage,
1695
+ purpose: 'summary',
1696
+ });
1697
+ summary = result.text;
1698
+ }
1587
1699
  summary = summary.trim();
1588
1700
  if (summary.length === 0) {
1589
1701
  const last = getLastStreamError?.();
@@ -1622,8 +1734,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1622
1734
  provider: parsedModelEffective.provider,
1623
1735
  model: parsedModelEffective.canonical,
1624
1736
  maxCompletionTokens: maxOutputTokensForCall,
1625
- strategy,
1626
- chunkCount,
1737
+ strategy: 'single',
1627
1738
  },
1628
1739
  metrics: metricsEnabled ? finishReport : null,
1629
1740
  summary,
@@ -1638,8 +1749,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1638
1749
  stderr,
1639
1750
  elapsedMs: Date.now() - runStartedAtMs,
1640
1751
  model: parsedModelEffective.canonical,
1641
- strategy,
1642
- chunkCount,
1643
1752
  report: finishReport,
1644
1753
  costUsd,
1645
1754
  color: verboseColor,
@@ -1670,8 +1779,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1670
1779
  stderr,
1671
1780
  elapsedMs: Date.now() - runStartedAtMs,
1672
1781
  model: parsedModelEffective.canonical,
1673
- strategy,
1674
- chunkCount,
1675
1782
  report,
1676
1783
  costUsd,
1677
1784
  color: verboseColor,