@steipete/summarize 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +47 -1
  2. package/README.md +6 -0
  3. package/dist/cli.cjs +2737 -1010
  4. package/dist/cli.cjs.map +4 -4
  5. package/dist/esm/content/asset.js +18 -0
  6. package/dist/esm/content/asset.js.map +1 -1
  7. package/dist/esm/content/link-preview/client.js +2 -0
  8. package/dist/esm/content/link-preview/client.js.map +1 -1
  9. package/dist/esm/content/link-preview/content/article.js +15 -1
  10. package/dist/esm/content/link-preview/content/article.js.map +1 -1
  11. package/dist/esm/content/link-preview/content/index.js +151 -4
  12. package/dist/esm/content/link-preview/content/index.js.map +1 -1
  13. package/dist/esm/flags.js +12 -2
  14. package/dist/esm/flags.js.map +1 -1
  15. package/dist/esm/llm/generate-text.js +74 -7
  16. package/dist/esm/llm/generate-text.js.map +1 -1
  17. package/dist/esm/pricing/litellm.js +4 -1
  18. package/dist/esm/pricing/litellm.js.map +1 -1
  19. package/dist/esm/prompts/file.js +15 -4
  20. package/dist/esm/prompts/file.js.map +1 -1
  21. package/dist/esm/prompts/link-summary.js +20 -6
  22. package/dist/esm/prompts/link-summary.js.map +1 -1
  23. package/dist/esm/run.js +517 -396
  24. package/dist/esm/run.js.map +1 -1
  25. package/dist/esm/version.js +1 -1
  26. package/dist/types/content/link-preview/client.d.ts +2 -1
  27. package/dist/types/content/link-preview/deps.d.ts +30 -0
  28. package/dist/types/content/link-preview/types.d.ts +1 -1
  29. package/dist/types/costs.d.ts +1 -1
  30. package/dist/types/pricing/litellm.d.ts +1 -0
  31. package/dist/types/prompts/file.d.ts +2 -1
  32. package/dist/types/version.d.ts +1 -1
  33. package/docs/extract-only.md +1 -1
  34. package/docs/firecrawl.md +2 -2
  35. package/docs/llm.md +7 -0
  36. package/docs/site/docs/config.html +1 -1
  37. package/docs/site/docs/firecrawl.html +1 -1
  38. package/docs/website.md +3 -3
  39. package/package.json +3 -2
package/dist/esm/run.js CHANGED
@@ -1,6 +1,11 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { accessSync, constants as fsConstants } from 'node:fs';
1
3
  import fs from 'node:fs/promises';
4
+ import path from 'node:path';
2
5
  import { Command, CommanderError, Option } from 'commander';
6
+ import { countTokens } from 'gpt-tokenizer';
3
7
  import { createLiveRenderer, render as renderMarkdownAnsi } from 'markdansi';
8
+ import { normalizeTokenUsage, tallyCosts } from 'tokentally';
4
9
  import { loadSummarizeConfig } from './config.js';
5
10
  import { buildAssetPromptMessages, classifyUrl, loadLocalAsset, loadRemoteAsset, resolveInputTarget, } from './content/asset.js';
6
11
  import { createLinkPreviewClient } from './content/index.js';
@@ -11,21 +16,107 @@ import { generateTextWithModelId, streamTextWithModelId } from './llm/generate-t
11
16
  import { resolveGoogleModelForUsage } from './llm/google-models.js';
12
17
  import { createHtmlToMarkdownConverter } from './llm/html-to-markdown.js';
13
18
  import { normalizeGatewayStyleModelId, parseGatewayStyleModelId } from './llm/model-id.js';
14
- import { loadLiteLlmCatalog, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
15
- import { buildFileSummaryPrompt, buildLinkSummaryPrompt, SUMMARY_LENGTH_TO_TOKENS, } from './prompts/index.js';
19
+ import { loadLiteLlmCatalog, resolveLiteLlmMaxInputTokensForModelId, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
20
+ import { buildFileSummaryPrompt, buildLinkSummaryPrompt } from './prompts/index.js';
16
21
  import { startOscProgress } from './tty/osc-progress.js';
17
22
  import { startSpinner } from './tty/spinner.js';
18
23
  import { resolvePackageVersion } from './version.js';
19
- const MAP_REDUCE_TRIGGER_CHARACTERS = 120_000;
20
- const MAP_REDUCE_CHUNK_CHARACTERS = 60_000;
24
+ const BIRD_TIP = 'Tip: Install bird🐦 for better Twitter support: https://github.com/steipete/bird';
25
+ const TWITTER_HOSTS = new Set(['x.com', 'twitter.com', 'mobile.twitter.com']);
26
+ const SUMMARY_LENGTH_MAX_CHARACTERS = {
27
+ short: 1200,
28
+ medium: 2500,
29
+ long: 6000,
30
+ xl: 14000,
31
+ xxl: Number.POSITIVE_INFINITY,
32
+ };
33
+ function resolveTargetCharacters(lengthArg) {
34
+ return lengthArg.kind === 'chars'
35
+ ? lengthArg.maxCharacters
36
+ : SUMMARY_LENGTH_MAX_CHARACTERS[lengthArg.preset];
37
+ }
38
+ function isTwitterStatusUrl(raw) {
39
+ try {
40
+ const parsed = new URL(raw);
41
+ const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
42
+ if (!TWITTER_HOSTS.has(host))
43
+ return false;
44
+ return /\/status\/\d+/.test(parsed.pathname);
45
+ }
46
+ catch {
47
+ return false;
48
+ }
49
+ }
50
+ function isExecutable(filePath) {
51
+ try {
52
+ accessSync(filePath, fsConstants.X_OK);
53
+ return true;
54
+ }
55
+ catch {
56
+ return false;
57
+ }
58
+ }
59
+ function hasBirdCli(env) {
60
+ const candidates = [];
61
+ const pathEnv = env.PATH ?? process.env.PATH ?? '';
62
+ for (const entry of pathEnv.split(path.delimiter)) {
63
+ if (!entry)
64
+ continue;
65
+ candidates.push(path.join(entry, 'bird'));
66
+ }
67
+ return candidates.some((candidate) => isExecutable(candidate));
68
+ }
69
+ async function readTweetWithBird(args) {
70
+ return await new Promise((resolve, reject) => {
71
+ execFile('bird', ['read', args.url, '--json'], {
72
+ timeout: args.timeoutMs,
73
+ env: { ...process.env, ...args.env },
74
+ maxBuffer: 1024 * 1024,
75
+ }, (error, stdout, stderr) => {
76
+ if (error) {
77
+ const detail = stderr?.trim();
78
+ const suffix = detail ? `: ${detail}` : '';
79
+ reject(new Error(`bird read failed${suffix}`));
80
+ return;
81
+ }
82
+ const trimmed = stdout.trim();
83
+ if (!trimmed) {
84
+ reject(new Error('bird read returned empty output'));
85
+ return;
86
+ }
87
+ try {
88
+ const parsed = JSON.parse(trimmed);
89
+ const tweet = Array.isArray(parsed) ? parsed[0] : parsed;
90
+ if (!tweet || typeof tweet.text !== 'string') {
91
+ reject(new Error('bird read returned invalid payload'));
92
+ return;
93
+ }
94
+ resolve(tweet);
95
+ }
96
+ catch (parseError) {
97
+ const message = parseError instanceof Error ? parseError.message : String(parseError);
98
+ reject(new Error(`bird read returned invalid JSON: ${message}`));
99
+ }
100
+ });
101
+ });
102
+ }
103
+ function withBirdTip(error, url, env) {
104
+ if (!url || !isTwitterStatusUrl(url) || hasBirdCli(env)) {
105
+ return error instanceof Error ? error : new Error(String(error));
106
+ }
107
+ const message = error instanceof Error ? error.message : String(error);
108
+ const combined = `${message}\n${BIRD_TIP}`;
109
+ return error instanceof Error ? new Error(combined, { cause: error }) : new Error(combined);
110
+ }
111
+ const MAX_TEXT_BYTES_DEFAULT = 10 * 1024 * 1024;
21
112
  function buildProgram() {
22
113
  return new Command()
23
114
  .name('summarize')
24
115
  .description('Summarize web pages and YouTube links (uses direct provider API keys).')
25
116
  .argument('[input]', 'URL or local file path to summarize')
26
117
  .option('--youtube <mode>', 'YouTube transcript source: auto (web then apify), web (youtubei/captionTracks), apify', 'auto')
27
- .option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first). Note: in --extract-only website mode, defaults to always when FIRECRAWL_API_KEY is set.', 'auto')
28
- .option('--markdown <mode>', 'Website Markdown output: off, auto (prefer Firecrawl, then LLM when configured), llm (force LLM). Only affects --extract-only for non-YouTube URLs.', 'auto')
118
+ .option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first).', 'auto')
119
+ .option('--markdown <mode>', 'Website Markdown output: off, auto (use LLM when configured), llm (force LLM). Only affects --extract-only for non-YouTube URLs.', 'auto')
29
120
  .option('--length <length>', 'Summary length: short|medium|long|xl|xxl or a character limit like 20000, 20k', 'medium')
30
121
  .option('--max-output-tokens <count>', 'Hard cap for LLM output tokens (e.g. 2000, 2k). Overrides provider defaults.', undefined)
31
122
  .option('--timeout <duration>', 'Timeout for content fetching and LLM request: 30 (seconds), 30s, 2m, 5000ms', '2m')
@@ -139,19 +230,26 @@ function assertAssetMediaTypeSupported({ attachment, sizeLabel, }) {
139
230
  `Archive formats (zip/tar/7z/rar) can’t be sent to the model.\n` +
140
231
  `Unzip and summarize a specific file instead (e.g. README.md).`);
141
232
  }
142
- function buildAssetPromptPayload({ promptText, attachment, }) {
143
- if (attachment.part.type === 'file' && isTextLikeMediaType(attachment.mediaType)) {
144
- const data = attachment.part.data;
145
- const content = typeof data === 'string'
146
- ? data
147
- : data instanceof Uint8Array
148
- ? new TextDecoder().decode(data)
149
- : '';
233
+ function buildAssetPromptPayload({ promptText, attachment, textContent, }) {
234
+ if (textContent && attachment.part.type === 'file' && isTextLikeMediaType(attachment.mediaType)) {
150
235
  const header = `File: ${attachment.filename ?? 'unknown'} (${attachment.mediaType})`;
151
- return `${promptText}\n\n---\n${header}\n\n${content}`.trim();
236
+ return `${promptText}\n\n---\n${header}\n\n${textContent.content}`.trim();
152
237
  }
153
238
  return buildAssetPromptMessages({ promptText, attachment });
154
239
  }
240
+ function getTextContentFromAttachment(attachment) {
241
+ if (attachment.part.type !== 'file' || !isTextLikeMediaType(attachment.mediaType)) {
242
+ return null;
243
+ }
244
+ const data = attachment.part.data;
245
+ if (typeof data === 'string') {
246
+ return { content: data, bytes: Buffer.byteLength(data, 'utf8') };
247
+ }
248
+ if (data instanceof Uint8Array) {
249
+ return { content: new TextDecoder().decode(data), bytes: data.byteLength };
250
+ }
251
+ return { content: '', bytes: 0 };
252
+ }
155
253
  function assertProviderSupportsAttachment({ provider, modelId, attachment, }) {
156
254
  // xAI via AI SDK currently supports image parts, but not generic file parts (e.g. PDFs).
157
255
  if (provider === 'xai' &&
@@ -196,6 +294,18 @@ function isGoogleStreamingUnsupportedError(error) {
196
294
  /Call ListModels/i.test(errorText) ||
197
295
  /supported methods/i.test(errorText));
198
296
  }
297
+ function isStreamingTimeoutError(error) {
298
+ if (!error)
299
+ return false;
300
+ const message = typeof error === 'string'
301
+ ? error
302
+ : error instanceof Error
303
+ ? error.message
304
+ : typeof error.message === 'string'
305
+ ? String(error.message)
306
+ : '';
307
+ return /timed out/i.test(message);
308
+ }
199
309
  function attachRichHelp(program, env, stdout) {
200
310
  const color = supportsColor(stdout, env);
201
311
  const heading = (text) => ansi('1;36', text, color);
@@ -204,7 +314,7 @@ function attachRichHelp(program, env, stdout) {
204
314
  program.addHelpText('after', () => `
205
315
  ${heading('Examples')}
206
316
  ${cmd('summarize "https://example.com"')}
207
- ${cmd('summarize "https://example.com" --extract-only')} ${dim('# website markdown (prefers Firecrawl when configured)')}
317
+ ${cmd('summarize "https://example.com" --extract-only')} ${dim('# website markdown (LLM if configured)')}
208
318
  ${cmd('summarize "https://example.com" --extract-only --markdown llm')} ${dim('# website markdown via LLM')}
209
319
  ${cmd('summarize "https://www.youtube.com/watch?v=I845O57ZSy4&t=11s" --extract-only --youtube web')}
210
320
  ${cmd('summarize "https://example.com" --length 20k --max-output-tokens 2k --timeout 2m --model openai/gpt-5.2')}
@@ -240,38 +350,6 @@ async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutM
240
350
  usage: result.usage,
241
351
  };
242
352
  }
243
- function splitTextIntoChunks(input, maxCharacters) {
244
- if (maxCharacters <= 0) {
245
- return [input];
246
- }
247
- const text = input.trim();
248
- if (text.length <= maxCharacters) {
249
- return [text];
250
- }
251
- const chunks = [];
252
- let offset = 0;
253
- while (offset < text.length) {
254
- const end = Math.min(offset + maxCharacters, text.length);
255
- const slice = text.slice(offset, end);
256
- if (end === text.length) {
257
- chunks.push(slice.trim());
258
- break;
259
- }
260
- const candidateBreaks = [
261
- slice.lastIndexOf('\n\n'),
262
- slice.lastIndexOf('\n'),
263
- slice.lastIndexOf('. '),
264
- ];
265
- const lastBreak = Math.max(...candidateBreaks);
266
- const splitAt = lastBreak > Math.floor(maxCharacters * 0.5) ? lastBreak + 1 : slice.length;
267
- const chunk = slice.slice(0, splitAt).trim();
268
- if (chunk.length > 0) {
269
- chunks.push(chunk);
270
- }
271
- offset += splitAt;
272
- }
273
- return chunks.filter((chunk) => chunk.length > 0);
274
- }
275
353
  const VERBOSE_PREFIX = '[summarize]';
276
354
  function writeVerbose(stderr, verbose, message, color) {
277
355
  if (!verbose) {
@@ -315,6 +393,11 @@ function formatBytes(bytes) {
315
393
  }
316
394
  return `${value.toFixed(value >= 10 ? 0 : 1)} ${units[unitIndex]}`;
317
395
  }
396
+ function formatCount(value) {
397
+ if (!Number.isFinite(value))
398
+ return 'unknown';
399
+ return value.toLocaleString('en-US');
400
+ }
318
401
  function sumNumbersOrNull(values) {
319
402
  let sum = 0;
320
403
  let any = false;
@@ -331,7 +414,15 @@ function formatUSD(value) {
331
414
  return 'n/a';
332
415
  return `$${value.toFixed(4)}`;
333
416
  }
334
- function writeFinishLine({ stderr, elapsedMs, model, strategy, chunkCount, report, costUsd, color, }) {
417
+ function mergeStreamingChunk(previous, chunk) {
418
+ if (!chunk)
419
+ return { next: previous, appended: '' };
420
+ if (chunk.startsWith(previous)) {
421
+ return { next: chunk, appended: chunk.slice(previous.length) };
422
+ }
423
+ return { next: previous + chunk, appended: chunk };
424
+ }
425
+ function writeFinishLine({ stderr, elapsedMs, model, report, costUsd, color, }) {
335
426
  const promptTokens = sumNumbersOrNull(report.llm.map((row) => row.promptTokens));
336
427
  const completionTokens = sumNumbersOrNull(report.llm.map((row) => row.completionTokens));
337
428
  const totalTokens = sumNumbersOrNull(report.llm.map((row) => row.totalTokens));
@@ -349,25 +440,10 @@ function writeFinishLine({ stderr, elapsedMs, model, strategy, chunkCount, repor
349
440
  if (report.services.apify.requests > 0) {
350
441
  parts.push(`apify=${report.services.apify.requests}`);
351
442
  }
352
- if (strategy === 'map-reduce') {
353
- parts.push('strategy=map-reduce');
354
- if (typeof chunkCount === 'number' && Number.isFinite(chunkCount) && chunkCount > 0) {
355
- parts.push(`chunks=${chunkCount}`);
356
- }
357
- }
358
443
  const line = `Finished in ${formatElapsedMs(elapsedMs)} (${parts.join(' | ')})`;
359
444
  stderr.write('\n');
360
445
  stderr.write(`${ansi('1;32', line, color)}\n`);
361
446
  }
362
- function buildChunkNotesPrompt({ content }) {
363
- return `Return 10 bullet points summarizing the content below (Markdown).
364
-
365
- CONTENT:
366
- """
367
- ${content}
368
- """
369
- `;
370
- }
371
447
  export async function runCli(argv, { env, fetch, stdout, stderr }) {
372
448
  ;
373
449
  globalThis.AI_SDK_LOG_WARNINGS = false;
@@ -419,7 +495,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
419
495
  const markdownMode = parseMarkdownMode(program.opts().markdown);
420
496
  const shouldComputeReport = metricsEnabled;
421
497
  const isYoutubeUrl = typeof url === 'string' ? /youtube\.com|youtu\.be/i.test(url) : false;
422
- const firecrawlExplicitlySet = normalizedArgv.some((arg) => arg === '--firecrawl' || arg.startsWith('--firecrawl='));
423
498
  const requestedFirecrawlMode = parseFirecrawlMode(program.opts().firecrawl);
424
499
  const modelArg = typeof program.opts().model === 'string' ? program.opts().model : null;
425
500
  const { config, path: configPath } = loadSummarizeConfig({ env });
@@ -477,29 +552,41 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
477
552
  return null;
478
553
  return capMaxOutputTokensForModel({ modelId, requested: maxOutputTokensArg });
479
554
  };
555
+ const resolveMaxInputTokensForCall = async (modelId) => {
556
+ const catalog = await getLiteLlmCatalog();
557
+ if (!catalog)
558
+ return null;
559
+ const limit = resolveLiteLlmMaxInputTokensForModelId(catalog, modelId);
560
+ if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0) {
561
+ return limit;
562
+ }
563
+ return null;
564
+ };
480
565
  const estimateCostUsd = async () => {
481
566
  const catalog = await getLiteLlmCatalog();
482
567
  if (!catalog)
483
568
  return null;
484
- let total = 0;
485
- let any = false;
486
- for (const call of llmCalls) {
569
+ const calls = llmCalls.map((call) => {
487
570
  const promptTokens = call.usage?.promptTokens ?? null;
488
571
  const completionTokens = call.usage?.completionTokens ?? null;
489
- if (typeof promptTokens !== 'number' ||
490
- !Number.isFinite(promptTokens) ||
491
- typeof completionTokens !== 'number' ||
492
- !Number.isFinite(completionTokens)) {
493
- continue;
494
- }
495
- const pricing = resolveLiteLlmPricingForModelId(catalog, call.model);
496
- if (!pricing)
497
- continue;
498
- total +=
499
- promptTokens * pricing.inputUsdPerToken + completionTokens * pricing.outputUsdPerToken;
500
- any = true;
501
- }
502
- return any ? total : null;
572
+ const hasTokens = typeof promptTokens === 'number' &&
573
+ Number.isFinite(promptTokens) &&
574
+ typeof completionTokens === 'number' &&
575
+ Number.isFinite(completionTokens);
576
+ const usage = hasTokens
577
+ ? normalizeTokenUsage({
578
+ inputTokens: promptTokens,
579
+ outputTokens: completionTokens,
580
+ totalTokens: call.usage?.totalTokens ?? undefined,
581
+ })
582
+ : null;
583
+ return { model: call.model, usage };
584
+ });
585
+ const result = await tallyCosts({
586
+ calls,
587
+ resolvePricing: (modelId) => resolveLiteLlmPricingForModelId(catalog, modelId),
588
+ });
589
+ return result.total?.totalUsd ?? null;
503
590
  };
504
591
  const buildReport = async () => {
505
592
  return buildRunMetricsReport({ llmCalls, firecrawlRequests, apifyRequests });
@@ -609,14 +696,29 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
609
696
  const effectiveModelId = modelResolution.modelId;
610
697
  const parsedModelEffective = parseGatewayStyleModelId(effectiveModelId);
611
698
  const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
699
+ const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
700
+ const textContent = getTextContentFromAttachment(attachment);
701
+ if (textContent && textContent.bytes > MAX_TEXT_BYTES_DEFAULT) {
702
+ throw new Error(`Text file too large (${formatBytes(textContent.bytes)}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
703
+ }
612
704
  const summaryLengthTarget = lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters };
613
705
  const promptText = buildFileSummaryPrompt({
614
706
  filename: attachment.filename,
615
707
  mediaType: attachment.mediaType,
616
708
  summaryLength: summaryLengthTarget,
709
+ contentLength: textContent?.content.length ?? null,
617
710
  });
618
- const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
619
- const promptPayload = buildAssetPromptPayload({ promptText, attachment });
711
+ const promptPayload = buildAssetPromptPayload({ promptText, attachment, textContent });
712
+ const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
713
+ if (typeof maxInputTokensForCall === 'number' &&
714
+ Number.isFinite(maxInputTokensForCall) &&
715
+ maxInputTokensForCall > 0 &&
716
+ typeof promptPayload === 'string') {
717
+ const tokenCount = countTokens(promptPayload);
718
+ if (tokenCount > maxInputTokensForCall) {
719
+ throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
720
+ }
721
+ }
620
722
  const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
621
723
  const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
622
724
  const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
@@ -637,7 +739,26 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
637
739
  });
638
740
  }
639
741
  catch (error) {
640
- if (parsedModelEffective.provider === 'google' &&
742
+ if (isStreamingTimeoutError(error)) {
743
+ writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
744
+ const result = await summarizeWithModelId({
745
+ modelId: parsedModelEffective.canonical,
746
+ prompt: promptPayload,
747
+ maxOutputTokens: maxOutputTokensForCall ?? undefined,
748
+ timeoutMs,
749
+ fetchImpl: trackedFetch,
750
+ apiKeys: apiKeysForLlm,
751
+ });
752
+ llmCalls.push({
753
+ provider: result.provider,
754
+ model: result.canonicalModelId,
755
+ usage: result.usage,
756
+ purpose: 'summary',
757
+ });
758
+ summary = result.text;
759
+ streamResult = null;
760
+ }
761
+ else if (parsedModelEffective.provider === 'google' &&
641
762
  isGoogleStreamingUnsupportedError(error)) {
642
763
  writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
643
764
  const result = await summarizeWithModelId({
@@ -690,9 +811,11 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
690
811
  clearProgressForStdout();
691
812
  cleared = true;
692
813
  }
693
- streamed += delta;
814
+ const merged = mergeStreamingChunk(streamed, delta);
815
+ streamed = merged.next;
694
816
  if (shouldStreamSummaryToStdout) {
695
- stdout.write(delta);
817
+ if (merged.appended)
818
+ stdout.write(merged.appended);
696
819
  continue;
697
820
  }
698
821
  if (liveRenderer) {
@@ -819,7 +942,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
819
942
  model: parsedModelEffective.canonical,
820
943
  maxCompletionTokens: maxOutputTokensForCall,
821
944
  strategy: 'single',
822
- chunkCount: 1,
823
945
  },
824
946
  metrics: metricsEnabled ? finishReport : null,
825
947
  summary,
@@ -834,8 +956,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
834
956
  stderr,
835
957
  elapsedMs: Date.now() - runStartedAtMs,
836
958
  model: parsedModelEffective.canonical,
837
- strategy: 'single',
838
- chunkCount: 1,
839
959
  report: finishReport,
840
960
  costUsd,
841
961
  color: verboseColor,
@@ -866,8 +986,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
866
986
  stderr,
867
987
  elapsedMs: Date.now() - runStartedAtMs,
868
988
  model: parsedModelEffective.canonical,
869
- strategy: 'single',
870
- chunkCount: 1,
871
989
  report,
872
990
  costUsd,
873
991
  color: verboseColor,
@@ -988,12 +1106,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
988
1106
  if (!url) {
989
1107
  throw new Error('Only HTTP and HTTPS URLs can be summarized');
990
1108
  }
991
- const firecrawlMode = (() => {
992
- if (extractOnly && !isYoutubeUrl && !firecrawlExplicitlySet && firecrawlConfigured) {
993
- return 'always';
994
- }
995
- return requestedFirecrawlMode;
996
- })();
1109
+ const firecrawlMode = requestedFirecrawlMode;
997
1110
  if (firecrawlMode === 'always' && !firecrawlConfigured) {
998
1111
  throw new Error('--firecrawl always requires FIRECRAWL_API_KEY');
999
1112
  }
@@ -1037,6 +1150,9 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1037
1150
  },
1038
1151
  })
1039
1152
  : null;
1153
+ const readTweetWithBirdClient = hasBirdCli(env)
1154
+ ? ({ url, timeoutMs }) => readTweetWithBird({ url, timeoutMs, env })
1155
+ : null;
1040
1156
  writeVerbose(stderr, verbose, 'extract start', verboseColor);
1041
1157
  const stopOscProgress = startOscProgress({
1042
1158
  label: 'Fetching website',
@@ -1057,22 +1173,65 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1057
1173
  phase: 'idle',
1058
1174
  htmlDownloadedBytes: 0,
1059
1175
  htmlTotalBytes: null,
1176
+ fetchStartedAtMs: null,
1060
1177
  lastSpinnerUpdateAtMs: 0,
1061
1178
  };
1062
- const updateSpinner = (text) => {
1179
+ let ticker = null;
1180
+ const updateSpinner = (text, options) => {
1063
1181
  const now = Date.now();
1064
- if (now - state.lastSpinnerUpdateAtMs < 100)
1182
+ if (!options?.force && now - state.lastSpinnerUpdateAtMs < 100)
1065
1183
  return;
1066
1184
  state.lastSpinnerUpdateAtMs = now;
1067
1185
  spinner.setText(text);
1068
1186
  };
1187
+ const formatFirecrawlReason = (reason) => {
1188
+ const lower = reason.toLowerCase();
1189
+ if (lower.includes('forced'))
1190
+ return 'forced';
1191
+ if (lower.includes('html fetch failed'))
1192
+ return 'fallback: HTML fetch failed';
1193
+ if (lower.includes('blocked') || lower.includes('thin'))
1194
+ return 'fallback: blocked/thin HTML';
1195
+ return reason;
1196
+ };
1197
+ const renderFetchLine = () => {
1198
+ const downloaded = formatBytes(state.htmlDownloadedBytes);
1199
+ const total = typeof state.htmlTotalBytes === 'number' ? `/${formatBytes(state.htmlTotalBytes)}` : '';
1200
+ const elapsedMs = typeof state.fetchStartedAtMs === 'number' ? Date.now() - state.fetchStartedAtMs : 0;
1201
+ const elapsed = formatElapsedMs(elapsedMs);
1202
+ if (state.htmlDownloadedBytes === 0 && !state.htmlTotalBytes) {
1203
+ return `Fetching website (connecting, ${elapsed})…`;
1204
+ }
1205
+ const rate = elapsedMs > 0 && state.htmlDownloadedBytes > 0
1206
+ ? `, ${formatBytes(state.htmlDownloadedBytes / (elapsedMs / 1000))}/s`
1207
+ : '';
1208
+ return `Fetching website (${downloaded}${total}, ${elapsed}${rate})…`;
1209
+ };
1210
+ const startTicker = () => {
1211
+ if (ticker)
1212
+ return;
1213
+ ticker = setInterval(() => {
1214
+ if (state.phase !== 'fetching')
1215
+ return;
1216
+ updateSpinner(renderFetchLine());
1217
+ }, 1000);
1218
+ };
1219
+ const stopTicker = () => {
1220
+ if (!ticker)
1221
+ return;
1222
+ clearInterval(ticker);
1223
+ ticker = null;
1224
+ };
1069
1225
  return {
1070
1226
  getHtmlDownloadedBytes: () => state.htmlDownloadedBytes,
1227
+ stop: stopTicker,
1071
1228
  onProgress: (event) => {
1072
1229
  if (event.kind === 'fetch-html-start') {
1073
1230
  state.phase = 'fetching';
1074
1231
  state.htmlDownloadedBytes = 0;
1075
1232
  state.htmlTotalBytes = null;
1233
+ state.fetchStartedAtMs = Date.now();
1234
+ startTicker();
1076
1235
  updateSpinner('Fetching website (connecting)…');
1077
1236
  return;
1078
1237
  }
@@ -1080,23 +1239,57 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1080
1239
  state.phase = 'fetching';
1081
1240
  state.htmlDownloadedBytes = event.downloadedBytes;
1082
1241
  state.htmlTotalBytes = event.totalBytes;
1083
- const downloaded = formatBytes(event.downloadedBytes);
1084
- const total = typeof event.totalBytes === 'number' ? `/${formatBytes(event.totalBytes)}` : '';
1085
- updateSpinner(`Fetching website (${downloaded}${total})…`);
1242
+ updateSpinner(renderFetchLine());
1243
+ return;
1244
+ }
1245
+ if (event.kind === 'bird-start') {
1246
+ state.phase = 'bird';
1247
+ stopTicker();
1248
+ updateSpinner('Bird: reading tweet…', { force: true });
1249
+ return;
1250
+ }
1251
+ if (event.kind === 'bird-done') {
1252
+ state.phase = 'bird';
1253
+ stopTicker();
1254
+ if (event.ok && typeof event.textBytes === 'number') {
1255
+ updateSpinner(`Bird: got ${formatBytes(event.textBytes)}…`, { force: true });
1256
+ return;
1257
+ }
1258
+ updateSpinner('Bird: failed; fallback…', { force: true });
1259
+ return;
1260
+ }
1261
+ if (event.kind === 'nitter-start') {
1262
+ state.phase = 'nitter';
1263
+ stopTicker();
1264
+ updateSpinner('Nitter: fetching…', { force: true });
1265
+ return;
1266
+ }
1267
+ if (event.kind === 'nitter-done') {
1268
+ state.phase = 'nitter';
1269
+ stopTicker();
1270
+ if (event.ok && typeof event.textBytes === 'number') {
1271
+ updateSpinner(`Nitter: got ${formatBytes(event.textBytes)}…`, { force: true });
1272
+ return;
1273
+ }
1274
+ updateSpinner('Nitter: failed; fallback…', { force: true });
1086
1275
  return;
1087
1276
  }
1088
1277
  if (event.kind === 'firecrawl-start') {
1089
1278
  state.phase = 'firecrawl';
1090
- updateSpinner('Firecrawl: scraping…');
1279
+ stopTicker();
1280
+ const reason = event.reason ? formatFirecrawlReason(event.reason) : '';
1281
+ const suffix = reason ? ` (${reason})` : '';
1282
+ updateSpinner(`Firecrawl: scraping${suffix}…`, { force: true });
1091
1283
  return;
1092
1284
  }
1093
1285
  if (event.kind === 'firecrawl-done') {
1094
1286
  state.phase = 'firecrawl';
1287
+ stopTicker();
1095
1288
  if (event.ok && typeof event.markdownBytes === 'number') {
1096
- updateSpinner(`Firecrawl: got ${formatBytes(event.markdownBytes)}…`);
1289
+ updateSpinner(`Firecrawl: got ${formatBytes(event.markdownBytes)}…`, { force: true });
1097
1290
  return;
1098
1291
  }
1099
- updateSpinner('Firecrawl: no content; fallback…');
1292
+ updateSpinner('Firecrawl: no content; fallback…', { force: true });
1100
1293
  }
1101
1294
  },
1102
1295
  };
@@ -1105,6 +1298,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1105
1298
  apifyApiToken: apifyToken,
1106
1299
  scrapeWithFirecrawl,
1107
1300
  convertHtmlToMarkdown,
1301
+ readTweetWithBird: readTweetWithBirdClient,
1108
1302
  fetch: trackedFetch,
1109
1303
  onProgress: websiteProgress?.onProgress ?? null,
1110
1304
  });
@@ -1113,24 +1307,42 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1113
1307
  if (stopped)
1114
1308
  return;
1115
1309
  stopped = true;
1310
+ websiteProgress?.stop?.();
1116
1311
  spinner.stopAndClear();
1117
1312
  stopOscProgress();
1118
1313
  };
1119
1314
  clearProgressBeforeStdout = stopProgress;
1120
1315
  try {
1121
- const extracted = await client.fetchLinkContent(url, {
1122
- timeoutMs,
1123
- youtubeTranscript: youtubeMode,
1124
- firecrawl: firecrawlMode,
1125
- format: markdownRequested ? 'markdown' : 'text',
1126
- });
1316
+ let extracted;
1317
+ try {
1318
+ extracted = await client.fetchLinkContent(url, {
1319
+ timeoutMs,
1320
+ youtubeTranscript: youtubeMode,
1321
+ firecrawl: firecrawlMode,
1322
+ format: markdownRequested ? 'markdown' : 'text',
1323
+ });
1324
+ }
1325
+ catch (error) {
1326
+ throw withBirdTip(error, url, env);
1327
+ }
1127
1328
  const extractedContentBytes = Buffer.byteLength(extracted.content, 'utf8');
1128
1329
  const extractedContentSize = formatBytes(extractedContentBytes);
1129
- const viaFirecrawl = extracted.diagnostics.firecrawl.used ? ', Firecrawl' : '';
1330
+ const viaSources = [];
1331
+ if (extracted.diagnostics.strategy === 'bird') {
1332
+ viaSources.push('bird');
1333
+ }
1334
+ if (extracted.diagnostics.strategy === 'nitter') {
1335
+ viaSources.push('Nitter');
1336
+ }
1337
+ if (extracted.diagnostics.firecrawl.used) {
1338
+ viaSources.push('Firecrawl');
1339
+ }
1340
+ const viaSourceLabel = viaSources.length > 0 ? `, ${viaSources.join('+')}` : '';
1130
1341
  if (progressEnabled) {
1342
+ websiteProgress?.stop?.();
1131
1343
  spinner.setText(extractOnly
1132
- ? `Extracted (${extractedContentSize})`
1133
- : `Summarizing (sent ${extractedContentSize}${viaFirecrawl})…`);
1344
+ ? `Extracted (${extractedContentSize}${viaSourceLabel})`
1345
+ : `Summarizing (sent ${extractedContentSize}${viaSourceLabel})…`);
1134
1346
  }
1135
1347
  writeVerbose(stderr, verbose, `extract done strategy=${extracted.diagnostics.strategy} siteName=${formatOptionalString(extracted.siteName)} title=${formatOptionalString(extracted.title)} transcriptSource=${formatOptionalString(extracted.transcriptSource)}`, verboseColor);
1136
1348
  writeVerbose(stderr, verbose, `extract stats characters=${extracted.totalCharacters} words=${extracted.wordCount} transcriptCharacters=${formatOptionalNumber(extracted.transcriptCharacters)} transcriptLines=${formatOptionalNumber(extracted.transcriptLines)}`, verboseColor);
@@ -1194,8 +1406,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1194
1406
  stderr,
1195
1407
  elapsedMs: Date.now() - runStartedAtMs,
1196
1408
  model,
1197
- strategy: 'none',
1198
- chunkCount: null,
1199
1409
  report: finishReport,
1200
1410
  costUsd,
1201
1411
  color: verboseColor,
@@ -1213,8 +1423,76 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1213
1423
  stderr,
1214
1424
  elapsedMs: Date.now() - runStartedAtMs,
1215
1425
  model,
1216
- strategy: 'none',
1217
- chunkCount: null,
1426
+ report,
1427
+ costUsd,
1428
+ color: verboseColor,
1429
+ });
1430
+ }
1431
+ return;
1432
+ }
1433
+ const shouldSkipTweetSummary = isTwitterStatusUrl(url) &&
1434
+ extracted.content.length > 0 &&
1435
+ extracted.content.length <= resolveTargetCharacters(lengthArg);
1436
+ if (shouldSkipTweetSummary) {
1437
+ clearProgressForStdout();
1438
+ writeVerbose(stderr, verbose, `skip summary: tweet content length=${extracted.content.length} target=${resolveTargetCharacters(lengthArg)}`, verboseColor);
1439
+ if (json) {
1440
+ const finishReport = shouldComputeReport ? await buildReport() : null;
1441
+ const payload = {
1442
+ input: {
1443
+ kind: 'url',
1444
+ url,
1445
+ timeoutMs,
1446
+ youtube: youtubeMode,
1447
+ firecrawl: firecrawlMode,
1448
+ markdown: effectiveMarkdownMode,
1449
+ length: lengthArg.kind === 'preset'
1450
+ ? { kind: 'preset', preset: lengthArg.preset }
1451
+ : { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
1452
+ maxOutputTokens: maxOutputTokensArg,
1453
+ model,
1454
+ },
1455
+ env: {
1456
+ hasXaiKey: Boolean(xaiApiKey),
1457
+ hasOpenAIKey: Boolean(apiKey),
1458
+ hasApifyToken: Boolean(apifyToken),
1459
+ hasFirecrawlKey: firecrawlConfigured,
1460
+ hasGoogleKey: googleConfigured,
1461
+ hasAnthropicKey: anthropicConfigured,
1462
+ },
1463
+ extracted,
1464
+ prompt,
1465
+ llm: null,
1466
+ metrics: metricsEnabled ? finishReport : null,
1467
+ summary: extracted.content,
1468
+ };
1469
+ if (metricsDetailed && finishReport) {
1470
+ writeMetricsReport(finishReport);
1471
+ }
1472
+ stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
1473
+ if (metricsEnabled && finishReport) {
1474
+ const costUsd = await estimateCostUsd();
1475
+ writeFinishLine({
1476
+ stderr,
1477
+ elapsedMs: Date.now() - runStartedAtMs,
1478
+ model,
1479
+ report: finishReport,
1480
+ costUsd,
1481
+ color: verboseColor,
1482
+ });
1483
+ }
1484
+ return;
1485
+ }
1486
+ stdout.write(`${extracted.content}\n`);
1487
+ const report = shouldComputeReport ? await buildReport() : null;
1488
+ if (metricsDetailed && report)
1489
+ writeMetricsReport(report);
1490
+ if (metricsEnabled && report) {
1491
+ const costUsd = await estimateCostUsd();
1492
+ writeFinishLine({
1493
+ stderr,
1494
+ elapsedMs: Date.now() - runStartedAtMs,
1495
+ model,
1218
1496
  report,
1219
1497
  costUsd,
1220
1498
  color: verboseColor,
@@ -1259,317 +1537,165 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1259
1537
  const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
1260
1538
  writeVerbose(stderr, verbose, `mode summarize provider=${parsedModelEffective.provider} model=${parsedModelEffective.canonical}`, verboseColor);
1261
1539
  const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
1262
- const isLargeContent = extracted.content.length >= MAP_REDUCE_TRIGGER_CHARACTERS;
1263
- let strategy = 'single';
1264
- let chunkCount = 1;
1540
+ const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
1541
+ if (typeof maxInputTokensForCall === 'number' &&
1542
+ Number.isFinite(maxInputTokensForCall) &&
1543
+ maxInputTokensForCall > 0) {
1544
+ const tokenCount = countTokens(prompt);
1545
+ if (tokenCount > maxInputTokensForCall) {
1546
+ throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
1547
+ }
1548
+ }
1265
1549
  const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
1266
1550
  const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
1267
1551
  const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
1268
1552
  let summaryAlreadyPrinted = false;
1269
1553
  let summary = '';
1270
1554
  let getLastStreamError = null;
1271
- if (!isLargeContent) {
1272
- writeVerbose(stderr, verbose, 'summarize strategy=single', verboseColor);
1273
- if (streamingEnabledForCall) {
1274
- writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
1275
- let streamResult = null;
1276
- try {
1277
- streamResult = await streamTextWithModelId({
1555
+ writeVerbose(stderr, verbose, 'summarize strategy=single', verboseColor);
1556
+ if (streamingEnabledForCall) {
1557
+ writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
1558
+ let streamResult = null;
1559
+ try {
1560
+ streamResult = await streamTextWithModelId({
1561
+ modelId: parsedModelEffective.canonical,
1562
+ apiKeys: apiKeysForLlm,
1563
+ prompt,
1564
+ temperature: 0,
1565
+ maxOutputTokens: maxOutputTokensForCall ?? undefined,
1566
+ timeoutMs,
1567
+ fetchImpl: trackedFetch,
1568
+ });
1569
+ }
1570
+ catch (error) {
1571
+ if (isStreamingTimeoutError(error)) {
1572
+ writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
1573
+ const result = await summarizeWithModelId({
1278
1574
  modelId: parsedModelEffective.canonical,
1279
- apiKeys: apiKeysForLlm,
1280
1575
  prompt,
1281
- temperature: 0,
1282
1576
  maxOutputTokens: maxOutputTokensForCall ?? undefined,
1283
1577
  timeoutMs,
1284
1578
  fetchImpl: trackedFetch,
1579
+ apiKeys: apiKeysForLlm,
1285
1580
  });
1286
- }
1287
- catch (error) {
1288
- if (parsedModelEffective.provider === 'google' &&
1289
- isGoogleStreamingUnsupportedError(error)) {
1290
- writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
1291
- const result = await summarizeWithModelId({
1292
- modelId: parsedModelEffective.canonical,
1293
- prompt,
1294
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1295
- timeoutMs,
1296
- fetchImpl: trackedFetch,
1297
- apiKeys: apiKeysForLlm,
1298
- });
1299
- llmCalls.push({
1300
- provider: result.provider,
1301
- model: result.canonicalModelId,
1302
- usage: result.usage,
1303
- purpose: 'summary',
1304
- });
1305
- summary = result.text;
1306
- streamResult = null;
1307
- }
1308
- else {
1309
- throw error;
1310
- }
1311
- }
1312
- if (streamResult) {
1313
- getLastStreamError = streamResult.lastError;
1314
- let streamed = '';
1315
- const liveRenderer = shouldLiveRenderSummary
1316
- ? createLiveRenderer({
1317
- write: (chunk) => {
1318
- clearProgressForStdout();
1319
- stdout.write(chunk);
1320
- },
1321
- width: markdownRenderWidth(stdout, env),
1322
- renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
1323
- width: markdownRenderWidth(stdout, env),
1324
- wrap: true,
1325
- color: supportsColor(stdout, env),
1326
- }),
1327
- })
1328
- : null;
1329
- let lastFrameAtMs = 0;
1330
- try {
1331
- let cleared = false;
1332
- for await (const delta of streamResult.textStream) {
1333
- streamed += delta;
1334
- if (shouldStreamSummaryToStdout) {
1335
- if (!cleared) {
1336
- clearProgressForStdout();
1337
- cleared = true;
1338
- }
1339
- stdout.write(delta);
1340
- continue;
1341
- }
1342
- if (liveRenderer) {
1343
- const now = Date.now();
1344
- const due = now - lastFrameAtMs >= 120;
1345
- const hasNewline = delta.includes('\n');
1346
- if (hasNewline || due) {
1347
- liveRenderer.render(streamed);
1348
- lastFrameAtMs = now;
1349
- }
1350
- }
1351
- }
1352
- const trimmed = streamed.trim();
1353
- streamed = trimmed;
1354
- if (liveRenderer) {
1355
- liveRenderer.render(trimmed);
1356
- summaryAlreadyPrinted = true;
1357
- }
1358
- }
1359
- finally {
1360
- liveRenderer?.finish();
1361
- }
1362
- const usage = await streamResult.usage;
1363
1581
  llmCalls.push({
1364
- provider: streamResult.provider,
1365
- model: streamResult.canonicalModelId,
1366
- usage,
1582
+ provider: result.provider,
1583
+ model: result.canonicalModelId,
1584
+ usage: result.usage,
1367
1585
  purpose: 'summary',
1368
1586
  });
1369
- summary = streamed;
1370
- if (shouldStreamSummaryToStdout) {
1371
- if (!streamed.endsWith('\n')) {
1372
- stdout.write('\n');
1373
- }
1374
- summaryAlreadyPrinted = true;
1375
- }
1587
+ summary = result.text;
1588
+ streamResult = null;
1376
1589
  }
1377
- }
1378
- else {
1379
- const result = await summarizeWithModelId({
1380
- modelId: parsedModelEffective.canonical,
1381
- prompt,
1382
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1383
- timeoutMs,
1384
- fetchImpl: trackedFetch,
1385
- apiKeys: apiKeysForLlm,
1386
- });
1387
- llmCalls.push({
1388
- provider: result.provider,
1389
- model: result.canonicalModelId,
1390
- usage: result.usage,
1391
- purpose: 'summary',
1392
- });
1393
- summary = result.text;
1394
- }
1395
- }
1396
- else {
1397
- strategy = 'map-reduce';
1398
- const chunks = splitTextIntoChunks(extracted.content, MAP_REDUCE_CHUNK_CHARACTERS);
1399
- chunkCount = chunks.length;
1400
- stderr.write(`Large input (${extracted.content.length} chars); summarizing in ${chunks.length} chunks.\n`);
1401
- writeVerbose(stderr, verbose, `summarize strategy=map-reduce chunks=${chunks.length}`, verboseColor);
1402
- const chunkNotes = [];
1403
- for (let i = 0; i < chunks.length; i += 1) {
1404
- writeVerbose(stderr, verbose, `summarize chunk ${i + 1}/${chunks.length} notes start`, verboseColor);
1405
- const chunkPrompt = buildChunkNotesPrompt({
1406
- content: chunks[i] ?? '',
1407
- });
1408
- const chunkNoteTokensRequested = typeof maxOutputTokensArg === 'number'
1409
- ? Math.min(SUMMARY_LENGTH_TO_TOKENS.medium, maxOutputTokensArg)
1410
- : SUMMARY_LENGTH_TO_TOKENS.medium;
1411
- const chunkNoteTokens = await capMaxOutputTokensForModel({
1412
- modelId: parsedModelEffective.canonical,
1413
- requested: chunkNoteTokensRequested,
1414
- });
1415
- const notesResult = await summarizeWithModelId({
1416
- modelId: parsedModelEffective.canonical,
1417
- prompt: chunkPrompt,
1418
- maxOutputTokens: chunkNoteTokens,
1419
- timeoutMs,
1420
- fetchImpl: trackedFetch,
1421
- apiKeys: apiKeysForLlm,
1422
- });
1423
- const notes = notesResult.text;
1424
- llmCalls.push({
1425
- provider: notesResult.provider,
1426
- model: notesResult.canonicalModelId,
1427
- usage: notesResult.usage,
1428
- purpose: 'chunk-notes',
1429
- });
1430
- chunkNotes.push(notes.trim());
1431
- }
1432
- writeVerbose(stderr, verbose, 'summarize merge chunk notes', verboseColor);
1433
- const mergedContent = `Chunk notes (generated from the full input):\n\n${chunkNotes
1434
- .filter((value) => value.length > 0)
1435
- .join('\n\n')}`;
1436
- const mergedPrompt = buildLinkSummaryPrompt({
1437
- url: extracted.url,
1438
- title: extracted.title,
1439
- siteName: extracted.siteName,
1440
- description: extracted.description,
1441
- content: mergedContent,
1442
- truncated: false,
1443
- hasTranscript: isYouTube ||
1444
- (extracted.transcriptSource !== null && extracted.transcriptSource !== 'unavailable'),
1445
- summaryLength: lengthArg.kind === 'preset'
1446
- ? lengthArg.preset
1447
- : { maxCharacters: lengthArg.maxCharacters },
1448
- shares: [],
1449
- });
1450
- if (streamingEnabledForCall) {
1451
- writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
1452
- let streamResult = null;
1453
- try {
1454
- streamResult = await streamTextWithModelId({
1590
+ else if (parsedModelEffective.provider === 'google' &&
1591
+ isGoogleStreamingUnsupportedError(error)) {
1592
+ writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
1593
+ const result = await summarizeWithModelId({
1455
1594
  modelId: parsedModelEffective.canonical,
1456
- apiKeys: apiKeysForLlm,
1457
- prompt: mergedPrompt,
1458
- temperature: 0,
1595
+ prompt,
1459
1596
  maxOutputTokens: maxOutputTokensForCall ?? undefined,
1460
1597
  timeoutMs,
1461
1598
  fetchImpl: trackedFetch,
1599
+ apiKeys: apiKeysForLlm,
1462
1600
  });
1601
+ llmCalls.push({
1602
+ provider: result.provider,
1603
+ model: result.canonicalModelId,
1604
+ usage: result.usage,
1605
+ purpose: 'summary',
1606
+ });
1607
+ summary = result.text;
1608
+ streamResult = null;
1463
1609
  }
1464
- catch (error) {
1465
- if (parsedModelEffective.provider === 'google' &&
1466
- isGoogleStreamingUnsupportedError(error)) {
1467
- writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
1468
- const mergedResult = await summarizeWithModelId({
1469
- modelId: parsedModelEffective.canonical,
1470
- prompt: mergedPrompt,
1471
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1472
- timeoutMs,
1473
- fetchImpl: trackedFetch,
1474
- apiKeys: apiKeysForLlm,
1475
- });
1476
- llmCalls.push({
1477
- provider: mergedResult.provider,
1478
- model: mergedResult.canonicalModelId,
1479
- usage: mergedResult.usage,
1480
- purpose: 'summary',
1481
- });
1482
- summary = mergedResult.text;
1483
- streamResult = null;
1484
- }
1485
- else {
1486
- throw error;
1487
- }
1610
+ else {
1611
+ throw error;
1488
1612
  }
1489
- if (streamResult) {
1490
- getLastStreamError = streamResult.lastError;
1491
- let streamed = '';
1492
- const liveRenderer = shouldLiveRenderSummary
1493
- ? createLiveRenderer({
1494
- write: (chunk) => {
1495
- clearProgressForStdout();
1496
- stdout.write(chunk);
1497
- },
1613
+ }
1614
+ if (streamResult) {
1615
+ getLastStreamError = streamResult.lastError;
1616
+ let streamed = '';
1617
+ const liveRenderer = shouldLiveRenderSummary
1618
+ ? createLiveRenderer({
1619
+ write: (chunk) => {
1620
+ clearProgressForStdout();
1621
+ stdout.write(chunk);
1622
+ },
1623
+ width: markdownRenderWidth(stdout, env),
1624
+ renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
1498
1625
  width: markdownRenderWidth(stdout, env),
1499
- renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
1500
- width: markdownRenderWidth(stdout, env),
1501
- wrap: true,
1502
- color: supportsColor(stdout, env),
1503
- }),
1504
- })
1505
- : null;
1506
- let lastFrameAtMs = 0;
1507
- try {
1508
- let cleared = false;
1509
- for await (const delta of streamResult.textStream) {
1626
+ wrap: true,
1627
+ color: supportsColor(stdout, env),
1628
+ }),
1629
+ })
1630
+ : null;
1631
+ let lastFrameAtMs = 0;
1632
+ try {
1633
+ let cleared = false;
1634
+ for await (const delta of streamResult.textStream) {
1635
+ const merged = mergeStreamingChunk(streamed, delta);
1636
+ streamed = merged.next;
1637
+ if (shouldStreamSummaryToStdout) {
1510
1638
  if (!cleared) {
1511
1639
  clearProgressForStdout();
1512
1640
  cleared = true;
1513
1641
  }
1514
- streamed += delta;
1515
- if (shouldStreamSummaryToStdout) {
1516
- stdout.write(delta);
1517
- continue;
1518
- }
1519
- if (liveRenderer) {
1520
- const now = Date.now();
1521
- const due = now - lastFrameAtMs >= 120;
1522
- const hasNewline = delta.includes('\n');
1523
- if (hasNewline || due) {
1524
- liveRenderer.render(streamed);
1525
- lastFrameAtMs = now;
1526
- }
1527
- }
1642
+ if (merged.appended)
1643
+ stdout.write(merged.appended);
1644
+ continue;
1528
1645
  }
1529
- const trimmed = streamed.trim();
1530
- streamed = trimmed;
1531
1646
  if (liveRenderer) {
1532
- liveRenderer.render(trimmed);
1533
- summaryAlreadyPrinted = true;
1647
+ const now = Date.now();
1648
+ const due = now - lastFrameAtMs >= 120;
1649
+ const hasNewline = delta.includes('\n');
1650
+ if (hasNewline || due) {
1651
+ liveRenderer.render(streamed);
1652
+ lastFrameAtMs = now;
1653
+ }
1534
1654
  }
1535
1655
  }
1536
- finally {
1537
- liveRenderer?.finish();
1538
- }
1539
- const usage = await streamResult.usage;
1540
- llmCalls.push({
1541
- provider: streamResult.provider,
1542
- model: streamResult.canonicalModelId,
1543
- usage,
1544
- purpose: 'summary',
1545
- });
1546
- summary = streamed;
1547
- if (shouldStreamSummaryToStdout) {
1548
- if (!streamed.endsWith('\n')) {
1549
- stdout.write('\n');
1550
- }
1656
+ const trimmed = streamed.trim();
1657
+ streamed = trimmed;
1658
+ if (liveRenderer) {
1659
+ liveRenderer.render(trimmed);
1551
1660
  summaryAlreadyPrinted = true;
1552
1661
  }
1553
1662
  }
1554
- }
1555
- else {
1556
- const mergedResult = await summarizeWithModelId({
1557
- modelId: parsedModelEffective.canonical,
1558
- prompt: mergedPrompt,
1559
- maxOutputTokens: maxOutputTokensForCall ?? undefined,
1560
- timeoutMs,
1561
- fetchImpl: trackedFetch,
1562
- apiKeys: apiKeysForLlm,
1563
- });
1663
+ finally {
1664
+ liveRenderer?.finish();
1665
+ }
1666
+ const usage = await streamResult.usage;
1564
1667
  llmCalls.push({
1565
- provider: mergedResult.provider,
1566
- model: mergedResult.canonicalModelId,
1567
- usage: mergedResult.usage,
1668
+ provider: streamResult.provider,
1669
+ model: streamResult.canonicalModelId,
1670
+ usage,
1568
1671
  purpose: 'summary',
1569
1672
  });
1570
- summary = mergedResult.text;
1673
+ summary = streamed;
1674
+ if (shouldStreamSummaryToStdout) {
1675
+ if (!streamed.endsWith('\n')) {
1676
+ stdout.write('\n');
1677
+ }
1678
+ summaryAlreadyPrinted = true;
1679
+ }
1571
1680
  }
1572
1681
  }
1682
+ else {
1683
+ const result = await summarizeWithModelId({
1684
+ modelId: parsedModelEffective.canonical,
1685
+ prompt,
1686
+ maxOutputTokens: maxOutputTokensForCall ?? undefined,
1687
+ timeoutMs,
1688
+ fetchImpl: trackedFetch,
1689
+ apiKeys: apiKeysForLlm,
1690
+ });
1691
+ llmCalls.push({
1692
+ provider: result.provider,
1693
+ model: result.canonicalModelId,
1694
+ usage: result.usage,
1695
+ purpose: 'summary',
1696
+ });
1697
+ summary = result.text;
1698
+ }
1573
1699
  summary = summary.trim();
1574
1700
  if (summary.length === 0) {
1575
1701
  const last = getLastStreamError?.();
@@ -1608,8 +1734,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1608
1734
  provider: parsedModelEffective.provider,
1609
1735
  model: parsedModelEffective.canonical,
1610
1736
  maxCompletionTokens: maxOutputTokensForCall,
1611
- strategy,
1612
- chunkCount,
1737
+ strategy: 'single',
1613
1738
  },
1614
1739
  metrics: metricsEnabled ? finishReport : null,
1615
1740
  summary,
@@ -1624,8 +1749,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1624
1749
  stderr,
1625
1750
  elapsedMs: Date.now() - runStartedAtMs,
1626
1751
  model: parsedModelEffective.canonical,
1627
- strategy,
1628
- chunkCount,
1629
1752
  report: finishReport,
1630
1753
  costUsd,
1631
1754
  color: verboseColor,
@@ -1656,8 +1779,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
1656
1779
  stderr,
1657
1780
  elapsedMs: Date.now() - runStartedAtMs,
1658
1781
  model: parsedModelEffective.canonical,
1659
- strategy,
1660
- chunkCount,
1661
1782
  report,
1662
1783
  costUsd,
1663
1784
  color: verboseColor,