@steipete/summarize 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/CHANGELOG.md +75 -0
  2. package/LICENSE +1 -1
  3. package/README.md +307 -184
  4. package/dist/cli.js +1 -1
  5. package/dist/esm/cache.js +72 -4
  6. package/dist/esm/cache.js.map +1 -1
  7. package/dist/esm/config.js +127 -1
  8. package/dist/esm/config.js.map +1 -1
  9. package/dist/esm/daemon/agent.js +547 -0
  10. package/dist/esm/daemon/agent.js.map +1 -0
  11. package/dist/esm/daemon/chat.js +83 -175
  12. package/dist/esm/daemon/chat.js.map +1 -1
  13. package/dist/esm/daemon/cli.js +35 -3
  14. package/dist/esm/daemon/cli.js.map +1 -1
  15. package/dist/esm/daemon/env-snapshot.js +3 -0
  16. package/dist/esm/daemon/env-snapshot.js.map +1 -1
  17. package/dist/esm/daemon/flow-context.js +31 -8
  18. package/dist/esm/daemon/flow-context.js.map +1 -1
  19. package/dist/esm/daemon/launchd.js +27 -0
  20. package/dist/esm/daemon/launchd.js.map +1 -1
  21. package/dist/esm/daemon/process-registry.js +206 -0
  22. package/dist/esm/daemon/process-registry.js.map +1 -0
  23. package/dist/esm/daemon/schtasks.js +64 -0
  24. package/dist/esm/daemon/schtasks.js.map +1 -1
  25. package/dist/esm/daemon/server.js +712 -42
  26. package/dist/esm/daemon/server.js.map +1 -1
  27. package/dist/esm/daemon/summarize.js +33 -4
  28. package/dist/esm/daemon/summarize.js.map +1 -1
  29. package/dist/esm/daemon/systemd.js +61 -0
  30. package/dist/esm/daemon/systemd.js.map +1 -1
  31. package/dist/esm/flags.js +24 -0
  32. package/dist/esm/flags.js.map +1 -1
  33. package/dist/esm/llm/providers/openai.js +1 -1
  34. package/dist/esm/llm/providers/openai.js.map +1 -1
  35. package/dist/esm/media-cache.js +251 -0
  36. package/dist/esm/media-cache.js.map +1 -0
  37. package/dist/esm/processes.js +2 -0
  38. package/dist/esm/processes.js.map +1 -0
  39. package/dist/esm/run/bird.js +118 -5
  40. package/dist/esm/run/bird.js.map +1 -1
  41. package/dist/esm/run/cli-preflight.js +19 -1
  42. package/dist/esm/run/cli-preflight.js.map +1 -1
  43. package/dist/esm/run/finish-line.js +11 -4
  44. package/dist/esm/run/finish-line.js.map +1 -1
  45. package/dist/esm/run/flows/asset/extract.js +70 -0
  46. package/dist/esm/run/flows/asset/extract.js.map +1 -0
  47. package/dist/esm/run/flows/asset/input.js +208 -24
  48. package/dist/esm/run/flows/asset/input.js.map +1 -1
  49. package/dist/esm/run/flows/asset/media-policy.js +3 -0
  50. package/dist/esm/run/flows/asset/media-policy.js.map +1 -0
  51. package/dist/esm/run/flows/asset/media.js +224 -0
  52. package/dist/esm/run/flows/asset/media.js.map +1 -0
  53. package/dist/esm/run/flows/asset/output.js +98 -0
  54. package/dist/esm/run/flows/asset/output.js.map +1 -0
  55. package/dist/esm/run/flows/asset/summary.js +157 -7
  56. package/dist/esm/run/flows/asset/summary.js.map +1 -1
  57. package/dist/esm/run/flows/url/extract.js +6 -6
  58. package/dist/esm/run/flows/url/extract.js.map +1 -1
  59. package/dist/esm/run/flows/url/flow.js +336 -36
  60. package/dist/esm/run/flows/url/flow.js.map +1 -1
  61. package/dist/esm/run/flows/url/markdown.js +6 -1
  62. package/dist/esm/run/flows/url/markdown.js.map +1 -1
  63. package/dist/esm/run/flows/url/slides-output.js +485 -0
  64. package/dist/esm/run/flows/url/slides-output.js.map +1 -0
  65. package/dist/esm/run/flows/url/slides-text.js +628 -0
  66. package/dist/esm/run/flows/url/slides-text.js.map +1 -0
  67. package/dist/esm/run/flows/url/summary.js +356 -82
  68. package/dist/esm/run/flows/url/summary.js.map +1 -1
  69. package/dist/esm/run/help.js +94 -5
  70. package/dist/esm/run/help.js.map +1 -1
  71. package/dist/esm/run/logging.js +12 -4
  72. package/dist/esm/run/logging.js.map +1 -1
  73. package/dist/esm/run/media-cache-state.js +33 -0
  74. package/dist/esm/run/media-cache-state.js.map +1 -0
  75. package/dist/esm/run/progress.js +19 -1
  76. package/dist/esm/run/progress.js.map +1 -1
  77. package/dist/esm/run/run-settings.js +39 -1
  78. package/dist/esm/run/run-settings.js.map +1 -1
  79. package/dist/esm/run/runner.js +196 -8
  80. package/dist/esm/run/runner.js.map +1 -1
  81. package/dist/esm/run/slides-cli.js +225 -0
  82. package/dist/esm/run/slides-cli.js.map +1 -0
  83. package/dist/esm/run/slides-render.js +163 -0
  84. package/dist/esm/run/slides-render.js.map +1 -0
  85. package/dist/esm/run/stream-output.js +10 -3
  86. package/dist/esm/run/stream-output.js.map +1 -1
  87. package/dist/esm/run/summary-engine.js +33 -7
  88. package/dist/esm/run/summary-engine.js.map +1 -1
  89. package/dist/esm/run/transcriber-cli.js +148 -0
  90. package/dist/esm/run/transcriber-cli.js.map +1 -0
  91. package/dist/esm/shared/sse-events.js +4 -0
  92. package/dist/esm/shared/sse-events.js.map +1 -1
  93. package/dist/esm/slides/extract.js +1942 -0
  94. package/dist/esm/slides/extract.js.map +1 -0
  95. package/dist/esm/slides/index.js +4 -0
  96. package/dist/esm/slides/index.js.map +1 -0
  97. package/dist/esm/slides/settings.js +73 -0
  98. package/dist/esm/slides/settings.js.map +1 -0
  99. package/dist/esm/slides/store.js +111 -0
  100. package/dist/esm/slides/store.js.map +1 -0
  101. package/dist/esm/slides/types.js +2 -0
  102. package/dist/esm/slides/types.js.map +1 -0
  103. package/dist/esm/tty/osc-progress.js +21 -1
  104. package/dist/esm/tty/osc-progress.js.map +1 -1
  105. package/dist/esm/tty/progress/fetch-html.js +8 -4
  106. package/dist/esm/tty/progress/fetch-html.js.map +1 -1
  107. package/dist/esm/tty/progress/transcript.js +82 -31
  108. package/dist/esm/tty/progress/transcript.js.map +1 -1
  109. package/dist/esm/tty/spinner.js +2 -2
  110. package/dist/esm/tty/spinner.js.map +1 -1
  111. package/dist/esm/tty/theme.js +189 -0
  112. package/dist/esm/tty/theme.js.map +1 -0
  113. package/dist/esm/tty/website-progress.js +17 -13
  114. package/dist/esm/tty/website-progress.js.map +1 -1
  115. package/dist/esm/version.js +1 -1
  116. package/dist/esm/version.js.map +1 -1
  117. package/dist/types/cache.d.ts +14 -2
  118. package/dist/types/config.d.ts +23 -0
  119. package/dist/types/daemon/agent.d.ts +25 -0
  120. package/dist/types/daemon/chat.d.ts +10 -18
  121. package/dist/types/daemon/env-snapshot.d.ts +1 -1
  122. package/dist/types/daemon/flow-context.d.ts +21 -1
  123. package/dist/types/daemon/launchd.d.ts +4 -0
  124. package/dist/types/daemon/process-registry.d.ts +73 -0
  125. package/dist/types/daemon/schtasks.d.ts +4 -0
  126. package/dist/types/daemon/summarize.d.ts +36 -5
  127. package/dist/types/daemon/systemd.d.ts +4 -0
  128. package/dist/types/flags.d.ts +1 -0
  129. package/dist/types/media-cache.d.ts +22 -0
  130. package/dist/types/processes.d.ts +1 -0
  131. package/dist/types/run/bird.d.ts +7 -0
  132. package/dist/types/run/finish-line.d.ts +2 -1
  133. package/dist/types/run/flows/asset/extract.d.ts +18 -0
  134. package/dist/types/run/flows/asset/input.d.ts +12 -2
  135. package/dist/types/run/flows/asset/media-policy.d.ts +2 -0
  136. package/dist/types/run/flows/asset/media.d.ts +21 -0
  137. package/dist/types/run/flows/asset/output.d.ts +42 -0
  138. package/dist/types/run/flows/asset/summary.d.ts +6 -0
  139. package/dist/types/run/flows/url/extract.d.ts +2 -1
  140. package/dist/types/run/flows/url/slides-output.d.ts +66 -0
  141. package/dist/types/run/flows/url/slides-text.d.ts +87 -0
  142. package/dist/types/run/flows/url/summary.d.ts +11 -3
  143. package/dist/types/run/flows/url/types.d.ts +29 -2
  144. package/dist/types/run/help.d.ts +3 -0
  145. package/dist/types/run/logging.d.ts +3 -2
  146. package/dist/types/run/media-cache-state.d.ts +7 -0
  147. package/dist/types/run/progress.d.ts +2 -1
  148. package/dist/types/run/run-settings.d.ts +7 -1
  149. package/dist/types/run/slides-cli.d.ts +9 -0
  150. package/dist/types/run/slides-render.d.ts +30 -0
  151. package/dist/types/run/stream-output.d.ts +2 -1
  152. package/dist/types/run/summary-engine.d.ts +11 -1
  153. package/dist/types/run/transcriber-cli.d.ts +8 -0
  154. package/dist/types/shared/sse-events.d.ts +20 -0
  155. package/dist/types/slides/extract.d.ts +42 -0
  156. package/dist/types/slides/index.d.ts +5 -0
  157. package/dist/types/slides/settings.d.ts +20 -0
  158. package/dist/types/slides/store.d.ts +15 -0
  159. package/dist/types/slides/types.d.ts +40 -0
  160. package/dist/types/tty/osc-progress.d.ts +2 -2
  161. package/dist/types/tty/progress/fetch-html.d.ts +3 -1
  162. package/dist/types/tty/progress/transcript.d.ts +3 -1
  163. package/dist/types/tty/spinner.d.ts +3 -1
  164. package/dist/types/tty/theme.d.ts +44 -0
  165. package/dist/types/tty/website-progress.d.ts +3 -1
  166. package/dist/types/version.d.ts +1 -1
  167. package/docs/README.md +1 -1
  168. package/docs/_config.yml +26 -0
  169. package/docs/_layouts/default.html +60 -0
  170. package/docs/agent.md +333 -0
  171. package/docs/assets/site.css +748 -0
  172. package/docs/assets/site.js +72 -0
  173. package/docs/assets/summarize-cli.png +0 -0
  174. package/docs/assets/summarize-extension.png +0 -0
  175. package/docs/assets/youtube-slides.png +0 -0
  176. package/docs/cache.md +29 -3
  177. package/docs/chrome-extension.md +61 -11
  178. package/docs/config.md +50 -2
  179. package/docs/extract-only.md +8 -0
  180. package/docs/index.html +205 -0
  181. package/docs/index.md +25 -0
  182. package/docs/llm.md +11 -1
  183. package/docs/manual-tests.md +2 -0
  184. package/docs/media.md +3 -0
  185. package/docs/nvidia-onnx-transcription.md +55 -0
  186. package/docs/site/assets/site.css +399 -228
  187. package/docs/site/assets/summarize-cli.png +0 -0
  188. package/docs/site/assets/summarize-extension.png +0 -0
  189. package/docs/site/docs/chrome-extension.html +89 -0
  190. package/docs/site/docs/config.html +1 -0
  191. package/docs/site/docs/extract-only.html +1 -0
  192. package/docs/site/docs/firecrawl.html +1 -0
  193. package/docs/site/docs/index.html +5 -0
  194. package/docs/site/docs/llm.html +1 -0
  195. package/docs/site/docs/openai.html +1 -0
  196. package/docs/site/docs/website.html +1 -0
  197. package/docs/site/docs/youtube.html +1 -0
  198. package/docs/site/index.html +148 -84
  199. package/docs/slides.md +74 -0
  200. package/docs/timestamps.md +103 -0
  201. package/docs/website.md +12 -0
  202. package/docs/youtube.md +16 -0
  203. package/package.json +16 -15
@@ -1,19 +1,137 @@
1
+ import { isTwitterStatusUrl, isYouTubeUrl } from '@steipete/summarize-core/content/url';
1
2
  import { countTokens } from 'gpt-tokenizer';
2
3
  import { render as renderMarkdownAnsi } from 'markdansi';
3
4
  import { buildLanguageKey, buildLengthKey, buildPromptHash, buildSummaryCacheKey, hashString, normalizeContentForHash, } from '../../../cache.js';
4
5
  import { formatOutputLanguageForJson } from '../../../language.js';
5
6
  import { parseGatewayStyleModelId } from '../../../llm/model-id.js';
6
7
  import { buildAutoModelAttempts } from '../../../model-auto.js';
7
- import { buildLinkSummaryPrompt } from '../../../prompts/index.js';
8
+ import { buildLinkSummaryPrompt, SUMMARY_LENGTH_TARGET_CHARACTERS, SUMMARY_SYSTEM_PROMPT, } from '../../../prompts/index.js';
8
9
  import { parseCliUserModelId } from '../../env.js';
9
10
  import { buildExtractFinishLabel, buildLengthPartsForFinishLine, writeFinishLine, } from '../../finish-line.js';
11
+ import { resolveTargetCharacters } from '../../format.js';
10
12
  import { writeVerbose } from '../../logging.js';
11
13
  import { prepareMarkdownForTerminal } from '../../markdown.js';
12
14
  import { runModelAttempts } from '../../model-attempts.js';
13
15
  import { buildOpenRouterNoAllowedProvidersMessage } from '../../openrouter.js';
14
16
  import { isRichTty, markdownRenderWidth, supportsColor } from '../../terminal.js';
15
- export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOverride, lengthInstruction, languageInstruction, }) {
17
+ import { coerceSummaryWithSlides, interleaveSlidesIntoTranscript, normalizeSummarySlideHeadings, } from './slides-text.js';
18
+ const MAX_SLIDE_TRANSCRIPT_CHARS_BY_PRESET = {
19
+ short: 2500,
20
+ medium: 5000,
21
+ long: 9000,
22
+ xl: 15000,
23
+ xxl: 24000,
24
+ };
25
+ const SLIDE_TRANSCRIPT_DEFAULT_EDGE_SECONDS = 30;
26
+ const SLIDE_TRANSCRIPT_LEEWAY_SECONDS = 10;
27
+ function parseTimestampSeconds(value) {
28
+ const parts = value.split(':').map((item) => Number(item));
29
+ if (parts.some((item) => !Number.isFinite(item)))
30
+ return null;
31
+ if (parts.length === 2) {
32
+ const [minutes, seconds] = parts;
33
+ return minutes * 60 + seconds;
34
+ }
35
+ if (parts.length === 3) {
36
+ const [hours, minutes, seconds] = parts;
37
+ return hours * 3600 + minutes * 60 + seconds;
38
+ }
39
+ return null;
40
+ }
41
+ function parseTranscriptTimedText(input) {
42
+ if (!input)
43
+ return [];
44
+ const segments = [];
45
+ for (const line of input.split('\n')) {
46
+ const trimmed = line.trim();
47
+ if (!trimmed.startsWith('['))
48
+ continue;
49
+ const match = trimmed.match(/^\[(\d{1,2}:\d{2}(?::\d{2})?)\]\s*(.*)$/);
50
+ if (!match)
51
+ continue;
52
+ const seconds = parseTimestampSeconds(match[1]);
53
+ if (seconds == null)
54
+ continue;
55
+ const text = (match[2] ?? '').trim();
56
+ if (!text)
57
+ continue;
58
+ segments.push({ startSeconds: seconds, text });
59
+ }
60
+ segments.sort((a, b) => a.startSeconds - b.startSeconds);
61
+ return segments;
62
+ }
63
+ function formatTimestamp(seconds) {
64
+ const clamped = Math.max(0, Math.floor(seconds));
65
+ const hours = Math.floor(clamped / 3600);
66
+ const minutes = Math.floor((clamped % 3600) / 60);
67
+ const secs = clamped % 60;
68
+ const mm = String(minutes).padStart(2, '0');
69
+ const ss = String(secs).padStart(2, '0');
70
+ if (hours <= 0)
71
+ return `${minutes}:${ss}`;
72
+ const hh = String(hours).padStart(2, '0');
73
+ return `${hh}:${mm}:${ss}`;
74
+ }
75
+ function truncateTranscript(value, limit) {
76
+ if (value.length <= limit)
77
+ return value;
78
+ const truncated = value.slice(0, limit).trimEnd();
79
+ const clean = truncated.replace(/\s+\S*$/, '').trim();
80
+ const result = clean.length > 0 ? clean : truncated.trim();
81
+ return result.length > 0 ? `${result}…` : '';
82
+ }
83
+ function buildSlidesPromptText({ slides, transcriptTimedText, preset, }) {
84
+ if (!slides || slides.slides.length === 0)
85
+ return null;
86
+ const segments = parseTranscriptTimedText(transcriptTimedText);
87
+ const slidesWithTimestamps = slides.slides
88
+ .filter((slide) => Number.isFinite(slide.timestamp))
89
+ .map((slide) => ({ index: slide.index, timestamp: Math.max(0, Math.floor(slide.timestamp)) }))
90
+ .sort((a, b) => a.timestamp - b.timestamp);
91
+ if (slidesWithTimestamps.length === 0)
92
+ return null;
93
+ const totalBudget = Number(MAX_SLIDE_TRANSCRIPT_CHARS_BY_PRESET[preset]);
94
+ const perSlideBudget = Math.max(120, Math.floor(totalBudget / Math.max(1, slidesWithTimestamps.length)));
95
+ let remaining = totalBudget;
96
+ const blocks = [];
97
+ for (let i = 0; i < slidesWithTimestamps.length; i += 1) {
98
+ const slide = slidesWithTimestamps[i];
99
+ if (!slide)
100
+ continue;
101
+ const prev = slidesWithTimestamps[i - 1];
102
+ const next = slidesWithTimestamps[i + 1];
103
+ const startBase = prev ? Math.floor((prev.timestamp + slide.timestamp) / 2) : slide.timestamp;
104
+ const endBase = next ? Math.ceil((slide.timestamp + next.timestamp) / 2) : slide.timestamp;
105
+ const start = Math.max(0, (prev ? startBase : slide.timestamp - SLIDE_TRANSCRIPT_DEFAULT_EDGE_SECONDS) -
106
+ SLIDE_TRANSCRIPT_LEEWAY_SECONDS);
107
+ const end = (next ? endBase : slide.timestamp + SLIDE_TRANSCRIPT_DEFAULT_EDGE_SECONDS) +
108
+ SLIDE_TRANSCRIPT_LEEWAY_SECONDS;
109
+ const excerptParts = [];
110
+ for (const segment of segments) {
111
+ if (segment.startSeconds < start)
112
+ continue;
113
+ if (segment.startSeconds > end)
114
+ break;
115
+ excerptParts.push(segment.text);
116
+ }
117
+ const excerptRaw = excerptParts.join(' ').trim().replace(/\s+/g, ' ');
118
+ const excerptBudget = remaining > 0 ? Math.min(perSlideBudget, remaining) : 0;
119
+ const excerpt = excerptRaw && excerptBudget > 0 ? truncateTranscript(excerptRaw, excerptBudget) : '';
120
+ const label = `[slide:${slide.index}] [${formatTimestamp(start)}–${formatTimestamp(end)}]`;
121
+ const block = excerpt ? `${label}\n${excerpt}` : label;
122
+ blocks.push(block);
123
+ remaining = Math.max(0, remaining - block.length);
124
+ }
125
+ return blocks.length > 0 ? blocks.join('\n\n') : null;
126
+ }
127
+ export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOverride, lengthInstruction, languageInstruction, slides, }) {
16
128
  const isYouTube = extracted.siteName === 'YouTube';
129
+ const preset = lengthArg.kind === 'preset' ? lengthArg.preset : 'medium';
130
+ const slidesText = buildSlidesPromptText({
131
+ slides,
132
+ transcriptTimedText: extracted.transcriptTimedText,
133
+ preset,
134
+ });
17
135
  return buildLinkSummaryPrompt({
18
136
  url: extracted.url,
19
137
  title: extracted.title,
@@ -23,6 +141,8 @@ export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOve
23
141
  truncated: extracted.truncated,
24
142
  hasTranscript: isYouTube ||
25
143
  (extracted.transcriptSource !== null && extracted.transcriptSource !== 'unavailable'),
144
+ hasTranscriptTimestamps: Boolean(extracted.transcriptTimedText),
145
+ slides: slidesText ? { count: slides?.slides.length ?? 0, text: slidesText } : null,
26
146
  summaryLength: lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters },
27
147
  outputLanguage,
28
148
  shares: [],
@@ -31,6 +151,97 @@ export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOve
31
151
  languageInstruction: languageInstruction ?? null,
32
152
  });
33
153
  }
154
+ function shouldBypassShortContentSummary({ extracted, lengthArg, forceSummary, maxOutputTokensArg, json, }) {
155
+ if (forceSummary)
156
+ return false;
157
+ if (!extracted.content || extracted.content.length === 0)
158
+ return false;
159
+ const targetCharacters = resolveTargetCharacters(lengthArg, SUMMARY_LENGTH_TARGET_CHARACTERS);
160
+ if (!Number.isFinite(targetCharacters) || targetCharacters <= 0)
161
+ return false;
162
+ if (extracted.content.length > targetCharacters)
163
+ return false;
164
+ if (!json && typeof maxOutputTokensArg === 'number') {
165
+ const tokenCount = countTokens(extracted.content);
166
+ if (tokenCount > maxOutputTokensArg)
167
+ return false;
168
+ }
169
+ return true;
170
+ }
171
+ async function outputSummaryFromExtractedContent({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, slides, footerLabel, verboseMessage, }) {
172
+ const { io, flags, model, hooks } = ctx;
173
+ hooks.clearProgressForStdout();
174
+ const finishModel = pickModelForFinishLine(model.llmCalls, null);
175
+ if (flags.json) {
176
+ const finishReport = flags.shouldComputeReport ? await hooks.buildReport() : null;
177
+ const payload = {
178
+ input: {
179
+ kind: 'url',
180
+ url,
181
+ timeoutMs: flags.timeoutMs,
182
+ youtube: flags.youtubeMode,
183
+ firecrawl: flags.firecrawlMode,
184
+ format: flags.format,
185
+ markdown: effectiveMarkdownMode,
186
+ timestamps: flags.transcriptTimestamps,
187
+ length: flags.lengthArg.kind === 'preset'
188
+ ? { kind: 'preset', preset: flags.lengthArg.preset }
189
+ : { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
190
+ maxOutputTokens: flags.maxOutputTokensArg,
191
+ model: model.requestedModelLabel,
192
+ language: formatOutputLanguageForJson(flags.outputLanguage),
193
+ },
194
+ env: {
195
+ hasXaiKey: Boolean(model.apiStatus.xaiApiKey),
196
+ hasOpenAIKey: Boolean(model.apiStatus.apiKey),
197
+ hasOpenRouterKey: Boolean(model.apiStatus.openrouterApiKey),
198
+ hasApifyToken: Boolean(model.apiStatus.apifyToken),
199
+ hasFirecrawlKey: model.apiStatus.firecrawlConfigured,
200
+ hasGoogleKey: model.apiStatus.googleConfigured,
201
+ hasAnthropicKey: model.apiStatus.anthropicConfigured,
202
+ },
203
+ extracted,
204
+ slides,
205
+ prompt,
206
+ llm: null,
207
+ metrics: flags.metricsEnabled ? finishReport : null,
208
+ summary: extracted.content,
209
+ };
210
+ io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
211
+ if (flags.metricsEnabled && finishReport) {
212
+ const costUsd = await hooks.estimateCostUsd();
213
+ hooks.clearProgressForStdout();
214
+ writeFinishLine({
215
+ stderr: io.stderr,
216
+ env: io.envForRun,
217
+ elapsedMs: Date.now() - flags.runStartedAtMs,
218
+ label: extractionUi.finishSourceLabel,
219
+ model: finishModel,
220
+ report: finishReport,
221
+ costUsd,
222
+ detailed: flags.metricsDetailed,
223
+ extraParts: buildFinishExtras({
224
+ extracted,
225
+ metricsDetailed: flags.metricsDetailed,
226
+ transcriptionCostLabel,
227
+ }),
228
+ color: flags.verboseColor,
229
+ });
230
+ }
231
+ return;
232
+ }
233
+ io.stdout.write(`${extracted.content}\n`);
234
+ hooks.restoreProgressAfterStdout?.();
235
+ if (extractionUi.footerParts.length > 0) {
236
+ const footer = footerLabel
237
+ ? [...extractionUi.footerParts, footerLabel]
238
+ : extractionUi.footerParts;
239
+ hooks.writeViaFooter(footer);
240
+ }
241
+ if (verboseMessage && flags.verbose) {
242
+ writeVerbose(io.stderr, flags.verbose, verboseMessage, flags.verboseColor, io.envForRun);
243
+ }
244
+ }
34
245
  const buildFinishExtras = ({ extracted, metricsDetailed, transcriptionCostLabel, }) => {
35
246
  const parts = [
36
247
  ...(buildLengthPartsForFinishLine(extracted, metricsDetailed) ?? []),
@@ -62,7 +273,7 @@ const buildModelMetaFromAttempt = (attempt) => {
62
273
  : parsed.canonical;
63
274
  return { provider: parsed.provider, canonical };
64
275
  };
65
- export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, }) {
276
+ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, slides, slidesOutput, }) {
66
277
  const { io, flags, model, hooks } = ctx;
67
278
  hooks.clearProgressForStdout();
68
279
  const finishLabel = buildExtractFinishLabel({
@@ -83,6 +294,7 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
83
294
  firecrawl: flags.firecrawlMode,
84
295
  format: flags.format,
85
296
  markdown: effectiveMarkdownMode,
297
+ timestamps: flags.transcriptTimestamps,
86
298
  length: flags.lengthArg.kind === 'preset'
87
299
  ? { kind: 'preset', preset: flags.lengthArg.preset }
88
300
  : { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
@@ -100,16 +312,20 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
100
312
  hasAnthropicKey: model.apiStatus.anthropicConfigured,
101
313
  },
102
314
  extracted,
315
+ slides,
103
316
  prompt,
104
317
  llm: null,
105
318
  metrics: flags.metricsEnabled ? finishReport : null,
106
319
  summary: null,
107
320
  };
108
321
  io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
322
+ hooks.restoreProgressAfterStdout?.();
323
+ hooks.restoreProgressAfterStdout?.();
109
324
  if (flags.metricsEnabled && finishReport) {
110
325
  const costUsd = await hooks.estimateCostUsd();
111
326
  writeFinishLine({
112
327
  stderr: io.stderr,
328
+ env: io.envForRun,
113
329
  elapsedMs: Date.now() - flags.runStartedAtMs,
114
330
  label: finishLabel,
115
331
  model: finishModel,
@@ -126,14 +342,62 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
126
342
  }
127
343
  return;
128
344
  }
345
+ const extractCandidate = flags.transcriptTimestamps &&
346
+ extracted.transcriptTimedText &&
347
+ extracted.transcriptSource &&
348
+ extracted.content.toLowerCase().startsWith('transcript:')
349
+ ? `Transcript:\n${extracted.transcriptTimedText}`
350
+ : extracted.content;
351
+ const slideTags = slides?.slides && slides.slides.length > 0
352
+ ? slides.slides.map((slide) => `[slide:${slide.index}]`).join('\n')
353
+ : '';
354
+ if (slidesOutput && slides?.slides && slides.slides.length > 0) {
355
+ const transcriptText = extracted.transcriptTimedText
356
+ ? `Transcript:\n${extracted.transcriptTimedText}`
357
+ : null;
358
+ const interleaved = transcriptText
359
+ ? interleaveSlidesIntoTranscript({
360
+ transcriptTimedText: transcriptText,
361
+ slides: slides.slides.map((slide) => ({
362
+ index: slide.index,
363
+ timestamp: slide.timestamp,
364
+ })),
365
+ })
366
+ : `${extractCandidate.trimEnd()}\n\n${slideTags}`;
367
+ await slidesOutput.renderFromText(interleaved);
368
+ hooks.restoreProgressAfterStdout?.();
369
+ const slideFooter = slides ? [`slides ${slides.slides.length}`] : [];
370
+ hooks.writeViaFooter([...extractionUi.footerParts, ...slideFooter]);
371
+ const report = flags.shouldComputeReport ? await hooks.buildReport() : null;
372
+ if (flags.metricsEnabled && report) {
373
+ const costUsd = await hooks.estimateCostUsd();
374
+ writeFinishLine({
375
+ stderr: io.stderr,
376
+ env: io.envForRun,
377
+ elapsedMs: Date.now() - flags.runStartedAtMs,
378
+ label: finishLabel,
379
+ model: finishModel,
380
+ report,
381
+ costUsd,
382
+ detailed: flags.metricsDetailed,
383
+ extraParts: buildFinishExtras({
384
+ extracted,
385
+ metricsDetailed: flags.metricsDetailed,
386
+ transcriptionCostLabel,
387
+ }),
388
+ color: flags.verboseColor,
389
+ });
390
+ }
391
+ return;
392
+ }
129
393
  const renderedExtract = flags.format === 'markdown' && !flags.plain && isRichTty(io.stdout)
130
- ? renderMarkdownAnsi(prepareMarkdownForTerminal(extracted.content), {
394
+ ? renderMarkdownAnsi(prepareMarkdownForTerminal(extractCandidate), {
131
395
  width: markdownRenderWidth(io.stdout, io.env),
132
396
  wrap: true,
133
397
  color: supportsColor(io.stdout, io.envForRun),
134
398
  hyperlinks: true,
135
399
  })
136
- : extracted.content;
400
+ : extractCandidate;
137
401
  if (flags.format === 'markdown' && !flags.plain && isRichTty(io.stdout)) {
138
402
  io.stdout.write(`\n${renderedExtract.replace(/^\n+/, '')}`);
139
403
  }
@@ -143,12 +407,16 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
143
407
  if (!renderedExtract.endsWith('\n')) {
144
408
  io.stdout.write('\n');
145
409
  }
146
- hooks.writeViaFooter(extractionUi.footerParts);
410
+ hooks.restoreProgressAfterStdout?.();
411
+ const slideFooter = slides ? [`slides ${slides.slides.length}`] : [];
412
+ hooks.writeViaFooter([...extractionUi.footerParts, ...slideFooter]);
147
413
  const report = flags.shouldComputeReport ? await hooks.buildReport() : null;
148
414
  if (flags.metricsEnabled && report) {
149
415
  const costUsd = await hooks.estimateCostUsd();
416
+ hooks.clearProgressForStdout();
150
417
  writeFinishLine({
151
418
  stderr: io.stderr,
419
+ env: io.envForRun,
152
420
  elapsedMs: Date.now() - flags.runStartedAtMs,
153
421
  label: finishLabel,
154
422
  model: finishModel,
@@ -164,9 +432,9 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
164
432
  });
165
433
  }
166
434
  }
167
- export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, onModelChosen, }) {
435
+ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, onModelChosen, slides, slidesOutput, }) {
168
436
  const { io, flags, model, cache: cacheState, hooks } = ctx;
169
- const promptPayload = { userText: prompt };
437
+ const promptPayload = { system: SUMMARY_SYSTEM_PROMPT, userText: prompt };
170
438
  const promptTokens = countTokens(promptPayload.userText);
171
439
  const kindForAuto = extracted.siteName === 'YouTube' ? 'youtube' : 'website';
172
440
  const attempts = await (async () => {
@@ -185,7 +453,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
185
453
  });
186
454
  if (flags.verbose) {
187
455
  for (const attempt of list.slice(0, 8)) {
188
- writeVerbose(io.stderr, flags.verbose, `auto candidate ${attempt.debug}`, flags.verboseColor);
456
+ writeVerbose(io.stderr, flags.verbose, `auto candidate ${attempt.debug}`, flags.verboseColor, io.envForRun);
189
457
  }
190
458
  }
191
459
  return list.map((attempt) => {
@@ -232,7 +500,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
232
500
  },
233
501
  ];
234
502
  })();
235
- const cacheStore = cacheState.mode === 'default' ? cacheState.store : null;
503
+ const cacheStore = cacheState.mode === 'default' && !flags.summaryCacheBypass ? cacheState.store : null;
236
504
  const contentHash = cacheStore ? hashString(normalizeContentForHash(extracted.content)) : null;
237
505
  const promptHash = cacheStore ? buildPromptHash(prompt) : null;
238
506
  const lengthKey = buildLengthKey(flags.lengthArg);
@@ -241,6 +509,40 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
241
509
  let usedAttempt = null;
242
510
  let summaryFromCache = false;
243
511
  let cacheChecked = false;
512
+ const isTweet = extracted.siteName?.toLowerCase() === 'x' || isTwitterStatusUrl(extracted.url);
513
+ const isYouTube = extracted.siteName === 'YouTube' || isYouTubeUrl(url);
514
+ const hasMedia = Boolean(extracted.video) ||
515
+ (extracted.transcriptSource != null && extracted.transcriptSource !== 'unavailable') ||
516
+ (typeof extracted.mediaDurationSeconds === 'number' && extracted.mediaDurationSeconds > 0) ||
517
+ extracted.isVideoOnly === true;
518
+ const autoBypass = ctx.model.isFallbackModel && !ctx.model.isNamedModelSelection;
519
+ const canBypassShortContent = (autoBypass || isTweet) &&
520
+ !flags.slides &&
521
+ !hasMedia &&
522
+ flags.streamMode !== 'on' &&
523
+ !isYouTube &&
524
+ shouldBypassShortContentSummary({
525
+ extracted,
526
+ lengthArg: flags.lengthArg,
527
+ forceSummary: flags.forceSummary,
528
+ maxOutputTokensArg: flags.maxOutputTokensArg,
529
+ json: flags.json,
530
+ });
531
+ if (canBypassShortContent) {
532
+ await outputSummaryFromExtractedContent({
533
+ ctx,
534
+ url,
535
+ extracted,
536
+ extractionUi,
537
+ prompt,
538
+ effectiveMarkdownMode,
539
+ transcriptionCostLabel,
540
+ slides,
541
+ footerLabel: 'short content',
542
+ verboseMessage: 'short content: skipping summary',
543
+ });
544
+ return;
545
+ }
244
546
  if (cacheStore && contentHash && promptHash) {
245
547
  cacheChecked = true;
246
548
  for (const attempt of attempts) {
@@ -256,7 +558,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
256
558
  const cached = cacheStore.getText('summary', key);
257
559
  if (!cached)
258
560
  continue;
259
- writeVerbose(io.stderr, flags.verbose, 'cache hit summary', flags.verboseColor);
561
+ writeVerbose(io.stderr, flags.verbose, 'cache hit summary', flags.verboseColor, io.envForRun);
260
562
  onModelChosen?.(attempt.userModelId);
261
563
  summaryResult = {
262
564
  summary: cached,
@@ -270,7 +572,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
270
572
  }
271
573
  }
272
574
  if (cacheChecked && !summaryFromCache) {
273
- writeVerbose(io.stderr, flags.verbose, 'cache miss summary', flags.verboseColor);
575
+ writeVerbose(io.stderr, flags.verbose, 'cache miss summary', flags.verboseColor, io.envForRun);
274
576
  }
275
577
  ctx.hooks.onSummaryCached?.(summaryFromCache);
276
578
  let lastError = null;
@@ -284,10 +586,10 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
284
586
  envHasKeyFor: model.summaryEngine.envHasKeyFor,
285
587
  formatMissingModelError: model.summaryEngine.formatMissingModelError,
286
588
  onAutoSkip: (attempt) => {
287
- writeVerbose(io.stderr, flags.verbose, `auto skip ${attempt.userModelId}: missing ${attempt.requiredEnv}`, flags.verboseColor);
589
+ writeVerbose(io.stderr, flags.verbose, `auto skip ${attempt.userModelId}: missing ${attempt.requiredEnv}`, flags.verboseColor, io.envForRun);
288
590
  },
289
591
  onAutoFailure: (attempt, error) => {
290
- writeVerbose(io.stderr, flags.verbose, `auto failed ${attempt.userModelId}: ${error instanceof Error ? error.message : String(error)}`, flags.verboseColor);
592
+ writeVerbose(io.stderr, flags.verbose, `auto failed ${attempt.userModelId}: ${error instanceof Error ? error.message : String(error)}`, flags.verboseColor, io.envForRun);
291
593
  },
292
594
  onFixedModelError: (_attempt, error) => {
293
595
  throw error;
@@ -297,6 +599,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
297
599
  prompt: promptPayload,
298
600
  allowStreaming: flags.streamingEnabled,
299
601
  onModelChosen: onModelChosen ?? null,
602
+ streamHandler: slidesOutput?.streamHandler ?? null,
300
603
  }),
301
604
  });
302
605
  summaryResult = attemptOutcome.result;
@@ -330,69 +633,18 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
330
633
  }
331
634
  throw new Error(withFreeTip(`No model available for --model ${model.requestedModelInput}`));
332
635
  }
333
- hooks.clearProgressForStdout();
334
- if (flags.json) {
335
- const finishReport = flags.shouldComputeReport ? await hooks.buildReport() : null;
336
- const finishModel = pickModelForFinishLine(model.llmCalls, null);
337
- const payload = {
338
- input: {
339
- kind: 'url',
340
- url,
341
- timeoutMs: flags.timeoutMs,
342
- youtube: flags.youtubeMode,
343
- firecrawl: flags.firecrawlMode,
344
- format: flags.format,
345
- markdown: effectiveMarkdownMode,
346
- length: flags.lengthArg.kind === 'preset'
347
- ? { kind: 'preset', preset: flags.lengthArg.preset }
348
- : { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
349
- maxOutputTokens: flags.maxOutputTokensArg,
350
- model: model.requestedModelLabel,
351
- language: formatOutputLanguageForJson(flags.outputLanguage),
352
- },
353
- env: {
354
- hasXaiKey: Boolean(model.apiStatus.xaiApiKey),
355
- hasOpenAIKey: Boolean(model.apiStatus.apiKey),
356
- hasOpenRouterKey: Boolean(model.apiStatus.openrouterApiKey),
357
- hasApifyToken: Boolean(model.apiStatus.apifyToken),
358
- hasFirecrawlKey: model.apiStatus.firecrawlConfigured,
359
- hasGoogleKey: model.apiStatus.googleConfigured,
360
- hasAnthropicKey: model.apiStatus.anthropicConfigured,
361
- },
362
- extracted,
363
- prompt,
364
- llm: null,
365
- metrics: flags.metricsEnabled ? finishReport : null,
366
- summary: extracted.content,
367
- };
368
- io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
369
- if (flags.metricsEnabled && finishReport) {
370
- const costUsd = await hooks.estimateCostUsd();
371
- writeFinishLine({
372
- stderr: io.stderr,
373
- elapsedMs: Date.now() - flags.runStartedAtMs,
374
- label: extractionUi.finishSourceLabel,
375
- model: finishModel,
376
- report: finishReport,
377
- costUsd,
378
- detailed: flags.metricsDetailed,
379
- extraParts: buildFinishExtras({
380
- extracted,
381
- metricsDetailed: flags.metricsDetailed,
382
- transcriptionCostLabel,
383
- }),
384
- color: flags.verboseColor,
385
- });
386
- }
387
- return;
388
- }
389
- io.stdout.write(`${extracted.content}\n`);
390
- if (extractionUi.footerParts.length > 0) {
391
- hooks.writeViaFooter([...extractionUi.footerParts, 'no model']);
392
- }
393
- if (lastError instanceof Error && flags.verbose) {
394
- writeVerbose(io.stderr, flags.verbose, `auto failed all models: ${lastError.message}`, flags.verboseColor);
395
- }
636
+ await outputSummaryFromExtractedContent({
637
+ ctx,
638
+ url,
639
+ extracted,
640
+ extractionUi,
641
+ prompt,
642
+ effectiveMarkdownMode,
643
+ transcriptionCostLabel,
644
+ slides,
645
+ footerLabel: 'no model',
646
+ verboseMessage: lastError instanceof Error ? `auto failed all models: ${lastError.message}` : null,
647
+ });
396
648
  return;
397
649
  }
398
650
  if (!summaryFromCache && cacheStore && contentHash && promptHash) {
@@ -404,9 +656,10 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
404
656
  languageKey,
405
657
  });
406
658
  cacheStore.setText('summary', key, summaryResult.summary, cacheState.ttlMs);
407
- writeVerbose(io.stderr, flags.verbose, 'cache write summary', flags.verboseColor);
659
+ writeVerbose(io.stderr, flags.verbose, 'cache write summary', flags.verboseColor, io.envForRun);
408
660
  }
409
661
  const { summary, summaryAlreadyPrinted, modelMeta, maxOutputTokensForCall } = summaryResult;
662
+ const normalizedSummary = slides && slides.slides.length > 0 ? normalizeSummarySlideHeadings(summary) : summary;
410
663
  if (flags.json) {
411
664
  const finishReport = flags.shouldComputeReport ? await hooks.buildReport() : null;
412
665
  const payload = {
@@ -418,6 +671,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
418
671
  firecrawl: flags.firecrawlMode,
419
672
  format: flags.format,
420
673
  markdown: effectiveMarkdownMode,
674
+ timestamps: flags.transcriptTimestamps,
421
675
  length: flags.lengthArg.kind === 'preset'
422
676
  ? { kind: 'preset', preset: flags.lengthArg.preset }
423
677
  : { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
@@ -435,6 +689,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
435
689
  hasAnthropicKey: model.apiStatus.anthropicConfigured,
436
690
  },
437
691
  extracted,
692
+ slides,
438
693
  prompt,
439
694
  llm: {
440
695
  provider: modelMeta.provider,
@@ -443,13 +698,14 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
443
698
  strategy: 'single',
444
699
  },
445
700
  metrics: flags.metricsEnabled ? finishReport : null,
446
- summary,
701
+ summary: normalizedSummary,
447
702
  };
448
703
  io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
449
704
  if (flags.metricsEnabled && finishReport) {
450
705
  const costUsd = await hooks.estimateCostUsd();
451
706
  writeFinishLine({
452
707
  stderr: io.stderr,
708
+ env: io.envForRun,
453
709
  elapsedMs: Date.now() - flags.runStartedAtMs,
454
710
  elapsedLabel: summaryFromCache ? 'Cached' : null,
455
711
  label: extractionUi.finishSourceLabel,
@@ -467,16 +723,32 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
467
723
  }
468
724
  return;
469
725
  }
470
- if (!summaryAlreadyPrinted) {
726
+ if (slidesOutput) {
727
+ if (!summaryAlreadyPrinted) {
728
+ const summaryForSlides = slides && slides.slides.length > 0
729
+ ? coerceSummaryWithSlides({
730
+ markdown: normalizedSummary,
731
+ slides: slides.slides.map((slide) => ({
732
+ index: slide.index,
733
+ timestamp: slide.timestamp,
734
+ })),
735
+ transcriptTimedText: extracted.transcriptTimedText ?? null,
736
+ lengthArg: flags.lengthArg,
737
+ })
738
+ : normalizedSummary;
739
+ await slidesOutput.renderFromText(summaryForSlides);
740
+ }
741
+ }
742
+ else if (!summaryAlreadyPrinted) {
471
743
  hooks.clearProgressForStdout();
472
744
  const rendered = !flags.plain && isRichTty(io.stdout)
473
- ? renderMarkdownAnsi(prepareMarkdownForTerminal(summary), {
745
+ ? renderMarkdownAnsi(prepareMarkdownForTerminal(normalizedSummary), {
474
746
  width: markdownRenderWidth(io.stdout, io.env),
475
747
  wrap: true,
476
748
  color: supportsColor(io.stdout, io.envForRun),
477
749
  hyperlinks: true,
478
750
  })
479
- : summary;
751
+ : normalizedSummary;
480
752
  if (!flags.plain && isRichTty(io.stdout)) {
481
753
  io.stdout.write(`\n${rendered.replace(/^\n+/, '')}`);
482
754
  }
@@ -488,12 +760,14 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
488
760
  if (!rendered.endsWith('\n')) {
489
761
  io.stdout.write('\n');
490
762
  }
763
+ hooks.restoreProgressAfterStdout?.();
491
764
  }
492
765
  const report = flags.shouldComputeReport ? await hooks.buildReport() : null;
493
766
  if (flags.metricsEnabled && report) {
494
767
  const costUsd = await hooks.estimateCostUsd();
495
768
  writeFinishLine({
496
769
  stderr: io.stderr,
770
+ env: io.envForRun,
497
771
  elapsedMs: Date.now() - flags.runStartedAtMs,
498
772
  elapsedLabel: summaryFromCache ? 'Cached' : null,
499
773
  label: extractionUi.finishSourceLabel,