@steipete/summarize 0.8.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +114 -1
- package/LICENSE +1 -1
- package/README.md +309 -182
- package/dist/cli.js +1 -1
- package/dist/esm/cache.js +72 -4
- package/dist/esm/cache.js.map +1 -1
- package/dist/esm/config.js +197 -1
- package/dist/esm/config.js.map +1 -1
- package/dist/esm/content/asset.js +75 -2
- package/dist/esm/content/asset.js.map +1 -1
- package/dist/esm/daemon/agent.js +547 -0
- package/dist/esm/daemon/agent.js.map +1 -0
- package/dist/esm/daemon/chat.js +97 -0
- package/dist/esm/daemon/chat.js.map +1 -0
- package/dist/esm/daemon/cli.js +105 -10
- package/dist/esm/daemon/cli.js.map +1 -1
- package/dist/esm/daemon/env-snapshot.js +3 -0
- package/dist/esm/daemon/env-snapshot.js.map +1 -1
- package/dist/esm/daemon/flow-context.js +53 -28
- package/dist/esm/daemon/flow-context.js.map +1 -1
- package/dist/esm/daemon/launchd.js +27 -0
- package/dist/esm/daemon/launchd.js.map +1 -1
- package/dist/esm/daemon/process-registry.js +206 -0
- package/dist/esm/daemon/process-registry.js.map +1 -0
- package/dist/esm/daemon/schtasks.js +64 -0
- package/dist/esm/daemon/schtasks.js.map +1 -1
- package/dist/esm/daemon/server.js +1034 -52
- package/dist/esm/daemon/server.js.map +1 -1
- package/dist/esm/daemon/summarize.js +66 -18
- package/dist/esm/daemon/summarize.js.map +1 -1
- package/dist/esm/daemon/systemd.js +61 -0
- package/dist/esm/daemon/systemd.js.map +1 -1
- package/dist/esm/flags.js +24 -0
- package/dist/esm/flags.js.map +1 -1
- package/dist/esm/llm/attachments.js +2 -0
- package/dist/esm/llm/attachments.js.map +1 -0
- package/dist/esm/llm/errors.js +6 -0
- package/dist/esm/llm/errors.js.map +1 -0
- package/dist/esm/llm/generate-text.js +206 -356
- package/dist/esm/llm/generate-text.js.map +1 -1
- package/dist/esm/llm/html-to-markdown.js +1 -2
- package/dist/esm/llm/html-to-markdown.js.map +1 -1
- package/dist/esm/llm/prompt.js.map +1 -1
- package/dist/esm/llm/providers/anthropic.js +126 -0
- package/dist/esm/llm/providers/anthropic.js.map +1 -0
- package/dist/esm/llm/providers/google.js +78 -0
- package/dist/esm/llm/providers/google.js.map +1 -0
- package/dist/esm/llm/providers/models.js +111 -0
- package/dist/esm/llm/providers/models.js.map +1 -0
- package/dist/esm/llm/providers/openai.js +150 -0
- package/dist/esm/llm/providers/openai.js.map +1 -0
- package/dist/esm/llm/providers/shared.js +48 -0
- package/dist/esm/llm/providers/shared.js.map +1 -0
- package/dist/esm/llm/providers/types.js +2 -0
- package/dist/esm/llm/providers/types.js.map +1 -0
- package/dist/esm/llm/transcript-to-markdown.js +1 -2
- package/dist/esm/llm/transcript-to-markdown.js.map +1 -1
- package/dist/esm/llm/types.js +2 -0
- package/dist/esm/llm/types.js.map +1 -0
- package/dist/esm/llm/usage.js +69 -0
- package/dist/esm/llm/usage.js.map +1 -0
- package/dist/esm/logging/daemon.js +124 -0
- package/dist/esm/logging/daemon.js.map +1 -0
- package/dist/esm/logging/ring-file.js +66 -0
- package/dist/esm/logging/ring-file.js.map +1 -0
- package/dist/esm/media-cache.js +251 -0
- package/dist/esm/media-cache.js.map +1 -0
- package/dist/esm/model-auto.js +103 -5
- package/dist/esm/model-auto.js.map +1 -1
- package/dist/esm/processes.js +2 -0
- package/dist/esm/processes.js.map +1 -0
- package/dist/esm/refresh-free.js +3 -3
- package/dist/esm/refresh-free.js.map +1 -1
- package/dist/esm/run/attachments.js +8 -4
- package/dist/esm/run/attachments.js.map +1 -1
- package/dist/esm/run/bird.js +118 -5
- package/dist/esm/run/bird.js.map +1 -1
- package/dist/esm/run/cache-state.js +3 -2
- package/dist/esm/run/cache-state.js.map +1 -1
- package/dist/esm/run/cli-preflight.js +19 -1
- package/dist/esm/run/cli-preflight.js.map +1 -1
- package/dist/esm/run/constants.js +0 -7
- package/dist/esm/run/constants.js.map +1 -1
- package/dist/esm/run/finish-line.js +58 -11
- package/dist/esm/run/finish-line.js.map +1 -1
- package/dist/esm/run/flows/asset/extract.js +70 -0
- package/dist/esm/run/flows/asset/extract.js.map +1 -0
- package/dist/esm/run/flows/asset/input.js +209 -25
- package/dist/esm/run/flows/asset/input.js.map +1 -1
- package/dist/esm/run/flows/asset/media-policy.js +3 -0
- package/dist/esm/run/flows/asset/media-policy.js.map +1 -0
- package/dist/esm/run/flows/asset/media.js +224 -0
- package/dist/esm/run/flows/asset/media.js.map +1 -0
- package/dist/esm/run/flows/asset/output.js +98 -0
- package/dist/esm/run/flows/asset/output.js.map +1 -0
- package/dist/esm/run/flows/asset/preprocess.js +92 -16
- package/dist/esm/run/flows/asset/preprocess.js.map +1 -1
- package/dist/esm/run/flows/asset/summary.js +165 -11
- package/dist/esm/run/flows/asset/summary.js.map +1 -1
- package/dist/esm/run/flows/url/extract.js +6 -6
- package/dist/esm/run/flows/url/extract.js.map +1 -1
- package/dist/esm/run/flows/url/flow.js +338 -36
- package/dist/esm/run/flows/url/flow.js.map +1 -1
- package/dist/esm/run/flows/url/markdown.js +6 -1
- package/dist/esm/run/flows/url/markdown.js.map +1 -1
- package/dist/esm/run/flows/url/slides-output.js +485 -0
- package/dist/esm/run/flows/url/slides-output.js.map +1 -0
- package/dist/esm/run/flows/url/slides-text.js +628 -0
- package/dist/esm/run/flows/url/slides-text.js.map +1 -0
- package/dist/esm/run/flows/url/summary.js +358 -83
- package/dist/esm/run/flows/url/summary.js.map +1 -1
- package/dist/esm/run/help.js +94 -5
- package/dist/esm/run/help.js.map +1 -1
- package/dist/esm/run/logging.js +12 -4
- package/dist/esm/run/logging.js.map +1 -1
- package/dist/esm/run/media-cache-state.js +33 -0
- package/dist/esm/run/media-cache-state.js.map +1 -0
- package/dist/esm/run/progress.js +19 -1
- package/dist/esm/run/progress.js.map +1 -1
- package/dist/esm/run/run-context.js +19 -0
- package/dist/esm/run/run-context.js.map +1 -0
- package/dist/esm/run/run-output.js +1 -1
- package/dist/esm/run/run-output.js.map +1 -1
- package/dist/esm/run/run-settings.js +182 -0
- package/dist/esm/run/run-settings.js.map +1 -0
- package/dist/esm/run/runner.js +225 -32
- package/dist/esm/run/runner.js.map +1 -1
- package/dist/esm/run/slides-cli.js +225 -0
- package/dist/esm/run/slides-cli.js.map +1 -0
- package/dist/esm/run/slides-render.js +163 -0
- package/dist/esm/run/slides-render.js.map +1 -0
- package/dist/esm/run/stream-output.js +63 -0
- package/dist/esm/run/stream-output.js.map +1 -0
- package/dist/esm/run/streaming.js +16 -43
- package/dist/esm/run/streaming.js.map +1 -1
- package/dist/esm/run/summary-engine.js +59 -41
- package/dist/esm/run/summary-engine.js.map +1 -1
- package/dist/esm/run/transcriber-cli.js +148 -0
- package/dist/esm/run/transcriber-cli.js.map +1 -0
- package/dist/esm/shared/sse-events.js +26 -0
- package/dist/esm/shared/sse-events.js.map +1 -0
- package/dist/esm/shared/streaming-merge.js +44 -0
- package/dist/esm/shared/streaming-merge.js.map +1 -0
- package/dist/esm/slides/extract.js +1942 -0
- package/dist/esm/slides/extract.js.map +1 -0
- package/dist/esm/slides/index.js +4 -0
- package/dist/esm/slides/index.js.map +1 -0
- package/dist/esm/slides/settings.js +73 -0
- package/dist/esm/slides/settings.js.map +1 -0
- package/dist/esm/slides/store.js +111 -0
- package/dist/esm/slides/store.js.map +1 -0
- package/dist/esm/slides/types.js +2 -0
- package/dist/esm/slides/types.js.map +1 -0
- package/dist/esm/tty/osc-progress.js +21 -1
- package/dist/esm/tty/osc-progress.js.map +1 -1
- package/dist/esm/tty/progress/fetch-html.js +8 -4
- package/dist/esm/tty/progress/fetch-html.js.map +1 -1
- package/dist/esm/tty/progress/transcript.js +82 -31
- package/dist/esm/tty/progress/transcript.js.map +1 -1
- package/dist/esm/tty/spinner.js +2 -2
- package/dist/esm/tty/spinner.js.map +1 -1
- package/dist/esm/tty/theme.js +189 -0
- package/dist/esm/tty/theme.js.map +1 -0
- package/dist/esm/tty/website-progress.js +17 -13
- package/dist/esm/tty/website-progress.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/dist/esm/version.js.map +1 -1
- package/dist/types/cache.d.ts +14 -2
- package/dist/types/config.d.ts +34 -0
- package/dist/types/daemon/agent.d.ts +25 -0
- package/dist/types/daemon/chat.d.ts +27 -0
- package/dist/types/daemon/env-snapshot.d.ts +1 -1
- package/dist/types/daemon/flow-context.d.ts +24 -3
- package/dist/types/daemon/launchd.d.ts +4 -0
- package/dist/types/daemon/process-registry.d.ts +73 -0
- package/dist/types/daemon/schtasks.d.ts +4 -0
- package/dist/types/daemon/server.d.ts +7 -1
- package/dist/types/daemon/summarize.d.ts +47 -5
- package/dist/types/daemon/systemd.d.ts +4 -0
- package/dist/types/flags.d.ts +1 -0
- package/dist/types/llm/attachments.d.ts +6 -0
- package/dist/types/llm/errors.d.ts +1 -0
- package/dist/types/llm/generate-text.d.ts +29 -13
- package/dist/types/llm/prompt.d.ts +7 -2
- package/dist/types/llm/providers/anthropic.d.ts +30 -0
- package/dist/types/llm/providers/google.d.ts +29 -0
- package/dist/types/llm/providers/models.d.ts +27 -0
- package/dist/types/llm/providers/openai.d.ts +38 -0
- package/dist/types/llm/providers/shared.d.ts +14 -0
- package/dist/types/llm/providers/types.d.ts +6 -0
- package/dist/types/llm/types.d.ts +5 -0
- package/dist/types/llm/usage.d.ts +5 -0
- package/dist/types/logging/daemon.d.ts +26 -0
- package/dist/types/logging/ring-file.d.ts +10 -0
- package/dist/types/media-cache.d.ts +22 -0
- package/dist/types/model-auto.d.ts +1 -0
- package/dist/types/processes.d.ts +1 -0
- package/dist/types/run/attachments.d.ts +9 -6
- package/dist/types/run/bird.d.ts +7 -0
- package/dist/types/run/constants.d.ts +0 -2
- package/dist/types/run/finish-line.d.ts +59 -1
- package/dist/types/run/flows/asset/extract.d.ts +18 -0
- package/dist/types/run/flows/asset/input.d.ts +12 -2
- package/dist/types/run/flows/asset/media-policy.d.ts +2 -0
- package/dist/types/run/flows/asset/media.d.ts +21 -0
- package/dist/types/run/flows/asset/output.d.ts +42 -0
- package/dist/types/run/flows/asset/preprocess.d.ts +22 -2
- package/dist/types/run/flows/asset/summary.d.ts +6 -0
- package/dist/types/run/flows/url/extract.d.ts +2 -1
- package/dist/types/run/flows/url/slides-output.d.ts +66 -0
- package/dist/types/run/flows/url/slides-text.d.ts +87 -0
- package/dist/types/run/flows/url/summary.d.ts +11 -3
- package/dist/types/run/flows/url/types.d.ts +29 -2
- package/dist/types/run/help.d.ts +3 -0
- package/dist/types/run/logging.d.ts +3 -2
- package/dist/types/run/media-cache-state.d.ts +7 -0
- package/dist/types/run/progress.d.ts +2 -1
- package/dist/types/run/run-context.d.ts +44 -0
- package/dist/types/run/run-settings.d.ts +62 -0
- package/dist/types/run/slides-cli.d.ts +9 -0
- package/dist/types/run/slides-render.d.ts +30 -0
- package/dist/types/run/stream-output.d.ts +12 -0
- package/dist/types/run/streaming.d.ts +10 -4
- package/dist/types/run/summary-engine.d.ts +15 -3
- package/dist/types/run/summary-llm.d.ts +2 -2
- package/dist/types/run/transcriber-cli.d.ts +8 -0
- package/dist/types/shared/sse-events.d.ts +64 -0
- package/dist/types/shared/streaming-merge.d.ts +4 -0
- package/dist/types/slides/extract.d.ts +42 -0
- package/dist/types/slides/index.d.ts +5 -0
- package/dist/types/slides/settings.d.ts +20 -0
- package/dist/types/slides/store.d.ts +15 -0
- package/dist/types/slides/types.d.ts +40 -0
- package/dist/types/tty/osc-progress.d.ts +2 -2
- package/dist/types/tty/progress/fetch-html.d.ts +3 -1
- package/dist/types/tty/progress/transcript.d.ts +3 -1
- package/dist/types/tty/spinner.d.ts +3 -1
- package/dist/types/tty/theme.d.ts +44 -0
- package/dist/types/tty/website-progress.d.ts +3 -1
- package/dist/types/version.d.ts +1 -1
- package/docs/README.md +13 -8
- package/docs/_config.yml +26 -0
- package/docs/_layouts/default.html +60 -0
- package/docs/agent.md +333 -0
- package/docs/assets/site.css +748 -0
- package/docs/assets/site.js +72 -0
- package/docs/assets/summarize-cli.png +0 -0
- package/docs/assets/summarize-extension.png +0 -0
- package/docs/assets/youtube-slides.png +0 -0
- package/docs/cache.md +29 -3
- package/docs/chrome-extension.md +85 -7
- package/docs/config.md +74 -2
- package/docs/extract-only.md +10 -2
- package/docs/index.html +205 -0
- package/docs/index.md +25 -0
- package/docs/language.md +1 -1
- package/docs/llm.md +17 -1
- package/docs/manual-tests.md +2 -0
- package/docs/media.md +37 -0
- package/docs/model-auto.md +2 -1
- package/docs/nvidia-onnx-transcription.md +55 -0
- package/docs/openai.md +5 -0
- package/docs/releasing.md +26 -0
- package/docs/site/assets/site.css +399 -228
- package/docs/site/assets/summarize-cli.png +0 -0
- package/docs/site/assets/summarize-extension.png +0 -0
- package/docs/site/docs/chrome-extension.html +89 -0
- package/docs/site/docs/config.html +1 -0
- package/docs/site/docs/extract-only.html +1 -0
- package/docs/site/docs/firecrawl.html +1 -0
- package/docs/site/docs/index.html +5 -0
- package/docs/site/docs/llm.html +1 -0
- package/docs/site/docs/openai.html +1 -0
- package/docs/site/docs/website.html +1 -0
- package/docs/site/docs/youtube.html +1 -0
- package/docs/site/index.html +148 -84
- package/docs/slides.md +74 -0
- package/docs/timestamps.md +103 -0
- package/docs/website.md +13 -0
- package/docs/youtube.md +16 -0
- package/package.json +22 -18
- package/dist/esm/daemon/request-settings.js +0 -91
- package/dist/esm/daemon/request-settings.js.map +0 -1
- package/dist/types/daemon/request-settings.d.ts +0 -27
|
@@ -1,19 +1,137 @@
|
|
|
1
|
+
import { isTwitterStatusUrl, isYouTubeUrl } from '@steipete/summarize-core/content/url';
|
|
1
2
|
import { countTokens } from 'gpt-tokenizer';
|
|
2
3
|
import { render as renderMarkdownAnsi } from 'markdansi';
|
|
3
4
|
import { buildLanguageKey, buildLengthKey, buildPromptHash, buildSummaryCacheKey, hashString, normalizeContentForHash, } from '../../../cache.js';
|
|
4
5
|
import { formatOutputLanguageForJson } from '../../../language.js';
|
|
5
6
|
import { parseGatewayStyleModelId } from '../../../llm/model-id.js';
|
|
6
7
|
import { buildAutoModelAttempts } from '../../../model-auto.js';
|
|
7
|
-
import { buildLinkSummaryPrompt } from '../../../prompts/index.js';
|
|
8
|
+
import { buildLinkSummaryPrompt, SUMMARY_LENGTH_TARGET_CHARACTERS, SUMMARY_SYSTEM_PROMPT, } from '../../../prompts/index.js';
|
|
8
9
|
import { parseCliUserModelId } from '../../env.js';
|
|
9
10
|
import { buildExtractFinishLabel, buildLengthPartsForFinishLine, writeFinishLine, } from '../../finish-line.js';
|
|
11
|
+
import { resolveTargetCharacters } from '../../format.js';
|
|
10
12
|
import { writeVerbose } from '../../logging.js';
|
|
11
13
|
import { prepareMarkdownForTerminal } from '../../markdown.js';
|
|
12
14
|
import { runModelAttempts } from '../../model-attempts.js';
|
|
13
15
|
import { buildOpenRouterNoAllowedProvidersMessage } from '../../openrouter.js';
|
|
14
16
|
import { isRichTty, markdownRenderWidth, supportsColor } from '../../terminal.js';
|
|
15
|
-
|
|
17
|
+
import { coerceSummaryWithSlides, interleaveSlidesIntoTranscript, normalizeSummarySlideHeadings, } from './slides-text.js';
|
|
18
|
+
const MAX_SLIDE_TRANSCRIPT_CHARS_BY_PRESET = {
|
|
19
|
+
short: 2500,
|
|
20
|
+
medium: 5000,
|
|
21
|
+
long: 9000,
|
|
22
|
+
xl: 15000,
|
|
23
|
+
xxl: 24000,
|
|
24
|
+
};
|
|
25
|
+
const SLIDE_TRANSCRIPT_DEFAULT_EDGE_SECONDS = 30;
|
|
26
|
+
const SLIDE_TRANSCRIPT_LEEWAY_SECONDS = 10;
|
|
27
|
+
function parseTimestampSeconds(value) {
|
|
28
|
+
const parts = value.split(':').map((item) => Number(item));
|
|
29
|
+
if (parts.some((item) => !Number.isFinite(item)))
|
|
30
|
+
return null;
|
|
31
|
+
if (parts.length === 2) {
|
|
32
|
+
const [minutes, seconds] = parts;
|
|
33
|
+
return minutes * 60 + seconds;
|
|
34
|
+
}
|
|
35
|
+
if (parts.length === 3) {
|
|
36
|
+
const [hours, minutes, seconds] = parts;
|
|
37
|
+
return hours * 3600 + minutes * 60 + seconds;
|
|
38
|
+
}
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
function parseTranscriptTimedText(input) {
|
|
42
|
+
if (!input)
|
|
43
|
+
return [];
|
|
44
|
+
const segments = [];
|
|
45
|
+
for (const line of input.split('\n')) {
|
|
46
|
+
const trimmed = line.trim();
|
|
47
|
+
if (!trimmed.startsWith('['))
|
|
48
|
+
continue;
|
|
49
|
+
const match = trimmed.match(/^\[(\d{1,2}:\d{2}(?::\d{2})?)\]\s*(.*)$/);
|
|
50
|
+
if (!match)
|
|
51
|
+
continue;
|
|
52
|
+
const seconds = parseTimestampSeconds(match[1]);
|
|
53
|
+
if (seconds == null)
|
|
54
|
+
continue;
|
|
55
|
+
const text = (match[2] ?? '').trim();
|
|
56
|
+
if (!text)
|
|
57
|
+
continue;
|
|
58
|
+
segments.push({ startSeconds: seconds, text });
|
|
59
|
+
}
|
|
60
|
+
segments.sort((a, b) => a.startSeconds - b.startSeconds);
|
|
61
|
+
return segments;
|
|
62
|
+
}
|
|
63
|
+
function formatTimestamp(seconds) {
|
|
64
|
+
const clamped = Math.max(0, Math.floor(seconds));
|
|
65
|
+
const hours = Math.floor(clamped / 3600);
|
|
66
|
+
const minutes = Math.floor((clamped % 3600) / 60);
|
|
67
|
+
const secs = clamped % 60;
|
|
68
|
+
const mm = String(minutes).padStart(2, '0');
|
|
69
|
+
const ss = String(secs).padStart(2, '0');
|
|
70
|
+
if (hours <= 0)
|
|
71
|
+
return `${minutes}:${ss}`;
|
|
72
|
+
const hh = String(hours).padStart(2, '0');
|
|
73
|
+
return `${hh}:${mm}:${ss}`;
|
|
74
|
+
}
|
|
75
|
+
function truncateTranscript(value, limit) {
|
|
76
|
+
if (value.length <= limit)
|
|
77
|
+
return value;
|
|
78
|
+
const truncated = value.slice(0, limit).trimEnd();
|
|
79
|
+
const clean = truncated.replace(/\s+\S*$/, '').trim();
|
|
80
|
+
const result = clean.length > 0 ? clean : truncated.trim();
|
|
81
|
+
return result.length > 0 ? `${result}…` : '';
|
|
82
|
+
}
|
|
83
|
+
function buildSlidesPromptText({ slides, transcriptTimedText, preset, }) {
|
|
84
|
+
if (!slides || slides.slides.length === 0)
|
|
85
|
+
return null;
|
|
86
|
+
const segments = parseTranscriptTimedText(transcriptTimedText);
|
|
87
|
+
const slidesWithTimestamps = slides.slides
|
|
88
|
+
.filter((slide) => Number.isFinite(slide.timestamp))
|
|
89
|
+
.map((slide) => ({ index: slide.index, timestamp: Math.max(0, Math.floor(slide.timestamp)) }))
|
|
90
|
+
.sort((a, b) => a.timestamp - b.timestamp);
|
|
91
|
+
if (slidesWithTimestamps.length === 0)
|
|
92
|
+
return null;
|
|
93
|
+
const totalBudget = Number(MAX_SLIDE_TRANSCRIPT_CHARS_BY_PRESET[preset]);
|
|
94
|
+
const perSlideBudget = Math.max(120, Math.floor(totalBudget / Math.max(1, slidesWithTimestamps.length)));
|
|
95
|
+
let remaining = totalBudget;
|
|
96
|
+
const blocks = [];
|
|
97
|
+
for (let i = 0; i < slidesWithTimestamps.length; i += 1) {
|
|
98
|
+
const slide = slidesWithTimestamps[i];
|
|
99
|
+
if (!slide)
|
|
100
|
+
continue;
|
|
101
|
+
const prev = slidesWithTimestamps[i - 1];
|
|
102
|
+
const next = slidesWithTimestamps[i + 1];
|
|
103
|
+
const startBase = prev ? Math.floor((prev.timestamp + slide.timestamp) / 2) : slide.timestamp;
|
|
104
|
+
const endBase = next ? Math.ceil((slide.timestamp + next.timestamp) / 2) : slide.timestamp;
|
|
105
|
+
const start = Math.max(0, (prev ? startBase : slide.timestamp - SLIDE_TRANSCRIPT_DEFAULT_EDGE_SECONDS) -
|
|
106
|
+
SLIDE_TRANSCRIPT_LEEWAY_SECONDS);
|
|
107
|
+
const end = (next ? endBase : slide.timestamp + SLIDE_TRANSCRIPT_DEFAULT_EDGE_SECONDS) +
|
|
108
|
+
SLIDE_TRANSCRIPT_LEEWAY_SECONDS;
|
|
109
|
+
const excerptParts = [];
|
|
110
|
+
for (const segment of segments) {
|
|
111
|
+
if (segment.startSeconds < start)
|
|
112
|
+
continue;
|
|
113
|
+
if (segment.startSeconds > end)
|
|
114
|
+
break;
|
|
115
|
+
excerptParts.push(segment.text);
|
|
116
|
+
}
|
|
117
|
+
const excerptRaw = excerptParts.join(' ').trim().replace(/\s+/g, ' ');
|
|
118
|
+
const excerptBudget = remaining > 0 ? Math.min(perSlideBudget, remaining) : 0;
|
|
119
|
+
const excerpt = excerptRaw && excerptBudget > 0 ? truncateTranscript(excerptRaw, excerptBudget) : '';
|
|
120
|
+
const label = `[slide:${slide.index}] [${formatTimestamp(start)}–${formatTimestamp(end)}]`;
|
|
121
|
+
const block = excerpt ? `${label}\n${excerpt}` : label;
|
|
122
|
+
blocks.push(block);
|
|
123
|
+
remaining = Math.max(0, remaining - block.length);
|
|
124
|
+
}
|
|
125
|
+
return blocks.length > 0 ? blocks.join('\n\n') : null;
|
|
126
|
+
}
|
|
127
|
+
export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOverride, lengthInstruction, languageInstruction, slides, }) {
|
|
16
128
|
const isYouTube = extracted.siteName === 'YouTube';
|
|
129
|
+
const preset = lengthArg.kind === 'preset' ? lengthArg.preset : 'medium';
|
|
130
|
+
const slidesText = buildSlidesPromptText({
|
|
131
|
+
slides,
|
|
132
|
+
transcriptTimedText: extracted.transcriptTimedText,
|
|
133
|
+
preset,
|
|
134
|
+
});
|
|
17
135
|
return buildLinkSummaryPrompt({
|
|
18
136
|
url: extracted.url,
|
|
19
137
|
title: extracted.title,
|
|
@@ -23,6 +141,8 @@ export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOve
|
|
|
23
141
|
truncated: extracted.truncated,
|
|
24
142
|
hasTranscript: isYouTube ||
|
|
25
143
|
(extracted.transcriptSource !== null && extracted.transcriptSource !== 'unavailable'),
|
|
144
|
+
hasTranscriptTimestamps: Boolean(extracted.transcriptTimedText),
|
|
145
|
+
slides: slidesText ? { count: slides?.slides.length ?? 0, text: slidesText } : null,
|
|
26
146
|
summaryLength: lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters },
|
|
27
147
|
outputLanguage,
|
|
28
148
|
shares: [],
|
|
@@ -31,6 +151,97 @@ export function buildUrlPrompt({ extracted, outputLanguage, lengthArg, promptOve
|
|
|
31
151
|
languageInstruction: languageInstruction ?? null,
|
|
32
152
|
});
|
|
33
153
|
}
|
|
154
|
+
function shouldBypassShortContentSummary({ extracted, lengthArg, forceSummary, maxOutputTokensArg, json, }) {
|
|
155
|
+
if (forceSummary)
|
|
156
|
+
return false;
|
|
157
|
+
if (!extracted.content || extracted.content.length === 0)
|
|
158
|
+
return false;
|
|
159
|
+
const targetCharacters = resolveTargetCharacters(lengthArg, SUMMARY_LENGTH_TARGET_CHARACTERS);
|
|
160
|
+
if (!Number.isFinite(targetCharacters) || targetCharacters <= 0)
|
|
161
|
+
return false;
|
|
162
|
+
if (extracted.content.length > targetCharacters)
|
|
163
|
+
return false;
|
|
164
|
+
if (!json && typeof maxOutputTokensArg === 'number') {
|
|
165
|
+
const tokenCount = countTokens(extracted.content);
|
|
166
|
+
if (tokenCount > maxOutputTokensArg)
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
return true;
|
|
170
|
+
}
|
|
171
|
+
async function outputSummaryFromExtractedContent({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, slides, footerLabel, verboseMessage, }) {
|
|
172
|
+
const { io, flags, model, hooks } = ctx;
|
|
173
|
+
hooks.clearProgressForStdout();
|
|
174
|
+
const finishModel = pickModelForFinishLine(model.llmCalls, null);
|
|
175
|
+
if (flags.json) {
|
|
176
|
+
const finishReport = flags.shouldComputeReport ? await hooks.buildReport() : null;
|
|
177
|
+
const payload = {
|
|
178
|
+
input: {
|
|
179
|
+
kind: 'url',
|
|
180
|
+
url,
|
|
181
|
+
timeoutMs: flags.timeoutMs,
|
|
182
|
+
youtube: flags.youtubeMode,
|
|
183
|
+
firecrawl: flags.firecrawlMode,
|
|
184
|
+
format: flags.format,
|
|
185
|
+
markdown: effectiveMarkdownMode,
|
|
186
|
+
timestamps: flags.transcriptTimestamps,
|
|
187
|
+
length: flags.lengthArg.kind === 'preset'
|
|
188
|
+
? { kind: 'preset', preset: flags.lengthArg.preset }
|
|
189
|
+
: { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
|
|
190
|
+
maxOutputTokens: flags.maxOutputTokensArg,
|
|
191
|
+
model: model.requestedModelLabel,
|
|
192
|
+
language: formatOutputLanguageForJson(flags.outputLanguage),
|
|
193
|
+
},
|
|
194
|
+
env: {
|
|
195
|
+
hasXaiKey: Boolean(model.apiStatus.xaiApiKey),
|
|
196
|
+
hasOpenAIKey: Boolean(model.apiStatus.apiKey),
|
|
197
|
+
hasOpenRouterKey: Boolean(model.apiStatus.openrouterApiKey),
|
|
198
|
+
hasApifyToken: Boolean(model.apiStatus.apifyToken),
|
|
199
|
+
hasFirecrawlKey: model.apiStatus.firecrawlConfigured,
|
|
200
|
+
hasGoogleKey: model.apiStatus.googleConfigured,
|
|
201
|
+
hasAnthropicKey: model.apiStatus.anthropicConfigured,
|
|
202
|
+
},
|
|
203
|
+
extracted,
|
|
204
|
+
slides,
|
|
205
|
+
prompt,
|
|
206
|
+
llm: null,
|
|
207
|
+
metrics: flags.metricsEnabled ? finishReport : null,
|
|
208
|
+
summary: extracted.content,
|
|
209
|
+
};
|
|
210
|
+
io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
211
|
+
if (flags.metricsEnabled && finishReport) {
|
|
212
|
+
const costUsd = await hooks.estimateCostUsd();
|
|
213
|
+
hooks.clearProgressForStdout();
|
|
214
|
+
writeFinishLine({
|
|
215
|
+
stderr: io.stderr,
|
|
216
|
+
env: io.envForRun,
|
|
217
|
+
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
218
|
+
label: extractionUi.finishSourceLabel,
|
|
219
|
+
model: finishModel,
|
|
220
|
+
report: finishReport,
|
|
221
|
+
costUsd,
|
|
222
|
+
detailed: flags.metricsDetailed,
|
|
223
|
+
extraParts: buildFinishExtras({
|
|
224
|
+
extracted,
|
|
225
|
+
metricsDetailed: flags.metricsDetailed,
|
|
226
|
+
transcriptionCostLabel,
|
|
227
|
+
}),
|
|
228
|
+
color: flags.verboseColor,
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
io.stdout.write(`${extracted.content}\n`);
|
|
234
|
+
hooks.restoreProgressAfterStdout?.();
|
|
235
|
+
if (extractionUi.footerParts.length > 0) {
|
|
236
|
+
const footer = footerLabel
|
|
237
|
+
? [...extractionUi.footerParts, footerLabel]
|
|
238
|
+
: extractionUi.footerParts;
|
|
239
|
+
hooks.writeViaFooter(footer);
|
|
240
|
+
}
|
|
241
|
+
if (verboseMessage && flags.verbose) {
|
|
242
|
+
writeVerbose(io.stderr, flags.verbose, verboseMessage, flags.verboseColor, io.envForRun);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
34
245
|
const buildFinishExtras = ({ extracted, metricsDetailed, transcriptionCostLabel, }) => {
|
|
35
246
|
const parts = [
|
|
36
247
|
...(buildLengthPartsForFinishLine(extracted, metricsDetailed) ?? []),
|
|
@@ -62,7 +273,7 @@ const buildModelMetaFromAttempt = (attempt) => {
|
|
|
62
273
|
: parsed.canonical;
|
|
63
274
|
return { provider: parsed.provider, canonical };
|
|
64
275
|
};
|
|
65
|
-
export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, }) {
|
|
276
|
+
export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, slides, slidesOutput, }) {
|
|
66
277
|
const { io, flags, model, hooks } = ctx;
|
|
67
278
|
hooks.clearProgressForStdout();
|
|
68
279
|
const finishLabel = buildExtractFinishLabel({
|
|
@@ -83,6 +294,7 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
|
|
|
83
294
|
firecrawl: flags.firecrawlMode,
|
|
84
295
|
format: flags.format,
|
|
85
296
|
markdown: effectiveMarkdownMode,
|
|
297
|
+
timestamps: flags.transcriptTimestamps,
|
|
86
298
|
length: flags.lengthArg.kind === 'preset'
|
|
87
299
|
? { kind: 'preset', preset: flags.lengthArg.preset }
|
|
88
300
|
: { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
|
|
@@ -100,16 +312,20 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
|
|
|
100
312
|
hasAnthropicKey: model.apiStatus.anthropicConfigured,
|
|
101
313
|
},
|
|
102
314
|
extracted,
|
|
315
|
+
slides,
|
|
103
316
|
prompt,
|
|
104
317
|
llm: null,
|
|
105
318
|
metrics: flags.metricsEnabled ? finishReport : null,
|
|
106
319
|
summary: null,
|
|
107
320
|
};
|
|
108
321
|
io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
322
|
+
hooks.restoreProgressAfterStdout?.();
|
|
323
|
+
hooks.restoreProgressAfterStdout?.();
|
|
109
324
|
if (flags.metricsEnabled && finishReport) {
|
|
110
325
|
const costUsd = await hooks.estimateCostUsd();
|
|
111
326
|
writeFinishLine({
|
|
112
327
|
stderr: io.stderr,
|
|
328
|
+
env: io.envForRun,
|
|
113
329
|
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
114
330
|
label: finishLabel,
|
|
115
331
|
model: finishModel,
|
|
@@ -126,14 +342,62 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
|
|
|
126
342
|
}
|
|
127
343
|
return;
|
|
128
344
|
}
|
|
345
|
+
const extractCandidate = flags.transcriptTimestamps &&
|
|
346
|
+
extracted.transcriptTimedText &&
|
|
347
|
+
extracted.transcriptSource &&
|
|
348
|
+
extracted.content.toLowerCase().startsWith('transcript:')
|
|
349
|
+
? `Transcript:\n${extracted.transcriptTimedText}`
|
|
350
|
+
: extracted.content;
|
|
351
|
+
const slideTags = slides?.slides && slides.slides.length > 0
|
|
352
|
+
? slides.slides.map((slide) => `[slide:${slide.index}]`).join('\n')
|
|
353
|
+
: '';
|
|
354
|
+
if (slidesOutput && slides?.slides && slides.slides.length > 0) {
|
|
355
|
+
const transcriptText = extracted.transcriptTimedText
|
|
356
|
+
? `Transcript:\n${extracted.transcriptTimedText}`
|
|
357
|
+
: null;
|
|
358
|
+
const interleaved = transcriptText
|
|
359
|
+
? interleaveSlidesIntoTranscript({
|
|
360
|
+
transcriptTimedText: transcriptText,
|
|
361
|
+
slides: slides.slides.map((slide) => ({
|
|
362
|
+
index: slide.index,
|
|
363
|
+
timestamp: slide.timestamp,
|
|
364
|
+
})),
|
|
365
|
+
})
|
|
366
|
+
: `${extractCandidate.trimEnd()}\n\n${slideTags}`;
|
|
367
|
+
await slidesOutput.renderFromText(interleaved);
|
|
368
|
+
hooks.restoreProgressAfterStdout?.();
|
|
369
|
+
const slideFooter = slides ? [`slides ${slides.slides.length}`] : [];
|
|
370
|
+
hooks.writeViaFooter([...extractionUi.footerParts, ...slideFooter]);
|
|
371
|
+
const report = flags.shouldComputeReport ? await hooks.buildReport() : null;
|
|
372
|
+
if (flags.metricsEnabled && report) {
|
|
373
|
+
const costUsd = await hooks.estimateCostUsd();
|
|
374
|
+
writeFinishLine({
|
|
375
|
+
stderr: io.stderr,
|
|
376
|
+
env: io.envForRun,
|
|
377
|
+
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
378
|
+
label: finishLabel,
|
|
379
|
+
model: finishModel,
|
|
380
|
+
report,
|
|
381
|
+
costUsd,
|
|
382
|
+
detailed: flags.metricsDetailed,
|
|
383
|
+
extraParts: buildFinishExtras({
|
|
384
|
+
extracted,
|
|
385
|
+
metricsDetailed: flags.metricsDetailed,
|
|
386
|
+
transcriptionCostLabel,
|
|
387
|
+
}),
|
|
388
|
+
color: flags.verboseColor,
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
129
393
|
const renderedExtract = flags.format === 'markdown' && !flags.plain && isRichTty(io.stdout)
|
|
130
|
-
? renderMarkdownAnsi(prepareMarkdownForTerminal(
|
|
394
|
+
? renderMarkdownAnsi(prepareMarkdownForTerminal(extractCandidate), {
|
|
131
395
|
width: markdownRenderWidth(io.stdout, io.env),
|
|
132
396
|
wrap: true,
|
|
133
397
|
color: supportsColor(io.stdout, io.envForRun),
|
|
134
398
|
hyperlinks: true,
|
|
135
399
|
})
|
|
136
|
-
:
|
|
400
|
+
: extractCandidate;
|
|
137
401
|
if (flags.format === 'markdown' && !flags.plain && isRichTty(io.stdout)) {
|
|
138
402
|
io.stdout.write(`\n${renderedExtract.replace(/^\n+/, '')}`);
|
|
139
403
|
}
|
|
@@ -143,12 +407,16 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
|
|
|
143
407
|
if (!renderedExtract.endsWith('\n')) {
|
|
144
408
|
io.stdout.write('\n');
|
|
145
409
|
}
|
|
146
|
-
hooks.
|
|
410
|
+
hooks.restoreProgressAfterStdout?.();
|
|
411
|
+
const slideFooter = slides ? [`slides ${slides.slides.length}`] : [];
|
|
412
|
+
hooks.writeViaFooter([...extractionUi.footerParts, ...slideFooter]);
|
|
147
413
|
const report = flags.shouldComputeReport ? await hooks.buildReport() : null;
|
|
148
414
|
if (flags.metricsEnabled && report) {
|
|
149
415
|
const costUsd = await hooks.estimateCostUsd();
|
|
416
|
+
hooks.clearProgressForStdout();
|
|
150
417
|
writeFinishLine({
|
|
151
418
|
stderr: io.stderr,
|
|
419
|
+
env: io.envForRun,
|
|
152
420
|
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
153
421
|
label: finishLabel,
|
|
154
422
|
model: finishModel,
|
|
@@ -164,9 +432,10 @@ export async function outputExtractedUrl({ ctx, url, extracted, extractionUi, pr
|
|
|
164
432
|
});
|
|
165
433
|
}
|
|
166
434
|
}
|
|
167
|
-
export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, onModelChosen, }) {
|
|
435
|
+
export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi, prompt, effectiveMarkdownMode, transcriptionCostLabel, onModelChosen, slides, slidesOutput, }) {
|
|
168
436
|
const { io, flags, model, cache: cacheState, hooks } = ctx;
|
|
169
|
-
const
|
|
437
|
+
const promptPayload = { system: SUMMARY_SYSTEM_PROMPT, userText: prompt };
|
|
438
|
+
const promptTokens = countTokens(promptPayload.userText);
|
|
170
439
|
const kindForAuto = extracted.siteName === 'YouTube' ? 'youtube' : 'website';
|
|
171
440
|
const attempts = await (async () => {
|
|
172
441
|
if (model.isFallbackModel) {
|
|
@@ -184,7 +453,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
184
453
|
});
|
|
185
454
|
if (flags.verbose) {
|
|
186
455
|
for (const attempt of list.slice(0, 8)) {
|
|
187
|
-
writeVerbose(io.stderr, flags.verbose, `auto candidate ${attempt.debug}`, flags.verboseColor);
|
|
456
|
+
writeVerbose(io.stderr, flags.verbose, `auto candidate ${attempt.debug}`, flags.verboseColor, io.envForRun);
|
|
188
457
|
}
|
|
189
458
|
}
|
|
190
459
|
return list.map((attempt) => {
|
|
@@ -231,7 +500,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
231
500
|
},
|
|
232
501
|
];
|
|
233
502
|
})();
|
|
234
|
-
const cacheStore = cacheState.mode === 'default' ? cacheState.store : null;
|
|
503
|
+
const cacheStore = cacheState.mode === 'default' && !flags.summaryCacheBypass ? cacheState.store : null;
|
|
235
504
|
const contentHash = cacheStore ? hashString(normalizeContentForHash(extracted.content)) : null;
|
|
236
505
|
const promptHash = cacheStore ? buildPromptHash(prompt) : null;
|
|
237
506
|
const lengthKey = buildLengthKey(flags.lengthArg);
|
|
@@ -240,6 +509,40 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
240
509
|
let usedAttempt = null;
|
|
241
510
|
let summaryFromCache = false;
|
|
242
511
|
let cacheChecked = false;
|
|
512
|
+
const isTweet = extracted.siteName?.toLowerCase() === 'x' || isTwitterStatusUrl(extracted.url);
|
|
513
|
+
const isYouTube = extracted.siteName === 'YouTube' || isYouTubeUrl(url);
|
|
514
|
+
const hasMedia = Boolean(extracted.video) ||
|
|
515
|
+
(extracted.transcriptSource != null && extracted.transcriptSource !== 'unavailable') ||
|
|
516
|
+
(typeof extracted.mediaDurationSeconds === 'number' && extracted.mediaDurationSeconds > 0) ||
|
|
517
|
+
extracted.isVideoOnly === true;
|
|
518
|
+
const autoBypass = ctx.model.isFallbackModel && !ctx.model.isNamedModelSelection;
|
|
519
|
+
const canBypassShortContent = (autoBypass || isTweet) &&
|
|
520
|
+
!flags.slides &&
|
|
521
|
+
!hasMedia &&
|
|
522
|
+
flags.streamMode !== 'on' &&
|
|
523
|
+
!isYouTube &&
|
|
524
|
+
shouldBypassShortContentSummary({
|
|
525
|
+
extracted,
|
|
526
|
+
lengthArg: flags.lengthArg,
|
|
527
|
+
forceSummary: flags.forceSummary,
|
|
528
|
+
maxOutputTokensArg: flags.maxOutputTokensArg,
|
|
529
|
+
json: flags.json,
|
|
530
|
+
});
|
|
531
|
+
if (canBypassShortContent) {
|
|
532
|
+
await outputSummaryFromExtractedContent({
|
|
533
|
+
ctx,
|
|
534
|
+
url,
|
|
535
|
+
extracted,
|
|
536
|
+
extractionUi,
|
|
537
|
+
prompt,
|
|
538
|
+
effectiveMarkdownMode,
|
|
539
|
+
transcriptionCostLabel,
|
|
540
|
+
slides,
|
|
541
|
+
footerLabel: 'short content',
|
|
542
|
+
verboseMessage: 'short content: skipping summary',
|
|
543
|
+
});
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
243
546
|
if (cacheStore && contentHash && promptHash) {
|
|
244
547
|
cacheChecked = true;
|
|
245
548
|
for (const attempt of attempts) {
|
|
@@ -255,7 +558,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
255
558
|
const cached = cacheStore.getText('summary', key);
|
|
256
559
|
if (!cached)
|
|
257
560
|
continue;
|
|
258
|
-
writeVerbose(io.stderr, flags.verbose, 'cache hit summary', flags.verboseColor);
|
|
561
|
+
writeVerbose(io.stderr, flags.verbose, 'cache hit summary', flags.verboseColor, io.envForRun);
|
|
259
562
|
onModelChosen?.(attempt.userModelId);
|
|
260
563
|
summaryResult = {
|
|
261
564
|
summary: cached,
|
|
@@ -269,7 +572,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
269
572
|
}
|
|
270
573
|
}
|
|
271
574
|
if (cacheChecked && !summaryFromCache) {
|
|
272
|
-
writeVerbose(io.stderr, flags.verbose, 'cache miss summary', flags.verboseColor);
|
|
575
|
+
writeVerbose(io.stderr, flags.verbose, 'cache miss summary', flags.verboseColor, io.envForRun);
|
|
273
576
|
}
|
|
274
577
|
ctx.hooks.onSummaryCached?.(summaryFromCache);
|
|
275
578
|
let lastError = null;
|
|
@@ -283,19 +586,20 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
283
586
|
envHasKeyFor: model.summaryEngine.envHasKeyFor,
|
|
284
587
|
formatMissingModelError: model.summaryEngine.formatMissingModelError,
|
|
285
588
|
onAutoSkip: (attempt) => {
|
|
286
|
-
writeVerbose(io.stderr, flags.verbose, `auto skip ${attempt.userModelId}: missing ${attempt.requiredEnv}`, flags.verboseColor);
|
|
589
|
+
writeVerbose(io.stderr, flags.verbose, `auto skip ${attempt.userModelId}: missing ${attempt.requiredEnv}`, flags.verboseColor, io.envForRun);
|
|
287
590
|
},
|
|
288
591
|
onAutoFailure: (attempt, error) => {
|
|
289
|
-
writeVerbose(io.stderr, flags.verbose, `auto failed ${attempt.userModelId}: ${error instanceof Error ? error.message : String(error)}`, flags.verboseColor);
|
|
592
|
+
writeVerbose(io.stderr, flags.verbose, `auto failed ${attempt.userModelId}: ${error instanceof Error ? error.message : String(error)}`, flags.verboseColor, io.envForRun);
|
|
290
593
|
},
|
|
291
594
|
onFixedModelError: (_attempt, error) => {
|
|
292
595
|
throw error;
|
|
293
596
|
},
|
|
294
597
|
runAttempt: (attempt) => model.summaryEngine.runSummaryAttempt({
|
|
295
598
|
attempt,
|
|
296
|
-
prompt,
|
|
599
|
+
prompt: promptPayload,
|
|
297
600
|
allowStreaming: flags.streamingEnabled,
|
|
298
601
|
onModelChosen: onModelChosen ?? null,
|
|
602
|
+
streamHandler: slidesOutput?.streamHandler ?? null,
|
|
299
603
|
}),
|
|
300
604
|
});
|
|
301
605
|
summaryResult = attemptOutcome.result;
|
|
@@ -329,69 +633,18 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
329
633
|
}
|
|
330
634
|
throw new Error(withFreeTip(`No model available for --model ${model.requestedModelInput}`));
|
|
331
635
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
markdown: effectiveMarkdownMode,
|
|
345
|
-
length: flags.lengthArg.kind === 'preset'
|
|
346
|
-
? { kind: 'preset', preset: flags.lengthArg.preset }
|
|
347
|
-
: { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
|
|
348
|
-
maxOutputTokens: flags.maxOutputTokensArg,
|
|
349
|
-
model: model.requestedModelLabel,
|
|
350
|
-
language: formatOutputLanguageForJson(flags.outputLanguage),
|
|
351
|
-
},
|
|
352
|
-
env: {
|
|
353
|
-
hasXaiKey: Boolean(model.apiStatus.xaiApiKey),
|
|
354
|
-
hasOpenAIKey: Boolean(model.apiStatus.apiKey),
|
|
355
|
-
hasOpenRouterKey: Boolean(model.apiStatus.openrouterApiKey),
|
|
356
|
-
hasApifyToken: Boolean(model.apiStatus.apifyToken),
|
|
357
|
-
hasFirecrawlKey: model.apiStatus.firecrawlConfigured,
|
|
358
|
-
hasGoogleKey: model.apiStatus.googleConfigured,
|
|
359
|
-
hasAnthropicKey: model.apiStatus.anthropicConfigured,
|
|
360
|
-
},
|
|
361
|
-
extracted,
|
|
362
|
-
prompt,
|
|
363
|
-
llm: null,
|
|
364
|
-
metrics: flags.metricsEnabled ? finishReport : null,
|
|
365
|
-
summary: extracted.content,
|
|
366
|
-
};
|
|
367
|
-
io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
368
|
-
if (flags.metricsEnabled && finishReport) {
|
|
369
|
-
const costUsd = await hooks.estimateCostUsd();
|
|
370
|
-
writeFinishLine({
|
|
371
|
-
stderr: io.stderr,
|
|
372
|
-
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
373
|
-
label: extractionUi.finishSourceLabel,
|
|
374
|
-
model: finishModel,
|
|
375
|
-
report: finishReport,
|
|
376
|
-
costUsd,
|
|
377
|
-
detailed: flags.metricsDetailed,
|
|
378
|
-
extraParts: buildFinishExtras({
|
|
379
|
-
extracted,
|
|
380
|
-
metricsDetailed: flags.metricsDetailed,
|
|
381
|
-
transcriptionCostLabel,
|
|
382
|
-
}),
|
|
383
|
-
color: flags.verboseColor,
|
|
384
|
-
});
|
|
385
|
-
}
|
|
386
|
-
return;
|
|
387
|
-
}
|
|
388
|
-
io.stdout.write(`${extracted.content}\n`);
|
|
389
|
-
if (extractionUi.footerParts.length > 0) {
|
|
390
|
-
hooks.writeViaFooter([...extractionUi.footerParts, 'no model']);
|
|
391
|
-
}
|
|
392
|
-
if (lastError instanceof Error && flags.verbose) {
|
|
393
|
-
writeVerbose(io.stderr, flags.verbose, `auto failed all models: ${lastError.message}`, flags.verboseColor);
|
|
394
|
-
}
|
|
636
|
+
await outputSummaryFromExtractedContent({
|
|
637
|
+
ctx,
|
|
638
|
+
url,
|
|
639
|
+
extracted,
|
|
640
|
+
extractionUi,
|
|
641
|
+
prompt,
|
|
642
|
+
effectiveMarkdownMode,
|
|
643
|
+
transcriptionCostLabel,
|
|
644
|
+
slides,
|
|
645
|
+
footerLabel: 'no model',
|
|
646
|
+
verboseMessage: lastError instanceof Error ? `auto failed all models: ${lastError.message}` : null,
|
|
647
|
+
});
|
|
395
648
|
return;
|
|
396
649
|
}
|
|
397
650
|
if (!summaryFromCache && cacheStore && contentHash && promptHash) {
|
|
@@ -403,9 +656,10 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
403
656
|
languageKey,
|
|
404
657
|
});
|
|
405
658
|
cacheStore.setText('summary', key, summaryResult.summary, cacheState.ttlMs);
|
|
406
|
-
writeVerbose(io.stderr, flags.verbose, 'cache write summary', flags.verboseColor);
|
|
659
|
+
writeVerbose(io.stderr, flags.verbose, 'cache write summary', flags.verboseColor, io.envForRun);
|
|
407
660
|
}
|
|
408
661
|
const { summary, summaryAlreadyPrinted, modelMeta, maxOutputTokensForCall } = summaryResult;
|
|
662
|
+
const normalizedSummary = slides && slides.slides.length > 0 ? normalizeSummarySlideHeadings(summary) : summary;
|
|
409
663
|
if (flags.json) {
|
|
410
664
|
const finishReport = flags.shouldComputeReport ? await hooks.buildReport() : null;
|
|
411
665
|
const payload = {
|
|
@@ -417,6 +671,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
417
671
|
firecrawl: flags.firecrawlMode,
|
|
418
672
|
format: flags.format,
|
|
419
673
|
markdown: effectiveMarkdownMode,
|
|
674
|
+
timestamps: flags.transcriptTimestamps,
|
|
420
675
|
length: flags.lengthArg.kind === 'preset'
|
|
421
676
|
? { kind: 'preset', preset: flags.lengthArg.preset }
|
|
422
677
|
: { kind: 'chars', maxCharacters: flags.lengthArg.maxCharacters },
|
|
@@ -434,6 +689,7 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
434
689
|
hasAnthropicKey: model.apiStatus.anthropicConfigured,
|
|
435
690
|
},
|
|
436
691
|
extracted,
|
|
692
|
+
slides,
|
|
437
693
|
prompt,
|
|
438
694
|
llm: {
|
|
439
695
|
provider: modelMeta.provider,
|
|
@@ -442,13 +698,14 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
442
698
|
strategy: 'single',
|
|
443
699
|
},
|
|
444
700
|
metrics: flags.metricsEnabled ? finishReport : null,
|
|
445
|
-
summary,
|
|
701
|
+
summary: normalizedSummary,
|
|
446
702
|
};
|
|
447
703
|
io.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
448
704
|
if (flags.metricsEnabled && finishReport) {
|
|
449
705
|
const costUsd = await hooks.estimateCostUsd();
|
|
450
706
|
writeFinishLine({
|
|
451
707
|
stderr: io.stderr,
|
|
708
|
+
env: io.envForRun,
|
|
452
709
|
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
453
710
|
elapsedLabel: summaryFromCache ? 'Cached' : null,
|
|
454
711
|
label: extractionUi.finishSourceLabel,
|
|
@@ -466,16 +723,32 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
466
723
|
}
|
|
467
724
|
return;
|
|
468
725
|
}
|
|
469
|
-
if (
|
|
726
|
+
if (slidesOutput) {
|
|
727
|
+
if (!summaryAlreadyPrinted) {
|
|
728
|
+
const summaryForSlides = slides && slides.slides.length > 0
|
|
729
|
+
? coerceSummaryWithSlides({
|
|
730
|
+
markdown: normalizedSummary,
|
|
731
|
+
slides: slides.slides.map((slide) => ({
|
|
732
|
+
index: slide.index,
|
|
733
|
+
timestamp: slide.timestamp,
|
|
734
|
+
})),
|
|
735
|
+
transcriptTimedText: extracted.transcriptTimedText ?? null,
|
|
736
|
+
lengthArg: flags.lengthArg,
|
|
737
|
+
})
|
|
738
|
+
: normalizedSummary;
|
|
739
|
+
await slidesOutput.renderFromText(summaryForSlides);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
else if (!summaryAlreadyPrinted) {
|
|
470
743
|
hooks.clearProgressForStdout();
|
|
471
744
|
const rendered = !flags.plain && isRichTty(io.stdout)
|
|
472
|
-
? renderMarkdownAnsi(prepareMarkdownForTerminal(
|
|
745
|
+
? renderMarkdownAnsi(prepareMarkdownForTerminal(normalizedSummary), {
|
|
473
746
|
width: markdownRenderWidth(io.stdout, io.env),
|
|
474
747
|
wrap: true,
|
|
475
748
|
color: supportsColor(io.stdout, io.envForRun),
|
|
476
749
|
hyperlinks: true,
|
|
477
750
|
})
|
|
478
|
-
:
|
|
751
|
+
: normalizedSummary;
|
|
479
752
|
if (!flags.plain && isRichTty(io.stdout)) {
|
|
480
753
|
io.stdout.write(`\n${rendered.replace(/^\n+/, '')}`);
|
|
481
754
|
}
|
|
@@ -487,12 +760,14 @@ export async function summarizeExtractedUrl({ ctx, url, extracted, extractionUi,
|
|
|
487
760
|
if (!rendered.endsWith('\n')) {
|
|
488
761
|
io.stdout.write('\n');
|
|
489
762
|
}
|
|
763
|
+
hooks.restoreProgressAfterStdout?.();
|
|
490
764
|
}
|
|
491
765
|
const report = flags.shouldComputeReport ? await hooks.buildReport() : null;
|
|
492
766
|
if (flags.metricsEnabled && report) {
|
|
493
767
|
const costUsd = await hooks.estimateCostUsd();
|
|
494
768
|
writeFinishLine({
|
|
495
769
|
stderr: io.stderr,
|
|
770
|
+
env: io.envForRun,
|
|
496
771
|
elapsedMs: Date.now() - flags.runStartedAtMs,
|
|
497
772
|
elapsedLabel: summaryFromCache ? 'Cached' : null,
|
|
498
773
|
label: extractionUi.finishSourceLabel,
|