@steipete/summarize 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -5
- package/README.md +122 -20
- package/dist/cli.cjs +8446 -4360
- package/dist/cli.cjs.map +4 -4
- package/dist/esm/cli-main.js +47 -2
- package/dist/esm/cli-main.js.map +1 -1
- package/dist/esm/config.js +368 -3
- package/dist/esm/config.js.map +1 -1
- package/dist/esm/content/link-preview/content/index.js +13 -0
- package/dist/esm/content/link-preview/content/index.js.map +1 -1
- package/dist/esm/content/link-preview/content/utils.js +3 -1
- package/dist/esm/content/link-preview/content/utils.js.map +1 -1
- package/dist/esm/content/link-preview/content/video.js +96 -0
- package/dist/esm/content/link-preview/content/video.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/captions.js +21 -21
- package/dist/esm/content/link-preview/transcript/providers/youtube/captions.js.map +1 -1
- package/dist/esm/costs.js.map +1 -1
- package/dist/esm/flags.js +41 -1
- package/dist/esm/flags.js.map +1 -1
- package/dist/esm/generate-free.js +616 -0
- package/dist/esm/generate-free.js.map +1 -0
- package/dist/esm/llm/cli.js +290 -0
- package/dist/esm/llm/cli.js.map +1 -0
- package/dist/esm/llm/generate-text.js +159 -105
- package/dist/esm/llm/generate-text.js.map +1 -1
- package/dist/esm/llm/html-to-markdown.js +4 -2
- package/dist/esm/llm/html-to-markdown.js.map +1 -1
- package/dist/esm/markitdown.js +54 -0
- package/dist/esm/markitdown.js.map +1 -0
- package/dist/esm/model-auto.js +353 -0
- package/dist/esm/model-auto.js.map +1 -0
- package/dist/esm/model-spec.js +82 -0
- package/dist/esm/model-spec.js.map +1 -0
- package/dist/esm/prompts/cli.js +18 -0
- package/dist/esm/prompts/cli.js.map +1 -0
- package/dist/esm/prompts/file.js +21 -2
- package/dist/esm/prompts/file.js.map +1 -1
- package/dist/esm/prompts/index.js +2 -1
- package/dist/esm/prompts/index.js.map +1 -1
- package/dist/esm/prompts/link-summary.js +3 -8
- package/dist/esm/prompts/link-summary.js.map +1 -1
- package/dist/esm/refresh-free.js +667 -0
- package/dist/esm/refresh-free.js.map +1 -0
- package/dist/esm/run.js +1612 -533
- package/dist/esm/run.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/dist/types/config.d.ts +58 -5
- package/dist/types/content/link-preview/content/types.d.ts +10 -0
- package/dist/types/content/link-preview/content/utils.d.ts +1 -1
- package/dist/types/content/link-preview/content/video.d.ts +5 -0
- package/dist/types/costs.d.ts +2 -1
- package/dist/types/flags.d.ts +7 -0
- package/dist/types/generate-free.d.ts +17 -0
- package/dist/types/llm/cli.d.ts +24 -0
- package/dist/types/llm/generate-text.d.ts +13 -4
- package/dist/types/llm/html-to-markdown.d.ts +9 -3
- package/dist/types/markitdown.d.ts +10 -0
- package/dist/types/model-auto.d.ts +23 -0
- package/dist/types/model-spec.d.ts +33 -0
- package/dist/types/prompts/cli.d.ts +8 -0
- package/dist/types/prompts/file.d.ts +7 -0
- package/dist/types/prompts/index.d.ts +2 -1
- package/dist/types/refresh-free.d.ts +19 -0
- package/dist/types/run.d.ts +3 -1
- package/dist/types/version.d.ts +1 -1
- package/docs/README.md +4 -1
- package/docs/cli.md +95 -0
- package/docs/config.md +123 -1
- package/docs/extract-only.md +10 -7
- package/docs/firecrawl.md +2 -2
- package/docs/llm.md +24 -4
- package/docs/manual-tests.md +40 -0
- package/docs/model-auto.md +92 -0
- package/docs/site/assets/site.js +20 -17
- package/docs/site/docs/config.html +3 -3
- package/docs/site/docs/extract-only.html +7 -5
- package/docs/site/docs/firecrawl.html +6 -6
- package/docs/site/docs/index.html +2 -2
- package/docs/site/docs/llm.html +2 -2
- package/docs/site/docs/openai.html +2 -2
- package/docs/site/docs/website.html +7 -4
- package/docs/site/docs/youtube.html +2 -2
- package/docs/site/index.html +1 -1
- package/docs/smoketest.md +58 -0
- package/docs/website.md +13 -8
- package/docs/youtube.md +1 -1
- package/package.json +8 -4
- package/dist/esm/content/link-preview/transcript/providers/twitter.js +0 -12
- package/dist/esm/content/link-preview/transcript/providers/twitter.js.map +0 -1
- package/dist/esm/content/link-preview/transcript/providers/youtube/ytdlp.js +0 -114
- package/dist/esm/content/link-preview/transcript/providers/youtube/ytdlp.js.map +0 -1
- package/dist/esm/summarizeHome.js +0 -20
- package/dist/esm/summarizeHome.js.map +0 -1
- package/dist/esm/tty/live-markdown.js +0 -52
- package/dist/esm/tty/live-markdown.js.map +0 -1
- package/dist/types/content/link-preview/transcript/providers/twitter.d.ts +0 -3
- package/dist/types/content/link-preview/transcript/providers/youtube/ytdlp.d.ts +0 -3
- package/dist/types/summarizeHome.d.ts +0 -6
- package/dist/types/tty/live-markdown.d.ts +0 -10
package/dist/esm/run.js
CHANGED
|
@@ -1,27 +1,36 @@
|
|
|
1
1
|
import { execFile } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants as fsConstants } from 'node:fs';
|
|
3
3
|
import fs from 'node:fs/promises';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
4
5
|
import path from 'node:path';
|
|
5
6
|
import { Command, CommanderError, Option } from 'commander';
|
|
6
7
|
import { countTokens } from 'gpt-tokenizer';
|
|
7
8
|
import { createLiveRenderer, render as renderMarkdownAnsi } from 'markdansi';
|
|
9
|
+
import mime from 'mime';
|
|
8
10
|
import { normalizeTokenUsage, tallyCosts } from 'tokentally';
|
|
9
11
|
import { loadSummarizeConfig } from './config.js';
|
|
10
12
|
import { buildAssetPromptMessages, classifyUrl, loadLocalAsset, loadRemoteAsset, resolveInputTarget, } from './content/asset.js';
|
|
11
13
|
import { createLinkPreviewClient } from './content/index.js';
|
|
14
|
+
import { fetchWithTimeout } from './content/link-preview/fetch-with-timeout.js';
|
|
12
15
|
import { buildRunMetricsReport } from './costs.js';
|
|
13
16
|
import { createFirecrawlScraper } from './firecrawl.js';
|
|
14
|
-
import { parseDurationMs, parseFirecrawlMode, parseLengthArg, parseMarkdownMode, parseMaxOutputTokensArg, parseMetricsMode, parseRenderMode, parseStreamMode, parseYoutubeMode, } from './flags.js';
|
|
17
|
+
import { parseDurationMs, parseExtractFormat, parseFirecrawlMode, parseLengthArg, parseMarkdownMode, parseMaxOutputTokensArg, parseMetricsMode, parsePreprocessMode, parseRenderMode, parseRetriesArg, parseStreamMode, parseVideoMode, parseYoutubeMode, } from './flags.js';
|
|
18
|
+
import { isCliDisabled, resolveCliBinary, runCliModel } from './llm/cli.js';
|
|
15
19
|
import { generateTextWithModelId, streamTextWithModelId } from './llm/generate-text.js';
|
|
16
20
|
import { resolveGoogleModelForUsage } from './llm/google-models.js';
|
|
17
21
|
import { createHtmlToMarkdownConverter } from './llm/html-to-markdown.js';
|
|
18
|
-
import {
|
|
22
|
+
import { parseGatewayStyleModelId } from './llm/model-id.js';
|
|
23
|
+
import { convertToMarkdownWithMarkitdown } from './markitdown.js';
|
|
24
|
+
import { buildAutoModelAttempts } from './model-auto.js';
|
|
25
|
+
import { parseRequestedModelId } from './model-spec.js';
|
|
19
26
|
import { loadLiteLlmCatalog, resolveLiteLlmMaxInputTokensForModelId, resolveLiteLlmMaxOutputTokensForModelId, resolveLiteLlmPricingForModelId, } from './pricing/litellm.js';
|
|
20
|
-
import { buildFileSummaryPrompt, buildLinkSummaryPrompt } from './prompts/index.js';
|
|
27
|
+
import { buildFileSummaryPrompt, buildFileTextSummaryPrompt, buildLinkSummaryPrompt, buildPathSummaryPrompt, } from './prompts/index.js';
|
|
28
|
+
import { refreshFree } from './refresh-free.js';
|
|
21
29
|
import { startOscProgress } from './tty/osc-progress.js';
|
|
22
30
|
import { startSpinner } from './tty/spinner.js';
|
|
23
31
|
import { resolvePackageVersion } from './version.js';
|
|
24
32
|
const BIRD_TIP = 'Tip: Install bird🐦 for better Twitter support: https://github.com/steipete/bird';
|
|
33
|
+
const UVX_TIP = 'Tip: Install uv (uvx) for local Markdown conversion: brew install uv (or set UVX_PATH to your uvx binary).';
|
|
25
34
|
const TWITTER_HOSTS = new Set(['x.com', 'twitter.com', 'mobile.twitter.com']);
|
|
26
35
|
const SUMMARY_LENGTH_MAX_CHARACTERS = {
|
|
27
36
|
short: 1200,
|
|
@@ -30,6 +39,61 @@ const SUMMARY_LENGTH_MAX_CHARACTERS = {
|
|
|
30
39
|
xl: 14000,
|
|
31
40
|
xxl: Number.POSITIVE_INFINITY,
|
|
32
41
|
};
|
|
42
|
+
function truncateList(items, max) {
|
|
43
|
+
const normalized = items.map((item) => item.trim()).filter(Boolean);
|
|
44
|
+
if (normalized.length <= max)
|
|
45
|
+
return normalized.join(', ');
|
|
46
|
+
return `${normalized.slice(0, max).join(', ')} (+${normalized.length - max} more)`;
|
|
47
|
+
}
|
|
48
|
+
function parseOpenRouterModelId(modelId) {
|
|
49
|
+
const normalized = modelId.trim();
|
|
50
|
+
if (!normalized.startsWith('openrouter/'))
|
|
51
|
+
return null;
|
|
52
|
+
const rest = normalized.slice('openrouter/'.length);
|
|
53
|
+
const [author, ...slugParts] = rest.split('/');
|
|
54
|
+
if (!author || slugParts.length === 0)
|
|
55
|
+
return null;
|
|
56
|
+
return { author, slug: slugParts.join('/') };
|
|
57
|
+
}
|
|
58
|
+
async function resolveOpenRouterProvidersForModels({ modelIds, fetchImpl, timeoutMs, }) {
|
|
59
|
+
const results = new Map();
|
|
60
|
+
const unique = Array.from(new Set(modelIds.map((id) => id.trim()).filter(Boolean)));
|
|
61
|
+
await Promise.all(unique.map(async (modelId) => {
|
|
62
|
+
const parsed = parseOpenRouterModelId(modelId);
|
|
63
|
+
if (!parsed)
|
|
64
|
+
return;
|
|
65
|
+
const url = `https://openrouter.ai/api/v1/models/${encodeURIComponent(parsed.author)}/${encodeURIComponent(parsed.slug)}/endpoints`;
|
|
66
|
+
try {
|
|
67
|
+
const response = await fetchWithTimeout(fetchImpl, url, { headers: { Accept: 'application/json' } }, Math.min(timeoutMs, 15_000));
|
|
68
|
+
if (!response.ok)
|
|
69
|
+
return;
|
|
70
|
+
const payload = (await response.json());
|
|
71
|
+
const endpoints = Array.isArray(payload.data?.endpoints) ? payload.data?.endpoints : [];
|
|
72
|
+
const providers = endpoints
|
|
73
|
+
.map((endpoint) => endpoint && typeof endpoint.provider_name === 'string'
|
|
74
|
+
? endpoint.provider_name.trim()
|
|
75
|
+
: null)
|
|
76
|
+
.filter((value) => Boolean(value));
|
|
77
|
+
const uniqueProviders = Array.from(new Set(providers)).sort((a, b) => a.localeCompare(b));
|
|
78
|
+
if (uniqueProviders.length > 0)
|
|
79
|
+
results.set(modelId, uniqueProviders);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// best-effort only
|
|
83
|
+
}
|
|
84
|
+
}));
|
|
85
|
+
return results;
|
|
86
|
+
}
|
|
87
|
+
async function buildOpenRouterNoAllowedProvidersMessage({ attempts, fetchImpl, timeoutMs, }) {
|
|
88
|
+
const modelIds = attempts
|
|
89
|
+
.map((attempt) => attempt.userModelId)
|
|
90
|
+
.filter((id) => id.startsWith('openrouter/'));
|
|
91
|
+
const tried = truncateList(modelIds, 6);
|
|
92
|
+
const providerMap = await resolveOpenRouterProvidersForModels({ modelIds, fetchImpl, timeoutMs });
|
|
93
|
+
const allProviders = Array.from(new Set(Array.from(providerMap.values()).flat())).sort((a, b) => a.localeCompare(b));
|
|
94
|
+
const providersHint = allProviders.length > 0 ? ` Providers to allow: ${truncateList(allProviders, 10)}.` : '';
|
|
95
|
+
return `OpenRouter could not route any models with this API key (no allowed providers). Tried: ${tried}.${providersHint} Hint: increase --timeout (e.g. 10m) and/or use --debug/--verbose to see per-model failures. (OpenRouter: Settings → API Keys → edit key → Allowed providers.)`;
|
|
96
|
+
}
|
|
33
97
|
function resolveTargetCharacters(lengthArg) {
|
|
34
98
|
return lengthArg.kind === 'chars'
|
|
35
99
|
? lengthArg.maxCharacters
|
|
@@ -56,15 +120,63 @@ function isExecutable(filePath) {
|
|
|
56
120
|
return false;
|
|
57
121
|
}
|
|
58
122
|
}
|
|
59
|
-
function
|
|
60
|
-
|
|
61
|
-
|
|
123
|
+
function resolveExecutableInPath(binary, env) {
|
|
124
|
+
if (!binary)
|
|
125
|
+
return null;
|
|
126
|
+
if (path.isAbsolute(binary)) {
|
|
127
|
+
return isExecutable(binary) ? binary : null;
|
|
128
|
+
}
|
|
129
|
+
const pathEnv = env.PATH ?? '';
|
|
62
130
|
for (const entry of pathEnv.split(path.delimiter)) {
|
|
63
131
|
if (!entry)
|
|
64
132
|
continue;
|
|
65
|
-
|
|
133
|
+
const candidate = path.join(entry, binary);
|
|
134
|
+
if (isExecutable(candidate))
|
|
135
|
+
return candidate;
|
|
136
|
+
}
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
function hasBirdCli(env) {
|
|
140
|
+
return resolveExecutableInPath('bird', env) !== null;
|
|
141
|
+
}
|
|
142
|
+
function hasUvxCli(env) {
|
|
143
|
+
if (typeof env.UVX_PATH === 'string' && env.UVX_PATH.trim().length > 0) {
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
return resolveExecutableInPath('uvx', env) !== null;
|
|
147
|
+
}
|
|
148
|
+
function resolveCliAvailability({ env, config, }) {
|
|
149
|
+
const cliConfig = config?.cli ?? null;
|
|
150
|
+
const providers = ['claude', 'codex', 'gemini'];
|
|
151
|
+
const availability = {};
|
|
152
|
+
for (const provider of providers) {
|
|
153
|
+
if (isCliDisabled(provider, cliConfig)) {
|
|
154
|
+
availability[provider] = false;
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
const binary = resolveCliBinary(provider, cliConfig, env);
|
|
158
|
+
availability[provider] = resolveExecutableInPath(binary, env) !== null;
|
|
159
|
+
}
|
|
160
|
+
return availability;
|
|
161
|
+
}
|
|
162
|
+
function parseCliUserModelId(modelId) {
|
|
163
|
+
const parts = modelId
|
|
164
|
+
.trim()
|
|
165
|
+
.split('/')
|
|
166
|
+
.map((part) => part.trim());
|
|
167
|
+
const provider = parts[1]?.toLowerCase();
|
|
168
|
+
if (provider !== 'claude' && provider !== 'codex' && provider !== 'gemini') {
|
|
169
|
+
throw new Error(`Invalid CLI model id "${modelId}". Expected cli/<provider>/<model>.`);
|
|
170
|
+
}
|
|
171
|
+
const model = parts.slice(2).join('/').trim();
|
|
172
|
+
return { provider, model: model.length > 0 ? model : null };
|
|
173
|
+
}
|
|
174
|
+
function parseCliProviderArg(raw) {
|
|
175
|
+
const normalized = raw.trim().toLowerCase();
|
|
176
|
+
if (normalized === 'claude' || normalized === 'codex' || normalized === 'gemini') {
|
|
177
|
+
return normalized;
|
|
66
178
|
}
|
|
67
|
-
|
|
179
|
+
throw new Error(`Unsupported --cli: ${raw}`);
|
|
68
180
|
}
|
|
69
181
|
async function readTweetWithBird(args) {
|
|
70
182
|
return await new Promise((resolve, reject) => {
|
|
@@ -108,24 +220,62 @@ function withBirdTip(error, url, env) {
|
|
|
108
220
|
const combined = `${message}\n${BIRD_TIP}`;
|
|
109
221
|
return error instanceof Error ? new Error(combined, { cause: error }) : new Error(combined);
|
|
110
222
|
}
|
|
223
|
+
function withUvxTip(error, env) {
|
|
224
|
+
if (hasUvxCli(env)) {
|
|
225
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
226
|
+
}
|
|
227
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
228
|
+
const combined = `${message}\n${UVX_TIP}`;
|
|
229
|
+
return error instanceof Error ? new Error(combined, { cause: error }) : new Error(combined);
|
|
230
|
+
}
|
|
111
231
|
const MAX_TEXT_BYTES_DEFAULT = 10 * 1024 * 1024;
|
|
232
|
+
const BUILTIN_MODELS = {
|
|
233
|
+
free: {
|
|
234
|
+
mode: 'auto',
|
|
235
|
+
rules: [
|
|
236
|
+
{
|
|
237
|
+
candidates: [
|
|
238
|
+
// Snapshot (2025-12-23): generated via `summarize refresh-free`.
|
|
239
|
+
'openrouter/xiaomi/mimo-v2-flash:free',
|
|
240
|
+
'openrouter/mistralai/devstral-2512:free',
|
|
241
|
+
'openrouter/qwen/qwen3-coder:free',
|
|
242
|
+
'openrouter/kwaipilot/kat-coder-pro:free',
|
|
243
|
+
'openrouter/moonshotai/kimi-k2:free',
|
|
244
|
+
'openrouter/nex-agi/deepseek-v3.1-nex-n1:free',
|
|
245
|
+
],
|
|
246
|
+
},
|
|
247
|
+
],
|
|
248
|
+
},
|
|
249
|
+
};
|
|
112
250
|
function buildProgram() {
|
|
113
251
|
return new Command()
|
|
114
252
|
.name('summarize')
|
|
115
253
|
.description('Summarize web pages and YouTube links (uses direct provider API keys).')
|
|
116
254
|
.argument('[input]', 'URL or local file path to summarize')
|
|
117
255
|
.option('--youtube <mode>', 'YouTube transcript source: auto, web (youtubei/captionTracks), yt-dlp (audio+whisper), apify', 'auto')
|
|
118
|
-
.
|
|
119
|
-
.
|
|
120
|
-
.
|
|
256
|
+
.addOption(new Option('--video-mode <mode>', 'Video handling: auto (prefer video understanding if supported), transcript, understand.')
|
|
257
|
+
.choices(['auto', 'transcript', 'understand'])
|
|
258
|
+
.default('auto'))
|
|
259
|
+
.option('--firecrawl <mode>', 'Firecrawl usage: off, auto (fallback), always (try Firecrawl first). Note: in --format md website mode, defaults to always when FIRECRAWL_API_KEY is set (unless --firecrawl is set explicitly).', 'auto')
|
|
260
|
+
.option('--format <format>', 'Website/file content format: md|text. For websites: controls the extraction format. For files: controls whether we try to preprocess to Markdown for model compatibility. (default: text)', 'text')
|
|
261
|
+
.addOption(new Option('--preprocess <mode>', 'Preprocess inputs for model compatibility: off, auto (fallback), always.')
|
|
262
|
+
.choices(['off', 'auto', 'always'])
|
|
263
|
+
.default('auto'))
|
|
264
|
+
.addOption(new Option('--markdown-mode <mode>', 'HTML→Markdown conversion: off, auto (prefer Firecrawl when configured, then LLM when configured, then markitdown when available), llm (force LLM). Only affects --format md for non-YouTube URLs.').default('auto'))
|
|
265
|
+
.addOption(new Option('--markdown <mode>', 'Deprecated alias for --markdown-mode (use --extract --format md --markdown-mode ...)').hideHelp())
|
|
266
|
+
.option('--length <length>', 'Summary length: short|medium|long|xl|xxl or a character limit like 20000, 20k', 'xl')
|
|
121
267
|
.option('--max-output-tokens <count>', 'Hard cap for LLM output tokens (e.g. 2000, 2k). Overrides provider defaults.', undefined)
|
|
122
268
|
.option('--timeout <duration>', 'Timeout for content fetching and LLM request: 30 (seconds), 30s, 2m, 5000ms', '2m')
|
|
123
|
-
.option('--
|
|
124
|
-
.option('--
|
|
269
|
+
.option('--retries <count>', 'LLM retry attempts on timeout (default: 1).', '1')
|
|
270
|
+
.option('--model <model>', 'LLM model id: auto, <name>, cli/<provider>/<model>, xai/..., openai/..., google/..., anthropic/... or openrouter/<author>/<slug> (default: auto)', undefined)
|
|
271
|
+
.addOption(new Option('--cli [provider]', 'Use a CLI provider: claude, gemini, codex (equivalent to --model cli/<provider>). If omitted, use auto selection with CLI enabled.'))
|
|
272
|
+
.option('--extract', 'Print extracted content and exit (no LLM summary)', false)
|
|
273
|
+
.addOption(new Option('--extract-only', 'Deprecated alias for --extract').hideHelp())
|
|
125
274
|
.option('--json', 'Output structured JSON (includes prompt + metrics)', false)
|
|
126
275
|
.option('--stream <mode>', 'Stream LLM output: auto (TTY only), on, off. Note: streaming is disabled in --json mode.', 'auto')
|
|
127
276
|
.option('--render <mode>', 'Render Markdown output: auto (TTY only), md-live, md, plain. Note: auto selects md-live when streaming to a TTY.', 'auto')
|
|
128
277
|
.option('--verbose', 'Print detailed progress info to stderr', false)
|
|
278
|
+
.option('--debug', 'Alias for --verbose (and defaults --metrics to detailed)', false)
|
|
129
279
|
.addOption(new Option('--metrics <mode>', 'Metrics output: off, on, detailed')
|
|
130
280
|
.choices(['off', 'on', 'detailed'])
|
|
131
281
|
.default('on'))
|
|
@@ -250,6 +400,55 @@ function getTextContentFromAttachment(attachment) {
|
|
|
250
400
|
}
|
|
251
401
|
return { content: '', bytes: 0 };
|
|
252
402
|
}
|
|
403
|
+
function getFileBytesFromAttachment(attachment) {
|
|
404
|
+
if (attachment.part.type !== 'file')
|
|
405
|
+
return null;
|
|
406
|
+
const data = attachment.part.data;
|
|
407
|
+
return data instanceof Uint8Array ? data : null;
|
|
408
|
+
}
|
|
409
|
+
function getAttachmentBytes(attachment) {
|
|
410
|
+
if (attachment.part.type === 'image') {
|
|
411
|
+
const image = attachment.part.image;
|
|
412
|
+
return image instanceof Uint8Array ? image : null;
|
|
413
|
+
}
|
|
414
|
+
return getFileBytesFromAttachment(attachment);
|
|
415
|
+
}
|
|
416
|
+
async function ensureCliAttachmentPath({ sourceKind, sourceLabel, attachment, }) {
|
|
417
|
+
if (sourceKind === 'file')
|
|
418
|
+
return sourceLabel;
|
|
419
|
+
const bytes = getAttachmentBytes(attachment);
|
|
420
|
+
if (!bytes) {
|
|
421
|
+
throw new Error('CLI attachment missing bytes');
|
|
422
|
+
}
|
|
423
|
+
const ext = attachment.filename && path.extname(attachment.filename)
|
|
424
|
+
? path.extname(attachment.filename)
|
|
425
|
+
: attachment.mediaType
|
|
426
|
+
? `.${mime.getExtension(attachment.mediaType) ?? 'bin'}`
|
|
427
|
+
: '.bin';
|
|
428
|
+
const filename = attachment.filename?.trim() || `asset${ext}`;
|
|
429
|
+
const dir = await fs.mkdtemp(path.join(tmpdir(), 'summarize-cli-asset-'));
|
|
430
|
+
const filePath = path.join(dir, filename);
|
|
431
|
+
await fs.writeFile(filePath, bytes);
|
|
432
|
+
return filePath;
|
|
433
|
+
}
|
|
434
|
+
function shouldMarkitdownConvertMediaType(mediaType) {
|
|
435
|
+
const mt = mediaType.toLowerCase();
|
|
436
|
+
if (mt === 'application/pdf')
|
|
437
|
+
return true;
|
|
438
|
+
if (mt === 'application/rtf')
|
|
439
|
+
return true;
|
|
440
|
+
if (mt === 'text/html' || mt === 'application/xhtml+xml')
|
|
441
|
+
return true;
|
|
442
|
+
if (mt === 'application/msword')
|
|
443
|
+
return true;
|
|
444
|
+
if (mt.startsWith('application/vnd.openxmlformats-officedocument.'))
|
|
445
|
+
return true;
|
|
446
|
+
if (mt === 'application/vnd.ms-excel')
|
|
447
|
+
return true;
|
|
448
|
+
if (mt === 'application/vnd.ms-powerpoint')
|
|
449
|
+
return true;
|
|
450
|
+
return false;
|
|
451
|
+
}
|
|
253
452
|
function assertProviderSupportsAttachment({ provider, modelId, attachment, }) {
|
|
254
453
|
// xAI via AI SDK currently supports image parts, but not generic file parts (e.g. PDFs).
|
|
255
454
|
if (provider === 'xai' &&
|
|
@@ -314,11 +513,12 @@ function attachRichHelp(program, env, stdout) {
|
|
|
314
513
|
program.addHelpText('after', () => `
|
|
315
514
|
${heading('Examples')}
|
|
316
515
|
${cmd('summarize "https://example.com"')}
|
|
317
|
-
${cmd('summarize "https://example.com" --extract
|
|
318
|
-
${cmd('summarize "https://example.com" --extract
|
|
319
|
-
${cmd('summarize "https://
|
|
320
|
-
${cmd('summarize "https://
|
|
321
|
-
${cmd('
|
|
516
|
+
${cmd('summarize "https://example.com" --extract')} ${dim('# extracted plain text')}
|
|
517
|
+
${cmd('summarize "https://example.com" --extract --format md')} ${dim('# extracted markdown (prefers Firecrawl when configured)')}
|
|
518
|
+
${cmd('summarize "https://example.com" --extract --format md --markdown-mode llm')} ${dim('# extracted markdown via LLM')}
|
|
519
|
+
${cmd('summarize "https://www.youtube.com/watch?v=I845O57ZSy4&t=11s" --extract --youtube web')}
|
|
520
|
+
${cmd('summarize "https://example.com" --length 20k --max-output-tokens 2k --timeout 2m --model openai/gpt-5-mini')}
|
|
521
|
+
${cmd('summarize "https://example.com" --model mymodel')} ${dim('# config preset')}
|
|
322
522
|
${cmd('summarize "https://example.com" --json --verbose')}
|
|
323
523
|
|
|
324
524
|
${heading('Env Vars')}
|
|
@@ -326,9 +526,11 @@ ${heading('Env Vars')}
|
|
|
326
526
|
OPENAI_API_KEY optional (required for openai/... models)
|
|
327
527
|
OPENAI_BASE_URL optional (OpenAI-compatible API endpoint; e.g. OpenRouter)
|
|
328
528
|
OPENROUTER_API_KEY optional (routes openai/... models through OpenRouter)
|
|
329
|
-
OPENROUTER_PROVIDERS optional (provider fallback order, e.g. "groq,google-vertex")
|
|
330
529
|
GEMINI_API_KEY optional (required for google/... models)
|
|
331
530
|
ANTHROPIC_API_KEY optional (required for anthropic/... models)
|
|
531
|
+
CLAUDE_PATH optional (path to Claude CLI binary)
|
|
532
|
+
CODEX_PATH optional (path to Codex CLI binary)
|
|
533
|
+
GEMINI_PATH optional (path to Gemini CLI binary)
|
|
332
534
|
SUMMARIZE_MODEL optional (overrides default model selection)
|
|
333
535
|
FIRECRAWL_API_KEY optional website extraction fallback (Markdown)
|
|
334
536
|
APIFY_API_TOKEN optional YouTube transcript fallback
|
|
@@ -336,16 +538,18 @@ ${heading('Env Vars')}
|
|
|
336
538
|
FAL_KEY optional FAL AI API key for audio transcription
|
|
337
539
|
`);
|
|
338
540
|
}
|
|
339
|
-
async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutMs, fetchImpl, apiKeys,
|
|
541
|
+
async function summarizeWithModelId({ modelId, prompt, maxOutputTokens, timeoutMs, fetchImpl, apiKeys, forceOpenRouter, retries, onRetry, }) {
|
|
340
542
|
const result = await generateTextWithModelId({
|
|
341
543
|
modelId,
|
|
342
544
|
apiKeys,
|
|
343
|
-
|
|
545
|
+
forceOpenRouter,
|
|
344
546
|
prompt,
|
|
345
547
|
temperature: 0,
|
|
346
548
|
maxOutputTokens,
|
|
347
549
|
timeoutMs,
|
|
348
550
|
fetchImpl,
|
|
551
|
+
retries,
|
|
552
|
+
onRetry,
|
|
349
553
|
});
|
|
350
554
|
return {
|
|
351
555
|
text: result.text,
|
|
@@ -362,6 +566,23 @@ function writeVerbose(stderr, verbose, message, color) {
|
|
|
362
566
|
const prefix = ansi('36', VERBOSE_PREFIX, color);
|
|
363
567
|
stderr.write(`${prefix} ${message}\n`);
|
|
364
568
|
}
|
|
569
|
+
function createRetryLogger({ stderr, verbose, color, modelId, }) {
|
|
570
|
+
return (notice) => {
|
|
571
|
+
const message = typeof notice.error === 'string'
|
|
572
|
+
? notice.error
|
|
573
|
+
: notice.error instanceof Error
|
|
574
|
+
? notice.error.message
|
|
575
|
+
: typeof notice.error?.message === 'string'
|
|
576
|
+
? String(notice.error.message)
|
|
577
|
+
: '';
|
|
578
|
+
const reason = /empty summary/i.test(message)
|
|
579
|
+
? 'empty output'
|
|
580
|
+
: /timed out/i.test(message)
|
|
581
|
+
? 'timeout'
|
|
582
|
+
: 'error';
|
|
583
|
+
writeVerbose(stderr, verbose, `LLM ${reason} for ${modelId}; retry ${notice.attempt}/${notice.maxRetries} in ${notice.delayMs}ms.`, color);
|
|
584
|
+
};
|
|
585
|
+
}
|
|
365
586
|
function formatOptionalString(value) {
|
|
366
587
|
if (typeof value === 'string' && value.trim().length > 0) {
|
|
367
588
|
return value.trim();
|
|
@@ -418,40 +639,208 @@ function formatUSD(value) {
|
|
|
418
639
|
return 'n/a';
|
|
419
640
|
return `$${value.toFixed(4)}`;
|
|
420
641
|
}
|
|
642
|
+
function normalizeStreamText(input) {
|
|
643
|
+
return input.replace(/\r\n?/g, '\n');
|
|
644
|
+
}
|
|
645
|
+
function commonPrefixLength(a, b, limit = 4096) {
|
|
646
|
+
const max = Math.min(a.length, b.length, limit);
|
|
647
|
+
let i = 0;
|
|
648
|
+
for (; i < max; i += 1) {
|
|
649
|
+
if (a[i] !== b[i])
|
|
650
|
+
break;
|
|
651
|
+
}
|
|
652
|
+
return i;
|
|
653
|
+
}
|
|
421
654
|
function mergeStreamingChunk(previous, chunk) {
|
|
422
655
|
if (!chunk)
|
|
423
656
|
return { next: previous, appended: '' };
|
|
424
|
-
|
|
425
|
-
|
|
657
|
+
const prev = normalizeStreamText(previous);
|
|
658
|
+
const nextChunk = normalizeStreamText(chunk);
|
|
659
|
+
if (!prev)
|
|
660
|
+
return { next: nextChunk, appended: nextChunk };
|
|
661
|
+
if (nextChunk.startsWith(prev)) {
|
|
662
|
+
return { next: nextChunk, appended: nextChunk.slice(prev.length) };
|
|
663
|
+
}
|
|
664
|
+
if (prev.startsWith(nextChunk)) {
|
|
665
|
+
return { next: prev, appended: '' };
|
|
666
|
+
}
|
|
667
|
+
if (nextChunk.length >= prev.length) {
|
|
668
|
+
const prefixLen = commonPrefixLength(prev, nextChunk);
|
|
669
|
+
if (prefixLen > 0) {
|
|
670
|
+
const minPrefix = Math.max(prev.length - 64, Math.floor(prev.length * 0.9));
|
|
671
|
+
if (prefixLen >= minPrefix) {
|
|
672
|
+
return { next: nextChunk, appended: nextChunk.slice(prefixLen) };
|
|
673
|
+
}
|
|
674
|
+
}
|
|
426
675
|
}
|
|
427
|
-
|
|
676
|
+
const maxOverlap = Math.min(prev.length, nextChunk.length, 2048);
|
|
677
|
+
for (let len = maxOverlap; len > 0; len -= 1) {
|
|
678
|
+
if (prev.slice(-len) === nextChunk.slice(0, len)) {
|
|
679
|
+
return { next: prev + nextChunk.slice(len), appended: nextChunk.slice(len) };
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
return { next: prev + nextChunk, appended: nextChunk };
|
|
428
683
|
}
|
|
429
|
-
function writeFinishLine({ stderr, elapsedMs, model, report, costUsd, color, }) {
|
|
684
|
+
function writeFinishLine({ stderr, elapsedMs, model, report, costUsd, detailed, extraParts, color, }) {
|
|
430
685
|
const promptTokens = sumNumbersOrNull(report.llm.map((row) => row.promptTokens));
|
|
431
686
|
const completionTokens = sumNumbersOrNull(report.llm.map((row) => row.completionTokens));
|
|
432
687
|
const totalTokens = sumNumbersOrNull(report.llm.map((row) => row.totalTokens));
|
|
433
|
-
const
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
688
|
+
const hasAnyTokens = promptTokens !== null || completionTokens !== null || totalTokens !== null;
|
|
689
|
+
const tokensPart = hasAnyTokens
|
|
690
|
+
? `${promptTokens?.toLocaleString() ?? 'unknown'}/${completionTokens?.toLocaleString() ?? 'unknown'}/${totalTokens?.toLocaleString() ?? 'unknown'} (in/out/Σ)`
|
|
691
|
+
: null;
|
|
692
|
+
const summaryParts = [
|
|
693
|
+
formatElapsedMs(elapsedMs),
|
|
694
|
+
costUsd != null ? formatUSD(costUsd) : null,
|
|
437
695
|
model,
|
|
438
|
-
|
|
439
|
-
tokPart,
|
|
696
|
+
tokensPart,
|
|
440
697
|
];
|
|
441
|
-
|
|
442
|
-
|
|
698
|
+
const line1 = summaryParts.filter((part) => typeof part === 'string').join(' · ');
|
|
699
|
+
const totalCalls = report.llm.reduce((sum, row) => sum + row.calls, 0);
|
|
700
|
+
stderr.write('\n');
|
|
701
|
+
stderr.write(`${ansi('1;32', line1, color)}\n`);
|
|
702
|
+
if (detailed) {
|
|
703
|
+
const lenParts = extraParts?.filter((part) => part.startsWith('input=') || part.startsWith('transcript=')) ??
|
|
704
|
+
[];
|
|
705
|
+
const miscParts = extraParts?.filter((part) => !part.startsWith('input=') && !part.startsWith('transcript=')) ??
|
|
706
|
+
[];
|
|
707
|
+
const line2Segments = [];
|
|
708
|
+
if (lenParts.length > 0) {
|
|
709
|
+
line2Segments.push(`len ${lenParts.join(' ')}`);
|
|
710
|
+
}
|
|
711
|
+
line2Segments.push(`calls=${totalCalls.toLocaleString()}`);
|
|
712
|
+
if (report.services.firecrawl.requests > 0 || report.services.apify.requests > 0) {
|
|
713
|
+
const svcParts = [];
|
|
714
|
+
if (report.services.firecrawl.requests > 0) {
|
|
715
|
+
svcParts.push(`firecrawl=${report.services.firecrawl.requests.toLocaleString()}`);
|
|
716
|
+
}
|
|
717
|
+
if (report.services.apify.requests > 0) {
|
|
718
|
+
svcParts.push(`apify=${report.services.apify.requests.toLocaleString()}`);
|
|
719
|
+
}
|
|
720
|
+
line2Segments.push(`svc ${svcParts.join(' ')}`);
|
|
721
|
+
}
|
|
722
|
+
if (miscParts.length > 0) {
|
|
723
|
+
line2Segments.push(...miscParts);
|
|
724
|
+
}
|
|
725
|
+
if (line2Segments.length > 0) {
|
|
726
|
+
stderr.write(`${ansi('0;90', line2Segments.join(' | '), color)}\n`);
|
|
727
|
+
}
|
|
443
728
|
}
|
|
444
|
-
|
|
445
|
-
|
|
729
|
+
}
|
|
730
|
+
function formatCompactCount(value) {
|
|
731
|
+
if (!Number.isFinite(value))
|
|
732
|
+
return 'unknown';
|
|
733
|
+
const abs = Math.abs(value);
|
|
734
|
+
const format = (n, suffix) => {
|
|
735
|
+
const decimals = n >= 10 ? 0 : 1;
|
|
736
|
+
return `${n.toFixed(decimals)}${suffix}`;
|
|
737
|
+
};
|
|
738
|
+
if (abs >= 1_000_000_000)
|
|
739
|
+
return format(value / 1_000_000_000, 'B');
|
|
740
|
+
if (abs >= 1_000_000)
|
|
741
|
+
return format(value / 1_000_000, 'M');
|
|
742
|
+
if (abs >= 10_000)
|
|
743
|
+
return format(value / 1_000, 'k');
|
|
744
|
+
if (abs >= 1_000)
|
|
745
|
+
return `${(value / 1_000).toFixed(1)}k`;
|
|
746
|
+
return String(Math.floor(value));
|
|
747
|
+
}
|
|
748
|
+
function buildDetailedLengthPartsForExtracted(extracted) {
|
|
749
|
+
const parts = [];
|
|
750
|
+
const isYouTube = extracted.siteName === 'YouTube' || /youtube\.com|youtu\.be/i.test(extracted.url);
|
|
751
|
+
if (!isYouTube && !extracted.transcriptCharacters)
|
|
752
|
+
return parts;
|
|
753
|
+
parts.push(`input=${formatCompactCount(extracted.totalCharacters)} chars (~${formatCompactCount(extracted.wordCount)} words)`);
|
|
754
|
+
if (typeof extracted.transcriptCharacters === 'number' && extracted.transcriptCharacters > 0) {
|
|
755
|
+
const wordEstimate = Math.max(0, Math.round(extracted.transcriptCharacters / 6));
|
|
756
|
+
const minutesEstimate = Math.max(1, Math.round(wordEstimate / 160));
|
|
757
|
+
const details = [`${formatCompactCount(extracted.transcriptCharacters)} chars`];
|
|
758
|
+
if (typeof extracted.transcriptLines === 'number' && extracted.transcriptLines > 0) {
|
|
759
|
+
details.push(`${formatCompactCount(extracted.transcriptLines)} lines`);
|
|
760
|
+
}
|
|
761
|
+
parts.push(`transcript=~${minutesEstimate}m (${details.join(', ')})`);
|
|
446
762
|
}
|
|
447
|
-
|
|
448
|
-
stderr.write('\n');
|
|
449
|
-
stderr.write(`${ansi('1;32', line, color)}\n`);
|
|
763
|
+
return parts;
|
|
450
764
|
}
|
|
451
|
-
export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
765
|
+
export async function runCli(argv, { env, fetch, execFile: execFileOverride, stdout, stderr }) {
|
|
452
766
|
;
|
|
453
767
|
globalThis.AI_SDK_LOG_WARNINGS = false;
|
|
454
768
|
const normalizedArgv = argv.filter((arg) => arg !== '--');
|
|
769
|
+
if (normalizedArgv[0]?.toLowerCase() === 'refresh-free') {
|
|
770
|
+
const verbose = normalizedArgv.includes('--verbose') || normalizedArgv.includes('--debug');
|
|
771
|
+
const setDefault = normalizedArgv.includes('--set-default');
|
|
772
|
+
const help = normalizedArgv.includes('--help') ||
|
|
773
|
+
normalizedArgv.includes('-h') ||
|
|
774
|
+
normalizedArgv.includes('help');
|
|
775
|
+
const readArgValue = (name) => {
|
|
776
|
+
const eq = normalizedArgv.find((a) => a.startsWith(`${name}=`));
|
|
777
|
+
if (eq)
|
|
778
|
+
return eq.slice(`${name}=`.length).trim() || null;
|
|
779
|
+
const index = normalizedArgv.indexOf(name);
|
|
780
|
+
if (index === -1)
|
|
781
|
+
return null;
|
|
782
|
+
const next = normalizedArgv[index + 1];
|
|
783
|
+
if (!next || next.startsWith('-'))
|
|
784
|
+
return null;
|
|
785
|
+
return next.trim() || null;
|
|
786
|
+
};
|
|
787
|
+
const runsRaw = readArgValue('--runs');
|
|
788
|
+
const smartRaw = readArgValue('--smart');
|
|
789
|
+
const minParamsRaw = readArgValue('--min-params');
|
|
790
|
+
const maxAgeDaysRaw = readArgValue('--max-age-days');
|
|
791
|
+
const runs = runsRaw ? Number(runsRaw) : 2;
|
|
792
|
+
const smart = smartRaw ? Number(smartRaw) : 3;
|
|
793
|
+
const minParams = (() => {
|
|
794
|
+
if (!minParamsRaw)
|
|
795
|
+
return 27;
|
|
796
|
+
const raw = minParamsRaw.trim().toLowerCase();
|
|
797
|
+
const normalized = raw.endsWith('b') ? raw.slice(0, -1).trim() : raw;
|
|
798
|
+
const value = Number(normalized);
|
|
799
|
+
return value;
|
|
800
|
+
})();
|
|
801
|
+
const maxAgeDays = (() => {
|
|
802
|
+
if (!maxAgeDaysRaw)
|
|
803
|
+
return 180;
|
|
804
|
+
const value = Number(maxAgeDaysRaw.trim());
|
|
805
|
+
return value;
|
|
806
|
+
})();
|
|
807
|
+
if (help) {
|
|
808
|
+
stdout.write(`${[
|
|
809
|
+
'Usage: summarize refresh-free [--runs 2] [--smart 3] [--min-params 27b] [--max-age-days 180] [--set-default] [--verbose]',
|
|
810
|
+
'',
|
|
811
|
+
'Writes ~/.summarize/config.json (models.free) with working OpenRouter :free candidates.',
|
|
812
|
+
'With --set-default: also sets `model` to "free".',
|
|
813
|
+
].join('\n')}\n`);
|
|
814
|
+
return;
|
|
815
|
+
}
|
|
816
|
+
if (!Number.isFinite(runs) || runs < 0)
|
|
817
|
+
throw new Error('--runs must be >= 0');
|
|
818
|
+
if (!Number.isFinite(smart) || smart < 0)
|
|
819
|
+
throw new Error('--smart must be >= 0');
|
|
820
|
+
if (!Number.isFinite(minParams) || minParams < 0)
|
|
821
|
+
throw new Error('--min-params must be >= 0 (e.g. 27b)');
|
|
822
|
+
if (!Number.isFinite(maxAgeDays) || maxAgeDays < 0)
|
|
823
|
+
throw new Error('--max-age-days must be >= 0');
|
|
824
|
+
await refreshFree({
|
|
825
|
+
env,
|
|
826
|
+
fetchImpl: fetch,
|
|
827
|
+
stdout,
|
|
828
|
+
stderr,
|
|
829
|
+
verbose,
|
|
830
|
+
options: {
|
|
831
|
+
runs,
|
|
832
|
+
smart,
|
|
833
|
+
minParamB: minParams,
|
|
834
|
+
maxAgeDays,
|
|
835
|
+
setDefault,
|
|
836
|
+
maxCandidates: 10,
|
|
837
|
+
concurrency: 4,
|
|
838
|
+
timeoutMs: 10_000,
|
|
839
|
+
},
|
|
840
|
+
});
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
const execFileImpl = execFileOverride ?? execFile;
|
|
455
844
|
const version = resolvePackageVersion();
|
|
456
845
|
const program = buildProgram();
|
|
457
846
|
program.configureOutput({
|
|
@@ -477,7 +866,19 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
477
866
|
stdout.write(`${version}\n`);
|
|
478
867
|
return;
|
|
479
868
|
}
|
|
480
|
-
const
|
|
869
|
+
const cliFlagPresent = normalizedArgv.some((arg) => arg === '--cli' || arg.startsWith('--cli='));
|
|
870
|
+
let cliProviderArgRaw = typeof program.opts().cli === 'string' ? program.opts().cli : null;
|
|
871
|
+
let rawInput = program.args[0];
|
|
872
|
+
if (!rawInput && cliFlagPresent && cliProviderArgRaw) {
|
|
873
|
+
try {
|
|
874
|
+
resolveInputTarget(cliProviderArgRaw);
|
|
875
|
+
rawInput = cliProviderArgRaw;
|
|
876
|
+
cliProviderArgRaw = null;
|
|
877
|
+
}
|
|
878
|
+
catch {
|
|
879
|
+
// keep rawInput as-is
|
|
880
|
+
}
|
|
881
|
+
}
|
|
481
882
|
if (!rawInput) {
|
|
482
883
|
throw new Error('Usage: summarize <url-or-file> [--youtube auto|web|apify] [--length 20k] [--max-output-tokens 2k] [--timeout 2m] [--json]');
|
|
483
884
|
}
|
|
@@ -485,33 +886,68 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
485
886
|
const url = inputTarget.kind === 'url' ? inputTarget.url : null;
|
|
486
887
|
const runStartedAtMs = Date.now();
|
|
487
888
|
const youtubeMode = parseYoutubeMode(program.opts().youtube);
|
|
889
|
+
const videoModeExplicitlySet = normalizedArgv.some((arg) => arg === '--video-mode' || arg.startsWith('--video-mode='));
|
|
488
890
|
const lengthArg = parseLengthArg(program.opts().length);
|
|
489
891
|
const maxOutputTokensArg = parseMaxOutputTokensArg(program.opts().maxOutputTokens);
|
|
490
892
|
const timeoutMs = parseDurationMs(program.opts().timeout);
|
|
491
|
-
const
|
|
893
|
+
const retries = parseRetriesArg(program.opts().retries);
|
|
894
|
+
const extractMode = Boolean(program.opts().extract) || Boolean(program.opts().extractOnly);
|
|
492
895
|
const json = Boolean(program.opts().json);
|
|
493
896
|
const streamMode = parseStreamMode(program.opts().stream);
|
|
494
897
|
const renderMode = parseRenderMode(program.opts().render);
|
|
495
|
-
const
|
|
496
|
-
const
|
|
898
|
+
const debug = Boolean(program.opts().debug);
|
|
899
|
+
const verbose = Boolean(program.opts().verbose) || debug;
|
|
900
|
+
const metricsExplicitlySet = normalizedArgv.some((arg) => arg === '--metrics' || arg.startsWith('--metrics='));
|
|
901
|
+
const metricsMode = parseMetricsMode(debug && !metricsExplicitlySet ? 'detailed' : program.opts().metrics);
|
|
497
902
|
const metricsEnabled = metricsMode !== 'off';
|
|
498
903
|
const metricsDetailed = metricsMode === 'detailed';
|
|
499
|
-
const
|
|
904
|
+
const preprocessMode = parsePreprocessMode(program.opts().preprocess);
|
|
905
|
+
const format = parseExtractFormat(program.opts().format);
|
|
500
906
|
const shouldComputeReport = metricsEnabled;
|
|
501
907
|
const isYoutubeUrl = typeof url === 'string' ? /youtube\.com|youtu\.be/i.test(url) : false;
|
|
908
|
+
const firecrawlExplicitlySet = normalizedArgv.some((arg) => arg === '--firecrawl' || arg.startsWith('--firecrawl='));
|
|
909
|
+
const markdownModeExplicitlySet = normalizedArgv.some((arg) => arg === '--markdown-mode' ||
|
|
910
|
+
arg.startsWith('--markdown-mode=') ||
|
|
911
|
+
arg === '--markdown' ||
|
|
912
|
+
arg.startsWith('--markdown='));
|
|
913
|
+
const markdownMode = format === 'markdown'
|
|
914
|
+
? parseMarkdownMode(program.opts().markdownMode ??
|
|
915
|
+
program.opts().markdown ??
|
|
916
|
+
'auto')
|
|
917
|
+
: 'off';
|
|
502
918
|
const requestedFirecrawlMode = parseFirecrawlMode(program.opts().firecrawl);
|
|
503
919
|
const modelArg = typeof program.opts().model === 'string' ? program.opts().model : null;
|
|
920
|
+
const cliProviderArg = typeof cliProviderArgRaw === 'string' && cliProviderArgRaw.trim().length > 0
|
|
921
|
+
? parseCliProviderArg(cliProviderArgRaw)
|
|
922
|
+
: null;
|
|
923
|
+
if (cliFlagPresent && modelArg) {
|
|
924
|
+
throw new Error('Use either --model or --cli (not both).');
|
|
925
|
+
}
|
|
926
|
+
const explicitModelArg = cliProviderArg
|
|
927
|
+
? `cli/${cliProviderArg}`
|
|
928
|
+
: cliFlagPresent
|
|
929
|
+
? 'auto'
|
|
930
|
+
: modelArg;
|
|
504
931
|
const { config, path: configPath } = loadSummarizeConfig({ env });
|
|
932
|
+
const videoMode = parseVideoMode(videoModeExplicitlySet
|
|
933
|
+
? program.opts().videoMode
|
|
934
|
+
: (config?.media?.videoMode ?? program.opts().videoMode));
|
|
935
|
+
const cliEnabledOverride = (() => {
|
|
936
|
+
if (!cliFlagPresent || cliProviderArg)
|
|
937
|
+
return null;
|
|
938
|
+
if (Array.isArray(config?.cli?.enabled))
|
|
939
|
+
return config.cli.enabled;
|
|
940
|
+
return ['gemini', 'claude', 'codex'];
|
|
941
|
+
})();
|
|
942
|
+
const cliConfigForRun = cliEnabledOverride
|
|
943
|
+
? { ...(config?.cli ?? {}), enabled: cliEnabledOverride }
|
|
944
|
+
: config?.cli;
|
|
945
|
+
const configForCli = cliEnabledOverride !== null
|
|
946
|
+
? { ...(config ?? {}), ...(cliConfigForRun ? { cli: cliConfigForRun } : {}) }
|
|
947
|
+
: config;
|
|
505
948
|
const xaiKeyRaw = typeof env.XAI_API_KEY === 'string' ? env.XAI_API_KEY : null;
|
|
506
949
|
const openaiBaseUrl = typeof env.OPENAI_BASE_URL === 'string' ? env.OPENAI_BASE_URL : null;
|
|
507
950
|
const openRouterKeyRaw = typeof env.OPENROUTER_API_KEY === 'string' ? env.OPENROUTER_API_KEY : null;
|
|
508
|
-
const openRouterProvidersRaw = typeof env.OPENROUTER_PROVIDERS === 'string' ? env.OPENROUTER_PROVIDERS : null;
|
|
509
|
-
const openRouterProviders = openRouterProvidersRaw
|
|
510
|
-
? openRouterProvidersRaw
|
|
511
|
-
.split(',')
|
|
512
|
-
.map((p) => p.trim())
|
|
513
|
-
.filter(Boolean)
|
|
514
|
-
: null;
|
|
515
951
|
const openaiKeyRaw = typeof env.OPENAI_API_KEY === 'string' ? env.OPENAI_API_KEY : null;
|
|
516
952
|
const apiKey = typeof openaiBaseUrl === 'string' && /openrouter\.ai/i.test(openaiBaseUrl)
|
|
517
953
|
? (openRouterKeyRaw ?? openaiKeyRaw)
|
|
@@ -533,13 +969,30 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
533
969
|
const xaiApiKey = xaiKeyRaw?.trim() ?? null;
|
|
534
970
|
const googleApiKey = googleKeyRaw?.trim() ?? null;
|
|
535
971
|
const anthropicApiKey = anthropicKeyRaw?.trim() ?? null;
|
|
536
|
-
const openrouterApiKey =
|
|
972
|
+
const openrouterApiKey = (() => {
|
|
973
|
+
const explicit = openRouterKeyRaw?.trim() ?? '';
|
|
974
|
+
if (explicit.length > 0)
|
|
975
|
+
return explicit;
|
|
976
|
+
const baseUrl = openaiBaseUrl?.trim() ?? '';
|
|
977
|
+
const openaiKey = openaiKeyRaw?.trim() ?? '';
|
|
978
|
+
if (baseUrl.length > 0 && /openrouter\.ai/i.test(baseUrl) && openaiKey.length > 0) {
|
|
979
|
+
return openaiKey;
|
|
980
|
+
}
|
|
981
|
+
return null;
|
|
982
|
+
})();
|
|
537
983
|
const openaiTranscriptionKey = openaiKeyRaw?.trim() ?? null;
|
|
538
984
|
const googleConfigured = typeof googleApiKey === 'string' && googleApiKey.length > 0;
|
|
539
985
|
const xaiConfigured = typeof xaiApiKey === 'string' && xaiApiKey.length > 0;
|
|
540
986
|
const anthropicConfigured = typeof anthropicApiKey === 'string' && anthropicApiKey.length > 0;
|
|
541
987
|
const openrouterConfigured = typeof openrouterApiKey === 'string' && openrouterApiKey.length > 0;
|
|
542
|
-
const
|
|
988
|
+
const cliAvailability = resolveCliAvailability({ env, config: configForCli });
|
|
989
|
+
const envForAuto = openrouterApiKey ? { ...env, OPENROUTER_API_KEY: openrouterApiKey } : env;
|
|
990
|
+
if (markdownModeExplicitlySet && format !== 'markdown') {
|
|
991
|
+
throw new Error('--markdown-mode is only supported with --format md');
|
|
992
|
+
}
|
|
993
|
+
if (markdownModeExplicitlySet && inputTarget.kind !== 'url') {
|
|
994
|
+
throw new Error('--markdown-mode is only supported for website URLs');
|
|
995
|
+
}
|
|
543
996
|
const llmCalls = [];
|
|
544
997
|
let firecrawlRequests = 0;
|
|
545
998
|
let apifyRequests = 0;
|
|
@@ -580,10 +1033,13 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
580
1033
|
return null;
|
|
581
1034
|
};
|
|
582
1035
|
const estimateCostUsd = async () => {
|
|
583
|
-
const
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
const
|
|
1036
|
+
const explicitCosts = llmCalls
|
|
1037
|
+
.map((call) => typeof call.costUsd === 'number' && Number.isFinite(call.costUsd) ? call.costUsd : null)
|
|
1038
|
+
.filter((value) => typeof value === 'number');
|
|
1039
|
+
const explicitTotal = explicitCosts.length > 0 ? explicitCosts.reduce((sum, value) => sum + value, 0) : 0;
|
|
1040
|
+
const calls = llmCalls
|
|
1041
|
+
.filter((call) => !(typeof call.costUsd === 'number' && Number.isFinite(call.costUsd)))
|
|
1042
|
+
.map((call) => {
|
|
587
1043
|
const promptTokens = call.usage?.promptTokens ?? null;
|
|
588
1044
|
const completionTokens = call.usage?.completionTokens ?? null;
|
|
589
1045
|
const hasTokens = typeof promptTokens === 'number' &&
|
|
@@ -599,11 +1055,21 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
599
1055
|
: null;
|
|
600
1056
|
return { model: call.model, usage };
|
|
601
1057
|
});
|
|
1058
|
+
if (calls.length === 0) {
|
|
1059
|
+
return explicitCosts.length > 0 ? explicitTotal : null;
|
|
1060
|
+
}
|
|
1061
|
+
const catalog = await getLiteLlmCatalog();
|
|
1062
|
+
if (!catalog) {
|
|
1063
|
+
return explicitCosts.length > 0 ? explicitTotal : null;
|
|
1064
|
+
}
|
|
602
1065
|
const result = await tallyCosts({
|
|
603
1066
|
calls,
|
|
604
1067
|
resolvePricing: (modelId) => resolveLiteLlmPricingForModelId(catalog, modelId),
|
|
605
1068
|
});
|
|
606
|
-
|
|
1069
|
+
const catalogTotal = result.total?.totalUsd ?? null;
|
|
1070
|
+
if (catalogTotal === null && explicitCosts.length === 0)
|
|
1071
|
+
return null;
|
|
1072
|
+
return (catalogTotal ?? 0) + explicitTotal;
|
|
607
1073
|
};
|
|
608
1074
|
const buildReport = async () => {
|
|
609
1075
|
return buildRunMetricsReport({ llmCalls, firecrawlRequests, apifyRequests });
|
|
@@ -625,24 +1091,75 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
625
1091
|
}
|
|
626
1092
|
return fetch(input, init);
|
|
627
1093
|
};
|
|
1094
|
+
const modelMap = (() => {
|
|
1095
|
+
const out = new Map();
|
|
1096
|
+
for (const [name, model] of Object.entries(BUILTIN_MODELS)) {
|
|
1097
|
+
out.set(name.toLowerCase(), { name, model });
|
|
1098
|
+
}
|
|
1099
|
+
const raw = config?.models;
|
|
1100
|
+
if (!raw)
|
|
1101
|
+
return out;
|
|
1102
|
+
for (const [name, model] of Object.entries(raw)) {
|
|
1103
|
+
out.set(name.toLowerCase(), { name, model });
|
|
1104
|
+
}
|
|
1105
|
+
return out;
|
|
1106
|
+
})();
|
|
628
1107
|
const resolvedDefaultModel = (() => {
|
|
629
1108
|
if (typeof env.SUMMARIZE_MODEL === 'string' && env.SUMMARIZE_MODEL.trim().length > 0) {
|
|
630
1109
|
return env.SUMMARIZE_MODEL.trim();
|
|
631
1110
|
}
|
|
632
|
-
|
|
633
|
-
|
|
1111
|
+
const modelFromConfig = config?.model;
|
|
1112
|
+
if (modelFromConfig) {
|
|
1113
|
+
if ('id' in modelFromConfig && typeof modelFromConfig.id === 'string') {
|
|
1114
|
+
const id = modelFromConfig.id.trim();
|
|
1115
|
+
if (id.length > 0)
|
|
1116
|
+
return id;
|
|
1117
|
+
}
|
|
1118
|
+
if ('name' in modelFromConfig && typeof modelFromConfig.name === 'string') {
|
|
1119
|
+
const name = modelFromConfig.name.trim();
|
|
1120
|
+
if (name.length > 0)
|
|
1121
|
+
return name;
|
|
1122
|
+
}
|
|
1123
|
+
if ('mode' in modelFromConfig && modelFromConfig.mode === 'auto')
|
|
1124
|
+
return 'auto';
|
|
1125
|
+
}
|
|
1126
|
+
return 'auto';
|
|
1127
|
+
})();
|
|
1128
|
+
const requestedModelInput = ((explicitModelArg?.trim() ?? '') || resolvedDefaultModel).trim();
|
|
1129
|
+
const requestedModelInputLower = requestedModelInput.toLowerCase();
|
|
1130
|
+
const wantsFreeNamedModel = requestedModelInputLower === 'free';
|
|
1131
|
+
const namedModelMatch = requestedModelInputLower !== 'auto' ? (modelMap.get(requestedModelInputLower) ?? null) : null;
|
|
1132
|
+
const namedModelConfig = namedModelMatch?.model ?? null;
|
|
1133
|
+
const isNamedModelSelection = Boolean(namedModelMatch);
|
|
1134
|
+
const configForModelSelection = isNamedModelSelection && namedModelConfig
|
|
1135
|
+
? { ...(configForCli ?? {}), model: namedModelConfig }
|
|
1136
|
+
: configForCli;
|
|
1137
|
+
const requestedModel = (() => {
|
|
1138
|
+
if (isNamedModelSelection && namedModelConfig) {
|
|
1139
|
+
if ('id' in namedModelConfig)
|
|
1140
|
+
return parseRequestedModelId(namedModelConfig.id);
|
|
1141
|
+
if ('mode' in namedModelConfig && namedModelConfig.mode === 'auto')
|
|
1142
|
+
return { kind: 'auto' };
|
|
1143
|
+
throw new Error(`Invalid model "${namedModelMatch?.name ?? requestedModelInput}": unsupported model config`);
|
|
1144
|
+
}
|
|
1145
|
+
if (requestedModelInputLower !== 'auto' && !requestedModelInput.includes('/')) {
|
|
1146
|
+
throw new Error(`Unknown model "${requestedModelInput}". Define it in ${configPath ?? '~/.summarize/config.json'} under "models", or use a provider-prefixed id like openai/...`);
|
|
634
1147
|
}
|
|
635
|
-
return
|
|
1148
|
+
return parseRequestedModelId(requestedModelInput);
|
|
636
1149
|
})();
|
|
637
|
-
const
|
|
638
|
-
|
|
1150
|
+
const requestedModelLabel = isNamedModelSelection
|
|
1151
|
+
? requestedModelInput
|
|
1152
|
+
: requestedModel.kind === 'auto'
|
|
1153
|
+
? 'auto'
|
|
1154
|
+
: requestedModel.userModelId;
|
|
1155
|
+
const isFallbackModel = requestedModel.kind === 'auto';
|
|
639
1156
|
const verboseColor = supportsColor(stderr, env);
|
|
640
1157
|
const effectiveStreamMode = (() => {
|
|
641
1158
|
if (streamMode !== 'auto')
|
|
642
1159
|
return streamMode;
|
|
643
1160
|
return isRichTty(stdout) ? 'on' : 'off';
|
|
644
1161
|
})();
|
|
645
|
-
const streamingEnabled = effectiveStreamMode === 'on' && !json && !
|
|
1162
|
+
const streamingEnabled = effectiveStreamMode === 'on' && !json && !extractMode;
|
|
646
1163
|
const effectiveRenderMode = (() => {
|
|
647
1164
|
if (renderMode !== 'auto')
|
|
648
1165
|
return renderMode;
|
|
@@ -650,19 +1167,8 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
650
1167
|
return 'plain';
|
|
651
1168
|
return streamingEnabled ? 'md-live' : 'md';
|
|
652
1169
|
})();
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
const completionTokens = sumNumbersOrNull(report.llm.map((row) => row.completionTokens));
|
|
656
|
-
const totalTokens = sumNumbersOrNull(report.llm.map((row) => row.totalTokens));
|
|
657
|
-
for (const row of report.llm) {
|
|
658
|
-
stderr.write(`metrics llm provider=${row.provider} model=${row.model} calls=${row.calls} promptTokens=${row.promptTokens ?? 'unknown'} completionTokens=${row.completionTokens ?? 'unknown'} totalTokens=${row.totalTokens ?? 'unknown'}\n`);
|
|
659
|
-
}
|
|
660
|
-
stderr.write(`metrics firecrawl requests=${report.services.firecrawl.requests}\n`);
|
|
661
|
-
stderr.write(`metrics apify requests=${report.services.apify.requests}\n`);
|
|
662
|
-
stderr.write(`metrics total tok(i/o/t)=${promptTokens ?? 'unknown'}/${completionTokens ?? 'unknown'}/${totalTokens ?? 'unknown'}\n`);
|
|
663
|
-
};
|
|
664
|
-
if (extractOnly && inputTarget.kind !== 'url') {
|
|
665
|
-
throw new Error('--extract-only is only supported for website/YouTube URLs');
|
|
1170
|
+
if (extractMode && inputTarget.kind !== 'url') {
|
|
1171
|
+
throw new Error('--extract is only supported for website/YouTube URLs');
|
|
666
1172
|
}
|
|
667
1173
|
const progressEnabled = isRichTty(stderr) && !verbose && !json;
|
|
668
1174
|
let clearProgressBeforeStdout = null;
|
|
@@ -671,8 +1177,103 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
671
1177
|
clearProgressBeforeStdout = null;
|
|
672
1178
|
fn?.();
|
|
673
1179
|
};
|
|
674
|
-
const
|
|
675
|
-
|
|
1180
|
+
const fixedModelSpec = requestedModel.kind === 'fixed' ? requestedModel : null;
|
|
1181
|
+
const desiredOutputTokens = (() => {
|
|
1182
|
+
if (typeof maxOutputTokensArg === 'number')
|
|
1183
|
+
return maxOutputTokensArg;
|
|
1184
|
+
const targetChars = resolveTargetCharacters(lengthArg);
|
|
1185
|
+
if (!Number.isFinite(targetChars) ||
|
|
1186
|
+
targetChars <= 0 ||
|
|
1187
|
+
targetChars === Number.POSITIVE_INFINITY) {
|
|
1188
|
+
return null;
|
|
1189
|
+
}
|
|
1190
|
+
// Rough heuristic (chars → tokens). Used for auto selection + cost estimation.
|
|
1191
|
+
return Math.max(16, Math.ceil(targetChars / 4));
|
|
1192
|
+
})();
|
|
1193
|
+
const envHasKeyFor = (requiredEnv) => {
|
|
1194
|
+
if (requiredEnv === 'CLI_CLAUDE') {
|
|
1195
|
+
return Boolean(cliAvailability.claude);
|
|
1196
|
+
}
|
|
1197
|
+
if (requiredEnv === 'CLI_CODEX') {
|
|
1198
|
+
return Boolean(cliAvailability.codex);
|
|
1199
|
+
}
|
|
1200
|
+
if (requiredEnv === 'CLI_GEMINI') {
|
|
1201
|
+
return Boolean(cliAvailability.gemini);
|
|
1202
|
+
}
|
|
1203
|
+
if (requiredEnv === 'GEMINI_API_KEY') {
|
|
1204
|
+
return googleConfigured;
|
|
1205
|
+
}
|
|
1206
|
+
if (requiredEnv === 'OPENROUTER_API_KEY') {
|
|
1207
|
+
return openrouterConfigured;
|
|
1208
|
+
}
|
|
1209
|
+
if (requiredEnv === 'OPENAI_API_KEY') {
|
|
1210
|
+
return Boolean(apiKey);
|
|
1211
|
+
}
|
|
1212
|
+
if (requiredEnv === 'XAI_API_KEY') {
|
|
1213
|
+
return Boolean(xaiApiKey);
|
|
1214
|
+
}
|
|
1215
|
+
return Boolean(anthropicApiKey);
|
|
1216
|
+
};
|
|
1217
|
+
const formatMissingModelError = (attempt) => {
|
|
1218
|
+
if (attempt.requiredEnv === 'CLI_CLAUDE') {
|
|
1219
|
+
return `Claude CLI not found for model ${attempt.userModelId}. Install Claude CLI or set CLAUDE_PATH.`;
|
|
1220
|
+
}
|
|
1221
|
+
if (attempt.requiredEnv === 'CLI_CODEX') {
|
|
1222
|
+
return `Codex CLI not found for model ${attempt.userModelId}. Install Codex CLI or set CODEX_PATH.`;
|
|
1223
|
+
}
|
|
1224
|
+
if (attempt.requiredEnv === 'CLI_GEMINI') {
|
|
1225
|
+
return `Gemini CLI not found for model ${attempt.userModelId}. Install Gemini CLI or set GEMINI_PATH.`;
|
|
1226
|
+
}
|
|
1227
|
+
return `Missing ${attempt.requiredEnv} for model ${attempt.userModelId}. Set the env var or choose a different --model.`;
|
|
1228
|
+
};
|
|
1229
|
+
const runSummaryAttempt = async ({ attempt, prompt, allowStreaming, onModelChosen, cli, }) => {
|
|
1230
|
+
onModelChosen?.(attempt.userModelId);
|
|
1231
|
+
if (attempt.transport === 'cli') {
|
|
1232
|
+
const cliPrompt = typeof prompt === 'string' ? prompt : (cli?.promptOverride ?? null);
|
|
1233
|
+
if (!cliPrompt) {
|
|
1234
|
+
throw new Error('CLI models require a text prompt (no binary attachments).');
|
|
1235
|
+
}
|
|
1236
|
+
if (!attempt.cliProvider) {
|
|
1237
|
+
throw new Error(`Missing CLI provider for model ${attempt.userModelId}.`);
|
|
1238
|
+
}
|
|
1239
|
+
if (isCliDisabled(attempt.cliProvider, cliConfigForRun)) {
|
|
1240
|
+
throw new Error(`CLI provider ${attempt.cliProvider} is disabled by cli.enabled. Update your config to enable it.`);
|
|
1241
|
+
}
|
|
1242
|
+
const result = await runCliModel({
|
|
1243
|
+
provider: attempt.cliProvider,
|
|
1244
|
+
prompt: cliPrompt,
|
|
1245
|
+
model: attempt.cliModel ?? null,
|
|
1246
|
+
allowTools: Boolean(cli?.allowTools),
|
|
1247
|
+
timeoutMs,
|
|
1248
|
+
env,
|
|
1249
|
+
execFileImpl,
|
|
1250
|
+
config: cliConfigForRun ?? null,
|
|
1251
|
+
cwd: cli?.cwd,
|
|
1252
|
+
extraArgs: cli?.extraArgsByProvider?.[attempt.cliProvider],
|
|
1253
|
+
});
|
|
1254
|
+
const summary = result.text.trim();
|
|
1255
|
+
if (!summary)
|
|
1256
|
+
throw new Error('CLI returned an empty summary');
|
|
1257
|
+
if (result.usage || typeof result.costUsd === 'number') {
|
|
1258
|
+
llmCalls.push({
|
|
1259
|
+
provider: 'cli',
|
|
1260
|
+
model: attempt.userModelId,
|
|
1261
|
+
usage: result.usage ?? null,
|
|
1262
|
+
costUsd: result.costUsd ?? null,
|
|
1263
|
+
purpose: 'summary',
|
|
1264
|
+
});
|
|
1265
|
+
}
|
|
1266
|
+
return {
|
|
1267
|
+
summary,
|
|
1268
|
+
summaryAlreadyPrinted: false,
|
|
1269
|
+
modelMeta: { provider: 'cli', canonical: attempt.userModelId },
|
|
1270
|
+
maxOutputTokensForCall: null,
|
|
1271
|
+
};
|
|
1272
|
+
}
|
|
1273
|
+
if (!attempt.llmModelId) {
|
|
1274
|
+
throw new Error(`Missing model id for ${attempt.userModelId}.`);
|
|
1275
|
+
}
|
|
1276
|
+
const parsedModel = parseGatewayStyleModelId(attempt.llmModelId);
|
|
676
1277
|
const apiKeysForLlm = {
|
|
677
1278
|
xaiApiKey,
|
|
678
1279
|
openaiApiKey: apiKey,
|
|
@@ -680,28 +1281,6 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
680
1281
|
anthropicApiKey: anthropicConfigured ? anthropicApiKey : null,
|
|
681
1282
|
openrouterApiKey: openrouterConfigured ? openrouterApiKey : null,
|
|
682
1283
|
};
|
|
683
|
-
const requiredKeyEnv = parsedModel.provider === 'xai'
|
|
684
|
-
? 'XAI_API_KEY'
|
|
685
|
-
: parsedModel.provider === 'google'
|
|
686
|
-
? 'GEMINI_API_KEY (or GOOGLE_GENERATIVE_AI_API_KEY / GOOGLE_API_KEY)'
|
|
687
|
-
: parsedModel.provider === 'anthropic'
|
|
688
|
-
? 'ANTHROPIC_API_KEY'
|
|
689
|
-
: 'OPENAI_API_KEY (or OPENROUTER_API_KEY)';
|
|
690
|
-
const hasRequiredKey = parsedModel.provider === 'xai'
|
|
691
|
-
? Boolean(xaiApiKey)
|
|
692
|
-
: parsedModel.provider === 'google'
|
|
693
|
-
? googleConfigured
|
|
694
|
-
: parsedModel.provider === 'anthropic'
|
|
695
|
-
? anthropicConfigured
|
|
696
|
-
: Boolean(apiKey) || openrouterConfigured;
|
|
697
|
-
if (!hasRequiredKey) {
|
|
698
|
-
throw new Error(`Missing ${requiredKeyEnv} for model ${parsedModel.canonical}. Set the env var or choose a different --model.`);
|
|
699
|
-
}
|
|
700
|
-
assertProviderSupportsAttachment({
|
|
701
|
-
provider: parsedModel.provider,
|
|
702
|
-
modelId: parsedModel.canonical,
|
|
703
|
-
attachment: { part: attachment.part, mediaType: attachment.mediaType },
|
|
704
|
-
});
|
|
705
1284
|
const modelResolution = await resolveModelIdForLlmCall({
|
|
706
1285
|
parsedModel,
|
|
707
1286
|
apiKeys: { googleApiKey: apiKeysForLlm.googleApiKey },
|
|
@@ -711,203 +1290,200 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
711
1290
|
if (modelResolution.note && verbose) {
|
|
712
1291
|
writeVerbose(stderr, verbose, modelResolution.note, verboseColor);
|
|
713
1292
|
}
|
|
714
|
-
const
|
|
715
|
-
const
|
|
716
|
-
const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
|
|
1293
|
+
const parsedModelEffective = parseGatewayStyleModelId(modelResolution.modelId);
|
|
1294
|
+
const streamingEnabledForCall = allowStreaming && streamingEnabled && !modelResolution.forceStreamOff;
|
|
717
1295
|
const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
|
|
718
|
-
const textContent = getTextContentFromAttachment(attachment);
|
|
719
|
-
if (textContent && textContent.bytes > MAX_TEXT_BYTES_DEFAULT) {
|
|
720
|
-
throw new Error(`Text file too large (${formatBytes(textContent.bytes)}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
|
|
721
|
-
}
|
|
722
|
-
const summaryLengthTarget = lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters };
|
|
723
|
-
const promptText = buildFileSummaryPrompt({
|
|
724
|
-
filename: attachment.filename,
|
|
725
|
-
mediaType: attachment.mediaType,
|
|
726
|
-
summaryLength: summaryLengthTarget,
|
|
727
|
-
contentLength: textContent?.content.length ?? null,
|
|
728
|
-
});
|
|
729
|
-
const promptPayload = buildAssetPromptPayload({ promptText, attachment, textContent });
|
|
730
1296
|
const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
|
|
731
1297
|
if (typeof maxInputTokensForCall === 'number' &&
|
|
732
1298
|
Number.isFinite(maxInputTokensForCall) &&
|
|
733
1299
|
maxInputTokensForCall > 0 &&
|
|
734
|
-
typeof
|
|
735
|
-
const tokenCount = countTokens(
|
|
1300
|
+
typeof prompt === 'string') {
|
|
1301
|
+
const tokenCount = countTokens(prompt);
|
|
736
1302
|
if (tokenCount > maxInputTokensForCall) {
|
|
737
1303
|
throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
|
|
738
1304
|
}
|
|
739
1305
|
}
|
|
1306
|
+
if (!streamingEnabledForCall) {
|
|
1307
|
+
const result = await summarizeWithModelId({
|
|
1308
|
+
modelId: parsedModelEffective.canonical,
|
|
1309
|
+
prompt,
|
|
1310
|
+
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1311
|
+
timeoutMs,
|
|
1312
|
+
fetchImpl: trackedFetch,
|
|
1313
|
+
apiKeys: apiKeysForLlm,
|
|
1314
|
+
forceOpenRouter: attempt.forceOpenRouter,
|
|
1315
|
+
retries,
|
|
1316
|
+
onRetry: createRetryLogger({
|
|
1317
|
+
stderr,
|
|
1318
|
+
verbose,
|
|
1319
|
+
color: verboseColor,
|
|
1320
|
+
modelId: parsedModelEffective.canonical,
|
|
1321
|
+
}),
|
|
1322
|
+
});
|
|
1323
|
+
llmCalls.push({
|
|
1324
|
+
provider: result.provider,
|
|
1325
|
+
model: result.canonicalModelId,
|
|
1326
|
+
usage: result.usage,
|
|
1327
|
+
purpose: 'summary',
|
|
1328
|
+
});
|
|
1329
|
+
const summary = result.text.trim();
|
|
1330
|
+
if (!summary)
|
|
1331
|
+
throw new Error('LLM returned an empty summary');
|
|
1332
|
+
return {
|
|
1333
|
+
summary,
|
|
1334
|
+
summaryAlreadyPrinted: false,
|
|
1335
|
+
modelMeta: {
|
|
1336
|
+
provider: parsedModelEffective.provider,
|
|
1337
|
+
canonical: parsedModelEffective.canonical,
|
|
1338
|
+
},
|
|
1339
|
+
maxOutputTokensForCall: maxOutputTokensForCall ?? null,
|
|
1340
|
+
};
|
|
1341
|
+
}
|
|
740
1342
|
const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
|
|
741
1343
|
const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
|
|
742
1344
|
const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
|
|
743
1345
|
let summaryAlreadyPrinted = false;
|
|
744
1346
|
let summary = '';
|
|
745
1347
|
let getLastStreamError = null;
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
1348
|
+
let streamResult = null;
|
|
1349
|
+
try {
|
|
1350
|
+
streamResult = await streamTextWithModelId({
|
|
1351
|
+
modelId: parsedModelEffective.canonical,
|
|
1352
|
+
apiKeys: apiKeysForLlm,
|
|
1353
|
+
forceOpenRouter: attempt.forceOpenRouter,
|
|
1354
|
+
prompt,
|
|
1355
|
+
temperature: 0,
|
|
1356
|
+
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1357
|
+
timeoutMs,
|
|
1358
|
+
fetchImpl: trackedFetch,
|
|
1359
|
+
});
|
|
1360
|
+
}
|
|
1361
|
+
catch (error) {
|
|
1362
|
+
if (isStreamingTimeoutError(error)) {
|
|
1363
|
+
writeVerbose(stderr, verbose, `Streaming timed out for ${parsedModelEffective.canonical}; falling back to non-streaming.`, verboseColor);
|
|
1364
|
+
const result = await summarizeWithModelId({
|
|
750
1365
|
modelId: parsedModelEffective.canonical,
|
|
751
|
-
|
|
752
|
-
openrouter: openrouterOptions,
|
|
753
|
-
prompt: promptPayload,
|
|
754
|
-
temperature: 0,
|
|
1366
|
+
prompt,
|
|
755
1367
|
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
756
1368
|
timeoutMs,
|
|
757
1369
|
fetchImpl: trackedFetch,
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
prompt: promptPayload,
|
|
766
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
767
|
-
timeoutMs,
|
|
768
|
-
fetchImpl: trackedFetch,
|
|
769
|
-
apiKeys: apiKeysForLlm,
|
|
770
|
-
openrouter: openrouterOptions,
|
|
771
|
-
});
|
|
772
|
-
llmCalls.push({
|
|
773
|
-
provider: result.provider,
|
|
774
|
-
model: result.canonicalModelId,
|
|
775
|
-
usage: result.usage,
|
|
776
|
-
purpose: 'summary',
|
|
777
|
-
});
|
|
778
|
-
summary = result.text;
|
|
779
|
-
streamResult = null;
|
|
780
|
-
}
|
|
781
|
-
else if (parsedModelEffective.provider === 'google' &&
|
|
782
|
-
isGoogleStreamingUnsupportedError(error)) {
|
|
783
|
-
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
784
|
-
const result = await summarizeWithModelId({
|
|
1370
|
+
apiKeys: apiKeysForLlm,
|
|
1371
|
+
forceOpenRouter: attempt.forceOpenRouter,
|
|
1372
|
+
retries,
|
|
1373
|
+
onRetry: createRetryLogger({
|
|
1374
|
+
stderr,
|
|
1375
|
+
verbose,
|
|
1376
|
+
color: verboseColor,
|
|
785
1377
|
modelId: parsedModelEffective.canonical,
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
timeoutMs,
|
|
789
|
-
fetchImpl: trackedFetch,
|
|
790
|
-
apiKeys: apiKeysForLlm,
|
|
791
|
-
openrouter: openrouterOptions,
|
|
792
|
-
});
|
|
793
|
-
llmCalls.push({
|
|
794
|
-
provider: result.provider,
|
|
795
|
-
model: result.canonicalModelId,
|
|
796
|
-
usage: result.usage,
|
|
797
|
-
purpose: 'summary',
|
|
798
|
-
});
|
|
799
|
-
summary = result.text;
|
|
800
|
-
streamResult = null;
|
|
801
|
-
}
|
|
802
|
-
else if (isUnsupportedAttachmentError(error)) {
|
|
803
|
-
throw new Error(`Model ${parsedModel.canonical} does not support attaching files of type ${attachment.mediaType}. Try a different --model (e.g. google/gemini-3-flash-preview).`, { cause: error });
|
|
804
|
-
}
|
|
805
|
-
else {
|
|
806
|
-
throw error;
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
if (streamResult) {
|
|
810
|
-
getLastStreamError = streamResult.lastError;
|
|
811
|
-
let streamed = '';
|
|
812
|
-
const liveRenderer = shouldLiveRenderSummary
|
|
813
|
-
? createLiveRenderer({
|
|
814
|
-
write: (chunk) => {
|
|
815
|
-
clearProgressForStdout();
|
|
816
|
-
stdout.write(chunk);
|
|
817
|
-
},
|
|
818
|
-
width: markdownRenderWidth(stdout, env),
|
|
819
|
-
renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
|
|
820
|
-
width: markdownRenderWidth(stdout, env),
|
|
821
|
-
wrap: true,
|
|
822
|
-
color: supportsColor(stdout, env),
|
|
823
|
-
}),
|
|
824
|
-
})
|
|
825
|
-
: null;
|
|
826
|
-
let lastFrameAtMs = 0;
|
|
827
|
-
try {
|
|
828
|
-
try {
|
|
829
|
-
let cleared = false;
|
|
830
|
-
for await (const delta of streamResult.textStream) {
|
|
831
|
-
if (!cleared) {
|
|
832
|
-
clearProgressForStdout();
|
|
833
|
-
cleared = true;
|
|
834
|
-
}
|
|
835
|
-
const merged = mergeStreamingChunk(streamed, delta);
|
|
836
|
-
streamed = merged.next;
|
|
837
|
-
if (shouldStreamSummaryToStdout) {
|
|
838
|
-
if (merged.appended)
|
|
839
|
-
stdout.write(merged.appended);
|
|
840
|
-
continue;
|
|
841
|
-
}
|
|
842
|
-
if (liveRenderer) {
|
|
843
|
-
const now = Date.now();
|
|
844
|
-
const due = now - lastFrameAtMs >= 120;
|
|
845
|
-
const hasNewline = delta.includes('\n');
|
|
846
|
-
if (hasNewline || due) {
|
|
847
|
-
liveRenderer.render(streamed);
|
|
848
|
-
lastFrameAtMs = now;
|
|
849
|
-
}
|
|
850
|
-
}
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
catch (error) {
|
|
854
|
-
if (isUnsupportedAttachmentError(error)) {
|
|
855
|
-
throw new Error(`Model ${parsedModel.canonical} does not support attaching files of type ${attachment.mediaType}. Try a different --model (e.g. google/gemini-3-flash-preview).`, { cause: error });
|
|
856
|
-
}
|
|
857
|
-
throw error;
|
|
858
|
-
}
|
|
859
|
-
const trimmed = streamed.trim();
|
|
860
|
-
streamed = trimmed;
|
|
861
|
-
if (liveRenderer) {
|
|
862
|
-
liveRenderer.render(trimmed);
|
|
863
|
-
summaryAlreadyPrinted = true;
|
|
864
|
-
}
|
|
865
|
-
}
|
|
866
|
-
finally {
|
|
867
|
-
liveRenderer?.finish();
|
|
868
|
-
}
|
|
869
|
-
const usage = await streamResult.usage;
|
|
1378
|
+
}),
|
|
1379
|
+
});
|
|
870
1380
|
llmCalls.push({
|
|
871
|
-
provider:
|
|
872
|
-
model:
|
|
873
|
-
usage,
|
|
1381
|
+
provider: result.provider,
|
|
1382
|
+
model: result.canonicalModelId,
|
|
1383
|
+
usage: result.usage,
|
|
874
1384
|
purpose: 'summary',
|
|
875
1385
|
});
|
|
876
|
-
summary =
|
|
877
|
-
|
|
878
|
-
if (!streamed.endsWith('\n')) {
|
|
879
|
-
stdout.write('\n');
|
|
880
|
-
}
|
|
881
|
-
summaryAlreadyPrinted = true;
|
|
882
|
-
}
|
|
1386
|
+
summary = result.text;
|
|
1387
|
+
streamResult = null;
|
|
883
1388
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
result = await summarizeWithModelId({
|
|
1389
|
+
else if (parsedModelEffective.provider === 'google' &&
|
|
1390
|
+
isGoogleStreamingUnsupportedError(error)) {
|
|
1391
|
+
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
1392
|
+
const result = await summarizeWithModelId({
|
|
889
1393
|
modelId: parsedModelEffective.canonical,
|
|
890
|
-
prompt
|
|
1394
|
+
prompt,
|
|
891
1395
|
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
892
1396
|
timeoutMs,
|
|
893
1397
|
fetchImpl: trackedFetch,
|
|
894
1398
|
apiKeys: apiKeysForLlm,
|
|
895
|
-
|
|
1399
|
+
forceOpenRouter: attempt.forceOpenRouter,
|
|
1400
|
+
retries,
|
|
1401
|
+
onRetry: createRetryLogger({
|
|
1402
|
+
stderr,
|
|
1403
|
+
verbose,
|
|
1404
|
+
color: verboseColor,
|
|
1405
|
+
modelId: parsedModelEffective.canonical,
|
|
1406
|
+
}),
|
|
1407
|
+
});
|
|
1408
|
+
llmCalls.push({
|
|
1409
|
+
provider: result.provider,
|
|
1410
|
+
model: result.canonicalModelId,
|
|
1411
|
+
usage: result.usage,
|
|
1412
|
+
purpose: 'summary',
|
|
896
1413
|
});
|
|
1414
|
+
summary = result.text;
|
|
1415
|
+
streamResult = null;
|
|
897
1416
|
}
|
|
898
|
-
|
|
899
|
-
if (isUnsupportedAttachmentError(error)) {
|
|
900
|
-
throw new Error(`Model ${parsedModel.canonical} does not support attaching files of type ${attachment.mediaType}. Try a different --model (e.g. google/gemini-3-flash-preview).`, { cause: error });
|
|
901
|
-
}
|
|
1417
|
+
else {
|
|
902
1418
|
throw error;
|
|
903
1419
|
}
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
1420
|
+
}
|
|
1421
|
+
if (streamResult) {
|
|
1422
|
+
getLastStreamError = streamResult.lastError;
|
|
1423
|
+
let streamed = '';
|
|
1424
|
+
const liveRenderer = shouldLiveRenderSummary
|
|
1425
|
+
? createLiveRenderer({
|
|
1426
|
+
write: (chunk) => {
|
|
1427
|
+
clearProgressForStdout();
|
|
1428
|
+
stdout.write(chunk);
|
|
1429
|
+
},
|
|
1430
|
+
width: markdownRenderWidth(stdout, env),
|
|
1431
|
+
renderFrame: (markdown) => renderMarkdownAnsi(markdown, {
|
|
1432
|
+
width: markdownRenderWidth(stdout, env),
|
|
1433
|
+
wrap: true,
|
|
1434
|
+
color: supportsColor(stdout, env),
|
|
1435
|
+
}),
|
|
1436
|
+
})
|
|
1437
|
+
: null;
|
|
1438
|
+
let lastFrameAtMs = 0;
|
|
1439
|
+
try {
|
|
1440
|
+
let cleared = false;
|
|
1441
|
+
for await (const delta of streamResult.textStream) {
|
|
1442
|
+
const merged = mergeStreamingChunk(streamed, delta);
|
|
1443
|
+
streamed = merged.next;
|
|
1444
|
+
if (shouldStreamSummaryToStdout) {
|
|
1445
|
+
if (!cleared) {
|
|
1446
|
+
clearProgressForStdout();
|
|
1447
|
+
cleared = true;
|
|
1448
|
+
}
|
|
1449
|
+
if (merged.appended)
|
|
1450
|
+
stdout.write(merged.appended);
|
|
1451
|
+
continue;
|
|
1452
|
+
}
|
|
1453
|
+
if (liveRenderer) {
|
|
1454
|
+
const now = Date.now();
|
|
1455
|
+
const due = now - lastFrameAtMs >= 120;
|
|
1456
|
+
const hasNewline = delta.includes('\n');
|
|
1457
|
+
if (hasNewline || due) {
|
|
1458
|
+
liveRenderer.render(streamed);
|
|
1459
|
+
lastFrameAtMs = now;
|
|
1460
|
+
}
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
const trimmed = streamed.trim();
|
|
1464
|
+
streamed = trimmed;
|
|
1465
|
+
if (liveRenderer) {
|
|
1466
|
+
liveRenderer.render(trimmed);
|
|
1467
|
+
summaryAlreadyPrinted = true;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
finally {
|
|
1471
|
+
liveRenderer?.finish();
|
|
1472
|
+
}
|
|
1473
|
+
const usage = await streamResult.usage;
|
|
1474
|
+
llmCalls.push({
|
|
1475
|
+
provider: streamResult.provider,
|
|
1476
|
+
model: streamResult.canonicalModelId,
|
|
1477
|
+
usage,
|
|
1478
|
+
purpose: 'summary',
|
|
1479
|
+
});
|
|
1480
|
+
summary = streamed;
|
|
1481
|
+
if (shouldStreamSummaryToStdout) {
|
|
1482
|
+
if (!streamed.endsWith('\n')) {
|
|
1483
|
+
stdout.write('\n');
|
|
1484
|
+
}
|
|
1485
|
+
summaryAlreadyPrinted = true;
|
|
1486
|
+
}
|
|
911
1487
|
}
|
|
912
1488
|
summary = summary.trim();
|
|
913
1489
|
if (summary.length === 0) {
|
|
@@ -917,6 +1493,326 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
917
1493
|
}
|
|
918
1494
|
throw new Error('LLM returned an empty summary');
|
|
919
1495
|
}
|
|
1496
|
+
return {
|
|
1497
|
+
summary,
|
|
1498
|
+
summaryAlreadyPrinted,
|
|
1499
|
+
modelMeta: {
|
|
1500
|
+
provider: parsedModelEffective.provider,
|
|
1501
|
+
canonical: parsedModelEffective.canonical,
|
|
1502
|
+
},
|
|
1503
|
+
maxOutputTokensForCall: maxOutputTokensForCall ?? null,
|
|
1504
|
+
};
|
|
1505
|
+
};
|
|
1506
|
+
const writeViaFooter = (parts) => {
|
|
1507
|
+
if (json)
|
|
1508
|
+
return;
|
|
1509
|
+
const filtered = parts.map((p) => p.trim()).filter(Boolean);
|
|
1510
|
+
if (filtered.length === 0)
|
|
1511
|
+
return;
|
|
1512
|
+
clearProgressForStdout();
|
|
1513
|
+
stderr.write(`${ansi('2', `via ${filtered.join(', ')}`, verboseColor)}\n`);
|
|
1514
|
+
};
|
|
1515
|
+
const summarizeAsset = async ({ sourceKind, sourceLabel, attachment, onModelChosen, }) => {
|
|
1516
|
+
const textContent = getTextContentFromAttachment(attachment);
|
|
1517
|
+
if (textContent && textContent.bytes > MAX_TEXT_BYTES_DEFAULT) {
|
|
1518
|
+
throw new Error(`Text file too large (${formatBytes(textContent.bytes)}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
|
|
1519
|
+
}
|
|
1520
|
+
const fileBytes = getFileBytesFromAttachment(attachment);
|
|
1521
|
+
const canPreprocessWithMarkitdown = format === 'markdown' &&
|
|
1522
|
+
preprocessMode !== 'off' &&
|
|
1523
|
+
hasUvxCli(env) &&
|
|
1524
|
+
attachment.part.type === 'file' &&
|
|
1525
|
+
fileBytes !== null &&
|
|
1526
|
+
shouldMarkitdownConvertMediaType(attachment.mediaType);
|
|
1527
|
+
const summaryLengthTarget = lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters };
|
|
1528
|
+
let promptText = '';
|
|
1529
|
+
const assetFooterParts = [];
|
|
1530
|
+
const buildAttachmentPromptPayload = () => {
|
|
1531
|
+
promptText = buildFileSummaryPrompt({
|
|
1532
|
+
filename: attachment.filename,
|
|
1533
|
+
mediaType: attachment.mediaType,
|
|
1534
|
+
summaryLength: summaryLengthTarget,
|
|
1535
|
+
contentLength: textContent?.content.length ?? null,
|
|
1536
|
+
});
|
|
1537
|
+
return buildAssetPromptPayload({ promptText, attachment, textContent });
|
|
1538
|
+
};
|
|
1539
|
+
const buildMarkitdownPromptPayload = (markdown) => {
|
|
1540
|
+
promptText = buildFileTextSummaryPrompt({
|
|
1541
|
+
filename: attachment.filename,
|
|
1542
|
+
originalMediaType: attachment.mediaType,
|
|
1543
|
+
contentMediaType: 'text/markdown',
|
|
1544
|
+
summaryLength: summaryLengthTarget,
|
|
1545
|
+
contentLength: markdown.length,
|
|
1546
|
+
});
|
|
1547
|
+
return `${promptText}\n\n---\n\n${markdown}`.trim();
|
|
1548
|
+
};
|
|
1549
|
+
let preprocessedMarkdown = null;
|
|
1550
|
+
let usingPreprocessedMarkdown = false;
|
|
1551
|
+
if (preprocessMode === 'always' && canPreprocessWithMarkitdown) {
|
|
1552
|
+
if (!fileBytes) {
|
|
1553
|
+
throw new Error('Internal error: missing file bytes for markitdown preprocessing');
|
|
1554
|
+
}
|
|
1555
|
+
try {
|
|
1556
|
+
preprocessedMarkdown = await convertToMarkdownWithMarkitdown({
|
|
1557
|
+
bytes: fileBytes,
|
|
1558
|
+
filenameHint: attachment.filename,
|
|
1559
|
+
mediaTypeHint: attachment.mediaType,
|
|
1560
|
+
uvxCommand: env.UVX_PATH,
|
|
1561
|
+
timeoutMs,
|
|
1562
|
+
env,
|
|
1563
|
+
execFileImpl,
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
catch (error) {
|
|
1567
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1568
|
+
throw new Error(`Failed to preprocess ${attachment.mediaType} with markitdown: ${message} (disable with --preprocess off).`);
|
|
1569
|
+
}
|
|
1570
|
+
if (Buffer.byteLength(preprocessedMarkdown, 'utf8') > MAX_TEXT_BYTES_DEFAULT) {
|
|
1571
|
+
throw new Error(`Preprocessed Markdown too large (${formatBytes(Buffer.byteLength(preprocessedMarkdown, 'utf8'))}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
|
|
1572
|
+
}
|
|
1573
|
+
usingPreprocessedMarkdown = true;
|
|
1574
|
+
assetFooterParts.push(`markitdown(${attachment.mediaType})`);
|
|
1575
|
+
}
|
|
1576
|
+
let promptPayload = buildAttachmentPromptPayload();
|
|
1577
|
+
if (usingPreprocessedMarkdown) {
|
|
1578
|
+
if (!preprocessedMarkdown) {
|
|
1579
|
+
throw new Error('Internal error: missing markitdown content for preprocessing');
|
|
1580
|
+
}
|
|
1581
|
+
promptPayload = buildMarkitdownPromptPayload(preprocessedMarkdown);
|
|
1582
|
+
}
|
|
1583
|
+
if (!usingPreprocessedMarkdown &&
|
|
1584
|
+
fixedModelSpec &&
|
|
1585
|
+
fixedModelSpec.transport !== 'cli' &&
|
|
1586
|
+
preprocessMode !== 'off') {
|
|
1587
|
+
const fixedParsed = parseGatewayStyleModelId(fixedModelSpec.llmModelId);
|
|
1588
|
+
try {
|
|
1589
|
+
assertProviderSupportsAttachment({
|
|
1590
|
+
provider: fixedParsed.provider,
|
|
1591
|
+
modelId: fixedModelSpec.userModelId,
|
|
1592
|
+
attachment: { part: attachment.part, mediaType: attachment.mediaType },
|
|
1593
|
+
});
|
|
1594
|
+
}
|
|
1595
|
+
catch (error) {
|
|
1596
|
+
if (!canPreprocessWithMarkitdown) {
|
|
1597
|
+
if (format === 'markdown' &&
|
|
1598
|
+
attachment.part.type === 'file' &&
|
|
1599
|
+
shouldMarkitdownConvertMediaType(attachment.mediaType) &&
|
|
1600
|
+
!hasUvxCli(env)) {
|
|
1601
|
+
throw withUvxTip(error, env);
|
|
1602
|
+
}
|
|
1603
|
+
throw error;
|
|
1604
|
+
}
|
|
1605
|
+
if (!fileBytes) {
|
|
1606
|
+
throw new Error('Internal error: missing file bytes for markitdown preprocessing');
|
|
1607
|
+
}
|
|
1608
|
+
try {
|
|
1609
|
+
preprocessedMarkdown = await convertToMarkdownWithMarkitdown({
|
|
1610
|
+
bytes: fileBytes,
|
|
1611
|
+
filenameHint: attachment.filename,
|
|
1612
|
+
mediaTypeHint: attachment.mediaType,
|
|
1613
|
+
uvxCommand: env.UVX_PATH,
|
|
1614
|
+
timeoutMs,
|
|
1615
|
+
env,
|
|
1616
|
+
execFileImpl,
|
|
1617
|
+
});
|
|
1618
|
+
}
|
|
1619
|
+
catch (markitdownError) {
|
|
1620
|
+
if (preprocessMode === 'auto') {
|
|
1621
|
+
throw error;
|
|
1622
|
+
}
|
|
1623
|
+
const message = markitdownError instanceof Error ? markitdownError.message : String(markitdownError);
|
|
1624
|
+
throw new Error(`Failed to preprocess ${attachment.mediaType} with markitdown: ${message} (disable with --preprocess off).`);
|
|
1625
|
+
}
|
|
1626
|
+
if (Buffer.byteLength(preprocessedMarkdown, 'utf8') > MAX_TEXT_BYTES_DEFAULT) {
|
|
1627
|
+
throw new Error(`Preprocessed Markdown too large (${formatBytes(Buffer.byteLength(preprocessedMarkdown, 'utf8'))}). Limit is ${formatBytes(MAX_TEXT_BYTES_DEFAULT)}.`);
|
|
1628
|
+
}
|
|
1629
|
+
usingPreprocessedMarkdown = true;
|
|
1630
|
+
assetFooterParts.push(`markitdown(${attachment.mediaType})`);
|
|
1631
|
+
promptPayload = buildMarkitdownPromptPayload(preprocessedMarkdown);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1634
|
+
const promptTokensForAuto = typeof promptPayload === 'string' ? countTokens(promptPayload) : null;
|
|
1635
|
+
const lowerMediaType = attachment.mediaType.toLowerCase();
|
|
1636
|
+
const kind = lowerMediaType.startsWith('video/')
|
|
1637
|
+
? 'video'
|
|
1638
|
+
: lowerMediaType.startsWith('image/')
|
|
1639
|
+
? 'image'
|
|
1640
|
+
: textContent
|
|
1641
|
+
? 'text'
|
|
1642
|
+
: 'file';
|
|
1643
|
+
const requiresVideoUnderstanding = kind === 'video' && videoMode !== 'transcript';
|
|
1644
|
+
const attempts = await (async () => {
|
|
1645
|
+
if (isFallbackModel) {
|
|
1646
|
+
const catalog = await getLiteLlmCatalog();
|
|
1647
|
+
const all = buildAutoModelAttempts({
|
|
1648
|
+
kind,
|
|
1649
|
+
promptTokens: promptTokensForAuto,
|
|
1650
|
+
desiredOutputTokens,
|
|
1651
|
+
requiresVideoUnderstanding,
|
|
1652
|
+
env: envForAuto,
|
|
1653
|
+
config: configForModelSelection,
|
|
1654
|
+
catalog,
|
|
1655
|
+
openrouterProvidersFromEnv: null,
|
|
1656
|
+
cliAvailability,
|
|
1657
|
+
});
|
|
1658
|
+
const mapped = all.map((attempt) => {
|
|
1659
|
+
if (attempt.transport !== 'cli')
|
|
1660
|
+
return attempt;
|
|
1661
|
+
const parsed = parseCliUserModelId(attempt.userModelId);
|
|
1662
|
+
return { ...attempt, cliProvider: parsed.provider, cliModel: parsed.model };
|
|
1663
|
+
});
|
|
1664
|
+
const filtered = mapped.filter((a) => {
|
|
1665
|
+
if (a.transport === 'cli')
|
|
1666
|
+
return true;
|
|
1667
|
+
if (!a.llmModelId)
|
|
1668
|
+
return false;
|
|
1669
|
+
const parsed = parseGatewayStyleModelId(a.llmModelId);
|
|
1670
|
+
if (parsed.provider === 'xai' &&
|
|
1671
|
+
attachment.part.type === 'file' &&
|
|
1672
|
+
!isTextLikeMediaType(attachment.mediaType)) {
|
|
1673
|
+
return false;
|
|
1674
|
+
}
|
|
1675
|
+
return true;
|
|
1676
|
+
});
|
|
1677
|
+
return filtered;
|
|
1678
|
+
}
|
|
1679
|
+
if (!fixedModelSpec) {
|
|
1680
|
+
throw new Error('Internal error: missing fixed model spec');
|
|
1681
|
+
}
|
|
1682
|
+
if (fixedModelSpec.transport === 'cli') {
|
|
1683
|
+
return [
|
|
1684
|
+
{
|
|
1685
|
+
transport: 'cli',
|
|
1686
|
+
userModelId: fixedModelSpec.userModelId,
|
|
1687
|
+
llmModelId: null,
|
|
1688
|
+
cliProvider: fixedModelSpec.cliProvider,
|
|
1689
|
+
cliModel: fixedModelSpec.cliModel,
|
|
1690
|
+
openrouterProviders: null,
|
|
1691
|
+
forceOpenRouter: false,
|
|
1692
|
+
requiredEnv: fixedModelSpec.requiredEnv,
|
|
1693
|
+
},
|
|
1694
|
+
];
|
|
1695
|
+
}
|
|
1696
|
+
return [
|
|
1697
|
+
{
|
|
1698
|
+
transport: fixedModelSpec.transport === 'openrouter' ? 'openrouter' : 'native',
|
|
1699
|
+
userModelId: fixedModelSpec.userModelId,
|
|
1700
|
+
llmModelId: fixedModelSpec.llmModelId,
|
|
1701
|
+
openrouterProviders: fixedModelSpec.openrouterProviders,
|
|
1702
|
+
forceOpenRouter: fixedModelSpec.forceOpenRouter,
|
|
1703
|
+
requiredEnv: fixedModelSpec.requiredEnv,
|
|
1704
|
+
},
|
|
1705
|
+
];
|
|
1706
|
+
})();
|
|
1707
|
+
const cliContext = await (async () => {
|
|
1708
|
+
if (!attempts.some((a) => a.transport === 'cli'))
|
|
1709
|
+
return null;
|
|
1710
|
+
if (typeof promptPayload === 'string')
|
|
1711
|
+
return null;
|
|
1712
|
+
const needsPathPrompt = attachment.part.type === 'image' || attachment.part.type === 'file';
|
|
1713
|
+
if (!needsPathPrompt)
|
|
1714
|
+
return null;
|
|
1715
|
+
const filePath = await ensureCliAttachmentPath({ sourceKind, sourceLabel, attachment });
|
|
1716
|
+
const dir = path.dirname(filePath);
|
|
1717
|
+
const extraArgsByProvider = {
|
|
1718
|
+
gemini: ['--include-directories', dir],
|
|
1719
|
+
codex: attachment.part.type === 'image' ? ['-i', filePath] : undefined,
|
|
1720
|
+
};
|
|
1721
|
+
return {
|
|
1722
|
+
promptOverride: buildPathSummaryPrompt({
|
|
1723
|
+
kindLabel: attachment.part.type === 'image' ? 'image' : 'file',
|
|
1724
|
+
filePath,
|
|
1725
|
+
filename: attachment.filename,
|
|
1726
|
+
mediaType: attachment.mediaType,
|
|
1727
|
+
summaryLength: summaryLengthTarget,
|
|
1728
|
+
}),
|
|
1729
|
+
allowTools: true,
|
|
1730
|
+
cwd: dir,
|
|
1731
|
+
extraArgsByProvider,
|
|
1732
|
+
};
|
|
1733
|
+
})();
|
|
1734
|
+
let summaryResult = null;
|
|
1735
|
+
let usedAttempt = null;
|
|
1736
|
+
let lastError = null;
|
|
1737
|
+
let sawOpenRouterNoAllowedProviders = false;
|
|
1738
|
+
const missingRequiredEnvs = new Set();
|
|
1739
|
+
for (const attempt of attempts) {
|
|
1740
|
+
const hasKey = envHasKeyFor(attempt.requiredEnv);
|
|
1741
|
+
if (!hasKey) {
|
|
1742
|
+
if (isFallbackModel) {
|
|
1743
|
+
if (isNamedModelSelection) {
|
|
1744
|
+
missingRequiredEnvs.add(attempt.requiredEnv);
|
|
1745
|
+
continue;
|
|
1746
|
+
}
|
|
1747
|
+
writeVerbose(stderr, verbose, `auto skip ${attempt.userModelId}: missing ${attempt.requiredEnv}`, verboseColor);
|
|
1748
|
+
continue;
|
|
1749
|
+
}
|
|
1750
|
+
throw new Error(formatMissingModelError(attempt));
|
|
1751
|
+
}
|
|
1752
|
+
try {
|
|
1753
|
+
summaryResult = await runSummaryAttempt({
|
|
1754
|
+
attempt,
|
|
1755
|
+
prompt: promptPayload,
|
|
1756
|
+
allowStreaming: requestedModel.kind === 'fixed',
|
|
1757
|
+
onModelChosen: onModelChosen ?? null,
|
|
1758
|
+
cli: cliContext,
|
|
1759
|
+
});
|
|
1760
|
+
usedAttempt = attempt;
|
|
1761
|
+
break;
|
|
1762
|
+
}
|
|
1763
|
+
catch (error) {
|
|
1764
|
+
lastError = error;
|
|
1765
|
+
if (isNamedModelSelection &&
|
|
1766
|
+
error instanceof Error &&
|
|
1767
|
+
/No allowed providers are available for the selected model/i.test(error.message)) {
|
|
1768
|
+
sawOpenRouterNoAllowedProviders = true;
|
|
1769
|
+
}
|
|
1770
|
+
if (requestedModel.kind === 'fixed') {
|
|
1771
|
+
if (isUnsupportedAttachmentError(error)) {
|
|
1772
|
+
throw new Error(`Model ${attempt.userModelId} does not support attaching files of type ${attachment.mediaType}. Try a different --model.`, { cause: error });
|
|
1773
|
+
}
|
|
1774
|
+
throw error;
|
|
1775
|
+
}
|
|
1776
|
+
writeVerbose(stderr, verbose, `auto failed ${attempt.userModelId}: ${error instanceof Error ? error.message : String(error)}`, verboseColor);
|
|
1777
|
+
}
|
|
1778
|
+
}
|
|
1779
|
+
if (!summaryResult || !usedAttempt) {
|
|
1780
|
+
const withFreeTip = (message) => {
|
|
1781
|
+
if (!isNamedModelSelection || !wantsFreeNamedModel)
|
|
1782
|
+
return message;
|
|
1783
|
+
return (`${message}\n` +
|
|
1784
|
+
`Tip: run "summarize refresh-free" to refresh the free model candidates (writes ~/.summarize/config.json).`);
|
|
1785
|
+
};
|
|
1786
|
+
if (isNamedModelSelection) {
|
|
1787
|
+
if (lastError === null && missingRequiredEnvs.size > 0) {
|
|
1788
|
+
throw new Error(withFreeTip(`Missing ${Array.from(missingRequiredEnvs).sort().join(', ')} for --model ${requestedModelInput}.`));
|
|
1789
|
+
}
|
|
1790
|
+
if (lastError instanceof Error) {
|
|
1791
|
+
if (sawOpenRouterNoAllowedProviders) {
|
|
1792
|
+
const message = await buildOpenRouterNoAllowedProvidersMessage({
|
|
1793
|
+
attempts,
|
|
1794
|
+
fetchImpl: trackedFetch,
|
|
1795
|
+
timeoutMs,
|
|
1796
|
+
});
|
|
1797
|
+
throw new Error(withFreeTip(message), { cause: lastError });
|
|
1798
|
+
}
|
|
1799
|
+
throw new Error(withFreeTip(lastError.message), { cause: lastError });
|
|
1800
|
+
}
|
|
1801
|
+
throw new Error(withFreeTip(`No model available for --model ${requestedModelInput}`));
|
|
1802
|
+
}
|
|
1803
|
+
if (textContent) {
|
|
1804
|
+
clearProgressForStdout();
|
|
1805
|
+
stdout.write(`${textContent.content.trim()}\n`);
|
|
1806
|
+
if (assetFooterParts.length > 0) {
|
|
1807
|
+
writeViaFooter([...assetFooterParts, 'no model']);
|
|
1808
|
+
}
|
|
1809
|
+
return;
|
|
1810
|
+
}
|
|
1811
|
+
if (lastError instanceof Error)
|
|
1812
|
+
throw lastError;
|
|
1813
|
+
throw new Error('No model available for this input');
|
|
1814
|
+
}
|
|
1815
|
+
const { summary, summaryAlreadyPrinted, modelMeta, maxOutputTokensForCall } = summaryResult;
|
|
920
1816
|
const extracted = {
|
|
921
1817
|
kind: 'asset',
|
|
922
1818
|
source: sourceLabel,
|
|
@@ -935,7 +1831,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
935
1831
|
? { kind: 'preset', preset: lengthArg.preset }
|
|
936
1832
|
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
937
1833
|
maxOutputTokens: maxOutputTokensArg,
|
|
938
|
-
model,
|
|
1834
|
+
model: requestedModelLabel,
|
|
939
1835
|
}
|
|
940
1836
|
: {
|
|
941
1837
|
kind: 'asset-url',
|
|
@@ -945,13 +1841,14 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
945
1841
|
? { kind: 'preset', preset: lengthArg.preset }
|
|
946
1842
|
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
947
1843
|
maxOutputTokens: maxOutputTokensArg,
|
|
948
|
-
model,
|
|
1844
|
+
model: requestedModelLabel,
|
|
949
1845
|
};
|
|
950
1846
|
const payload = {
|
|
951
1847
|
input,
|
|
952
1848
|
env: {
|
|
953
1849
|
hasXaiKey: Boolean(xaiApiKey),
|
|
954
1850
|
hasOpenAIKey: Boolean(apiKey),
|
|
1851
|
+
hasOpenRouterKey: Boolean(openrouterApiKey),
|
|
955
1852
|
hasApifyToken: Boolean(apifyToken),
|
|
956
1853
|
hasFirecrawlKey: firecrawlConfigured,
|
|
957
1854
|
hasGoogleKey: googleConfigured,
|
|
@@ -960,26 +1857,25 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
960
1857
|
extracted,
|
|
961
1858
|
prompt: promptText,
|
|
962
1859
|
llm: {
|
|
963
|
-
provider:
|
|
964
|
-
model:
|
|
1860
|
+
provider: modelMeta.provider,
|
|
1861
|
+
model: usedAttempt.userModelId,
|
|
965
1862
|
maxCompletionTokens: maxOutputTokensForCall,
|
|
966
1863
|
strategy: 'single',
|
|
967
1864
|
},
|
|
968
1865
|
metrics: metricsEnabled ? finishReport : null,
|
|
969
1866
|
summary,
|
|
970
1867
|
};
|
|
971
|
-
if (metricsDetailed && finishReport) {
|
|
972
|
-
writeMetricsReport(finishReport);
|
|
973
|
-
}
|
|
974
1868
|
stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
975
1869
|
if (metricsEnabled && finishReport) {
|
|
976
1870
|
const costUsd = await estimateCostUsd();
|
|
977
1871
|
writeFinishLine({
|
|
978
1872
|
stderr,
|
|
979
1873
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
980
|
-
model:
|
|
1874
|
+
model: usedAttempt.userModelId,
|
|
981
1875
|
report: finishReport,
|
|
982
1876
|
costUsd,
|
|
1877
|
+
detailed: metricsDetailed,
|
|
1878
|
+
extraParts: null,
|
|
983
1879
|
color: verboseColor,
|
|
984
1880
|
});
|
|
985
1881
|
}
|
|
@@ -999,17 +1895,18 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
999
1895
|
stdout.write('\n');
|
|
1000
1896
|
}
|
|
1001
1897
|
}
|
|
1898
|
+
writeViaFooter([...assetFooterParts, `model ${usedAttempt.userModelId}`]);
|
|
1002
1899
|
const report = shouldComputeReport ? await buildReport() : null;
|
|
1003
|
-
if (metricsDetailed && report)
|
|
1004
|
-
writeMetricsReport(report);
|
|
1005
1900
|
if (metricsEnabled && report) {
|
|
1006
1901
|
const costUsd = await estimateCostUsd();
|
|
1007
1902
|
writeFinishLine({
|
|
1008
1903
|
stderr,
|
|
1009
1904
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1010
|
-
model:
|
|
1905
|
+
model: usedAttempt.userModelId,
|
|
1011
1906
|
report,
|
|
1012
1907
|
costUsd,
|
|
1908
|
+
detailed: metricsDetailed,
|
|
1909
|
+
extraParts: null,
|
|
1013
1910
|
color: verboseColor,
|
|
1014
1911
|
});
|
|
1015
1912
|
}
|
|
@@ -1059,6 +1956,16 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1059
1956
|
sourceKind: 'file',
|
|
1060
1957
|
sourceLabel: loaded.sourceLabel,
|
|
1061
1958
|
attachment: loaded.attachment,
|
|
1959
|
+
onModelChosen: (modelId) => {
|
|
1960
|
+
if (!progressEnabled)
|
|
1961
|
+
return;
|
|
1962
|
+
const mt = loaded.attachment.mediaType;
|
|
1963
|
+
const name = loaded.attachment.filename;
|
|
1964
|
+
const details = sizeLabel ? `${mt}, ${sizeLabel}` : mt;
|
|
1965
|
+
spinner.setText(name
|
|
1966
|
+
? `Summarizing ${name} (${details}, model: ${modelId})…`
|
|
1967
|
+
: `Summarizing ${details} (model: ${modelId})…`);
|
|
1968
|
+
},
|
|
1062
1969
|
});
|
|
1063
1970
|
return;
|
|
1064
1971
|
}
|
|
@@ -1114,6 +2021,11 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1114
2021
|
sourceKind: 'asset-url',
|
|
1115
2022
|
sourceLabel: loaded.sourceLabel,
|
|
1116
2023
|
attachment: loaded.attachment,
|
|
2024
|
+
onModelChosen: (modelId) => {
|
|
2025
|
+
if (!progressEnabled)
|
|
2026
|
+
return;
|
|
2027
|
+
spinner.setText(`Summarizing (model: ${modelId})…`);
|
|
2028
|
+
},
|
|
1117
2029
|
});
|
|
1118
2030
|
return;
|
|
1119
2031
|
}
|
|
@@ -1128,52 +2040,166 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1128
2040
|
if (!url) {
|
|
1129
2041
|
throw new Error('Only HTTP and HTTPS URLs can be summarized');
|
|
1130
2042
|
}
|
|
1131
|
-
const
|
|
2043
|
+
const wantsMarkdown = format === 'markdown' && !isYoutubeUrl;
|
|
2044
|
+
if (wantsMarkdown && markdownMode === 'off') {
|
|
2045
|
+
throw new Error('--format md conflicts with --markdown-mode off (use --format text)');
|
|
2046
|
+
}
|
|
2047
|
+
const firecrawlMode = (() => {
|
|
2048
|
+
if (wantsMarkdown && !isYoutubeUrl && !firecrawlExplicitlySet && firecrawlConfigured) {
|
|
2049
|
+
return 'always';
|
|
2050
|
+
}
|
|
2051
|
+
return requestedFirecrawlMode;
|
|
2052
|
+
})();
|
|
1132
2053
|
if (firecrawlMode === 'always' && !firecrawlConfigured) {
|
|
1133
2054
|
throw new Error('--firecrawl always requires FIRECRAWL_API_KEY');
|
|
1134
2055
|
}
|
|
1135
|
-
const
|
|
1136
|
-
const
|
|
1137
|
-
const
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
2056
|
+
const markdownRequested = wantsMarkdown;
|
|
2057
|
+
const effectiveMarkdownMode = markdownRequested ? markdownMode : 'off';
|
|
2058
|
+
const markdownModel = (() => {
|
|
2059
|
+
if (!markdownRequested)
|
|
2060
|
+
return null;
|
|
2061
|
+
// Prefer the explicitly chosen model when it is a native provider (keeps behavior stable).
|
|
2062
|
+
if (requestedModel.kind === 'fixed' && requestedModel.transport === 'native') {
|
|
2063
|
+
return { llmModelId: requestedModel.llmModelId, forceOpenRouter: false };
|
|
2064
|
+
}
|
|
2065
|
+
// Otherwise pick a safe, broadly-capable default for HTML→Markdown conversion.
|
|
2066
|
+
if (googleConfigured) {
|
|
2067
|
+
return { llmModelId: 'google/gemini-3-flash-preview', forceOpenRouter: false };
|
|
2068
|
+
}
|
|
2069
|
+
if (apiKey) {
|
|
2070
|
+
return { llmModelId: 'openai/gpt-5-mini', forceOpenRouter: false };
|
|
2071
|
+
}
|
|
2072
|
+
if (openrouterConfigured) {
|
|
2073
|
+
return { llmModelId: 'openai/openai/gpt-5-mini', forceOpenRouter: true };
|
|
2074
|
+
}
|
|
2075
|
+
if (anthropicConfigured) {
|
|
2076
|
+
return { llmModelId: 'anthropic/claude-sonnet-4-5', forceOpenRouter: false };
|
|
2077
|
+
}
|
|
2078
|
+
if (xaiConfigured) {
|
|
2079
|
+
return { llmModelId: 'xai/grok-4-fast-non-reasoning', forceOpenRouter: false };
|
|
2080
|
+
}
|
|
2081
|
+
return null;
|
|
2082
|
+
})();
|
|
2083
|
+
const markdownProvider = (() => {
|
|
2084
|
+
if (!markdownModel)
|
|
2085
|
+
return 'none';
|
|
2086
|
+
const parsed = parseGatewayStyleModelId(markdownModel.llmModelId);
|
|
2087
|
+
return parsed.provider;
|
|
2088
|
+
})();
|
|
2089
|
+
const hasKeyForMarkdownModel = (() => {
|
|
2090
|
+
if (!markdownModel)
|
|
2091
|
+
return false;
|
|
2092
|
+
if (markdownModel.forceOpenRouter)
|
|
2093
|
+
return openrouterConfigured;
|
|
2094
|
+
const parsed = parseGatewayStyleModelId(markdownModel.llmModelId);
|
|
2095
|
+
return parsed.provider === 'xai'
|
|
2096
|
+
? xaiConfigured
|
|
2097
|
+
: parsed.provider === 'google'
|
|
2098
|
+
? googleConfigured
|
|
2099
|
+
: parsed.provider === 'anthropic'
|
|
2100
|
+
? anthropicConfigured
|
|
2101
|
+
: Boolean(apiKey);
|
|
2102
|
+
})();
|
|
2103
|
+
if (markdownRequested && effectiveMarkdownMode === 'llm' && !hasKeyForMarkdownModel) {
|
|
2104
|
+
const required = (() => {
|
|
2105
|
+
if (markdownModel?.forceOpenRouter)
|
|
2106
|
+
return 'OPENROUTER_API_KEY';
|
|
2107
|
+
if (markdownModel) {
|
|
2108
|
+
const parsed = parseGatewayStyleModelId(markdownModel.llmModelId);
|
|
2109
|
+
return parsed.provider === 'xai'
|
|
2110
|
+
? 'XAI_API_KEY'
|
|
2111
|
+
: parsed.provider === 'google'
|
|
2112
|
+
? 'GEMINI_API_KEY (or GOOGLE_GENERATIVE_AI_API_KEY / GOOGLE_API_KEY)'
|
|
2113
|
+
: parsed.provider === 'anthropic'
|
|
2114
|
+
? 'ANTHROPIC_API_KEY'
|
|
2115
|
+
: 'OPENAI_API_KEY';
|
|
2116
|
+
}
|
|
2117
|
+
return 'GEMINI_API_KEY (or GOOGLE_GENERATIVE_AI_API_KEY / GOOGLE_API_KEY)';
|
|
2118
|
+
})();
|
|
2119
|
+
throw new Error(`--markdown-mode llm requires ${required}`);
|
|
1154
2120
|
}
|
|
1155
|
-
writeVerbose(stderr, verbose, `config url=${url} timeoutMs=${timeoutMs} youtube=${youtubeMode} firecrawl=${firecrawlMode} length=${lengthArg.kind === 'preset' ? lengthArg.preset : `${lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(maxOutputTokensArg)} json=${json}
|
|
1156
|
-
writeVerbose(stderr, verbose, `configFile path=${formatOptionalString(configPath)} model=${formatOptionalString(
|
|
2121
|
+
writeVerbose(stderr, verbose, `config url=${url} timeoutMs=${timeoutMs} youtube=${youtubeMode} firecrawl=${firecrawlMode} length=${lengthArg.kind === 'preset' ? lengthArg.preset : `${lengthArg.maxCharacters} chars`} maxOutputTokens=${formatOptionalNumber(maxOutputTokensArg)} retries=${retries} json=${json} extract=${extractMode} format=${format} preprocess=${preprocessMode} markdownMode=${markdownMode} model=${requestedModelLabel} videoMode=${videoMode} stream=${effectiveStreamMode} render=${effectiveRenderMode}`, verboseColor);
|
|
2122
|
+
writeVerbose(stderr, verbose, `configFile path=${formatOptionalString(configPath)} model=${formatOptionalString((() => {
|
|
2123
|
+
const model = config?.model;
|
|
2124
|
+
if (!model)
|
|
2125
|
+
return null;
|
|
2126
|
+
if ('id' in model)
|
|
2127
|
+
return model.id;
|
|
2128
|
+
if ('name' in model)
|
|
2129
|
+
return model.name;
|
|
2130
|
+
if ('mode' in model && model.mode === 'auto')
|
|
2131
|
+
return 'auto';
|
|
2132
|
+
return null;
|
|
2133
|
+
})())}`, verboseColor);
|
|
1157
2134
|
writeVerbose(stderr, verbose, `env xaiKey=${xaiConfigured} openaiKey=${Boolean(apiKey)} googleKey=${googleConfigured} anthropicKey=${anthropicConfigured} openrouterKey=${openrouterConfigured} apifyToken=${Boolean(apifyToken)} firecrawlKey=${firecrawlConfigured}`, verboseColor);
|
|
1158
2135
|
writeVerbose(stderr, verbose, `markdown requested=${markdownRequested} provider=${markdownProvider}`, verboseColor);
|
|
1159
2136
|
const scrapeWithFirecrawl = firecrawlConfigured && firecrawlMode !== 'off'
|
|
1160
2137
|
? createFirecrawlScraper({ apiKey: firecrawlApiKey, fetchImpl: trackedFetch })
|
|
1161
2138
|
: null;
|
|
1162
|
-
const
|
|
2139
|
+
const llmHtmlToMarkdown = markdownRequested &&
|
|
2140
|
+
markdownModel !== null &&
|
|
2141
|
+
(effectiveMarkdownMode === 'llm' || markdownProvider !== 'none')
|
|
1163
2142
|
? createHtmlToMarkdownConverter({
|
|
1164
|
-
modelId:
|
|
2143
|
+
modelId: markdownModel.llmModelId,
|
|
2144
|
+
forceOpenRouter: markdownModel.forceOpenRouter,
|
|
1165
2145
|
xaiApiKey: xaiConfigured ? xaiApiKey : null,
|
|
1166
2146
|
googleApiKey: googleConfigured ? googleApiKey : null,
|
|
1167
2147
|
openaiApiKey: apiKey,
|
|
1168
2148
|
anthropicApiKey: anthropicConfigured ? anthropicApiKey : null,
|
|
1169
2149
|
openrouterApiKey: openrouterConfigured ? openrouterApiKey : null,
|
|
1170
|
-
openrouter: openrouterOptions,
|
|
1171
2150
|
fetchImpl: trackedFetch,
|
|
2151
|
+
retries,
|
|
2152
|
+
onRetry: createRetryLogger({
|
|
2153
|
+
stderr,
|
|
2154
|
+
verbose,
|
|
2155
|
+
color: verboseColor,
|
|
2156
|
+
modelId: markdownModel.llmModelId,
|
|
2157
|
+
}),
|
|
1172
2158
|
onUsage: ({ model: usedModel, provider, usage }) => {
|
|
1173
2159
|
llmCalls.push({ provider, model: usedModel, usage, purpose: 'markdown' });
|
|
1174
2160
|
},
|
|
1175
2161
|
})
|
|
1176
2162
|
: null;
|
|
2163
|
+
const markitdownHtmlToMarkdown = markdownRequested && preprocessMode !== 'off' && hasUvxCli(env)
|
|
2164
|
+
? async (args) => {
|
|
2165
|
+
void args.url;
|
|
2166
|
+
void args.title;
|
|
2167
|
+
void args.siteName;
|
|
2168
|
+
return convertToMarkdownWithMarkitdown({
|
|
2169
|
+
bytes: new TextEncoder().encode(args.html),
|
|
2170
|
+
filenameHint: 'page.html',
|
|
2171
|
+
mediaTypeHint: 'text/html',
|
|
2172
|
+
uvxCommand: env.UVX_PATH,
|
|
2173
|
+
timeoutMs: args.timeoutMs,
|
|
2174
|
+
env,
|
|
2175
|
+
execFileImpl,
|
|
2176
|
+
});
|
|
2177
|
+
}
|
|
2178
|
+
: null;
|
|
2179
|
+
const convertHtmlToMarkdown = markdownRequested
|
|
2180
|
+
? async (args) => {
|
|
2181
|
+
if (effectiveMarkdownMode === 'llm') {
|
|
2182
|
+
if (!llmHtmlToMarkdown) {
|
|
2183
|
+
throw new Error('No HTML→Markdown converter configured');
|
|
2184
|
+
}
|
|
2185
|
+
return llmHtmlToMarkdown(args);
|
|
2186
|
+
}
|
|
2187
|
+
if (llmHtmlToMarkdown) {
|
|
2188
|
+
try {
|
|
2189
|
+
return await llmHtmlToMarkdown(args);
|
|
2190
|
+
}
|
|
2191
|
+
catch (error) {
|
|
2192
|
+
if (!markitdownHtmlToMarkdown)
|
|
2193
|
+
throw error;
|
|
2194
|
+
return await markitdownHtmlToMarkdown(args);
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
if (markitdownHtmlToMarkdown) {
|
|
2198
|
+
return await markitdownHtmlToMarkdown(args);
|
|
2199
|
+
}
|
|
2200
|
+
throw new Error('No HTML→Markdown converter configured');
|
|
2201
|
+
}
|
|
2202
|
+
: null;
|
|
1177
2203
|
const readTweetWithBirdClient = hasBirdCli(env)
|
|
1178
2204
|
? ({ url, timeoutMs }) => readTweetWithBird({ url, timeoutMs, env })
|
|
1179
2205
|
: null;
|
|
@@ -1352,22 +2378,46 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1352
2378
|
catch (error) {
|
|
1353
2379
|
throw withBirdTip(error, url, env);
|
|
1354
2380
|
}
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
2381
|
+
let extractedContentSize = 'unknown';
|
|
2382
|
+
let viaSourceLabel = '';
|
|
2383
|
+
let footerBaseParts = [];
|
|
2384
|
+
const recomputeExtractionUi = () => {
|
|
2385
|
+
const extractedContentBytes = Buffer.byteLength(extracted.content, 'utf8');
|
|
2386
|
+
extractedContentSize = formatBytes(extractedContentBytes);
|
|
2387
|
+
const viaSources = [];
|
|
2388
|
+
if (extracted.diagnostics.strategy === 'bird') {
|
|
2389
|
+
viaSources.push('bird');
|
|
2390
|
+
}
|
|
2391
|
+
if (extracted.diagnostics.strategy === 'nitter') {
|
|
2392
|
+
viaSources.push('Nitter');
|
|
2393
|
+
}
|
|
2394
|
+
if (extracted.diagnostics.firecrawl.used) {
|
|
2395
|
+
viaSources.push('Firecrawl');
|
|
2396
|
+
}
|
|
2397
|
+
viaSourceLabel = viaSources.length > 0 ? `, ${viaSources.join('+')}` : '';
|
|
2398
|
+
footerBaseParts = [];
|
|
2399
|
+
if (extracted.diagnostics.strategy === 'html')
|
|
2400
|
+
footerBaseParts.push('html');
|
|
2401
|
+
if (extracted.diagnostics.strategy === 'bird')
|
|
2402
|
+
footerBaseParts.push('bird');
|
|
2403
|
+
if (extracted.diagnostics.strategy === 'nitter')
|
|
2404
|
+
footerBaseParts.push('nitter');
|
|
2405
|
+
if (extracted.diagnostics.firecrawl.used)
|
|
2406
|
+
footerBaseParts.push('firecrawl');
|
|
2407
|
+
if (extracted.diagnostics.markdown.used) {
|
|
2408
|
+
footerBaseParts.push(extracted.diagnostics.markdown.provider === 'llm' ? 'html→md llm' : 'markdown');
|
|
2409
|
+
}
|
|
2410
|
+
if (extracted.diagnostics.transcript.textProvided) {
|
|
2411
|
+
footerBaseParts.push(`transcript ${extracted.diagnostics.transcript.provider ?? 'unknown'}`);
|
|
2412
|
+
}
|
|
2413
|
+
if (extracted.isVideoOnly && extracted.video) {
|
|
2414
|
+
footerBaseParts.push(extracted.video.kind === 'youtube' ? 'video youtube' : 'video url');
|
|
2415
|
+
}
|
|
2416
|
+
};
|
|
2417
|
+
recomputeExtractionUi();
|
|
1368
2418
|
if (progressEnabled) {
|
|
1369
2419
|
websiteProgress?.stop?.();
|
|
1370
|
-
spinner.setText(
|
|
2420
|
+
spinner.setText(extractMode
|
|
1371
2421
|
? `Extracted (${extractedContentSize}${viaSourceLabel})`
|
|
1372
2422
|
: `Summarizing (sent ${extractedContentSize}${viaSourceLabel})…`);
|
|
1373
2423
|
}
|
|
@@ -1378,6 +2428,66 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1378
2428
|
writeVerbose(stderr, verbose, `extract transcript textProvided=${extracted.diagnostics.transcript.textProvided} provider=${formatOptionalString(extracted.diagnostics.transcript.provider ?? null)} attemptedProviders=${extracted.diagnostics.transcript.attemptedProviders.length > 0
|
|
1379
2429
|
? extracted.diagnostics.transcript.attemptedProviders.join(',')
|
|
1380
2430
|
: 'none'} notes=${formatOptionalString(extracted.diagnostics.transcript.notes ?? null)}`, verboseColor);
|
|
2431
|
+
if (extractMode &&
|
|
2432
|
+
markdownRequested &&
|
|
2433
|
+
preprocessMode !== 'off' &&
|
|
2434
|
+
effectiveMarkdownMode === 'auto' &&
|
|
2435
|
+
!extracted.diagnostics.markdown.used &&
|
|
2436
|
+
!hasUvxCli(env)) {
|
|
2437
|
+
stderr.write(`${UVX_TIP}\n`);
|
|
2438
|
+
}
|
|
2439
|
+
if (!isYoutubeUrl && extracted.isVideoOnly && extracted.video) {
|
|
2440
|
+
if (extracted.video.kind === 'youtube') {
|
|
2441
|
+
writeVerbose(stderr, verbose, `video-only page detected; switching to YouTube URL ${extracted.video.url}`, verboseColor);
|
|
2442
|
+
if (progressEnabled) {
|
|
2443
|
+
spinner.setText('Video-only page: fetching YouTube transcript…');
|
|
2444
|
+
}
|
|
2445
|
+
extracted = await client.fetchLinkContent(extracted.video.url, {
|
|
2446
|
+
timeoutMs,
|
|
2447
|
+
youtubeTranscript: youtubeMode,
|
|
2448
|
+
firecrawl: firecrawlMode,
|
|
2449
|
+
format: markdownRequested ? 'markdown' : 'text',
|
|
2450
|
+
});
|
|
2451
|
+
recomputeExtractionUi();
|
|
2452
|
+
if (progressEnabled) {
|
|
2453
|
+
spinner.setText(extractMode
|
|
2454
|
+
? `Extracted (${extractedContentSize}${viaSourceLabel})`
|
|
2455
|
+
: `Summarizing (sent ${extractedContentSize}${viaSourceLabel})…`);
|
|
2456
|
+
}
|
|
2457
|
+
}
|
|
2458
|
+
else if (extracted.video.kind === 'direct') {
|
|
2459
|
+
const wantsVideoUnderstanding = videoMode === 'understand' || videoMode === 'auto';
|
|
2460
|
+
const canVideoUnderstand = wantsVideoUnderstanding &&
|
|
2461
|
+
googleConfigured &&
|
|
2462
|
+
(requestedModel.kind === 'auto' ||
|
|
2463
|
+
(fixedModelSpec?.transport === 'native' && fixedModelSpec.provider === 'google'));
|
|
2464
|
+
if (canVideoUnderstand) {
|
|
2465
|
+
if (progressEnabled)
|
|
2466
|
+
spinner.setText('Downloading video…');
|
|
2467
|
+
const loadedVideo = await loadRemoteAsset({
|
|
2468
|
+
url: extracted.video.url,
|
|
2469
|
+
fetchImpl: trackedFetch,
|
|
2470
|
+
timeoutMs,
|
|
2471
|
+
});
|
|
2472
|
+
assertAssetMediaTypeSupported({ attachment: loadedVideo.attachment, sizeLabel: null });
|
|
2473
|
+
let chosenModel = null;
|
|
2474
|
+
if (progressEnabled)
|
|
2475
|
+
spinner.setText('Summarizing video…');
|
|
2476
|
+
await summarizeAsset({
|
|
2477
|
+
sourceKind: 'asset-url',
|
|
2478
|
+
sourceLabel: loadedVideo.sourceLabel,
|
|
2479
|
+
attachment: loadedVideo.attachment,
|
|
2480
|
+
onModelChosen: (modelId) => {
|
|
2481
|
+
chosenModel = modelId;
|
|
2482
|
+
if (progressEnabled)
|
|
2483
|
+
spinner.setText(`Summarizing video (model: ${modelId})…`);
|
|
2484
|
+
},
|
|
2485
|
+
});
|
|
2486
|
+
writeViaFooter([...footerBaseParts, ...(chosenModel ? [`model ${chosenModel}`] : [])]);
|
|
2487
|
+
return;
|
|
2488
|
+
}
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
1381
2491
|
const isYouTube = extracted.siteName === 'YouTube';
|
|
1382
2492
|
const prompt = buildLinkSummaryPrompt({
|
|
1383
2493
|
url: extracted.url,
|
|
@@ -1391,7 +2501,7 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1391
2501
|
summaryLength: lengthArg.kind === 'preset' ? lengthArg.preset : { maxCharacters: lengthArg.maxCharacters },
|
|
1392
2502
|
shares: [],
|
|
1393
2503
|
});
|
|
1394
|
-
if (
|
|
2504
|
+
if (extractMode) {
|
|
1395
2505
|
clearProgressForStdout();
|
|
1396
2506
|
if (json) {
|
|
1397
2507
|
const finishReport = shouldComputeReport ? await buildReport() : null;
|
|
@@ -1402,16 +2512,18 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1402
2512
|
timeoutMs,
|
|
1403
2513
|
youtube: youtubeMode,
|
|
1404
2514
|
firecrawl: firecrawlMode,
|
|
2515
|
+
format,
|
|
1405
2516
|
markdown: effectiveMarkdownMode,
|
|
1406
2517
|
length: lengthArg.kind === 'preset'
|
|
1407
2518
|
? { kind: 'preset', preset: lengthArg.preset }
|
|
1408
2519
|
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
1409
2520
|
maxOutputTokens: maxOutputTokensArg,
|
|
1410
|
-
model,
|
|
2521
|
+
model: requestedModelLabel,
|
|
1411
2522
|
},
|
|
1412
2523
|
env: {
|
|
1413
2524
|
hasXaiKey: Boolean(xaiApiKey),
|
|
1414
2525
|
hasOpenAIKey: Boolean(apiKey),
|
|
2526
|
+
hasOpenRouterKey: Boolean(openrouterApiKey),
|
|
1415
2527
|
hasApifyToken: Boolean(apifyToken),
|
|
1416
2528
|
hasFirecrawlKey: firecrawlConfigured,
|
|
1417
2529
|
hasGoogleKey: googleConfigured,
|
|
@@ -1423,35 +2535,35 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1423
2535
|
metrics: metricsEnabled ? finishReport : null,
|
|
1424
2536
|
summary: null,
|
|
1425
2537
|
};
|
|
1426
|
-
if (metricsDetailed && finishReport) {
|
|
1427
|
-
writeMetricsReport(finishReport);
|
|
1428
|
-
}
|
|
1429
2538
|
stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
1430
2539
|
if (metricsEnabled && finishReport) {
|
|
1431
2540
|
const costUsd = await estimateCostUsd();
|
|
1432
2541
|
writeFinishLine({
|
|
1433
2542
|
stderr,
|
|
1434
2543
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1435
|
-
model,
|
|
2544
|
+
model: requestedModelLabel,
|
|
1436
2545
|
report: finishReport,
|
|
1437
2546
|
costUsd,
|
|
2547
|
+
detailed: metricsDetailed,
|
|
2548
|
+
extraParts: metricsDetailed ? buildDetailedLengthPartsForExtracted(extracted) : null,
|
|
1438
2549
|
color: verboseColor,
|
|
1439
2550
|
});
|
|
1440
2551
|
}
|
|
1441
2552
|
return;
|
|
1442
2553
|
}
|
|
1443
2554
|
stdout.write(`${extracted.content}\n`);
|
|
2555
|
+
writeViaFooter(footerBaseParts);
|
|
1444
2556
|
const report = shouldComputeReport ? await buildReport() : null;
|
|
1445
|
-
if (metricsDetailed && report)
|
|
1446
|
-
writeMetricsReport(report);
|
|
1447
2557
|
if (metricsEnabled && report) {
|
|
1448
2558
|
const costUsd = await estimateCostUsd();
|
|
1449
2559
|
writeFinishLine({
|
|
1450
2560
|
stderr,
|
|
1451
2561
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1452
|
-
model,
|
|
2562
|
+
model: requestedModelLabel,
|
|
1453
2563
|
report,
|
|
1454
2564
|
costUsd,
|
|
2565
|
+
detailed: metricsDetailed,
|
|
2566
|
+
extraParts: metricsDetailed ? buildDetailedLengthPartsForExtracted(extracted) : null,
|
|
1455
2567
|
color: verboseColor,
|
|
1456
2568
|
});
|
|
1457
2569
|
}
|
|
@@ -1472,16 +2584,18 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1472
2584
|
timeoutMs,
|
|
1473
2585
|
youtube: youtubeMode,
|
|
1474
2586
|
firecrawl: firecrawlMode,
|
|
2587
|
+
format,
|
|
1475
2588
|
markdown: effectiveMarkdownMode,
|
|
1476
2589
|
length: lengthArg.kind === 'preset'
|
|
1477
2590
|
? { kind: 'preset', preset: lengthArg.preset }
|
|
1478
2591
|
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
1479
2592
|
maxOutputTokens: maxOutputTokensArg,
|
|
1480
|
-
model,
|
|
2593
|
+
model: requestedModelLabel,
|
|
1481
2594
|
},
|
|
1482
2595
|
env: {
|
|
1483
2596
|
hasXaiKey: Boolean(xaiApiKey),
|
|
1484
2597
|
hasOpenAIKey: Boolean(apiKey),
|
|
2598
|
+
hasOpenRouterKey: Boolean(openrouterApiKey),
|
|
1485
2599
|
hasApifyToken: Boolean(apifyToken),
|
|
1486
2600
|
hasFirecrawlKey: firecrawlConfigured,
|
|
1487
2601
|
hasGoogleKey: googleConfigured,
|
|
@@ -1493,248 +2607,212 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1493
2607
|
metrics: metricsEnabled ? finishReport : null,
|
|
1494
2608
|
summary: extracted.content,
|
|
1495
2609
|
};
|
|
1496
|
-
if (metricsDetailed && finishReport) {
|
|
1497
|
-
writeMetricsReport(finishReport);
|
|
1498
|
-
}
|
|
1499
2610
|
stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
1500
2611
|
if (metricsEnabled && finishReport) {
|
|
1501
2612
|
const costUsd = await estimateCostUsd();
|
|
1502
2613
|
writeFinishLine({
|
|
1503
2614
|
stderr,
|
|
1504
2615
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1505
|
-
model,
|
|
2616
|
+
model: requestedModelLabel,
|
|
1506
2617
|
report: finishReport,
|
|
1507
2618
|
costUsd,
|
|
2619
|
+
detailed: metricsDetailed,
|
|
2620
|
+
extraParts: metricsDetailed ? buildDetailedLengthPartsForExtracted(extracted) : null,
|
|
1508
2621
|
color: verboseColor,
|
|
1509
2622
|
});
|
|
1510
2623
|
}
|
|
1511
2624
|
return;
|
|
1512
2625
|
}
|
|
1513
2626
|
stdout.write(`${extracted.content}\n`);
|
|
2627
|
+
writeViaFooter(footerBaseParts);
|
|
1514
2628
|
const report = shouldComputeReport ? await buildReport() : null;
|
|
1515
|
-
if (metricsDetailed && report)
|
|
1516
|
-
writeMetricsReport(report);
|
|
1517
2629
|
if (metricsEnabled && report) {
|
|
1518
2630
|
const costUsd = await estimateCostUsd();
|
|
1519
2631
|
writeFinishLine({
|
|
1520
2632
|
stderr,
|
|
1521
2633
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1522
|
-
model,
|
|
2634
|
+
model: requestedModelLabel,
|
|
1523
2635
|
report,
|
|
1524
2636
|
costUsd,
|
|
2637
|
+
detailed: metricsDetailed,
|
|
2638
|
+
extraParts: metricsDetailed ? buildDetailedLengthPartsForExtracted(extracted) : null,
|
|
1525
2639
|
color: verboseColor,
|
|
1526
2640
|
});
|
|
1527
2641
|
}
|
|
1528
2642
|
return;
|
|
1529
2643
|
}
|
|
1530
|
-
const
|
|
1531
|
-
const
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
2644
|
+
const promptTokens = countTokens(prompt);
|
|
2645
|
+
const kindForAuto = isYouTube ? 'youtube' : 'website';
|
|
2646
|
+
const attempts = await (async () => {
|
|
2647
|
+
if (isFallbackModel) {
|
|
2648
|
+
const catalog = await getLiteLlmCatalog();
|
|
2649
|
+
const list = buildAutoModelAttempts({
|
|
2650
|
+
kind: kindForAuto,
|
|
2651
|
+
promptTokens,
|
|
2652
|
+
desiredOutputTokens,
|
|
2653
|
+
requiresVideoUnderstanding: false,
|
|
2654
|
+
env: envForAuto,
|
|
2655
|
+
config: configForModelSelection,
|
|
2656
|
+
catalog,
|
|
2657
|
+
openrouterProvidersFromEnv: null,
|
|
2658
|
+
cliAvailability,
|
|
2659
|
+
});
|
|
2660
|
+
if (verbose) {
|
|
2661
|
+
for (const a of list.slice(0, 8)) {
|
|
2662
|
+
writeVerbose(stderr, verbose, `auto candidate ${a.debug}`, verboseColor);
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
return list.map((attempt) => {
|
|
2666
|
+
if (attempt.transport !== 'cli')
|
|
2667
|
+
return attempt;
|
|
2668
|
+
const parsed = parseCliUserModelId(attempt.userModelId);
|
|
2669
|
+
return { ...attempt, cliProvider: parsed.provider, cliModel: parsed.model };
|
|
2670
|
+
});
|
|
2671
|
+
}
|
|
2672
|
+
if (!fixedModelSpec) {
|
|
2673
|
+
throw new Error('Internal error: missing fixed model spec');
|
|
2674
|
+
}
|
|
2675
|
+
if (fixedModelSpec.transport === 'cli') {
|
|
2676
|
+
return [
|
|
2677
|
+
{
|
|
2678
|
+
transport: 'cli',
|
|
2679
|
+
userModelId: fixedModelSpec.userModelId,
|
|
2680
|
+
llmModelId: null,
|
|
2681
|
+
cliProvider: fixedModelSpec.cliProvider,
|
|
2682
|
+
cliModel: fixedModelSpec.cliModel,
|
|
2683
|
+
openrouterProviders: null,
|
|
2684
|
+
forceOpenRouter: false,
|
|
2685
|
+
requiredEnv: fixedModelSpec.requiredEnv,
|
|
2686
|
+
},
|
|
2687
|
+
];
|
|
2688
|
+
}
|
|
2689
|
+
return [
|
|
2690
|
+
{
|
|
2691
|
+
transport: fixedModelSpec.transport === 'openrouter' ? 'openrouter' : 'native',
|
|
2692
|
+
userModelId: fixedModelSpec.userModelId,
|
|
2693
|
+
llmModelId: fixedModelSpec.llmModelId,
|
|
2694
|
+
openrouterProviders: fixedModelSpec.openrouterProviders,
|
|
2695
|
+
forceOpenRouter: fixedModelSpec.forceOpenRouter,
|
|
2696
|
+
requiredEnv: fixedModelSpec.requiredEnv,
|
|
2697
|
+
},
|
|
2698
|
+
];
|
|
2699
|
+
})();
|
|
2700
|
+
const onModelChosen = (modelId) => {
|
|
2701
|
+
if (!progressEnabled)
|
|
2702
|
+
return;
|
|
2703
|
+
spinner.setText(`Summarizing (sent ${extractedContentSize}${viaSourceLabel}, model: ${modelId})…`);
|
|
1537
2704
|
};
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
:
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
const modelResolution = await resolveModelIdForLlmCall({
|
|
1556
|
-
parsedModel,
|
|
1557
|
-
apiKeys: { googleApiKey: apiKeysForLlm.googleApiKey },
|
|
1558
|
-
fetchImpl: trackedFetch,
|
|
1559
|
-
timeoutMs,
|
|
1560
|
-
});
|
|
1561
|
-
if (modelResolution.note && verbose) {
|
|
1562
|
-
writeVerbose(stderr, verbose, modelResolution.note, verboseColor);
|
|
1563
|
-
}
|
|
1564
|
-
const parsedModelEffective = parseGatewayStyleModelId(modelResolution.modelId);
|
|
1565
|
-
const streamingEnabledForCall = streamingEnabled && !modelResolution.forceStreamOff;
|
|
1566
|
-
writeVerbose(stderr, verbose, `mode summarize provider=${parsedModelEffective.provider} model=${parsedModelEffective.canonical}`, verboseColor);
|
|
1567
|
-
const maxOutputTokensForCall = await resolveMaxOutputTokensForCall(parsedModelEffective.canonical);
|
|
1568
|
-
const maxInputTokensForCall = await resolveMaxInputTokensForCall(parsedModelEffective.canonical);
|
|
1569
|
-
if (typeof maxInputTokensForCall === 'number' &&
|
|
1570
|
-
Number.isFinite(maxInputTokensForCall) &&
|
|
1571
|
-
maxInputTokensForCall > 0) {
|
|
1572
|
-
const tokenCount = countTokens(prompt);
|
|
1573
|
-
if (tokenCount > maxInputTokensForCall) {
|
|
1574
|
-
throw new Error(`Input token count (${formatCount(tokenCount)}) exceeds model input limit (${formatCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`);
|
|
2705
|
+
let summaryResult = null;
|
|
2706
|
+
let usedAttempt = null;
|
|
2707
|
+
let lastError = null;
|
|
2708
|
+
let sawOpenRouterNoAllowedProviders = false;
|
|
2709
|
+
const missingRequiredEnvs = new Set();
|
|
2710
|
+
for (const attempt of attempts) {
|
|
2711
|
+
const hasKey = envHasKeyFor(attempt.requiredEnv);
|
|
2712
|
+
if (!hasKey) {
|
|
2713
|
+
if (isFallbackModel) {
|
|
2714
|
+
if (isNamedModelSelection) {
|
|
2715
|
+
missingRequiredEnvs.add(attempt.requiredEnv);
|
|
2716
|
+
continue;
|
|
2717
|
+
}
|
|
2718
|
+
writeVerbose(stderr, verbose, `auto skip ${attempt.userModelId}: missing ${attempt.requiredEnv}`, verboseColor);
|
|
2719
|
+
continue;
|
|
2720
|
+
}
|
|
2721
|
+
throw new Error(formatMissingModelError(attempt));
|
|
1575
2722
|
}
|
|
1576
|
-
}
|
|
1577
|
-
const shouldBufferSummaryForRender = streamingEnabledForCall && effectiveRenderMode === 'md' && isRichTty(stdout);
|
|
1578
|
-
const shouldLiveRenderSummary = streamingEnabledForCall && effectiveRenderMode === 'md-live' && isRichTty(stdout);
|
|
1579
|
-
const shouldStreamSummaryToStdout = streamingEnabledForCall && !shouldBufferSummaryForRender && !shouldLiveRenderSummary;
|
|
1580
|
-
let summaryAlreadyPrinted = false;
|
|
1581
|
-
let summary = '';
|
|
1582
|
-
let getLastStreamError = null;
|
|
1583
|
-
writeVerbose(stderr, verbose, 'summarize strategy=single', verboseColor);
|
|
1584
|
-
if (streamingEnabledForCall) {
|
|
1585
|
-
writeVerbose(stderr, verbose, `summarize stream=on buffered=${shouldBufferSummaryForRender}`, verboseColor);
|
|
1586
|
-
let streamResult = null;
|
|
1587
2723
|
try {
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
apiKeys: apiKeysForLlm,
|
|
2724
|
+
summaryResult = await runSummaryAttempt({
|
|
2725
|
+
attempt,
|
|
1591
2726
|
prompt,
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
timeoutMs,
|
|
1595
|
-
fetchImpl: trackedFetch,
|
|
2727
|
+
allowStreaming: requestedModel.kind === 'fixed',
|
|
2728
|
+
onModelChosen,
|
|
1596
2729
|
});
|
|
2730
|
+
usedAttempt = attempt;
|
|
2731
|
+
break;
|
|
1597
2732
|
}
|
|
1598
2733
|
catch (error) {
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1605
|
-
timeoutMs,
|
|
1606
|
-
fetchImpl: trackedFetch,
|
|
1607
|
-
apiKeys: apiKeysForLlm,
|
|
1608
|
-
openrouter: openrouterOptions,
|
|
1609
|
-
});
|
|
1610
|
-
llmCalls.push({
|
|
1611
|
-
provider: result.provider,
|
|
1612
|
-
model: result.canonicalModelId,
|
|
1613
|
-
usage: result.usage,
|
|
1614
|
-
purpose: 'summary',
|
|
1615
|
-
});
|
|
1616
|
-
summary = result.text;
|
|
1617
|
-
streamResult = null;
|
|
1618
|
-
}
|
|
1619
|
-
else if (parsedModelEffective.provider === 'google' &&
|
|
1620
|
-
isGoogleStreamingUnsupportedError(error)) {
|
|
1621
|
-
writeVerbose(stderr, verbose, `Google model ${parsedModelEffective.canonical} rejected streamGenerateContent; falling back to non-streaming.`, verboseColor);
|
|
1622
|
-
const result = await summarizeWithModelId({
|
|
1623
|
-
modelId: parsedModelEffective.canonical,
|
|
1624
|
-
prompt,
|
|
1625
|
-
maxOutputTokens: maxOutputTokensForCall ?? undefined,
|
|
1626
|
-
timeoutMs,
|
|
1627
|
-
fetchImpl: trackedFetch,
|
|
1628
|
-
apiKeys: apiKeysForLlm,
|
|
1629
|
-
openrouter: openrouterOptions,
|
|
1630
|
-
});
|
|
1631
|
-
llmCalls.push({
|
|
1632
|
-
provider: result.provider,
|
|
1633
|
-
model: result.canonicalModelId,
|
|
1634
|
-
usage: result.usage,
|
|
1635
|
-
purpose: 'summary',
|
|
1636
|
-
});
|
|
1637
|
-
summary = result.text;
|
|
1638
|
-
streamResult = null;
|
|
2734
|
+
lastError = error;
|
|
2735
|
+
if (isNamedModelSelection &&
|
|
2736
|
+
error instanceof Error &&
|
|
2737
|
+
/No allowed providers are available for the selected model/i.test(error.message)) {
|
|
2738
|
+
sawOpenRouterNoAllowedProviders = true;
|
|
1639
2739
|
}
|
|
1640
|
-
|
|
2740
|
+
if (requestedModel.kind === 'fixed') {
|
|
1641
2741
|
throw error;
|
|
1642
2742
|
}
|
|
2743
|
+
writeVerbose(stderr, verbose, `auto failed ${attempt.userModelId}: ${error instanceof Error ? error.message : String(error)}`, verboseColor);
|
|
1643
2744
|
}
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
width: markdownRenderWidth(stdout, env),
|
|
1656
|
-
wrap: true,
|
|
1657
|
-
color: supportsColor(stdout, env),
|
|
1658
|
-
}),
|
|
1659
|
-
})
|
|
1660
|
-
: null;
|
|
1661
|
-
let lastFrameAtMs = 0;
|
|
1662
|
-
try {
|
|
1663
|
-
let cleared = false;
|
|
1664
|
-
for await (const delta of streamResult.textStream) {
|
|
1665
|
-
const merged = mergeStreamingChunk(streamed, delta);
|
|
1666
|
-
streamed = merged.next;
|
|
1667
|
-
if (shouldStreamSummaryToStdout) {
|
|
1668
|
-
if (!cleared) {
|
|
1669
|
-
clearProgressForStdout();
|
|
1670
|
-
cleared = true;
|
|
1671
|
-
}
|
|
1672
|
-
if (merged.appended)
|
|
1673
|
-
stdout.write(merged.appended);
|
|
1674
|
-
continue;
|
|
1675
|
-
}
|
|
1676
|
-
if (liveRenderer) {
|
|
1677
|
-
const now = Date.now();
|
|
1678
|
-
const due = now - lastFrameAtMs >= 120;
|
|
1679
|
-
const hasNewline = delta.includes('\n');
|
|
1680
|
-
if (hasNewline || due) {
|
|
1681
|
-
liveRenderer.render(streamed);
|
|
1682
|
-
lastFrameAtMs = now;
|
|
1683
|
-
}
|
|
1684
|
-
}
|
|
1685
|
-
}
|
|
1686
|
-
const trimmed = streamed.trim();
|
|
1687
|
-
streamed = trimmed;
|
|
1688
|
-
if (liveRenderer) {
|
|
1689
|
-
liveRenderer.render(trimmed);
|
|
1690
|
-
summaryAlreadyPrinted = true;
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
finally {
|
|
1694
|
-
liveRenderer?.finish();
|
|
2745
|
+
}
|
|
2746
|
+
if (!summaryResult || !usedAttempt) {
|
|
2747
|
+
const withFreeTip = (message) => {
|
|
2748
|
+
if (!isNamedModelSelection || !wantsFreeNamedModel)
|
|
2749
|
+
return message;
|
|
2750
|
+
return (`${message}\n` +
|
|
2751
|
+
`Tip: run "summarize refresh-free" to refresh the free model candidates (writes ~/.summarize/config.json).`);
|
|
2752
|
+
};
|
|
2753
|
+
if (isNamedModelSelection) {
|
|
2754
|
+
if (lastError === null && missingRequiredEnvs.size > 0) {
|
|
2755
|
+
throw new Error(withFreeTip(`Missing ${Array.from(missingRequiredEnvs).sort().join(', ')} for --model ${requestedModelInput}.`));
|
|
1695
2756
|
}
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
if (shouldStreamSummaryToStdout) {
|
|
1705
|
-
if (!streamed.endsWith('\n')) {
|
|
1706
|
-
stdout.write('\n');
|
|
2757
|
+
if (lastError instanceof Error) {
|
|
2758
|
+
if (sawOpenRouterNoAllowedProviders) {
|
|
2759
|
+
const message = await buildOpenRouterNoAllowedProvidersMessage({
|
|
2760
|
+
attempts,
|
|
2761
|
+
fetchImpl: trackedFetch,
|
|
2762
|
+
timeoutMs,
|
|
2763
|
+
});
|
|
2764
|
+
throw new Error(withFreeTip(message), { cause: lastError });
|
|
1707
2765
|
}
|
|
1708
|
-
|
|
2766
|
+
throw new Error(withFreeTip(lastError.message), { cause: lastError });
|
|
1709
2767
|
}
|
|
2768
|
+
throw new Error(withFreeTip(`No model available for --model ${requestedModelInput}`));
|
|
1710
2769
|
}
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
2770
|
+
clearProgressForStdout();
|
|
2771
|
+
if (json) {
|
|
2772
|
+
const finishReport = shouldComputeReport ? await buildReport() : null;
|
|
2773
|
+
const payload = {
|
|
2774
|
+
input: {
|
|
2775
|
+
kind: 'url',
|
|
2776
|
+
url,
|
|
2777
|
+
timeoutMs,
|
|
2778
|
+
youtube: youtubeMode,
|
|
2779
|
+
firecrawl: firecrawlMode,
|
|
2780
|
+
format,
|
|
2781
|
+
markdown: effectiveMarkdownMode,
|
|
2782
|
+
length: lengthArg.kind === 'preset'
|
|
2783
|
+
? { kind: 'preset', preset: lengthArg.preset }
|
|
2784
|
+
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
2785
|
+
maxOutputTokens: maxOutputTokensArg,
|
|
2786
|
+
model: requestedModelLabel,
|
|
2787
|
+
},
|
|
2788
|
+
env: {
|
|
2789
|
+
hasXaiKey: Boolean(xaiApiKey),
|
|
2790
|
+
hasOpenAIKey: Boolean(apiKey),
|
|
2791
|
+
hasOpenRouterKey: Boolean(openrouterApiKey),
|
|
2792
|
+
hasApifyToken: Boolean(apifyToken),
|
|
2793
|
+
hasFirecrawlKey: firecrawlConfigured,
|
|
2794
|
+
hasGoogleKey: googleConfigured,
|
|
2795
|
+
hasAnthropicKey: anthropicConfigured,
|
|
2796
|
+
},
|
|
2797
|
+
extracted,
|
|
2798
|
+
prompt,
|
|
2799
|
+
llm: null,
|
|
2800
|
+
metrics: metricsEnabled ? finishReport : null,
|
|
2801
|
+
summary: extracted.content,
|
|
2802
|
+
};
|
|
2803
|
+
stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
2804
|
+
return;
|
|
1735
2805
|
}
|
|
1736
|
-
|
|
2806
|
+
stdout.write(`${extracted.content}\n`);
|
|
2807
|
+
if (footerBaseParts.length > 0) {
|
|
2808
|
+
writeViaFooter([...footerBaseParts, 'no model']);
|
|
2809
|
+
}
|
|
2810
|
+
if (lastError instanceof Error && verbose) {
|
|
2811
|
+
writeVerbose(stderr, verbose, `auto failed all models: ${lastError.message}`, verboseColor);
|
|
2812
|
+
}
|
|
2813
|
+
return;
|
|
1737
2814
|
}
|
|
2815
|
+
const { summary, summaryAlreadyPrinted, modelMeta, maxOutputTokensForCall } = summaryResult;
|
|
1738
2816
|
if (json) {
|
|
1739
2817
|
const finishReport = shouldComputeReport ? await buildReport() : null;
|
|
1740
2818
|
const payload = {
|
|
@@ -1744,16 +2822,18 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1744
2822
|
timeoutMs,
|
|
1745
2823
|
youtube: youtubeMode,
|
|
1746
2824
|
firecrawl: firecrawlMode,
|
|
2825
|
+
format,
|
|
1747
2826
|
markdown: effectiveMarkdownMode,
|
|
1748
2827
|
length: lengthArg.kind === 'preset'
|
|
1749
2828
|
? { kind: 'preset', preset: lengthArg.preset }
|
|
1750
2829
|
: { kind: 'chars', maxCharacters: lengthArg.maxCharacters },
|
|
1751
2830
|
maxOutputTokens: maxOutputTokensArg,
|
|
1752
|
-
model,
|
|
2831
|
+
model: requestedModelLabel,
|
|
1753
2832
|
},
|
|
1754
2833
|
env: {
|
|
1755
2834
|
hasXaiKey: Boolean(xaiApiKey),
|
|
1756
2835
|
hasOpenAIKey: Boolean(apiKey),
|
|
2836
|
+
hasOpenRouterKey: Boolean(openrouterApiKey),
|
|
1757
2837
|
hasApifyToken: Boolean(apifyToken),
|
|
1758
2838
|
hasFirecrawlKey: firecrawlConfigured,
|
|
1759
2839
|
hasGoogleKey: googleConfigured,
|
|
@@ -1762,26 +2842,25 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1762
2842
|
extracted,
|
|
1763
2843
|
prompt,
|
|
1764
2844
|
llm: {
|
|
1765
|
-
provider:
|
|
1766
|
-
model:
|
|
2845
|
+
provider: modelMeta.provider,
|
|
2846
|
+
model: usedAttempt.userModelId,
|
|
1767
2847
|
maxCompletionTokens: maxOutputTokensForCall,
|
|
1768
2848
|
strategy: 'single',
|
|
1769
2849
|
},
|
|
1770
2850
|
metrics: metricsEnabled ? finishReport : null,
|
|
1771
2851
|
summary,
|
|
1772
2852
|
};
|
|
1773
|
-
if (metricsDetailed && finishReport) {
|
|
1774
|
-
writeMetricsReport(finishReport);
|
|
1775
|
-
}
|
|
1776
2853
|
stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
1777
2854
|
if (metricsEnabled && finishReport) {
|
|
1778
2855
|
const costUsd = await estimateCostUsd();
|
|
1779
2856
|
writeFinishLine({
|
|
1780
2857
|
stderr,
|
|
1781
2858
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1782
|
-
model:
|
|
2859
|
+
model: usedAttempt.userModelId,
|
|
1783
2860
|
report: finishReport,
|
|
1784
2861
|
costUsd,
|
|
2862
|
+
detailed: metricsDetailed,
|
|
2863
|
+
extraParts: metricsDetailed ? buildDetailedLengthPartsForExtracted(extracted) : null,
|
|
1785
2864
|
color: verboseColor,
|
|
1786
2865
|
});
|
|
1787
2866
|
}
|
|
@@ -1802,16 +2881,16 @@ export async function runCli(argv, { env, fetch, stdout, stderr }) {
|
|
|
1802
2881
|
}
|
|
1803
2882
|
}
|
|
1804
2883
|
const report = shouldComputeReport ? await buildReport() : null;
|
|
1805
|
-
if (metricsDetailed && report)
|
|
1806
|
-
writeMetricsReport(report);
|
|
1807
2884
|
if (metricsEnabled && report) {
|
|
1808
2885
|
const costUsd = await estimateCostUsd();
|
|
1809
2886
|
writeFinishLine({
|
|
1810
2887
|
stderr,
|
|
1811
2888
|
elapsedMs: Date.now() - runStartedAtMs,
|
|
1812
|
-
model:
|
|
2889
|
+
model: modelMeta.canonical,
|
|
1813
2890
|
report,
|
|
1814
2891
|
costUsd,
|
|
2892
|
+
detailed: metricsDetailed,
|
|
2893
|
+
extraParts: metricsDetailed ? buildDetailedLengthPartsForExtracted(extracted) : null,
|
|
1815
2894
|
color: verboseColor,
|
|
1816
2895
|
});
|
|
1817
2896
|
}
|