@steipete/summarize 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/LICENSE +21 -0
- package/README.md +185 -0
- package/dist/cli.cjs +74333 -0
- package/dist/cli.cjs.map +7 -0
- package/dist/esm/cli-main.js +80 -0
- package/dist/esm/cli-main.js.map +1 -0
- package/dist/esm/cli.js +18 -0
- package/dist/esm/cli.js.map +1 -0
- package/dist/esm/config.js +33 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/content/asset.js +167 -0
- package/dist/esm/content/asset.js.map +1 -0
- package/dist/esm/content/index.js +4 -0
- package/dist/esm/content/index.js.map +1 -0
- package/dist/esm/content/link-preview/client.js +20 -0
- package/dist/esm/content/link-preview/client.js.map +1 -0
- package/dist/esm/content/link-preview/content/article.js +150 -0
- package/dist/esm/content/link-preview/content/article.js.map +1 -0
- package/dist/esm/content/link-preview/content/cleaner.js +55 -0
- package/dist/esm/content/link-preview/content/cleaner.js.map +1 -0
- package/dist/esm/content/link-preview/content/fetcher.js +120 -0
- package/dist/esm/content/link-preview/content/fetcher.js.map +1 -0
- package/dist/esm/content/link-preview/content/index.js +275 -0
- package/dist/esm/content/link-preview/content/index.js.map +1 -0
- package/dist/esm/content/link-preview/content/parsers.js +77 -0
- package/dist/esm/content/link-preview/content/parsers.js.map +1 -0
- package/dist/esm/content/link-preview/content/types.js +4 -0
- package/dist/esm/content/link-preview/content/types.js.map +1 -0
- package/dist/esm/content/link-preview/content/utils.js +127 -0
- package/dist/esm/content/link-preview/content/utils.js.map +1 -0
- package/dist/esm/content/link-preview/content/youtube.js +82 -0
- package/dist/esm/content/link-preview/content/youtube.js.map +1 -0
- package/dist/esm/content/link-preview/deps.js +2 -0
- package/dist/esm/content/link-preview/deps.js.map +1 -0
- package/dist/esm/content/link-preview/fetch-with-timeout.js +35 -0
- package/dist/esm/content/link-preview/fetch-with-timeout.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/cache.js +73 -0
- package/dist/esm/content/link-preview/transcript/cache.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/index.js +95 -0
- package/dist/esm/content/link-preview/transcript/index.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/normalize.js +43 -0
- package/dist/esm/content/link-preview/transcript/normalize.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/generic.js +11 -0
- package/dist/esm/content/link-preview/transcript/providers/generic.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/podcast.js +12 -0
- package/dist/esm/content/link-preview/transcript/providers/podcast.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/twitter.js +12 -0
- package/dist/esm/content/link-preview/transcript/providers/twitter.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/api.js +257 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/api.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/apify.js +55 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/apify.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/captions.js +409 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/captions.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/ytdlp.js +114 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube/ytdlp.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube.js +74 -0
- package/dist/esm/content/link-preview/transcript/providers/youtube.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/types.js +2 -0
- package/dist/esm/content/link-preview/transcript/types.js.map +1 -0
- package/dist/esm/content/link-preview/transcript/utils.js +193 -0
- package/dist/esm/content/link-preview/transcript/utils.js.map +1 -0
- package/dist/esm/content/link-preview/types.js +2 -0
- package/dist/esm/content/link-preview/types.js.map +1 -0
- package/dist/esm/costs.js +57 -0
- package/dist/esm/costs.js.map +1 -0
- package/dist/esm/firecrawl.js +54 -0
- package/dist/esm/firecrawl.js.map +1 -0
- package/dist/esm/flags.js +97 -0
- package/dist/esm/flags.js.map +1 -0
- package/dist/esm/index.js +4 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/llm/generate-text.js +296 -0
- package/dist/esm/llm/generate-text.js.map +1 -0
- package/dist/esm/llm/google-models.js +112 -0
- package/dist/esm/llm/google-models.js.map +1 -0
- package/dist/esm/llm/html-to-markdown.js +44 -0
- package/dist/esm/llm/html-to-markdown.js.map +1 -0
- package/dist/esm/llm/model-id.js +45 -0
- package/dist/esm/llm/model-id.js.map +1 -0
- package/dist/esm/pricing/litellm.js +25 -0
- package/dist/esm/pricing/litellm.js.map +1 -0
- package/dist/esm/prompts/file.js +14 -0
- package/dist/esm/prompts/file.js.map +1 -0
- package/dist/esm/prompts/index.js +3 -0
- package/dist/esm/prompts/index.js.map +1 -0
- package/dist/esm/prompts/link-summary.js +105 -0
- package/dist/esm/prompts/link-summary.js.map +1 -0
- package/dist/esm/run.js +1674 -0
- package/dist/esm/run.js.map +1 -0
- package/dist/esm/shared/contracts.js +2 -0
- package/dist/esm/shared/contracts.js.map +1 -0
- package/dist/esm/summarizeHome.js +20 -0
- package/dist/esm/summarizeHome.js.map +1 -0
- package/dist/esm/tty/live-markdown.js +52 -0
- package/dist/esm/tty/live-markdown.js.map +1 -0
- package/dist/esm/tty/osc-progress.js +8 -0
- package/dist/esm/tty/osc-progress.js.map +1 -0
- package/dist/esm/tty/spinner.js +33 -0
- package/dist/esm/tty/spinner.js.map +1 -0
- package/dist/esm/version.js +44 -0
- package/dist/esm/version.js.map +1 -0
- package/dist/types/cli-main.d.ts +11 -0
- package/dist/types/cli.d.ts +1 -0
- package/dist/types/config.d.ts +15 -0
- package/dist/types/content/asset.d.ts +44 -0
- package/dist/types/content/index.d.ts +4 -0
- package/dist/types/content/link-preview/client.d.ts +14 -0
- package/dist/types/content/link-preview/content/article.d.ts +4 -0
- package/dist/types/content/link-preview/content/cleaner.d.ts +12 -0
- package/dist/types/content/link-preview/content/fetcher.d.ts +16 -0
- package/dist/types/content/link-preview/content/index.d.ts +4 -0
- package/dist/types/content/link-preview/content/parsers.d.ts +7 -0
- package/dist/types/content/link-preview/content/types.d.ts +44 -0
- package/dist/types/content/link-preview/content/utils.d.ts +16 -0
- package/dist/types/content/link-preview/content/youtube.d.ts +1 -0
- package/dist/types/content/link-preview/deps.d.ts +70 -0
- package/dist/types/content/link-preview/fetch-with-timeout.d.ts +4 -0
- package/dist/types/content/link-preview/transcript/cache.d.ts +29 -0
- package/dist/types/content/link-preview/transcript/index.d.ts +9 -0
- package/dist/types/content/link-preview/transcript/normalize.d.ts +3 -0
- package/dist/types/content/link-preview/transcript/providers/generic.d.ts +3 -0
- package/dist/types/content/link-preview/transcript/providers/podcast.d.ts +3 -0
- package/dist/types/content/link-preview/transcript/providers/twitter.d.ts +3 -0
- package/dist/types/content/link-preview/transcript/providers/youtube/api.d.ts +26 -0
- package/dist/types/content/link-preview/transcript/providers/youtube/apify.d.ts +1 -0
- package/dist/types/content/link-preview/transcript/providers/youtube/captions.d.ts +7 -0
- package/dist/types/content/link-preview/transcript/providers/youtube/ytdlp.d.ts +3 -0
- package/dist/types/content/link-preview/transcript/providers/youtube.d.ts +3 -0
- package/dist/types/content/link-preview/transcript/types.d.ts +23 -0
- package/dist/types/content/link-preview/transcript/utils.d.ts +7 -0
- package/dist/types/content/link-preview/types.d.ts +36 -0
- package/dist/types/costs.d.ts +31 -0
- package/dist/types/firecrawl.d.ts +5 -0
- package/dist/types/flags.d.ts +23 -0
- package/dist/types/index.d.ts +4 -0
- package/dist/types/llm/generate-text.d.ts +43 -0
- package/dist/types/llm/google-models.d.ts +10 -0
- package/dist/types/llm/html-to-markdown.d.ts +15 -0
- package/dist/types/llm/model-id.d.ts +14 -0
- package/dist/types/pricing/litellm.d.ts +13 -0
- package/dist/types/prompts/file.d.ts +6 -0
- package/dist/types/prompts/index.d.ts +3 -0
- package/dist/types/prompts/link-summary.d.ts +27 -0
- package/dist/types/run.d.ts +8 -0
- package/dist/types/shared/contracts.d.ts +2 -0
- package/dist/types/summarizeHome.d.ts +6 -0
- package/dist/types/tty/live-markdown.d.ts +10 -0
- package/dist/types/tty/osc-progress.d.ts +3 -0
- package/dist/types/tty/spinner.d.ts +10 -0
- package/dist/types/version.d.ts +2 -0
- package/docs/README.md +11 -0
- package/docs/config.md +28 -0
- package/docs/extract-only.md +13 -0
- package/docs/firecrawl.md +17 -0
- package/docs/llm.md +33 -0
- package/docs/openai.md +18 -0
- package/docs/site/.nojekyll +1 -0
- package/docs/site/404.html +37 -0
- package/docs/site/assets/site.css +577 -0
- package/docs/site/assets/site.js +69 -0
- package/docs/site/docs/config.html +73 -0
- package/docs/site/docs/extract-only.html +79 -0
- package/docs/site/docs/firecrawl.html +72 -0
- package/docs/site/docs/index.html +89 -0
- package/docs/site/docs/llm.html +70 -0
- package/docs/site/docs/openai.html +66 -0
- package/docs/site/docs/website.html +70 -0
- package/docs/site/docs/youtube.html +62 -0
- package/docs/site/index.html +125 -0
- package/docs/website.md +27 -0
- package/docs/youtube.md +32 -0
- package/package.json +76 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { load } from 'cheerio';
|
|
2
|
+
export function isRecord(value) {
|
|
3
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
4
|
+
}
|
|
5
|
+
export const isYouTubeUrl = (rawUrl) => {
|
|
6
|
+
try {
|
|
7
|
+
const hostname = new URL(rawUrl).hostname.toLowerCase();
|
|
8
|
+
return hostname.includes('youtube.com') || hostname.includes('youtu.be');
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
const lower = rawUrl.toLowerCase();
|
|
12
|
+
return lower.includes('youtube.com') || lower.includes('youtu.be');
|
|
13
|
+
}
|
|
14
|
+
};
|
|
15
|
+
const YOUTUBE_VIDEO_ID_PATTERN = /^[a-zA-Z0-9_-]{11}$/;
|
|
16
|
+
export function isYouTubeVideoUrl(rawUrl) {
|
|
17
|
+
try {
|
|
18
|
+
const url = new URL(rawUrl);
|
|
19
|
+
const hostname = url.hostname.toLowerCase();
|
|
20
|
+
if (hostname === 'youtu.be') {
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
if (hostname.includes('youtube.com')) {
|
|
24
|
+
return (url.pathname.startsWith('/watch') ||
|
|
25
|
+
url.pathname.startsWith('/shorts/') ||
|
|
26
|
+
url.pathname.startsWith('/embed/') ||
|
|
27
|
+
url.pathname.startsWith('/v/'));
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
export function extractYouTubeVideoId(rawUrl) {
|
|
36
|
+
try {
|
|
37
|
+
const url = new URL(rawUrl);
|
|
38
|
+
const hostname = url.hostname.toLowerCase();
|
|
39
|
+
let candidate = null;
|
|
40
|
+
if (hostname === 'youtu.be') {
|
|
41
|
+
candidate = url.pathname.split('/')[1] ?? null;
|
|
42
|
+
}
|
|
43
|
+
if (hostname.includes('youtube.com')) {
|
|
44
|
+
if (url.pathname.startsWith('/watch')) {
|
|
45
|
+
candidate = url.searchParams.get('v');
|
|
46
|
+
}
|
|
47
|
+
else if (url.pathname.startsWith('/shorts/')) {
|
|
48
|
+
candidate = url.pathname.split('/')[2] ?? null;
|
|
49
|
+
}
|
|
50
|
+
else if (url.pathname.startsWith('/embed/')) {
|
|
51
|
+
candidate = url.pathname.split('/')[2] ?? null;
|
|
52
|
+
}
|
|
53
|
+
else if (url.pathname.startsWith('/v/')) {
|
|
54
|
+
candidate = url.pathname.split('/')[2] ?? null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const trimmed = candidate?.trim() ?? '';
|
|
58
|
+
if (!trimmed) {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
return YOUTUBE_VIDEO_ID_PATTERN.test(trimmed) ? trimmed : null;
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
// Ignore parsing errors for malformed URLs
|
|
65
|
+
}
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
export function sanitizeYoutubeJsonResponse(input) {
|
|
69
|
+
const trimmed = input.trimStart();
|
|
70
|
+
if (trimmed.startsWith(")]}'")) {
|
|
71
|
+
return trimmed.slice(4);
|
|
72
|
+
}
|
|
73
|
+
return trimmed;
|
|
74
|
+
}
|
|
75
|
+
export function decodeHtmlEntities(input) {
|
|
76
|
+
return input
|
|
77
|
+
.replaceAll('&', '&')
|
|
78
|
+
.replaceAll('<', '<')
|
|
79
|
+
.replaceAll('>', '>')
|
|
80
|
+
.replaceAll('"', '"')
|
|
81
|
+
.replaceAll(''', "'")
|
|
82
|
+
.replaceAll(''', "'")
|
|
83
|
+
.replaceAll('/', '/')
|
|
84
|
+
.replaceAll(' ', ' ');
|
|
85
|
+
}
|
|
86
|
+
export function extractYoutubeBootstrapConfig(html) {
|
|
87
|
+
try {
|
|
88
|
+
const $ = load(html);
|
|
89
|
+
const scripts = $('script').toArray();
|
|
90
|
+
for (const script of scripts) {
|
|
91
|
+
const source = $(script).html();
|
|
92
|
+
if (!source) {
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
const config = parseBootstrapFromScript(source);
|
|
96
|
+
if (config) {
|
|
97
|
+
return config;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
// fall through to legacy regex
|
|
103
|
+
}
|
|
104
|
+
return parseBootstrapFromScript(html);
|
|
105
|
+
}
|
|
106
|
+
const YTCFG_SET_TOKEN = 'ytcfg.set';
|
|
107
|
+
const YTCFG_VAR_TOKEN = 'var ytcfg';
|
|
108
|
+
function extractBalancedJsonObject(source, startAt) {
|
|
109
|
+
const start = source.indexOf('{', startAt);
|
|
110
|
+
if (start < 0) {
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
let depth = 0;
|
|
114
|
+
let inString = false;
|
|
115
|
+
let quote = null;
|
|
116
|
+
let escaping = false;
|
|
117
|
+
for (let i = start; i < source.length; i += 1) {
|
|
118
|
+
const ch = source[i];
|
|
119
|
+
if (!ch) {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
if (inString) {
|
|
123
|
+
if (escaping) {
|
|
124
|
+
escaping = false;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (ch === '\\') {
|
|
128
|
+
escaping = true;
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
if (quote && ch === quote) {
|
|
132
|
+
inString = false;
|
|
133
|
+
quote = null;
|
|
134
|
+
}
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (ch === '"' || ch === "'") {
|
|
138
|
+
inString = true;
|
|
139
|
+
quote = ch;
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (ch === '{') {
|
|
143
|
+
depth += 1;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (ch === '}') {
|
|
147
|
+
depth -= 1;
|
|
148
|
+
if (depth === 0) {
|
|
149
|
+
return source.slice(start, i + 1);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
function parseBootstrapFromScript(source) {
|
|
156
|
+
const sanitizedSource = sanitizeYoutubeJsonResponse(source.trimStart());
|
|
157
|
+
for (let index = 0; index >= 0;) {
|
|
158
|
+
index = sanitizedSource.indexOf(YTCFG_SET_TOKEN, index);
|
|
159
|
+
if (index < 0) {
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
const object = extractBalancedJsonObject(sanitizedSource, index);
|
|
163
|
+
if (object) {
|
|
164
|
+
try {
|
|
165
|
+
const parsed = JSON.parse(object);
|
|
166
|
+
if (isRecord(parsed)) {
|
|
167
|
+
return parsed;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
// keep searching
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
index += YTCFG_SET_TOKEN.length;
|
|
175
|
+
}
|
|
176
|
+
const varIndex = sanitizedSource.indexOf(YTCFG_VAR_TOKEN);
|
|
177
|
+
if (varIndex >= 0) {
|
|
178
|
+
const object = extractBalancedJsonObject(sanitizedSource, varIndex);
|
|
179
|
+
if (object) {
|
|
180
|
+
try {
|
|
181
|
+
const parsed = JSON.parse(object);
|
|
182
|
+
if (isRecord(parsed)) {
|
|
183
|
+
return parsed;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return null;
|
|
192
|
+
}
|
|
193
|
+
//# sourceMappingURL=utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../../src/content/link-preview/transcript/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAE9B,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC7E,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,EAAW,EAAE;IACtD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;QACvD,OAAO,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAC1E,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;QAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IACpE,CAAC;AACH,CAAC,CAAA;AAED,MAAM,wBAAwB,GAAG,qBAAqB,CAAA;AAEtD,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAA;QAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;QAC3C,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;YAC5B,OAAO,IAAI,CAAA;QACb,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACrC,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC;gBACjC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;gBACnC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC;gBAClC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAC/B,CAAA;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAClD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAA;QAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAA;QAC3C,IAAI,SAAS,GAAkB,IAAI,CAAA;QACnC,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;YAC5B,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;QAChD,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACtC,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACvC,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/C,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,CAAC;iBAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1C,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QACvC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,IAAI,CAAA;QACb,CAAC;QACD,OAAO,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAA;IAChE,CAAC;IAAC,MAAM,CAAC;QACP,2CAA2C;IAC7C,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,KAAa;IACvD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,EAAE,CAAA;IACjC,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC/B,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACzB,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,OAAO,KAAK;SACT,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC;SACvB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC;SACxB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC;SACzB,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,IAAY;IACxD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAA;QACpB,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAA;QAErC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAA;YAC/B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAQ;YACV,CAAC;YAED,MAAM,MAAM,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAA;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,MAAM,CAAA;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IAED,OAAO,wBAAwB,CAAC,IAAI,CAAC,CAAA;AACvC,CAAC;AAED,MAAM,eAAe,GAAG,WAAW,CAAA;AACnC,MAAM,eAAe,GAAG,WAAW,CAAA;AAEnC,SAAS,yBAAyB,CAAC,MAAc,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAC1C,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,QAAQ,GAAG,KAAK,CAAA;IACpB,IAAI,KAAK,GAAqB,IAAI,CAAA;IAClC,IAAI,QAAQ,GAAG,KAAK,CAAA;IAEpB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QACpB,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,SAAQ;QACV,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,GAAG,KAAK,CAAA;gBAChB,SAAQ;YACV,CAAC;YACD,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBAChB,QAAQ,GAAG,IAAI,CAAA;gBACf,SAAQ;YACV,CAAC;YACD,IAAI,KAAK,IAAI,EAAE,KAAK,KAAK,EAAE,CAAC;gBAC1B,QAAQ,GAAG,KAAK,CAAA;gBAChB,KAAK,GAAG,IAAI,CAAA;YACd,CAAC;YACD,SAAQ;QACV,CAAC;QAED,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAC7B,QAAQ,GAAG,IAAI,CAAA;YACf,KAAK,GAAG,EAAE,CAAA;YACV,SAAQ;QACV,CAAC;QAED,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,KAAK,IAAI,CAAC,CAAA;YACV,SAAQ;QACV,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,KAAK,IAAI,CAAC,CAAA;YACV,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAA;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,wBAAwB,CAAC,MAAc;IAC9C,MAAM,eAAe,GAAG,2BAA2B,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAA;IAEvE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,GAAI,CAAC;QACjC,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,eAAe,EAAE,KAAK,CAAC,CAAA;QACvD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,MAAK;QACP,CAAC;QACD,MAAM,MAAM,GAAG,yBAAyB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAA;QAChE,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;gBAC1C,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACrB,OAAO,MAAM,CAAA;gBACf,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB;YACnB,CAAC;QACH,CAAC;QACD,KAAK,IAAI,eAAe,CAAC,MAAM,CAAA;IACjC,CAAC;IAED,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,CAAC,eAAe,CAAC,CAAA;IACzD,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,yBAAyB,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAA;QACnE,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;gBAC1C,IAAI,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACrB,OAAO,MAAM,CAAA;gBACf,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACb,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/content/link-preview/types.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAU,CAAA"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
function sumOrNull(values) {
|
|
2
|
+
let sum = 0;
|
|
3
|
+
let any = false;
|
|
4
|
+
for (const value of values) {
|
|
5
|
+
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
6
|
+
sum += value;
|
|
7
|
+
any = true;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
return any ? sum : null;
|
|
11
|
+
}
|
|
12
|
+
export function buildRunMetricsReport({ llmCalls, firecrawlRequests, apifyRequests, }) {
|
|
13
|
+
const llmMap = new Map();
|
|
14
|
+
for (const call of llmCalls) {
|
|
15
|
+
const key = `${call.provider}:${call.model}`;
|
|
16
|
+
const existing = llmMap.get(key);
|
|
17
|
+
const promptTokens = call.usage?.promptTokens ?? null;
|
|
18
|
+
const completionTokens = call.usage?.completionTokens ?? null;
|
|
19
|
+
const totalTokens = call.usage?.totalTokens ?? null;
|
|
20
|
+
if (!existing) {
|
|
21
|
+
llmMap.set(key, {
|
|
22
|
+
provider: call.provider,
|
|
23
|
+
model: call.model,
|
|
24
|
+
calls: 1,
|
|
25
|
+
promptTokens: [promptTokens],
|
|
26
|
+
completionTokens: [completionTokens],
|
|
27
|
+
totalTokens: [totalTokens],
|
|
28
|
+
});
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
existing.calls += 1;
|
|
32
|
+
existing.promptTokens.push(promptTokens);
|
|
33
|
+
existing.completionTokens.push(completionTokens);
|
|
34
|
+
existing.totalTokens.push(totalTokens);
|
|
35
|
+
}
|
|
36
|
+
const llm = Array.from(llmMap.values()).map((row) => {
|
|
37
|
+
const promptTokens = sumOrNull(row.promptTokens);
|
|
38
|
+
const completionTokens = sumOrNull(row.completionTokens);
|
|
39
|
+
const totalTokens = sumOrNull(row.totalTokens);
|
|
40
|
+
return {
|
|
41
|
+
provider: row.provider,
|
|
42
|
+
model: row.model,
|
|
43
|
+
calls: row.calls,
|
|
44
|
+
promptTokens,
|
|
45
|
+
completionTokens,
|
|
46
|
+
totalTokens,
|
|
47
|
+
};
|
|
48
|
+
});
|
|
49
|
+
return {
|
|
50
|
+
llm,
|
|
51
|
+
services: {
|
|
52
|
+
firecrawl: { requests: firecrawlRequests },
|
|
53
|
+
apify: { requests: apifyRequests },
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=costs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"costs.js","sourceRoot":"","sources":["../../src/costs.ts"],"names":[],"mappings":"AA0BA,SAAS,SAAS,CAAC,MAA4B;IAC7C,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,IAAI,GAAG,GAAG,KAAK,CAAA;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,GAAG,IAAI,KAAK,CAAA;YACZ,GAAG,GAAG,IAAI,CAAA;QACZ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAA;AACzB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,EACpC,QAAQ,EACR,iBAAiB,EACjB,aAAa,GAKd;IACC,MAAM,MAAM,GAAG,IAAI,GAAG,EAUnB,CAAA;IAEH,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,EAAE,CAAA;QAC5C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAChC,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,IAAI,CAAA;QACrD,MAAM,gBAAgB,GAAG,IAAI,CAAC,KAAK,EAAE,gBAAgB,IAAI,IAAI,CAAA;QAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,WAAW,IAAI,IAAI,CAAA;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE;gBACd,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,CAAC;gBACR,YAAY,EAAE,CAAC,YAAY,CAAC;gBAC5B,gBAAgB,EAAE,CAAC,gBAAgB,CAAC;gBACpC,WAAW,EAAE,CAAC,WAAW,CAAC;aAC3B,CAAC,CAAA;YACF,SAAQ;QACV,CAAC;QACD,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAA;QACnB,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;QACxC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAChD,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACxC,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAClD,MAAM,YAAY,GAAG,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAA;QAChD,MAAM,gBAAgB,GAAG,SAAS,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;QACxD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QAC9C,OAAO;YACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,YAAY;YACZ,gBAAgB;YAChB,WAAW;SACZ,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,OAAO;QACL,GAAG;QACH,QAAQ,EAAE;YACR,SAAS,EAAE,EAAE,QAAQ,EAAE,iBAAiB,EAAE;YAC1C,KAAK,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE;SACnC;KACF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export function createFirecrawlScraper({ apiKey, fetchImpl, }) {
|
|
2
|
+
return async (url, options) => {
|
|
3
|
+
const controller = new AbortController();
|
|
4
|
+
const timeoutMs = options?.timeoutMs;
|
|
5
|
+
const hasTimeout = typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs > 0;
|
|
6
|
+
const timeout = hasTimeout ? setTimeout(() => controller.abort(), timeoutMs) : null;
|
|
7
|
+
try {
|
|
8
|
+
const response = await fetchImpl('https://api.firecrawl.dev/v1/scrape', {
|
|
9
|
+
method: 'POST',
|
|
10
|
+
headers: {
|
|
11
|
+
Authorization: `Bearer ${apiKey}`,
|
|
12
|
+
'Content-Type': 'application/json',
|
|
13
|
+
},
|
|
14
|
+
signal: controller.signal,
|
|
15
|
+
body: JSON.stringify({
|
|
16
|
+
url,
|
|
17
|
+
formats: ['markdown', 'html'],
|
|
18
|
+
onlyMainContent: true,
|
|
19
|
+
proxy: 'auto',
|
|
20
|
+
maxAge: 0,
|
|
21
|
+
}),
|
|
22
|
+
});
|
|
23
|
+
const payload = (await response.json().catch(() => null));
|
|
24
|
+
if (!response.ok) {
|
|
25
|
+
const message = payload?.error ? `: ${payload.error}` : '';
|
|
26
|
+
throw new Error(`Firecrawl request failed (${response.status})${message}`);
|
|
27
|
+
}
|
|
28
|
+
if (!payload?.success) {
|
|
29
|
+
throw new Error(payload?.error ?? 'Firecrawl response was not successful');
|
|
30
|
+
}
|
|
31
|
+
const data = payload.data;
|
|
32
|
+
const markdown = data?.markdown ?? null;
|
|
33
|
+
if (typeof markdown !== 'string' || markdown.trim().length === 0) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
markdown,
|
|
38
|
+
html: typeof data?.html === 'string' ? data.html : null,
|
|
39
|
+
metadata: data?.metadata ?? null,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
44
|
+
throw new Error('Firecrawl request timed out');
|
|
45
|
+
}
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
if (timeout)
|
|
50
|
+
clearTimeout(timeout);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=firecrawl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../src/firecrawl.ts"],"names":[],"mappings":"AAYA,MAAM,UAAU,sBAAsB,CAAC,EACrC,MAAM,EACN,SAAS,GAIV;IACC,OAAO,KAAK,EACV,GAAW,EACX,OAAgC,EACO,EAAE;QACzC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;QACxC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,CAAA;QACpC,MAAM,UAAU,GAAG,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,CAAA;QAC/F,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAEnF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,qCAAqC,EAAE;gBACtE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,MAAM,EAAE;oBACjC,cAAc,EAAE,kBAAkB;iBACnC;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,GAAG;oBACH,OAAO,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;oBAC7B,eAAe,EAAE,IAAI;oBACrB,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,CAAC;iBACV,CAAC;aACH,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAA6B,CAAA;YAErF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,OAAO,GAAG,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;gBAC1D,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,CAAC,MAAM,IAAI,OAAO,EAAE,CAAC,CAAA;YAC5E,CAAC;YAED,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,IAAI,uCAAuC,CAAC,CAAA;YAC5E,CAAC;YAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAA;YACzB,MAAM,QAAQ,GAAG,IAAI,EAAE,QAAQ,IAAI,IAAI,CAAA;YACvC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjE,OAAO,IAAI,CAAA;YACb,CAAC;YAED,OAAO;gBACL,QAAQ;gBACR,IAAI,EAAE,OAAO,IAAI,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;gBACvD,QAAQ,EAAE,IAAI,EAAE,QAAQ,IAAI,IAAI;aACjC,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACjE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;YAChD,CAAC;YACD,MAAM,KAAK,CAAA;QACb,CAAC;gBAAS,CAAC;YACT,IAAI,OAAO;gBAAE,YAAY,CAAC,OAAO,CAAC,CAAA;QACpC,CAAC;IACH,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
const SUMMARY_LENGTHS = ['short', 'medium', 'long', 'xl', 'xxl'];
|
|
2
|
+
const DURATION_PATTERN = /^(?<value>\d+(?:\.\d+)?)(?<unit>ms|s|m|h)?$/i;
|
|
3
|
+
const COUNT_PATTERN = /^(?<value>\d+(?:\.\d+)?)(?<unit>k|m)?$/i;
|
|
4
|
+
export function parseYoutubeMode(raw) {
|
|
5
|
+
const normalized = raw.trim().toLowerCase();
|
|
6
|
+
if (normalized === 'autp')
|
|
7
|
+
return 'auto';
|
|
8
|
+
if (normalized === 'auto' || normalized === 'web' || normalized === 'apify')
|
|
9
|
+
return normalized;
|
|
10
|
+
throw new Error(`Unsupported --youtube: ${raw}`);
|
|
11
|
+
}
|
|
12
|
+
export function parseFirecrawlMode(raw) {
|
|
13
|
+
const normalized = raw.trim().toLowerCase();
|
|
14
|
+
if (normalized === 'off' || normalized === 'auto' || normalized === 'always')
|
|
15
|
+
return normalized;
|
|
16
|
+
throw new Error(`Unsupported --firecrawl: ${raw}`);
|
|
17
|
+
}
|
|
18
|
+
export function parseMarkdownMode(raw) {
|
|
19
|
+
const normalized = raw.trim().toLowerCase();
|
|
20
|
+
if (normalized === 'off' || normalized === 'auto' || normalized === 'llm')
|
|
21
|
+
return normalized;
|
|
22
|
+
throw new Error(`Unsupported --markdown: ${raw}`);
|
|
23
|
+
}
|
|
24
|
+
export function parseStreamMode(raw) {
|
|
25
|
+
const normalized = raw.trim().toLowerCase();
|
|
26
|
+
if (normalized === 'auto' || normalized === 'on' || normalized === 'off')
|
|
27
|
+
return normalized;
|
|
28
|
+
throw new Error(`Unsupported --stream: ${raw}`);
|
|
29
|
+
}
|
|
30
|
+
export function parseRenderMode(raw) {
|
|
31
|
+
const normalized = raw.trim().toLowerCase();
|
|
32
|
+
if (normalized === 'auto' || normalized === 'plain')
|
|
33
|
+
return normalized;
|
|
34
|
+
if (normalized === 'md-live' || normalized === 'live' || normalized === 'mdlive')
|
|
35
|
+
return 'md-live';
|
|
36
|
+
if (normalized === 'md' || normalized === 'markdown')
|
|
37
|
+
return 'md';
|
|
38
|
+
throw new Error(`Unsupported --render: ${raw}`);
|
|
39
|
+
}
|
|
40
|
+
export function parseMetricsMode(raw) {
|
|
41
|
+
const normalized = raw.trim().toLowerCase();
|
|
42
|
+
if (normalized === 'off' || normalized === 'on' || normalized === 'detailed') {
|
|
43
|
+
return normalized;
|
|
44
|
+
}
|
|
45
|
+
throw new Error(`Unsupported --metrics: ${raw}`);
|
|
46
|
+
}
|
|
47
|
+
export function parseDurationMs(raw) {
|
|
48
|
+
const normalized = raw.trim();
|
|
49
|
+
const match = DURATION_PATTERN.exec(normalized);
|
|
50
|
+
if (!match?.groups) {
|
|
51
|
+
throw new Error(`Unsupported --timeout: ${raw}`);
|
|
52
|
+
}
|
|
53
|
+
const numeric = Number(match.groups.value);
|
|
54
|
+
if (!Number.isFinite(numeric) || numeric <= 0) {
|
|
55
|
+
throw new Error(`Unsupported --timeout: ${raw}`);
|
|
56
|
+
}
|
|
57
|
+
const unit = match.groups.unit?.toLowerCase() ?? 's';
|
|
58
|
+
const multiplier = unit === 'ms' ? 1 : unit === 's' ? 1000 : unit === 'm' ? 60_000 : 3_600_000;
|
|
59
|
+
return Math.floor(numeric * multiplier);
|
|
60
|
+
}
|
|
61
|
+
export function parseLengthArg(raw) {
|
|
62
|
+
const normalized = raw.trim().toLowerCase();
|
|
63
|
+
if (SUMMARY_LENGTHS.includes(normalized)) {
|
|
64
|
+
return { kind: 'preset', preset: normalized };
|
|
65
|
+
}
|
|
66
|
+
const match = COUNT_PATTERN.exec(normalized);
|
|
67
|
+
if (!match?.groups) {
|
|
68
|
+
throw new Error(`Unsupported --length: ${raw}`);
|
|
69
|
+
}
|
|
70
|
+
const numeric = Number(match.groups.value);
|
|
71
|
+
if (!Number.isFinite(numeric) || numeric <= 0) {
|
|
72
|
+
throw new Error(`Unsupported --length: ${raw}`);
|
|
73
|
+
}
|
|
74
|
+
const unit = match.groups.unit?.toLowerCase() ?? null;
|
|
75
|
+
const multiplier = unit === 'k' ? 1000 : unit === 'm' ? 1_000_000 : 1;
|
|
76
|
+
return { kind: 'chars', maxCharacters: Math.floor(numeric * multiplier) };
|
|
77
|
+
}
|
|
78
|
+
export function parseMaxOutputTokensArg(raw) {
|
|
79
|
+
if (raw === undefined || raw === null)
|
|
80
|
+
return null;
|
|
81
|
+
const normalized = raw.trim().toLowerCase();
|
|
82
|
+
if (!normalized) {
|
|
83
|
+
throw new Error(`Unsupported --max-output-tokens: ${raw}`);
|
|
84
|
+
}
|
|
85
|
+
const match = COUNT_PATTERN.exec(normalized);
|
|
86
|
+
if (!match?.groups) {
|
|
87
|
+
throw new Error(`Unsupported --max-output-tokens: ${raw}`);
|
|
88
|
+
}
|
|
89
|
+
const numeric = Number(match.groups.value);
|
|
90
|
+
if (!Number.isFinite(numeric) || numeric <= 0) {
|
|
91
|
+
throw new Error(`Unsupported --max-output-tokens: ${raw}`);
|
|
92
|
+
}
|
|
93
|
+
const unit = match.groups.unit?.toLowerCase() ?? null;
|
|
94
|
+
const multiplier = unit === 'k' ? 1000 : unit === 'm' ? 1_000_000 : 1;
|
|
95
|
+
return Math.floor(numeric * multiplier);
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=flags.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"flags.js","sourceRoot":"","sources":["../../src/flags.ts"],"names":[],"mappings":"AAaA,MAAM,eAAe,GAAoB,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;AACjF,MAAM,gBAAgB,GAAG,8CAA8C,CAAA;AACvE,MAAM,aAAa,GAAG,yCAAyC,CAAA;AAE/D,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,UAAU,KAAK,MAAM;QAAE,OAAO,MAAM,CAAA;IACxC,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,OAAO;QAAE,OAAO,UAAU,CAAA;IAC9F,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAA;IAC/F,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAA;AACpD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAW;IAC3C,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,KAAK;QAAE,OAAO,UAAU,CAAA;IAC5F,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,KAAK;QAAE,OAAO,UAAU,CAAA;IAC3F,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,OAAO;QAAE,OAAO,UAAwB,CAAA;IACpF,IAAI,UAAU,KAAK,SAAS,IAAI,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAClG,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,UAAU;QAAE,OAAO,IAAI,CAAA;IACjE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;QAC7E,OAAO,UAAyB,CAAA;IAClC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAC7B,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC/C,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,GAAG,CAAA;IACpD,MAAM,UAAU,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;IAC9F,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,UAAU,CAAC,CAAA;AACzC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,eAAe,CAAC,QAAQ,CAAC,UAA2B,CAAC,EAAE,CAAC;QAC1D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAA2B,EAAE,CAAA;IAChE,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC5C,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,IAAI,CAAA;IACrD,MAAM,UAAU,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,UAAU,CAAC,EAAE,CAAA;AAC3E,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,GAAuB;IAC7D,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC5C,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,IAAI,CAAA;IACrD,MAAM,UAAU,GAAG,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;IACrE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,UAAU,CAAC,CAAA;AACzC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAA;AAClC,cAAc,oBAAoB,CAAA;AAElC,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA"}
|