@asterxsk/kiln 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +170 -170
  3. package/agent/AGENTS.md +67 -67
  4. package/agent/README.md +5 -5
  5. package/agent/extensions/AGENTS.md +68 -68
  6. package/agent/extensions/ask-user/index.ts +418 -418
  7. package/agent/extensions/ask-user/package-lock.json +769 -769
  8. package/agent/extensions/ask-user/package.json +19 -19
  9. package/agent/extensions/ask-user/prompt.ts +45 -45
  10. package/agent/extensions/ask-user/tsconfig.json +7 -7
  11. package/agent/extensions/background-terminals/docs/implementation-guide.md +942 -942
  12. package/agent/extensions/background-terminals/index.ts +627 -627
  13. package/agent/extensions/background-terminals/manager.test.ts +735 -735
  14. package/agent/extensions/background-terminals/output.test.ts +109 -109
  15. package/agent/extensions/background-terminals/package-lock.json +769 -769
  16. package/agent/extensions/background-terminals/package.json +17 -17
  17. package/agent/extensions/background-terminals/prompt.test.ts +125 -125
  18. package/agent/extensions/background-terminals/ps.test.ts +82 -82
  19. package/agent/extensions/background-terminals/result-delivery.test.ts +44 -44
  20. package/agent/extensions/background-terminals/src/domain.ts +87 -87
  21. package/agent/extensions/background-terminals/src/manager.ts +907 -907
  22. package/agent/extensions/background-terminals/src/output.ts +84 -84
  23. package/agent/extensions/background-terminals/src/prompt.ts +142 -142
  24. package/agent/extensions/background-terminals/src/result-delivery.ts +27 -27
  25. package/agent/extensions/background-terminals/src/runtime.ts +36 -36
  26. package/agent/extensions/background-terminals/src/ui/output-view.ts +79 -79
  27. package/agent/extensions/background-terminals/src/ui/ps.ts +621 -621
  28. package/agent/extensions/background-terminals/tsconfig.json +7 -7
  29. package/agent/extensions/file-search/index.spec.ts +443 -443
  30. package/agent/extensions/file-search/index.ts +459 -459
  31. package/agent/extensions/file-search/package-lock.json +2253 -2253
  32. package/agent/extensions/file-search/package.json +23 -23
  33. package/agent/extensions/file-search/src/args.ts +122 -122
  34. package/agent/extensions/file-search/src/binaries.ts +422 -422
  35. package/agent/extensions/file-search/src/output.ts +126 -126
  36. package/agent/extensions/file-search/src/process.ts +146 -146
  37. package/agent/extensions/file-search/src/prompt.ts +52 -52
  38. package/agent/extensions/file-search/tsconfig.json +7 -7
  39. package/agent/extensions/modelconf/PLAN.md +915 -915
  40. package/agent/extensions/modelconf/index.ts +296 -296
  41. package/agent/extensions/modelconf/src/ui/ModelConfView.ts +1101 -1101
  42. package/agent/extensions/pi-web-access/CHANGELOG.md +690 -690
  43. package/agent/extensions/pi-web-access/LICENSE +21 -21
  44. package/agent/extensions/pi-web-access/README.md +470 -470
  45. package/agent/extensions/pi-web-access/SECURITY.md +5 -5
  46. package/agent/extensions/pi-web-access/activity.ts +101 -101
  47. package/agent/extensions/pi-web-access/auth-fetch.ts +148 -148
  48. package/agent/extensions/pi-web-access/brightdata-unlocker.ts +272 -272
  49. package/agent/extensions/pi-web-access/chrome-cookies.ts +669 -669
  50. package/agent/extensions/pi-web-access/content-find.ts +139 -139
  51. package/agent/extensions/pi-web-access/credential-source.ts +191 -191
  52. package/agent/extensions/pi-web-access/data-uri-sanitize.ts +406 -406
  53. package/agent/extensions/pi-web-access/datalab-pdf-extract.ts +568 -568
  54. package/agent/extensions/pi-web-access/declared-web-links.ts +173 -173
  55. package/agent/extensions/pi-web-access/evidence/CONTRACT-EVIDENCE.md +496 -496
  56. package/agent/extensions/pi-web-access/evidence/contract-probe.mjs +140 -140
  57. package/agent/extensions/pi-web-access/exa.ts +526 -526
  58. package/agent/extensions/pi-web-access/extract.ts +1196 -1196
  59. package/agent/extensions/pi-web-access/feature-config.ts +29 -29
  60. package/agent/extensions/pi-web-access/fetch-params.ts +111 -111
  61. package/agent/extensions/pi-web-access/gemini-adc.ts +298 -298
  62. package/agent/extensions/pi-web-access/gemini-api.ts +353 -353
  63. package/agent/extensions/pi-web-access/gemini-pdf-extract.ts +108 -108
  64. package/agent/extensions/pi-web-access/gemini-url-context.ts +128 -128
  65. package/agent/extensions/pi-web-access/gemini-web-config.ts +101 -101
  66. package/agent/extensions/pi-web-access/gemini-web.ts +487 -487
  67. package/agent/extensions/pi-web-access/github-api.ts +197 -197
  68. package/agent/extensions/pi-web-access/github-extract.ts +746 -746
  69. package/agent/extensions/pi-web-access/github-issue-pr.ts +700 -700
  70. package/agent/extensions/pi-web-access/index.ts +1737 -1737
  71. package/agent/extensions/pi-web-access/package-lock.json +5808 -5808
  72. package/agent/extensions/pi-web-access/package.json +64 -64
  73. package/agent/extensions/pi-web-access/page-query.ts +96 -96
  74. package/agent/extensions/pi-web-access/pdf-extract.ts +409 -409
  75. package/agent/extensions/pi-web-access/promise-try.d.ts +7 -7
  76. package/agent/extensions/pi-web-access/query-rewrite.ts +51 -51
  77. package/agent/extensions/pi-web-access/render-search-error.ts +170 -170
  78. package/agent/extensions/pi-web-access/rsc-extract.ts +338 -338
  79. package/agent/extensions/pi-web-access/source-check.ts +282 -282
  80. package/agent/extensions/pi-web-access/ssrf-protection.ts +526 -526
  81. package/agent/extensions/pi-web-access/storage.ts +521 -521
  82. package/agent/extensions/pi-web-access/summary-model-scope.ts +125 -125
  83. package/agent/extensions/pi-web-access/test/auth-fetch.test.mjs +208 -208
  84. package/agent/extensions/pi-web-access/test/brightdata-unlocker.test.mjs +840 -840
  85. package/agent/extensions/pi-web-access/test/chrome-cookie-extraction.test.mjs +441 -441
  86. package/agent/extensions/pi-web-access/test/config-path.test.mjs +283 -283
  87. package/agent/extensions/pi-web-access/test/content-find.test.mjs +25 -25
  88. package/agent/extensions/pi-web-access/test/credential-source.test.mjs +118 -118
  89. package/agent/extensions/pi-web-access/test/data-uri-sanitize.test.mjs +210 -210
  90. package/agent/extensions/pi-web-access/test/datalab-pdf-extract.test.mjs +552 -552
  91. package/agent/extensions/pi-web-access/test/declared-web-links.test.mjs +212 -212
  92. package/agent/extensions/pi-web-access/test/fetch-answer-storage.test.mjs +40 -40
  93. package/agent/extensions/pi-web-access/test/fetch-cache-storage.test.mjs +334 -334
  94. package/agent/extensions/pi-web-access/test/fetch-content-domain-policy.test.mjs +95 -95
  95. package/agent/extensions/pi-web-access/test/fetch-modes.test.mjs +53 -53
  96. package/agent/extensions/pi-web-access/test/fetch-not-found-guidance.test.mjs +92 -92
  97. package/agent/extensions/pi-web-access/test/fetch-params.test.mjs +86 -86
  98. package/agent/extensions/pi-web-access/test/fetch-render-call.test.mjs +34 -34
  99. package/agent/extensions/pi-web-access/test/fetch-routing.test.mjs +173 -173
  100. package/agent/extensions/pi-web-access/test/gemini-adc-auth.test.mjs +257 -257
  101. package/agent/extensions/pi-web-access/test/gemini-api-transport.test.mjs +170 -170
  102. package/agent/extensions/pi-web-access/test/gemini-pdf-extract.test.mjs +133 -133
  103. package/agent/extensions/pi-web-access/test/gemini-web-cookie-opt-in.test.mjs +178 -178
  104. package/agent/extensions/pi-web-access/test/gemini-web-header-overflow.test.mjs +148 -148
  105. package/agent/extensions/pi-web-access/test/get-search-content.test.mjs +223 -223
  106. package/agent/extensions/pi-web-access/test/github-extract.test.mjs +378 -378
  107. package/agent/extensions/pi-web-access/test/github-issue-pr.test.mjs +565 -565
  108. package/agent/extensions/pi-web-access/test/inline-content-config.test.mjs +99 -99
  109. package/agent/extensions/pi-web-access/test/lazy-extract-load.test.mjs +118 -118
  110. package/agent/extensions/pi-web-access/test/local-video-oversize.test.mjs +52 -52
  111. package/agent/extensions/pi-web-access/test/package-typebox-dependency.test.mjs +50 -50
  112. package/agent/extensions/pi-web-access/test/page-query.test.mjs +51 -51
  113. package/agent/extensions/pi-web-access/test/pdf-config.test.mjs +140 -140
  114. package/agent/extensions/pi-web-access/test/pdf-extract.test.mjs +500 -500
  115. package/agent/extensions/pi-web-access/test/proxy-transport.test.mjs +286 -286
  116. package/agent/extensions/pi-web-access/test/query-rewrite.test.mjs +52 -52
  117. package/agent/extensions/pi-web-access/test/rsc-fallback.test.mjs +102 -102
  118. package/agent/extensions/pi-web-access/test/search-error-render.test.mjs +152 -152
  119. package/agent/extensions/pi-web-access/test/search-providers.test.mjs +274 -274
  120. package/agent/extensions/pi-web-access/test/source-check.test.mjs +179 -179
  121. package/agent/extensions/pi-web-access/test/ssrf-allow-ranges-config.test.mjs +205 -205
  122. package/agent/extensions/pi-web-access/test/ssrf-protection.test.mjs +456 -456
  123. package/agent/extensions/pi-web-access/test/tool-registration-config.test.mjs +182 -182
  124. package/agent/extensions/pi-web-access/test/youtube-extract-errors.test.mjs +64 -64
  125. package/agent/extensions/pi-web-access/tsconfig.json +11 -11
  126. package/agent/extensions/pi-web-access/utils.ts +451 -451
  127. package/agent/extensions/pi-web-access/video-extract.ts +392 -392
  128. package/agent/extensions/pi-web-access/youtube-extract.ts +328 -328
  129. package/agent/extensions/shared/activity-status.ts +31 -31
  130. package/agent/extensions/shared/child-session.test.ts +270 -270
  131. package/agent/extensions/shared/child-session.ts +148 -148
  132. package/agent/extensions/shared/context-utilization.test.ts +48 -48
  133. package/agent/extensions/shared/context-utilization.ts +47 -47
  134. package/agent/extensions/shared/dashboard-state.ts +99 -99
  135. package/agent/extensions/shared/tool-call-timeout.test.ts +117 -117
  136. package/agent/extensions/shared/tool-call-timeout.ts +104 -104
  137. package/agent/extensions/subagents/by-the-way.test.ts +29 -29
  138. package/agent/extensions/subagents/claude.test.ts +119 -119
  139. package/agent/extensions/subagents/codex.test.ts +102 -102
  140. package/agent/extensions/subagents/context-usage.test.ts +107 -107
  141. package/agent/extensions/subagents/docs/design-plan.md +568 -568
  142. package/agent/extensions/subagents/docs/effect-v4-extension-guide.md +354 -354
  143. package/agent/extensions/subagents/docs/effect-v4-notes.md +571 -571
  144. package/agent/extensions/subagents/index.ts +779 -779
  145. package/agent/extensions/subagents/manager.test.ts +276 -276
  146. package/agent/extensions/subagents/package-lock.json +2244 -2244
  147. package/agent/extensions/subagents/package.json +19 -19
  148. package/agent/extensions/subagents/result-delivery.test.ts +27 -27
  149. package/agent/extensions/subagents/src/backend.ts +73 -73
  150. package/agent/extensions/subagents/src/backends/claude.ts +701 -701
  151. package/agent/extensions/subagents/src/backends/codex.ts +1060 -1060
  152. package/agent/extensions/subagents/src/backends/pi.ts +575 -575
  153. package/agent/extensions/subagents/src/backends/stub.ts +300 -300
  154. package/agent/extensions/subagents/src/by-the-way.ts +21 -21
  155. package/agent/extensions/subagents/src/domain.ts +253 -253
  156. package/agent/extensions/subagents/src/format.ts +74 -74
  157. package/agent/extensions/subagents/src/manager.ts +736 -736
  158. package/agent/extensions/subagents/src/prompt.ts +92 -92
  159. package/agent/extensions/subagents/src/result-delivery.ts +20 -20
  160. package/agent/extensions/subagents/src/runtime.ts +53 -53
  161. package/agent/extensions/subagents/src/ui/takeover.ts +583 -583
  162. package/agent/extensions/subagents/src/ui/transcript.ts +201 -201
  163. package/agent/extensions/subagents/takeover.test.ts +29 -29
  164. package/agent/extensions/subagents/tsconfig.json +7 -7
  165. package/agent/extensions/todo/AGENTS.md +38 -38
  166. package/agent/extensions/todo/LICENSE +21 -21
  167. package/agent/extensions/todo/config.ts +55 -55
  168. package/agent/extensions/todo/index.ts +151 -151
  169. package/agent/extensions/todo/locales/de.json +17 -17
  170. package/agent/extensions/todo/locales/en.json +15 -15
  171. package/agent/extensions/todo/locales/es.json +17 -17
  172. package/agent/extensions/todo/locales/fr.json +17 -17
  173. package/agent/extensions/todo/locales/pt-BR.json +17 -17
  174. package/agent/extensions/todo/locales/pt.json +17 -17
  175. package/agent/extensions/todo/locales/ru.json +17 -17
  176. package/agent/extensions/todo/locales/uk.json +17 -17
  177. package/agent/extensions/todo/locales/zh.json +17 -17
  178. package/agent/extensions/todo/package-lock.json +3358 -3358
  179. package/agent/extensions/todo/package.json +67 -67
  180. package/agent/extensions/todo/state/i18n-bridge.ts +64 -64
  181. package/agent/extensions/todo/state/invariants.ts +20 -20
  182. package/agent/extensions/todo/state/replay.ts +38 -38
  183. package/agent/extensions/todo/state/selectors.ts +107 -107
  184. package/agent/extensions/todo/state/state-reducer.ts +326 -326
  185. package/agent/extensions/todo/state/state.ts +18 -18
  186. package/agent/extensions/todo/state/store.ts +82 -82
  187. package/agent/extensions/todo/state/task-graph.ts +57 -57
  188. package/agent/extensions/todo/todo-overlay.ts +200 -200
  189. package/agent/extensions/todo/todo.ts +155 -155
  190. package/agent/extensions/todo/tool/response-envelope.ts +109 -109
  191. package/agent/extensions/todo/tool/types.ts +206 -206
  192. package/agent/extensions/todo/view/format.ts +177 -177
  193. package/agent/install.ps1 +637 -527
  194. package/agent/install.sh +620 -511
  195. package/agent/keybindings.json +7 -7
  196. package/bin/kiln.js +124 -11
  197. package/package.json +8 -2
  198. package/agent/extensions/taste/index.ts +0 -443
  199. package/agent/extensions/taste/install.ps1 +0 -23
  200. package/agent/extensions/taste/install.sh +0 -21
  201. /package/agent/extensions/{status line → statusline}/index.ts +0 -0
  202. /package/agent/extensions/{status line → statusline}/install.ps1 +0 -0
  203. /package/agent/extensions/{status line → statusline}/install.sh +0 -0
@@ -1,392 +1,392 @@
1
- import { execFileSync } from "node:child_process";
2
- import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
3
- import { readFile } from "node:fs/promises";
4
- import { resolve, extname, basename, join, dirname } from "node:path";
5
- import { activityMonitor } from "./activity.ts";
6
- import { canAttachImages } from "./feature-config.ts";
7
- import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.ts";
8
- import { queryGeminiApiWithVideo, getApiKey, fetchGeminiApi, getVersionedApiBase, getUploadBase, redactGeminiApiResponse } from "./gemini-api.ts";
9
- import { extractHeadingTitle, type ExtractedContent, type ExtractOptions, type FrameResult } from "./extract.ts";
10
- import { readExecError, trimErrorText, mapFfmpegError, getWebSearchConfigPath } from "./utils.ts";
11
-
12
- const CONFIG_PATH = getWebSearchConfigPath();
13
-
14
- const DEFAULT_VIDEO_PROMPT = `Extract the complete content of this video. Include:
15
- 1. Video title (infer from content if not explicit), duration
16
- 2. A brief summary (2-3 sentences)
17
- 3. Full transcript with timestamps
18
- 4. Descriptions of any code, terminal commands, diagrams, slides, or UI shown on screen
19
-
20
- Format as markdown.`;
21
-
22
- const VIDEO_EXTENSIONS: Record<string, string> = {
23
- ".mp4": "video/mp4",
24
- ".mov": "video/quicktime",
25
- ".webm": "video/webm",
26
- ".avi": "video/x-msvideo",
27
- ".mpeg": "video/mpeg",
28
- ".mpg": "video/mpeg",
29
- ".wmv": "video/x-ms-wmv",
30
- ".flv": "video/x-flv",
31
- ".3gp": "video/3gpp",
32
- ".3gpp": "video/3gpp",
33
- };
34
-
35
- function shouldRethrow(err: unknown): boolean {
36
- const message = err instanceof Error ? err.message : String(err);
37
- return message.startsWith("Failed to parse ");
38
- }
39
-
40
- interface VideoFileInfo {
41
- absolutePath: string;
42
- mimeType: string;
43
- sizeBytes: number;
44
- maxSizeBytes: number;
45
- withinUploadLimit: boolean;
46
- }
47
-
48
- interface VideoConfig {
49
- enabled: boolean;
50
- preferredModel: string;
51
- maxSizeMB: number;
52
- }
53
-
54
- function normalizePreferredModel(value: unknown, fallback: string): string {
55
- if (typeof value !== "string") return fallback;
56
- const normalized = value.trim();
57
- return normalized.length > 0 ? normalized : fallback;
58
- }
59
-
60
- function normalizeEnabled(value: unknown, fallback: boolean): boolean {
61
- return typeof value === "boolean" ? value : fallback;
62
- }
63
-
64
- function normalizeMaxSizeMB(value: unknown, fallback: number): number {
65
- if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
66
- return value > 0 ? value : fallback;
67
- }
68
-
69
- const VIDEO_CONFIG_DEFAULTS: VideoConfig = {
70
- enabled: true,
71
- preferredModel: "gemini-3.6-flash",
72
- maxSizeMB: 50,
73
- };
74
-
75
- let cachedVideoConfig: VideoConfig | null = null;
76
-
77
- function loadVideoConfig(): VideoConfig {
78
- if (cachedVideoConfig) return cachedVideoConfig;
79
- if (!existsSync(CONFIG_PATH)) {
80
- cachedVideoConfig = { ...VIDEO_CONFIG_DEFAULTS };
81
- return cachedVideoConfig;
82
- }
83
-
84
- const rawText = readFileSync(CONFIG_PATH, "utf-8");
85
- let raw: { video?: { enabled?: boolean; preferredModel?: string; maxSizeMB?: number } };
86
- try {
87
- raw = JSON.parse(rawText) as { video?: { enabled?: boolean; preferredModel?: string; maxSizeMB?: number } };
88
- } catch (err) {
89
- const message = err instanceof Error ? err.message : String(err);
90
- throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
91
- }
92
-
93
- const v = raw.video ?? {};
94
- cachedVideoConfig = {
95
- enabled: normalizeEnabled(v.enabled, VIDEO_CONFIG_DEFAULTS.enabled),
96
- preferredModel: normalizePreferredModel(v.preferredModel, VIDEO_CONFIG_DEFAULTS.preferredModel),
97
- maxSizeMB: normalizeMaxSizeMB(v.maxSizeMB, VIDEO_CONFIG_DEFAULTS.maxSizeMB),
98
- };
99
- return cachedVideoConfig;
100
- }
101
-
102
- export function isVideoFile(input: string): VideoFileInfo | null {
103
- const config = loadVideoConfig();
104
- if (!config.enabled) return null;
105
-
106
- const isFilePath = input.startsWith("/") || input.startsWith("./") || input.startsWith("../") || input.startsWith("file://")
107
- || input.startsWith("\\") || /^[A-Za-z]:[\\/]/.test(input);
108
- if (!isFilePath) return null;
109
-
110
- let filePath = input;
111
- if (input.startsWith("file://")) {
112
- try {
113
- filePath = decodeURIComponent(new URL(input).pathname);
114
- } catch {
115
- return null;
116
- }
117
- }
118
-
119
- const ext = extname(filePath).toLowerCase();
120
- const mimeType = VIDEO_EXTENSIONS[ext];
121
- if (!mimeType) return null;
122
-
123
- const absolutePath = resolveFilePath(filePath);
124
- if (!absolutePath) return null;
125
-
126
- let stat: ReturnType<typeof statSync>;
127
- try {
128
- stat = statSync(absolutePath);
129
- } catch {
130
- return null;
131
- }
132
- if (!stat.isFile()) return null;
133
-
134
- const maxBytes = config.maxSizeMB * 1024 * 1024;
135
- return {
136
- absolutePath,
137
- mimeType,
138
- sizeBytes: stat.size,
139
- maxSizeBytes: maxBytes,
140
- withinUploadLimit: stat.size <= maxBytes,
141
- };
142
- }
143
-
144
- function resolveFilePath(filePath: string): string | null {
145
- const absolutePath = resolve(filePath);
146
- if (existsSync(absolutePath)) return absolutePath;
147
-
148
- const dir = dirname(absolutePath);
149
- const base = basename(absolutePath);
150
- if (!existsSync(dir)) return null;
151
-
152
- try {
153
- const normalizedBase = normalizeSpaces(base);
154
- const match = readdirSync(dir).find(f => normalizeSpaces(f) === normalizedBase);
155
- return match ? join(dir, match) : null;
156
- } catch {
157
- return null;
158
- }
159
- }
160
-
161
- function normalizeSpaces(s: string): string {
162
- return s.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000\uFEFF]/g, " ");
163
- }
164
-
165
- export async function extractVideo(
166
- info: VideoFileInfo,
167
- signal?: AbortSignal,
168
- options?: ExtractOptions,
169
- ): Promise<ExtractedContent | null> {
170
- const config = loadVideoConfig();
171
- const effectivePrompt = options?.prompt ?? DEFAULT_VIDEO_PROMPT;
172
- const effectiveModel = options?.model ?? config.preferredModel;
173
- const displayName = basename(info.absolutePath);
174
- if (!info.withinUploadLimit) {
175
- const sizeMB = (info.sizeBytes / 1024 / 1024).toFixed(1);
176
- const maxSizeMB = (info.maxSizeBytes / 1024 / 1024).toFixed(1);
177
- const error = `Local video ${displayName} is ${sizeMB} MiB, above configured video.maxSizeMB (${maxSizeMB} MiB) for Gemini analysis. Use timestamp/frames for ffmpeg frame extraction, increase video.maxSizeMB, or compress the file.`;
178
- return { url: info.absolutePath, title: displayName, content: error, error };
179
- }
180
- const activityId = activityMonitor.logStart({ type: "fetch", url: `video:${displayName}` });
181
-
182
- const result = await tryVideoGeminiApi(info, effectivePrompt, effectiveModel, signal)
183
- ?? await tryVideoGeminiWeb(info, effectivePrompt, effectiveModel, signal);
184
-
185
- if (result) {
186
- if (canAttachImages()) {
187
- const thumbnail = await extractVideoFrame(info.absolutePath);
188
- if (!("error" in thumbnail)) {
189
- result.thumbnail = thumbnail;
190
- }
191
- }
192
- activityMonitor.logComplete(activityId, 200);
193
- return result;
194
- }
195
-
196
- if (signal?.aborted) {
197
- activityMonitor.logComplete(activityId, 0);
198
- return null;
199
- }
200
-
201
- activityMonitor.logError(activityId, "all video extraction paths failed");
202
- return null;
203
- }
204
-
205
- function mapFfprobeError(err: unknown): string {
206
- const { code, stderr, message } = readExecError(err);
207
- if (code === "ENOENT") return "ffprobe is not installed. Install ffmpeg which includes ffprobe";
208
- const snippet = trimErrorText(stderr || message);
209
- return snippet ? `ffprobe failed: ${snippet}` : "ffprobe failed";
210
- }
211
-
212
- export async function extractVideoFrame(filePath: string, seconds: number = 1): Promise<FrameResult> {
213
- try {
214
- const buffer = execFileSync("ffmpeg", [
215
- "-ss", String(seconds), "-i", filePath,
216
- "-frames:v", "1", "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1",
217
- ], { maxBuffer: 5 * 1024 * 1024, timeout: 10000, stdio: ["pipe", "pipe", "pipe"] });
218
- if (buffer.length === 0) return { error: "ffmpeg failed: empty output" };
219
- return { data: buffer.toString("base64"), mimeType: "image/jpeg" };
220
- } catch (err) {
221
- return { error: mapFfmpegError(err) };
222
- }
223
- }
224
-
225
- export async function getLocalVideoDuration(filePath: string): Promise<number | { error: string }> {
226
- try {
227
- const output = execFileSync("ffprobe", [
228
- "-v", "quiet",
229
- "-show_entries", "format=duration",
230
- "-of", "csv=p=0",
231
- filePath,
232
- ], { timeout: 10000, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
233
- const duration = Number.parseFloat(output);
234
- if (!Number.isFinite(duration)) return { error: "ffprobe failed: invalid duration output" };
235
- return duration;
236
- } catch (err) {
237
- return { error: mapFfprobeError(err) };
238
- }
239
- }
240
-
241
- async function tryVideoGeminiWeb(
242
- info: VideoFileInfo,
243
- prompt: string,
244
- model: string,
245
- signal?: AbortSignal,
246
- ): Promise<ExtractedContent | null> {
247
- try {
248
- const cookies = await isGeminiWebAvailable();
249
- if (!cookies) return null;
250
- if (signal?.aborted) return null;
251
-
252
- const text = await queryWithCookies(prompt, cookies, {
253
- files: [info.absolutePath],
254
- ...(model !== "gemini-3.6-flash" ? { model } : {}),
255
- signal,
256
- timeoutMs: 180000,
257
- });
258
-
259
- return {
260
- url: info.absolutePath,
261
- title: extractVideoTitle(text, info.absolutePath),
262
- content: text,
263
- error: null,
264
- };
265
- } catch (err) {
266
- if (shouldRethrow(err)) throw err;
267
- return null;
268
- }
269
- }
270
-
271
- async function tryVideoGeminiApi(
272
- info: VideoFileInfo,
273
- prompt: string,
274
- model: string,
275
- signal?: AbortSignal,
276
- ): Promise<ExtractedContent | null> {
277
- const apiKey = await getApiKey(signal);
278
- if (!apiKey) return null;
279
- if (signal?.aborted) return null;
280
-
281
- let fileName: string | null = null;
282
- try {
283
- const uploaded = await uploadToFilesApi(info, apiKey, signal);
284
- fileName = uploaded.name;
285
-
286
- await pollFileState(fileName, apiKey, signal, 120000);
287
-
288
- const text = await queryGeminiApiWithVideo(prompt, uploaded.uri, {
289
- apiKey,
290
- model,
291
- mimeType: info.mimeType,
292
- signal,
293
- timeoutMs: 120000,
294
- });
295
-
296
- return {
297
- url: info.absolutePath,
298
- title: extractVideoTitle(text, info.absolutePath),
299
- content: text,
300
- error: null,
301
- };
302
- } catch (err) {
303
- if (shouldRethrow(err)) throw err;
304
- return null;
305
- } finally {
306
- if (fileName) deleteGeminiFile(fileName, apiKey);
307
- }
308
- }
309
-
310
- async function uploadToFilesApi(
311
- info: VideoFileInfo,
312
- apiKey: string,
313
- signal?: AbortSignal,
314
- ): Promise<{ name: string; uri: string }> {
315
- const displayName = basename(info.absolutePath);
316
-
317
- const initRes = await fetchGeminiApi(`${getUploadBase()}/files`, {
318
- method: "POST",
319
- headers: {
320
- "X-Goog-Upload-Protocol": "resumable",
321
- "X-Goog-Upload-Command": "start",
322
- "X-Goog-Upload-Header-Content-Length": String(info.sizeBytes),
323
- "X-Goog-Upload-Header-Content-Type": info.mimeType,
324
- "Content-Type": "application/json",
325
- },
326
- body: JSON.stringify({ file: { display_name: displayName } }),
327
- signal,
328
- }, apiKey);
329
-
330
- if (!initRes.ok) {
331
- const text = redactGeminiApiResponse(initRes, await initRes.text(), apiKey);
332
- throw new Error(`File upload init failed: ${initRes.status} (${text.slice(0, 200)})`);
333
- }
334
-
335
- const uploadUrl = initRes.headers.get("x-goog-upload-url");
336
- if (!uploadUrl) throw new Error("No upload URL in response headers");
337
-
338
- const fileData = await readFile(info.absolutePath);
339
- const uploadRes = await fetchGeminiApi(uploadUrl, {
340
- method: "PUT",
341
- headers: {
342
- "Content-Length": String(info.sizeBytes),
343
- "X-Goog-Upload-Offset": "0",
344
- "X-Goog-Upload-Command": "upload, finalize",
345
- },
346
- body: fileData,
347
- signal,
348
- }, apiKey);
349
-
350
- if (!uploadRes.ok) {
351
- const text = redactGeminiApiResponse(uploadRes, await uploadRes.text(), apiKey);
352
- throw new Error(`File upload failed: ${uploadRes.status} (${text.slice(0, 200)})`);
353
- }
354
-
355
- const result = await uploadRes.json() as { file: { name: string; uri: string } };
356
- return result.file;
357
- }
358
-
359
- async function pollFileState(
360
- fileName: string,
361
- apiKey: string,
362
- signal?: AbortSignal,
363
- timeoutMs: number = 120000,
364
- ): Promise<void> {
365
- const deadline = Date.now() + timeoutMs;
366
-
367
- while (Date.now() < deadline) {
368
- if (signal?.aborted) throw new Error("Aborted");
369
-
370
- const res = await fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`, { signal }, apiKey);
371
- if (!res.ok) throw new Error(`File state check failed: ${res.status}`);
372
-
373
- const data = await res.json() as { state: string };
374
- if (data.state === "ACTIVE") return;
375
- if (data.state === "FAILED") throw new Error("File processing failed");
376
-
377
- await new Promise(r => setTimeout(r, 5000));
378
- }
379
-
380
- throw new Error("File processing timed out");
381
- }
382
-
383
- function deleteGeminiFile(fileName: string, apiKey: string): void {
384
- void fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`, { method: "DELETE" }, apiKey).catch((err) => {
385
- const message = err instanceof Error ? err.message : String(err);
386
- console.error(`Failed to delete Gemini file ${fileName}: ${message}`);
387
- });
388
- }
389
-
390
- function extractVideoTitle(text: string, filePath: string): string {
391
- return extractHeadingTitle(text) ?? basename(filePath, extname(filePath));
392
- }
1
+ import { execFileSync } from "node:child_process";
2
+ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
3
+ import { readFile } from "node:fs/promises";
4
+ import { resolve, extname, basename, join, dirname } from "node:path";
5
+ import { activityMonitor } from "./activity.ts";
6
+ import { canAttachImages } from "./feature-config.ts";
7
+ import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.ts";
8
+ import { queryGeminiApiWithVideo, getApiKey, fetchGeminiApi, getVersionedApiBase, getUploadBase, redactGeminiApiResponse } from "./gemini-api.ts";
9
+ import { extractHeadingTitle, type ExtractedContent, type ExtractOptions, type FrameResult } from "./extract.ts";
10
+ import { readExecError, trimErrorText, mapFfmpegError, getWebSearchConfigPath } from "./utils.ts";
11
+
12
+ const CONFIG_PATH = getWebSearchConfigPath();
13
+
14
+ const DEFAULT_VIDEO_PROMPT = `Extract the complete content of this video. Include:
15
+ 1. Video title (infer from content if not explicit), duration
16
+ 2. A brief summary (2-3 sentences)
17
+ 3. Full transcript with timestamps
18
+ 4. Descriptions of any code, terminal commands, diagrams, slides, or UI shown on screen
19
+
20
+ Format as markdown.`;
21
+
22
+ const VIDEO_EXTENSIONS: Record<string, string> = {
23
+ ".mp4": "video/mp4",
24
+ ".mov": "video/quicktime",
25
+ ".webm": "video/webm",
26
+ ".avi": "video/x-msvideo",
27
+ ".mpeg": "video/mpeg",
28
+ ".mpg": "video/mpeg",
29
+ ".wmv": "video/x-ms-wmv",
30
+ ".flv": "video/x-flv",
31
+ ".3gp": "video/3gpp",
32
+ ".3gpp": "video/3gpp",
33
+ };
34
+
35
+ function shouldRethrow(err: unknown): boolean {
36
+ const message = err instanceof Error ? err.message : String(err);
37
+ return message.startsWith("Failed to parse ");
38
+ }
39
+
40
+ interface VideoFileInfo {
41
+ absolutePath: string;
42
+ mimeType: string;
43
+ sizeBytes: number;
44
+ maxSizeBytes: number;
45
+ withinUploadLimit: boolean;
46
+ }
47
+
48
+ interface VideoConfig {
49
+ enabled: boolean;
50
+ preferredModel: string;
51
+ maxSizeMB: number;
52
+ }
53
+
54
+ function normalizePreferredModel(value: unknown, fallback: string): string {
55
+ if (typeof value !== "string") return fallback;
56
+ const normalized = value.trim();
57
+ return normalized.length > 0 ? normalized : fallback;
58
+ }
59
+
60
+ function normalizeEnabled(value: unknown, fallback: boolean): boolean {
61
+ return typeof value === "boolean" ? value : fallback;
62
+ }
63
+
64
+ function normalizeMaxSizeMB(value: unknown, fallback: number): number {
65
+ if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
66
+ return value > 0 ? value : fallback;
67
+ }
68
+
69
+ const VIDEO_CONFIG_DEFAULTS: VideoConfig = {
70
+ enabled: true,
71
+ preferredModel: "gemini-3.6-flash",
72
+ maxSizeMB: 50,
73
+ };
74
+
75
+ let cachedVideoConfig: VideoConfig | null = null;
76
+
77
+ function loadVideoConfig(): VideoConfig {
78
+ if (cachedVideoConfig) return cachedVideoConfig;
79
+ if (!existsSync(CONFIG_PATH)) {
80
+ cachedVideoConfig = { ...VIDEO_CONFIG_DEFAULTS };
81
+ return cachedVideoConfig;
82
+ }
83
+
84
+ const rawText = readFileSync(CONFIG_PATH, "utf-8");
85
+ let raw: { video?: { enabled?: boolean; preferredModel?: string; maxSizeMB?: number } };
86
+ try {
87
+ raw = JSON.parse(rawText) as { video?: { enabled?: boolean; preferredModel?: string; maxSizeMB?: number } };
88
+ } catch (err) {
89
+ const message = err instanceof Error ? err.message : String(err);
90
+ throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
91
+ }
92
+
93
+ const v = raw.video ?? {};
94
+ cachedVideoConfig = {
95
+ enabled: normalizeEnabled(v.enabled, VIDEO_CONFIG_DEFAULTS.enabled),
96
+ preferredModel: normalizePreferredModel(v.preferredModel, VIDEO_CONFIG_DEFAULTS.preferredModel),
97
+ maxSizeMB: normalizeMaxSizeMB(v.maxSizeMB, VIDEO_CONFIG_DEFAULTS.maxSizeMB),
98
+ };
99
+ return cachedVideoConfig;
100
+ }
101
+
102
+ export function isVideoFile(input: string): VideoFileInfo | null {
103
+ const config = loadVideoConfig();
104
+ if (!config.enabled) return null;
105
+
106
+ const isFilePath = input.startsWith("/") || input.startsWith("./") || input.startsWith("../") || input.startsWith("file://")
107
+ || input.startsWith("\\") || /^[A-Za-z]:[\\/]/.test(input);
108
+ if (!isFilePath) return null;
109
+
110
+ let filePath = input;
111
+ if (input.startsWith("file://")) {
112
+ try {
113
+ filePath = decodeURIComponent(new URL(input).pathname);
114
+ } catch {
115
+ return null;
116
+ }
117
+ }
118
+
119
+ const ext = extname(filePath).toLowerCase();
120
+ const mimeType = VIDEO_EXTENSIONS[ext];
121
+ if (!mimeType) return null;
122
+
123
+ const absolutePath = resolveFilePath(filePath);
124
+ if (!absolutePath) return null;
125
+
126
+ let stat: ReturnType<typeof statSync>;
127
+ try {
128
+ stat = statSync(absolutePath);
129
+ } catch {
130
+ return null;
131
+ }
132
+ if (!stat.isFile()) return null;
133
+
134
+ const maxBytes = config.maxSizeMB * 1024 * 1024;
135
+ return {
136
+ absolutePath,
137
+ mimeType,
138
+ sizeBytes: stat.size,
139
+ maxSizeBytes: maxBytes,
140
+ withinUploadLimit: stat.size <= maxBytes,
141
+ };
142
+ }
143
+
144
+ function resolveFilePath(filePath: string): string | null {
145
+ const absolutePath = resolve(filePath);
146
+ if (existsSync(absolutePath)) return absolutePath;
147
+
148
+ const dir = dirname(absolutePath);
149
+ const base = basename(absolutePath);
150
+ if (!existsSync(dir)) return null;
151
+
152
+ try {
153
+ const normalizedBase = normalizeSpaces(base);
154
+ const match = readdirSync(dir).find(f => normalizeSpaces(f) === normalizedBase);
155
+ return match ? join(dir, match) : null;
156
+ } catch {
157
+ return null;
158
+ }
159
+ }
160
+
161
+ function normalizeSpaces(s: string): string {
162
+ return s.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000\uFEFF]/g, " ");
163
+ }
164
+
165
+ export async function extractVideo(
166
+ info: VideoFileInfo,
167
+ signal?: AbortSignal,
168
+ options?: ExtractOptions,
169
+ ): Promise<ExtractedContent | null> {
170
+ const config = loadVideoConfig();
171
+ const effectivePrompt = options?.prompt ?? DEFAULT_VIDEO_PROMPT;
172
+ const effectiveModel = options?.model ?? config.preferredModel;
173
+ const displayName = basename(info.absolutePath);
174
+ if (!info.withinUploadLimit) {
175
+ const sizeMB = (info.sizeBytes / 1024 / 1024).toFixed(1);
176
+ const maxSizeMB = (info.maxSizeBytes / 1024 / 1024).toFixed(1);
177
+ const error = `Local video ${displayName} is ${sizeMB} MiB, above configured video.maxSizeMB (${maxSizeMB} MiB) for Gemini analysis. Use timestamp/frames for ffmpeg frame extraction, increase video.maxSizeMB, or compress the file.`;
178
+ return { url: info.absolutePath, title: displayName, content: error, error };
179
+ }
180
+ const activityId = activityMonitor.logStart({ type: "fetch", url: `video:${displayName}` });
181
+
182
+ const result = await tryVideoGeminiApi(info, effectivePrompt, effectiveModel, signal)
183
+ ?? await tryVideoGeminiWeb(info, effectivePrompt, effectiveModel, signal);
184
+
185
+ if (result) {
186
+ if (canAttachImages()) {
187
+ const thumbnail = await extractVideoFrame(info.absolutePath);
188
+ if (!("error" in thumbnail)) {
189
+ result.thumbnail = thumbnail;
190
+ }
191
+ }
192
+ activityMonitor.logComplete(activityId, 200);
193
+ return result;
194
+ }
195
+
196
+ if (signal?.aborted) {
197
+ activityMonitor.logComplete(activityId, 0);
198
+ return null;
199
+ }
200
+
201
+ activityMonitor.logError(activityId, "all video extraction paths failed");
202
+ return null;
203
+ }
204
+
205
+ function mapFfprobeError(err: unknown): string {
206
+ const { code, stderr, message } = readExecError(err);
207
+ if (code === "ENOENT") return "ffprobe is not installed. Install ffmpeg which includes ffprobe";
208
+ const snippet = trimErrorText(stderr || message);
209
+ return snippet ? `ffprobe failed: ${snippet}` : "ffprobe failed";
210
+ }
211
+
212
+ export async function extractVideoFrame(filePath: string, seconds: number = 1): Promise<FrameResult> {
213
+ try {
214
+ const buffer = execFileSync("ffmpeg", [
215
+ "-ss", String(seconds), "-i", filePath,
216
+ "-frames:v", "1", "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1",
217
+ ], { maxBuffer: 5 * 1024 * 1024, timeout: 10000, stdio: ["pipe", "pipe", "pipe"] });
218
+ if (buffer.length === 0) return { error: "ffmpeg failed: empty output" };
219
+ return { data: buffer.toString("base64"), mimeType: "image/jpeg" };
220
+ } catch (err) {
221
+ return { error: mapFfmpegError(err) };
222
+ }
223
+ }
224
+
225
+ export async function getLocalVideoDuration(filePath: string): Promise<number | { error: string }> {
226
+ try {
227
+ const output = execFileSync("ffprobe", [
228
+ "-v", "quiet",
229
+ "-show_entries", "format=duration",
230
+ "-of", "csv=p=0",
231
+ filePath,
232
+ ], { timeout: 10000, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
233
+ const duration = Number.parseFloat(output);
234
+ if (!Number.isFinite(duration)) return { error: "ffprobe failed: invalid duration output" };
235
+ return duration;
236
+ } catch (err) {
237
+ return { error: mapFfprobeError(err) };
238
+ }
239
+ }
240
+
241
+ async function tryVideoGeminiWeb(
242
+ info: VideoFileInfo,
243
+ prompt: string,
244
+ model: string,
245
+ signal?: AbortSignal,
246
+ ): Promise<ExtractedContent | null> {
247
+ try {
248
+ const cookies = await isGeminiWebAvailable();
249
+ if (!cookies) return null;
250
+ if (signal?.aborted) return null;
251
+
252
+ const text = await queryWithCookies(prompt, cookies, {
253
+ files: [info.absolutePath],
254
+ ...(model !== "gemini-3.6-flash" ? { model } : {}),
255
+ signal,
256
+ timeoutMs: 180000,
257
+ });
258
+
259
+ return {
260
+ url: info.absolutePath,
261
+ title: extractVideoTitle(text, info.absolutePath),
262
+ content: text,
263
+ error: null,
264
+ };
265
+ } catch (err) {
266
+ if (shouldRethrow(err)) throw err;
267
+ return null;
268
+ }
269
+ }
270
+
271
+ async function tryVideoGeminiApi(
272
+ info: VideoFileInfo,
273
+ prompt: string,
274
+ model: string,
275
+ signal?: AbortSignal,
276
+ ): Promise<ExtractedContent | null> {
277
+ const apiKey = await getApiKey(signal);
278
+ if (!apiKey) return null;
279
+ if (signal?.aborted) return null;
280
+
281
+ let fileName: string | null = null;
282
+ try {
283
+ const uploaded = await uploadToFilesApi(info, apiKey, signal);
284
+ fileName = uploaded.name;
285
+
286
+ await pollFileState(fileName, apiKey, signal, 120000);
287
+
288
+ const text = await queryGeminiApiWithVideo(prompt, uploaded.uri, {
289
+ apiKey,
290
+ model,
291
+ mimeType: info.mimeType,
292
+ signal,
293
+ timeoutMs: 120000,
294
+ });
295
+
296
+ return {
297
+ url: info.absolutePath,
298
+ title: extractVideoTitle(text, info.absolutePath),
299
+ content: text,
300
+ error: null,
301
+ };
302
+ } catch (err) {
303
+ if (shouldRethrow(err)) throw err;
304
+ return null;
305
+ } finally {
306
+ if (fileName) deleteGeminiFile(fileName, apiKey);
307
+ }
308
+ }
309
+
310
+ async function uploadToFilesApi(
311
+ info: VideoFileInfo,
312
+ apiKey: string,
313
+ signal?: AbortSignal,
314
+ ): Promise<{ name: string; uri: string }> {
315
+ const displayName = basename(info.absolutePath);
316
+
317
+ const initRes = await fetchGeminiApi(`${getUploadBase()}/files`, {
318
+ method: "POST",
319
+ headers: {
320
+ "X-Goog-Upload-Protocol": "resumable",
321
+ "X-Goog-Upload-Command": "start",
322
+ "X-Goog-Upload-Header-Content-Length": String(info.sizeBytes),
323
+ "X-Goog-Upload-Header-Content-Type": info.mimeType,
324
+ "Content-Type": "application/json",
325
+ },
326
+ body: JSON.stringify({ file: { display_name: displayName } }),
327
+ signal,
328
+ }, apiKey);
329
+
330
+ if (!initRes.ok) {
331
+ const text = redactGeminiApiResponse(initRes, await initRes.text(), apiKey);
332
+ throw new Error(`File upload init failed: ${initRes.status} (${text.slice(0, 200)})`);
333
+ }
334
+
335
+ const uploadUrl = initRes.headers.get("x-goog-upload-url");
336
+ if (!uploadUrl) throw new Error("No upload URL in response headers");
337
+
338
+ const fileData = await readFile(info.absolutePath);
339
+ const uploadRes = await fetchGeminiApi(uploadUrl, {
340
+ method: "PUT",
341
+ headers: {
342
+ "Content-Length": String(info.sizeBytes),
343
+ "X-Goog-Upload-Offset": "0",
344
+ "X-Goog-Upload-Command": "upload, finalize",
345
+ },
346
+ body: fileData,
347
+ signal,
348
+ }, apiKey);
349
+
350
+ if (!uploadRes.ok) {
351
+ const text = redactGeminiApiResponse(uploadRes, await uploadRes.text(), apiKey);
352
+ throw new Error(`File upload failed: ${uploadRes.status} (${text.slice(0, 200)})`);
353
+ }
354
+
355
+ const result = await uploadRes.json() as { file: { name: string; uri: string } };
356
+ return result.file;
357
+ }
358
+
359
+ async function pollFileState(
360
+ fileName: string,
361
+ apiKey: string,
362
+ signal?: AbortSignal,
363
+ timeoutMs: number = 120000,
364
+ ): Promise<void> {
365
+ const deadline = Date.now() + timeoutMs;
366
+
367
+ while (Date.now() < deadline) {
368
+ if (signal?.aborted) throw new Error("Aborted");
369
+
370
+ const res = await fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`, { signal }, apiKey);
371
+ if (!res.ok) throw new Error(`File state check failed: ${res.status}`);
372
+
373
+ const data = await res.json() as { state: string };
374
+ if (data.state === "ACTIVE") return;
375
+ if (data.state === "FAILED") throw new Error("File processing failed");
376
+
377
+ await new Promise(r => setTimeout(r, 5000));
378
+ }
379
+
380
+ throw new Error("File processing timed out");
381
+ }
382
+
383
+ function deleteGeminiFile(fileName: string, apiKey: string): void {
384
+ void fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`, { method: "DELETE" }, apiKey).catch((err) => {
385
+ const message = err instanceof Error ? err.message : String(err);
386
+ console.error(`Failed to delete Gemini file ${fileName}: ${message}`);
387
+ });
388
+ }
389
+
390
+ function extractVideoTitle(text: string, filePath: string): string {
391
+ return extractHeadingTitle(text) ?? basename(filePath, extname(filePath));
392
+ }