@larkup/tool-video-intelligence 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/.env.example +150 -0
  2. package/LICENSE +176 -0
  3. package/README.md +281 -0
  4. package/compose.gpu.yaml +14 -0
  5. package/compose.yaml +71 -0
  6. package/dist/agent.d.ts +131 -0
  7. package/dist/agent.js +2087 -0
  8. package/dist/brief.d.ts +2 -0
  9. package/dist/brief.js +37 -0
  10. package/dist/client.d.ts +46 -0
  11. package/dist/client.js +139 -0
  12. package/dist/contracts.d.ts +331 -0
  13. package/dist/contracts.js +1 -0
  14. package/dist/index.d.ts +87 -0
  15. package/dist/index.js +391 -0
  16. package/dist/runtime.d.ts +96 -0
  17. package/dist/runtime.js +592 -0
  18. package/dist/ui.d.ts +82 -0
  19. package/dist/ui.js +87 -0
  20. package/package.json +84 -0
  21. package/runtime/Dockerfile +119 -0
  22. package/runtime/app/__init__.py +3 -0
  23. package/runtime/app/__main__.py +19 -0
  24. package/runtime/app/api/__init__.py +0 -0
  25. package/runtime/app/api/deps.py +69 -0
  26. package/runtime/app/api/v1.py +166 -0
  27. package/runtime/app/config.py +78 -0
  28. package/runtime/app/db/__init__.py +0 -0
  29. package/runtime/app/db/schemas.py +162 -0
  30. package/runtime/app/db/store.py +466 -0
  31. package/runtime/app/main.py +27 -0
  32. package/runtime/app/model_configuration.py +157 -0
  33. package/runtime/app/services/__init__.py +0 -0
  34. package/runtime/app/services/brain.py +2221 -0
  35. package/runtime/app/services/embedding.py +473 -0
  36. package/runtime/app/services/jobs.py +237 -0
  37. package/runtime/app/services/motion.py +66 -0
  38. package/runtime/app/services/pipeline.py +1911 -0
  39. package/runtime/app/services/scene.py +161 -0
  40. package/runtime/app/services/storage.py +44 -0
  41. package/runtime/app/services/transcription.py +667 -0
  42. package/runtime/app/services/vision.py +1441 -0
  43. package/runtime/app/utils/__init__.py +0 -0
  44. package/runtime/app/utils/timing.py +99 -0
  45. package/runtime/app/worker.py +20 -0
  46. package/runtime/pyproject.toml +56 -0
  47. package/runtime/requirements-cpu.txt +15 -0
  48. package/runtime/requirements-smoke.txt +7 -0
  49. package/runtime/requirements.txt +14 -0
  50. package/runtime/uv.lock +3637 -0
  51. package/scripts/grant-cloud-credits.sh +43 -0
  52. package/scripts/runtime.mjs +156 -0
  53. package/scripts/validate-indexing.mjs +168 -0
  54. package/tool.manifest.json +617 -0
@@ -0,0 +1,592 @@
1
+ import { execFile, spawn } from 'node:child_process';
2
+ import { existsSync, mkdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from 'node:fs';
3
+ import { fileURLToPath } from 'node:url';
4
+ import os from 'node:os';
5
+ import path from 'node:path';
6
+ import { promisify } from 'node:util';
7
+ import { VideoIntelligenceClient } from './client.js';
8
+ const execute = promisify(execFile);
9
+ const LOCAL_DOCKER_IMAGE = 'ghcr.io/larkup-ai/video-intelligence:0.1.0';
10
+ /** Starts the shipped local runtime only after the user selected local Docker mode. */
11
+ export async function ensureVideoRuntime(client, mode, localApiKey, localRuntimeUrl, understanding) {
12
+ try {
13
+ await client.health();
14
+ return;
15
+ }
16
+ catch (error) {
17
+ if (mode !== 'local-docker' && mode !== 'local-process')
18
+ throw error;
19
+ }
20
+ if (mode === 'local-process') {
21
+ await startNativeVideoRuntime(client, localApiKey, localRuntimeUrl, understanding);
22
+ return;
23
+ }
24
+ const packageDirectory = resolvePackageDirectory();
25
+ const acceleration = await detectLocalAcceleration();
26
+ try {
27
+ await execute('docker', ['compose', ...dockerComposeFiles(packageDirectory, acceleration), 'up', '-d', '--wait'], {
28
+ timeout: 15 * 60_000,
29
+ maxBuffer: 1024 * 1024,
30
+ env: localRuntimeEnvironment(localApiKey, localRuntimeUrl, understanding, acceleration),
31
+ });
32
+ }
33
+ catch (error) {
34
+ if (error.code === 'ENOENT') {
35
+ throw new Error('Docker Desktop is required for the local runtime. Install it and make sure Docker is running.');
36
+ }
37
+ throw error;
38
+ }
39
+ await client.health();
40
+ }
41
+ /** Recreates the local container so a changed shared key is applied immediately. */
42
+ export async function restartVideoRuntime(mode, localApiKey, localRuntimeUrl, understanding) {
43
+ if (mode === 'local-process') {
44
+ const packageDirectory = resolvePackageDirectory();
45
+ stopNativeVideoRuntime(nativePidPath(packageDirectory));
46
+ await new Promise((resolve) => setTimeout(resolve, 250));
47
+ const client = new VideoIntelligenceClient({
48
+ mode: 'local-process',
49
+ endpoint: localRuntimeUrl,
50
+ apiKey: localApiKey,
51
+ });
52
+ await startNativeVideoRuntime(client, localApiKey, localRuntimeUrl, understanding);
53
+ return;
54
+ }
55
+ const packageDirectory = resolvePackageDirectory();
56
+ const acceleration = await detectLocalAcceleration();
57
+ try {
58
+ await execute('docker', [
59
+ 'compose',
60
+ ...dockerComposeFiles(packageDirectory, acceleration),
61
+ 'up',
62
+ '-d',
63
+ '--force-recreate',
64
+ '--wait',
65
+ ], {
66
+ timeout: 15 * 60_000,
67
+ maxBuffer: 1024 * 1024,
68
+ env: localRuntimeEnvironment(localApiKey, localRuntimeUrl, understanding, acceleration),
69
+ });
70
+ }
71
+ catch (error) {
72
+ if (error.code === 'ENOENT') {
73
+ throw new Error('Docker Desktop is required for the local runtime. Install it and make sure Docker is running.');
74
+ }
75
+ throw error;
76
+ }
77
+ }
78
+ /** Stops the local runtime without removing the installed image/dependencies. */
79
+ export async function stopVideoRuntime() {
80
+ const packageDirectory = resolvePackageDirectory();
81
+ // The preferred runtime can change after installation (for example when a
82
+ // GPU or Docker becomes available). Stop both owned variants so the status
83
+ // never remains Running because we guessed the wrong one.
84
+ stopNativeVideoRuntime(nativePidPath(packageDirectory));
85
+ try {
86
+ await execute('docker', ['compose', '-f', path.join(packageDirectory, 'compose.yaml'), 'stop'], {
87
+ timeout: 60_000,
88
+ maxBuffer: 1024 * 1024,
89
+ });
90
+ }
91
+ catch {
92
+ // A native runtime may be the active one while Docker is unavailable.
93
+ // Health is checked by the caller, so only a still-running endpoint is an
94
+ // actionable stop failure.
95
+ }
96
+ }
97
+ /** Removes local runtime state only after the user explicitly removes the tool. */
98
+ export async function removeVideoRuntime(kind) {
99
+ const packageDirectory = resolvePackageDirectory();
100
+ if (kind === 'local-process') {
101
+ stopNativeVideoRuntime(nativePidPath(packageDirectory));
102
+ rmSync(path.join(process.cwd(), '.larkup', 'video-intelligence'), {
103
+ recursive: true,
104
+ force: true,
105
+ });
106
+ return;
107
+ }
108
+ try {
109
+ await execute('docker', [
110
+ 'compose',
111
+ '-f',
112
+ path.join(packageDirectory, 'compose.yaml'),
113
+ 'down',
114
+ '--volumes',
115
+ '--rmi',
116
+ 'local',
117
+ ], { timeout: 2 * 60_000, maxBuffer: 1024 * 1024 });
118
+ }
119
+ catch (error) {
120
+ if (error.code === 'ENOENT')
121
+ return;
122
+ throw error;
123
+ }
124
+ }
125
+ /**
126
+ * Prepares whatever a local kind needs, without starting it: pulls the Docker
127
+ * image, or installs uv (via astral's official curl|sh installer when it is
128
+ * missing) and syncs the native runtime's Python dependencies.
129
+ */
130
+ export async function installLocalRuntime(kind, localApiKey, localRuntimeUrl, understanding) {
131
+ if (kind === 'local-docker') {
132
+ try {
133
+ const packageDirectory = resolvePackageDirectory();
134
+ const acceleration = await detectLocalAcceleration();
135
+ if (acceleration.dockerSupported) {
136
+ await execute('docker', ['compose', ...dockerComposeFiles(packageDirectory, acceleration), 'build'], { timeout: 30 * 60_000, maxBuffer: 1024 * 1024 });
137
+ return;
138
+ }
139
+ await execute('docker', ['pull', LOCAL_DOCKER_IMAGE], {
140
+ timeout: 20 * 60_000,
141
+ maxBuffer: 1024 * 1024,
142
+ });
143
+ }
144
+ catch (error) {
145
+ if (error.code === 'ENOENT') {
146
+ throw new Error('Docker Desktop is required for the local runtime. Install it and make sure Docker is running.');
147
+ }
148
+ throw error;
149
+ }
150
+ return;
151
+ }
152
+ const packageDirectory = resolvePackageDirectory();
153
+ const runtimeDirectory = path.join(packageDirectory, 'runtime');
154
+ const native = await detectNativeHost();
155
+ const acceleration = await detectLocalAcceleration();
156
+ if (!native.uvInstalled) {
157
+ try {
158
+ await execute('sh', ['-c', 'curl -LsSf https://astral.sh/uv/install.sh | sh'], {
159
+ timeout: 5 * 60_000,
160
+ maxBuffer: 1024 * 1024,
161
+ });
162
+ }
163
+ catch (error) {
164
+ throw new Error(`Could not install uv automatically. Install it yourself from https://docs.astral.sh/uv/ and try again. (${error instanceof Error ? error.message : 'unknown error'})`);
165
+ }
166
+ }
167
+ await execute('uv', [
168
+ 'sync',
169
+ '--directory',
170
+ runtimeDirectory,
171
+ '--extra',
172
+ acceleration.nativeSupported ? 'gpu' : 'cpu',
173
+ ], {
174
+ timeout: 15 * 60_000,
175
+ maxBuffer: 1024 * 1024,
176
+ env: nativeRuntimeEnvironment(packageDirectory, localApiKey, localRuntimeUrl, understanding, acceleration),
177
+ });
178
+ }
179
+ /** `docker info` (not just `docker --version`) so a stopped daemon is distinguished from a missing CLI. */
180
+ export async function detectDockerHost() {
181
+ try {
182
+ await execute('docker', ['info'], { timeout: 8_000, maxBuffer: 1024 * 1024 });
183
+ }
184
+ catch (error) {
185
+ if (error.code === 'ENOENT') {
186
+ return {
187
+ cliInstalled: false,
188
+ daemonRunning: false,
189
+ imagePulled: false,
190
+ message: "Docker isn't installed on this machine.",
191
+ };
192
+ }
193
+ return {
194
+ cliInstalled: true,
195
+ daemonRunning: false,
196
+ imagePulled: false,
197
+ message: 'Docker is installed but not running. Start Docker Desktop (or the Docker engine) and try again.',
198
+ };
199
+ }
200
+ let imagePulled = false;
201
+ try {
202
+ await execute('docker', ['image', 'inspect', LOCAL_DOCKER_IMAGE], {
203
+ timeout: 8_000,
204
+ maxBuffer: 1024 * 1024,
205
+ });
206
+ imagePulled = true;
207
+ }
208
+ catch {
209
+ imagePulled = false;
210
+ }
211
+ return { cliInstalled: true, daemonRunning: true, imagePulled, message: 'Docker is ready.' };
212
+ }
213
+ /**
214
+ * Detect NVIDIA through its driver utility rather than assuming any GPU can
215
+ * run CUDA. Apple/AMD devices correctly stay on the efficient CPU path until
216
+ * there is a supported local operator build for them.
217
+ */
218
+ export async function detectNvidiaGpu() {
219
+ try {
220
+ const { stdout } = await execute('nvidia-smi', ['--query-gpu=name,memory.total', '--format=csv,noheader,nounits'], { timeout: 8_000, maxBuffer: 64 * 1024 });
221
+ const first = stdout.trim().split(/\r?\n/, 1)[0];
222
+ const match = first?.match(/^\s*(.+?)\s*,\s*([\d.]+)\s*$/);
223
+ if (!match)
224
+ return { available: false, message: 'No supported NVIDIA GPU was detected.' };
225
+ const memoryMB = Number(match[2]);
226
+ return {
227
+ available: true,
228
+ name: match[1].trim(),
229
+ ...(Number.isFinite(memoryMB) ? { memoryGB: memoryMB / 1024 } : {}),
230
+ message: `${match[1].trim()} detected${Number.isFinite(memoryMB) ? ` (${(memoryMB / 1024).toFixed(1)} GB)` : ''}.`,
231
+ };
232
+ }
233
+ catch {
234
+ return { available: false, message: 'No supported NVIDIA GPU was detected.' };
235
+ }
236
+ }
237
+ /** Selects the fastest supported local path; it never selects a Larkup-managed worker. */
238
+ export async function detectLocalAcceleration() {
239
+ const [gpu, docker] = await Promise.all([detectNvidiaGpu(), detectDockerHost()]);
240
+ let dockerSupported = false;
241
+ if (gpu.available && docker.daemonRunning && os.platform() === 'linux') {
242
+ try {
243
+ const { stdout } = await execute('docker', ['info', '--format', '{{json .Runtimes}}'], {
244
+ timeout: 8_000,
245
+ maxBuffer: 64 * 1024,
246
+ });
247
+ const runtimes = JSON.parse(stdout);
248
+ dockerSupported = 'nvidia' in runtimes;
249
+ }
250
+ catch {
251
+ dockerSupported = false;
252
+ }
253
+ }
254
+ if (gpu.available) {
255
+ const memory = gpu.memoryGB ? ` with ${gpu.memoryGB.toFixed(1)} GB VRAM` : '';
256
+ return {
257
+ device: 'cuda',
258
+ dockerSupported,
259
+ nativeSupported: true,
260
+ gpuName: gpu.name,
261
+ gpuMemoryGB: gpu.memoryGB,
262
+ message: dockerSupported
263
+ ? `${gpu.name} will accelerate local video processing${memory}.`
264
+ : `${gpu.name} is available for the native local runtime${memory}. Docker GPU support is not available on this machine.`,
265
+ };
266
+ }
267
+ return {
268
+ device: 'cpu',
269
+ dockerSupported: false,
270
+ nativeSupported: false,
271
+ message: 'No CUDA-capable NVIDIA GPU is available, so local processing will use this computer’s CPU.',
272
+ };
273
+ }
274
+ export async function detectNativeHost() {
275
+ try {
276
+ await execute('uv', ['--version'], {
277
+ timeout: 8_000,
278
+ maxBuffer: 1024 * 1024,
279
+ env: withUvPath(),
280
+ });
281
+ }
282
+ catch {
283
+ return {
284
+ uvInstalled: false,
285
+ depsInstalled: false,
286
+ message: 'uv is not installed on this machine.',
287
+ };
288
+ }
289
+ const runtimeDirectory = path.join(resolvePackageDirectory(), 'runtime');
290
+ const depsInstalled = existsSync(path.join(runtimeDirectory, '.venv'));
291
+ return {
292
+ uvInstalled: true,
293
+ depsInstalled,
294
+ message: depsInstalled
295
+ ? 'Native runtime is ready.'
296
+ : 'uv is installed; Python dependencies are not synced yet.',
297
+ };
298
+ }
299
+ /** Live-detects Docker vs. native every call so the UI never trusts a stale persisted choice. */
300
+ export async function detectLocalRuntimeHost() {
301
+ const [docker, native, acceleration] = await Promise.all([
302
+ detectDockerHost(),
303
+ detectNativeHost(),
304
+ detectLocalAcceleration(),
305
+ ]);
306
+ const recommendedKind = acceleration.dockerSupported
307
+ ? 'local-docker'
308
+ : acceleration.nativeSupported
309
+ ? 'local-process'
310
+ : docker.daemonRunning
311
+ ? 'local-docker'
312
+ : native.uvInstalled
313
+ ? 'local-process'
314
+ : null;
315
+ const totalMemGB = os.totalmem() / 1024 ** 3;
316
+ const freeMemGB = os.freemem() / 1024 ** 3;
317
+ const cpus = os.cpus().length;
318
+ const suitability = recommendedKind === 'local-docker'
319
+ ? freeMemGB < 6
320
+ ? {
321
+ level: 'tight',
322
+ message: `Only ${freeMemGB.toFixed(1)} GB RAM is free; the ~8 GB Docker image runs better with more headroom. The native runtime is lighter, or close other apps first.`,
323
+ }
324
+ : {
325
+ level: 'good',
326
+ message: `${freeMemGB.toFixed(1)} GB RAM free across ${cpus} CPU cores — comfortable for the Docker runtime.`,
327
+ }
328
+ : recommendedKind === 'local-process'
329
+ ? acceleration.nativeSupported
330
+ ? {
331
+ level: 'good',
332
+ message: `${acceleration.message} The native runtime is recommended so it can use the GPU directly.`,
333
+ }
334
+ : freeMemGB < 3
335
+ ? {
336
+ level: 'tight',
337
+ message: `Only ${freeMemGB.toFixed(1)} GB RAM is free; the native CPU runtime may run slowly.`,
338
+ }
339
+ : {
340
+ level: 'good',
341
+ message: `${freeMemGB.toFixed(1)} GB RAM free across ${cpus} CPU cores — the native runtime should run well.`,
342
+ }
343
+ : {
344
+ level: 'unknown',
345
+ message: 'Neither Docker nor uv was detected yet. Installing will set up uv automatically.',
346
+ };
347
+ return {
348
+ docker,
349
+ native,
350
+ recommendedKind,
351
+ installed: docker.imagePulled || native.depsInstalled,
352
+ system: { platform: os.platform(), cpus, totalMemGB, freeMemGB },
353
+ acceleration,
354
+ suitability,
355
+ };
356
+ }
357
+ /**
358
+ * Starts the portable Python runtime through uv. This keeps Docker optional
359
+ * while retaining the exact same HTTP and job contract as the container.
360
+ */
361
+ async function startNativeVideoRuntime(client, localApiKey, localRuntimeUrl, understanding) {
362
+ const packageDirectory = resolvePackageDirectory();
363
+ const runtimeDirectory = path.join(packageDirectory, 'runtime');
364
+ const acceleration = await detectLocalAcceleration();
365
+ const environment = nativeRuntimeEnvironment(packageDirectory, localApiKey, localRuntimeUrl, understanding, acceleration);
366
+ const child = spawn('uv', [
367
+ 'run',
368
+ '--directory',
369
+ runtimeDirectory,
370
+ '--extra',
371
+ acceleration.nativeSupported ? 'gpu' : 'cpu',
372
+ 'larkup-video-runtime',
373
+ ], { detached: true, stdio: 'ignore', env: environment });
374
+ await new Promise((resolve, reject) => {
375
+ const ready = setTimeout(resolve, 100);
376
+ child.once('error', (error) => {
377
+ clearTimeout(ready);
378
+ if (error.code === 'ENOENT') {
379
+ reject(new Error('The native runtime requires uv. Install it from https://docs.astral.sh/uv/.'));
380
+ return;
381
+ }
382
+ reject(error);
383
+ });
384
+ });
385
+ child.unref();
386
+ const pidFile = nativePidPath(packageDirectory);
387
+ mkdirSync(path.dirname(pidFile), { recursive: true });
388
+ writeFileSync(pidFile, `${child.pid ?? ''}\n`, 'utf8');
389
+ const deadline = Date.now() + 5 * 60_000;
390
+ let lastError;
391
+ while (Date.now() < deadline) {
392
+ try {
393
+ await client.health();
394
+ return;
395
+ }
396
+ catch (error) {
397
+ lastError = error;
398
+ await new Promise((resolve) => setTimeout(resolve, 1_000));
399
+ }
400
+ }
401
+ throw new Error(`The native Video Intelligence runtime did not become ready. ${lastError instanceof Error
402
+ ? lastError.message
403
+ : 'Check that uv can install the CPU dependencies.'}`);
404
+ }
405
+ function resolvePackageDirectory() {
406
+ return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
407
+ }
408
+ function nativePidPath(packageDirectory) {
409
+ return path.join(process.cwd(), '.larkup', 'video-intelligence', 'runtime.pid');
410
+ }
411
+ export function stopNativeVideoRuntime(pidFile) {
412
+ if (!existsSync(pidFile))
413
+ return;
414
+ const pid = Number.parseInt(readFileSync(pidFile, 'utf8').trim(), 10);
415
+ try {
416
+ if (Number.isSafeInteger(pid) && pid > 0)
417
+ process.kill(pid, 'SIGTERM');
418
+ }
419
+ catch {
420
+ // The process may already have exited; a new runtime can still be launched.
421
+ }
422
+ finally {
423
+ unlinkSync(pidFile);
424
+ }
425
+ }
426
+ /** Prepends common uv install locations so a runtime started right after a fresh curl|sh install can find it. */
427
+ function withUvPath(env = process.env) {
428
+ const home = os.homedir();
429
+ const extra = [path.join(home, '.local', 'bin'), path.join(home, '.cargo', 'bin')];
430
+ return { ...env, PATH: [...extra, env.PATH ?? ''].filter(Boolean).join(path.delimiter) };
431
+ }
432
+ function localRuntimeEnvironment(localApiKey, localRuntimeUrl, understanding, acceleration) {
433
+ const port = portFromUrl(localRuntimeUrl);
434
+ return {
435
+ ...process.env,
436
+ // This selects only hardware on the user's machine. It does not contact
437
+ // Modal, RunPod, or any Larkup-managed worker.
438
+ LARKUP_VIDEO_DEVICE: acceleration?.dockerSupported ? 'cuda' : 'cpu',
439
+ ...(port ? { LARKUP_VIDEO_PORT: port } : {}),
440
+ ...(localApiKey
441
+ ? { LARKUP_VIDEO_REQUIRE_AUTH: 'true', LARKUP_VIDEO_SHARED_API_KEY: localApiKey }
442
+ : {}),
443
+ ...videoUnderstandingEnvironment(understanding),
444
+ };
445
+ }
446
+ function nativeRuntimeEnvironment(packageDirectory, localApiKey, localRuntimeUrl, understanding, acceleration) {
447
+ const port = portFromUrl(localRuntimeUrl) ?? '8787';
448
+ const hostname = hostnameFromUrl(localRuntimeUrl);
449
+ const dataDirectory = path.join(process.cwd(), '.larkup', 'video-intelligence');
450
+ return withUvPath({
451
+ ...process.env,
452
+ LARKUP_VIDEO_PORT: port,
453
+ LARKUP_VIDEO_RUNTIME_KIND: 'local-process',
454
+ LARKUP_VIDEO_HOST: hostname && hostname !== '127.0.0.1' && hostname !== 'localhost' ? '0.0.0.0' : '127.0.0.1',
455
+ LARKUP_VIDEO_DATA_DIR: path.join(dataDirectory, 'data'),
456
+ LARKUP_VIDEO_MODEL_DIR: path.join(dataDirectory, 'models'),
457
+ LARKUP_VIDEO_DEVICE: acceleration?.nativeSupported ? 'cuda' : 'cpu',
458
+ ...(localApiKey
459
+ ? { LARKUP_VIDEO_REQUIRE_AUTH: 'true', LARKUP_VIDEO_SHARED_API_KEY: localApiKey }
460
+ : {}),
461
+ ...videoUnderstandingEnvironment(understanding),
462
+ });
463
+ }
464
+ export function videoUnderstandingEnvironment(config) {
465
+ // Explicit blanks avoid a developer shell/.env accidentally changing a
466
+ // user's Local runtime. Remote calls below are always direct calls using the
467
+ // user's selected provider and key—not Larkup-managed compute.
468
+ const env = {
469
+ LARKUP_VIDEO_SEMANTIC_VISION: 'false',
470
+ LARKUP_VIDEO_VISION_PROVIDER: 'vercel_ai_gateway',
471
+ LARKUP_VIDEO_VISION_API_KEY: '',
472
+ AI_GATEWAY_API_KEY: '',
473
+ LARKUP_VIDEO_EMBEDDING_PROVIDER: 'disabled',
474
+ LARKUP_VIDEO_AGENT_ENABLED: 'false',
475
+ LARKUP_VIDEO_AGENT_PROVIDER: 'vercel_ai_gateway',
476
+ LARKUP_VIDEO_AGENT_API_KEY: '',
477
+ LARKUP_VIDEO_AGENT_MODEL: 'openai/gpt-5-mini',
478
+ LARKUP_VIDEO_TRANSCRIPTION_PROVIDER: '',
479
+ LARKUP_VIDEO_TRANSCRIPTION_FALLBACK: '',
480
+ LARKUP_VIDEO_DEEPGRAM_MODEL: '',
481
+ LARKUP_VIDEO_DEEPGRAM_AUTO_MODEL: '',
482
+ LARKUP_VIDEO_OPENAI_TRANSCRIPTION_MODEL: '',
483
+ LARKUP_VIDEO_GROQ_TRANSCRIPTION_MODEL: '',
484
+ LARKUP_VIDEO_ELEVENLABS_TRANSCRIPTION_MODEL: '',
485
+ DEEPGRAM_API_KEY: '',
486
+ OPENAI_API_KEY: '',
487
+ GROQ_API_KEY: '',
488
+ ELEVENLABS_API_KEY: '',
489
+ };
490
+ if (!config)
491
+ return env;
492
+ if (config.visionProvider)
493
+ env.LARKUP_VIDEO_VISION_PROVIDER = config.visionProvider;
494
+ if (config.semanticVisionModel)
495
+ env.LARKUP_VIDEO_SEMANTIC_VISION_MODEL = config.semanticVisionModel;
496
+ if (config.visionApiKey) {
497
+ env.LARKUP_VIDEO_SEMANTIC_VISION = 'true';
498
+ env.LARKUP_VIDEO_VISION_API_KEY = config.visionApiKey;
499
+ if (config.visionProvider === 'vercel_ai_gateway') {
500
+ env.AI_GATEWAY_API_KEY = config.visionApiKey;
501
+ }
502
+ }
503
+ if (config.agentProvider)
504
+ env.LARKUP_VIDEO_AGENT_PROVIDER = config.agentProvider;
505
+ if (config.agentModel)
506
+ env.LARKUP_VIDEO_AGENT_MODEL = config.agentModel;
507
+ if (config.agentApiKey) {
508
+ env.LARKUP_VIDEO_AGENT_ENABLED = 'true';
509
+ env.LARKUP_VIDEO_AGENT_API_KEY = config.agentApiKey;
510
+ }
511
+ if (config.audioProvider)
512
+ env.LARKUP_VIDEO_TRANSCRIPTION_PROVIDER =
513
+ config.audioProvider === 'local' || config.audioProvider === 'larkup-cloud'
514
+ ? 'whisper'
515
+ : config.audioProvider;
516
+ if (config.audioModel) {
517
+ if (config.audioProvider === 'deepgram') {
518
+ env.LARKUP_VIDEO_DEEPGRAM_MODEL = config.audioModel;
519
+ env.LARKUP_VIDEO_DEEPGRAM_AUTO_MODEL = config.audioModel;
520
+ }
521
+ if (config.audioProvider === 'openai') {
522
+ env.LARKUP_VIDEO_OPENAI_TRANSCRIPTION_MODEL = config.audioModel;
523
+ }
524
+ if (config.audioProvider === 'groq') {
525
+ env.LARKUP_VIDEO_GROQ_TRANSCRIPTION_MODEL = config.audioModel;
526
+ }
527
+ if (config.audioProvider === 'elevenlabs') {
528
+ env.LARKUP_VIDEO_ELEVENLABS_TRANSCRIPTION_MODEL = config.audioModel;
529
+ }
530
+ }
531
+ if (config.audioProvider === 'deepgram' && config.audioApiKey)
532
+ env.DEEPGRAM_API_KEY = config.audioApiKey;
533
+ if (config.audioProvider === 'openai' && config.audioApiKey)
534
+ env.OPENAI_API_KEY = config.audioApiKey;
535
+ if (config.audioProvider === 'groq' && config.audioApiKey)
536
+ env.GROQ_API_KEY = config.audioApiKey;
537
+ if (config.audioProvider === 'elevenlabs' && config.audioApiKey)
538
+ env.ELEVENLABS_API_KEY = config.audioApiKey;
539
+ if (config.videoEmbeddingProvider) {
540
+ env.LARKUP_VIDEO_EMBEDDING_PROVIDER = config.videoEmbeddingProvider;
541
+ }
542
+ else if (config.visionProvider === 'vercel_ai_gateway' && config.visionApiKey) {
543
+ // The same user-owned Gateway key can create multimodal document/query
544
+ // vectors. Enabling it here keeps Local retrieval RAG-first without
545
+ // requiring a second provider setting; an explicit `disabled` still wins.
546
+ env.LARKUP_VIDEO_EMBEDDING_PROVIDER = 'gateway-gemini-embedding-2';
547
+ }
548
+ if (config.dashscopeApiKey)
549
+ env.DASHSCOPE_API_KEY = config.dashscopeApiKey;
550
+ if (config.dashscopeWorkspaceId)
551
+ env.DASHSCOPE_WORKSPACE_ID = config.dashscopeWorkspaceId;
552
+ if (config.dashscopeRegion)
553
+ env.DASHSCOPE_REGION = config.dashscopeRegion;
554
+ if (config.runpodEmbeddingApiKey)
555
+ env.RUNPOD_API_KEY = config.runpodEmbeddingApiKey;
556
+ if (config.runpodEmbeddingEndpointId)
557
+ env.LARKUP_VIDEO_RUNPOD_EMBEDDING_ENDPOINT_ID = config.runpodEmbeddingEndpointId;
558
+ if (config.hfEmbeddingUrl)
559
+ env.LARKUP_VIDEO_HF_EMBEDDING_URL = config.hfEmbeddingUrl;
560
+ if (config.hfEmbeddingApiKey)
561
+ env.HF_TOKEN = config.hfEmbeddingApiKey;
562
+ return env;
563
+ }
564
+ function dockerComposeFiles(packageDirectory, acceleration) {
565
+ const files = ['-f', path.join(packageDirectory, 'compose.yaml')];
566
+ if (acceleration.dockerSupported)
567
+ files.push('-f', path.join(packageDirectory, 'compose.gpu.yaml'));
568
+ return files;
569
+ }
570
+ function portFromUrl(value) {
571
+ if (!value)
572
+ return undefined;
573
+ try {
574
+ const port = new URL(value).port;
575
+ return port && Number.isInteger(Number(port)) && Number(port) > 0 && Number(port) < 65_536
576
+ ? port
577
+ : undefined;
578
+ }
579
+ catch {
580
+ return undefined;
581
+ }
582
+ }
583
+ function hostnameFromUrl(value) {
584
+ if (!value)
585
+ return undefined;
586
+ try {
587
+ return new URL(value).hostname;
588
+ }
589
+ catch {
590
+ return undefined;
591
+ }
592
+ }
package/dist/ui.d.ts ADDED
@@ -0,0 +1,82 @@
1
+ export declare const VIDEO_INDEXING_BRIEF_SURFACE: {
2
+ readonly id: "video-indexing-brief";
3
+ readonly version: 1;
4
+ readonly slot: "data-indexing";
5
+ readonly title: "Let your AI understand this video";
6
+ readonly description: "Choose how deeply to analyze it and optionally point it to what matters most.";
7
+ readonly appliesTo: readonly ["video"];
8
+ readonly estimate: {
9
+ readonly modeField: "indexingMode";
10
+ readonly variants: readonly [{
11
+ readonly value: "fast";
12
+ readonly analyzedFramesPerSourceMinute: 5;
13
+ readonly ocrFramesPerSourceMinute: 3;
14
+ readonly processingSecondsPerSourceMinute: 4;
15
+ readonly maxProcessingSecondsPerSourceMinute: 5;
16
+ readonly fixedOverheadSeconds: 60;
17
+ readonly maxFixedOverheadSeconds: 60;
18
+ readonly creditsPerSourceMinute: 1;
19
+ }, {
20
+ readonly value: "balanced";
21
+ readonly analyzedFramesPerSourceMinute: 12;
22
+ readonly ocrFramesPerSourceMinute: 8;
23
+ readonly processingSecondsPerSourceMinute: 16;
24
+ readonly maxProcessingSecondsPerSourceMinute: 30;
25
+ readonly fixedOverheadSeconds: 120;
26
+ readonly maxFixedOverheadSeconds: 240;
27
+ readonly creditsPerSourceMinute: 2;
28
+ }, {
29
+ readonly value: "thorough";
30
+ readonly analyzedFramesPerSourceMinute: 30;
31
+ readonly ocrFramesPerSourceMinute: 20;
32
+ readonly processingSecondsPerSourceMinute: 32;
33
+ readonly maxProcessingSecondsPerSourceMinute: 60;
34
+ readonly fixedOverheadSeconds: 180;
35
+ readonly maxFixedOverheadSeconds: 360;
36
+ readonly creditsPerSourceMinute: 4;
37
+ }];
38
+ };
39
+ readonly form: {
40
+ readonly submitLabel: "Start indexing";
41
+ readonly fields: readonly [{
42
+ readonly key: "goal";
43
+ readonly type: "textarea";
44
+ readonly label: "What should your AI look for? (optional)";
45
+ readonly placeholder: "For example: find the final score, the moment the package is dropped, or every mention of pricing.";
46
+ }, {
47
+ readonly key: "indexingMode";
48
+ readonly type: "select";
49
+ readonly label: "Coverage";
50
+ readonly defaultValue: "balanced";
51
+ readonly options: readonly [{
52
+ readonly label: "Fast";
53
+ readonly value: "fast";
54
+ readonly description: "Sample key frames for a quick overview.";
55
+ readonly setValues: {
56
+ readonly processingAuthorityConfirmed: false;
57
+ };
58
+ }, {
59
+ readonly label: "Balanced";
60
+ readonly value: "balanced";
61
+ readonly description: "Sample visual and OCR evidence across the video.";
62
+ readonly setValues: {
63
+ readonly processingAuthorityConfirmed: false;
64
+ };
65
+ }, {
66
+ readonly label: "Thorough";
67
+ readonly value: "thorough";
68
+ readonly description: "Let the agent inspect denser evidence around subtle or infrequent details.";
69
+ readonly setValues: {
70
+ readonly processingAuthorityConfirmed: false;
71
+ };
72
+ }];
73
+ }];
74
+ };
75
+ };
76
+ export declare const VIDEO_JOB_RESULT_SURFACE: {
77
+ readonly id: "video-evidence-result";
78
+ readonly version: 1;
79
+ readonly slot: "chat-result";
80
+ readonly title: "Video evidence";
81
+ readonly resultType: "application/vnd.larkup.video-evidence+json;version=1";
82
+ };