@larkup/tool-video-intelligence 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +150 -0
- package/LICENSE +176 -0
- package/README.md +281 -0
- package/compose.gpu.yaml +14 -0
- package/compose.yaml +71 -0
- package/dist/agent.d.ts +131 -0
- package/dist/agent.js +2087 -0
- package/dist/brief.d.ts +2 -0
- package/dist/brief.js +37 -0
- package/dist/client.d.ts +46 -0
- package/dist/client.js +139 -0
- package/dist/contracts.d.ts +331 -0
- package/dist/contracts.js +1 -0
- package/dist/index.d.ts +87 -0
- package/dist/index.js +391 -0
- package/dist/runtime.d.ts +96 -0
- package/dist/runtime.js +592 -0
- package/dist/ui.d.ts +82 -0
- package/dist/ui.js +87 -0
- package/package.json +84 -0
- package/runtime/Dockerfile +119 -0
- package/runtime/app/__init__.py +3 -0
- package/runtime/app/__main__.py +19 -0
- package/runtime/app/api/__init__.py +0 -0
- package/runtime/app/api/deps.py +69 -0
- package/runtime/app/api/v1.py +166 -0
- package/runtime/app/config.py +78 -0
- package/runtime/app/db/__init__.py +0 -0
- package/runtime/app/db/schemas.py +162 -0
- package/runtime/app/db/store.py +466 -0
- package/runtime/app/main.py +27 -0
- package/runtime/app/model_configuration.py +157 -0
- package/runtime/app/services/__init__.py +0 -0
- package/runtime/app/services/brain.py +2221 -0
- package/runtime/app/services/embedding.py +473 -0
- package/runtime/app/services/jobs.py +237 -0
- package/runtime/app/services/motion.py +66 -0
- package/runtime/app/services/pipeline.py +1911 -0
- package/runtime/app/services/scene.py +161 -0
- package/runtime/app/services/storage.py +44 -0
- package/runtime/app/services/transcription.py +667 -0
- package/runtime/app/services/vision.py +1441 -0
- package/runtime/app/utils/__init__.py +0 -0
- package/runtime/app/utils/timing.py +99 -0
- package/runtime/app/worker.py +20 -0
- package/runtime/pyproject.toml +56 -0
- package/runtime/requirements-cpu.txt +15 -0
- package/runtime/requirements-smoke.txt +7 -0
- package/runtime/requirements.txt +14 -0
- package/runtime/uv.lock +3637 -0
- package/scripts/grant-cloud-credits.sh +43 -0
- package/scripts/runtime.mjs +156 -0
- package/scripts/validate-indexing.mjs +168 -0
- package/tool.manifest.json +617 -0
package/dist/ui.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
export const VIDEO_INDEXING_BRIEF_SURFACE = {
|
|
2
|
+
id: 'video-indexing-brief',
|
|
3
|
+
version: 1,
|
|
4
|
+
slot: 'data-indexing',
|
|
5
|
+
title: 'Let your AI understand this video',
|
|
6
|
+
description: 'Choose how deeply to analyze it and optionally point it to what matters most.',
|
|
7
|
+
appliesTo: ['video'],
|
|
8
|
+
estimate: {
|
|
9
|
+
modeField: 'indexingMode',
|
|
10
|
+
variants: [
|
|
11
|
+
{
|
|
12
|
+
value: 'fast',
|
|
13
|
+
analyzedFramesPerSourceMinute: 5,
|
|
14
|
+
ocrFramesPerSourceMinute: 3,
|
|
15
|
+
processingSecondsPerSourceMinute: 4,
|
|
16
|
+
maxProcessingSecondsPerSourceMinute: 5,
|
|
17
|
+
fixedOverheadSeconds: 60,
|
|
18
|
+
maxFixedOverheadSeconds: 60,
|
|
19
|
+
creditsPerSourceMinute: 1,
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
value: 'balanced',
|
|
23
|
+
analyzedFramesPerSourceMinute: 12,
|
|
24
|
+
ocrFramesPerSourceMinute: 8,
|
|
25
|
+
processingSecondsPerSourceMinute: 16,
|
|
26
|
+
maxProcessingSecondsPerSourceMinute: 30,
|
|
27
|
+
fixedOverheadSeconds: 120,
|
|
28
|
+
maxFixedOverheadSeconds: 240,
|
|
29
|
+
creditsPerSourceMinute: 2,
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
value: 'thorough',
|
|
33
|
+
analyzedFramesPerSourceMinute: 30,
|
|
34
|
+
ocrFramesPerSourceMinute: 20,
|
|
35
|
+
processingSecondsPerSourceMinute: 32,
|
|
36
|
+
maxProcessingSecondsPerSourceMinute: 60,
|
|
37
|
+
fixedOverheadSeconds: 180,
|
|
38
|
+
maxFixedOverheadSeconds: 360,
|
|
39
|
+
creditsPerSourceMinute: 4,
|
|
40
|
+
},
|
|
41
|
+
],
|
|
42
|
+
},
|
|
43
|
+
form: {
|
|
44
|
+
submitLabel: 'Start indexing',
|
|
45
|
+
fields: [
|
|
46
|
+
{
|
|
47
|
+
key: 'goal',
|
|
48
|
+
type: 'textarea',
|
|
49
|
+
label: 'What should your AI look for? (optional)',
|
|
50
|
+
placeholder: 'For example: find the final score, the moment the package is dropped, or every mention of pricing.',
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
key: 'indexingMode',
|
|
54
|
+
type: 'select',
|
|
55
|
+
label: 'Coverage',
|
|
56
|
+
defaultValue: 'balanced',
|
|
57
|
+
options: [
|
|
58
|
+
{
|
|
59
|
+
label: 'Fast',
|
|
60
|
+
value: 'fast',
|
|
61
|
+
description: 'Sample key frames for a quick overview.',
|
|
62
|
+
setValues: { processingAuthorityConfirmed: false },
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
label: 'Balanced',
|
|
66
|
+
value: 'balanced',
|
|
67
|
+
description: 'Sample visual and OCR evidence across the video.',
|
|
68
|
+
setValues: { processingAuthorityConfirmed: false },
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
label: 'Thorough',
|
|
72
|
+
value: 'thorough',
|
|
73
|
+
description: 'Let the agent inspect denser evidence around subtle or infrequent details.',
|
|
74
|
+
setValues: { processingAuthorityConfirmed: false },
|
|
75
|
+
},
|
|
76
|
+
],
|
|
77
|
+
},
|
|
78
|
+
],
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
export const VIDEO_JOB_RESULT_SURFACE = {
|
|
82
|
+
id: 'video-evidence-result',
|
|
83
|
+
version: 1,
|
|
84
|
+
slot: 'chat-result',
|
|
85
|
+
title: 'Video evidence',
|
|
86
|
+
resultType: 'application/vnd.larkup.video-evidence+json;version=1',
|
|
87
|
+
};
|
package/package.json
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@larkup/tool-video-intelligence",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Installable local and cloud video intelligence with evidence-first indexing.",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"default": "./dist/index.js"
|
|
13
|
+
},
|
|
14
|
+
"./contracts": {
|
|
15
|
+
"types": "./dist/contracts.d.ts",
|
|
16
|
+
"import": "./dist/contracts.js",
|
|
17
|
+
"default": "./dist/contracts.js"
|
|
18
|
+
},
|
|
19
|
+
"./ui": {
|
|
20
|
+
"types": "./dist/ui.d.ts",
|
|
21
|
+
"import": "./dist/ui.js",
|
|
22
|
+
"default": "./dist/ui.js"
|
|
23
|
+
},
|
|
24
|
+
"./runtime": {
|
|
25
|
+
"types": "./dist/runtime.d.ts",
|
|
26
|
+
"import": "./dist/runtime.js",
|
|
27
|
+
"default": "./dist/runtime.js"
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"bin": {
|
|
31
|
+
"larkup-video-intelligence": "./scripts/runtime.mjs"
|
|
32
|
+
},
|
|
33
|
+
"files": [
|
|
34
|
+
"dist",
|
|
35
|
+
"runtime/app",
|
|
36
|
+
"runtime/Dockerfile",
|
|
37
|
+
"runtime/pyproject.toml",
|
|
38
|
+
"runtime/uv.lock",
|
|
39
|
+
"runtime/requirements.txt",
|
|
40
|
+
"runtime/requirements-cpu.txt",
|
|
41
|
+
"runtime/requirements-smoke.txt",
|
|
42
|
+
"scripts",
|
|
43
|
+
".env.example",
|
|
44
|
+
"compose.yaml",
|
|
45
|
+
"compose.gpu.yaml",
|
|
46
|
+
"tool.manifest.json",
|
|
47
|
+
"README.md"
|
|
48
|
+
],
|
|
49
|
+
"peerDependencies": {
|
|
50
|
+
"@larkup/marketplace": ">=0.1.26"
|
|
51
|
+
},
|
|
52
|
+
"peerDependenciesMeta": {
|
|
53
|
+
"@larkup/marketplace": {
|
|
54
|
+
"optional": true
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
"dependencies": {
|
|
58
|
+
"@larkup/core": "0.5.0"
|
|
59
|
+
},
|
|
60
|
+
"devDependencies": {
|
|
61
|
+
"@types/node": "^26",
|
|
62
|
+
"typescript": "^7.0.2",
|
|
63
|
+
"vitest": "^4.1.10",
|
|
64
|
+
"@larkup/marketplace": "0.2.0"
|
|
65
|
+
},
|
|
66
|
+
"publishConfig": {
|
|
67
|
+
"access": "public"
|
|
68
|
+
},
|
|
69
|
+
"license": "Apache-2.0",
|
|
70
|
+
"repository": {
|
|
71
|
+
"type": "git",
|
|
72
|
+
"url": "https://github.com/Larkup-AI/larkup",
|
|
73
|
+
"directory": "packages/marketplace-tools/video-intelligence"
|
|
74
|
+
},
|
|
75
|
+
"scripts": {
|
|
76
|
+
"dev": "tsc --outDir dist --watch",
|
|
77
|
+
"build": "tsc --outDir dist",
|
|
78
|
+
"type-check": "tsc --noEmit",
|
|
79
|
+
"test": "vitest run",
|
|
80
|
+
"runtime:test": "cd runtime && uv run --extra cpu --extra test python -m unittest discover -s tests -p 'test_*.py'",
|
|
81
|
+
"runtime:start": "node scripts/runtime.mjs start",
|
|
82
|
+
"runtime:stop": "node scripts/runtime.mjs stop"
|
|
83
|
+
}
|
|
84
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
FROM python:3.12-slim AS smoke
|
|
2
|
+
|
|
3
|
+
WORKDIR /service
|
|
4
|
+
RUN apt-get update \
|
|
5
|
+
&& apt-get install -y --no-install-recommends ffmpeg \
|
|
6
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
7
|
+
COPY runtime/requirements-smoke.txt ./requirements-smoke.txt
|
|
8
|
+
RUN python3 -m pip install --no-cache-dir -r requirements-smoke.txt
|
|
9
|
+
COPY runtime/app ./app
|
|
10
|
+
ENV PYTHONUNBUFFERED=1 \
|
|
11
|
+
LARKUP_VIDEO_DATA_DIR=/data \
|
|
12
|
+
LARKUP_VIDEO_MODEL_DIR=/models \
|
|
13
|
+
LARKUP_VIDEO_DEVICE=cpu \
|
|
14
|
+
LARKUP_VIDEO_DISABLE_HEAVY_OPERATORS=true
|
|
15
|
+
EXPOSE 8787
|
|
16
|
+
CMD ["python3", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8787"]
|
|
17
|
+
|
|
18
|
+
FROM python:3.12-slim AS cpu
|
|
19
|
+
|
|
20
|
+
ARG YOLOX_MODEL_URL=https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_s.onnx
|
|
21
|
+
|
|
22
|
+
WORKDIR /service
|
|
23
|
+
RUN apt-get update \
|
|
24
|
+
&& apt-get install -y --no-install-recommends ffmpeg libgomp1 \
|
|
25
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
26
|
+
COPY runtime/requirements-cpu.txt ./requirements-cpu.txt
|
|
27
|
+
RUN python3 -m pip install --no-cache-dir --upgrade pip \
|
|
28
|
+
&& python3 -m pip install --no-cache-dir -r requirements-cpu.txt \
|
|
29
|
+
&& python3 -m pip install --no-cache-dir --no-deps rapidocr-onnxruntime==1.4.4
|
|
30
|
+
RUN mkdir -p /models /data \
|
|
31
|
+
&& python3 -c "import urllib.request; urllib.request.urlretrieve('$YOLOX_MODEL_URL', '/models/yolox_s.onnx')"
|
|
32
|
+
COPY runtime/app ./app
|
|
33
|
+
ENV PYTHONUNBUFFERED=1 \
|
|
34
|
+
LARKUP_VIDEO_DATA_DIR=/data \
|
|
35
|
+
LARKUP_VIDEO_MODEL_DIR=/models \
|
|
36
|
+
LARKUP_VIDEO_DEVICE=cpu \
|
|
37
|
+
LARKUP_VIDEO_OCR_ENGINE=RapidOCR
|
|
38
|
+
EXPOSE 8787
|
|
39
|
+
HEALTHCHECK --interval=15s --timeout=5s --retries=10 CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8787/v1/health', timeout=4)" || exit 1
|
|
40
|
+
CMD ["python3", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8787"]
|
|
41
|
+
|
|
42
|
+
# Blackwell workers require a CUDA 12.8-compatible runtime for onnxruntime-gpu
|
|
43
|
+
# (YOLOX detection) and PaddleOCR. Vision captioning and video embeddings
|
|
44
|
+
# are both HTTPS calls from app/services/ -- this image only ever runs
|
|
45
|
+
# local decode/detect/OCR/transcribe.
|
|
46
|
+
FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu22.04 AS system
|
|
47
|
+
|
|
48
|
+
ARG DEBIAN_FRONTEND=noninteractive
|
|
49
|
+
ARG YOLOX_MODEL_URL=https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_s.onnx
|
|
50
|
+
|
|
51
|
+
RUN apt-get update \
|
|
52
|
+
&& apt-get dist-upgrade -y \
|
|
53
|
+
&& apt-get install -y --no-install-recommends ca-certificates curl ffmpeg libgomp1 python3 python3-pip \
|
|
54
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
55
|
+
|
|
56
|
+
WORKDIR /service
|
|
57
|
+
|
|
58
|
+
FROM system AS runtime
|
|
59
|
+
COPY runtime/requirements.txt ./requirements.txt
|
|
60
|
+
RUN python3 -m pip install --no-cache-dir --upgrade pip \
|
|
61
|
+
&& python3 -m pip install --no-cache-dir paddlepaddle==3.0.0 -i https://www.paddlepaddle.org.cn/packages/stable/cpu/ \
|
|
62
|
+
&& python3 -m pip install --no-cache-dir -r requirements.txt
|
|
63
|
+
|
|
64
|
+
RUN mkdir -p /models /data \
|
|
65
|
+
&& curl --fail --location --retry 4 "$YOLOX_MODEL_URL" --output /models/yolox_s.onnx
|
|
66
|
+
|
|
67
|
+
COPY runtime/app ./app
|
|
68
|
+
|
|
69
|
+
ENV PYTHONUNBUFFERED=1 \
|
|
70
|
+
LARKUP_VIDEO_DATA_DIR=/data \
|
|
71
|
+
LARKUP_VIDEO_MODEL_DIR=/models \
|
|
72
|
+
LARKUP_VIDEO_DEVICE=auto \
|
|
73
|
+
LARKUP_VIDEO_SEMANTIC_VISION=true \
|
|
74
|
+
LARKUP_VIDEO_SEMANTIC_VISION_MODEL=google/gemini-3.6-flash \
|
|
75
|
+
LARKUP_VIDEO_EMBEDDING_PROVIDER=disabled \
|
|
76
|
+
LARKUP_VIDEO_EMBEDDING_FALLBACK_PROVIDER=disabled \
|
|
77
|
+
LARKUP_VIDEO_TRANSCRIPTION_PROVIDER="" \
|
|
78
|
+
LARKUP_VIDEO_TRANSCRIPTION_FALLBACK="" \
|
|
79
|
+
LARKUP_VIDEO_TRANSCRIPTION_CHUNK_SECONDS=300 \
|
|
80
|
+
LARKUP_VIDEO_TRANSCRIPTION_CONCURRENCY=6 \
|
|
81
|
+
LARKUP_VIDEO_TRANSCRIPTION_REQUEST_TIMEOUT_SECONDS=120 \
|
|
82
|
+
LARKUP_VIDEO_REQUIRE_AUTH=false
|
|
83
|
+
|
|
84
|
+
EXPOSE 8787
|
|
85
|
+
HEALTHCHECK --interval=15s --timeout=5s --retries=10 CMD curl -fsS http://127.0.0.1:8787/v1/health || exit 1
|
|
86
|
+
CMD ["python3", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8787"]
|
|
87
|
+
|
|
88
|
+
FROM runtime AS runpod
|
|
89
|
+
|
|
90
|
+
# The RunPod entrypoint is deliberately kept out of runtime/app (and out of
|
|
91
|
+
# the published npm package) -- see deploy/gpu_providers/README.md. It still
|
|
92
|
+
# has to land inside the `app` package here because its imports and the
|
|
93
|
+
# `python -m app.runpod_worker` entrypoint below both assume that. Only this
|
|
94
|
+
# stage needs it: a local (non-RunPod) `runtime`-target build never imports it.
|
|
95
|
+
COPY deploy/gpu_providers/runpod_worker_entrypoint.py ./app/runpod_worker.py
|
|
96
|
+
COPY deploy/gpu_providers/remote_source.py ./app/remote_source.py
|
|
97
|
+
COPY deploy/gpu_providers/progress.py ./app/progress.py
|
|
98
|
+
|
|
99
|
+
# Fail the image build if the exact RunPod entrypoint cannot import. This
|
|
100
|
+
# catches native-library and Python dependency failures before a worker is
|
|
101
|
+
# scheduled, where logs are much harder to recover.
|
|
102
|
+
RUN python3 -c "import cv2, runpod; from app.runpod_worker import handler; assert callable(handler)"
|
|
103
|
+
|
|
104
|
+
# This stage is the immutable serverless image. Do not rely on a template
|
|
105
|
+
# command override: it must start the worker even when deployed as-is.
|
|
106
|
+
HEALTHCHECK NONE
|
|
107
|
+
CMD ["python3", "-m", "app.runpod_worker"]
|
|
108
|
+
|
|
109
|
+
FROM runtime AS cloud-worker
|
|
110
|
+
|
|
111
|
+
# Shared image for the "rent-a-VM" providers (vast, shadeform, salad,
|
|
112
|
+
# scaleway, thundercompute, gpuai, northflank, hyperstack): each just starts
|
|
113
|
+
# this image with job-specific env vars, see deploy/gpu_providers/base.py.
|
|
114
|
+
COPY deploy/gpu_providers/cloud_worker_entrypoint.py ./app/cloud_worker.py
|
|
115
|
+
|
|
116
|
+
RUN python3 -c "import boto3; from app.cloud_worker import main; assert callable(main)"
|
|
117
|
+
|
|
118
|
+
HEALTHCHECK NONE
|
|
119
|
+
CMD ["python3", "-m", "app.cloud_worker"]
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Portable command-line entry point for the Video Intelligence API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
|
|
7
|
+
import uvicorn
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def main() -> None:
|
|
11
|
+
uvicorn.run(
|
|
12
|
+
"app.main:app",
|
|
13
|
+
host=os.getenv("LARKUP_VIDEO_HOST", "0.0.0.0"),
|
|
14
|
+
port=int(os.getenv("LARKUP_VIDEO_PORT", "8787")),
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
if __name__ == "__main__":
|
|
19
|
+
main()
|
|
File without changes
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Shared app state and FastAPI dependencies: settings, the store, the job
|
|
2
|
+
runner, request-rate limiting, and principal/admin authentication. Built
|
|
3
|
+
once here so main.py and v1.py both import the same instances.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hmac
|
|
9
|
+
import os
|
|
10
|
+
import time
|
|
11
|
+
from collections import defaultdict, deque
|
|
12
|
+
|
|
13
|
+
from fastapi import HTTPException, Request, status
|
|
14
|
+
|
|
15
|
+
from app.config import Settings
|
|
16
|
+
from app.db.store import AuthenticationError, Principal, Store
|
|
17
|
+
from app.services.jobs import JobService
|
|
18
|
+
|
|
19
|
+
settings = Settings.from_env()
|
|
20
|
+
store = Store(settings.data_dir / "video-intelligence.sqlite3")
|
|
21
|
+
jobs = JobService(settings, store)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SlidingWindowLimiter:
|
|
25
|
+
def __init__(self, requests_per_minute: int):
|
|
26
|
+
self.limit = requests_per_minute
|
|
27
|
+
self.windows: defaultdict[str, deque[float]] = defaultdict(deque)
|
|
28
|
+
|
|
29
|
+
def check(self, key: str) -> None:
|
|
30
|
+
now = time.monotonic()
|
|
31
|
+
window = self.windows[key]
|
|
32
|
+
while window and window[0] <= now - 60:
|
|
33
|
+
window.popleft()
|
|
34
|
+
if len(window) >= self.limit:
|
|
35
|
+
raise HTTPException(
|
|
36
|
+
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
|
37
|
+
detail="request rate limit reached; retry in one minute",
|
|
38
|
+
headers={"Retry-After": "60"},
|
|
39
|
+
)
|
|
40
|
+
window.append(now)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
limiter = SlidingWindowLimiter(max(1, int(os.getenv("LARKUP_VIDEO_REQUESTS_PER_MINUTE", "120"))))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _bearer(request: Request) -> str | None:
|
|
47
|
+
authorization = request.headers.get("authorization", "")
|
|
48
|
+
scheme, _, token = authorization.partition(" ")
|
|
49
|
+
return token.strip() if scheme.lower() == "bearer" and token.strip() else None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def principal(request: Request) -> Principal:
|
|
53
|
+
token = _bearer(request)
|
|
54
|
+
forwarded = request.headers.get("x-forwarded-for", "").partition(",")[0].strip()
|
|
55
|
+
limiter.check(token or forwarded or request.client.host if request.client else "local")
|
|
56
|
+
try:
|
|
57
|
+
return store.resolve_principal(token, settings.require_auth, settings.shared_api_key)
|
|
58
|
+
except AuthenticationError as error:
|
|
59
|
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=str(error)) from error
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def require_admin(token: str | None) -> None:
|
|
63
|
+
if not settings.admin_token:
|
|
64
|
+
raise HTTPException(
|
|
65
|
+
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
66
|
+
detail="admin access-code management is not configured",
|
|
67
|
+
)
|
|
68
|
+
if not token or not hmac.compare_digest(token, settings.admin_token):
|
|
69
|
+
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="invalid admin token")
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""Every /v1 route: health, uploads, jobs, usage, and access codes.
|
|
2
|
+
|
|
3
|
+
The local-Docker runtime's whole HTTP surface is small enough (eight
|
|
4
|
+
endpoints) that splitting it across more files would cost more to navigate
|
|
5
|
+
than it would save -- see api/deps.py for the auth/rate-limit dependencies
|
|
6
|
+
these handlers use.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import secrets
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Annotated
|
|
15
|
+
|
|
16
|
+
from fastapi import APIRouter, BackgroundTasks, Depends, File, Header, HTTPException, UploadFile
|
|
17
|
+
from fastapi.responses import JSONResponse, Response
|
|
18
|
+
|
|
19
|
+
from app.db.schemas import (
|
|
20
|
+
CreateAccessCodeRequest,
|
|
21
|
+
CreateAccessCodeResponse,
|
|
22
|
+
CreateJobRequest,
|
|
23
|
+
JobResponse,
|
|
24
|
+
RedeemAccessCodeRequest,
|
|
25
|
+
RedeemAccessCodeResponse,
|
|
26
|
+
UsageSummary,
|
|
27
|
+
)
|
|
28
|
+
from app.db.store import AuthenticationError, Principal, QuotaExceededError, StoreError
|
|
29
|
+
from app.services.pipeline import probe_video
|
|
30
|
+
from app.api.deps import jobs, limiter, principal, require_admin, settings, store
|
|
31
|
+
|
|
32
|
+
router = APIRouter()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@router.get("/v1/health")
|
|
36
|
+
def health() -> dict[str, object]:
|
|
37
|
+
return {
|
|
38
|
+
"status": "ok",
|
|
39
|
+
"version": "0.1.0",
|
|
40
|
+
"runtime": os.getenv("LARKUP_VIDEO_RUNTIME_KIND", "local-docker"),
|
|
41
|
+
"authRequired": settings.require_auth,
|
|
42
|
+
"device": settings.device,
|
|
43
|
+
"operators": {
|
|
44
|
+
"transcription": os.getenv("LARKUP_VIDEO_TRANSCRIPTION_PROVIDER", "whisper"),
|
|
45
|
+
"ocr": os.getenv("LARKUP_VIDEO_OCR_ENGINE", "PaddleOCR"),
|
|
46
|
+
"detection": "YOLOX",
|
|
47
|
+
"tracking": "anonymous-iou",
|
|
48
|
+
"semanticVision": settings.semantic_vision_model if settings.semantic_vision_enabled else None,
|
|
49
|
+
"agentBrain": settings.agent_model if settings.agent_enabled else None,
|
|
50
|
+
"agentProvider": settings.agent_provider if settings.agent_enabled else None,
|
|
51
|
+
},
|
|
52
|
+
"capabilities": ["agent-planning", "transcription", "ocr", "object-detection", "semantic-vision"],
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@router.post("/v1/uploads", status_code=201)
|
|
57
|
+
def upload_video(
|
|
58
|
+
user: Annotated[Principal, Depends(principal)],
|
|
59
|
+
file: Annotated[UploadFile, File()],
|
|
60
|
+
) -> dict[str, object]:
|
|
61
|
+
upload_id = "upl_" + secrets.token_hex(12)
|
|
62
|
+
original_name = Path(file.filename or "video.bin").name
|
|
63
|
+
destination = settings.data_dir / "uploads" / user.id / f"{upload_id}{Path(original_name).suffix}"
|
|
64
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
65
|
+
size = 0
|
|
66
|
+
try:
|
|
67
|
+
with destination.open("wb") as output:
|
|
68
|
+
while chunk := file.file.read(1024 * 1024):
|
|
69
|
+
size += len(chunk)
|
|
70
|
+
if size > settings.max_upload_bytes:
|
|
71
|
+
raise HTTPException(status_code=413, detail="video exceeds the configured upload limit")
|
|
72
|
+
output.write(chunk)
|
|
73
|
+
probe = probe_video(destination)
|
|
74
|
+
store.create_upload(user.id, upload_id, original_name, destination, size)
|
|
75
|
+
return {
|
|
76
|
+
"uploadId": upload_id,
|
|
77
|
+
"fileName": original_name,
|
|
78
|
+
"sizeBytes": size,
|
|
79
|
+
"durationMs": round(probe.duration_seconds * 1_000),
|
|
80
|
+
}
|
|
81
|
+
except Exception:
|
|
82
|
+
destination.unlink(missing_ok=True)
|
|
83
|
+
raise
|
|
84
|
+
finally:
|
|
85
|
+
file.file.close()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@router.post("/v1/jobs", response_model=JobResponse, status_code=202)
|
|
89
|
+
def create_job(
|
|
90
|
+
request: CreateJobRequest,
|
|
91
|
+
user: Annotated[Principal, Depends(principal)],
|
|
92
|
+
background_tasks: BackgroundTasks,
|
|
93
|
+
) -> dict[str, object]:
|
|
94
|
+
upload = store.get_upload(user.id, request.source.upload_id)
|
|
95
|
+
probe = probe_video(Path(upload["path"]))
|
|
96
|
+
job_id = "job_" + secrets.token_hex(12)
|
|
97
|
+
payload = request.model_dump(by_alias=True)
|
|
98
|
+
model_configuration = payload.pop("modelConfiguration", None)
|
|
99
|
+
store.create_job(user, job_id, request.source.upload_id, payload, probe.duration_seconds / 60)
|
|
100
|
+
# Dispatch after the response commits. This avoids losing a local job
|
|
101
|
+
# during a container restart before the thread-pool worker starts.
|
|
102
|
+
background_tasks.add_task(jobs.run, job_id, model_configuration)
|
|
103
|
+
return store.get_job(user.id, job_id)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@router.get("/v1/jobs/{job_id}", response_model=JobResponse)
|
|
107
|
+
def get_job(job_id: str, user: Annotated[Principal, Depends(principal)]) -> dict[str, object]:
|
|
108
|
+
return store.get_job(user.id, job_id)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@router.delete("/v1/jobs/{job_id}", response_model=JobResponse)
|
|
112
|
+
def cancel_job(job_id: str, user: Annotated[Principal, Depends(principal)]) -> dict[str, object]:
|
|
113
|
+
store.cancel_job(user.id, job_id)
|
|
114
|
+
return store.get_job(user.id, job_id)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@router.delete("/v1/jobs/{job_id}/data", response_class=Response)
|
|
118
|
+
def purge_job_data(job_id: str, user: Annotated[Principal, Depends(principal)]) -> Response:
|
|
119
|
+
"""Delete the local source/result cache after the host removes its media asset."""
|
|
120
|
+
store.purge_job_data(user.id, job_id)
|
|
121
|
+
return Response(status_code=204)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@router.get("/v1/usage", response_model=UsageSummary)
|
|
125
|
+
def usage(user: Annotated[Principal, Depends(principal)]) -> dict[str, object]:
|
|
126
|
+
return store.usage(user)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@router.post("/v1/access-codes/redeem", response_model=RedeemAccessCodeResponse)
|
|
130
|
+
def redeem_access_code(request: RedeemAccessCodeRequest) -> dict[str, object]:
|
|
131
|
+
limiter.check("redeem")
|
|
132
|
+
try:
|
|
133
|
+
api_key, entitlement = store.redeem_access_code(request.code, request.label)
|
|
134
|
+
except AuthenticationError as error:
|
|
135
|
+
raise HTTPException(status_code=401, detail=str(error)) from error
|
|
136
|
+
return {"apiKey": api_key, "entitlement": entitlement}
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@router.post("/v1/admin/access-codes", response_model=CreateAccessCodeResponse)
|
|
140
|
+
def create_access_code(
|
|
141
|
+
request: CreateAccessCodeRequest,
|
|
142
|
+
admin_token: Annotated[str | None, Header(alias="X-Larkup-Admin-Token")] = None,
|
|
143
|
+
) -> dict[str, object]:
|
|
144
|
+
require_admin(admin_token)
|
|
145
|
+
entitlement = {
|
|
146
|
+
"sourceMinutesPerMonth": request.source_minutes_per_month,
|
|
147
|
+
"maxConcurrentJobs": request.max_concurrent_jobs,
|
|
148
|
+
"plan": "access-code",
|
|
149
|
+
}
|
|
150
|
+
code = store.create_access_code(
|
|
151
|
+
label=request.label,
|
|
152
|
+
entitlement=entitlement,
|
|
153
|
+
max_uses=request.max_uses,
|
|
154
|
+
expires_at=request.expires_at,
|
|
155
|
+
)
|
|
156
|
+
return {"code": code, "label": request.label, "maxUses": request.max_uses, "expiresAt": request.expires_at}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def register_exception_handlers(app) -> None:
|
|
160
|
+
@app.exception_handler(QuotaExceededError)
|
|
161
|
+
async def _quota_error(_: object, error: QuotaExceededError) -> JSONResponse:
|
|
162
|
+
return JSONResponse(status_code=429, content={"detail": str(error)}, headers={"Retry-After": "60"})
|
|
163
|
+
|
|
164
|
+
@app.exception_handler(StoreError)
|
|
165
|
+
async def _store_error(_: object, error: StoreError) -> JSONResponse:
|
|
166
|
+
return JSONResponse(status_code=404, content={"detail": str(error)})
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _bool(name: str, default: bool) -> bool:
|
|
9
|
+
raw = os.getenv(name)
|
|
10
|
+
if raw is None:
|
|
11
|
+
return default
|
|
12
|
+
return raw.strip().lower() in {"1", "true", "yes", "on"}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Settings:
|
|
17
|
+
data_dir: Path
|
|
18
|
+
model_dir: Path
|
|
19
|
+
require_auth: bool
|
|
20
|
+
shared_api_key: str | None
|
|
21
|
+
admin_token: str | None
|
|
22
|
+
workers: int
|
|
23
|
+
device: str
|
|
24
|
+
allow_local_paths: bool
|
|
25
|
+
max_upload_bytes: int
|
|
26
|
+
disable_heavy_operators: bool
|
|
27
|
+
semantic_vision_enabled: bool
|
|
28
|
+
semantic_vision_model: str
|
|
29
|
+
reasoning_vision_model: str
|
|
30
|
+
agent_enabled: bool
|
|
31
|
+
agent_provider: str
|
|
32
|
+
agent_model: str
|
|
33
|
+
allowed_origins: tuple[str, ...]
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_env(cls) -> "Settings":
|
|
37
|
+
data_dir = Path(os.getenv("LARKUP_VIDEO_DATA_DIR", "/data")).resolve()
|
|
38
|
+
model_dir = Path(os.getenv("LARKUP_VIDEO_MODEL_DIR", "/models")).resolve()
|
|
39
|
+
data_dir.mkdir(parents=True, exist_ok=True)
|
|
40
|
+
model_dir.mkdir(parents=True, exist_ok=True)
|
|
41
|
+
return cls(
|
|
42
|
+
data_dir=data_dir,
|
|
43
|
+
model_dir=model_dir,
|
|
44
|
+
require_auth=_bool("LARKUP_VIDEO_REQUIRE_AUTH", False),
|
|
45
|
+
shared_api_key=os.getenv("LARKUP_VIDEO_SHARED_API_KEY") or None,
|
|
46
|
+
admin_token=os.getenv("LARKUP_VIDEO_ADMIN_TOKEN") or None,
|
|
47
|
+
workers=max(1, min(8, int(os.getenv("LARKUP_VIDEO_WORKERS", "1")))),
|
|
48
|
+
device=os.getenv("LARKUP_VIDEO_DEVICE", "auto"),
|
|
49
|
+
allow_local_paths=_bool("LARKUP_VIDEO_ALLOW_LOCAL_PATHS", False),
|
|
50
|
+
max_upload_bytes=max(
|
|
51
|
+
1,
|
|
52
|
+
int(os.getenv("LARKUP_VIDEO_MAX_UPLOAD_BYTES", str(20 * 1024**3))),
|
|
53
|
+
),
|
|
54
|
+
disable_heavy_operators=_bool("LARKUP_VIDEO_DISABLE_HEAVY_OPERATORS", False),
|
|
55
|
+
semantic_vision_enabled=_bool("LARKUP_VIDEO_SEMANTIC_VISION", True),
|
|
56
|
+
# Vercel AI Gateway model ids, not local HuggingFace paths -- both
|
|
57
|
+
# run entirely through gateway_vision.GatewayVisionClient, which
|
|
58
|
+
# reads these same env vars itself. Kept here too so /v1/health
|
|
59
|
+
# can report them. The bulk model captions every clip cheaply;
|
|
60
|
+
# the reasoning model is reserved for watch_original's final
|
|
61
|
+
# dense-verification pass, where accuracy matters more than cost.
|
|
62
|
+
semantic_vision_model=os.getenv(
|
|
63
|
+
"LARKUP_VIDEO_SEMANTIC_VISION_MODEL", "google/gemini-3.6-flash"
|
|
64
|
+
),
|
|
65
|
+
reasoning_vision_model=os.getenv(
|
|
66
|
+
"LARKUP_VIDEO_REASONING_VISION_MODEL", "google/gemini-3.6-flash"
|
|
67
|
+
),
|
|
68
|
+
agent_enabled=_bool("LARKUP_VIDEO_AGENT_ENABLED", True),
|
|
69
|
+
agent_provider=os.getenv("LARKUP_VIDEO_AGENT_PROVIDER", "vercel_ai_gateway"),
|
|
70
|
+
agent_model=os.getenv("LARKUP_VIDEO_AGENT_MODEL", "openai/gpt-5-mini"),
|
|
71
|
+
allowed_origins=tuple(
|
|
72
|
+
origin.strip()
|
|
73
|
+
for origin in os.getenv(
|
|
74
|
+
"LARKUP_VIDEO_ALLOWED_ORIGINS", "http://localhost:3000"
|
|
75
|
+
).split(",")
|
|
76
|
+
if origin.strip()
|
|
77
|
+
),
|
|
78
|
+
)
|
|
File without changes
|