@larkup/tool-video-intelligence 0.2.0

package/tool.manifest.json

@@ -0,0 +1,617 @@
+{
+  "$schema": "https://hub.larkup.de/schemas/tool-manifest.v3.json",
+  "manifestVersion": "3.0",
+  "kind": "tool",
+  "id": "video-intelligence",
+  "name": "Video Intelligence",
+  "description": "Help your AI understand your videos, so you can ask questions and get answers from the right moments.",
+  "longDescription": "Turn videos into knowledge your AI can understand. It listens to what is said, reads what appears on screen, and helps you find the moments that matter.",
+  "category": "media",
+  "version": "0.2.0",
+  "pricing": "free",
+  "emoji": "🎥",
+  "icon": "Film",
+  "packageName": "@larkup/tool-video-intelligence",
+  "installSize": "~120 KB",
+  "requiresSandbox": false,
+  "author": "Larkup",
+  "capabilities": [
+    "video-indexing",
+    "video-indexing-brief",
+    "agentic-video-indexing",
+    "cloud-video-runtime",
+    "timestamped-transcription",
+    "ocr",
+    "object-detection",
+    "anonymous-object-tracking",
+    "usage-metering"
+  ],
+  "entrypoints": {
+    "server": ".",
+    "agent": ".",
+    "ui": "./ui"
+  },
+  "runtime": {
+    "protocolVersion": "1",
+    "defaultMode": "managed-cloud",
+    "healthPath": "/v1/health",
+    "modes": [
+      {
+        "id": "managed-cloud",
+        "label": "Larkup Cloud",
+        "description": "Index with managed private GPU capacity and your Larkup Cloud allowance.",
+        "requiresGpu": true
+      },
+      {
+        "id": "local",
+        "label": "Local runtime",
+        "icon": "/docker.png",
+        "description": "Run the application locally on your computer.",
+        "endpointConfigKey": "localRuntimeUrl",
+        "credentialConfigKey": "localRuntimeApiKey",
+        "image": "ghcr.io/larkup-ai/video-intelligence:0.1.0",
+        "composeFile": "compose.yaml",
+        "setupNotice": "The local application and its models download to this computer. The initial download can be large."
+      },
+      {
+        "id": "custom-remote",
+        "label": "Custom runtime",
+        "description": "Connect a compatible Video Intelligence deployment you operate.",
+        "endpointConfigKey": "customRuntimeUrl",
+        "credentialConfigKey": "customRuntimeApiKey",
+        "requiresGpu": true
+      }
+    ]
+  },
+  "billing": {
+    "model": "hybrid",
+    "entitlementVersion": "1",
+    "meters": [
+      {
+        "id": "source-minute",
+        "unit": "minute",
+        "description": "Minutes of source media indexed."
+      },
+      {
+        "id": "thorough-inspection-minute",
+        "unit": "minute",
+        "description": "Minutes inspected with Thorough agent coverage."
+      },
+      {
+        "id": "gpu-second",
+        "unit": "second",
+        "description": "GPU worker time used."
+      },
+      {
+        "id": "ocr-frame",
+        "unit": "frame",
+        "description": "Frames sent through OCR."
+      }
+    ]
+  },
+  "usage": {
+    "label": "Usage this month",
+    "visibleWhenRuntimeMode": ["managed-cloud"],
+    "visualization": {
+      "usedKey": "sourceMinutesUsed",
+      "limitKey": "sourceMinutesLimit"
+    },
+    "support": {
+      "contactLabel": "Request more usage",
+      "description": "Each installation receives Cloud access automatically. Request more video-processing usage when your allowance is low.",
+      "userIdConfigKey": "cloudInstallationId"
+    },
+    "fields": [
+      {
+        "key": "sourceMinutesUsed",
+        "label": "Used",
+        "format": "minutes"
+      },
+      {
+        "key": "sourceMinutesLimit",
+        "label": "Included",
+        "format": "minutes"
+      },
+      {
+        "key": "activeJobs",
+        "label": "Active jobs",
+        "format": "number"
+      }
+    ]
+  },
+  "ui": {
+    "isolation": "sandboxed-iframe",
+    "surfaces": [
+      {
+        "id": "video-indexing-brief",
+        "slot": "data-indexing",
+        "title": "Let your AI understand this video",
+        "entrypoint": "./ui",
+        "appliesTo": ["video"],
+        "description": "Choose how deeply to analyze it and optionally point it to what matters most.",
+        "estimate": {
+          "modeField": "indexingMode",
+          "variants": [
+            {
+              "value": "fast",
+              "analyzedFramesPerSourceMinute": 5,
+              "ocrFramesPerSourceMinute": 3,
+              "processingSecondsPerSourceMinute": 4,
+              "maxProcessingSecondsPerSourceMinute": 5,
+              "fixedOverheadSeconds": 60,
+              "maxFixedOverheadSeconds": 60,
+              "creditsPerSourceMinute": 1
+            },
+            {
+              "value": "balanced",
+              "analyzedFramesPerSourceMinute": 12,
+              "ocrFramesPerSourceMinute": 8,
+              "processingSecondsPerSourceMinute": 16,
+              "maxProcessingSecondsPerSourceMinute": 30,
+              "fixedOverheadSeconds": 120,
+              "maxFixedOverheadSeconds": 240,
+              "creditsPerSourceMinute": 2
+            },
+            {
+              "value": "thorough",
+              "analyzedFramesPerSourceMinute": 30,
+              "ocrFramesPerSourceMinute": 20,
+              "processingSecondsPerSourceMinute": 32,
+              "maxProcessingSecondsPerSourceMinute": 60,
+              "fixedOverheadSeconds": 180,
+              "maxFixedOverheadSeconds": 360,
+              "creditsPerSourceMinute": 4
+            }
+          ]
+        },
+        "form": {
+          "submitLabel": "Start indexing",
+          "fields": [
+            {
+              "key": "goal",
+              "type": "textarea",
+              "label": "What should your AI look for? (optional)",
+              "placeholder": "For example: find the final score, the moment the package is dropped, or every mention of pricing."
+            },
+            {
+              "key": "indexingMode",
+              "type": "select",
+              "label": "Coverage",
+              "defaultValue": "balanced",
+              "options": [
+                {
+                  "label": "Fast",
+                  "value": "fast",
+                  "description": "Sample key frames for a quick overview.",
+                  "setValues": {
+                    "processingAuthorityConfirmed": false
+                  }
+                },
+                {
+                  "label": "Balanced",
+                  "value": "balanced",
+                  "description": "Sample visual and OCR evidence across the video.",
+                  "setValues": {
+                    "processingAuthorityConfirmed": false
+                  }
+                },
+                {
+                  "label": "Thorough",
+                  "value": "thorough",
+                  "description": "Let the agent inspect denser evidence around subtle or infrequent details.",
+                  "setValues": {
+                    "processingAuthorityConfirmed": false
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      },
+      {
+        "id": "video-evidence-result",
+        "slot": "chat-result",
+        "title": "Video evidence",
+        "entrypoint": "./ui"
+      }
+    ]
+  },
+  "configSchema": [
+    {
+      "key": "runtimeMode",
+      "label": "Runtime",
+      "type": "select",
+      "group": "Runtime",
+      "defaultValue": "managed-cloud",
+      "help": "Choose managed Larkup Cloud, a local runtime that automatically selects Docker or native, or your own compatible deployment.",
+      "options": [
+        {
+          "label": "Larkup Cloud",
+          "value": "managed-cloud",
+          "icon": "/logo.png",
+          "description": "Managed GPU capacity and automatic device access."
+        },
+        {
+          "label": "Local runtime",
+          "value": "local",
+          "icon": "/docker.png",
+          "description": "Run the application locally on your computer."
+        },
+        {
+          "label": "Custom runtime",
+          "value": "custom-remote",
+          "icon": "/icons/cloudflare.png",
+          "description": "Use your compatible deployment URL and key."
+        }
+      ]
+    },
+    {
+      "key": "localRuntimeUrl",
+      "label": "Local runtime URL",
+      "type": "text",
+      "group": "Local runtime",
+      "defaultValue": "http://127.0.0.1:8787",
+      "help": "Use this machine's LAN address instead of 127.0.0.1 when another Larkup user should connect to it.",
+      "verification": {
+        "endpoint": "/api/tools/video-intelligence/verify",
+        "method": "POST",
+        "fields": {
+          "localRuntimeUrl": "localRuntimeUrl",
+          "localRuntimeApiKey": "localRuntimeApiKey",
+          "runtimeMode": "runtimeMode"
+        }
+      },
+      "visibleWhen": {
+        "field": "runtimeMode",
+        "equals": "local"
+      }
+    },
+    {
+      "key": "localRuntimeApiKey",
+      "label": "Shared local API key",
+      "type": "password",
+      "group": "Local runtime",
+      "help": "Optional bearer key to use when your local runtime authentication is enabled. Share it only with trusted users on your private network.",
+      "visibleWhen": {
+        "field": "runtimeMode",
+        "equals": "local"
+      }
+    },
+    {
+      "key": "videoVisionProvider",
+      "label": "Vision provider",
+      "type": "select",
+      "group": "Video understanding",
+      "defaultValue": "vercel_ai_gateway",
+      "defaultFromGlobalConfigKey": "visionProvider",
+      "help": "Defaults to your saved AI Models vision provider. Choose another provider here only for this tool.",
+      "options": [
+        {
+          "label": "Use AI Models",
+          "value": "auto",
+          "description": "Follow the workspace Vision Model setting."
+        },
+        {
+          "label": "Google Gemini",
+          "value": "google",
+          "icon": "/icons/gemini.svg",
+          "description": "Use Gemini directly with a Google API key."
+        },
+        {
+          "label": "Vercel AI Gateway",
+          "value": "vercel_ai_gateway",
+          "icon": "/icons/vercel.svg",
+          "description": "Use a Gateway vision model and key."
+        },
+        {
+          "label": "OpenAI",
+          "value": "openai",
+          "icon": "/icons/openai.svg",
+          "description": "Use an OpenAI vision model directly."
+        }
+      ]
+    },
+    {
+      "key": "videoVisionApiKey",
+      "label": "Vision provider API key",
+      "type": "password",
+      "group": "Video understanding",
+      "help": "Optional tool override. Leave empty to use the matching provider key from AI Models; it is forwarded only for the active job."
+    },
+    {
+      "key": "semanticVisionModel",
+      "label": "Video vision model",
+      "type": "select",
+      "group": "Video understanding",
+      "defaultValue": "google/gemini-3.6-flash",
+      "defaultFromGlobalConfigKey": "visionModelId",
+      "providerField": "videoVisionProvider",
+      "defaultValueByProvider": {
+        "auto": "auto",
+        "google": "google/gemini-3.6-flash",
+        "vercel_ai_gateway": "google/gemini-3.6-flash",
+        "openai": "openai/gpt-4o-mini"
+      },
+      "help": "Defaults to the Vision Model in AI Models. Choose another video-capable model to override it only for this tool.",
+      "verification": {
+        "endpoint": "/api/tools/video-intelligence/verify",
+        "method": "POST",
+        "fields": {
+          "videoVisionProvider": "videoVisionProvider",
+          "videoVisionApiKey": "videoVisionApiKey",
+          "semanticVisionModel": "semanticVisionModel"
+        }
+      },
+      "options": [
+        {
+          "label": "Use AI Models",
+          "value": "auto",
+          "description": "Follow the workspace Vision Model setting."
+        },
+        {
+          "label": "Gemini 3.6 Flash",
+          "value": "google/gemini-3.6-flash",
+          "icon": "/icons/gemini.svg",
+          "description": "Current Google multimodal model for video understanding."
+        },
+        {
+          "label": "Gemini 3.5 Flash",
+          "value": "google/gemini-3.5-flash",
+          "icon": "/icons/gemini.svg",
+          "description": "Fast video and image understanding."
+        },
+        {
+          "label": "GPT-4o mini",
+          "value": "openai/gpt-4o-mini",
+          "icon": "/icons/openai.svg",
+          "description": "Efficient OpenAI vision model."
+        }
+      ]
+    },
+    {
+      "key": "videoAgentProvider",
+      "label": "Agent provider",
+      "type": "select",
+      "group": "Agent planning",
+      "defaultValue": "vercel_ai_gateway",
+      "defaultFromGlobalConfigKey": "chatProvider",
+      "help": "Defaults to your saved AI Models chat provider. Choose another provider here only for this tool.",
+      "options": [
+        {
+          "label": "Use AI Models",
+          "value": "auto",
+          "description": "Follow the workspace Chat Model setting."
+        },
+        { "label": "Google Gemini", "value": "google", "icon": "/icons/gemini.svg" },
+        { "label": "Vercel AI Gateway", "value": "vercel_ai_gateway", "icon": "/icons/vercel.svg" },
+        { "label": "OpenAI", "value": "openai", "icon": "/icons/openai.svg" },
+        {
+          "label": "DeepSeek",
+          "value": "deepseek",
+          "icon": "https://models.dev/logos/deepseek.svg"
+        },
+        {
+          "label": "Anthropic",
+          "value": "anthropic",
+          "icon": "https://models.dev/logos/anthropic.svg"
+        },
+        { "label": "Mistral", "value": "mistral", "icon": "https://models.dev/logos/mistral.svg" },
+        { "label": "Cohere", "value": "cohere", "icon": "https://models.dev/logos/cohere.svg" }
+      ]
+    },
+    {
+      "key": "videoAgentApiKey",
+      "label": "Agent provider API key",
+      "type": "password",
+      "group": "Agent planning",
+      "help": "Optional tool override. Leave empty to use the matching chat-provider key from AI Models; it is forwarded only for the active job."
+    },
+    {
+      "key": "agentModel",
+      "label": "Agent / tool-brain model",
+      "type": "select",
+      "group": "Agent planning",
+      "defaultValue": "openai/gpt-5-mini",
+      "defaultFromGlobalConfigKey": "chatModelId",
+      "providerField": "videoAgentProvider",
+      "defaultValueByProvider": {
+        "auto": "auto",
+        "google": "google/gemini-3.5-flash-lite",
+        "vercel_ai_gateway": "openai/gpt-5-mini",
+        "openai": "openai/gpt-5-mini",
+        "deepseek": "deepseek/deepseek-v3.2",
+        "anthropic": "anthropic/claude-sonnet-4",
+        "mistral": "mistral/mistral-large-3",
+        "cohere": "cohere/command-a"
+      },
+      "help": "Defaults to the Chat Model in AI Models. Choose another model to override it only for video planning and synthesis.",
+      "verification": {
+        "endpoint": "/api/tools/video-intelligence/verify",
+        "method": "POST",
+        "fields": {
+          "videoAgentProvider": "videoAgentProvider",
+          "videoAgentApiKey": "videoAgentApiKey",
+          "agentModel": "agentModel"
+        }
+      },
+      "options": [
+        {
+          "label": "Use AI Models",
+          "value": "auto",
+          "description": "Follow the workspace Chat Model setting."
+        },
+        {
+          "label": "Gemini 3.5 Flash Lite",
+          "value": "google/gemini-3.5-flash-lite",
+          "icon": "/icons/gemini.svg"
+        },
+        {
+          "label": "Gemini 3.6 Flash",
+          "value": "google/gemini-3.6-flash",
+          "icon": "/icons/gemini.svg"
+        },
+        { "label": "GPT-5 mini", "value": "openai/gpt-5-mini", "icon": "/icons/openai.svg" },
+        { "label": "GPT-5.2", "value": "openai/gpt-5.2", "icon": "/icons/openai.svg" },
+        {
+          "label": "DeepSeek V3.2",
+          "value": "deepseek/deepseek-v3.2",
+          "icon": "https://models.dev/logos/deepseek.svg"
+        },
+        {
+          "label": "DeepSeek V4 Pro",
+          "value": "deepseek/deepseek-v4-pro",
+          "icon": "https://models.dev/logos/deepseek.svg"
+        },
+        {
+          "label": "Claude Sonnet 4",
+          "value": "anthropic/claude-sonnet-4",
+          "icon": "https://models.dev/logos/anthropic.svg"
+        },
+        {
+          "label": "Mistral Large 3",
+          "value": "mistral/mistral-large-3",
+          "icon": "https://models.dev/logos/mistral.svg"
+        },
+        {
+          "label": "Command A",
+          "value": "cohere/command-a",
+          "icon": "https://models.dev/logos/cohere.svg"
+        }
+      ]
+    },
+    {
+      "key": "customRuntimeUrl",
+      "label": "Custom runtime URL",
+      "type": "text",
+      "group": "Custom runtime",
+      "required": true,
+      "help": "HTTPS URL of a compatible Video Intelligence /v1 runtime.",
+      "verification": {
+        "endpoint": "/api/tools/video-intelligence/verify",
+        "method": "POST",
+        "fields": {
+          "customRuntimeUrl": "customRuntimeUrl",
+          "customRuntimeApiKey": "customRuntimeApiKey",
+          "runtimeMode": "runtimeMode"
+        }
+      },
+      "visibleWhen": {
+        "field": "runtimeMode",
+        "equals": "custom-remote"
+      }
+    },
+    {
+      "key": "customRuntimeApiKey",
+      "label": "Custom runtime API key",
+      "type": "password",
+      "group": "Custom runtime",
+      "required": true,
+      "visibleWhen": {
+        "field": "runtimeMode",
+        "equals": "custom-remote"
+      }
+    },
+    {
+      "key": "audioProvider",
+      "label": "Audio provider",
+      "type": "select",
+      "group": "Audio",
+      "defaultValue": "deepgram",
+      "help": "Required for every runtime. Transcription is billed directly to your selected provider; Larkup Cloud does not supply audio-model credits.",
+      "options": [
+        {
+          "label": "OpenAI",
+          "value": "openai",
+          "icon": "/icons/openai.svg",
+          "description": "Use your OpenAI transcription allowance."
+        },
+        {
+          "label": "Groq",
+          "value": "groq",
+          "icon": "https://models.dev/logos/groq.svg",
+          "description": "Fast hosted Whisper inference."
+        },
+        {
+          "label": "Deepgram",
+          "value": "deepgram",
+          "icon": "/icons/deepgram.png",
+          "description": "Speech-to-text with language features."
+        },
+        {
+          "label": "ElevenLabs",
+          "value": "elevenlabs",
+          "icon": "/icons/elevenlabs.png",
+          "description": "Use your ElevenLabs speech-to-text account."
+        }
+      ]
+    },
+    {
+      "key": "audioApiKey",
+      "label": "Audio provider API key",
+      "type": "password",
+      "group": "Audio",
+      "help": "Required. It stays in this installation and is forwarded only for the active video job.",
+      "verification": {
+        "endpoint": "/api/tools/video-intelligence/verify",
+        "method": "POST",
+        "fields": {
+          "audioProvider": "audioProvider",
+          "audioApiKey": "audioApiKey"
+        }
+      },
+      "visibleWhen": {
+        "field": "audioProvider",
+        "equals": ["openai", "groq", "deepgram", "elevenlabs"]
+      }
+    },
+    {
+      "key": "defaultIndexingMode",
+      "label": "Default indexing mode",
+      "type": "select",
+      "group": "Indexing",
+      "defaultValue": "balanced",
+      "options": [
+        {
+          "label": "Fast",
+          "value": "fast"
+        },
+        {
+          "label": "Balanced",
+          "value": "balanced"
+        },
+        {
+          "label": "Thorough",
+          "value": "thorough"
+        }
+      ]
+    },
+    {
+      "key": "cloudAccessKey",
+      "label": "Larkup Cloud API key",
+      "type": "password",
+      "group": "Managed cloud connection",
+      "help": "Paste the key Larkup provides for managed cloud access. Leave this empty only when automatic project provisioning is enabled.",
+      "visibleWhen": {
+        "field": "runtimeMode",
+        "equals": "managed-cloud"
+      }
+    },
+    {
+      "key": "cloudInstallationId",
+      "label": "Managed cloud installation ID",
+      "type": "text",
+      "group": "Managed cloud connection",
+      "help": "Provisioned automatically on first use; never entered or shown to a client.",
+      "serverManaged": true
+    }
+  ],
+  "tags": ["video", "ocr", "tracking", "cloud", "gpu", "private", "gdpr"],
+  "downloads": 0,
+  "repositoryUrl": "https://github.com/Larkup-AI/larkup",
+  "license": "Apache-2.0",
+  "updatedAt": "2026-08-20",
+  "trustLevel": "elevated",
+  "permissions": {
+    "httpAllow": ["video.larkup.de"]
+  }
+}