@overshoot/sdk 0.1.0-alpha.2 → 0.1.0-alpha.4

package/README.md

@@ -1,19 +1,19 @@
 # Overshoot SDK
 
-> **⚠️ Alpha Release**: This is an alpha version (0.1.0-alpha.0). The API may change in future versions.
+> **Warning: Alpha Release**: This is an alpha version (0.1.0-alpha.3). The API may change in future versions.
 
 TypeScript SDK for real-time AI vision analysis on live video streams.
 
 ## Installation
 
 ```bash
-npm install overshoot@alpha
+npm install @overshoot/sdk@alpha
 ```
 
 Or install a specific alpha version:
 
 ```bash
-npm install overshoot@0.1.0-alpha.0
+npm install @overshoot/sdk@0.1.0-alpha.3
 ```
 
 ## Quick Start
@@ -21,10 +21,10 @@ npm install overshoot@0.1.0-alpha.0
 ### Camera Source
 
 ```typescript
-import { RealtimeVision } from "overshoot";
+import { RealtimeVision } from "@overshoot/sdk";
 
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key-here",
   prompt:
     "Read any visible text and return JSON: {text: string | null, confidence: number}",
@@ -41,7 +41,7 @@ await vision.start();
 
 ```typescript
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key-here",
   prompt: "Detect all objects in the video and count them",
   source: {
@@ -56,6 +56,8 @@ const vision = new RealtimeVision({
 await vision.start();
 ```
 
+> **Note:** Video files automatically loop continuously until you call `stop()`.
+
 ## Configuration
 
 ### RealtimeVisionConfig
@@ -70,20 +72,20 @@ interface RealtimeVisionConfig {
 
   // Optional
   source?: StreamSource; // Video source (default: environment-facing camera)
-  backend?: "overshoot" | "gemini"; // Model backend (default: "overshoot")
-  model?: string; // Model name (default: "Qwen/Qwen3-VL-30B-A3B-Instruct")
+  backend?: "overshoot"; // Model backend (default: "overshoot")
+  model?: string; // Model name (see Available Models below)
   outputSchema?: Record<string, any>; // JSON schema for structured output
   onError?: (error: Error) => void;
   debug?: boolean; // Enable debug logging (default: false)
 
   processing?: {
-    fps?: number; // Actual source frames per second (1-120)
+    fps?: number; // Source frames per second (1-120, auto-detected for cameras)
     sampling_ratio?: number; // Fraction of frames to process (0-1, default: 0.1)
-    clip_length_seconds?: number; // Size of each clip that the VLM infers on (0.1-60, default: 1.0)
-    delay_seconds?: number; // Shift between clips (0-60, default: 1.0)
+    clip_length_seconds?: number; // Duration of each clip sent to the model (0.1-60, default: 1.0)
+    delay_seconds?: number; // Interval between inference runs (0-60, default: 1.0)
   };
 
-  iceServers?: RTCIceServer[]; // Custom WebRTC ICE servers
+  iceServers?: RTCIceServer[]; // Custom WebRTC ICE servers (uses Overshoot TURN servers by default, see RealtimeVision.ts)
 }
 ```
 
@@ -95,6 +97,84 @@ type StreamSource =
   | { type: "video"; file: File };
 ```
 
+### Available Models
+
+| Model                            | Description                                                                          |
+| -------------------------------- | ------------------------------------------------------------------------------------ |
+| `Qwen/Qwen3-VL-30B-A3B-Instruct` | Default. Very fast and performant general-purpose vision-language model.             |
+| `Qwen/Qwen3-VL-8B-Instruct`      | Similar latency to 30B. Particularly good at OCR and text extraction tasks.          |
+| `OpenGVLab/InternVL3_5-30B-A3B`  | Excels at capturing visual detail. More verbose output, higher latency.              |
+
+### Processing Parameters Explained
+
+The processing parameters control how video frames are sampled and sent to the model:
+
+- **`fps`**: The frame rate of your video source. Auto-detected for camera streams; defaults to 30 for video files.
+- **`sampling_ratio`**: What fraction of frames to include in each clip (0.1 = 10% of frames).
+- **`clip_length_seconds`**: Duration of video captured for each inference (e.g., 1.0 = 1 second of video).
+- **`delay_seconds`**: How often inference runs (e.g., 1.0 = one inference per second).
+
+**Example:** With `fps=30`, `clip_length_seconds=1.0`, `sampling_ratio=0.1`:
+
+- Each clip captures 1 second of video (30 frames at 30fps)
+- 10% of frames are sampled = 3 frames sent to the model
+- If `delay_seconds=1.0`, you get ~1 inference result per second
+
+#### Configuration by Use Case
+
+Different applications need different processing configurations:
+
+**Real-time tracking** (low latency, frequent updates):
+
+```typescript
+processing: {
+  sampling_ratio: 0.7,
+  clip_length_seconds: 0.5,
+  delay_seconds: 0.3,
+}
+```
+
+**Event detection** (monitoring for specific occurrences):
+
+```typescript
+processing: {
+  sampling_ratio: 0.2,
+  clip_length_seconds: 5.0,
+  delay_seconds: 5.0,
+}
+```
+
+### Structured Output (JSON Schema)
+
+Use `outputSchema` to constrain the model's output to a specific JSON structure. The schema follows [JSON Schema](https://json-schema.org/) specification.
+
+```typescript
+const vision = new RealtimeVision({
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
+  apiKey: "your-api-key",
+  prompt: "Detect objects and return structured data",
+  outputSchema: {
+    type: "object",
+    properties: {
+      objects: {
+        type: "array",
+        items: { type: "string" },
+      },
+      count: { type: "integer" },
+    },
+    required: ["objects", "count"],
+  },
+  onResult: (result) => {
+    const data = JSON.parse(result.result);
+    console.log(`Found ${data.count} objects:`, data.objects);
+  },
+});
+```
+
+The model will return valid JSON matching your schema. If the model cannot produce valid output, `result.ok` will be `false` and `result.error` will contain details.
+
+> **Note:** `result.result` is always a string. When using `outputSchema`, you must parse it with `JSON.parse()`.
+
 ## API Methods
 
 ```typescript
@@ -106,18 +186,203 @@ await vision.stop(); // Stop and cleanup resources
 await vision.updatePrompt(newPrompt); // Update task while running
 
 // State access
-vision.getMediaStream(); // Get MediaStream for video preview
-vision.getStreamId(); // Get current stream ID
+vision.getMediaStream(); // Get MediaStream for video preview (null if not started)
+vision.getStreamId(); // Get current stream ID (null if not started)
 vision.isActive(); // Check if stream is running
 ```
 
+## Stream Lifecycle
+
+### Keepalive
+
+Streams have a server-side lease (typically 300 seconds). The SDK automatically sends keepalive requests to maintain the connection. If the keepalive fails (e.g., network issues), the stream will stop and `onError` will be called.
+
+You don't need to manage keepalives manually - just call `start()` and the SDK handles the rest.
+
+### State and Memory
+
+The SDK does not maintain memory or state between inference calls - each frame clip is processed independently. If your application needs to track state over time (e.g., counting repetitions, detecting transitions), implement this in your `onResult` callback:
+
+```typescript
+let lastPosition = "up";
+let repCount = 0;
+
+const vision = new RealtimeVision({
+  // ...config
+  onResult: (result) => {
+    const data = JSON.parse(result.result);
+
+    // Track state transitions externally
+    if (lastPosition === "down" && data.position === "up") {
+      repCount++;
+    }
+    lastPosition = data.position;
+  },
+});
+```
+
+For result deduplication (e.g., avoiding repeated announcements), track previous results and implement cooldown logic in your application code.
+
+## Prompt Engineering
+
+Prompt quality significantly affects results. Here are some tips:
+
+**Be specific about output format:**
+
+```typescript
+prompt: "Count the people visible. Return only a number.";
+```
+
+**Include examples for complex tasks:**
+
+```typescript
+prompt: `Describe the primary action happening. Examples:
+- "Person walking left"
+- "Car turning right"
+- "Dog sitting still"`;
+```
+
+**Request minimal output for lower latency:**
+
+```typescript
+prompt: "Is there a person in frame? Answer only 'yes' or 'no'.";
+```
+
+**Provide context when needed:**
+
+```typescript
+prompt: `You are monitoring a ${locationName}. Alert if you see: ${alertConditions.join(", ")}.`;
+```
+
+**Use JSON schema for structured data:**
+
+```typescript
+prompt: "Analyze the scene",
+outputSchema: {
+  type: "object",
+  properties: {
+    description: { type: "string" },
+    alert: { type: "boolean" }
+  },
+  required: ["description", "alert"]
+}
+```
+
+> **Note:** Prompt effectiveness varies by model. Test different approaches to find what works best for your use case.
+
+## React Integration
+
+When using the SDK in React applications, ensure proper cleanup:
+
+```typescript
+import { useEffect, useRef, useState } from "react";
+import { RealtimeVision } from "@overshoot/sdk";
+
+function VisionComponent() {
+  const visionRef = useRef<RealtimeVision | null>(null);
+  const videoRef = useRef<HTMLVideoElement>(null);
+  const [isRunning, setIsRunning] = useState(false);
+
+  const startVision = async () => {
+    const vision = new RealtimeVision({
+      apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
+      apiKey: "your-api-key",
+      prompt: "Describe what you see",
+      onResult: (result) => {
+        console.log(result.result);
+      },
+      onError: (error) => {
+        console.error("Vision error:", error);
+        setIsRunning(false);
+      },
+    });
+
+    await vision.start();
+    visionRef.current = vision;
+    setIsRunning(true);
+
+    // Attach stream to video element for preview
+    const stream = vision.getMediaStream();
+    if (stream && videoRef.current) {
+      videoRef.current.srcObject = stream;
+      videoRef.current.play().catch(console.error);
+    }
+  };
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      visionRef.current?.stop();
+    };
+  }, []);
+
+  return (
+    <div>
+      <video ref={videoRef} autoPlay playsInline muted />
+      <button onClick={startVision} disabled={isRunning}>
+        Start
+      </button>
+      <button onClick={() => visionRef.current?.stop()} disabled={!isRunning}>
+        Stop
+      </button>
+    </div>
+  );
+}
+```
+
+## Advanced: Custom Video Sources with StreamClient
+
+For advanced use cases like streaming from a canvas, screen capture, or other custom sources, use `StreamClient` directly:
+
+```typescript
+import { StreamClient } from "@overshoot/sdk";
+
+const client = new StreamClient({
+  baseUrl: "https://cluster1.overshoot.ai/api/v0.2",
+  apiKey: "your-api-key",
+});
+
+// Get stream from any source (canvas, screen capture, etc.)
+const canvas = document.querySelector("canvas");
+const stream = canvas.captureStream(30);
+const videoTrack = stream.getVideoTracks()[0];
+
+// Set up WebRTC connection
+const peerConnection = new RTCPeerConnection({ iceServers: [...] }); // See default ice servers in RealtimeVison.ts file
+peerConnection.addTrack(videoTrack, stream);
+
+const offer = await peerConnection.createOffer();
+await peerConnection.setLocalDescription(offer);
+
+// Create stream on server
+const response = await client.createStream({
+  webrtc: { type: "offer", sdp: peerConnection.localDescription.sdp },
+  processing: { sampling_ratio: 0.5, fps: 30, clip_length_seconds: 1.0, delay_seconds: 1.0 },
+  inference: {
+    prompt: "Analyze the content",
+    backend: "overshoot",
+    model: "Qwen/Qwen3-VL-30B-A3B-Instruct",
+  },
+});
+
+await peerConnection.setRemoteDescription(response.webrtc);
+
+// Connect WebSocket for results
+const ws = client.connectWebSocket(response.stream_id);
+ws.onopen = () => ws.send(JSON.stringify({ api_key: "your-api-key" }));
+ws.onmessage = (event) => {
+  const result = JSON.parse(event.data);
+  console.log("Result:", result);
+};
+```
+
 ## Examples
 
 ### Object Detection with Structured Output
 
 ```typescript
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key",
   prompt: "Detect objects and return JSON: {objects: string[], count: number}",
   outputSchema: {
@@ -141,7 +406,7 @@ await vision.start();
 
 ```typescript
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key",
   prompt: "Read all visible text in the image",
   onResult: (result) => {
@@ -152,31 +417,11 @@ const vision = new RealtimeVision({
 await vision.start();
 ```
 
-### Video Preview Display
-
-```typescript
-const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
-  apiKey: "your-api-key",
-  prompt: "Describe what you see",
-  onResult: (result) => console.log(result.result),
-});
-
-await vision.start();
-
-// Attach to video element for preview
-const videoElement = document.querySelector("video");
-const stream = vision.getMediaStream();
-if (stream) {
-  videoElement.srcObject = stream;
-}
-```
-
 ### Dynamic Prompt Updates
 
 ```typescript
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key",
   prompt: "Count people",
   onResult: (result) => console.log(result.result),
@@ -192,7 +437,7 @@ await vision.updatePrompt("Detect vehicles instead");
 
 ```typescript
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key",
   prompt: "Detect objects",
   debug: true, // Enable detailed logging
@@ -207,7 +452,7 @@ await vision.start();
 
 ```typescript
 const vision = new RealtimeVision({
-  apiUrl: "https://api.overshoot.ai",
+  apiUrl: "https://cluster1.overshoot.ai/api/v0.2",
   apiKey: "your-api-key",
   prompt: "Detect objects",
   onResult: (result) => {
@@ -243,10 +488,10 @@ The `onResult` callback receives a `StreamInferenceResult` object:
 interface StreamInferenceResult {
   id: string; // Result ID
   stream_id: string; // Stream ID
-  model_backend: "gemini" | "overshoot";
+  model_backend: "overshoot";
   model_name: string; // Model used
   prompt: string; // Task that was run
-  result: string; // Model output (text or JSON string)
+  result: string; // Model output (always a string - parse JSON if using outputSchema)
   inference_latency_ms: number; // Model inference time
   total_latency_ms: number; // End-to-end latency
   ok: boolean; // Success status

package/dist/index.d.mts

@@ -21,7 +21,7 @@ type StreamProcessingConfig = {
 };
 type StreamInferenceConfig = {
     prompt: string;
-    backend: "gemini" | "overshoot";
+    backend: "overshoot";
     model: string;
     output_schema_json?: Record<string, any>;
 };
@@ -45,7 +45,7 @@ type StreamCreateResponse = {
 type StreamInferenceResult = {
     id: string;
     stream_id: string;
-    model_backend: "gemini" | "overshoot";
+    model_backend: "overshoot";
     model_name: string;
     prompt: string;
     result: string;
@@ -58,26 +58,12 @@ type StreamConfigResponse = {
     id: string;
     stream_id: string;
     prompt: string;
-    backend: "gemini" | "overshoot";
+    backend: "overshoot";
     model: string;
     output_schema_json?: Record<string, any>;
     created_at?: string;
     updated_at?: string;
 };
-type FeedbackCreateRequest = {
-    rating: number;
-    category: string;
-    feedback?: string;
-};
-type FeedbackResponse = {
-    id: string;
-    stream_id: string;
-    rating: number;
-    category: string;
-    feedback: string;
-    created_at?: string;
-    updated_at?: string;
-};
 type KeepaliveResponse = {
     status: "ok";
     stream_id: string;
@@ -105,8 +91,6 @@ declare class StreamClient {
     createStream(request: StreamCreateRequest): Promise<StreamCreateResponse>;
     renewLease(streamId: string): Promise<KeepaliveResponse>;
     updatePrompt(streamId: string, prompt: string): Promise<StreamConfigResponse>;
-    submitFeedback(streamId: string, feedback: FeedbackCreateRequest): Promise<StatusResponse>;
-    getAllFeedback(): Promise<FeedbackResponse[]>;
     connectWebSocket(streamId: string): WebSocket;
     /**
      * Health check endpoint (for testing, uses internal port if available)
@@ -143,7 +127,7 @@ interface RealtimeVisionConfig {
     /**
      * Model backend to use
      */
-    backend?: "gemini" | "overshoot";
+    backend?: "overshoot";
     /**
      * Model name to use for inference
      */
@@ -203,6 +187,8 @@ declare class RealtimeVision {
     private streamId;
     private keepaliveInterval;
     private videoElement;
+    private canvasElement;
+    private canvasAnimationFrameId;
     private isRunning;
     constructor(config: RealtimeVisionConfig);
     /**
@@ -253,14 +239,6 @@ declare class RealtimeVision {
      * Stop the vision stream and clean up resources
      */
     stop(): Promise<void>;
-    /**
-     * Submit feedback for the stream
-     */
-    submitFeedback(feedback: {
-        rating: number;
-        category: string;
-        feedback?: string;
-    }): Promise<void>;
     /**
      * Get the current stream ID
      */
@@ -299,4 +277,4 @@ declare class ServerError extends ApiError {
     constructor(message: string, requestId?: string, details?: any);
 }
 
-export { ApiError, type ErrorResponse, type FeedbackCreateRequest, type FeedbackResponse, type KeepaliveResponse, NetworkError, NotFoundError, RealtimeVision, type RealtimeVisionConfig, ServerError, type StatusResponse, StreamClient, type StreamClientMeta, type StreamConfigResponse, type StreamCreateRequest, type StreamCreateResponse, type StreamInferenceConfig, type StreamInferenceResult, type StreamProcessingConfig, type StreamSource, UnauthorizedError, ValidationError, type WebRtcAnswer, type WebRtcOffer };
+export { ApiError, type ErrorResponse, type KeepaliveResponse, NetworkError, NotFoundError, RealtimeVision, type RealtimeVisionConfig, ServerError, type StatusResponse, StreamClient, type StreamClientMeta, type StreamConfigResponse, type StreamCreateRequest, type StreamCreateResponse, type StreamInferenceConfig, type StreamInferenceResult, type StreamProcessingConfig, type StreamSource, UnauthorizedError, ValidationError, type WebRtcAnswer, type WebRtcOffer };

package/dist/index.d.ts

@@ -21,7 +21,7 @@ type StreamProcessingConfig = {
 };
 type StreamInferenceConfig = {
     prompt: string;
-    backend: "gemini" | "overshoot";
+    backend: "overshoot";
     model: string;
     output_schema_json?: Record<string, any>;
 };
@@ -45,7 +45,7 @@ type StreamCreateResponse = {
 type StreamInferenceResult = {
     id: string;
     stream_id: string;
-    model_backend: "gemini" | "overshoot";
+    model_backend: "overshoot";
     model_name: string;
     prompt: string;
     result: string;
@@ -58,26 +58,12 @@ type StreamConfigResponse = {
     id: string;
     stream_id: string;
     prompt: string;
-    backend: "gemini" | "overshoot";
+    backend: "overshoot";
     model: string;
     output_schema_json?: Record<string, any>;
     created_at?: string;
     updated_at?: string;
 };
-type FeedbackCreateRequest = {
-    rating: number;
-    category: string;
-    feedback?: string;
-};
-type FeedbackResponse = {
-    id: string;
-    stream_id: string;
-    rating: number;
-    category: string;
-    feedback: string;
-    created_at?: string;
-    updated_at?: string;
-};
 type KeepaliveResponse = {
     status: "ok";
     stream_id: string;
@@ -105,8 +91,6 @@ declare class StreamClient {
     createStream(request: StreamCreateRequest): Promise<StreamCreateResponse>;
     renewLease(streamId: string): Promise<KeepaliveResponse>;
     updatePrompt(streamId: string, prompt: string): Promise<StreamConfigResponse>;
-    submitFeedback(streamId: string, feedback: FeedbackCreateRequest): Promise<StatusResponse>;
-    getAllFeedback(): Promise<FeedbackResponse[]>;
     connectWebSocket(streamId: string): WebSocket;
     /**
      * Health check endpoint (for testing, uses internal port if available)
@@ -143,7 +127,7 @@ interface RealtimeVisionConfig {
     /**
      * Model backend to use
      */
-    backend?: "gemini" | "overshoot";
+    backend?: "overshoot";
     /**
      * Model name to use for inference
      */
@@ -203,6 +187,8 @@ declare class RealtimeVision {
     private streamId;
     private keepaliveInterval;
     private videoElement;
+    private canvasElement;
+    private canvasAnimationFrameId;
     private isRunning;
     constructor(config: RealtimeVisionConfig);
     /**
@@ -253,14 +239,6 @@ declare class RealtimeVision {
      * Stop the vision stream and clean up resources
      */
     stop(): Promise<void>;
-    /**
-     * Submit feedback for the stream
-     */
-    submitFeedback(feedback: {
-        rating: number;
-        category: string;
-        feedback?: string;
-    }): Promise<void>;
     /**
      * Get the current stream ID
      */
@@ -299,4 +277,4 @@ declare class ServerError extends ApiError {
     constructor(message: string, requestId?: string, details?: any);
 }
 
-export { ApiError, type ErrorResponse, type FeedbackCreateRequest, type FeedbackResponse, type KeepaliveResponse, NetworkError, NotFoundError, RealtimeVision, type RealtimeVisionConfig, ServerError, type StatusResponse, StreamClient, type StreamClientMeta, type StreamConfigResponse, type StreamCreateRequest, type StreamCreateResponse, type StreamInferenceConfig, type StreamInferenceResult, type StreamProcessingConfig, type StreamSource, UnauthorizedError, ValidationError, type WebRtcAnswer, type WebRtcOffer };
+export { ApiError, type ErrorResponse, type KeepaliveResponse, NetworkError, NotFoundError, RealtimeVision, type RealtimeVisionConfig, ServerError, type StatusResponse, StreamClient, type StreamClientMeta, type StreamConfigResponse, type StreamCreateRequest, type StreamCreateResponse, type StreamInferenceConfig, type StreamInferenceResult, type StreamProcessingConfig, type StreamSource, UnauthorizedError, ValidationError, type WebRtcAnswer, type WebRtcOffer };