@blueharford/scrypted-spatial-awareness 0.6.9 → 0.6.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blueharford/scrypted-spatial-awareness",
-  "version": "0.6.9",
+  "version": "0.6.10",
   "description": "Cross-camera object tracking for Scrypted NVR with spatial awareness",
   "author": "Joshua Seidel <blueharford>",
   "license": "Apache-2.0",

package/src/core/spatial-reasoning.ts

@@ -1077,33 +1077,58 @@ Examples of good descriptions:
 
 Generate ONLY the description, nothing else:`;
 
-      // Build message content - use multimodal format if we have an image
-      let messageContent: any;
+      // Try multimodal format first, fall back to text-only if it fails
+      let result: any;
+      let usedVision = false;
+
       if (imageData) {
-        messageContent = [
-          { type: 'text', text: prompt },
-          buildImageContent(imageData, this.llmProviderType),
-        ];
-      } else {
-        messageContent = prompt;
+        // First attempt: Try multimodal with image
+        try {
+          this.console.log(`[LLM] Attempting multimodal ${eventType} call with image...`);
+          const multimodalContent = [
+            { type: 'text', text: prompt },
+            buildImageContent(imageData, this.llmProviderType),
+          ];
+
+          result = await llm.getChatCompletion({
+            messages: [
+              {
+                role: 'user',
+                content: multimodalContent,
+              },
+            ],
+            max_tokens: 100,
+            temperature: 0.7,
+          });
+          usedVision = true;
+        } catch (visionError: any) {
+          // If vision format fails, try text-only
+          if (isVisionFormatError(visionError)) {
+            this.console.warn(`[LLM] Vision format not supported, falling back to text-only: ${visionError.message || visionError}`);
+          } else {
+            this.console.warn(`[LLM] Multimodal call failed, trying text-only: ${visionError.message || visionError}`);
+          }
+        }
       }
 
-      // Call LLM using ChatCompletion interface
-      this.console.log(`[LLM] Calling getChatCompletion for ${eventType}...`);
-      const result = await llm.getChatCompletion({
-        messages: [
-          {
-            role: 'user',
-            content: messageContent,
-          },
-        ],
-        max_tokens: 100,
-        temperature: 0.7,
-      });
+      // If no result yet, try text-only
+      if (!result) {
+        this.console.log(`[LLM] Calling text-only getChatCompletion for ${eventType}...`);
+        result = await llm.getChatCompletion({
+          messages: [
+            {
+              role: 'user',
+              content: prompt,
+            },
+          ],
+          max_tokens: 100,
+          temperature: 0.7,
+        });
+      }
 
       const content = result?.choices?.[0]?.message?.content;
       if (content && typeof content === 'string') {
-        this.console.log(`[LLM] Got ${eventType} description: ${content.trim().substring(0, 50)}...`);
+        this.console.log(`[LLM] Got ${eventType} description (vision=${usedVision}): ${content.trim().substring(0, 50)}...`);
         return content.trim();
       }