@axiom-lattice/core 2.1.102 → 2.1.103

package/dist/index.js

@@ -1683,6 +1683,8 @@ __export(index_exports, {
   QueueMode: () => QueueMode,
   RemoteSandboxInstance: () => RemoteSandboxInstance,
   RemoteSandboxProvider: () => RemoteSandboxProvider,
+  STTModelLattice: () => STTModelLattice,
+  STTModelLatticeManager: () => STTModelLatticeManager,
   SandboxFilesystem: () => SandboxFilesystem,
   SandboxLatticeManager: () => SandboxLatticeManager,
   SandboxSkillStore: () => SandboxSkillStore,
@@ -1788,6 +1790,9 @@ __export(index_exports, {
   getNextCronTime: () => getNextCronTime,
   getOrCreateCollectionVectorStore: () => getOrCreateCollectionVectorStore,
   getQueueLattice: () => getQueueLattice,
+  getSTTClient: () => getSTTClient,
+  getSTTClientWithTenant: () => getSTTClientWithTenant,
+  getSTTModelLattice: () => getSTTModelLattice,
   getSandBoxManager: () => getSandBoxManager,
   getScheduleLattice: () => getScheduleLattice,
   getStoreLattice: () => getStoreLattice,
@@ -1831,6 +1836,7 @@ __export(index_exports, {
   registerLoggerLattice: () => registerLoggerLattice,
   registerModelLattice: () => registerModelLattice,
   registerQueueLattice: () => registerQueueLattice,
+  registerSTTModelLattice: () => registerSTTModelLattice,
   registerSandboxProviderType: () => registerSandboxProviderType,
   registerScheduleLattice: () => registerScheduleLattice,
   registerStoreLattice: () => registerStoreLattice,
@@ -1852,6 +1858,7 @@ __export(index_exports, {
   skillLatticeManager: () => skillLatticeManager,
   sqlDatabaseManager: () => sqlDatabaseManager,
   storeLatticeManager: () => storeLatticeManager,
+  sttModelLatticeManager: () => sttModelLatticeManager,
   toJsonSchema: () => toJsonSchema,
   toSafeStateExpr: () => toSafeStateExpr,
   toolLatticeManager: () => toolLatticeManager,
@@ -9656,6 +9663,34 @@ function validateImageSize(sizeBytes) {
   return null;
 }
 
+// src/deep_agent_new/backends/audioUtils.ts
+var AUDIO_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".webm",
+  ".wav",
+  ".mp3",
+  ".m4a",
+  ".ogg",
+  ".flac",
+  ".aac",
+  ".wma",
+  ".opus",
+  ".amr"
+]);
+function isAudioFile(filePath) {
+  const ext = filePath.toLowerCase().slice(filePath.lastIndexOf("."));
+  return AUDIO_EXTENSIONS.has(ext);
+}
+function detectAudioFormat(filePath) {
+  return filePath.toLowerCase().slice(filePath.lastIndexOf(".") + 1);
+}
+var MAX_AUDIO_SIZE = 25 * 1024 * 1024;
+function validateAudioSize(sizeBytes) {
+  if (sizeBytes > MAX_AUDIO_SIZE) {
+    return `Audio file too large (${(sizeBytes / 1024 / 1024).toFixed(1)}MB). Maximum is 25MB.`;
+  }
+  return null;
+}
+
 // src/deep_agent_new/backends/describeImage.ts
 var import_messages = require("@langchain/core/messages");
 async function describeImage(options) {
@@ -9681,6 +9716,215 @@ async function describeImage(options) {
   return result.content || "";
 }
 
+// src/stt_model_lattice/STTModelLatticeManager.ts
+init_BaseLatticeManager();
+
+// src/stt_model_lattice/STTModelLattice.ts
+var import_openai2 = require("@langchain/openai");
+var STTModelLattice = class {
+  constructor(key4, config) {
+    this.key = key4;
+    this.config = config;
+    this.client = this;
+    const apiKey = config.apiKey || (config.apiKeyEnvName ? process.env[config.apiKeyEnvName] : void 0) || process.env.OPENAI_API_KEY || "";
+    if (!apiKey) {
+      throw new Error("No API key configured for STT. Set apiKey, apiKeyEnvName, or OPENAI_API_KEY.");
+    }
+    this.openaiClient = new import_openai2.OpenAIClient({
+      apiKey,
+      baseURL: config.baseURL,
+      timeout: config.timeout || 3e4
+    });
+  }
+  /**
+   * Transcribe audio buffer to text.
+   * Routes to whisper or chat API mode based on config.
+   */
+  async transcribe(audio, format) {
+    const mode = this.config.apiMode || "whisper";
+    if (mode === "chat") {
+      return this.transcribeViaChat(audio, format);
+    }
+    return this.transcribeViaWhisper(audio, format);
+  }
+  /**
+   * Transcribe via OpenAI /v1/audio/transcriptions endpoint (multipart).
+   */
+  async transcribeViaWhisper(audio, format) {
+    const mimeType = formatToMimeType(format);
+    const file = typeof File !== "undefined" ? new File([audio], `audio.${format}`, { type: mimeType }) : await (0, import_openai2.toFile)(audio, `audio.${format}`, { type: mimeType });
+    const extra = this.config.extra || {};
+    const response = await this.openaiClient.audio.transcriptions.create({
+      file,
+      model: this.config.model || "whisper-1",
+      response_format: "verbose_json",
+      ...extra
+    });
+    const result = response;
+    return {
+      text: result.text,
+      segments: result.segments?.map((s) => ({
+        start: s.start,
+        end: s.end,
+        text: s.text
+      })),
+      confidence: result.segments ? result.segments.reduce((sum, s) => sum + Math.exp(s.avg_logprob ?? -Infinity), 0) / result.segments.length : void 0
+    };
+  }
+  /**
+   * Transcribe via OpenAI /v1/chat/completions with input_audio.
+   * Used by Qwen3-ASR-Flash and similar audio-via-chat models.
+   */
+  async transcribeViaChat(audio, format) {
+    const mimeType = formatToMimeType(format);
+    const dataUri = `data:${mimeType};base64,${audio.toString("base64")}`;
+    const extra = this.config.extra || {};
+    const response = await this.openaiClient.chat.completions.create({
+      model: this.config.model || "qwen3-asr-flash",
+      messages: [
+        {
+          role: "user",
+          content: [
+            {
+              type: "input_audio",
+              input_audio: {
+                data: dataUri,
+                format
+              }
+            }
+          ]
+        }
+      ],
+      ...extra
+    });
+    const result = response;
+    const text = result.choices?.[0]?.message?.content || "";
+    return { text };
+  }
+};
+function formatToMimeType(format) {
+  const mimeTypes = {
+    webm: "audio/webm",
+    wav: "audio/wav",
+    mp3: "audio/mpeg",
+    m4a: "audio/mp4",
+    ogg: "audio/ogg",
+    flac: "audio/flac",
+    aac: "audio/aac"
+  };
+  const lower = format.toLowerCase();
+  if (!mimeTypes[lower]) {
+    console.warn(`Unknown STT audio format "${format}", falling back to audio/webm`);
+  }
+  return mimeTypes[lower] || "audio/webm";
+}
+
+// src/stt_model_lattice/STTModelLatticeManager.ts
+var STTModelLatticeManager = class _STTModelLatticeManager extends BaseLatticeManager {
+  static getInstance() {
+    if (!_STTModelLatticeManager._instance) {
+      _STTModelLatticeManager._instance = new _STTModelLatticeManager();
+    }
+    return _STTModelLatticeManager._instance;
+  }
+  getLatticeType() {
+    return "stt";
+  }
+  /**
+   * Register an STT model lattice.
+   * @param key - Lattice key name
+   * @param config - STT provider configuration
+   */
+  registerLattice(key4, config) {
+    const client = new STTModelLattice(key4, config);
+    const label = config.model || key4;
+    const sttLattice = {
+      key: key4,
+      client,
+      config,
+      label
+    };
+    this.register(key4, sttLattice);
+  }
+  /**
+   * Get an STT model lattice by key.
+   */
+  getSTTModelLattice(key4) {
+    const lattice = this.get(key4);
+    if (!lattice) {
+      throw new Error(`STTModelLattice "${key4}" not found`);
+    }
+    return lattice;
+  }
+  /**
+   * Get STT client instance by key (default tenant).
+   */
+  getSTTClient(key4) {
+    return this.getSTTModelLattice(key4).client;
+  }
+  /**
+   * Get STT client instance by key and tenant.
+   * @param tenantId - Tenant ID for isolation
+   * @param key - Lattice key name
+   */
+  getSTTClientWithTenant(tenantId2, key4) {
+    let lattice = this.getWithTenant(tenantId2, key4);
+    if (!lattice && tenantId2 !== "default") {
+      lattice = this.getWithTenant("default", key4);
+    }
+    if (!lattice) {
+      throw new Error(`STTModelLattice "${key4}" not found for tenant "${tenantId2}"`);
+    }
+    return lattice.client;
+  }
+  /**
+   * Get all registered STT models as info list.
+   */
+  getAllSTTModelInfo() {
+    return this.getAllLattices().map((l) => ({
+      key: l.key,
+      label: l.label,
+      provider: l.config.provider || "openai-compatible",
+      model: l.config.model || "unknown"
+    }));
+  }
+  getAllLattices() {
+    return this.getAll();
+  }
+  hasLattice(key4) {
+    return this.has(key4);
+  }
+  removeLattice(key4) {
+    return this.remove(key4);
+  }
+  clearLattices() {
+    this.clear();
+  }
+  getLatticeCount() {
+    return this.count();
+  }
+  getLatticeKeys() {
+    return this.keys();
+  }
+};
+var sttModelLatticeManager = STTModelLatticeManager.getInstance();
+var registerSTTModelLattice = (key4, config) => sttModelLatticeManager.registerLattice(key4, config);
+var getSTTModelLattice = (key4) => sttModelLatticeManager.getSTTModelLattice(key4);
+var getSTTClient = (key4) => sttModelLatticeManager.getSTTClient(key4);
+var getSTTClientWithTenant = (tenantId2, key4) => sttModelLatticeManager.getSTTClientWithTenant(tenantId2, key4);
+
+// src/deep_agent_new/backends/transcribeAudio.ts
+async function transcribeAudio(options) {
+  const { audioBuffer, format } = options;
+  if (!sttModelLatticeManager.hasLattice("default")) {
+    throw new Error(
+      "No default STT model registered. Use registerSTTModelLattice('default', { ... }) first."
+    );
+  }
+  const client = sttModelLatticeManager.getSTTClient("default");
+  return await client.transcribe(audioBuffer, format);
+}
+
 // src/deep_agent_new/middleware/fs.ts
 var FileDataSchema = import_v3.z.object({
   content: import_v3.z.array(import_v3.z.string()),
@@ -9739,7 +9983,7 @@ Path conventions:
 - glob: find files matching a pattern (e.g., "/project/**/*.py")
 - grep: search for text within files`;
 var LS_TOOL_DESCRIPTION = "List files and directories in a directory";
-var READ_FILE_TOOL_DESCRIPTION = "Read the contents of a file. For image files (png, jpg, gif, webp, bmp, svg), returns a visual description when the current model supports vision. For unsupported models, returns an error suggesting to switch to a vision-capable model.";
+var READ_FILE_TOOL_DESCRIPTION = "Read the contents of a file. For image files (png, jpg, gif, webp, bmp, svg), returns a visual description when the current model supports vision; otherwise returns an error suggesting a vision-capable model. For audio files (webm, wav, mp3, m4a, ogg, flac, aac, wma, opus, amr), transcribes the content using the default STT model; if none is registered, returns an error with registration instructions.";
 var WRITE_FILE_TOOL_DESCRIPTION = "Write content to a new file. Returns an error if the file already exists";
 var EDIT_FILE_TOOL_DESCRIPTION = "Edit a file by replacing a specific string with a new string";
 var GLOB_TOOL_DESCRIPTION = "Find files matching a glob pattern (e.g., '**/*.py' for all Python files)";
@@ -9824,6 +10068,34 @@ ${description}`;
 \u8BFB\u53D6\u56FE\u7247\u5931\u8D25\uFF1A${error instanceof Error ? error.message : "\u672A\u77E5\u9519\u8BEF"}`;
         }
       }
+      if (isAudioFile(file_path)) {
+        if (!resolvedBackend.readBinary) {
+          return `[Audio] ${file_path}
+The current backend does not support binary reading, unable to process audio.`;
+        }
+        try {
+          const buffer2 = await resolvedBackend.readBinary(file_path);
+          const sizeWarning = validateAudioSize(buffer2.length);
+          if (sizeWarning) {
+            return `[Audio] ${file_path}
+${sizeWarning}`;
+          }
+          const format = detectAudioFormat(file_path);
+          const result = await transcribeAudio({ audioBuffer: buffer2, format });
+          let output = `[Audio] ${file_path} (${format}, ${(buffer2.length / 1024).toFixed(1)}KB)`;
+          if (result.confidence !== void 0) {
+            output += `
+Confidence: ${(result.confidence * 100).toFixed(1)}%`;
+          }
+          output += `
+
+${result.text}`;
+          return output;
+        } catch (error) {
+          return `[Audio] ${file_path}
+Audio transcription failed: ${error instanceof Error ? error.message : "Unknown error"}`;
+        }
+      }
       return await resolvedBackend.read(file_path, offset, limit);
     },
     {
@@ -29430,6 +29702,8 @@ registerBuiltinPlugins();
   QueueMode,
   RemoteSandboxInstance,
   RemoteSandboxProvider,
+  STTModelLattice,
+  STTModelLatticeManager,
   SandboxFilesystem,
   SandboxLatticeManager,
   SandboxSkillStore,
@@ -29535,6 +29809,9 @@ registerBuiltinPlugins();
   getNextCronTime,
   getOrCreateCollectionVectorStore,
   getQueueLattice,
+  getSTTClient,
+  getSTTClientWithTenant,
+  getSTTModelLattice,
   getSandBoxManager,
   getScheduleLattice,
   getStoreLattice,
@@ -29578,6 +29855,7 @@ registerBuiltinPlugins();
   registerLoggerLattice,
   registerModelLattice,
   registerQueueLattice,
+  registerSTTModelLattice,
   registerSandboxProviderType,
   registerScheduleLattice,
   registerStoreLattice,
@@ -29599,6 +29877,7 @@ registerBuiltinPlugins();
   skillLatticeManager,
   sqlDatabaseManager,
   storeLatticeManager,
+  sttModelLatticeManager,
   toJsonSchema,
   toSafeStateExpr,
   toolLatticeManager,