@tyvm/knowhow 0.0.68 → 0.0.70

package/src/services/KnowhowClient.ts

@@ -7,6 +7,20 @@ import {
   CompletionResponse,
   EmbeddingOptions,
   EmbeddingResponse,
+  AudioTranscriptionOptions,
+  AudioTranscriptionResponse,
+  AudioGenerationOptions,
+  AudioGenerationResponse,
+  ImageGenerationOptions,
+  ImageGenerationResponse,
+  VideoGenerationOptions,
+  VideoGenerationResponse,
+  VideoStatusOptions,
+  VideoStatusResponse,
+  FileUploadOptions,
+  FileUploadResponse,
+  FileDownloadOptions,
+  FileDownloadResponse,
 } from "../clients";
 import { Config } from "../types";
 
@@ -202,6 +216,98 @@ export class KnowhowSimpleClient {
     });
   }
 
+  async createAudioTranscription(options: AudioTranscriptionOptions) {
+    await this.checkJwt();
+    const formData = new FormData();
+    // options.file can be a Buffer, ReadStream, Blob, or File
+    if (Buffer.isBuffer(options.file)) {
+      formData.append("file", new Blob([options.file]), options["fileName"] || "audio.mp3");
+    } else {
+      formData.append("file", options.file);
+    }
+    if (options.model) formData.append("model", options.model);
+    if (options.language) formData.append("language", options.language);
+    if (options.prompt) formData.append("prompt", options.prompt);
+    if (options.response_format) formData.append("response_format", options.response_format);
+    if (options.temperature != null) formData.append("temperature", String(options.temperature));
+
+    return axios.post<AudioTranscriptionResponse>(
+      `${this.baseUrl}/api/proxy/v1/audio/transcriptions`,
+      formData,
+      { headers: { ...this.headers } }
+    );
+  }
+
+  async createAudioGeneration(options: AudioGenerationOptions) {
+    await this.checkJwt();
+    return axios.post<AudioGenerationResponse>(
+      `${this.baseUrl}/api/proxy/v1/audio/generations`,
+      options,
+      { headers: this.headers }
+    );
+  }
+
+  async createImageGeneration(options: ImageGenerationOptions) {
+    await this.checkJwt();
+    return axios.post<ImageGenerationResponse>(
+      `${this.baseUrl}/api/proxy/v1/images/generations`,
+      options,
+      { headers: this.headers }
+    );
+  }
+
+  async createVideoGeneration(options: VideoGenerationOptions) {
+    await this.checkJwt();
+    return axios.post<VideoGenerationResponse>(
+      `${this.baseUrl}/api/proxy/v1/videos/generations`,
+      options,
+      { headers: this.headers }
+    );
+  }
+
+  async getVideoStatus(options: VideoStatusOptions) {
+    await this.checkJwt();
+    const { jobId, ...rest } = options;
+    return axios.get<VideoStatusResponse>(
+      `${this.baseUrl}/api/proxy/v1/videos/${jobId}/status`,
+      { headers: this.headers, params: rest }
+    );
+  }
+
+  async downloadVideo(options: FileDownloadOptions) {
+    await this.checkJwt();
+    const { fileId } = options;
+    return axios.get<ArrayBuffer>(
+      `${this.baseUrl}/api/proxy/v1/videos/${fileId}/content`,
+      { headers: this.headers, responseType: "arraybuffer" }
+    );
+  }
+
+  async uploadFile(options: FileUploadOptions) {
+    await this.checkJwt();
+    // Send as JSON with base64-encoded data
+    const body = {
+      data: options.data.toString("base64"),
+      mimeType: options.mimeType,
+      fileName: options.fileName,
+      displayName: options.displayName,
+    };
+    return axios.post<FileUploadResponse>(
+      `${this.baseUrl}/api/proxy/v1/files`,
+      body,
+      { headers: this.headers }
+    );
+  }
+
+  async downloadFile(options: FileDownloadOptions) {
+    await this.checkJwt();
+    const { fileId } = options;
+    return axios.get<ArrayBuffer>(
+      `${this.baseUrl}/api/proxy/v1/files/${fileId}/content`,
+      { headers: this.headers, responseType: "arraybuffer" }
+    );
+  }
+
   async createChatTask(request: CreateMessageTaskRequest) {
     await this.checkJwt();
     return axios.post<CreateMessageTaskResponse>(

package/src/services/index.ts

@@ -10,7 +10,8 @@ import { S3Service } from "./S3";
 import { ToolsService } from "./Tools";
 import { PluginService } from "../plugins/plugins";
 import { DockerService } from "./DockerService";
-import { AgentSynchronization } from "./AgentSynchronization";
+import { AgentSyncKnowhowWeb } from "./AgentSyncKnowhowWeb";
+import { AgentSyncFs } from "./AgentSyncFs";
 import { SessionManager } from "./SessionManager";
 import { TaskRegistry } from "./TaskRegistry";
 
@@ -24,7 +25,8 @@ export * from "./LazyToolsService";
 export * as MCP from "./Mcp";
 export * from "./EmbeddingService";
 export * from "./DockerService";
-export * from "./AgentSynchronization";
+export * from "./AgentSyncKnowhowWeb";
+export * from "./AgentSyncFs";
 export * from "./SessionManager";
 export * from "./TaskRegistry";
 export { Clients } from "../clients";

package/src/types.ts

@@ -46,7 +46,8 @@ export type Config = {
   embedSources: EmbedSource[];
   embeddingModel: string;
 
-  plugins: string[];
+  plugins: { enabled: string[]; disabled: string[] };
+
   modules: string[];
 
   agents: Assistant[];
@@ -140,6 +141,7 @@ export type Language = {
     events: string[];
     sources: IDatasource[];
     context?: string;
+    handled?: boolean;
   };
 };
 
@@ -177,6 +179,8 @@ export const Models = {
     Grok3MiniFastBeta: "grok-3-mini-fast-beta",
     Grok21212: "grok-2-1212",
     Grok2Vision1212: "grok-2-vision-1212",
+    GrokImagineImage: "grok-imagine-image",
+    GrokImagineVideo: "grok-imagine-video",
   },
   openai: {
     GPT_5_2: "gpt-5.2",
@@ -203,6 +207,14 @@ export const Models = {
     o1_Mini: "o1-mini-2024-09-12",
     GPT_4o_Mini_Search: "gpt-4o-mini-search-preview-2025-03-11",
     GPT_4o_Search: "gpt-4o-search-preview-2025-03-11",
+
+    TTS_1: "tts-1",
+    Whisper_1: "whisper-1",
+    DALL_E_3: "dall-e-3",
+    DALL_E_2: "dall-e-2",
+    Sora: "sora",
+    Sora_2: "sora-2",
+    Sora_2_Pro: "sora-2-pro",
     // Computer_Use: "computer-use-preview-2025-03-11",
     // Codex_Mini: "codex-mini-latest",
   },
@@ -212,14 +224,17 @@ export const Models = {
     Gemini_25_Pro_Preview: "gemini-2.5-pro-preview-05-06",
     Gemini_20_Flash: "gemini-2.0-flash",
     Gemini_20_Flash_Preview_Image_Generation:
-      "gemini-2.0-flash-preview-image-generation",
+      "gemini-2.0-flash-exp-image-generation",
     Gemini_20_Flash_Lite: "gemini-2.0-flash-lite",
     Gemini_15_Flash: "gemini-1.5-flash",
     Gemini_15_Flash_8B: "gemini-1.5-flash-8b",
     Gemini_15_Pro: "gemini-1.5-pro",
-    Imagen_3: "imagen-3.0-generate-002",
+    Imagen_3: "imagen-4.0-generate-001",
     Veo_2: "veo-2.0-generate-001",
+    Veo_3_1: "veo-3.1-generate-preview",
     Gemini_20_Flash_Live: "gemini-2.0-flash-live-001",
+    Gemini_25_Flash_TTS: "gemini-2.5-flash-preview-tts",
+    Gemini_20_Flash_TTS: "gemini-2.0-flash-preview-tts",
   },
 };
 
@@ -234,6 +249,20 @@ export const EmbeddingModels = {
   },
 };
 
+export function getEnabledPlugins(
+  plugins: Config["plugins"] | undefined
+): string[] {
+  if (!plugins) return [];
+  return plugins.enabled ?? [];
+}
+
+export function getDisabledPlugins(
+  plugins: Config["plugins"] | undefined
+): string[] {
+  if (!plugins) return [];
+  return plugins.disabled ?? [];
+}
+
 export const Providers = Object.keys(Models).reduce((obj, key) => {
   obj[key] = key;
   return obj;
@@ -275,6 +304,30 @@ export const GoogleImageModels = [
   Models.google.Imagen_3,
 ];
 
-export const GoogleVideoModels = [Models.google.Veo_2];
+export const OpenAiImageModels = [
+  Models.openai.DALL_E_3,
+  Models.openai.DALL_E_2,
+];
+
+export const OpenAiVideoModels = [
+  Models.openai.Sora,
+  Models.openai.Sora_2,
+  Models.openai.Sora_2_Pro,
+];
+
+export const OpenAiTTSModels = [Models.openai.TTS_1];
+
+export const OpenAiTranscriptionModels = [Models.openai.Whisper_1];
+
+export const XaiImageModels = [Models.xai.GrokImagineImage];
+
+export const XaiVideoModels = [Models.xai.GrokImagineVideo];
+
+export const GoogleTTSModels = [
+  Models.google.Gemini_25_Flash_TTS,
+  Models.google.Gemini_20_Flash_TTS,
+];
+
+export const GoogleVideoModels = [Models.google.Veo_2, Models.google.Veo_3_1];
 
 export const GoogleEmbeddingModels = [EmbeddingModels.google.Gemini_Embedding];

package/src/worker.ts

@@ -306,14 +306,37 @@ export async function worker(options?: {
         // Get the allowedPorts configuration
         const allowedPorts = config.worker?.tunnel?.allowedPorts || [];
 
+        // Create URL rewriter callback that returns the hostname (without protocol)
+        // The tunnel package will add the protocol based on the useHttps config
+        // This receives port and metadata from the tunnel request
+        const urlRewriter = (port: number, metadata?: any) => {
+          const workerId = metadata?.workerId;
+          const secret = metadata?.secret;
+
+          // Build the hostname/domain (without protocol) based on metadata
+          // The tunnel handler will add the protocol using the useHttps config
+          // Examples:
+          // - secret-p3000.worker.example.com
+          // - workerId-p3000.worker.example.com
+          const subdomain = secret
+            ? `${secret}-p${port}`
+            : `${workerId}-p${port}`;
+
+          // Return just the hostname - the tunnel package should add the protocol
+          // based on the useHttps configuration passed below
+          const replacementUrl = `${subdomain}.${tunnelDomain}`;
+          return replacementUrl;
+        };
+
         // Initialize tunnel handler with the tunnel-specific WebSocket
+        // Pass useHttps flag so the tunnel package can add the correct protocol
         tunnelHandler = createTunnelHandler(tunnelConnection!, {
           allowedPorts,
           maxConcurrentStreams:
             config.worker?.tunnel?.maxConcurrentStreams || 50,
+          tunnelUseHttps: tunnelUseHttps,
           localHost: tunnelLocalHost,
-          tunnelDomain,
-          tunnelUseHttps,
+          urlRewriter,
           enableUrlRewriting:
             config.worker?.tunnel?.enableUrlRewriting !== false,
           portMapping,

package/tests/manual/modalities/README.md

@@ -0,0 +1,157 @@
+# Modalities Manual Tests
+
+Manual integration tests for audio, image, vision, and video generation across all supported AI providers.
+
+## Overview
+
+These tests exercise every output/input modality supported by each provider:
+
+| Modality             | OpenAI       | Google             | XAI          |
+|----------------------|--------------|--------------------|--------------|
+| Audio Generation     | TTS-1 / TTS-1-HD | Gemini 2.0 Flash TTS | ❌ Not supported |
+| Audio Transcription  | Whisper-1    | ❌ Not supported   | ❌ Not supported |
+| Image Generation     | DALL-E 3     | Imagen 3 / Gemini 2.0 Flash | Aurora |
+| Vision (image input) | GPT-4o       | Gemini 2.0 Flash   | Grok-2-Vision |
+| Video Generation     | Sora (stub)  | Veo 2              | ❌ Not supported |
+
+---
+
+## Prerequisites
+
+Set the appropriate API keys in your environment before running tests:
+
+```bash
+export OPENAI_KEY="sk-..."
+export GEMINI_API_KEY="AIza..."
+export XAI_API_KEY="xai-..."
+```
+
+---
+
+## Running the Tests
+
+### Run all modality tests
+
+```bash
+npx jest tests/manual/modalities --testTimeout=300000 --runInBand
+```
+
+> **`--runInBand`** is recommended so tests within a file run sequentially (some tests depend on outputs from prior tests in the same file).
+
+### Run a single provider
+
+```bash
+# OpenAI only
+npx jest tests/manual/modalities/openai.modalities.test.ts --testTimeout=120000 --runInBand
+
+# Google only
+npx jest tests/manual/modalities/google.modalities.test.ts --testTimeout=300000 --runInBand
+
+# XAI only
+npx jest tests/manual/modalities/xai.modalities.test.ts --testTimeout=120000 --runInBand
+```
+
+### Run a single test by name
+
+```bash
+npx jest tests/manual/modalities/openai.modalities.test.ts \
+  --testNamePattern="DALL-E 3" \
+  --testTimeout=60000
+```
+
+---
+
+## Output Files
+
+All generated artifacts are saved under `tests/manual/modalities/outputs/<provider>/` so they can be reviewed after the tests run.
+
+### OpenAI (`outputs/openai/`)
+
+| File | Test | Description |
+|------|------|-------------|
+| `tts-output.mp3` | Test 1 | TTS-1 generated speech audio |
+| `transcription.json` | Test 2 | Whisper transcription of the TTS audio |
+| `dalle3-output.png` | Test 3 | DALL-E 3 generated image |
+| `dalle3-output-url.txt` | Test 3 | URL fallback if b64_json not returned |
+| `vision-description.txt` | Test 4 | GPT-4o description of the DALL-E image |
+
+### Google (`outputs/google/`)
+
+| File | Test | Description |
+|------|------|-------------|
+| `tts-output.wav` | Test 1 | Gemini 2.0 Flash TTS audio |
+| `gemini-flash-image.png` | Test 2 | Gemini 2.0 Flash inline image |
+| `gemini-flash-image-url.txt` | Test 2 | URL fallback |
+| `imagen3-output.png` | Test 3 | Imagen 3 generated image |
+| `imagen3-output-url.txt` | Test 3 | URL fallback |
+| `vision-description.txt` | Test 4 | Gemini description of the Imagen 3 image |
+| `veo2-output.mp4` | Test 5 | Veo 2 generated video |
+| `veo2-output-url.txt` | Test 5 | URL fallback |
+
+### XAI (`outputs/xai/`)
+
+| File | Test | Description |
+|------|------|-------------|
+| `aurora-output.png` | Test 1 | Aurora generated image |
+| `aurora-output-url.txt` | Test 1 | URL fallback |
+| `vision-description.txt` | Test 2 | Grok-2-Vision description of Aurora image |
+
+---
+
+## Test Dependency Order
+
+Several tests depend on output from a previous test **within the same file**. Always run tests sequentially (`--runInBand`):
+
+- **OpenAI**: Test 2 (Whisper) needs `tts-output.mp3` from Test 1
+- **OpenAI**: Test 4 (Vision) needs `dalle3-output.png` from Test 3
+- **Google**: Test 4 (Vision) needs `imagen3-output.png` from Test 3
+- **XAI**: Test 2 (Vision) needs `aurora-output.png` from Test 1
+
+If a dependency file is missing, the dependent test will be skipped with a clear log message.
+
+---
+
+## Provider Notes
+
+### OpenAI
+
+- **TTS models**: `tts-1` (faster, lower quality) and `tts-1-hd` (higher quality). Voices: `alloy`, `echo`, `fable`, `onyx`, `nova`, `shimmer`.
+- **Whisper**: Supports `mp3`, `mp4`, `mpeg`, `mpga`, `m4a`, `wav`, `webm`. Response formats: `json`, `text`, `srt`, `verbose_json`, `vtt`.
+- **DALL-E 3**: Sizes `1024x1024`, `1792x1024`, `1024x1792`. Quality: `standard` or `hd`.
+- **Sora**: Not yet implemented in the client. See Test 5 for the intended API.
+
+### Google
+
+- **Gemini TTS**: Uses `gemini-2.0-flash-preview-tts` model. Supports multi-speaker synthesis.
+- **Gemini 2.0 Flash image**: Native inline image generation using the `gemini-2.0-flash-preview-image-generation` model.
+- **Imagen 3**: High-quality image generation via the Vertex-style Gemini API.
+- **Veo 2**: Video generation via `veo-2.0-generate-001`. Generation is asynchronous and polls for completion — allow up to 5 minutes.
+
+### XAI
+
+- **Aurora**: XAI's image generation model. Uses the OpenAI-compatible `/images/generations` endpoint.
+- **Grok-2-Vision**: Vision model for image understanding (`grok-2-vision-1212`).
+- **Audio**: XAI does not support audio generation or transcription. Tests 3 and 4 verify these throw errors.
+- **Video**: XAI has no public video generation API yet. Test 5 is a documented placeholder.
+
+---
+
+## Adding New Tests
+
+1. Create a new file: `tests/manual/modalities/<provider>.modalities.test.ts`
+2. Save all outputs to `path.join(__dirname, "outputs", "<provider>", "<filename>")`
+3. Guard each test with an API key check and skip gracefully if not set
+4. Add `--runInBand` if tests depend on each other's outputs
+5. Update this README with the new provider and output files
+
+---
+
+## Troubleshooting
+
+**Tests time out**: Increase `--testTimeout`. Video generation (Veo 2) can take 2–5 minutes.
+
+**"API key not set" skip**: Export the relevant environment variable before running.
+
+**Dependency file missing**: Run tests in order with `--runInBand` rather than in parallel.
+
+**TypeScript errors**: Run `npm run compile` from `packages/knowhow/` to check for type issues.