@tyvm/knowhow 0.0.158 → 0.0.160

package/tests/manual/file-edits/patching.test.ts

@@ -1,139 +0,0 @@
-import { readFile, writeFile } from "../../../src/utils";
-import { createPatch, applyPatch } from "diff";
-import { scanFile } from "../../../src/agents/tools";
-import { patchFile } from "../../../src/agents/tools/patch";
-import { agents } from "../../../src/agents";
-import { services } from "../../../src/services";
-import { includedTools } from "../../../src/agents/tools/list";
-import * as allTools from "../../../src/agents/tools";
-
-const inputPath = "tests/integration/patching/input.txt";
-const outputPath = "tests/integration/patching/output.txt";
-const patchPath = "tests/integration/patching/patch.txt";
-
-const brokenPatch = `Index: tests/integration/patching/input.txt
-===================================================================
---- tests/integration/patching/input.txt
-+++ tests/integration/patching/input.txt
-@@ -116,9 +116,9 @@
-         // Add the tool responses to the thread
-         messages.push(
-           ...(toolMessages as Array<ChatCompletionToolMessageParam>)
-         );
--        const finalMessage = toolMessages.find((m) => m.name === "finalAnswer");
-+        const finalMessage = toolMessages.find((m) => m.name === "FinalAnswer");
-         if (finalMessage) {
-           return finalMessage.content;
-         }
-       }\n`;
-
-describe("Patcher", () => {
-  let Agents: any, Tools: any;
-  
-  beforeAll(async () => {
-    const { Patcher } = agents();
-    ({ Agents, Tools } = services());
-    Agents.registerAgent(Patcher);
-    Tools.addTools(includedTools);
-    const toolFunctions = Object.entries(allTools)
-      .filter(([_, value]) => typeof value === "function")
-      .reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {});
-    Tools.addFunctions(toolFunctions);
-  });
-
-  test("should be able to apply actual patch", async () => {
-    const oldString = (await readFile(inputPath)).toString();
-    const newString = (await readFile(outputPath)).toString();
-    const workingPatch = (await readFile(patchPath)).toString();
-
-    const applied = await applyPatch(oldString, patchPath);
-
-    // Expect this not to work
-    expect(applied).toBe(oldString);
-
-    // Expect this to work
-    const workingApply = applyPatch(oldString, workingPatch);
-    expect(workingApply).toBe(newString);
-
-    console.log({
-      brokenPatch: Buffer.from(brokenPatch),
-      workingPatch: Buffer.from(workingPatch),
-    });
-
-    const buffer1 = Buffer.from(brokenPatch);
-    const buffer2 = Buffer.from(workingPatch);
-
-    console.log("bufferLength", buffer1.length, buffer2.length);
-    const length = Math.max(buffer1.length, buffer2.length);
-    for (let i = 0; i < length; i++) {
-      if (buffer1[i] !== buffer2[i]) {
-        console.log("not equal", i, buffer1[i], buffer2[i]);
-      }
-    }
-
-    expect(brokenPatch).toBe(workingPatch);
-  });
-
-  test("should generate the diff", async () => {
-    const oldString = (await readFile(inputPath)).toString();
-    const newString = (await readFile(outputPath)).toString();
-
-    const patch = createPatch(inputPath, oldString, newString);
-    await writeFile(patchPath, patch);
-
-    // const actualPatch = (await readFile(patchPath)).toString();
-
-    console.log(patch);
-
-    const scanned = await scanFile(inputPath, 120, 120);
-    console.log(JSON.parse(scanned));
-
-    expect(patch).toBe(brokenPatch);
-  });
-
-  test("should be able to patch a codebase", async () => {
-    const originalText = (await readFile(inputPath)).toString();
-    console.log(
-      await Agents.callAgent(
-        "Patcher",
-        `Update the file in ${inputPath} change the string "finalAnswer" to "FinalAnswer", as we are changing the tool name to be FinalAnswer instead of finalAnswer. This should be around line 120. Do not modify anything else. Treat this file as a typescript file. Only try one patch. If it fails stop`
-      )
-    );
-
-    const updatedText = (await readFile(inputPath)).toString();
-    await writeFile(inputPath, originalText);
-
-    expect(updatedText).toBe(
-      originalText.replace(`"finalAnswer"`, `"FinalAnswer"`)
-    );
-  });
-
-  it("should be able to patch when given the answer", async () => {
-    const originalText = (await readFile(inputPath)).toString();
-    await Agents.callAgent(
-      "Patcher",
-      `Update the file in ${inputPath} change the string "finalAnswer" to "FinalAnswer" on line 120. Do not modify anything else. Heres the answer: ${brokenPatch}.`
-    );
-
-    const updatedText = (await readFile(inputPath)).toString();
-    await writeFile(inputPath, originalText);
-
-    expect(updatedText).toBe(
-      originalText.replace(`"finalAnswer"`, `"FinalAnswer"`)
-    );
-  });
-
-  test("should be able to update the unseen file", async () => {
-    const unseenPath = "tests/integration/patching/unseen.txt";
-    const originalText = (await readFile(unseenPath)).toString();
-    await Agents.callAgent(
-      "Patcher",
-      `Update the file in ${unseenPath} change the file additions and deletions count from 200 to 300. Do not modify anything else. Treat this file as a typescript file.`
-    );
-
-    const updatedText = (await readFile(unseenPath)).toString();
-    await writeFile(unseenPath, originalText);
-
-    expect(updatedText).toBe(originalText.replaceAll("200", "300"));
-  });
-});

package/tests/manual/modalities/README.md

@@ -1,157 +0,0 @@
-# Modalities Manual Tests
-
-Manual integration tests for audio, image, vision, and video generation across all supported AI providers.
-
-## Overview
-
-These tests exercise every output/input modality supported by each provider:
-
-| Modality             | OpenAI       | Google             | XAI          |
-|----------------------|--------------|--------------------|--------------|
-| Audio Generation     | TTS-1 / TTS-1-HD | Gemini 2.0 Flash TTS | ❌ Not supported |
-| Audio Transcription  | Whisper-1    | ❌ Not supported   | ❌ Not supported |
-| Image Generation     | DALL-E 3     | Imagen 3 / Gemini 2.0 Flash | Aurora |
-| Vision (image input) | GPT-4o       | Gemini 2.0 Flash   | Grok-2-Vision |
-| Video Generation     | Sora (stub)  | Veo 2              | ❌ Not supported |
-
----
-
-## Prerequisites
-
-Set the appropriate API keys in your environment before running tests:
-
-```bash
-export OPENAI_KEY="sk-..."
-export GEMINI_API_KEY="AIza..."
-export XAI_API_KEY="xai-..."
-```
-
----
-
-## Running the Tests
-
-### Run all modality tests
-
-```bash
-npx jest tests/manual/modalities --testTimeout=300000 --runInBand
-```
-
-> **`--runInBand`** is recommended so tests within a file run sequentially (some tests depend on outputs from prior tests in the same file).
-
-### Run a single provider
-
-```bash
-# OpenAI only
-npx jest tests/manual/modalities/openai.modalities.test.ts --testTimeout=120000 --runInBand
-
-# Google only
-npx jest tests/manual/modalities/google.modalities.test.ts --testTimeout=300000 --runInBand
-
-# XAI only
-npx jest tests/manual/modalities/xai.modalities.test.ts --testTimeout=120000 --runInBand
-```
-
-### Run a single test by name
-
-```bash
-npx jest tests/manual/modalities/openai.modalities.test.ts \
-  --testNamePattern="DALL-E 3" \
-  --testTimeout=60000
-```
-
----
-
-## Output Files
-
-All generated artifacts are saved under `tests/manual/modalities/outputs/<provider>/` so they can be reviewed after the tests run.
-
-### OpenAI (`outputs/openai/`)
-
-| File | Test | Description |
-|------|------|-------------|
-| `tts-output.mp3` | Test 1 | TTS-1 generated speech audio |
-| `transcription.json` | Test 2 | Whisper transcription of the TTS audio |
-| `dalle3-output.png` | Test 3 | DALL-E 3 generated image |
-| `dalle3-output-url.txt` | Test 3 | URL fallback if b64_json not returned |
-| `vision-description.txt` | Test 4 | GPT-4o description of the DALL-E image |
-
-### Google (`outputs/google/`)
-
-| File | Test | Description |
-|------|------|-------------|
-| `tts-output.wav` | Test 1 | Gemini 2.0 Flash TTS audio |
-| `gemini-flash-image.png` | Test 2 | Gemini 2.0 Flash inline image |
-| `gemini-flash-image-url.txt` | Test 2 | URL fallback |
-| `imagen3-output.png` | Test 3 | Imagen 3 generated image |
-| `imagen3-output-url.txt` | Test 3 | URL fallback |
-| `vision-description.txt` | Test 4 | Gemini description of the Imagen 3 image |
-| `veo2-output.mp4` | Test 5 | Veo 2 generated video |
-| `veo2-output-url.txt` | Test 5 | URL fallback |
-
-### XAI (`outputs/xai/`)
-
-| File | Test | Description |
-|------|------|-------------|
-| `aurora-output.png` | Test 1 | Aurora generated image |
-| `aurora-output-url.txt` | Test 1 | URL fallback |
-| `vision-description.txt` | Test 2 | Grok-2-Vision description of Aurora image |
-
----
-
-## Test Dependency Order
-
-Several tests depend on output from a previous test **within the same file**. Always run tests sequentially (`--runInBand`):
-
-- **OpenAI**: Test 2 (Whisper) needs `tts-output.mp3` from Test 1
-- **OpenAI**: Test 4 (Vision) needs `dalle3-output.png` from Test 3
-- **Google**: Test 4 (Vision) needs `imagen3-output.png` from Test 3
-- **XAI**: Test 2 (Vision) needs `aurora-output.png` from Test 1
-
-If a dependency file is missing, the dependent test will be skipped with a clear log message.
-
----
-
-## Provider Notes
-
-### OpenAI
-
-- **TTS models**: `tts-1` (faster, lower quality) and `tts-1-hd` (higher quality). Voices: `alloy`, `echo`, `fable`, `onyx`, `nova`, `shimmer`.
-- **Whisper**: Supports `mp3`, `mp4`, `mpeg`, `mpga`, `m4a`, `wav`, `webm`. Response formats: `json`, `text`, `srt`, `verbose_json`, `vtt`.
-- **DALL-E 3**: Sizes `1024x1024`, `1792x1024`, `1024x1792`. Quality: `standard` or `hd`.
-- **Sora**: Not yet implemented in the client. See Test 5 for the intended API.
-
-### Google
-
-- **Gemini TTS**: Uses `gemini-2.0-flash-preview-tts` model. Supports multi-speaker synthesis.
-- **Gemini 2.0 Flash image**: Native inline image generation using the `gemini-2.0-flash-preview-image-generation` model.
-- **Imagen 3**: High-quality image generation via the Vertex-style Gemini API.
-- **Veo 2**: Video generation via `veo-2.0-generate-001`. Generation is asynchronous and polls for completion — allow up to 5 minutes.
-
-### XAI
-
-- **Aurora**: XAI's image generation model. Uses the OpenAI-compatible `/images/generations` endpoint.
-- **Grok-2-Vision**: Vision model for image understanding (`grok-2-vision-1212`).
-- **Audio**: XAI does not support audio generation or transcription. Tests 3 and 4 verify these throw errors.
-- **Video**: XAI has no public video generation API yet. Test 5 is a documented placeholder.
-
----
-
-## Adding New Tests
-
-1. Create a new file: `tests/manual/modalities/<provider>.modalities.test.ts`
-2. Save all outputs to `path.join(__dirname, "outputs", "<provider>", "<filename>")`
-3. Guard each test with an API key check and skip gracefully if not set
-4. Add `--runInBand` if tests depend on each other's outputs
-5. Update this README with the new provider and output files
-
----
-
-## Troubleshooting
-
-**Tests time out**: Increase `--testTimeout`. Video generation (Veo 2) can take 2–5 minutes.
-
-**"API key not set" skip**: Export the relevant environment variable before running.
-
-**Dependency file missing**: Run tests in order with `--runInBand` rather than in parallel.
-
-**TypeScript errors**: Run `npm run compile` from `packages/knowhow/` to check for type issues.

package/tests/manual/modalities/google.modalities.test.ts

@@ -1,335 +0,0 @@
-/**
- * Google (Gemini) Modalities Manual Test
- *
- * Tests:
- *  1. Audio generation (Gemini TTS) → saved to disk
- *  2. Image generation (Gemini 2.0 Flash inline) → saved to disk
- *  3. Image generation (Imagen 3) → saved to disk
- *  4. Send generated image to Gemini vision model and describe it
- *  5. Video generation (Veo 2) → saved to disk
- *
- * Run with:
- *   npx jest tests/manual/modalities/google.modalities.test.ts --testTimeout=300000
- */
-
-import * as fs from "fs";
-import * as path from "path";
-import { AIClient } from "../../../src/clients";
-import { Models } from "../../../src/types";
-
-const OUTPUT_DIR = path.join(__dirname, "outputs", "google");
-
-function ensureOutputDir() {
-  fs.mkdirSync(OUTPUT_DIR, { recursive: true });
-}
-
-describe("Google (Gemini) Modalities", () => {
-  let client: AIClient;
-
-  beforeAll(() => {
-    if (!process.env.GEMINI_API_KEY) {
-      console.warn("GEMINI_API_KEY not set – skipping Google modality tests");
-    }
-    ensureOutputDir();
-    client = new AIClient();
-  });
-
-  // ─── 1. Audio Generation (Gemini TTS) ───────────────────────────────────────
-
-  test("1. Gemini TTS – generate audio and save to disk", async () => {
-    if (!process.env.GEMINI_API_KEY) {
-      console.log("Skipping: GEMINI_API_KEY not set");
-      return;
-    }
-
-    const outputPath = path.join(OUTPUT_DIR, "tts-output.wav");
-    if (fs.existsSync(outputPath)) {
-      console.log(`⏭️  Skipping: ${outputPath} already exists`);
-      return;
-    }
-
-    const text =
-      "Hello! This is a test of the Google Gemini text-to-speech system. " +
-      "We are verifying that Gemini audio generation is working correctly.";
-
-    const response = await client.createAudioGeneration("google", {
-      model: Models.google.Gemini_25_Flash_TTS,
-      input: text,
-      voice: "Puck",
-    });
-
-    fs.writeFileSync(outputPath, response.audio);
-
-    console.log(`✅ Audio saved to: ${outputPath}`);
-    console.log(`   Format: ${response.format}`);
-    console.log(`   Size: ${response.audio.length} bytes`);
-
-    expect(response.audio).toBeInstanceOf(Buffer);
-    expect(response.audio.length).toBeGreaterThan(0);
-    expect(fs.existsSync(outputPath)).toBe(true);
-  }, 60000);
-
-  // ─── 2. Image Generation (Gemini 2.0 Flash inline) ──────────────────────────
-
-  test("2. Gemini 2.0 Flash – inline image generation and save to disk", async () => {
-    if (!process.env.GEMINI_API_KEY) {
-      console.log("Skipping: GEMINI_API_KEY not set");
-      return;
-    }
-
-    const outputPath = path.join(OUTPUT_DIR, "gemini-flash-image.png");
-    if (fs.existsSync(outputPath)) {
-      console.log(`⏭️  Skipping: ${outputPath} already exists`);
-      return;
-    }
-
-    const prompt =
-      "A watercolor painting of a serene mountain lake at sunrise, " +
-      "with reflections of pine trees in the calm water";
-
-    const response = await client.createImageGeneration("google", {
-      model: Models.google.Gemini_25_Flash_Image,
-      prompt,
-      n: 1,
-    });
-
-    expect(response.data.length).toBeGreaterThan(0);
-
-    const imageData = response.data[0];
-
-    if (imageData.b64_json) {
-      const buffer = Buffer.from(imageData.b64_json, "base64");
-      fs.writeFileSync(outputPath, buffer);
-      console.log(`✅ Gemini Flash image saved to: ${outputPath}`);
-      console.log(`   Size: ${buffer.length} bytes`);
-    } else if (imageData.url) {
-      fs.writeFileSync(
-        path.join(OUTPUT_DIR, "gemini-flash-image-url.txt"),
-        imageData.url
-      );
-      console.log(`✅ Gemini Flash image URL saved`);
-    }
-
-    console.log(`   Estimated cost: $${response.usd_cost?.toFixed(6)}`);
-
-    expect(response.created).toBeGreaterThan(0);
-    expect(response.data.length).toBeGreaterThan(0);
-  }, 90000);
-
-  // ─── 3. Image Generation (Imagen 3) ─────────────────────────────────────────
-
-  test("3. Imagen 3 – generate image and save to disk", async () => {
-    if (!process.env.GEMINI_API_KEY) {
-      console.log("Skipping: GEMINI_API_KEY not set");
-      return;
-    }
-
-    const outputPath = path.join(OUTPUT_DIR, "imagen3-output.png");
-    if (fs.existsSync(outputPath)) {
-      console.log(`⏭️  Skipping: ${outputPath} already exists`);
-      return;
-    }
-
-    const prompt =
-      "A photorealistic close-up of a red rose with dewdrops, " +
-      "dramatic lighting, shallow depth of field, professional photography";
-
-    const response = await client.createImageGeneration("google", {
-      model: Models.google.Imagen_3,
-      prompt,
-      n: 1,
-    });
-
-    expect(response.data.length).toBeGreaterThan(0);
-
-    const imageData = response.data[0];
-
-    if (imageData.b64_json) {
-      console.log(imageData.b64_json.slice(0, 100) + "..."); // Log the beginning of the base64 string for debugging
-      const buffer = Buffer.from(imageData.b64_json, "base64");
-      fs.writeFileSync(outputPath, buffer);
-      console.log(`✅ Imagen 3 image saved to: ${outputPath}`);
-      console.log(`   Size: ${buffer.length} bytes`);
-    } else if (imageData.url) {
-      fs.writeFileSync(
-        path.join(OUTPUT_DIR, "imagen3-output-url.txt"),
-        imageData.url
-      );
-      console.log(`✅ Imagen 3 image URL saved`);
-    }
-
-    console.log(`   Estimated cost: $${response.usd_cost?.toFixed(6)}`);
-
-    expect(response.created).toBeGreaterThan(0);
-  }, 90000);
-
-  // ─── 4. Vision – send generated image back to Gemini ────────────────────────
-
-  test("4. Gemini Vision – describe the generated Imagen 3 image", async () => {
-    if (!process.env.GEMINI_API_KEY) {
-      console.log("Skipping: GEMINI_API_KEY not set");
-      return;
-    }
-
-    const outputPath = path.join(OUTPUT_DIR, "vision-description.txt");
-    if (fs.existsSync(outputPath)) {
-      console.log(`⏭️  Skipping: ${outputPath} already exists`);
-      return;
-    }
-
-    const imagePath = path.join(OUTPUT_DIR, "imagen3-output.png");
-    if (!fs.existsSync(imagePath)) {
-      console.log(
-        "Skipping: imagen3-output.png not found – run Imagen 3 test first"
-      );
-      return;
-    }
-
-    const imageBuffer = fs.readFileSync(imagePath);
-    const base64Image = imageBuffer.toString("base64");
-    const dataUrl = `data:image/png;base64,${base64Image}`;
-
-    const response = await client.createCompletion("google", {
-      model: Models.google.Gemini_25_Flash,
-      messages: [
-        {
-          role: "user",
-          content: [
-            {
-              type: "text",
-              text: "Please describe this image in detail. What do you see?",
-            },
-            {
-              type: "image_url",
-              image_url: { url: dataUrl },
-            },
-          ],
-        },
-      ],
-      max_tokens: 500,
-    });
-
-    const description = response.choices[0]?.message?.content || "";
-    fs.writeFileSync(outputPath, description);
-
-    console.log(`✅ Vision description saved to: ${outputPath}`);
-    console.log(`   Description: ${description.substring(0, 200)}...`);
-    console.log(`   Estimated cost: $${response.usd_cost?.toFixed(6)}`);
-
-    expect(description).toBeTruthy();
-    expect(description.length).toBeGreaterThan(10);
-  }, 60000);
-
-  // ─── 5. Video Generation (Veo 3.1) ──────────────────────────────────────────
-
-  test("5. Google Veo 3.1 – generate video and save to disk", async () => {
-    if (!process.env.GEMINI_API_KEY) {
-      console.log("Skipping: GEMINI_API_KEY not set");
-      return;
-    }
-
-    const outputPath = path.join(OUTPUT_DIR, "veo31-output.mp4");
-    const jobNamePath = path.join(OUTPUT_DIR, "veo31-job-name.txt");
-
-    // If final output already exists, skip entirely
-    if (fs.existsSync(outputPath)) {
-      console.log(`⏭️  Skipping: ${outputPath} already exists`);
-      return;
-    }
-
-    // Helper: poll a known jobId until done, then download and save
-    async function pollAndDownload(jobId: string) {
-      const maxPollingTime = 20 * 60 * 1000; // 20 minutes
-      const pollingInterval = 10000; // 10 seconds
-      const startTime = Date.now();
-
-      console.log(`⏳ Polling job: ${jobId}`);
-
-      while (Date.now() - startTime < maxPollingTime) {
-        await new Promise((resolve) => setTimeout(resolve, pollingInterval));
-
-        const status = await client.getVideoStatus("google", { jobId });
-        console.log(`   Status: ${status.status}`);
-
-        if (status.status === "failed") {
-          throw new Error(`Veo video generation failed: ${status.error}`);
-        }
-
-        if (status.status === "completed") {
-          // Get the URI from the status response
-          const videoData = status.data?.[0];
-          const uri = videoData?.url || videoData?.fileUri;
-
-          if (!uri) {
-            throw new Error("No video URI in completed status response");
-          }
-
-          console.log(`   Video URI: ${uri}`);
-          console.log(`⏳ Downloading video via Files API...`);
-
-          // Use the client's downloadFile method, passing filePath so the SDK
-          // writes directly to the destination (no extra read/write cycle).
-          const downloaded = await client.downloadFile("google", {
-            fileId: uri,
-            uri,
-            filePath: outputPath,
-          });
-
-          console.log(
-            `✅ Veo 3.1 video saved to: ${outputPath} (${downloaded.data.length} bytes)`
-          );
-          console.log(`   MIME type: ${downloaded.mimeType}`);
-
-          // Clean up job name file
-          if (fs.existsSync(jobNamePath)) fs.unlinkSync(jobNamePath);
-          return;
-        }
-        // otherwise status is "in_progress" or "queued" – keep polling
-      }
-
-      throw new Error(
-        "Veo 3.1 video generation timed out after 20 minutes of polling"
-      );
-    }
-
-    // If we already have a job ID from a previous (timed-out) run, resume polling
-    if (fs.existsSync(jobNamePath)) {
-      const jobId = fs.readFileSync(jobNamePath, "utf8").trim();
-      console.log(`🔄 Resuming poll for existing job: ${jobId}`);
-      await pollAndDownload(jobId);
-      expect(fs.existsSync(outputPath)).toBe(true);
-      return;
-    }
-
-    // Otherwise start a new video generation job
-    const prompt =
-      "A timelapse of clouds moving over a mountain range at golden hour, " +
-      "cinematic quality, smooth motion";
-
-    console.log("⏳ Submitting Veo 3.1 video generation job...");
-
-    const response = await client.createVideoGeneration("google", {
-      model: Models.google.Veo_3_1,
-      prompt,
-      n: 1,
-      duration: 6,
-      aspect_ratio: "16:9",
-    });
-
-    const jobId = response.jobId;
-    if (!jobId) {
-      throw new Error(
-        `No jobId returned from video generation: ${JSON.stringify(response)}`
-      );
-    }
-
-    // Persist the job ID so subsequent runs can resume if this run times out
-    fs.writeFileSync(jobNamePath, jobId);
-    console.log(`📝 Job ID saved to: ${jobNamePath}`);
-    console.log(`   Job ID: ${jobId}`);
-
-    await pollAndDownload(jobId);
-
-    expect(fs.existsSync(outputPath)).toBe(true);
-  }, 1500000); // 25 minute timeout
-});