@tyvm/knowhow 0.0.159 → 0.0.161

package/tests/manual/clients/completions.test.ts

@@ -1,198 +0,0 @@
-/**
- * Text Completions Manual Test
- *
- * Tests text generation ("say hello in 5 words") across multiple models and providers.
- * Results are persisted to `completions.json` so that re-running the suite skips
- * models that have already been tested (avoiding unnecessary API spend).
- *
- * Run with:
- *   npx jest tests/manual/clients/completions.test.ts --testTimeout=60000 --runInBand
- */
-
-import * as fs from "fs";
-import * as path from "path";
-import { AIClient } from "../../../src/clients";
-import { Models } from "../../../src/types";
-import { OpenAiChatModels, OpenAiReasoningModels, OpenAiResponsesOnlyModels } from "../../../src/clients/pricing/openai";
-import { AnthropicTextModels, AnthropicLimitedAvailabilityModels } from "../../../src/clients/pricing/anthropic";
-import { GoogleTextModels } from "../../../src/clients/pricing/google";
-import { XaiTextModels, XaiDeprecatedTextModels } from "../../../src/clients/pricing/xai";
-import { FireworksTextPricing } from "../../../src/clients/pricing/fireworks";
-
-const OUTPUT_FILE = path.join(__dirname, "completions.json");
-const PROMPT = "Say hello in 5 words.";
-
-// ─── Models to test ──────────────────────────────────────────────────────────
-// Derived from the canonical model constant lists per provider.
-// Only active (non-deprecated) models are included.
-
-interface ModelEntry {
-  provider: string;
-  model: string;
-  envKey: string;
-  reasoningEffort?: "low";
-}
-
-function openaiEntries(models: string[]): ModelEntry[] {
-  return models.map((model) => ({ provider: "openai", model, envKey: "OPENAI_KEY" }));
-}
-
-function anthropicEntries(models: string[]): ModelEntry[] {
-  return models.map((model) => ({ provider: "anthropic", model, envKey: "ANTHROPIC_API_KEY" }));
-}
-
-function googleEntries(models: string[]): ModelEntry[] {
-  return models.map((model) => ({ provider: "google", model, envKey: "GEMINI_API_KEY" }));
-}
-
-function xaiEntries(models: string[]): ModelEntry[] {
-  return models.map((model) => ({ provider: "xai", model, envKey: "XAI_API_KEY" }));
-}
-
-function fireworksEntries(): ModelEntry[] {
-  return Object.entries(FireworksTextPricing)
-    .filter(([, pricing]) => !pricing.deprecated)
-    .map(([model]) => ({
-      provider: "fireworks",
-      model,
-      envKey: "FIREWORKS_API_KEY",
-      // Fireworks model support for reasoning_effort is model-specific. This
-      // suite exercises the request shape used by Patcher without that flag.
-    }));
-}
-
-const TEST_MODELS: ModelEntry[] = [
-  // OpenAI — all active chat + reasoning models + responses-API models
-  // Deduplicate since some models appear in multiple lists
-  ...openaiEntries([...new Set([...OpenAiChatModels, ...OpenAiReasoningModels, ...OpenAiResponsesOnlyModels])])
-    .map((entry) => ({ ...entry, reasoningEffort: "low" as const })),
-
-  // Anthropic — active text models (limited availability excluded)
-  ...anthropicEntries(AnthropicTextModels).map((entry) => ({ ...entry, reasoningEffort: "low" as const })),
-
-  // Google — all active text models
-  ...googleEntries(GoogleTextModels).map((entry) => ({ ...entry, reasoningEffort: "low" as const })),
-
-  // XAI — active text models (deprecated grok-2 models excluded)
-  ...xaiEntries(XaiTextModels).map((entry) => ({ ...entry, reasoningEffort: "low" as const })),
-
-  // Fireworks — all active catalog models
-  ...fireworksEntries(),
-];
-
-// ─── Persistence helpers ──────────────────────────────────────────────────────
-
-interface CompletionRecord {
-  provider: string;
-  model: string;
-  prompt: string;
-  response: string;
-  usage: {
-    prompt_tokens?: number;
-    completion_tokens?: number;
-    total_tokens?: number;
-    [key: string]: any;
-  };
-  usd_cost?: number;
-  durationMs?: number;
-  testedAt: string;
-}
-
-type CompletionsJSON = Record<string, CompletionRecord>;
-
-function loadResults(): CompletionsJSON {
-  if (fs.existsSync(OUTPUT_FILE)) {
-    try {
-      return JSON.parse(fs.readFileSync(OUTPUT_FILE, "utf-8")) as CompletionsJSON;
-    } catch {
-      return {};
-    }
-  }
-  return {};
-}
-
-function saveResults(results: CompletionsJSON): void {
-  fs.writeFileSync(OUTPUT_FILE, JSON.stringify(results, null, 2), "utf-8");
-}
-
-function recordKey(provider: string, model: string): string {
-  return `${provider}::${model}`;
-}
-
-// ─── Test suite ───────────────────────────────────────────────────────────────
-
-describe("Text Completions – multi-provider model verification", () => {
-  let client: AIClient;
-  let results: CompletionsJSON;
-
-  beforeAll(() => {
-    client = new AIClient();
-    results = loadResults();
-  });
-
-  afterAll(() => {
-    saveResults(results);
-    console.log(`\n📄 Results saved to: ${OUTPUT_FILE}`);
-    console.log(`   Total records: ${Object.keys(results).length}`);
-  });
-
-  for (const { provider, model, envKey, reasoningEffort } of TEST_MODELS) {
-    const key = recordKey(provider, model);
-
-    test(`${provider} / ${model}`, async () => {
-      // ── Skip if already recorded ──────────────────────────────────────────
-      if (results[key]) {
-        const rec = results[key];
-        console.log(`⏭  Skipping (already tested on ${rec.testedAt})`);
-        console.log(`   Response : "${rec.response}"`);
-        console.log(`   Cost     : $${rec.usd_cost?.toFixed(8) ?? "unknown"}`);
-        console.log(`   Duration : ${rec.durationMs != null ? rec.durationMs + "ms" : "unknown"}`);
-        expect(rec.response).toBeTruthy();
-        return;
-      }
-
-      // ── Skip if API key missing ───────────────────────────────────────────
-      if (!process.env[envKey]) {
-        console.log(`⏭  Skipping: ${envKey} not set`);
-        return;
-      }
-
-      // ── Run completion ────────────────────────────────────────────────────
-      const startMs = Date.now();
-      const response = await client.createCompletion(provider, {
-        model,
-        messages: [{ role: "user", content: PROMPT }],
-        max_tokens: 2000,
-        // Exercise the summary option enabled by Patcher. Provider clients must
-        // translate or remove this internal option rather than forwarding it.
-        reasoning_summary: true,
-        reasoning_effort: reasoningEffort,
-      });
-      const durationMs = Date.now() - startMs;
-
-      const text = response.choices[0]?.message?.content ?? "";
-      expect(text).toBeTruthy();
-
-      const record: CompletionRecord = {
-        provider,
-        model,
-        prompt: PROMPT,
-        response: text,
-        usage: response.usage ?? {},
-        durationMs,
-        usd_cost: response.usd_cost,
-        testedAt: new Date().toISOString(),
-      };
-
-      results[key] = record;
-      // Persist immediately so a mid-run failure doesn't lose earlier results
-      saveResults(results);
-
-      console.log(`✅ ${provider} / ${model}`);
-      console.log(`   Response : "${text}"`);
-      console.log(`   Tokens   : ${JSON.stringify(response.usage)}`);
-      console.log(`   Cost     : $${response.usd_cost?.toFixed(8) ?? "unknown"}`);
-      console.log(`   Duration : ${durationMs}ms`);
-    }, 60_000);
-  }
-});

package/tests/manual/file-edits/figma.test.ts

@@ -1,3 +0,0 @@
-// FigmaPlugin tests moved to @tyvm/knowhow-plugin-figma package
-// Install and configure @tyvm/knowhow-plugin-figma to use Figma integration
-describe.skip("FigmaPlugin (moved to @tyvm/knowhow-plugin-figma)", () => {});

package/tests/manual/file-edits/fileblocks/readwrite.test.ts

@@ -1,100 +0,0 @@
-import * as fs from "fs";
-import { readFile, writeFile } from "../../../../src/utils";
-import { agents } from "../../../../src/agents";
-import { services } from "../../../../src/services";
-import { includedTools } from "../../../../src/agents/tools/list";
-import { FlagsService } from "../../../../src/services/flags";
-
-describe("Developer", () => {
-  const { Patcher } = agents();
-  beforeAll(() => {
-    const { Tools } = services();
-    Tools.addTools(includedTools);
-    Patcher.disableTool("searchFiles");
-    Patcher.disableTool("execCommand");
-    Patcher.enableTool("modifyFile");
-    Patcher.enableTool("readFile");
-  });
-
-  test("should be able to patch a codebase", async () => {
-    const inputPath = "tests/integration/patching/input.txt";
-    const originalText = (await readFile(inputPath)).toString();
-    await Patcher.call(
-      `Update the file in ${inputPath} change the string "finalAnswer" to "FinalAnswer" on line 120. Do not modify anything else. Treat this file as a typescript file.`
-    );
-
-    const updatedText = (await readFile(inputPath)).toString();
-    await writeFile(inputPath, originalText);
-
-    expect(updatedText).toBe(
-      originalText.replace(`"finalAnswer"`, `"FinalAnswer"`)
-    );
-  });
-
-  test("should be able to update the unseen file", async () => {
-    const unseenPath = "tests/integration/patching/unseen.txt";
-    const originalText = (await readFile(unseenPath)).toString();
-    await Patcher.call(
-      `Update the file in ${unseenPath} change the file additions and deletions count from 200 to 300. Do not modify anything else. Treat this file as a typescript file.`
-    );
-
-    const updatedText = (await readFile(unseenPath)).toString();
-    await writeFile(unseenPath, originalText);
-
-    expect(updatedText).toBe(originalText.replaceAll("200", "300"));
-  });
-
-  test.only("should be able to create a simple flag service", async () => {
-    const outputPath = "tests/integration/fileblocks/outputs/flags.js";
-
-    const promptText = `
-  Create the following class
-
-    - **Class Name**: FlagsService
-    - **Internal Variables**:
-      - flags: Object
-      - log: Boolean
-    - **Constructor**:
-      - Parameters: flags (Array<string>), log (Boolean)
-    - **Methods**:
-      - register(flags: Array<string>): Initializes flags with false.
-      - flip(flag: string): Toggles and logs the state of a flag.
-      - enableLogging(): Enables logging.
-      - disableLogging(): Disables logging.
-      - enabled(flag: string): Checks if a flag is enabled.
-    - **Singleton Variable**: Flags
-
-      Make sure to module.exports the class and Singleton, as we will be importing it in the test via require("./outputs/flags.js").Flags
-`;
-    await Patcher.call(
-      `Modify the file in ${outputPath} for these instructions.: ${promptText}`
-    );
-
-    const fileContent = (await readFile(outputPath)).toString();
-    console.log(fileContent);
-    jest.resetModules();
-    const service: FlagsService = require("./outputs/flags.js").Flags;
-    try {
-      service.register(["test"]);
-      expect(service.enabled("test")).toEqual(false);
-      service.flip("test");
-      expect(service.enabled("test")).toEqual(true);
-
-      await Patcher.call(
-        `Modify the file in ${outputPath} and add a method called hasFlag that takes a string and returns a boolean. It should return true if the flag is defined at all, even if it is false. This is a unit test, only modify the file I've specified.`
-      );
-
-      jest.resetModules();
-
-      const newService: FlagsService & {
-        hasFlag: (flag: string) => boolean;
-      } = require("./outputs/flags.js").Flags;
-
-      newService.register(["test"]);
-      expect(newService.hasFlag("test")).toEqual(true);
-      fs.unlinkSync(outputPath);
-    } catch (e) {
-      console.log(e);
-    }
-  });
-});

package/tests/manual/file-edits/patching/input.txt

@@ -1,145 +0,0 @@
-import {
-  ChatCompletionMessageParam,
-  ChatCompletionToolMessageParam,
-  ChatCompletionMessageToolCall,
-} from "openai/resources/chat";
-import { openai } from "../../ai";
-import {
-  searchFiles,
-  scanFile,
-  writeFile,
-  applyPatchFile,
-  execCommand,
-  readFile,
-  finalAnswer,
-  addInternalTools,
-  callPlugin,
-  testPatch,
-} from "../tools";
-import { Tools } from "../tools/list";
-
-const availableFunctions = addInternalTools({
-  testPatch,
-  applyPatchFile,
-  callPlugin,
-  execCommand,
-  finalAnswer,
-  readFile,
-  scanFile,
-  searchFiles,
-  writeFile,
-});
-
-export class CodebaseAgent {
-  getInitialMessages(user_input: string) {
-    return [
-      {
-        role: "system",
-        content:
-          "Codebase Agent. You use the tools to read and write code, to help the developer implement features faster. Call final answer once you have finished implementing what is requested. As an agent you will receive multiple rounds of input until you call final answer. You are not able to request feedback from the user, so proceed with your plans and the developer will contact you afterwards if they need more help.",
-      },
-
-      { role: "user", content: user_input },
-    ] as Array<ChatCompletionMessageParam>;
-  }
-
-  async getToolMessages(toolCall: ChatCompletionMessageToolCall) {
-    const functionName = toolCall.function.name;
-    const functionToCall = availableFunctions[functionName];
-    const functionArgs = JSON.parse(toolCall.function.arguments);
-    const positionalArgs = Object.values(functionArgs);
-
-    console.log(
-      `Calling function ${functionName} with args:`,
-      JSON.stringify(positionalArgs, null, 2)
-    );
-
-    const functionResponse = await functionToCall(...positionalArgs);
-
-    if (functionName === "multi_tool_use.parallel") {
-      const args = positionalArgs[0] as {
-        recipient_name: string;
-        parameters: any;
-      }[];
-
-      return args.map((call, index) => {
-        return {
-          tool_call_id: toolCall.id + "_" + index,
-          role: "tool",
-          name: call.recipient_name.split(".").pop(),
-          content: functionResponse[index] || "Done",
-        };
-      });
-    }
-
-    const toolMessage = {
-      tool_call_id: toolCall.id,
-      role: "tool",
-      name: functionName,
-      content: functionResponse || "Done",
-    };
-
-    return [toolMessage];
-  }
-
-  logMessages(messages: Array<ChatCompletionMessageParam>) {
-    for (const message of messages) {
-      if (message.role == "assistant") {
-        console.log(message.content);
-      }
-    }
-  }
-
-  async call(
-    user_input: string,
-    _messages?: Array<ChatCompletionMessageParam>
-  ) {
-    const model = "gpt-4-1106-preview";
-    const messages = _messages || this.getInitialMessages(user_input);
-
-    const response = await openai.chat.completions.create({
-      model,
-      messages: messages,
-      tools: Tools,
-      tool_choice: "auto",
-    });
-    this.logMessages(response.choices.map((c) => c.message));
-    const responseMessage = response.choices[0].message;
-
-    const toolCalls = responseMessage.tool_calls;
-    if (responseMessage.tool_calls) {
-      // extend conversation with assistant's reply
-      messages.push(responseMessage);
-
-      for (const toolCall of toolCalls) {
-        const toolMessages = await this.getToolMessages(toolCall);
-        // Add the tool responses to the thread
-        messages.push(
-          ...(toolMessages as Array<ChatCompletionToolMessageParam>)
-        );
-        const finalMessage = toolMessages.find((m) => m.name === "finalAnswer");
-        if (finalMessage) {
-          return finalMessage.content;
-        }
-      }
-
-      // Send the tool responses back to the model
-      const secondResponse = await openai.chat.completions.create({
-        model,
-        messages: messages,
-      });
-
-      const aiResp = secondResponse.choices.map((c) => c.message);
-      this.logMessages(aiResp);
-      messages.push(...aiResp);
-
-      return this.call(user_input, messages);
-    }
-
-    if (responseMessage.content) {
-      return responseMessage.content;
-    }
-  }
-}
-
-export const Developer = new CodebaseAgent();

package/tests/manual/file-edits/patching/output.txt

@@ -1,145 +0,0 @@
-import {
-  ChatCompletionMessageParam,
-  ChatCompletionToolMessageParam,
-  ChatCompletionMessageToolCall,
-} from "openai/resources/chat";
-import { openai } from "../../ai";
-import {
-  searchFiles,
-  scanFile,
-  writeFile,
-  applyPatchFile,
-  execCommand,
-  readFile,
-  finalAnswer,
-  addInternalTools,
-  callPlugin,
-  testPatch,
-} from "../tools";
-import { Tools } from "../tools/list";
-
-const availableFunctions = addInternalTools({
-  testPatch,
-  applyPatchFile,
-  callPlugin,
-  execCommand,
-  finalAnswer,
-  readFile,
-  scanFile,
-  searchFiles,
-  writeFile,
-});
-
-export class CodebaseAgent {
-  getInitialMessages(user_input: string) {
-    return [
-      {
-        role: "system",
-        content:
-          "Codebase Agent. You use the tools to read and write code, to help the developer implement features faster. Call final answer once you have finished implementing what is requested. As an agent you will receive multiple rounds of input until you call final answer. You are not able to request feedback from the user, so proceed with your plans and the developer will contact you afterwards if they need more help.",
-      },
-
-      { role: "user", content: user_input },
-    ] as Array<ChatCompletionMessageParam>;
-  }
-
-  async getToolMessages(toolCall: ChatCompletionMessageToolCall) {
-    const functionName = toolCall.function.name;
-    const functionToCall = availableFunctions[functionName];
-    const functionArgs = JSON.parse(toolCall.function.arguments);
-    const positionalArgs = Object.values(functionArgs);
-
-    console.log(
-      `Calling function ${functionName} with args:`,
-      JSON.stringify(positionalArgs, null, 2)
-    );
-
-    const functionResponse = await functionToCall(...positionalArgs);
-
-    if (functionName === "multi_tool_use.parallel") {
-      const args = positionalArgs[0] as {
-        recipient_name: string;
-        parameters: any;
-      }[];
-
-      return args.map((call, index) => {
-        return {
-          tool_call_id: toolCall.id + "_" + index,
-          role: "tool",
-          name: call.recipient_name.split(".").pop(),
-          content: functionResponse[index] || "Done",
-        };
-      });
-    }
-
-    const toolMessage = {
-      tool_call_id: toolCall.id,
-      role: "tool",
-      name: functionName,
-      content: functionResponse || "Done",
-    };
-
-    return [toolMessage];
-  }
-
-  logMessages(messages: Array<ChatCompletionMessageParam>) {
-    for (const message of messages) {
-      if (message.role == "assistant") {
-        console.log(message.content);
-      }
-    }
-  }
-
-  async call(
-    user_input: string,
-    _messages?: Array<ChatCompletionMessageParam>
-  ) {
-    const model = "gpt-4-1106-preview";
-    const messages = _messages || this.getInitialMessages(user_input);
-
-    const response = await openai.chat.completions.create({
-      model,
-      messages: messages,
-      tools: Tools,
-      tool_choice: "auto",
-    });
-    this.logMessages(response.choices.map((c) => c.message));
-    const responseMessage = response.choices[0].message;
-
-    const toolCalls = responseMessage.tool_calls;
-    if (responseMessage.tool_calls) {
-      // extend conversation with assistant's reply
-      messages.push(responseMessage);
-
-      for (const toolCall of toolCalls) {
-        const toolMessages = await this.getToolMessages(toolCall);
-        // Add the tool responses to the thread
-        messages.push(
-          ...(toolMessages as Array<ChatCompletionToolMessageParam>)
-        );
-        const finalMessage = toolMessages.find((m) => m.name === "FinalAnswer");
-        if (finalMessage) {
-          return finalMessage.content;
-        }
-      }
-
-      // Send the tool responses back to the model
-      const secondResponse = await openai.chat.completions.create({
-        model,
-        messages: messages,
-      });
-
-      const aiResp = secondResponse.choices.map((c) => c.message);
-      this.logMessages(aiResp);
-      messages.push(...aiResp);
-
-      return this.call(user_input, messages);
-    }
-
-    if (responseMessage.content) {
-      return responseMessage.content;
-    }
-  }
-}
-
-export const Developer = new CodebaseAgent();

package/tests/manual/file-edits/patching/patch.txt

@@ -1,15 +0,0 @@
-Index: tests/integration/patching/input.txt
-===================================================================
---- tests/integration/patching/input.txt
-+++ tests/integration/patching/input.txt
-@@ -116,9 +116,9 @@
-         // Add the tool responses to the thread
-         messages.push(
-           ...(toolMessages as Array<ChatCompletionToolMessageParam>)
-         );
--        const finalMessage = toolMessages.find((m) => m.name === "finalAnswer");
-+        const finalMessage = toolMessages.find((m) => m.name === "FinalAnswer");
-         if (finalMessage) {
-           return finalMessage.content;
-         }
-       }

package/tests/manual/file-edits/patching/unseen.txt

@@ -1,80 +0,0 @@
-import { Octokit } from "@octokit/rest";
-import { Plugin } from "./types";
-import parseDiff from "parse-diff";
-import { MinimalEmbedding } from "../types";
-
-export class GitHubPlugin implements Plugin {
-  octokit: Octokit;
-
-  constructor() {
-    this.octokit = new Octokit({
-      auth: process.env.GITHUB_TOKEN,
-    });
-  }
-
-  async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
-    const urls = this.extractUrls(userPrompt);
-    const diffs = await this.getParsedDiffs(urls);
-    const diffsFiltered = diffs.filter((diff) => diff !== null);
-
-    return diffsFiltered.map((diff, index) => {
-      return {
-        id: urls[index],
-        text: JSON.stringify(diff),
-        metadata: {},
-      };
-    });
-  }
-
-  extractUrls(userPrompt: string): string[] {
-    const prUrlRegex =
-      /https:\/\/github\.com\/([\w-]+)\/([\w-]+)\/pull\/(\d+)/g;
-    const matches = userPrompt.match(prUrlRegex);
-    return matches;
-  }
-
-  async getDiff(url: string) {
-    const [owner, repo, _, pull_number] = url.split("/").slice(-4);
-    console.log(`Loading diff for ${owner}/${repo}#${pull_number}`);
-    const { data: diff } = await this.octokit.rest.pulls.get({
-      owner,
-      repo,
-      pull_number: parseInt(pull_number),
-      mediaType: {
-        format: "diff",
-      },
-    });
-
-    return diff;
-  }
-
-  async getParsedDiffs(urls: string[]) {
-    return Promise.all(
-      urls.map(async (url) => {
-        const diff = await this.getDiff(url);
-        let parsed = parseDiff(diff.toString());
-
-        parsed = parsed.filter((file) => {
-          return file.additions < 200 && file.deletions < 200;
-        });
-
-        return parsed;
-      })
-    );
-  }
-
-  async call(userPrompt: string): Promise<string> {
-    const urls = this.extractUrls(userPrompt);
-
-    if (urls) {
-      const responses = await this.getParsedDiffs(urls);
-      // Format the diffs in Markdown
-      const markdownDiffs = responses
-        .map((diff) => `\`\`\`diff\n${JSON.stringify(diff)}\n\`\`\``)
-        .join("\n\n");
-      return `GITHUB PLUGIN: ${urls} loaded:\n\n${markdownDiffs}`;
-    }
-
-    return "GITHUB PLUGIN: No pull request URLs detected.";
-  }
-}