@syntheticlab/synbad 0.0.5 → 0.0.6

package/dist/source/index.js

@@ -9,10 +9,11 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 };
 import * as http from "http";
 import * as https from "https";
-import fs from "fs/promises";
 import path from "path";
 import { Command } from "@commander-js/extra-typings";
 import OpenAI from "openai";
+import { getReasoning } from "./chat-completion.js";
+import { findTestFiles, evalName } from "./evals.js";
 const cli = new Command()
     .name("synbad")
     .description("A set of evals for LLM inference providers");
@@ -21,10 +22,12 @@ cli.command("eval")
     .requiredOption("--env-var <env var name>", "The env var to use to authenticate with the inference provider")
     .requiredOption("--base-url <base url>", "The base URL for the inference provider")
     .option("--skip-reasoning", "Skip reasoning evals (set this for non-reasoning models)")
+    .option("--reasoning-effort <level>", "Set the reasoning effort to high, medium, or low")
     .option("--only <eval path within synbad>", "Specific evals you want to run, e.g. evals/reasoning or evals/tools/claude-dash")
     .option("--count <num times>", "Number of times to run the eval. Any failures count as an overall failure")
+    .option("--stream", "Test streaming API calls")
     .requiredOption("--model <model name>", "The model name to test")
-    .action(async ({ model, envVar, baseUrl, only, count }) => {
+    .action(async ({ model, envVar, baseUrl, only, count, skipReasoning, reasoningEffort, stream }) => {
     if (!process.env[envVar]) {
         console.error(`No env var named ${envVar} exists for the current process`);
         process.exit(1);
@@ -35,29 +38,111 @@ cli.command("eval")
     });
     let found = 0;
     const failures = new Set();
-    const evalPath = only ? path.join(import.meta.dirname, "..", only) : path.join(import.meta.dirname, "../evals");
+    const evalPath = only ? path.join(import.meta.dirname, "..", only) : path.join(import.meta.dirname, "..", "evals");
     const maxRuns = count == null ? 1 : parseInt(count, 10);
-    for await (const testFile of findTestFiles(evalPath)) {
+    for await (const testFile of findTestFiles(evalPath, !!skipReasoning)) {
         found++;
         const test = await import(__rewriteRelativeImportExtension(testFile));
         const json = test.json;
         const name = evalName(testFile);
         process.stdout.write(`Running ${name}...`);
+        async function respond() {
+            const reasoning = reasoningEffort == null ? {} : {
+                reasoning_effort: reasoningEffort,
+            };
+            if (!stream) {
+                const response = await client.chat.completions.create({
+                    ...json,
+                    ...reasoning,
+                    stream: false,
+                    model,
+                });
+                return response.choices[0].message;
+            }
+            const msg = {};
+            const chunkStream = await client.chat.completions.create({
+                ...json,
+                ...reasoning,
+                model,
+                stream: true,
+            });
+            let lastIndex = null;
+            let toolBuffer = null;
+            for await (const chunk of chunkStream) {
+                if (!chunk.choices)
+                    continue;
+                const choice = chunk.choices[0];
+                if (!choice)
+                    continue;
+                const content = choice.delta.content;
+                const tools = choice.delta.tool_calls;
+                const reasoning = getReasoning(choice.delta);
+                if (content) {
+                    if (!msg.content)
+                        msg.content = "";
+                    msg.content += content;
+                }
+                if (tools) {
+                    for (const toolDelta of tools) {
+                        if (lastIndex == null)
+                            lastIndex = toolDelta.index;
+                        if (lastIndex !== toolDelta.index && toolBuffer != null) {
+                            msg.tool_calls ||= [];
+                            // @ts-ignore
+                            msg.tool_calls.push(toolBuffer);
+                            toolBuffer = {
+                                index: toolDelta.index,
+                                type: "function",
+                                function: {},
+                            };
+                        }
+                        if (!toolBuffer) {
+                            toolBuffer = {
+                                index: toolDelta.index,
+                                type: "function",
+                                function: {}
+                            };
+                        }
+                        lastIndex = toolDelta.index;
+                        if (toolDelta.id)
+                            toolBuffer.id = toolDelta.id;
+                        if (toolDelta.function) {
+                            if (toolDelta.function.name) {
+                                toolBuffer.function.name ||= "";
+                                toolBuffer.function.name += toolDelta.function.name;
+                            }
+                            if (toolDelta.function.arguments) {
+                                toolBuffer.function.arguments ||= "";
+                                toolBuffer.function.arguments += toolDelta.function.arguments;
+                            }
+                        }
+                    }
+                }
+                if (reasoning) {
+                    if (!msg.reasoning_content)
+                        msg.reasoning_content = "";
+                    msg.reasoning_content += reasoning;
+                }
+            }
+            if (toolBuffer) {
+                msg.tool_calls ||= [];
+                // @ts-ignore
+                msg.tool_calls.push(toolBuffer);
+            }
+            return msg;
+        }
         try {
             for (let i = 0; i < maxRuns; i++) {
                 if (maxRuns > 1) {
                     process.stdout.write(` ${i + 1}/${maxRuns}`);
                 }
-                const response = await client.chat.completions.create({
-                    model,
-                    ...json,
-                });
+                const response = await respond();
                 try {
                     test.test(response);
                 }
                 catch (e) {
                     console.error("Response:");
-                    console.error(JSON.stringify(response.choices[0], null, 2));
+                    console.error(JSON.stringify(response, null, 2));
                     throw e;
                 }
             }
@@ -84,6 +169,7 @@ ${passed}/${found} evals passed. Failures:
 cli.command("proxy")
     .requiredOption("-p, --port <number>", "Port to listen on")
     .requiredOption("-t, --target <url>", "Target URL to proxy to")
+    .option("--pretty", "Pretty-print the JSON")
     .action(async (options) => {
     const port = parseInt(options.port, 10);
     const targetUrl = new URL(options.target);
@@ -108,6 +194,7 @@ cli.command("proxy")
             stderrLog(`[${timestamp}] 📦 Writing request data to stdout...`);
             // Choose the right module based on target protocol
             const httpModule = targetUrl.protocol === "https:" ? https : http;
+            const buffer = [];
             // Create proxy request
             const proxyReq = httpModule.request({
                 hostname: targetUrl.hostname,
@@ -151,11 +238,16 @@ cli.command("proxy")
                 }
             });
             req.on("data", (chunk) => {
-                process.stdout.write(chunk);
+                buffer.push(chunk);
+                if (!options.pretty)
+                    process.stdout.write(chunk);
                 proxyReq.write(chunk);
             });
             req.on("end", () => {
-                process.stdout.write("\n");
+                if (options.pretty)
+                    console.log(JSON.stringify(JSON.parse(buffer.join()), null, 2));
+                else
+                    process.stdout.write("\n");
                 console.log(`[${timestamp}] ✅ Request complete`);
                 proxyReq.end();
             });
@@ -178,38 +270,6 @@ cli.command("proxy")
         stderrLog("🤓 Terminal UI messages (such as this one) will be logged to stderr");
     });
 });
-function evalName(file) {
-    return `${path.basename(path.dirname(file))}/${path.basename(file).replace(/.js$/, "")}`;
-}
-async function* findTestFiles(dir) {
-    try {
-        await fs.stat(dir);
-    }
-    catch (e) {
-        const pathname = `${dir}.js`;
-        const stat = await fs.stat(pathname);
-        if (stat.isFile()) {
-            yield pathname;
-            return;
-        }
-        throw e;
-    }
-    const entryNames = await fs.readdir(dir);
-    const entries = await Promise.all(entryNames.map(async (entry) => {
-        return {
-            path: path.join(dir, entry),
-            stat: await fs.stat(path.join(dir, entry)),
-        };
-    }));
-    for (const entry of entries) {
-        if (entry.stat.isFile() && entry.path.endsWith(".js")) {
-            yield entry.path;
-        }
-        if (entry.stat.isDirectory()) {
-            yield* findTestFiles(entry.path);
-        }
-    }
-}
 function stderrLog(item, ...items) {
     let formatted = item;
     if (items.length > 0) {

package/evals/reasoning/multiturn-reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
+import { ChatMessage, getReasoning } from "../../source/chat-completion.ts";
 
-export function test(response: ChatResponse) {
-  const reasoning = getReasoning(response.choices[0].message);
+export function test(message: ChatMessage) {
+  const reasoning = getReasoning(message);
   assert.isNotNullish(reasoning);
 }
 

package/evals/reasoning/reasoning-claude-tool-call.ts

@@ -1,8 +1,7 @@
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
 import * as assert from "../../source/asserts.ts";
 
-export function test(response: ChatResponse) {
-  const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }: ChatMessage) {
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);
   assert.strictEqual(tool_calls.length, 1);

package/evals/reasoning/reasoning-parsing.ts

@@ -1,8 +1,8 @@
 import * as assert from "../../source/asserts.ts";
-import { ChatResponse, getReasoning } from "../../source/chat-completion.ts";
+import { ChatMessage, getReasoning } from "../../source/chat-completion.ts";
 
-export function test(response: ChatResponse) {
-  const reasoning = getReasoning(response.choices[0].message);
+export function test(message: ChatMessage) {
+  const reasoning = getReasoning(message);
   assert.isNotNullish(reasoning);
 }
 

package/evals/reasoning/response-in-reasoning.ts

@@ -0,0 +1,65 @@
+import * as assert from "../../source/asserts.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
+
+export function test(message: ChatMessage) {
+  const content = message.content;
+  assert.or(
+    () => assert.isNotNullish(content),
+    () => assert.isNotEmptyArray(message.tool_calls),
+  );
+}
+
+export const json = {
+  "messages": [
+    {
+      "role": "system",
+      "content": "When I ask you to add a feature or resolve a problem: ALWAYS start the project explorer sub-agent to build complete understanding"
+    },
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": "Hello"
+        }
+      ]
+    }
+  ],
+  "temperature": 1,
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "task",
+        "description": "Launch a new agent to handle complex, multi-step tasks autonomously.\n\nAvailable agent types and the tools they have access to:\n- general: General-purpose agent for researching complex questions and executing multi-step tasks. Use this agent to execute multiple units of work in parallel.\n- explore: Fast agent specialized for exploring codebases. Use this when you need to quickly find files by patterns (eg. \"src/components/**/*.tsx\"), search code for keywords (eg. \"API endpoints\"), or answer questions about the codebase (eg. \"how do API endpoints work?\"). When calling this agent, specify the desired thoroughness level: \"quick\" for basic searches, \"medium\" for moderate exploration, or \"very thorough\" for comprehensive analysis across multiple locations and naming conventions.\n- code-reviewer: Expert code review specialist. Proactively reviews code for quality, security, and maintainability. Use immediately after writing or modifying code.\n\nWhen using the Task tool, you must specify a subagent_type parameter to select which agent type to use.\n\nWhen to use the Task tool:\n- When you are instructed to execute custom slash commands. Use the Task tool with the slash command invocation as the entire prompt. The slash command can take arguments. For example: Task(description=\"Check the file\", prompt=\"/check-file path/to/file.py\")\n\nWhen NOT to use the Task tool:\n- If you want to read a specific file path, use the Read or Glob tool instead of the Task tool, to find the match more quickly\n- If you are searching for a specific class definition like \"class Foo\", use the Glob tool instead, to find the match more quickly\n- If you are searching for code within a specific file or set of 2-3 files, use the Read tool instead of the Task tool, to find the match more quickly\n- Other tasks that are not related to the agent descriptions above\n\n\nUsage notes:\n1. Launch multiple agents concurrently whenever possible, to maximize performance; to do that, use a single message with multiple tool uses\n2. When the agent is done, it will return a single message back to you. The result returned by the agent is not visible to the user. To show the user the result, you should send a text message back to the user with a concise summary of the result.\n3. Each agent invocation is stateless unless you provide a session_id. Your prompt should contain a highly detailed task description for the agent to perform autonomously and you should specify exactly what information the agent should return back to you in its final and only message to you.\n4. The agent's outputs should generally be trusted\n5. Clearly tell the agent whether you expect it to write code or just to do research (search, file reads, web fetches, etc.), since it is not aware of the user's intent\n6. If the agent description mentions that it should be used proactively, then you should try your best to use it without the user having to ask for it first. Use your judgement.\n\n",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "description": {
+              "description": "A short (3-5 words) description of the task",
+              "type": "string"
+            },
+            "prompt": {
+              "description": "The task for the agent to perform",
+              "type": "string"
+            },
+            "subagent_type": {
+              "description": "The type of specialized agent to use for this task",
+              "type": "string"
+            },
+            "session_id": {
+              "description": "Existing Task session to continue",
+              "type": "string"
+            }
+          },
+          "required": [
+            "description",
+            "prompt",
+            "subagent_type"
+          ]
+        }
+      }
+    },
+  ],
+  "tool_choice": "auto"
+}

package/evals/tools/claude-dash.ts

@@ -1,8 +1,7 @@
-import OpenAI from "openai";
 import * as assert from "../../source/asserts.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
 
-export function test(response: OpenAI.ChatCompletion) {
-  const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }: ChatMessage) {
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);
 }

package/evals/tools/crush-list-files.ts

@@ -1,22 +1,23 @@
-import OpenAI from "openai";
 import * as assert from "../../source/asserts.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
 
-export function test(response: OpenAI.ChatCompletion) {
-  const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }: ChatMessage) {
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);
-  assert.strictEqual(tool_calls.length, 1);
+  assert.gte(tool_calls.length, 1);
   assert.strictEqual(tool_calls[0].type, "function");
   const fn = tool_calls[0].function;
   assert.or(
     () => {
       assert.strictEqual(fn.name, "ls");
-      const args = JSON.parse(fn.arguments);
-      assert.or(
-        () => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"),
-        () => assert.strictEqual(args.path, "."),
-        () => assert.isNullish(args.path),
-      );
+      if(fn.arguments) {
+        const args = JSON.parse(fn.arguments);
+        assert.or(
+          () => assert.strictEqual(args.path, "/home/reissbaker/Hack/scratch-scripts"),
+          () => assert.strictEqual(args.path, "."),
+          () => assert.isNullish(args.path),
+        );
+      }
     },
     () => {
       assert.strictEqual(fn.name, "bash");
@@ -448,7 +449,4 @@ export const json = {
   ],
   "tool_choice": "auto",
   "max_tokens": 60000,
-  "stream_options": {
-    "include_usage": true
-  }
 }

package/evals/tools/multi-turn-tools.ts

@@ -0,0 +1,104 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+import * as assert from "../../source/asserts.ts";
+
+export function test({ tool_calls }: ChatMessage) {
+  assert.isNotNullish(tool_calls);
+  assert.isNotEmptyArray(tool_calls);
+  assert.gte(tool_calls.length, 1);
+
+  assert.ok(tool_calls.some(tool_call => {
+    if (tool_call.type === "function" && tool_call.function.name === "get_weather") {
+      const location = JSON.parse(tool_call.function.arguments).location;
+      if (typeof location === "string") {
+        return location.toLowerCase().match(/las vegas/);
+      }
+    }
+    return false;
+  }), "At least one tool call must be get_weather({ location: 'las_vegas' })");
+}
+
+export const json = {
+  "messages": [
+    {
+      role: "user",
+      content: "What's the weather in Paris?"
+    },
+    {
+      role: "assistant",
+      tool_calls: [
+        {
+          id: "gw1",
+          type: "function",
+          function: {
+            name: "get_weather",
+            arguments: JSON.stringify({
+              location: "Paris, France",
+            }),
+          },
+        },
+      ],
+    },
+    {
+      role: "tool",
+      tool_call_id: "gw1",
+      content: "The weather in Paris is 24 degrees Celsius",
+    },
+    {
+      role: "assistant",
+      content: "I've looked up the weather in Paris, and it's a comfy 24 degrees Celsius today.",
+    },
+    {
+      role: "user",
+      content: "I meant Paris, Texas",
+    },
+    {
+      role: "assistant",
+      tool_calls: [
+        {
+          id: "gw2",
+          type: "function",
+          function: {
+            name: "get_weather",
+            arguments: JSON.stringify({
+              location: "Paris, Texas",
+            }),
+          },
+        },
+      ],
+    },
+    {
+      role: "tool",
+      tool_call_id: "gw2",
+      content: "The weather in Paris, Texas is 34 degrees Celsius",
+    },
+    {
+      role: "assistant",
+      content: "I've looked up the weather in Paris, Texas and it's a scorching 24 degrees Celsius today.",
+    },
+    {
+      role: "user",
+      content: "How about Las Vegas",
+    },
+  ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "get_weather",
+        "description": "Get current weather for a location",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {
+              "type": "string",
+              "description": "City name"
+            }
+          },
+          "required": ["location"]
+        }
+      }
+    }
+  ],
+  "parallel_tool_calls": true,
+  "tool_choice": "auto",
+}

package/evals/tools/no-fn-args.ts

@@ -0,0 +1,34 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+import * as assert from "../../source/asserts.ts";
+
+export function test({ tool_calls }: ChatMessage) {
+  assert.isNotNullish(tool_calls);
+  assert.isNotEmptyArray(tool_calls);
+  assert.strictEqual(tool_calls.length, 1);
+  assert.strictEqual(tool_calls[0].type, "function");
+}
+
+export const json = {
+  "messages": [
+    {
+      "role": "user",
+      "content": "read the todos",
+    },
+  ],
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "get_todo_items",
+        "description": "Retrieves the current list of todo items, including their names and completion statuses.",
+        "parameters": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
+          "type": "object",
+          "properties": {},
+          "additionalProperties": false
+        }
+      }
+    },
+  ],
+  "tool_choice": "auto",
+}

package/evals/tools/octo-list-no-optional-args.ts

@@ -0,0 +1,81 @@
+import { ChatMessage } from "../../source/chat-completion.ts";
+import * as assert from "../../source/asserts.ts";
+
+export function test({ tool_calls }: ChatMessage) {
+  assert.isNotNullish(tool_calls);
+  assert.isNotEmptyArray(tool_calls);
+  assert.strictEqual(tool_calls.length, 1);
+  assert.strictEqual(tool_calls[0].type, "function");
+  const parsed = JSON.parse(tool_calls[0].function.arguments);
+  assert.or(
+    () => assert.isNullish(parsed),
+    () => assert.deepEqual(parsed, {}),
+    () => assert.isNullish(parsed.dirPath),
+    () => assert.strictEqual(parsed.dirPath, ""),
+  );
+}
+
+export const json = {
+  "messages":[
+    {
+      "role":"system",
+      "content":"You are a coding assistant called Octo."
+    },
+    {
+      "role":"user",
+      "content":"call the list tool with no args"
+    }
+  ],
+  tools: [
+    {
+      "type":"function",
+      "function":{
+        "name":"append",
+        "description":"The append tool",
+        "parameters":{
+          "type":"object",
+          "required":["filePath","text"],
+          "properties":{
+            "filePath":{
+              "description":"The path to the file",
+              "type":"string"
+            },
+            "text":{
+              "description":"The text to append",
+              "type":"string"
+            }
+          }
+        },
+        "strict":true
+      }
+    },
+    {
+      "type":"function",
+      "function":{
+        "name":"create",
+        "description":"The create tool",
+        "parameters":{
+          "type":"object",
+          "required":["filePath","content"],
+          "properties":{
+            "filePath":{
+              "description":"Path where the file should be created",
+              "type":"string"
+            },"content":{
+              "description":"Content to write to the file",
+              "type":"string"
+            }
+          }
+        },
+        "strict":true
+      }
+    },
+    {"type":"function","function":{"name":"edit","description":"The edit tool","parameters":{"type":"object","required":["filePath","search","replace"],"properties":{"filePath":{"description":"The path to the file","type":"string"},"search":{"description":"The search string to replace. Must EXACTLY match the text you intend to replace, including\nwhitespace, punctuation, etc. Make sure to give a few lines of context above and below so you\ndon't accidentally replace a different matching substring in the same file.","type":"string"},"replace":{"description":"The string you want to insert into the file","type":"string"}}},"strict":true}},
+    {"type":"function","function":{"name":"fetch","description":"The fetch tool","parameters":{"type":"object","required":["url"],"properties":{"url":{"description":"Full url to fetch, e.g. https://...","type":"string"},"includeMarkup":{"description":"Include the HTML markup? Defaults to false. By default or when set to false, markup will be\nstripped and converted to plain text. Prefer markup stripping, and only set this to true if the\noutput is confusing: otherwise you may download a massive amount of data","type":"boolean"}}},"strict":true}},
+    {"type":"function","function":{"name":"list","description":"The list tool","parameters":{"type":"object","required":[],"properties":{"dirPath":{"description":"Path to the directory","type":"string"}}},"strict":true}},
+    {"type":"function","function":{"name":"prepend","description":"The prepend tool","parameters":{"type":"object","required":["filePath","text"],"properties":{"filePath":{"description":"The path to the file","type":"string"},"text":{"description":"The text to prepend","type":"string"}}},"strict":true}},
+    {"type":"function","function":{"name":"read","description":"The read tool","parameters":{"type":"object","required":["filePath"],"properties":{"filePath":{"description":"Path to file to read","type":"string"}}},"strict":true}},
+    {"type":"function","function":{"name":"rewrite","description":"The rewrite tool","parameters":{"type":"object","required":["filePath","text"],"properties":{"filePath":{"description":"The path to the file","type":"string"},"text":{"description":"The replaced file contents. This will rewrite and replace the entire file","type":"string"}}},"strict":true}},
+    {"type":"function","function":{"name":"shell","description":"The shell tool","parameters":{"type":"object","required":["timeout","cmd"],"properties":{"timeout":{"description":"A timeout for the command, in milliseconds. Be generous. You MUST specify this.","type":"number"},"cmd":{"description":"The command to run","type":"string"}}},"strict":true}}
+  ],
+};

package/evals/tools/parallel-tool.ts

@@ -1,8 +1,7 @@
-import OpenAI from "openai";
 import * as assert from "../../source/asserts.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
 
-export function test(response: OpenAI.ChatCompletion) {
-  const { tool_calls } = response.choices[0].message;
+export function test({ tool_calls }: ChatMessage) {
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);
   assert.strictEqual(tool_calls.length, 2);

package/evals/tools/simple-tool.ts

@@ -1,8 +1,7 @@
-import { ChatResponse } from "../../source/chat-completion.ts";
+import { ChatMessage } from "../../source/chat-completion.ts";
 import * as assert from "../../source/asserts.ts";
 
-export function test(response: ChatResponse) {
-  const { tool_calls } = response.choices[0].message;
+export function test({ content, tool_calls }: ChatMessage) {
   assert.isNotNullish(tool_calls);
   assert.isNotEmptyArray(tool_calls);
   assert.strictEqual(tool_calls.length, 1);
@@ -10,6 +9,8 @@ export function test(response: ChatResponse) {
   assert.strictEqual(tool_calls[0].function.name, "get_weather");
   const args = JSON.parse(tool_calls[0].function.arguments);
   assert.match(args.location.toLowerCase(), /paris/);
+  // Assert the tool call didn't leak into the content
+  assert.doesNotMatch(content || "", /get_weather/);
 }
 
 export const json = {