@chat-js/cli 0.1.4 → 0.2.1

package/templates/chat-app/lib/ai/models.ts

@@ -34,7 +34,7 @@ export const fetchModels = unstable_cache(
     const models = await fetchModelsRaw();
     return models.map(toModelData);
   },
-  [`ai-gateway-models-${config.models.gateway}`],
+  [`ai-gateway-models-${config.ai.gateway}`],
   {
     revalidate: 3600,
     tags: ["ai-gateway-models"],

package/templates/chat-app/lib/ai/providers.ts

@@ -51,6 +51,16 @@ export const getImageModel = (modelId: string) => {
   return imageModel;
 };
 
+export const getVideoModel = (modelId: string) => {
+  const videoModel = getActiveGateway().createVideoModel(modelId);
+  if (!videoModel) {
+    throw new Error(
+      `Gateway '${getActiveGateway().type}' does not support video models.`
+    );
+  }
+  return videoModel;
+};
+
 // Get a multimodal language model that can generate images via generateText
 export const getMultimodalImageModel = (modelId: string) =>
   getActiveGateway().createLanguageModel(modelId);

package/templates/chat-app/lib/ai/text-splitter.ts

@@ -1,8 +1,7 @@
-type TextSplitterParams = {
-  chunkSize: number;
-
+interface TextSplitterParams {
   chunkOverlap: number;
-};
+  chunkSize: number;
+}
 
 abstract class TextSplitter implements TextSplitterParams {
   chunkSize = 1000;

package/templates/chat-app/lib/ai/to-model-data.ts

@@ -28,6 +28,7 @@ export function toModelData(model: AiGatewayModel): ModelData {
       image: tags.includes("image-generation") || model.type === "image",
       text: model.type === "language",
       audio: false,
+      video: model.type === "video",
     },
   };
 }

package/templates/chat-app/lib/ai/tools/code-execution.ts

@@ -7,6 +7,7 @@ import { createModuleLogger } from "@/lib/logger";
 import { toolsDefinitions } from "./tools-definitions";
 
 const WHITESPACE_REGEX = /\s+/;
+const PACKAGE_SPEC_SPLIT_RE = /[=<>![\s]/;
 
 async function installBasePackages(
   sandbox: Sandbox,
@@ -39,8 +40,13 @@ async function installBasePackages(
   return { success: true };
 }
 
+function packageName(spec: string): string {
+  return spec.split(PACKAGE_SPEC_SPLIT_RE)[0].toLowerCase();
+}
+
 async function processExtraPackages(
   code: string,
+  basePackages: readonly string[],
   sandbox: Sandbox,
   log: ReturnType<typeof createModuleLogger>,
   requestId: string
@@ -51,43 +57,51 @@ async function processExtraPackages(
     result?: { message: string; chart: string };
   };
 }> {
+  const basePackageNames = new Set(basePackages.map((p) => p.toLowerCase()));
   const lines = code.split("\n");
   const pipLines = lines.filter((l) => l.trim().startsWith("!pip install "));
-  const extraPackages = pipLines.flatMap((l) =>
-    l
-      .trim()
-      .slice("!pip install ".length)
-      .split(WHITESPACE_REGEX)
-      .filter(Boolean)
-  );
+  const extraPackages = pipLines
+    .flatMap((l) =>
+      l
+        .trim()
+        .slice("!pip install ".length)
+        .split(WHITESPACE_REGEX)
+        .filter(Boolean)
+    )
+    .filter((spec) => !basePackageNames.has(packageName(spec)));
+
+  const codeWithoutPipLines = lines
+    .filter((l) => !l.trim().startsWith("!pip install "))
+    .join("\n");
+
+  if (extraPackages.length === 0) {
+    return { codeToRun: codeWithoutPipLines, installResult: { success: true } };
+  }
 
-  let codeToRun = code;
-  if (extraPackages.length > 0) {
-    log.info({ requestId, extraPackages }, "installing extra packages");
-    const dynamicInstall = await sandbox.runCommand({
-      cmd: "pip",
-      args: ["install", ...extraPackages],
-    });
-    if (dynamicInstall.exitCode !== 0) {
-      const stderr = await dynamicInstall.stderr();
-      log.error({ requestId, stderr }, "dynamic package installation failed");
-      return {
-        codeToRun: code,
-        installResult: {
-          success: false,
-          result: {
-            message: `Failed to install packages: ${stderr}`,
-            chart: "",
-          },
+  log.info({ requestId, extraPackages }, "installing extra packages");
+  const dynamicInstall = await sandbox.runCommand({
+    cmd: "pip",
+    args: ["install", ...extraPackages],
+  });
+  if (dynamicInstall.exitCode !== 0) {
+    const stderr = await dynamicInstall.stderr();
+    log.error({ requestId, stderr }, "dynamic package installation failed");
+    return {
+      codeToRun: code,
+      installResult: {
+        success: false,
+        result: {
+          message: `Failed to install packages: ${stderr}`,
+          chart: "",
         },
-      };
-    }
-    codeToRun = lines
-      .filter((l) => !l.trim().startsWith("!pip install "))
-      .join("\n");
+      },
+    };
   }
 
-  return { codeToRun, installResult: { success: true } };
+  return {
+    codeToRun: codeWithoutPipLines,
+    installResult: { success: true },
+  };
 }
 
 function createWrappedCode(codeToRun: string, chartPath: string): string {
@@ -96,19 +110,36 @@ import sys
 import json
 import traceback
 
+try:
+    import matplotlib.pyplot as _plt_module
+    _orig_savefig = _plt_module.savefig
+    def _intercepted_savefig(*args, **kwargs):
+        _orig_savefig('${chartPath}', format='png', bbox_inches='tight', dpi=100)
+        if args or kwargs.get('fname') not in (None, '${chartPath}'):
+            return _orig_savefig(*args, **kwargs)
+    _plt_module.savefig = _intercepted_savefig
+except ImportError:
+    pass
+
 try:
     exec(${JSON.stringify(codeToRun)})
     try:
         _locals = locals()
         _globals = globals()
-        if "result" in _locals:
-            print(_locals["result"])
-        elif "result" in _globals:
-            print(_globals["result"])
-        elif "results" in _locals:
-            print(_locals["results"])
-        elif "results" in _globals:
-            print(_globals["results"])
+        _chart_var = _locals.get("chart") or _globals.get("chart")
+        if (isinstance(_chart_var, dict)
+                and isinstance(_chart_var.get("type"), str)
+                and isinstance(_chart_var.get("elements"), list)):
+            print("__CHART_JSON__:" + json.dumps(_chart_var))
+        else:
+            if "result" in _locals:
+                print(_locals["result"])
+            elif "result" in _globals:
+                print(_globals["result"])
+            elif "results" in _locals:
+                print(_locals["results"])
+            elif "results" in _globals:
+                print(_globals["results"])
     except Exception:
         pass
     try:
@@ -126,11 +157,14 @@ except Exception as e:
 `;
 }
 
+const CHART_JSON_PREFIX = "__CHART_JSON__:";
+
 async function parseExecutionOutput(execResult: {
   stdout: () => Promise<string>;
   exitCode: number;
 }): Promise<{
   outputText: string;
+  chartData: Record<string, unknown> | null;
   execInfo: {
     success: boolean;
     error?: { name: string; value: string; traceback: string };
@@ -142,18 +176,33 @@ async function parseExecutionOutput(execResult: {
     error?: { name: string; value: string; traceback: string };
   } = { success: true };
   let outputText = "";
+  let chartData: Record<string, unknown> | null = null;
 
   try {
     const outLines = (stdout ?? "").trim().split("\n");
     const lastLine = outLines.at(-1);
     execInfo = JSON.parse(lastLine ?? "{}");
     outLines.pop();
+
+    const chartLineIdx = outLines.findIndex((l) =>
+      l.startsWith(CHART_JSON_PREFIX)
+    );
+    if (chartLineIdx !== -1) {
+      const raw = outLines[chartLineIdx].slice(CHART_JSON_PREFIX.length);
+      try {
+        chartData = JSON.parse(raw) as Record<string, unknown>;
+      } catch {
+        // ignore malformed chart JSON
+      }
+      outLines.splice(chartLineIdx, 1);
+    }
+
     outputText = outLines.join("\n");
   } catch {
     outputText = stdout ?? "";
   }
 
-  return { outputText, execInfo };
+  return { outputText, chartData, execInfo };
 }
 
 async function checkForChart(
@@ -248,7 +297,7 @@ async function executeInSandbox({
   requestId: string;
 }): Promise<{
   message: string;
-  chart: string | { base64: string; format: string };
+  chart: string | { base64: string; format: string } | Record<string, unknown>;
 }> {
   const baseInstallResult = await installBasePackages(
     sandbox,
@@ -262,6 +311,7 @@ async function executeInSandbox({
 
   const { codeToRun, installResult } = await processExtraPackages(
     code,
+    basePackages,
     sandbox,
     log,
     requestId
@@ -276,8 +326,8 @@ async function executeInSandbox({
     args: ["-c", wrappedCode],
   });
 
-  const { outputText, execInfo } = await parseExecutionOutput(execResult);
-  const chartOut = await checkForChart(sandbox, chartPath, log, requestId);
+  const { outputText, chartData, execInfo } =
+    await parseExecutionOutput(execResult);
 
   const message = buildResponseMessage({
     outputText,
@@ -287,6 +337,12 @@ async function executeInSandbox({
     requestId,
   });
 
+  if (chartData) {
+    log.info({ requestId }, "interactive chart data returned");
+    return { message: message.trim(), chart: chartData };
+  }
+
+  const chartOut = await checkForChart(sandbox, chartPath, log, requestId);
   return {
     message: message.trim(),
     chart: chartOut ?? "",
@@ -319,23 +375,36 @@ export const codeExecution = ({
   costAccumulator?: CostAccumulator;
 }) =>
   tool({
-    description: `Python-only sandbox for calculations, data analysis & simple visualisations.
+    description: `Python-only sandbox for calculations, data analysis & visualisations.
 
 Use for:
-- Execute Python (matplotlib, pandas, numpy, sympy, yfinance pre-installed)
-- Produce line / scatter / bar charts (no need to call 'plt.show()')
-- Install extra libs by adding lines like: '!pip install <pkg> [<pkg2> ...]' (we auto-install and strip these lines)
+- Execute Python (matplotlib, pandas, numpy, sympy, yfinance pre-installed — do NOT reinstall them)
+- Produce interactive line / scatter / bar charts OR matplotlib PNG charts
+- Install extra libs by adding lines like: '!pip install <pkg> [<pkg2> ...]' (we auto-install and strip these lines; pre-installed packages are ignored)
+
+Chart output — choose ONE:
+1. Interactive chart (preferred for line/scatter/bar): assign a 'chart' variable matching this schema:
+   chart = {
+     "type": "line" | "scatter" | "bar",
+     "title": "My Chart",
+     "x_label": "X",   # optional
+     "y_label": "Y",   # optional
+     "x_scale": "datetime" | None,  # optional, for time-series x axes
+     "elements": [
+       # for line/scatter: {"label": "Series A", "points": [[x1,y1],[x2,y2],...]}
+       # for bar: {"label": "Category", "group": "Group A", "value": 42}
+     ]
+   }
+2. Matplotlib PNG: use plt.plot()/plt.savefig() normally (no need to call plt.show())
 
 Restrictions:
 - No images in the assistant response; don't embed them
-
-Avoid:
-- Any non-Python language
-- Chart types other than line / scatter / bar
+- Interactive chart: only line / scatter / bar types
 
 Output rules:
-- Print the values you want returned (e.g. 'print(df.head())' or 'print(answer)')
-- Or assign to a variable named 'result' or 'results' and we'll print it automatically
+- Assign 'chart' dict for interactive charts (takes priority over matplotlib PNG)
+- Assign 'result' or 'results' for other computed values (auto-printed)
+- Or print explicitly: print(answer)
 - Don't rely on implicit REPL last-expression output`,
     inputSchema: z.object({
       title: z.string().describe("The title of the code snippet."),

package/templates/chat-app/lib/ai/tools/deep-research/configuration.ts

@@ -3,28 +3,16 @@ import { env } from "@/lib/env";
 
 export type SearchAPI = "firecrawl" | "tavily" | "none";
 
-export type DeepResearchRuntimeConfig = {
-  // General Configuration
-  max_structured_output_retries: number;
+export interface DeepResearchRuntimeConfig {
   allow_clarification: boolean;
-  max_concurrent_research_units: number;
-
-  // Research Configuration
-  search_api: SearchAPI;
-  search_api_max_queries: number;
-  max_researcher_iterations: number;
-
-  // Model Configuration
-  summarization_model: string;
-  summarization_model_max_tokens: number;
-  research_model: string;
-  research_model_max_tokens: number;
   compression_model: string;
   compression_model_max_tokens: number;
   final_report_model: string;
   final_report_model_max_tokens: number;
-  status_update_model: string;
-  status_update_model_max_tokens: number;
+  max_concurrent_research_units: number;
+  max_researcher_iterations: number;
+  // General Configuration
+  max_structured_output_retries: number;
 
   // MCP server configuration (not yet implemented)
   mcp_config?: {
@@ -33,7 +21,19 @@ export type DeepResearchRuntimeConfig = {
     headers?: Record<string, string>;
   };
   mcp_prompt?: string;
-};
+  research_model: string;
+  research_model_max_tokens: number;
+
+  // Research Configuration
+  search_api: SearchAPI;
+  search_api_max_queries: number;
+  status_update_model: string;
+  status_update_model_max_tokens: number;
+
+  // Model Configuration
+  summarization_model: string;
+  summarization_model_max_tokens: number;
+}
 
 function getSearchApi(): SearchAPI {
   if (env.TAVILY_API_KEY) {
@@ -46,33 +46,36 @@ function getSearchApi(): SearchAPI {
 }
 
 export function getDeepResearchConfig(): DeepResearchRuntimeConfig {
-  const deepResearchModel = config.models.defaults.deepResearch;
-  const deepResearchFinalReportModel =
-    config.models.defaults.deepResearchFinalReport;
-  const deepResearchSettings = config.deepResearch;
+  const {
+    defaultModel,
+    finalReportModel,
+    allowClarification,
+    maxConcurrentResearchUnits,
+    maxSearchQueries,
+    maxResearcherIterations,
+  } = config.ai.tools.deepResearch;
 
   return {
     // General Configuration
     max_structured_output_retries: 3,
-    allow_clarification: deepResearchSettings.allowClarification,
-    max_concurrent_research_units:
-      deepResearchSettings.maxConcurrentResearchUnits,
+    allow_clarification: allowClarification,
+    max_concurrent_research_units: maxConcurrentResearchUnits,
 
     // Research Configuration
     search_api: getSearchApi(),
-    search_api_max_queries: deepResearchSettings.maxSearchQueries,
-    max_researcher_iterations: deepResearchSettings.maxResearcherIterations,
+    search_api_max_queries: maxSearchQueries,
+    max_researcher_iterations: maxResearcherIterations,
 
     // Model Configuration - use same model for research/compression/summarization
-    summarization_model: deepResearchModel,
+    summarization_model: defaultModel,
     summarization_model_max_tokens: 4000,
-    research_model: deepResearchModel,
+    research_model: defaultModel,
     research_model_max_tokens: 4000,
-    compression_model: deepResearchModel,
+    compression_model: defaultModel,
     compression_model_max_tokens: 4000,
-    final_report_model: deepResearchFinalReportModel,
+    final_report_model: finalReportModel,
     final_report_model_max_tokens: 6000,
-    status_update_model: deepResearchModel,
+    status_update_model: defaultModel,
     status_update_model_max_tokens: 4000,
   };
 }

package/templates/chat-app/lib/ai/tools/deep-research/pipeline.ts

@@ -163,10 +163,10 @@ async function clarifyWithUser(
 
 // Step 2: Research Brief
 
-type ResearchBrief = {
+interface ResearchBrief {
   research_brief: string;
   title: string;
-};
+}
 
 async function writeResearchBrief(
   messages: ModelMessage[],

package/templates/chat-app/lib/ai/tools/deep-research/types.ts

@@ -9,15 +9,15 @@ import type { DeepResearchRuntimeConfig } from "./configuration";
 // Shared Agent Options
 //##################
 
-export type AgentOptions = {
+export interface AgentOptions {
+  abortSignal?: AbortSignal;
   config: DeepResearchRuntimeConfig;
+  costAccumulator?: CostAccumulator;
   dataStream: StreamWriter;
-  toolCallId: string;
   messageId: string;
   requestId: string;
-  costAccumulator?: CostAccumulator;
-  abortSignal?: AbortSignal;
-};
+  toolCallId: string;
+}
 
 //##################
 // Telemetry Helper
@@ -67,12 +67,12 @@ export const ResearchQuestionSchema = z.object({
 // Pipeline IO
 //##################
 
-export type DeepResearchInput = {
-  requestId: string;
+export interface DeepResearchInput {
   messageId: string;
-  toolCallId: string;
   messages: ModelMessage[];
-};
+  requestId: string;
+  toolCallId: string;
+}
 
 export type DeepResearchResult =
   | {

package/templates/chat-app/lib/ai/tools/documents/types.ts

@@ -15,13 +15,13 @@ export type DocumentToolResult =
       error: string;
     };
 
-export type DocumentToolContext = {
-  session: ToolSession;
+export interface DocumentToolContext {
+  costAccumulator?: CostAccumulator;
   // dataStream: StreamWriter;
   messageId: string;
   selectedModel: ModelId;
-  costAccumulator?: CostAccumulator;
-};
+  session: ToolSession;
+}
 
 // Document tool type names as they appear in ChatMessage parts
 export const createDocumentToolTypes = [

package/templates/chat-app/lib/ai/tools/generate-image.ts

@@ -6,14 +6,13 @@ import { uploadFile } from "@/lib/blob";
 import { config } from "@/lib/config";
 import type { CostAccumulator } from "@/lib/credits/cost-accumulator";
 import { createModuleLogger } from "@/lib/logger";
-import { toolsDefinitions } from "./tools-definitions";
 
-type GenerateImageProps = {
+interface GenerateImageProps {
   attachments?: FileUIPart[];
+  costAccumulator?: CostAccumulator;
   lastGeneratedImage?: { imageUrl: string; name: string } | null;
   selectedModel?: string;
-  costAccumulator?: CostAccumulator;
-};
+}
 
 const log = createModuleLogger("ai.tools.generate-image");
 
@@ -42,7 +41,7 @@ async function resolveImageModel(selectedModel?: string): Promise<{
   }
 
   // Fall back to the configured default image model
-  const defaultId = config.models.defaults.image;
+  const defaultId = config.ai.tools.image.default;
   try {
     const model = await getAppModelDefinition(defaultId as AppModelId);
     // Default could be a multimodal language model (e.g. gemini-3-pro-image)
@@ -126,12 +125,14 @@ async function runGenerateImageTraditional({
   imageParts,
   lastGeneratedImage,
   startMs,
+  costAccumulator,
 }: {
   mode: ImageMode;
   prompt: string;
   imageParts: FileUIPart[];
   lastGeneratedImage: { imageUrl: string; name: string } | null;
   startMs: number;
+  costAccumulator?: CostAccumulator;
 }): Promise<{ imageUrl: string; prompt: string }> {
   let promptInput:
     | string
@@ -160,7 +161,7 @@ async function runGenerateImageTraditional({
   }
 
   const res = await generateImage({
-    model: getImageModel(config.models.defaults.image),
+    model: getImageModel(config.ai.tools.image.default),
     prompt: promptInput,
     n: 1,
     providerOptions: {
@@ -181,6 +182,17 @@ async function runGenerateImageTraditional({
   const filename = `generated-image-${timestamp}.png`;
   const result = await uploadFile(filename, buffer);
 
+  if (res.usage) {
+    costAccumulator?.addLLMCost(
+      config.ai.tools.image.default as AppModelId,
+      {
+        inputTokens: res.usage.inputTokens,
+        outputTokens: res.usage.outputTokens,
+      },
+      "generateImage-traditional"
+    );
+  }
+
   log.info(
     {
       mode,
@@ -212,8 +224,14 @@ async function runGenerateImageMultimodal({
   costAccumulator?: CostAccumulator;
 }): Promise<{ imageUrl: string; prompt: string }> {
   // Build messages with image context if in edit mode
-  type ImageContent = { type: "image"; image: URL };
-  type TextContent = { type: "text"; text: string };
+  interface ImageContent {
+    image: URL;
+    type: "image";
+  }
+  interface TextContent {
+    text: string;
+    type: "text";
+  }
   const userContent: Array<TextContent | ImageContent> = [];
 
   // Add reference images if in edit mode
@@ -247,13 +265,24 @@ async function runGenerateImageMultimodal({
     "generateImage: using multimodal model"
   );
 
+  const isGoogleModel =
+    modelId.startsWith("google/") || modelId.includes("gemini");
+  const isOpenAIModel = modelId.startsWith("openai/");
+
   const res = await generateText({
     model: getMultimodalImageModel(modelId),
     messages: [{ role: "user", content: userContent }],
     providerOptions: {
-      google: {
-        responseModalities: ["TEXT", "IMAGE"],
-      },
+      ...(isGoogleModel && {
+        google: {
+          responseModalities: ["TEXT", "IMAGE"],
+        },
+      }),
+      ...(isOpenAIModel && {
+        openai: {
+          modalities: ["text", "image"],
+        },
+      }),
     },
   });
 
@@ -362,21 +391,14 @@ The assistant must not add new subjects, claims, branding, or alter the tone or
         }
 
         // Traditional image generation for dedicated image models
-        const result = await runGenerateImageTraditional({
+        return await runGenerateImageTraditional({
           mode,
           prompt,
           imageParts,
           lastGeneratedImage,
           startMs,
+          costAccumulator,
         });
-
-        // Report API cost for traditional image generation
-        costAccumulator?.addAPICost(
-          "generateImage",
-          toolsDefinitions.generateImage.cost
-        );
-
-        return result;
       } catch (error) {
         const resolvedError = await resolveError(error);
         log.error(