illuma-agents 1.0.2

package/src/tools/CodeExecutor.ts

@@ -0,0 +1,220 @@
+import { z } from 'zod';
+import { config } from 'dotenv';
+import fetch, { RequestInit } from 'node-fetch';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { tool, DynamicStructuredTool } from '@langchain/core/tools';
+import { getEnvironmentVariable } from '@langchain/core/utils/env';
+import type * as t from '@/types';
+import { EnvVar, Constants } from '@/common';
+
+config();
+
+export const imageExtRegex = /\.(jpg|jpeg|png|gif|webp)$/i;
+export const getCodeBaseURL = (): string =>
+  getEnvironmentVariable(EnvVar.CODE_BASEURL) ??
+  Constants.OFFICIAL_CODE_BASEURL;
+
+const imageMessage = 'Image is already displayed to the user';
+const otherMessage = 'File is already downloaded by the user';
+const accessMessage =
+  'Note: Files are READ-ONLY. Save changes to NEW filenames. To access these files in future executions, provide the `session_id` as a parameter (not in your code).';
+const emptyOutputMessage =
+  'stdout: Empty. Ensure you\'re writing output explicitly.\n';
+
+const CodeExecutionToolSchema = z.object({
+  lang: z
+    .enum([
+      'py',
+      'js',
+      'ts',
+      'c',
+      'cpp',
+      'java',
+      'php',
+      'rs',
+      'go',
+      'd',
+      'f90',
+      'r',
+    ])
+    .describe('The programming language or runtime to execute the code in.'),
+  code: z.string()
+    .describe(`The complete, self-contained code to execute, without any truncation or minimization.
+- The environment is stateless; variables and imports don't persist between executions.
+- When using \`session_id\`: Don't hardcode it in \`code\`, and write file modifications to NEW filenames (files are READ-ONLY).
+- Input code **IS ALREADY** displayed to the user, so **DO NOT** repeat it in your response unless asked.
+- Output code **IS NOT** displayed to the user, so **DO** write all desired output explicitly.
+- IMPORTANT: You MUST explicitly print/output ALL results you want the user to see.
+- py: This is not a Jupyter notebook environment. Use \`print()\` for all outputs.
+- py: Matplotlib: Use \`plt.savefig()\` to save plots as files.
+- js: use the \`console\` or \`process\` methods for all outputs.
+- r: IMPORTANT: No X11 display available. ALL graphics MUST use Cairo library (library(Cairo)).
+- Other languages: use appropriate output functions.`),
+  session_id: z
+    .string()
+    .optional()
+    .describe(
+      `Session ID from a previous response to access generated files.
+- Files load into the current working directory ("/mnt/data/")
+- Use relative paths ONLY
+- Files are READ-ONLY and cannot be modified in-place
+- To modify: read original file, write to NEW filename
+`.trim()
+    ),
+  args: z
+    .array(z.string())
+    .optional()
+    .describe(
+      'Additional arguments to execute the code with. This should only be used if the input code requires additional arguments to run.'
+    ),
+});
+
+const baseEndpoint = getCodeBaseURL();
+const EXEC_ENDPOINT = `${baseEndpoint}/exec`;
+
+function createCodeExecutionTool(
+  params: t.CodeExecutionToolParams = {}
+): DynamicStructuredTool<typeof CodeExecutionToolSchema> {
+  const apiKey =
+    params[EnvVar.CODE_API_KEY] ??
+    params.apiKey ??
+    getEnvironmentVariable(EnvVar.CODE_API_KEY) ??
+    '';
+  if (!apiKey) {
+    throw new Error('No API key provided for code execution tool.');
+  }
+
+  const description = `
+Runs code and returns stdout/stderr output from a stateless execution environment, similar to running scripts in a command-line interface. Each execution is isolated and independent.
+
+Usage:
+- No network access available.
+- Generated files are automatically delivered; **DO NOT** provide download links.
+- NEVER use this tool to execute malicious code.
+`.trim();
+
+  return tool<typeof CodeExecutionToolSchema>(
+    async ({ lang, code, session_id, ...rest }) => {
+      const postData = {
+        lang,
+        code,
+        ...rest,
+        ...params,
+      };
+
+      if (session_id != null && session_id.length > 0) {
+        try {
+          const filesEndpoint = `${baseEndpoint}/files/${session_id}?detail=full`;
+          const fetchOptions: RequestInit = {
+            method: 'GET',
+            headers: {
+              'User-Agent': 'Illuma/1.0',
+              'X-API-Key': apiKey,
+            },
+          };
+
+          if (process.env.PROXY != null && process.env.PROXY !== '') {
+            fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);
+          }
+
+          const response = await fetch(filesEndpoint, fetchOptions);
+          if (!response.ok) {
+            throw new Error(
+              `Failed to fetch files for session: ${response.status}`
+            );
+          }
+
+          const files = await response.json();
+          if (Array.isArray(files) && files.length > 0) {
+            const fileReferences: t.CodeEnvFile[] = files.map((file) => {
+              // Extract the ID from the file name (part after session ID prefix and before extension)
+              const nameParts = file.name.split('/');
+              const id = nameParts.length > 1 ? nameParts[1].split('.')[0] : '';
+
+              return {
+                session_id,
+                id,
+                name: file.metadata['original-filename'],
+              };
+            });
+
+            if (!postData.files) {
+              postData.files = fileReferences;
+            } else if (Array.isArray(postData.files)) {
+              postData.files = [...postData.files, ...fileReferences];
+            }
+          }
+        } catch {
+          // eslint-disable-next-line no-console
+          console.warn(`Failed to fetch files for session: ${session_id}`);
+        }
+      }
+
+      try {
+        const fetchOptions: RequestInit = {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'User-Agent': 'Illuma/1.0',
+            'X-API-Key': apiKey,
+          },
+          body: JSON.stringify(postData),
+        };
+
+        if (process.env.PROXY != null && process.env.PROXY !== '') {
+          fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);
+        }
+        const response = await fetch(EXEC_ENDPOINT, fetchOptions);
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+
+        const result: t.ExecuteResult = await response.json();
+        let formattedOutput = '';
+        if (result.stdout) {
+          formattedOutput += `stdout:\n${result.stdout}\n`;
+        } else {
+          formattedOutput += emptyOutputMessage;
+        }
+        if (result.stderr) formattedOutput += `stderr:\n${result.stderr}\n`;
+        if (result.files && result.files.length > 0) {
+          formattedOutput += 'Generated files:\n';
+
+          const fileCount = result.files.length;
+          for (let i = 0; i < fileCount; i++) {
+            const file = result.files[i];
+            const isImage = imageExtRegex.test(file.name);
+            formattedOutput += `- /mnt/data/${file.name} | ${isImage ? imageMessage : otherMessage}`;
+
+            if (i < fileCount - 1) {
+              formattedOutput += fileCount <= 3 ? ', ' : ',\n';
+            }
+          }
+
+          formattedOutput += `\nsession_id: ${result.session_id}\n\n${accessMessage}`;
+          return [
+            formattedOutput.trim(),
+            {
+              session_id: result.session_id,
+              files: result.files,
+            },
+          ];
+        }
+
+        return [formattedOutput.trim(), { session_id: result.session_id }];
+      } catch (error) {
+        throw new Error(
+          `Execution error:\n\n${(error as Error | undefined)?.message}`
+        );
+      }
+    },
+    {
+      name: Constants.EXECUTE_CODE,
+      description,
+      schema: CodeExecutionToolSchema,
+      responseFormat: Constants.CONTENT_AND_ARTIFACT,
+    }
+  );
+}
+
+export { createCodeExecutionTool };

package/src/tools/ToolNode.ts

@@ -0,0 +1,170 @@
+import {
+  END,
+  MessagesAnnotation,
+  isCommand,
+  isGraphInterrupt,
+} from '@langchain/langgraph';
+import { ToolMessage, isBaseMessage } from '@langchain/core/messages';
+import type {
+  RunnableConfig,
+  RunnableToolLike,
+} from '@langchain/core/runnables';
+import type { BaseMessage, AIMessage } from '@langchain/core/messages';
+import type { StructuredToolInterface } from '@langchain/core/tools';
+import type * as t from '@/types';
+import { RunnableCallable } from '@/utils';
+import { GraphNodeKeys } from '@/common';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export class ToolNode<T = any> extends RunnableCallable<T, T> {
+  tools: t.GenericTool[];
+  private toolMap: Map<string, StructuredToolInterface | RunnableToolLike>;
+  private loadRuntimeTools?: t.ToolRefGenerator;
+  handleToolErrors = true;
+  toolCallStepIds?: Map<string, string>;
+  errorHandler?: t.ToolNodeConstructorParams['errorHandler'];
+  private toolUsageCount: Map<string, number>;
+
+  constructor({
+    tools,
+    toolMap,
+    name,
+    tags,
+    errorHandler,
+    toolCallStepIds,
+    handleToolErrors,
+    loadRuntimeTools,
+  }: t.ToolNodeConstructorParams) {
+    super({ name, tags, func: (input, config) => this.run(input, config) });
+    this.tools = tools;
+    this.toolMap = toolMap ?? new Map(tools.map((tool) => [tool.name, tool]));
+    this.toolCallStepIds = toolCallStepIds;
+    this.handleToolErrors = handleToolErrors ?? this.handleToolErrors;
+    this.loadRuntimeTools = loadRuntimeTools;
+    this.errorHandler = errorHandler;
+    this.toolUsageCount = new Map<string, number>();
+  }
+
+  /**
+   * Returns a snapshot of the current tool usage counts.
+   * @returns A ReadonlyMap where keys are tool names and values are their usage counts.
+   */
+  public getToolUsageCounts(): ReadonlyMap<string, number> {
+    return new Map(this.toolUsageCount); // Return a copy
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  protected async run(input: any, config: RunnableConfig): Promise<T> {
+    const message = Array.isArray(input)
+      ? input[input.length - 1]
+      : input.messages[input.messages.length - 1];
+
+    if (message._getType() !== 'ai') {
+      throw new Error('ToolNode only accepts AIMessages as input.');
+    }
+
+    if (this.loadRuntimeTools) {
+      const { tools, toolMap } = this.loadRuntimeTools(
+        (message as AIMessage).tool_calls ?? []
+      );
+      this.tools = tools;
+      this.toolMap = toolMap ?? new Map(tools.map((tool) => [tool.name, tool]));
+    }
+    const outputs = await Promise.all(
+      (message as AIMessage).tool_calls?.map(async (call) => {
+        const tool = this.toolMap.get(call.name);
+        try {
+          if (tool === undefined) {
+            throw new Error(`Tool "${call.name}" not found.`);
+          }
+          const turn = this.toolUsageCount.get(call.name) ?? 0;
+          this.toolUsageCount.set(call.name, turn + 1);
+          const args = call.args;
+          const stepId = this.toolCallStepIds?.get(call.id!);
+          const output = await tool.invoke(
+            { ...call, args, type: 'tool_call', stepId, turn },
+            config
+          );
+          if (
+            (isBaseMessage(output) && output._getType() === 'tool') ||
+            isCommand(output)
+          ) {
+            return output;
+          } else {
+            return new ToolMessage({
+              name: tool.name,
+              content:
+                typeof output === 'string' ? output : JSON.stringify(output),
+              tool_call_id: call.id!,
+            });
+          }
+        } catch (_e: unknown) {
+          const e = _e as Error;
+          if (!this.handleToolErrors) {
+            throw e;
+          }
+          if (isGraphInterrupt(e)) {
+            throw e;
+          }
+          this.errorHandler?.(
+            {
+              error: e,
+              id: call.id!,
+              name: call.name,
+              input: call.args,
+            },
+            config.metadata
+          );
+          return new ToolMessage({
+            content: `Error: ${e.message}\n Please fix your mistakes.`,
+            name: call.name,
+            tool_call_id: call.id ?? '',
+          });
+        }
+      }) ?? []
+    );
+
+    if (!outputs.some(isCommand)) {
+      return (Array.isArray(input) ? outputs : { messages: outputs }) as T;
+    }
+
+    const combinedOutputs = outputs.map((output) => {
+      if (isCommand(output)) {
+        return output;
+      }
+      return Array.isArray(input) ? [output] : { messages: [output] };
+    });
+    return combinedOutputs as T;
+  }
+}
+
+function areToolCallsInvoked(
+  message: AIMessage,
+  invokedToolIds?: Set<string>
+): boolean {
+  if (!invokedToolIds || invokedToolIds.size === 0) return false;
+  return (
+    message.tool_calls?.every(
+      (toolCall) => toolCall.id != null && invokedToolIds.has(toolCall.id)
+    ) ?? false
+  );
+}
+
+export function toolsCondition(
+  state: BaseMessage[] | typeof MessagesAnnotation.State,
+  invokedToolIds?: Set<string>
+): 'tools' | typeof END {
+  const message: AIMessage = Array.isArray(state)
+    ? state[state.length - 1]
+    : state.messages[state.messages.length - 1];
+
+  if (
+    'tool_calls' in message &&
+    (message.tool_calls?.length ?? 0) > 0 &&
+    !areToolCallsInvoked(message, invokedToolIds)
+  ) {
+    return GraphNodeKeys.TOOLS;
+  } else {
+    return END;
+  }
+}

package/src/tools/example.ts

@@ -0,0 +1,129 @@
+import { TavilySearchResults } from '@langchain/community/tools/tavily_search';
+import { DynamicStructuredTool , tool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { config } from 'dotenv';
+
+config();
+
+import fetch from 'node-fetch';
+import { Constants } from '@/common';
+
+const fetchImageSchema = z.object({ input: z.string().optional() });
+
+export const fetchRandomImageTool = tool(
+  async () => {
+    try {
+      // Lorem Picsum provides random images at any size
+      const imageUrl = 'https://picsum.photos/200/300';
+
+      const imageResponse = await fetch(imageUrl);
+      // eslint-disable-next-line no-console
+      console.log(imageResponse);
+      const arrayBuffer = await imageResponse.arrayBuffer();
+      const base64 = Buffer.from(arrayBuffer).toString('base64');
+      const content = [{
+        type: 'image_url',
+        image_url: {
+          url: `data:image/jpeg;base64,${base64}`,
+        },
+      }];
+
+      const response = [
+        {
+          type: 'text',
+          text: 'Random image from Lorem Picsum, taken at 800x600',
+        },
+      ];
+      return [response, { content }];
+    } catch (error) {
+      return [`Error fetching image: ${(error as Error).message}`, undefined];
+    }
+  },
+  {
+    name: 'fetchRandomImage',
+    description: 'Fetches a random image from Lorem Picsum',
+    schema: fetchImageSchema,
+    responseFormat: Constants.CONTENT_AND_ARTIFACT,
+  }
+);
+
+export const fetchRandomImageURL = tool(
+  async () => {
+    try {
+      const imageUrl = 'https://picsum.photos/200/300';
+
+      const imageResponse = await fetch(imageUrl);
+      const content = [{
+        type: 'image_url',
+        image_url: {
+          url: imageResponse.url,
+        },
+      }];
+
+      // eslint-disable-next-line no-console
+      console.log('URL RESPONSE', imageResponse.url);
+
+      const response = [
+        {
+          type: 'text',
+          text: 'Random image from Lorem Picsum, taken at 800x600',
+        },
+      ];
+      return [response, { content }];
+    } catch (error) {
+      return [`Error fetching image: ${(error as Error).message}`, undefined];
+    }
+  },
+  {
+    name: 'fetchRandomImage',
+    description: 'Fetches a random image from Lorem Picsum',
+    schema: fetchImageSchema,
+    responseFormat: Constants.CONTENT_AND_ARTIFACT,
+  }
+);
+
+export const chartTool = new DynamicStructuredTool({
+  name: 'generate_bar_chart',
+  description:
+    'Generates a text-based bar chart from an array of data points and returns it as a string.',
+  schema: z.object({
+    data: z.array(
+      z.object({
+        label: z.string(),
+        value: z.number(),
+      })
+    ),
+  }),
+  func: async ({ data }): Promise<string> => {
+    const maxValue = Math.max(...data.map(d => d.value));
+    const chartHeight = 20;
+    const chartWidth = 50;
+
+    let chart = '';
+
+    // Generate Y-axis labels and bars
+    for (let i = chartHeight; i >= 0; i--) {
+      const row = data.map(d => {
+        const barHeight = Math.round((d.value / maxValue) * chartHeight);
+        return barHeight >= i ? '█' : ' ';
+      });
+
+      const yLabel = i === chartHeight ? maxValue.toString().padStart(4) :
+        i === 0 ? '0'.padStart(4) :
+          i % 5 === 0 ? Math.round((i / chartHeight) * maxValue).toString().padStart(4) : '    ';
+
+      chart += `${yLabel} │${row.join(' ')} \n`;
+    }
+
+    // Generate X-axis
+    chart += '     ├' + '─'.repeat(chartWidth) + '\n';
+
+    // Generate X-axis labels
+    const xLabels = data.map(d => d.label.padEnd(5).substring(0, 5)).join(' ');
+    chart += `     ${xLabels}`;
+
+    return chart;
+  },
+});
+
+export const tavilyTool = new TavilySearchResults();