@proteinjs/conversation 2.1.2 → 2.1.4

package/src/OpenAi.ts

@@ -22,14 +22,44 @@ function delay(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+/** Structured capture of each tool call during a single generateResponse loop. */
+export type ToolInvocationResult = {
+  id: string; // tool_call_id from the model
+  name: string; // function name invoked
+  startedAt: Date;
+  finishedAt: Date;
+  input: unknown; // parsed JSON args (or raw string if parse failed)
+  ok: boolean;
+  data?: unknown; // tool return value (JSON-serializable)
+  error?: { message: string; stack?: string };
+};
+
+/** Realtime progress hook for tool calls. */
+export type ToolInvocationProgressEvent =
+  | {
+      type: 'started';
+      id: string;
+      name: string;
+      startedAt: Date;
+      input: unknown;
+    }
+  | {
+      type: 'finished';
+      result: ToolInvocationResult;
+    };
+
 export type GenerateResponseParams = {
   messages: (string | ChatCompletionMessageParam)[];
   model?: TiktokenModel;
+  /** Optional realtime hook for tool-call lifecycle (started/finished). */
+  onToolInvocation?: (evt: ToolInvocationProgressEvent) => void;
 };
 
 export type GenerateResponseReturn = {
   message: string;
   usagedata: UsageData;
+  /** Structured ledger of tool calls executed while producing this message. */
+  toolInvocations: ToolInvocationResult[];
 };
 
 export type GenerateStreamingResponseParams = GenerateResponseParams & {
@@ -42,6 +72,8 @@ type GenerateResponseHelperParams = GenerateStreamingResponseParams & {
   stream: boolean;
   currentFunctionCalls?: number;
   usageDataAccumulator?: UsageDataAccumulator;
+  /** Accumulated across recursive tool loops. */
+  toolInvocations?: ToolInvocationResult[];
 };
 
 export type OpenAiParams = {
@@ -53,7 +85,7 @@ export type OpenAiParams = {
   logLevel?: LogLevel;
 };
 
-export const DEFAULT_MODEL: TiktokenModel = 'gpt-3.5-turbo';
+export const DEFAULT_MODEL: TiktokenModel = 'gpt-4o';
 export const DEFAULT_MAX_FUNCTION_CALLS = 50;
 
 export class OpenAi {
@@ -77,7 +109,7 @@ export class OpenAi {
     this.functions = functions;
     this.messageModerators = messageModerators;
     this.maxFunctionCalls = maxFunctionCalls;
-    this.logLevel = logLevel;
+    this.logLevel = (process.env.PROTEINJS_CONVERSATION_OPENAI_LOG_LEVEL as LogLevel | undefined) ?? logLevel;
   }
 
   async generateResponse({ model, ...rest }: GenerateResponseParams): Promise<GenerateResponseReturn> {
@@ -85,11 +117,17 @@ export class OpenAi {
       model: model ?? this.model,
       stream: false,
       ...rest,
+      toolInvocations: [],
     })) as GenerateResponseReturn;
   }
 
   async generateStreamingResponse({ model, ...rest }: GenerateStreamingResponseParams): Promise<Readable> {
-    return (await this.generateResponseHelper({ model: model ?? this.model, stream: true, ...rest })) as Readable;
+    return (await this.generateResponseHelper({
+      model: model ?? this.model,
+      stream: true,
+      ...rest,
+      toolInvocations: [],
+    })) as Readable;
   }
 
   private async generateResponseHelper({
@@ -98,8 +136,10 @@ export class OpenAi {
     stream,
     abortSignal,
     onUsageData,
+    onToolInvocation,
     usageDataAccumulator,
     currentFunctionCalls = 0,
+    toolInvocations = [],
   }: GenerateResponseHelperParams): Promise<GenerateResponseReturn | Readable> {
     const logger = new Logger({ name: 'OpenAi.generateResponseHelper', logLevel: this.logLevel });
     this.updateMessageHistory(messages);
@@ -130,7 +170,9 @@ export class OpenAi {
             currentFunctionCalls,
             resolvedUsageDataAccumulator,
             abortSignal,
-            onUsageData
+            onUsageData,
+            toolInvocations,
+            onToolInvocation
           )) as (toolCalls: ChatCompletionMessageToolCall[], currentFunctionCalls: number) => Promise<Readable>;
         const streamProcessor = new OpenAiStreamProcessor(
           inputStream,
@@ -152,7 +194,9 @@ export class OpenAi {
           currentFunctionCalls,
           resolvedUsageDataAccumulator,
           abortSignal,
-          onUsageData
+          onUsageData,
+          toolInvocations,
+          onToolInvocation
         );
       }
 
@@ -162,7 +206,7 @@ export class OpenAi {
       }
 
       this.history.push([responseMessage]);
-      return { message: responseText, usagedata: resolvedUsageDataAccumulator.usageData };
+      return { message: responseText, usagedata: resolvedUsageDataAccumulator.usageData, toolInvocations };
     };
 
     // Only wrap in context if this is the first call
@@ -208,7 +252,6 @@ export class OpenAi {
       const response = await openaiApi.chat.completions.create(
         {
           model,
-          temperature: 0,
           messages: this.history.getMessages(),
           ...(this.functions &&
             this.functions.length > 0 && {
@@ -310,7 +353,9 @@ export class OpenAi {
     currentFunctionCalls: number,
     usageDataAccumulator: UsageDataAccumulator,
     abortSignal?: AbortSignal,
-    onUsageData?: (usageData: UsageData) => Promise<void>
+    onUsageData?: (usageData: UsageData) => Promise<void>,
+    toolInvocations: ToolInvocationResult[] = [],
+    onToolInvocation?: (evt: ToolInvocationProgressEvent) => void
   ): Promise<GenerateResponseReturn | Readable> {
     if (currentFunctionCalls >= this.maxFunctionCalls) {
       throw new Error(`Max function calls (${this.maxFunctionCalls}) reached. Stopping execution.`);
@@ -327,7 +372,7 @@ export class OpenAi {
     this.history.push([toolCallMessage]);
 
     // Call the tools and get the responses
-    const toolMessageParams = await this.callTools(toolCalls, usageDataAccumulator);
+    const toolMessageParams = await this.callTools(toolCalls, usageDataAccumulator, toolInvocations, onToolInvocation);
 
     // Add the tool responses to the history
     this.history.push(toolMessageParams);
@@ -339,18 +384,31 @@ export class OpenAi {
       stream,
       abortSignal,
       onUsageData,
+      onToolInvocation,
       usageDataAccumulator,
       currentFunctionCalls: currentFunctionCalls + toolCalls.length,
+      toolInvocations,
     });
   }
 
   private async callTools(
     toolCalls: ChatCompletionMessageToolCall[],
-    usageDataAccumulator: UsageDataAccumulator
+    usageDataAccumulator: UsageDataAccumulator,
+    toolInvocations: ToolInvocationResult[],
+    onToolInvocation?: (evt: ToolInvocationProgressEvent) => void
   ): Promise<ChatCompletionMessageParam[]> {
     const toolMessageParams: ChatCompletionMessageParam[] = (
       await Promise.all(
-        toolCalls.map(async (toolCall) => await this.callFunction(toolCall.function, toolCall.id, usageDataAccumulator))
+        toolCalls.map(
+          async (toolCall) =>
+            await this.callFunction(
+              toolCall.function,
+              toolCall.id,
+              usageDataAccumulator,
+              toolInvocations,
+              onToolInvocation
+            )
+        )
       )
     ).reduce((acc, val) => acc.concat(val), []);
 
@@ -360,7 +418,9 @@ export class OpenAi {
   private async callFunction(
     functionCall: ChatCompletionMessageToolCall.Function,
     toolCallId: string,
-    usageDataAccumulator: UsageDataAccumulator
+    usageDataAccumulator: UsageDataAccumulator,
+    toolInvocations: ToolInvocationResult[],
+    onToolInvocation?: (evt: ToolInvocationProgressEvent) => void
   ): Promise<ChatCompletionMessageParam[]> {
     const logger = new Logger({ name: 'OpenAi.callFunction', logLevel: this.logLevel });
     if (!this.functions) {
@@ -370,7 +430,7 @@ export class OpenAi {
     }
 
     functionCall.name = functionCall.name.split('.').pop() as string;
-    const f = this.functions.find((f) => f.definition.name === functionCall.name);
+    const f = this.functions.find((fx) => fx.definition.name === functionCall.name);
     if (!f) {
       const errorMessage = `Assistant attempted to call nonexistent function`;
       logger.error({ message: errorMessage, obj: { functionName: functionCall.name } });
@@ -390,7 +450,33 @@ export class OpenAi {
         obj: { toolCallId, functionName: f.definition.name, args: parsedArguments },
       });
       usageDataAccumulator.recordToolCall(f.definition.name);
+
+      const startedAt = new Date();
+
+      onToolInvocation?.({
+        type: 'started',
+        id: toolCallId,
+        name: f.definition.name,
+        startedAt,
+        input: parsedArguments,
+      });
+
       const returnObject = await f.call(parsedArguments);
+      const finishedAt = new Date();
+
+      // Record success
+      const rec: ToolInvocationResult = {
+        id: toolCallId,
+        name: f.definition.name,
+        startedAt,
+        finishedAt,
+        input: parsedArguments,
+        ok: true,
+        data: returnObject,
+      };
+      toolInvocations.push(rec);
+
+      onToolInvocation?.({ type: 'finished', result: rec });
 
       const returnObjectCompletionParams: ChatCompletionMessageParam[] = [];
       if (isInstanceOf(returnObject, ChatCompletionMessageParamFactory)) {
@@ -433,11 +519,35 @@ export class OpenAi {
 
       return returnObjectCompletionParams;
     } catch (error: any) {
+      const now = new Date();
+      const attemptedArgs = (() => {
+        try {
+          return JSON.parse(functionCall.arguments);
+        } catch {
+          return functionCall.arguments;
+        }
+      })();
+
+      // Record failure
+      const rec: ToolInvocationResult = {
+        id: toolCallId,
+        name: functionCall.name,
+        startedAt: now,
+        finishedAt: now,
+        input: attemptedArgs,
+        ok: false,
+        error: { message: String(error?.message ?? error), stack: (error as any)?.stack },
+      };
+      toolInvocations.push(rec);
+
+      onToolInvocation?.({ type: 'finished', result: rec });
+
       logger.error({
         message: `An error occurred while executing function`,
         error,
         obj: { toolCallId, functionName: f.definition.name },
       });
+
       throw error;
     }
   }

package/src/fs/conversation_fs/ConversationFsModule.ts

@@ -6,7 +6,8 @@ import { ConversationFsModerator } from './ConversationFsModerator';
 import {
   fsFunctions,
   getRecentlyAccessedFilePathsFunction,
-  getRecentlyAccessedFilePathsFunctionName,
+  grepFunction,
+  grepFunctionName,
   readFilesFunction,
   readFilesFunctionName,
   writeFilesFunction,
@@ -36,7 +37,7 @@ export class ConversationFsModule implements ConversationModule {
       `When reading/writing a file in a specified package, join the package directory with the relative path to form the file path`,
       `When searching for source files, do not look in the dist or node_modules directories`,
       `If you don't know a file path, don't try to guess it, use the ${searchFilesFunctionName} function to find it`,
-      `When searching for something (ie. a file to work with/in), unless more context is specified, use the ${searchLibrariesFunctionName} function first, then fall back to functions: ${searchPackagesFunctionName}, ${searchFilesFunctionName}`,
+      `When searching for something (ie. a file to work with/in), unless more context is specified, use the ${grepFunctionName} function first, then fall back to functions: ${searchPackagesFunctionName}, ${searchFilesFunctionName}`,
       `After finding a file to work with, assume the user's following question pertains to that file and use ${readFilesFunctionName} to read the file if needed`,
     ];
   }
@@ -46,6 +47,7 @@ export class ConversationFsModule implements ConversationModule {
       readFilesFunction(this),
       writeFilesFunction(this),
       getRecentlyAccessedFilePathsFunction(this),
+      grepFunction(this),
       ...fsFunctions,
     ];
   }
@@ -61,6 +63,10 @@ export class ConversationFsModule implements ConversationModule {
   getRecentlyAccessedFilePaths() {
     return this.recentlyAccessedFilePaths;
   }
+
+  getRepoPath(): string {
+    return this.repoPath;
+  }
 }
 
 export class ConversationFsModuleFactory implements ConversationModuleFactory {

package/src/fs/conversation_fs/FsFunctions.ts

@@ -1,6 +1,50 @@
 import { File, Fs } from '@proteinjs/util-node';
 import { Function } from '../../Function';
 import { ConversationFsModule } from './ConversationFsModule';
+import path from 'path';
+
+const toRepoAbs = (mod: ConversationFsModule, p: string) => (path.isAbsolute(p) ? p : path.join(mod.getRepoPath(), p));
+
+// If path doesn’t exist, try to resolve "<repo>/<basename>" to the actual file under repo
+async function canonicalizePaths(mod: ConversationFsModule, paths: string[]): Promise<string[]> {
+  const repo = mod.getRepoPath();
+  const ignore = ['**/node_modules/**', '**/dist/**', '**/.git/**'];
+
+  const out: string[] = [];
+  for (const p of paths) {
+    const abs = toRepoAbs(mod, p);
+    if (await Fs.exists(abs)) {
+      out.push(abs);
+      continue;
+    }
+    const base = path.basename(p);
+    if (!base) {
+      out.push(abs);
+      continue;
+    }
+    const parsed = path.parse(base); // { name, ext }
+    const pattern = parsed.ext ? `**/${parsed.name}${parsed.ext}` : `**/${parsed.name}.*`;
+
+    let matches: string[] = [];
+    try {
+      matches = await (Fs as any).getFilePathsMatchingGlob(repo, pattern, ignore);
+    } catch {
+      // fall through
+    }
+
+    if (matches.length === 1) {
+      out.push(matches[0]);
+    } else if (matches.length > 1) {
+      // Prefer the shortest match (usually “src/...” beats deeper/duplicate locations)
+      matches.sort((a, b) => a.length - b.length);
+      out.push(matches[0]);
+    } else {
+      // No luck; keep the original absolute (will throw with a clear error)
+      out.push(abs);
+    }
+  }
+  return out;
+}
 
 export const readFilesFunctionName = 'readFiles';
 export function readFilesFunction(fsModule: ConversationFsModule) {
@@ -14,9 +58,7 @@ export function readFilesFunction(fsModule: ConversationFsModule) {
           filePaths: {
             type: 'array',
             description: 'Paths to the files',
-            items: {
-              type: 'string',
-            },
+            items: { type: 'string' },
           },
         },
         required: ['filePaths'],
@@ -24,7 +66,8 @@ export function readFilesFunction(fsModule: ConversationFsModule) {
     },
     call: async (params: { filePaths: string[] }) => {
       fsModule.pushRecentlyAccessedFilePath(params.filePaths);
-      return await Fs.readFiles(params.filePaths);
+      const absPaths = await canonicalizePaths(fsModule, params.filePaths);
+      return await Fs.readFiles(absPaths);
     },
     instructions: [`To read files from the local file system, use the ${readFilesFunctionName} function`],
   };
@@ -41,19 +84,10 @@ export function writeFilesFunction(fsModule: ConversationFsModule) {
         properties: {
           files: {
             type: 'array',
-            description: 'Files to write',
             items: {
               type: 'object',
-              properties: {
-                path: {
-                  type: 'string',
-                  description: 'the file path',
-                },
-                content: {
-                  type: 'string',
-                  description: 'the content to write to the file',
-                },
-              },
+              properties: { path: { type: 'string' }, content: { type: 'string' } },
+              required: ['path', 'content'],
             },
           },
         },
@@ -61,8 +95,13 @@ export function writeFilesFunction(fsModule: ConversationFsModule) {
       },
     },
     call: async (params: { files: File[] }) => {
-      fsModule.pushRecentlyAccessedFilePath(params.files.map((file) => file.path));
-      return await Fs.writeFiles(params.files);
+      fsModule.pushRecentlyAccessedFilePath(params.files.map((f) => f.path));
+      const canon = await canonicalizePaths(
+        fsModule,
+        params.files.map((f) => f.path)
+      );
+      const absFiles = params.files.map((f, i) => ({ ...f, path: canon[i] }));
+      return await Fs.writeFiles(absFiles);
     },
     instructions: [`To write files to the local file system, use the ${writeFilesFunctionName} function`],
   };
@@ -232,6 +271,47 @@ const moveFunction: Function = {
   instructions: [`To move a file or directory, use the ${moveFunctionName} function`],
 };
 
+export const grepFunctionName = 'grep';
+export function grepFunction(fsModule: ConversationFsModule) {
+  return {
+    definition: {
+      name: grepFunctionName,
+      description:
+        "Run system grep recursively (-F literal) within the repository and return raw stdout/stderr/code. Excludes node_modules, dist, and .git. Use 'maxResults' to cap output.",
+      parameters: {
+        type: 'object',
+        properties: {
+          pattern: {
+            type: 'string',
+            description:
+              'Literal text to search for (grep -F). For parentheses or special characters, pass them as-is; no regex needed.',
+          },
+          dir: {
+            type: 'string',
+            description:
+              'Directory to search under. If relative, it is resolved against the repo root. Defaults to the repo root.',
+          },
+          maxResults: {
+            type: 'number',
+            description: 'Maximum number of matching lines to return (uses grep -m N).',
+          },
+        },
+        required: ['pattern'],
+      },
+    },
+    call: async (params: { pattern: string; dir?: string; maxResults?: number }) => {
+      const repo = fsModule.getRepoPath();
+      const cwd = params.dir ? toRepoAbs(fsModule, params.dir) : repo;
+      return await Fs.grep({ pattern: params.pattern, dir: cwd, maxResults: params.maxResults });
+    },
+    instructions: [
+      `Use ${grepFunctionName} to search for literal text across the repo.`,
+      `Prefer small 'maxResults' (e.g., 5-20) to avoid flooding the context.`,
+      `Parse the returned stdout yourself (format: "<path>:<line>:<text>") to pick files to read.`,
+    ],
+  };
+}
+
 export const fsFunctions: Function[] = [
   createFolderFunction,
   fileOrDirectoryExistsFunction,

package/src/fs/keyword_to_files_index/KeywordToFilesIndexModule.ts

@@ -7,7 +7,8 @@ import { searchFilesFunction, searchFilesFunctionName } from './KeywordToFilesIn
 
 export type KeywordToFilesIndexModuleParams = {
   dir: string;
-  keywordFilesIndex: { [keyword: string]: string[] /** file paths */ };
+  // Map from lowercase filename *stem* (no extension) → file paths
+  keywordFilesIndex: { [keyword: string]: string[] };
 };
 
 export class KeywordToFilesIndexModule implements ConversationModule {
@@ -22,15 +23,19 @@ export class KeywordToFilesIndexModule implements ConversationModule {
     return 'Keyword to files index';
   }
 
+  /**
+   * Case-insensitive file name search that ignores extension.
+   */
   searchFiles(params: { keyword: string }) {
-    this.logger.info({ message: `Searching for file, keyword: ${params.keyword}` });
-    const filePaths = this.params.keywordFilesIndex[params.keyword];
+    this.logger.debug({ message: `Searching for file, keyword: ${params.keyword}` });
+    const keywordLowerNoExtension = path.parse(params.keyword).name.toLowerCase();
+    const filePaths = this.params.keywordFilesIndex[keywordLowerNoExtension];
     return filePaths || [];
   }
 
   getSystemMessages(): string[] {
     return [
-      `If you're searching for something, use the ${searchFilesFunctionName} function to find a file matching the search string`,
+      `If you're searching for something, use the ${searchFilesFunctionName} function to find a file (by name) matching the search string`,
     ];
   }
 
@@ -76,14 +81,14 @@ export class KeywordToFilesIndexModuleFactory implements ConversationModuleFacto
 
     // Process each file path
     for (const filePath of filePaths) {
-      const fileName = path.parse(filePath).name; // Get file name without extension
+      const fileNameLower = path.parse(filePath).name.toLowerCase(); // Get file name without extension
 
-      if (!keywordFilesIndex[fileName]) {
-        keywordFilesIndex[fileName] = [];
+      if (!keywordFilesIndex[fileNameLower]) {
+        keywordFilesIndex[fileNameLower] = [];
       }
 
-      this.logger.debug({ message: `fileName: ${fileName}, filePath: ${filePath}` });
-      keywordFilesIndex[fileName].push(filePath);
+      this.logger.debug({ message: `fileName: ${fileNameLower}, filePath: ${filePath}` });
+      keywordFilesIndex[fileNameLower].push(filePath);
     }
 
     return keywordFilesIndex;