@proteinjs/conversation 1.0.1

package/src/fs/keyword_to_files_index/KeywordToFilesIndexFunctions.ts

@@ -0,0 +1,55 @@
+import { KeywordToFilesIndexModule } from './KeywordToFilesIndexModule'
+
+export const searchFilesFunctionName = 'searchFiles';
+export const searchFilesFunction = (module: KeywordToFilesIndexModule) => {
+  return {
+    definition: {
+      name: searchFilesFunctionName,
+      description: 'Get file paths to files that contain keyword',
+      parameters: {
+        type: 'object',
+        properties: {
+          keyword: {
+            type: 'string',
+            description: 'Search files for this keyword'
+          },
+        },
+        required: ['keyword']
+      },
+    },
+    call: async (params: { keyword: string }) => module.searchFiles(params),
+    instructions: [
+      `If the user is trying to interact with a file, but does not provide a path, you can find file paths that match a keyword using the ${searchFilesFunctionName} function`,
+      `Only call functions that take in filePaths with valid file paths, if you don't know the valid file path try and search for it by keyword with the ${searchFilesFunctionName} function`,
+      `If the user references a file in a package without providing a path, use the ${searchFilesFunctionName} function on the keyword to find potentially relevant files, and choose the one that references the package name in its path`,
+    ],
+  }
+}
+
+// {
+      //   definition: {
+      //     name: 'getDeclarations',
+      //     description: 'Get the typescript declarations of files',
+      //     parameters: {
+      //       type: 'object',
+      //       properties: {
+      //         tsFilePaths: {
+      //           type: 'array',
+      //           description: 'Paths to the files',
+      //           items: {
+      //             type: 'string',
+      //           },
+      //         },
+      //         includeDependencyDeclarations: {
+      //           type: 'boolean',
+      //           description: 'if true, returns declarations for input tsFilePaths and all dependencies. defaults to false.'
+      //         },
+      //       },
+      //       required: ['tsFilePaths']
+      //     },
+      //   },
+      //   call: async (params: { tsFilePaths: string[] }) => this.repo.getDeclarations(params),
+      //   instructions: [
+      //     `Favor calling getDeclarations over readFiles if full file content is not needed`,
+      //   ],
+      // },

package/src/fs/keyword_to_files_index/KeywordToFilesIndexModule.ts

@@ -0,0 +1,90 @@
+import { Logger } from '@proteinjs/util';
+import { Fs } from '@proteinjs/util-node';
+import { ConversationModule, ConversationModuleFactory } from '../../ConversationModule';
+import { Function } from '../../Function';
+import path from 'path';
+import { searchFilesFunction, searchFilesFunctionName } from './KeywordToFilesIndexFunctions';
+
+export type KeywordToFilesIndexModuleParams = {
+  dir: string,
+  keywordFilesIndex: { [keyword: string]: string[] /** file paths */ },
+}
+
+export class KeywordToFilesIndexModule implements ConversationModule {
+  private logger = new Logger(this.constructor.name);
+  params: KeywordToFilesIndexModuleParams;
+
+  constructor(params: KeywordToFilesIndexModuleParams) {
+    this.params = params;
+  }
+
+  getName(): string {
+    return 'Keyword to files index';
+  }
+
+  searchFiles(params: { keyword: string }) {
+    this.logger.info(`Searching for file, keyword: ${params.keyword}`);
+    const filePaths = this.params.keywordFilesIndex[params.keyword];
+    return filePaths || [];
+  }
+
+  getSystemMessages(): string[] {
+    return [
+      `If you're searching for something, use the ${searchFilesFunctionName} function to find a file matching the search string`,
+    ];
+  }
+
+  getFunctions(): Function[] {
+    return [
+      searchFilesFunction(this),
+    ];
+  }
+
+  getMessageModerators() {
+    return [];
+  }
+}
+
+export class KeywordToFilesIndexModuleFactory implements ConversationModuleFactory {
+  private logger = new Logger(this.constructor.name);
+
+  async createModule(repoPath: string): Promise<KeywordToFilesIndexModule> {
+    this.logger.debug(`Creating module for repo: ${repoPath}`);
+    let repoParams: KeywordToFilesIndexModuleParams = { keywordFilesIndex: {}, dir: repoPath };
+    repoParams.keywordFilesIndex = await this.createKeywordFilesIndex(repoPath, ['**/node-typescript-parser/**']);
+    this.logger.debug(`Created module for repo: ${repoPath}`);
+    return new KeywordToFilesIndexModule(repoParams);
+  }
+
+  /**
+   * Create keyword-files index for the given base directory.
+   * 
+   * @param baseDir - The directory to start the file search from.
+   * @returns An index with keywords mapped to file paths.
+   */
+  async createKeywordFilesIndex(baseDir: string, globIgnorePatterns: string[] = []): Promise<{ [keyword: string]: string[] }> {
+    // Ensure the base directory has a trailing slash
+    if (!baseDir.endsWith(path.sep)) {
+        baseDir += path.sep;
+    }
+
+    // Get all file paths, recursively, excluding node_modules and dist directories
+    const filePaths = await Fs.getFilePaths(baseDir, ['**/node_modules/**', '**/dist/**', ...globIgnorePatterns]);
+
+    const keywordFilesIndex: { [keyword: string]: string[] } = {};
+
+    // Process each file path
+    for (const filePath of filePaths) {
+        const fileName = path.parse(filePath).name; // Get file name without extension
+
+        if (!keywordFilesIndex[fileName]) {
+            keywordFilesIndex[fileName] = [];
+        }
+        
+        this.logger.debug(`fileName: ${fileName}, filePath: ${filePath}`);
+        keywordFilesIndex[fileName].push(filePath);
+    }
+
+    return keywordFilesIndex;
+  }
+}

package/src/fs/package/PackageFunctions.ts

@@ -0,0 +1,210 @@
+import { Package, PackageUtil } from '@proteinjs/util-node';
+import { Function } from '../../Function';
+import { PackageModule } from './PackageModule';
+
+export const installPackagesFunctionName = 'installPackages';
+export const installPackagesFunction: Function = {
+  definition: {
+    name: installPackagesFunctionName,
+    description: 'Get the content of files',
+    parameters: {
+      type: 'object',
+      properties: {
+        packages: {
+          type: 'array',
+          description: 'Packages to install',
+          items: {
+            type: 'object',
+            properties: {
+              name: {
+                type: 'string',
+                description: 'Name of package',
+              },
+              version: {
+                type: 'string',
+                description: 'Version of package',
+              },
+              exactVersion: {
+                type: 'boolean',
+                description: 'If true, freeze the version so subsequent installs do not bump minor versions',
+              },
+              development: {
+                type: 'boolean',
+                description: 'If true, install as a dev dependency',
+              },
+            },
+            required: ['name'],
+          },
+        },
+        cwdPath: {
+          type: 'string',
+          description: 'The directory to install packages in',
+        }
+      },
+      required: ['packages', 'cwdPath']
+    },
+  },
+  call: async (params: { packages: Package[], cwdPath: string }) => await PackageUtil.installPackages(params.packages, params.cwdPath),
+  instructions: [
+    `To install a package, use the ${installPackagesFunctionName} function`,
+  ],
+}
+
+export const runPackageScriptFunctionName = 'runPackageScript';
+const runPackageScriptFunction: Function = {
+  definition: {
+    name: runPackageScriptFunctionName,
+    description: 'Run `npm run x`, where `x` is an existing script in a package.json',
+    parameters: {
+      type: 'object',
+      properties: {
+        name: {
+          type: 'string',
+          description: 'Name of the script',
+        },
+        cwdPath: {
+          type: 'string',
+          description: 'If omitted, defaults to process.cwd',
+        }
+      },
+      required: ['name']
+    },
+  },
+  call: async (params: { name: string, cwdPath?: string }) => await PackageUtil.runPackageScript(params.name, params.cwdPath),
+  instructions: [
+    `To run a npm script (such as start, test, or watch), use the ${runPackageScriptFunctionName} function`,
+  ],
+}
+
+export const searchPackagesFunctionName = 'searchPackages';
+export function searchPackagesFunction(packageModule: PackageModule) {
+  return {
+    definition: {
+      name: searchPackagesFunctionName,
+      description: 'Get package.json file paths for package names that include the keyword',
+      parameters: {
+        type: 'object',
+        properties: {
+          keyword: {
+            type: 'string',
+            description: 'Keyword to match package names',
+          }
+        },
+        required: ['keyword']
+      },
+    },
+    call: async (params: { keyword: string }) => await packageModule.searchPackages(params.keyword),
+    instructions: [
+      `To search for packages in the local repo, use the ${searchPackagesFunctionName} function`,
+    ],
+  }
+}
+
+export const searchLibrariesFunctionName = 'searchLibraries';
+export function searchLibrariesFunction(packageModule: PackageModule) {
+  return {
+    definition: {
+      name: searchLibrariesFunctionName,
+      description: 'Return libraries that match the keyword',
+      parameters: {
+        type: 'object',
+        properties: {
+          keyword: {
+            type: 'string',
+            description: 'Keyword to match file names',
+          }
+        },
+        required: ['keyword']
+      },
+    },
+    call: async (params: { keyword: string }) => await packageModule.searchLibraries(params.keyword),
+    instructions: [
+      `To search for libraries in the local repo, use the ${searchLibrariesFunctionName} function`,
+    ],
+  }
+}
+
+export const generateTypescriptDeclarationsFunctionName = 'generateTypescriptDesclarations';
+export const generateTypescriptDeclarationsFunction = {
+  definition: {
+    name: generateTypescriptDeclarationsFunctionName,
+    description: 'Return the typescript declarations for the files',
+    parameters: {
+      type: 'object',
+      properties: {
+        tsFilePaths: {
+          type: 'array',
+          description: 'File paths to generate declarations for',
+          items: {
+            type: 'string',
+          }
+        }
+      },
+      required: ['tsFilePaths']
+    },
+  },
+  call: async (params: { tsFilePaths: string[] }) => PackageUtil.generateTypescriptDeclarations(params),
+  instructions: [
+    `To generate typescript declarations for a local file, use the ${generateTypescriptDeclarationsFunctionName} function`,
+  ],
+}
+
+export const npmInstallFunctionName = 'npmInstall';
+export const npmInstallFunction: Function = {
+  definition: {
+    name: npmInstallFunctionName,
+    description: 'Run `npm install` in the specified directory',
+    parameters: {
+      type: 'object',
+      properties: {
+        cwdPath: {
+          type: 'string',
+          description: 'Directory to execute the command from',
+        }
+      },
+      required: ['cwdPath']
+    },
+  },
+  call: async (params: { cwdPath: string }) => await PackageUtil.npmInstall(params.cwdPath),
+  instructions: [
+    `To run npm install in a specific directory, use the ${npmInstallFunctionName} function`,
+  ],
+}
+
+export const uninstallPackagesFunctionName = 'uninstallPackages';
+export const uninstallPackagesFunction: Function = {
+  definition: {
+    name: uninstallPackagesFunctionName,
+    description: 'Uninstall packages',
+    parameters: {
+      type: 'object',
+      properties: {
+        packageNames: {
+          type: 'array',
+          description: 'Packages to uninstall',
+          items: {
+            type: 'string',
+            description: 'Name of package',
+          },
+        },
+        cwdPath: {
+          type: 'string',
+          description: 'The directory to uninstall packages from',
+        }
+      },
+      required: ['packageNames', 'cwdPath']
+    },
+  },
+  call: async (params: { packageNames: string[], cwdPath: string }) => await PackageUtil.uninstallPackages(params.packageNames, params.cwdPath),
+  instructions: [
+    `To uninstall a package, use the ${uninstallPackagesFunctionName} function`,
+  ],
+}
+
+export const packageFunctions: Function[] = [
+  installPackagesFunction,
+  runPackageScriptFunction,
+  generateTypescriptDeclarationsFunction,
+  npmInstallFunction,
+  uninstallPackagesFunction,
+]

package/src/fs/package/PackageModule.ts

@@ -0,0 +1,106 @@
+import { Fs } from '@proteinjs/util-node';
+import { ConversationModule, ConversationModuleFactory } from '../../ConversationModule';
+import { Function } from '../../Function';
+import { packageFunctions, searchLibrariesFunction, searchLibrariesFunctionName, searchPackagesFunction, searchPackagesFunctionName } from './PackageFunctions';
+import path from 'path';
+import { searchFilesFunctionName } from '../keyword_to_files_index/KeywordToFilesIndexFunctions';
+import { readFilesFunctionName } from '../conversation_fs/FsFunctions';
+
+export type Library = {
+  fileName: string,
+  filePath: string,
+  packageName: string,
+}
+
+export type LibraryImport = {
+  importStatements: string[],
+  typescriptDeclaration: string,
+}
+
+export class PackageModule implements ConversationModule {
+  private repoPath: string;
+
+  constructor(repoPath: string) {
+    this.repoPath = repoPath;
+  }
+
+  getName(): string {
+    return 'Package';
+  }
+
+  getSystemMessages(): string[] {
+    return [
+      `When generating code, prefer importing code from local packages`,
+      `Use the ${searchPackagesFunctionName} function on every package you plan to install to derermine if the package is in the local repo; if it is, calculate the relative path from the cwd package to the package being installed, use that path as the version when installing the package`,
+      // `When generating code, use the searchFiles function to find all file paths to index.ts files; these are the local apis we have access to`,
+      // `When generating import statements, use the searchFiles function to find all file paths to package.json files; if importing from a local package, make sure you import via its package if it is not a local file to the package we're generating code in`,
+      `When generating import statements from another package, do not use relative paths`,
+    ];
+  }
+
+  getFunctions(): Function[] {
+    return [
+      ...packageFunctions,
+      searchPackagesFunction(this),
+      searchLibrariesFunction(this),
+    ];
+  }
+
+  getMessageModerators() {
+    return [];
+  }
+
+  /**
+   * @param keyword either the package name or some substring
+   * @return string[] of file paths
+   */
+  async searchPackages(keyword: string): Promise<string[]> {
+    const matchingPackageJsonPaths: string[] = [];
+    const packageJsonFilePaths = await Fs.getFilePathsMatchingGlob(this.repoPath, '**/package.json', ['**/node_modules/**', '**/dist/**']);
+    const packageJsonFileMap = await Fs.readFiles(packageJsonFilePaths);
+    for (let packageJsonFilePath of Object.keys(packageJsonFileMap)) {
+      const packageJson = JSON.parse(packageJsonFileMap[packageJsonFilePath]);
+      if (packageJson.name.toLowerCase().includes(keyword.toLocaleLowerCase()))
+        matchingPackageJsonPaths.push(packageJsonFilePath);
+    }
+
+    return matchingPackageJsonPaths;
+  }
+
+  /**
+   * Search packages in repo for file names that include keyword
+   * @param keyword substring of file name to search for
+   * @returns all libraries in the repo matching the keyword
+   */
+  async searchLibraries(keyword: string): Promise<Library[]> {
+    const matchingLibraries: Library[] = [];
+    const packageJsonFilePaths = await Fs.getFilePathsMatchingGlob(this.repoPath, '**/package.json', ['**/node_modules/**', '**/dist/**']);
+    const packageJsonFileMap = await Fs.readFiles(packageJsonFilePaths);
+    for (let packageJsonFilePath of Object.keys(packageJsonFileMap)) {
+      const packageJson = JSON.parse(packageJsonFileMap[packageJsonFilePath]);
+      const packageJsonFilePathParts = packageJsonFilePath.split(path.sep);
+      packageJsonFilePathParts.pop();
+      const packageDirectory = packageJsonFilePathParts.join(path.sep);
+      const srcFilePaths = await Fs.getFilePathsMatchingGlob(path.join(packageDirectory, 'src'), '**/*.ts', ['**/node_modules/**', '**/dist/**']);
+      for (let srcFilePath of srcFilePaths) {
+        const fileNameWithExtension = path.basename(srcFilePath);
+        if (fileNameWithExtension.includes(keyword)) {
+          const fileName = path.basename(srcFilePath, path.extname(srcFilePath));
+          matchingLibraries.push({
+            fileName,
+            filePath: srcFilePath,
+            packageName: packageJson.name,
+          });
+        }
+      }
+    }
+
+    return matchingLibraries;
+  }
+}
+
+export class PackageModuleFactory implements ConversationModuleFactory {
+  async createModule(repoPath: string): Promise<PackageModule> {
+    return new PackageModule(repoPath);
+  }
+}

package/src/history/MessageHistory.ts

@@ -0,0 +1,57 @@
+import { ChatCompletionMessageParam } from 'openai/resources/chat';
+import { Logger } from '@proteinjs/util';
+
+export interface MessageHistoryParams {
+  enforceMessageLimit?: boolean;
+  maxMessages: number; // max number of non-system messages to retain, fifo
+}
+
+export class MessageHistory {
+  private logger = new Logger(this.constructor.name);
+  private messages: ChatCompletionMessageParam[] = [];
+  private params: MessageHistoryParams;
+
+  constructor(params?: Partial<MessageHistoryParams>) {
+    this.params = Object.assign({ maxMessages: 20 }, params);
+  }
+
+  getMessages() {
+    return this.messages;
+  }
+
+  toString() {
+    return this.messages.map(message => message.content).join('. ');
+  }
+
+  setMessages(messages: ChatCompletionMessageParam[]): MessageHistory {
+    this.messages = messages;
+    this.prune();
+    return this;
+  }
+
+  push(messages: ChatCompletionMessageParam[]): MessageHistory {
+    this.messages.push(...messages);
+    this.prune();
+    return this;
+  }
+
+  prune() {
+    if (this.params.enforceMessageLimit === false)
+      return;
+
+    let messageCount = 0;
+    const messagesToRemoveIndexes: number[] = [];
+    for (let i = this.messages.length - 1; i >= 0; i--) {
+      const message = this.messages[i];
+      if (message.role == 'system')
+        continue;
+
+      messageCount++;
+      if (messageCount > this.params.maxMessages)
+        messagesToRemoveIndexes.push(i);
+    }
+
+    this.messages = this.messages.filter((message, i) => !messagesToRemoveIndexes.includes(i));
+    this.logger.debug(`Pruned ${messagesToRemoveIndexes.length} messages`);
+  }
+}

package/src/history/MessageModerator.ts

@@ -0,0 +1,6 @@
+import { ChatCompletionMessageParam } from 'openai/resources/chat';
+
+export interface MessageModerator {
+  // given a set of messages, modify and return
+  observe(messages: ChatCompletionMessageParam[]): ChatCompletionMessageParam[];
+}

package/src/template/ConversationTemplate.ts

@@ -0,0 +1,12 @@
+export type Question = {
+  text: string,
+  optional?: boolean,
+}
+
+export type ConversationTemplate = {
+  name: string,
+  keywords: string[],
+  description: string,
+  questions: Question[],
+  instructions: () => Promise<string[]>,
+}

package/src/template/ConversationTemplateFunctions.ts

@@ -0,0 +1,43 @@
+import { ConversationTemplateModule } from './ConversationTemplateModule';
+
+export const searchConversationTemplatesFunctionName = 'searchConversationTemplates';
+export const searchConversationTemplatesFunction = (repo: ConversationTemplateModule) => {
+  return {
+    definition: {
+      name: searchConversationTemplatesFunctionName,
+      description: 'Get the conversation template names for templates matching the keyword',
+      parameters: {
+        type: 'object',
+        properties: {
+          keyword: {
+            type: 'string',
+            description: 'Search for conversation template names that match this keyword',
+          },
+        },
+        required: ['keyword']
+      },
+    },
+    call: async (params: { keyword: string }) => repo.searchConversationTemplates(params.keyword),
+  }
+}
+
+export const getConversationTemplateFunctionName = 'getConversationTemplate';
+export const getConversationTemplateFunction = (repo: ConversationTemplateModule) => {
+  return {
+    definition: {
+      name: getConversationTemplateFunctionName,
+      description: 'Get the conversation template matching the name',
+      parameters: {
+        type: 'object',
+        properties: {
+          conversationTemplateName: {
+            type: 'string',
+            description: 'Get the conversation template that has this name',
+          },
+        },
+        required: ['conversationTemplateName']
+      },
+    },
+    call: async (params: { conversationTemplateName: string }) => await repo.getConversationTemplate(params.conversationTemplateName),
+  }
+}

package/src/template/ConversationTemplateModule.ts

@@ -0,0 +1,83 @@
+import { Logger } from '@proteinjs/util';
+import { ConversationTemplate } from './ConversationTemplate';
+import { createPackageConversationTemplate } from './createPackage/CreatePackageConversationTemplate';
+import { ConversationModule, ConversationModuleFactory } from '../ConversationModule';
+import { getConversationTemplateFunction, getConversationTemplateFunctionName, searchConversationTemplatesFunction, searchConversationTemplatesFunctionName } from './ConversationTemplateFunctions';
+import { createCodeConversationTemplate } from './createCode/CreateCodeConversationTemplate';
+import { createAppTemplate } from './createApp/CreateAppTemplate';
+
+const conversationTemplates: ConversationTemplate[] = [
+  createPackageConversationTemplate,
+  createCodeConversationTemplate,
+  createAppTemplate,
+];
+
+export type ConversationTemplateModuleParams = {
+  conversationTemplates: { [conversationTemplateName: string]: ConversationTemplate},
+  conversationTemplateKeywordIndex: { [keyword: string]: string[] /** conversationTemplateNames */ }
+}
+
+export class ConversationTemplateModule implements ConversationModule {
+  private logger = new Logger(this.constructor.name);
+  params: ConversationTemplateModuleParams;
+
+  constructor(params: ConversationTemplateModuleParams) {
+    this.params = params;
+  }
+
+  getName(): string {
+    return 'Conversation Template';
+  }
+
+  searchConversationTemplates(keyword: string) {
+    this.logger.info(`Searching for conversation template, keyword: ${keyword}`);
+    const conversationNames = this.params.conversationTemplateKeywordIndex[keyword];
+    return conversationNames || [];
+  }
+
+  async getConversationTemplate(conversationTemplateName: string): Promise<Omit<ConversationTemplate, 'instructions'> & { instructions: string[] }> {
+    const conversationTemplate = this.params.conversationTemplates[conversationTemplateName];
+    if (!conversationTemplate)
+      return {} as any;
+
+    const instructions = await conversationTemplate.instructions();
+    return Object.assign(conversationTemplate, { instructions });
+  }
+
+  getSystemMessages() {
+    return [
+      `Whenever the user wants to create or talk about something, fisrt use the ${searchConversationTemplatesFunctionName} function to identify if there are relevant conversation templates to use`,
+      `Use the ${getConversationTemplateFunctionName} function to get the conversation template`,
+      `Once you've identified a conversation template that's relevant to the conversation, ask the user if they'd like to use that conversation template`,
+      `If they want to engage in the templated conversation, ask only the conversation template questions (template.questions), then use the user's answers to carry out the conversation template instructions (template.instructions)`,
+    ];
+  }
+
+  getFunctions() {
+    return [
+      searchConversationTemplatesFunction(this),
+      getConversationTemplateFunction(this),
+    ];
+  }
+
+  getMessageModerators() {
+    return [];
+  }
+}
+
+export class ConversationTemplateModuleFactory implements ConversationModuleFactory {
+  async createModule(repoPath: string) {
+    const params: ConversationTemplateModuleParams = { conversationTemplates: {}, conversationTemplateKeywordIndex: {} };
+    for (let conversationTemplate of conversationTemplates) {
+      params.conversationTemplates[conversationTemplate.name] = conversationTemplate;
+      for (let keyword of conversationTemplate.keywords) {
+        if (!params.conversationTemplateKeywordIndex[keyword])
+          params.conversationTemplateKeywordIndex[keyword] = [];
+
+        params.conversationTemplateKeywordIndex[keyword].push(conversationTemplate.name);
+      }
+    }
+
+    return new ConversationTemplateModule(params);
+  }
+}