@proteinjs/conversation 1.0.1

package/dist/src/code_template/Code.js

@@ -0,0 +1,41 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Code = void 0;
+const util_node_1 = require("@proteinjs/util-node");
+class Code {
+    constructor(args) {
+        this.args = args;
+    }
+    generate() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.args.imports)
+                this.addImports(this.args.imports, this.args.conversation);
+            return yield this.args.conversation.generateCode(this.args.description, 'gpt-4');
+        });
+    }
+    addImports(imports, conversation) {
+        conversation.addSystemMessagesToHistory([
+            this.declarationMessage(imports.map(d => d.sourceFilePath)),
+            this.importMessage(imports),
+        ]);
+    }
+    declarationMessage(tsFilePaths) {
+        const declarationMap = util_node_1.PackageUtil.generateTypescriptDeclarations({ tsFilePaths, includeDependencyDeclarations: true });
+        const declarations = Object.values(declarationMap).join('\n');
+        return `Assume the following code exists in other files:\n${declarations}`;
+    }
+    importMessage(imports) {
+        const importStatements = imports.map(i => `import { ${i.moduleNames.join(', ')} } from '${i.importPathFromGeneratedFile}'`);
+        return `Add the following imports:\n${importStatements}`;
+    }
+}
+exports.Code = Code;

package/dist/src/code_template/CodeTemplate.js

@@ -0,0 +1,39 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CodeTemplate = void 0;
+const util_1 = require("@proteinjs/util");
+const util_node_1 = require("@proteinjs/util-node");
+class CodeTemplate {
+    constructor(templateArgs) {
+        this.logger = new util_1.Logger(this.constructor.name);
+        this.templateArgs = templateArgs;
+    }
+    generate() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield util_node_1.PackageUtil.installPackages(this.resolvePackages());
+            for (let sourceFile of this.sourceFiles()) {
+                const filePath = util_node_1.Fs.baseContainedJoin(this.templateArgs.srcPath, sourceFile.relativePath);
+                this.logger.info(`Generating source file: ${filePath}`);
+                const code = yield sourceFile.code.generate();
+                yield util_node_1.Fs.writeFiles([{ path: filePath, content: code }]);
+                this.logger.info(`Generated source file: ${filePath}`);
+            }
+        });
+    }
+    resolvePackages() {
+        const packages = this.templateArgs.replacePackages ? [] : this.dependencyPackages();
+        if (this.templateArgs.additionalPackages)
+            packages.push(...this.templateArgs.additionalPackages);
+        return packages;
+    }
+}
+exports.CodeTemplate = CodeTemplate;

package/dist/src/code_template/CodeTemplateModule.js

@@ -0,0 +1,46 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CodeTemplateModule = void 0;
+class CodeTemplateModule {
+    getName() {
+        return 'Code Template';
+    }
+    getSystemMessages() {
+        return [
+            `If they want to create a function/class/object using an API we are familiar with, we will ask the user for the required information to fill in all mandatory parameters and ask them if they want to provide optional parameter values`,
+            `Once we have gotten the values for all parameters, respond with '${CodeTemplateModule.CODE_RESPONSE}' followed by the code to instantiate/call the function/class/object with the user's responses for parameter values`,
+            `If you're generating a call to a class that extends Template, require that the user provide Template's constructor parameters as well and combine them into the parameters passed into the base class you're instantiating`,
+            `Make sure you ask for a srcPath and pass that in to the Template base class constructor arg`,
+            `Surround generated code (not including imports) with a self-executing, anonymous async function like this (async function() =>{})()`,
+        ];
+    }
+    getFunctions() {
+        return [];
+    }
+    getMessageModerators() {
+        return [];
+    }
+}
+exports.CodeTemplateModule = CodeTemplateModule;
+CodeTemplateModule.CODE_RESPONSE = 'Code with user input:';
+// if (response.includes(CodegenConversation.CODE_RESPONSE))
+//         await this.generateCode(response, conversation);
+// private async generateCode(message: string, conversation: Conversation) {
+//   const code = OpenAi.parseCodeFromMarkdown(message);
+//   const srcPathToken = 'TOKEN';
+//   const responseSrcPath = await conversation.generateResponse([`Return the srcPath the user provided surrounded by the token ${srcPathToken}`], CodegenConversation.MODEL);
+//   const srcPath = responseSrcPath.replace(/["'`]/g, '').match(/TOKEN(.*?)TOKEN/)?.[1];
+//   if (!srcPath)
+//     throw new Error(`Failed to parse responseSrcPath: ${responseSrcPath}`);
+//   const codePath = path.join(process.cwd(), srcPath, 'template.ts');
+//   await fs.ensureFile(codePath);
+//   await fs.writeFile(codePath, code);
+//   console.log(`Wrote file: ${codePath}`);
+//   const command = 'node';
+//   const args = [codePath];
+//   const commandLog = `${command} ` + args.join(' ');
+//   console.info(`Running command: ${commandLog}`);
+//   await cmd(command, args, {OPENAI_API_KEY: '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'});
+//   console.info(`Ran command: ${commandLog}`);
+//   console.info(`Generated code from template: ${codePath}`);
+// }

package/dist/src/code_template/Repo.js

@@ -0,0 +1,127 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RepoFactory = exports.Repo = void 0;
+const util_1 = require("@proteinjs/util");
+const util_node_1 = require("@proteinjs/util-node");
+const promises_1 = __importDefault(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+class Repo {
+    constructor(params) {
+        this.logger = new util_1.Logger(this.constructor.name);
+        this.params = params;
+    }
+    packages() {
+        return this.params.packages;
+    }
+    slimPackages() {
+        return this.params.slimPackages;
+    }
+    tsFiles() {
+        return this.params.tsFiles;
+    }
+    keywordFilesIndex() {
+        return this.params.keywordFilesIndex;
+    }
+    searchFiles(params) {
+        this.logger.info(`Searching for file, keyword: ${params.keyword}`);
+        const filePaths = this.keywordFilesIndex()[params.keyword];
+        return filePaths || [];
+    }
+    getDeclarations(params) {
+        const queriedDeclarations = {};
+        for (let tsFilePath of params.tsFilePaths) {
+            queriedDeclarations[tsFilePath] = this.params.tsFiles[tsFilePath].declaration;
+            this.logger.info(`Accessed declaration: ${tsFilePath}`);
+        }
+        return queriedDeclarations;
+    }
+}
+exports.Repo = Repo;
+class RepoFactory {
+    static createRepo(dir) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.LOGGER.info(`Creating repo for dir: ${dir}`);
+            let repoParams = { packages: {}, slimPackages: {}, tsFiles: {}, keywordFilesIndex: {} };
+            function traverse(dir) {
+                return __awaiter(this, void 0, void 0, function* () {
+                    const childrenNames = yield promises_1.default.readdir(dir, { withFileTypes: true });
+                    let hasPackageJson = childrenNames.some(dirent => dirent.name === 'package.json');
+                    if (hasPackageJson) {
+                        const packageContent = yield promises_1.default.readFile(path_1.default.join(dir, 'package.json'), 'utf-8');
+                        const packageJSON = JSON.parse(packageContent);
+                        repoParams.packages[packageJSON.name] = {
+                            packageJSON: packageJSON,
+                            dirPath: dir,
+                            fileDescriptors: [],
+                            tsFiles: {},
+                        };
+                    }
+                    for (const dirent of childrenNames) {
+                        if (!dirent.isDirectory())
+                            continue;
+                        // Exclude directories 'node_modules' and 'dist' right at the beginning
+                        if (dirent.name.includes('node_modules') || dirent.name.includes('dist')) {
+                            continue;
+                        }
+                        // Continue with the traversal if it's a directory
+                        const childPath = path_1.default.join(dir, dirent.name);
+                        yield traverse(childPath);
+                    }
+                });
+            }
+            yield traverse(dir);
+            yield RepoFactory.loadFiles(repoParams);
+            Object.keys(repoParams.packages).forEach(packageName => {
+                const _a = repoParams.packages[packageName], { packageJSON, tsFiles } = _a, slimPackage = __rest(_a, ["packageJSON", "tsFiles"]);
+                repoParams.slimPackages[packageName] = slimPackage;
+            });
+            this.LOGGER.info(`Created repo for dir: ${dir}`);
+            return new Repo(repoParams);
+        });
+    }
+    static loadFiles(repoParams) {
+        return __awaiter(this, void 0, void 0, function* () {
+            for (let packageName of Object.keys(repoParams.packages)) {
+                this.LOGGER.info(`Loading files for package: ${packageName}`);
+                const dirPath = repoParams.packages[packageName].dirPath;
+                if (dirPath) {
+                    repoParams.packages[packageName].fileDescriptors.push(...yield util_node_1.Fs.getFilesInDirectory(dirPath, ['node_modules', 'dist']));
+                    for (let fileDescriptor of repoParams.packages[packageName].fileDescriptors) {
+                        const typescriptDeclaration = util_node_1.PackageUtil.generateTypescriptDeclarations({ tsFilePaths: [fileDescriptor.path] })[fileDescriptor.path];
+                        const tsFile = Object.assign({ declaration: typescriptDeclaration }, fileDescriptor);
+                        repoParams.packages[packageName].tsFiles[fileDescriptor.path] = tsFile;
+                        repoParams.tsFiles[fileDescriptor.path] = tsFile;
+                        if (!repoParams.keywordFilesIndex[fileDescriptor.nameWithoutExtension])
+                            repoParams.keywordFilesIndex[fileDescriptor.nameWithoutExtension] = [];
+                        repoParams.keywordFilesIndex[fileDescriptor.nameWithoutExtension].push(fileDescriptor.path);
+                    }
+                }
+            }
+        });
+    }
+}
+exports.RepoFactory = RepoFactory;
+RepoFactory.LOGGER = new util_1.Logger('RepoFactory');

package/dist/src/fs/conversation_fs/ConversationFsModerator.js

@@ -0,0 +1,99 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConversationFsModerator = exports.ConversationFsFactory = void 0;
+const util_1 = require("@proteinjs/util");
+class ConversationFsFactory {
+    constructor(params) {
+        this.params = Object.assign({ maxFiles: 3, logLevel: 'info' }, params);
+        this.logger = new util_1.Logger(this.constructor.name, this.params.logLevel);
+    }
+    merge(existingFs, updates) {
+        for (let filePath of Object.keys(updates)) {
+            // if the file already exists in the fs
+            if (existingFs.fileContentMap[filePath]) {
+                this.logger.debug(`Updating existing file: ${filePath}`);
+                existingFs.fileContentMap[filePath] = updates[filePath];
+                const oldIndex = existingFs.order.findIndex(item => item == filePath);
+                existingFs.order.splice(oldIndex, 1);
+                existingFs.order.push(filePath);
+                continue;
+            }
+            // if we have less than the max number of files in the fs
+            if (Object.keys(existingFs.fileContentMap).length < this.params.maxFiles) {
+                this.logger.debug(`Adding new file (under limit): ${filePath}`);
+                existingFs.fileContentMap[filePath] = updates[filePath];
+                existingFs.order.push(filePath);
+                continue;
+            }
+            else {
+                this.logger.debug(`Adding new file (over limit): ${filePath}`);
+                const removedFilePath = existingFs.order.splice(0, 1)[0];
+                delete existingFs.fileContentMap[removedFilePath];
+                existingFs.fileContentMap[filePath] = updates[filePath];
+                existingFs.order.push(filePath);
+            }
+        }
+        return existingFs;
+    }
+}
+exports.ConversationFsFactory = ConversationFsFactory;
+class ConversationFsModerator {
+    constructor(logLevel) {
+        this.logLevel = 'info';
+        if (logLevel)
+            this.logLevel = logLevel;
+    }
+    observe(messages) {
+        let conversationFileSystemMessageIndex = -1;
+        let conversationFileSystem;
+        let readFilesFunctionCallMessageIndexes = [];
+        let writeFilesFunctionCallMessageIndexes = [];
+        const readFilesConsolidatedOutput = {}; // newest version of file wins
+        for (let i = 0; i < messages.length; i++) {
+            const message = messages[i];
+            if (message.role == 'system' && message.content) {
+                let parsedContent;
+                try {
+                    parsedContent = JSON.parse(message.content);
+                }
+                catch (error) { }
+                if (!parsedContent || !parsedContent['fileSystem'])
+                    continue;
+                conversationFileSystem = parsedContent['fileSystem'];
+                conversationFileSystemMessageIndex = i;
+                continue;
+            }
+            if (message.role == 'function' && message.name == 'readFiles' && message.content) {
+                let parsedContent;
+                try {
+                    parsedContent = JSON.parse(message.content);
+                }
+                catch (error) { }
+                if (!parsedContent)
+                    continue;
+                for (let filePath of Object.keys(parsedContent)) {
+                    readFilesConsolidatedOutput[filePath] = parsedContent[filePath];
+                }
+                readFilesFunctionCallMessageIndexes.push(i);
+            }
+            if (message.role == 'function' && message.name == 'writeFiles') {
+                writeFilesFunctionCallMessageIndexes.push(i);
+            }
+        }
+        if (conversationFileSystem) {
+            conversationFileSystem = new ConversationFsFactory({ logLevel: this.logLevel }).merge(conversationFileSystem, readFilesConsolidatedOutput);
+            const content = JSON.stringify({ fileSystem: conversationFileSystem });
+            messages[conversationFileSystemMessageIndex].content = content;
+        }
+        else {
+            conversationFileSystem = { fileContentMap: readFilesConsolidatedOutput, order: Object.keys(readFilesConsolidatedOutput) };
+            messages.push({ role: 'system', content: `Whenever you make a call to readFiles, the file content will be loaded into the { fileSystem } object in the message history. Do not respond with fileSystem's content in a message.` });
+            const content = JSON.stringify({ fileSystem: conversationFileSystem });
+            messages.push({ role: 'system', content });
+        }
+        const moderatedMessages = messages
+            .filter((message, i) => !readFilesFunctionCallMessageIndexes.includes(i) && !writeFilesFunctionCallMessageIndexes.includes(i));
+        return moderatedMessages;
+    }
+}
+exports.ConversationFsModerator = ConversationFsModerator;

package/dist/src/fs/conversation_fs/ConversationFsModule.js

@@ -0,0 +1,68 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConversationFsModuleFactory = exports.ConversationFsModule = void 0;
+const KeywordToFilesIndexFunctions_1 = require("../keyword_to_files_index/KeywordToFilesIndexFunctions");
+const PackageFunctions_1 = require("../package/PackageFunctions");
+const ConversationFsModerator_1 = require("./ConversationFsModerator");
+const FsFunctions_1 = require("./FsFunctions");
+class ConversationFsModule {
+    constructor(repoPath) {
+        this.recentlyAccessedFilePaths = [];
+        this.repoPath = repoPath;
+    }
+    getName() {
+        return 'Conversation Fs';
+    }
+    getSystemMessages() {
+        return [
+            `Assume the current working directory is: ${this.repoPath} unless specified by the user`,
+            `Pre-pend the current working directory as the base path to file paths when performing file operations, unless specified otherwise by the user`,
+            `If the user asks to change the cwd, do not create a new folder, just fail if it's not a valid path`,
+            `If the user wants to work in a package, use the ${PackageFunctions_1.searchPackagesFunctionName} function and take the directory from the package.json file path and make that the cwd`,
+            `You have access to code in a local repo, you can read and write code to and from the file system with the ${FsFunctions_1.readFilesFunctionName} function and the ${FsFunctions_1.writeFilesFunctionName} function`,
+            `Before writing to a file that already exists, read the file first and make your changes to its contents`,
+            `When reading/writing a file in a specified package, join the package directory with the relative path to form the file path`,
+            `When searching for source files, do not look in the dist or node_modules directories`,
+            `If you don't know a file path, don't try to guess it, use the ${KeywordToFilesIndexFunctions_1.searchFilesFunctionName} function to find it`,
+            `When searching for something (ie. a file to work with/in), unless more context is specified, use the ${PackageFunctions_1.searchLibrariesFunctionName} function first, then fall back to functions: ${PackageFunctions_1.searchPackagesFunctionName}, ${KeywordToFilesIndexFunctions_1.searchFilesFunctionName}`,
+            `After finding a file to work with, assume the user's following question pertains to that file and use ${FsFunctions_1.readFilesFunctionName} to read the file if needed`,
+        ];
+    }
+    getFunctions() {
+        return [
+            (0, FsFunctions_1.readFilesFunction)(this),
+            (0, FsFunctions_1.writeFilesFunction)(this),
+            (0, FsFunctions_1.getRecentlyAccessedFilePathsFunction)(this),
+            ...FsFunctions_1.fsFunctions
+        ];
+    }
+    getMessageModerators() {
+        return [
+            new ConversationFsModerator_1.ConversationFsModerator(),
+        ];
+    }
+    pushRecentlyAccessedFilePath(filePaths) {
+        this.recentlyAccessedFilePaths.push(...filePaths);
+    }
+    getRecentlyAccessedFilePaths() {
+        return this.recentlyAccessedFilePaths;
+    }
+}
+exports.ConversationFsModule = ConversationFsModule;
+class ConversationFsModuleFactory {
+    createModule(repoPath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return new ConversationFsModule(repoPath);
+        });
+    }
+}
+exports.ConversationFsModuleFactory = ConversationFsModuleFactory;