cloving 0.1.21 → 0.1.23

package/dist/managers/ProxyManager.js

@@ -0,0 +1,163 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const express_1 = __importDefault(require("express"));
+const body_parser_1 = __importDefault(require("body-parser"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const cloving_gpt_1 = __importDefault(require("../cloving_gpt"));
+const ChunkManager_1 = __importDefault(require("./ChunkManager"));
+const config_utils_1 = require("../utils/config_utils");
+const command_utils_1 = require("../utils/command_utils");
+class ProxyManager {
+    constructor(options) {
+        this.options = options;
+        this.contextFiles = {};
+        this.options.silent = true;
+        this.options.stream = true;
+        this.gpt = new cloving_gpt_1.default(this.options);
+        this.app = (0, express_1.default)();
+        this.app.use(body_parser_1.default.json());
+        this.chunkManager = new ChunkManager_1.default();
+    }
+    async initialize() {
+        await this.setupRoutes();
+        await this.loadContextFiles();
+        this.startServer();
+    }
+    async setupRoutes() {
+        this.app.get('/', (_req, res) => {
+            res.json({ message: 'ClovingGPT proxy server running' });
+        });
+        this.app.post('/v1/chat/completions', this.handleChatCompletions.bind(this));
+    }
+    async loadContextFiles() {
+        let files = this.options.files || '.';
+        let expandedFiles = [];
+        for (const file of files) {
+            const filePath = path_1.default.resolve(file);
+            if (await fs_1.default.promises.stat(filePath).then(stat => stat.isDirectory()).catch(() => false)) {
+                const dirFiles = await (0, command_utils_1.getAllFilesInDirectory)(filePath);
+                expandedFiles = expandedFiles.concat(dirFiles.map(f => path_1.default.relative(process.cwd(), f)));
+            }
+            else {
+                expandedFiles.push(path_1.default.relative(process.cwd(), filePath));
+            }
+        }
+        files = expandedFiles;
+        for (const file of files) {
+            const filePath = path_1.default.resolve(file);
+            if (await fs_1.default.promises.stat(filePath).then(stat => stat.isFile()).catch(() => false)) {
+                const content = await fs_1.default.promises.readFile(filePath, 'utf-8');
+                this.contextFiles[file] = content;
+            }
+        }
+    }
+    async handleChatCompletions(req, res) {
+        try {
+            const { messages, stream } = req.body;
+            const content = messages.map((message) => message.content).join('\n\n');
+            const contextFileContents = Object.keys(this.contextFiles).map((file) => `### Contents of ${file}\n\n${this.contextFiles[file]}\n\n`).join('\n');
+            const prompt = `### Task
+
+${content}
+
+### Description of App
+
+${JSON.stringify((0, config_utils_1.getClovingConfig)(), null, 2)}
+
+${contextFileContents}
+
+### Task
+
+${content}`;
+            if (stream) {
+                await this.handleStreamResponse(prompt, res);
+            }
+            else {
+                await this.handleNonStreamResponse(prompt, res);
+            }
+        }
+        catch (error) {
+            console.error('Error processing request:', error);
+            res.status(500).json({ error: 'Internal server error' });
+        }
+    }
+    async handleStreamResponse(prompt, res) {
+        const shortPrompt = prompt.replace('### Task', '').trim().slice(0, 50).replace(/\n/g, ' ');
+        const promptTokens = Math.ceil(prompt.length / 4).toLocaleString();
+        console.log(`> sending ${promptTokens} token prompt:`, shortPrompt);
+        const responseStream = await this.gpt.streamText({ prompt });
+        res.setHeader('Content-Type', 'application/json');
+        res.setHeader('Transfer-Encoding', 'chunked');
+        let convertedStreams = [];
+        // Add initial stream to indicate the start of the response
+        convertedStreams.push('{"id":"chatcmpl-9tLvSuRQaYzQIiKRAwI5a7WqdJspq","object":"chat.completion.chunk","created":1722979349,"model":"cloving","system_fingerprint":"fp_3aa7262c27","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}]}');
+        this.chunkManager.on('content', (buffer) => {
+            let convertedStream = this.gpt.convertStream(buffer);
+            while (convertedStream !== null) {
+                const { output, lastChar } = convertedStream;
+                const content = JSON.stringify({ id: "chatcmpl-9tLvSuRQaYzQIiKRAwI5a7WqdJspq", object: "chat.completion.chunk", created: 1722979350, model: "cloving", system_fingerprint: "fp_3aa7262c27", choices: [{ index: 0, delta: { content: output }, logprobs: null, finish_reason: null }] });
+                convertedStreams.push(content);
+                if (convertedStreams.length >= 2) {
+                    const dataToSend = convertedStreams.map(c => `data: ${c}`).join('\n\n');
+                    res.write(`${dataToSend}\n\n`);
+                    convertedStreams = [];
+                }
+                this.chunkManager.clearBuffer(lastChar);
+                buffer = buffer.slice(lastChar);
+                convertedStream = this.gpt.convertStream(buffer);
+            }
+        });
+        responseStream.data.on('data', (chunk) => {
+            const chunkString = chunk.toString();
+            this.chunkManager.addChunk(chunkString);
+        });
+        responseStream.data.on('end', () => {
+            // Send any remaining converted streams
+            const dataToSend = convertedStreams.map(c => `data: ${c}`).join('\n\n');
+            res.write(`${dataToSend}\n\ndata: [DONE]\n\n`);
+            res.end();
+        });
+        responseStream.data.on('error', (error) => {
+            console.error('Error streaming response:', error);
+            res.status(500).json({ error: 'Internal server error' });
+        });
+    }
+    async handleNonStreamResponse(prompt, res) {
+        const content = await this.gpt.generateText({ prompt });
+        const response = {
+            id: 'chatcmpl-' + Date.now(),
+            object: 'chat.completion',
+            created: Math.floor(Date.now() / 1000),
+            model: 'cloving',
+            choices: [
+                {
+                    index: 0,
+                    message: {
+                        role: 'assistant',
+                        content,
+                    },
+                    logprobs: null,
+                    finish_reason: 'stop',
+                },
+            ],
+            usage: {
+                prompt_tokens: Math.ceil(prompt.length / 4),
+                completion_tokens: Math.ceil(content.length / 4),
+                total_tokens: Math.ceil(prompt.length / 4) + Math.ceil(content.length / 4),
+            },
+            system_fingerprint: 'fp_3cd8b62c3b',
+        };
+        res.json(response);
+    }
+    startServer() {
+        const port = parseInt(`${process.env.PORT || this.options.port || 3000}`, 10);
+        this.app.listen(port, () => {
+            console.log(`ClovingGPT proxy server running on port ${port}`);
+        });
+    }
+}
+exports.default = ProxyManager;

package/dist/managers/src/managers/ProxyManager.js

@@ -0,0 +1 @@
+"use strict";

package/dist/utils/command_utils.js

@@ -14,7 +14,7 @@ exports.SPECIAL_FILES = [
     '.csproj', 'packages.config', 'composer.json', 'CMakeLists.txt', 'conanfile.txt', 'conanfile.py',
     'go.mod', 'Cargo.toml', 'Package.swift', 'build.gradle.kts', 'Podfile', 'Cartfile', 'cpanfile',
     'DESCRIPTION', 'mix.exs', 'build.sbt', 'pubspec.yaml', 'stack.yaml', 'cabal.project', 'Project.toml',
-    'rockspec', 'rebar.config', 'project.clj'
+    'rockspec', 'rebar.config', 'project.clj', 'tsconfig.json'
 ];
 const collectSpecialFileContents = () => {
     const specialFileContents = {};

package/dist/utils/git_utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getCurrentBranchName = exports.getDefaultBranchName = exports.getGitDiff = void 0;
+exports.generateCommitMessagePrompt = exports.getCurrentBranchName = exports.getDefaultBranchName = exports.getGitDiff = void 0;
 const child_process_1 = require("child_process");
 const readline_1 = __importDefault(require("readline"));
 const askQuestion = (question) => {
@@ -61,3 +61,23 @@ const getCurrentBranchName = () => {
     }
 };
 exports.getCurrentBranchName = getCurrentBranchName;
+const generateCommitMessagePrompt = (diff) => {
+    return `Generate a concise and meaningful commit message based on a diff.
+
+Do not add any commentary or context to the message other than the commit message itself.
+
+An example of the output for this should look like the following:
+
+\`\`\`plaintext
+Update dependencies and package versions
+
+- Upgrade Ruby gems including aws-sdk, honeybadger, irb, and rubocop
+- Update Node.js packages including esbuild, tinymce, and trix
+- Bump TypeScript and ESLint related packages to latest versions
+\`\`\`
+
+Here is the diff to help you write the commit message:
+
+${diff}`;
+};
+exports.generateCommitMessagePrompt = generateCommitMessagePrompt;

package/dist/utils/string_utils.js

@@ -1,6 +1,11 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractFilesAndContent = exports.parseMarkdownInstructions = exports.extractMarkdown = exports.extractJsonMetadata = exports.estimateTokens = void 0;
+exports.saveGeneratedFiles = exports.extractFilesAndContent = exports.parseMarkdownInstructions = exports.extractMarkdown = exports.extractJsonMetadata = exports.estimateTokens = void 0;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
 // Function to estimate token count
 const estimateTokens = async (text) => {
     const charCount = text.length;
@@ -102,7 +107,7 @@ const extractFilesAndContent = (rawCodeCommand) => {
     for (const match of matches) {
         const fileName = match.replace(/\*{2}/g, '').trim();
         const escapedFileName = fileName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-        const regex = new RegExp(`\\*\\*${escapedFileName}\\*\\*\\n\\n\\\`{3}([\\s\\S]+?)\\\`{3}`, 'g');
+        const regex = new RegExp(`\\*\\*${escapedFileName}\\*\\*\\n\\n\\\`{3}([\\s\\S]+?)\\\`{3}(\n|$)`, 'g');
         const contentMatch = regex.exec(rawCodeCommand);
         if (contentMatch) {
             files.push(fileName);
@@ -115,3 +120,17 @@ const extractFilesAndContent = (rawCodeCommand) => {
     return [files, fileContents];
 };
 exports.extractFilesAndContent = extractFilesAndContent;
+const saveGeneratedFiles = async (files, fileContents) => {
+    for (const file of files) {
+        if (fileContents[file]) {
+            const filePath = path_1.default.resolve(file);
+            await fs_1.default.promises.mkdir(path_1.default.dirname(filePath), { recursive: true });
+            await fs_1.default.promises.writeFile(filePath, fileContents[file]);
+            console.log(`${file} has been saved.`);
+        }
+        else {
+            console.log(`File content not found for ${file}.`);
+        }
+    }
+};
+exports.saveGeneratedFiles = saveGeneratedFiles;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cloving",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "packageManager": "yarn@1.22.22",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -49,6 +49,7 @@
     "eslint-plugin-jsdoc": "^48.11.0",
     "eslint-plugin-n": "^17.10.1",
     "eslint-plugin-promise": "^7.0.0",
+    "express": "^4.19.2",
     "ignore": "^5.3.1",
     "ini": "^4.1.3",
     "isbinaryfile": "^5.0.2",
@@ -57,7 +58,9 @@
     "vite-node": "^2.0.5"
   },
   "devDependencies": {
+    "@types/body-parser": "^1.19.5",
     "@types/copy-paste": "^1.1.33",
+    "@types/express": "^4.17.21",
     "@types/ini": "^4.1.1",
     "@types/inquirer": "^9.0.7",
     "@types/jest": "^29.5.12",