cloving 0.3.17 → 0.3.19

package/src/commands/tokens.ts

@@ -0,0 +1,82 @@
+/**
+ * @module tokens
+ * This module provides functionality to estimate the number of tokens in the context files
+ * and project description for GPT processing.
+ */
+
+import fs from 'fs'
+import path from 'path'
+import { getNearestClovingConfig } from '../utils/config_utils'
+import { getAllFilesInDirectory } from '../utils/prompt_utils'
+import type { ClovingGPTOptions } from '../utils/types'
+
+/**
+ * Calculates the estimated number of tokens for the given context files and project description.
+ * @param {ClovingGPTOptions} options - The options for token calculation, including file paths.
+ * @returns {Promise<void>} A promise that resolves when the token estimation is complete.
+ */
+const tokens = async (options: ClovingGPTOptions): Promise<void> => {
+  let contextFiles: Record<string, string> = {}
+  let files = options.files || '.'
+
+  let prompt = ''
+
+  /**
+   * Expands the given file paths to include all files in directories.
+   * @type {string[]}
+   */
+  let expandedFiles: string[] = []
+  for (const file of files) {
+    const filePath = path.resolve(file)
+    if (
+      await fs.promises
+        .stat(filePath)
+        .then((stat) => stat.isDirectory())
+        .catch(() => false)
+    ) {
+      const dirFiles = await getAllFilesInDirectory(filePath)
+      expandedFiles = expandedFiles.concat(dirFiles.map((f) => path.relative(process.cwd(), f)))
+    } else {
+      expandedFiles.push(path.relative(process.cwd(), filePath))
+    }
+  }
+  files = expandedFiles
+
+  /**
+   * Reads the content of each file and stores it in the contextFiles object.
+   */
+  for (const file of files) {
+    const filePath = path.resolve(file)
+    if (
+      await fs.promises
+        .stat(filePath)
+        .then((stat) => stat.isFile())
+        .catch(() => false)
+    ) {
+      const content = await fs.promises.readFile(filePath, 'utf-8')
+      contextFiles[file] = content
+    }
+  }
+
+  /**
+   * Generates the prompt content by combining the app description and context file contents.
+   */
+  const contextFileContents = Object.keys(contextFiles)
+    .map((file) => `### Contents of ${file}\n\n${contextFiles[file]}\n\n`)
+    .join('\n')
+  prompt += `
+### Description of App
+
+${JSON.stringify(getNearestClovingConfig().config, null, 2)}
+
+${contextFileContents}`
+
+  /**
+   * Estimates the number of tokens based on the prompt length.
+   * This is a rough estimation and may not be exact.
+   */
+  const tokens = Math.ceil(prompt.length / 4).toLocaleString()
+  console.log(`Estimated tokens: ${tokens}`)
+}
+
+export default tokens

package/src/index.ts

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander'
+import config from './commands/config'
+import init from './commands/init'
+import proxy from './commands/proxy'
+import tokens from './commands/tokens'
+import models from './commands/models'
+import chat from './commands/chat'
+import shell from './commands/generate/shell'
+import code from './commands/generate/code'
+import commit from './commands/generate/commit'
+import context from './commands/generate/context'
+import unitTests from './commands/generate/unit_tests'
+import analyze from './commands/generate/review'
+import docs from './commands/generate/docs'
+import { getPackageVersion } from './utils/prompt_utils'
+
+const program = new Command()
+
+program
+  .name('cloving')
+  .description(
+    'Integrate AI into your development workflow for generating commit messages, code reviews, and unit tests.',
+  )
+  .version(getPackageVersion())
+
+program
+  .command('config')
+  .description('Configure cloving with your API key and models to use')
+  .action(config)
+
+program
+  .command('init')
+  .description('Setup cloving in the current directory')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .action(init)
+
+program
+  .command('chat')
+  .option(
+    '-f, --files <filenames...>',
+    'Specify filenames of files with context to use for generating code',
+  )
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5:sonnet-20240620)',
+  )
+  .description('Start an interactive chat with cloving')
+  .action(chat)
+
+program
+  .command('proxy')
+  .option(
+    '-f, --files <filenames...>',
+    'Specify filenames of files with context to use for generating code',
+  )
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5:sonnet-20240620)',
+  )
+  .description('Start a proxy server to use with cloving')
+  .action(proxy)
+
+program.command('models').alias('m').description('List available models').action(models)
+
+program
+  .command('tokens')
+  .alias('t')
+  .option(
+    '-f, --files <filenames...>',
+    'Specify filenames of files with context to use for generating code',
+  )
+  .description('Estimate the number of tokens in the current working directory or specified files')
+  .action(tokens)
+
+program
+  .command('commit')
+  .alias('c')
+  .description('Alias for cloving generate commit')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .action(commit)
+
+// Generate commands
+const generate = program
+  .command('generate')
+  .alias('g')
+  .description('Generate various items like unit-tests and code reviews')
+
+generate
+  .command('shell')
+  .alias('sh')
+  .description('Generate a shell command based on a prompt')
+  .option('-e, --exec', 'Automatically execute the generated shell command')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .option('-p, --prompt <prompt>', 'Specify the prompt to use')
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .action(shell)
+
+generate
+  .command('code')
+  .alias('c')
+  .description('Generate code based on a prompt')
+  .option('--save', 'Automatically save all files after generating the code')
+  .option(
+    '-i, --interactive',
+    'Auto-save generated code and then show a new prompt with existing context to revise the code further',
+  )
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5:sonnet-20240620)',
+  )
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .option('-p, --prompt <prompt>', 'Specify the prompt to use')
+  .option(
+    '-f, --files <filenames...>',
+    'Specify filenames of files with context to use for generating code',
+  )
+  .action(code)
+
+generate
+  .command('commit')
+  .description('Generate a commit message and commit the changes')
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .action(commit)
+
+generate
+  .command('context')
+  .description('Create a context string for generating code')
+  .option('-f, --files <filenames...>', 'Specify filenames of files with context')
+  .action(context)
+
+generate
+  .command('unit-tests')
+  .alias('u')
+  .description(
+    "Generate unit tests (if you don't specify filenames, it will generate tests for commited changes that differ from the main/master branch)",
+  )
+  .option('--save', 'Automatically save all files after generating the unit tests')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option('-f, --files <filenames...>', 'Specify filenames for the unit tests')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .action(unitTests)
+
+generate
+  .command('review')
+  .alias('r')
+  .description('Review the code for commited changes that differ from the main/master branch')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option('-f, --files <filenames...>', 'Specify filenames for files you want to review')
+  .option('-p, --prompt <prompt>', 'Add a prompt to the review')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .action(analyze)
+
+generate
+  .command('docs')
+  .alias('d')
+  .description('Generate documentation for the project')
+  .option('-s, --silent', 'Run the command without asking for confirmation of submitting prompts')
+  .option('-f, --files <filenames...>', 'Specify filenames for files you want to document')
+  .option(
+    '-m, --model <model>',
+    'Select the model to use (e.g., openai, claude, ollama, ollama:llama3, claude:claude-3-5-sonnet-20240620)',
+  )
+  .option('-t, --temperature <temperature>', 'Temperature for the model (default 0.2)')
+  .action(docs)
+
+program.parse(process.argv)

package/src/managers/BlockManager.ts

@@ -0,0 +1,126 @@
+import { EventEmitter } from 'events'
+import { extractCurrentNewBlocks } from '../utils/string_utils'
+import type { CurrentNewBlock } from '../utils/types'
+
+// Passes through string chunks, looking for code blocks, when it finds one, it starts buffering until it finds the end of the block
+// Then it emits the code block and clears the buffer and continues
+class BlockManager extends EventEmitter {
+  private buffer: string = ''
+  private codeBuffer: string = ''
+  private isBufferingCode: boolean = false
+  private isWaitingForContent: boolean = false
+
+  // add content to the buffer, watching for code blocks along the way
+  addContent(content: string) {
+    if (content.includes('`') || content.includes('\n')) {
+      this.isWaitingForContent = true
+      this.buffer += content
+      return
+    } else if (this.isWaitingForContent) {
+      content = this.buffer + content
+      this.buffer = ''
+    }
+
+    // if content has multiple \n``` in the string, then we need to split it up
+    const codeBlockRegex = /\n```[\s\S]*?\n```/g
+    const codeBlocks = content.match(codeBlockRegex)
+
+    if (codeBlocks) {
+      let lastIndex = 0
+      for (const block of codeBlocks) {
+        const index = content.indexOf(block, lastIndex)
+        this.processCodeBlocks(content.slice(lastIndex, index))
+        this.processCodeBlocks(block)
+        lastIndex = index + block.length
+      }
+      this.processCodeBlocks(content.slice(lastIndex))
+    } else {
+      this.processCodeBlocks(content)
+    }
+  }
+
+  private processCodeBlocks(content: string) {
+    const codeBlockMarker = '\n```'
+    let markerIndex = content.indexOf(codeBlockMarker)
+
+    if (markerIndex !== -1) {
+      if (this.isBufferingCode) {
+        this.codeBuffer += content.slice(0, markerIndex + codeBlockMarker.length)
+        const rest = content.slice(markerIndex + codeBlockMarker.length)
+        this.emit('endGeneratingCode')
+        this.emitCodeBlock()
+        this.emit('content', rest)
+      } else {
+        this.emitBuffer(content.slice(0, markerIndex))
+        this.codeBuffer = content.slice(markerIndex)
+        this.emit('startGeneratingCode')
+        this.isBufferingCode = true
+      }
+    } else {
+      if (this.isBufferingCode) {
+        this.codeBuffer += content
+      } else {
+        this.buffer += content
+      }
+    }
+
+    if (!this.isBufferingCode && this.buffer) {
+      this.emitBuffer()
+    }
+  }
+
+  private emitBuffer(additionalContent: string = '') {
+    const contentToEmit = this.buffer + additionalContent
+    if (contentToEmit) {
+      this.emit('content', contentToEmit)
+      this.clearBuffer()
+    }
+  }
+
+  private emitCodeBlock() {
+    if (this.codeBuffer.length > 0) {
+      const currentNewBlock = this.parseCodeBuffer()
+      this.emit('codeBlock', {
+        currentNewBlock,
+        raw: this.codeBuffer,
+      })
+      this.emit('content', this.buffer)
+      this.clearBuffer()
+    }
+  }
+
+  /**
+   * Parses a code buffer into its constituent parts.
+   *
+   * This function takes a code buffer as a string, splits it into lines, and identifies the language,
+   * current start index, divider index, and new end index. It then extracts the current code and new code
+   * from the buffer based on these indices.
+   *
+   * @return {CurrentNewBlock | null}
+   *   An object containing the language, current start, new end, current code, and new code.
+   */
+  private parseCodeBuffer(): CurrentNewBlock | null {
+    const results = extractCurrentNewBlocks(this.codeBuffer)
+
+    if (results.length === 0) {
+      return null
+    }
+
+    const block = results[0]
+    if (!block.currentContent || !block.newContent) {
+      console.warn('Incomplete code block detected. Skipping.')
+      return null
+    }
+
+    return block
+  }
+
+  clearBuffer() {
+    this.buffer = ''
+    this.codeBuffer = ''
+    this.isBufferingCode = false
+    this.isWaitingForContent = false
+  }
+}
+
+export default BlockManager