compose 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0181e80bc473a0e67c074cc18b299be97ecad391875a7578780a3f8a93286cdc'
-  data.tar.gz: a4d171f396fd690387c122958565eb047efd3b0ef11c99acd6fc6e557a78f1e4
+  metadata.gz: 050edb82f3b985eefdea7e2b791019968f7e8f13770a07652e12e0f273e159db
+  data.tar.gz: 8bce91cfa68da8141cca2ce29a8e9ab25be1051e5d72b93c6376a77bd84f222d
 SHA512:
-  metadata.gz: 172b6babed118da8b989bffb8dad311eef02a27c21200df782fccf29783d929559ca44e98023c4ac2e3607e1ece53a36e7490834c9fc09aaffe82aaf8dfc94ec
-  data.tar.gz: 89b5cd472edfe13a5007e753d23bca1be694ae056314f71376f442af323de21e1bdcf395c0560445534172c22a7304185df21d0b1835c7e65f05c26c325bf050
+  metadata.gz: 9e07206841afc24ee8daa040c5f11183198072f0e98913da0c5a1ac3d4605b14daf2ee130f93443eeb2f3fc6e7191021ccb1114210471d62353c5011de1fd8de
+  data.tar.gz: e0406eb23a9d5368cd9bc527237c152308716839afe9e649f3837434988dc07f246bbfd06281cbbfc0b62da7b98d2e3944af1783a181487b5c5c10935a967013

data/config/prompts/markdown_cleanup.txt

@@ -0,0 +1,30 @@
+You are tasked with cleaning up a markdown file that was converted from HTML. The file contains unnecessary elements that need to be removed to isolate the main content of the page. Here is the markdown content to clean:
+
+<markdown>
+{{MARKDOWN}}
+</markdown>
+
+Your goal is to remove all artifacts and unnecessary elements from the markdown, leaving only the main content of the page.
+
+Common artifacts and unnecessary elements to look for and remove include:
+
+1. Header and footer sections
+2. Navigation menus
+3. Sidebar content
+4. Social media sharing buttons
+5. Advertisement blocks
+6. Copyright notices
+
+Follow these steps to clean the markdown content:
+
+1. Identify the main content section of the page. This is typically the largest block of text that contains the primary information.
+2. If there is no H1 heading, try to identify the main title or heading of the page and format it as an H1 heading.
+3. Remove any content before the main content section, including headers, navigation menus, and introductory elements.
+4. Remove any content after the main content section, including footers, related links, and closing elements.
+5. Ensure that headings are properly formatted with the correct number of `#` symbols.
+6. Preserve any important links, image references, captions, lists, tables,
+   code blocks etc. that are part of the main content.
+
+Be careful not to remove any part of the main content while cleaning up the markdown.
+
+Once you have cleaned the markdown content, provide only the cleanup markdown. Ensure that only the main content of the page remains, formatted correctly in markdown. Do not surround the response with backticks.

data/lib/compose/ai_client.rb

@@ -17,7 +17,7 @@ module Compose
         response.dig('content', 0, 'text')
       when 'openai'
         response = openai_chat(system_prompt, user_prompt, temperature: temperature)
-        response.dig('choices', 0, 'message', 'content')
+        response
       else
         raise "Unsupported model: #{@model[:name]}"
       end
@@ -44,18 +44,27 @@ module Compose
     end
 
     def openai_chat(system_prompt, user_prompt, temperature: 0)
-      Whirly.start spinner: 'dots', status: 'Verifying change'
+      Whirly.start spinner: 'dots', status: 'Received 0 chunks'
+      token_count = 0
+      response = ""
       client = OpenAI::Client.new(access_token: ENV['OPENAI_API_KEY'])
-      response = client.chat(
+      client.chat(
         parameters: {
           model: @model[:name],
           messages: [
             { role: 'system', content: system_prompt },
             { role: 'user', content: user_prompt }
           ],
-          temperature: temperature
+          temperature: temperature,
+          stream: Proc.new { |chunk|
+            token_count += 1
+            Whirly.status = "Received #{token_count} chunks"
+            content = chunk.dig("choices", 0, "delta", "content")
+            response += content if content
+          }
         }
       )
+
       Whirly.stop
       response
     end

data/lib/compose/cli.rb

@@ -8,6 +8,7 @@ module Compose
     desc 'edit [FILES]', 'Edit files using AI assistance'
     method_option :model, type: :string, default: 'sonnet35', desc: 'AI model to use'
     method_option :include_imports, type: :boolean, default: true, aliases: '-a', desc: 'Include import statements when processing files'
+    method_option :clear_cache, type: :boolean, default: false, aliases: '-c', desc: 'Clear the cache before processing files'
     def edit(*files)
       puts 'Welcome to Compose!'.green
 
@@ -16,12 +17,21 @@ module Compose
         exit(1)
       end
 
-      code_model = Model.find(options[:model])
-      ApiKeyUtils.setup(code_model)
-      verifier_model = Model.preferred_verifier_model
-      ApiKeyUtils.setup(verifier_model)
+      if options[:clear_cache]
+        FileUtils.rm_rf(FileProcessor::CACHE_DIR)
+        puts 'Cache cleared'.green
+      end
+
+      # Set up configuration
+      Compose::Config.configure do |config|
+        config.code_model = Model.find(options[:model])
+        config.verifier_model = Model.preferred_verifier_model
+      end
+
+      ApiKeyUtils.setup(Compose::Config.code_model)
+      ApiKeyUtils.setup(Compose::Config.verifier_model)
 
-      file_processor = FileProcessor.new(files, include_imports: options[:include_imports], model: code_model)
+      file_processor = FileProcessor.new(files, include_imports: options[:include_imports])
       puts "Loaded #{file_processor.files.count} file(s): #{file_processor.files.keys.map { |path| File.exist?(path) ? Pathname.new(path).relative_path_from(Pathname.pwd).to_s : path }.join(', ')}"
 
       task = ask('What do you need me to do? (Type \'ask\' followed by your question to ask a question instead):')
@@ -33,7 +43,7 @@ module Compose
       else
         edits = file_processor.edit_files(task)
 
-        edit_verifier = EditVerifier.new(edits, model: verifier_model)
+        edit_verifier = EditVerifier.new(edits)
         verified_edits = edit_verifier.verify_edits
 
         edit_processor = EditProcessor.new(verified_edits)

data/lib/compose/config.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Compose
+  module Config
+    class << self
+      attr_accessor :verifier_model, :code_model
+
+      def configure
+        yield self
+      end
+    end
+  end
+end

data/lib/compose/edit_verifier.rb

@@ -2,10 +2,9 @@
 
 module Compose
   class EditVerifier
-    attr_reader :ai_client, :edits
+    attr_reader :edits
 
-    def initialize(edits, model:)
-      @ai_client = AIClient.new(model)
+    def initialize(edits)
       @edits = edits
     end
 
@@ -33,7 +32,7 @@ module Compose
       file_content = FileProcessor.load_file(File.expand_path(edit[:filename]))
       system_prompt = "CODE:\n#{file_content}\n"
 
-      response = ai_client.chat(system_prompt, prompt)
+      response = AIClient.new(Compose::Config.verifier_model).chat(system_prompt, prompt)
 
       begin
         verified_change = JSON.parse(response)

data/lib/compose/file_processor.rb

@@ -2,9 +2,16 @@
 
 require 'find'
 require 'pathname'
+require 'selenium-webdriver'
+require 'nokogiri'
+require 'reverse_markdown'
+require 'digest/md5'
+require 'fileutils'
 
 module Compose
   class FileProcessor
+    CACHE_DIR = File.join(ENV['HOME'], '.cache', 'compose')
+
     attr_reader :ai_client, :files, :tokens
 
     def self.load_file(file_path, include_imports: true)
@@ -20,12 +27,13 @@ module Compose
       "<#{relative_path}>\n#{processed_content}\n</#{relative_path}>"
     end
 
-    def initialize(input_files, include_imports: true, model:)
-      @ai_client = AIClient.new(model)
+    def initialize(input_files, include_imports: true)
       @files = {}
 
       input_files.each do |input|
-        if File.directory?(input)
+        if input.start_with?('http://', 'https://')
+          @files[input] = process_url(input)
+        elsif File.directory?(input)
           find_files_in_directory(input).each do |file|
             @files[File.expand_path(file)] = nil
           end
@@ -36,8 +44,8 @@ module Compose
         end
       end
 
-      @files.each do |file, _|
-        @files[file] = FileProcessor.load_file(file, include_imports: include_imports)
+      @files.each do |path, _|
+        @files[path] ||= FileProcessor.load_file(path, include_imports: include_imports)
       end
 
       self
@@ -52,7 +60,7 @@ module Compose
       prompt.gsub!('{{QUESTION}}', question)
       system_prompt = "CODE:\n#{content}\n"
 
-      ai_client.chat(system_prompt, prompt)
+      AIClient.new(Compose::Config.code_model).chat(system_prompt, prompt)
     end
 
     def edit_files(task)
@@ -60,7 +68,7 @@ module Compose
       prompt.gsub!('{{TASK}}', task)
       system_prompt = "CODE:\n#{content}\n"
 
-      response = ai_client.chat(system_prompt, prompt)
+      response = AIClient.new(Compose::Config.code_model).chat(system_prompt, prompt)
 
       begin
         edits = JSON.parse(response)['edits']
@@ -97,5 +105,57 @@ module Compose
     rescue
       false
     end
+
+    def process_url(url)
+      cached_file = File.join(CACHE_DIR, Digest::MD5.hexdigest(url))
+      content = if File.exist?(cached_file)
+                  File.read(cached_file)
+                else
+                  content = fetch_and_convert_url(url)
+                  return if content.nil?
+
+                  FileUtils.mkdir_p(File.dirname(cached_file))
+                  File.write(cached_file, content)
+                  content
+                end
+      return if content.nil?
+
+      "<url src='#{url}'>\n#{content}\n</url>"
+    end
+
+    def fetch_and_convert_url(url)
+      options = Selenium::WebDriver::Chrome::Options.new
+      options.add_argument('--headless')
+      options.add_argument('--disable-gpu')
+      options.add_argument('--no-sandbox')
+      options.add_argument('--disable-dev-shm-usage')
+      options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+
+      driver = Selenium::WebDriver.for :chrome, options: options
+
+      begin
+        driver.get(url)
+        html_content = driver.page_source
+        doc = Nokogiri::HTML(html_content)
+        doc.search('script, style').remove
+
+        parsed_markdown = ReverseMarkdown.convert(doc.to_html, unknown_tags: :bypass)
+        normalized_markdown = parsed_markdown.gsub(/ +/, ' ').strip
+
+        # Remove content before the first h1 in markdown
+        first_h1_index = normalized_markdown.index(/^# /)
+        if first_h1_index
+          normalized_markdown = normalized_markdown[first_h1_index..-1]
+        end
+
+        prompt = File.read(File.expand_path("../../config/prompts/markdown_cleanup.txt", __dir__))
+        prompt.gsub!('{{MARKDOWN}}', normalized_markdown)
+        cleaned_markdown = AIClient.new(Compose::Config.verifier_model).chat("", prompt)
+
+        cleaned_markdown
+      ensure
+        driver.quit
+      end
+    end
   end
 end

data/lib/compose/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Compose
-  VERSION = '0.1.6'
+  VERSION = '0.1.8'
 end

data/lib/compose.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'compose/version'
+require_relative 'compose/config'
 require_relative 'compose/cli'
 require_relative 'compose/api_key_utils'
 require_relative 'compose/file_processor'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: compose
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Dreaming Tulpa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-11 00:00:00.000000000 Z
+date: 2024-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-openai
@@ -164,6 +164,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: reverse_markdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: digest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -218,12 +274,14 @@ files:
 - bin/compose
 - config/models.yml
 - config/prompts/ask.txt
+- config/prompts/markdown_cleanup.txt
 - config/prompts/task.txt
 - config/prompts/verify.txt
 - lib/compose.rb
 - lib/compose/ai_client.rb
 - lib/compose/api_key_utils.rb
 - lib/compose/cli.rb
+- lib/compose/config.rb
 - lib/compose/edit_processor.rb
 - lib/compose/edit_verifier.rb
 - lib/compose/file_processor.rb