compose 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0181e80bc473a0e67c074cc18b299be97ecad391875a7578780a3f8a93286cdc'
-  data.tar.gz: a4d171f396fd690387c122958565eb047efd3b0ef11c99acd6fc6e557a78f1e4
+  metadata.gz: 3ac262b371cf1ff49a50e94ea83aaf75e7867564341c3bd3bff024c77831c2b4
+  data.tar.gz: 8577e99ba4dc26f7cd96a40589bb4c75cca3394e6e3302d8d43c9786df4633dd
 SHA512:
-  metadata.gz: 172b6babed118da8b989bffb8dad311eef02a27c21200df782fccf29783d929559ca44e98023c4ac2e3607e1ece53a36e7490834c9fc09aaffe82aaf8dfc94ec
-  data.tar.gz: 89b5cd472edfe13a5007e753d23bca1be694ae056314f71376f442af323de21e1bdcf395c0560445534172c22a7304185df21d0b1835c7e65f05c26c325bf050
+  metadata.gz: 5db9243359a8045ca48620bb0fb9f5dc0876fe763c01f0e3fba2cf46467b1feffc42d69afd35857aa1398b0977a8d0478fef6a338c847b0ef43651a934b386f3
+  data.tar.gz: 3e4de964ef36267bf768eba57c09d58e7037ae17e7313e61b068bb4611bb688ef914757b805a20d3ba7e4cc2a07f156d31bcb6270a0aebbca29a7708e2aa08a1

data/lib/compose/file_processor.rb

@@ -2,6 +2,11 @@
 
 require 'find'
 require 'pathname'
+require 'httparty'
+require 'nokogiri'
+require 'reverse_markdown'
+require 'digest/md5'
+require 'fileutils'
 
 module Compose
   class FileProcessor
@@ -25,21 +30,19 @@ module Compose
       @files = {}
 
       input_files.each do |input|
-        if File.directory?(input)
+        if input.start_with?('http://', 'https://')
+          @files[input] = process_url(input)
+        elsif File.directory?(input)
           find_files_in_directory(input).each do |file|
-            @files[File.expand_path(file)] = nil
+            @files[File.expand_path(file)] = FileProcessor.load_file(file, include_imports: include_imports)
           end
         elsif File.file?(input)
-          @files[File.expand_path(input)] = nil
+          @files[File.expand_path(input)] = FileProcessor.load_file(file, include_imports: include_imports)
         else
           puts "Skipping invalid input: #{input}".yellow
         end
       end
 
-      @files.each do |file, _|
-        @files[file] = FileProcessor.load_file(file, include_imports: include_imports)
-      end
-
       self
     end
 
@@ -97,5 +100,33 @@ module Compose
     rescue
       false
     end
+
+    def process_url(url)
+      cached_file = File.join(ENV['HOME'], '.compose', 'cache', Digest::MD5.hexdigest(url))
+      content = if File.exist?(cached_file)
+                  File.read(cached_file)
+                else
+                  content = fetch_and_convert_url(url)
+                  return if content.nil?
+
+                  FileUtils.mkdir_p(File.dirname(cached_file))
+                  File.write(cached_file, content)
+                  content
+                end
+      return if content.nil?
+
+      "<url src='#{url}'>\n#{content}\n</url>"
+    end
+
+    def fetch_and_convert_url(url)
+      response = HTTParty.get(url, follow_redirects: true)
+      return nil if response.code != 200
+
+      doc = Nokogiri::HTML(response.body)
+      doc.search('script, style').remove
+      parsed_markdown = ReverseMarkdown.convert(doc.to_html, unknown_tags: :bypass)
+      normalized_markdown = parsed_markdown.gsub(/\s+/, ' ').strip
+      normalized_markdown
+    end
   end
 end

data/lib/compose/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Compose
-  VERSION = '0.1.6'
+  VERSION = '0.1.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: compose
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Dreaming Tulpa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-09-11 00:00:00.000000000 Z
+date: 2024-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-openai
@@ -164,6 +164,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: reverse_markdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: digest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement