rubycrawl 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f463d9ba6ffa83c283954dd411a08dc0184ed065128f57da625b9c349447b77a
-  data.tar.gz: cc8adb28596fe65e54f18ec97d83152c9a8df6d38c3cf584c359f2a2230e6048
+  metadata.gz: 06ec3c9fa168060673012766dd0d217911aeee642c3ef545e31058927a202a72
+  data.tar.gz: 4ef9134d6398175ffc4e2cbb9e85a1b1a0d954d7db1f90d96388aa8319692cd7
 SHA512:
-  metadata.gz: 98c20cc8a1ff17df7a830e93f6aa49e5c630c7d43533e516648b4c4fdc301c7e733ab9aba6502d6de7bb5b5f1afe40f037d4fad59e77051322780dba5c575fa2
-  data.tar.gz: a789dea3bfbd3c63dc8d364da49b38904675d1311dc143dacbb0cf58631a0e8d59d8b3484148dde85b6efb03f8ec3caf6026e209239486219c7f55a7c955ff5c
+  metadata.gz: 0bbc9345c7cbb3e45b8af62baa473c0d01b07a71c24ec0f155b3532e54ffb93a98aed685794adba94b355cc6ccdfa9785d0c3e49dcc40213b386a060a4612b26
+  data.tar.gz: dec9d0d9333f44a642d296fe31d1585c18a289824c327bc59f5812b6726a7fd30d5ff4fffafdf8fdb74a37a69135725597fd292df5ed26088c3e8d3d8b792fc7

data/lib/rubycrawl/helpers.rb

@@ -11,7 +11,7 @@ class RubyCrawl
       uri = URI.parse(url)
 
       unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
-        raise ConfigurationError, "Only HTTP(S) URLs are supported, got: #{url}"
+        raise ConfigurationError, "Invalid URL: Only HTTP(S) URLs are supported, got: #{url}"
       end
 
       if uri.host&.match?(/^(localhost|127\.|192\.168\.|10\.|172\.(1[6-9]|2[0-9]|3[01]))/)

data/lib/rubycrawl/markdown_converter.rb

@@ -1,27 +1,54 @@
 # frozen_string_literal: true
 
+require 'uri'
+
 class RubyCrawl
   # Converts HTML to Markdown using reverse_markdown gem.
   module MarkdownConverter
+    # Patterns for relative URLs in markdown
+    MARKDOWN_URL_PATTERNS = [
+      %r{(!\[[^\]]*\])\((/[^)]+)\)}, # ![alt](/path)
+      %r{(\[[^\]]*\])\((/[^)]+)\)}   # [text](/path)
+    ].freeze
+
     module_function
 
-    # Convert HTML to Markdown.
+    # Convert HTML to Markdown with resolved URLs.
     #
     # @param html [String] The HTML content to convert
+    # @param base_url [String, nil] Base URL to resolve relative URLs
     # @param options [Hash] Options for conversion
-    # @option options [Boolean] :unknown_tags (:bypass) How to handle unknown tags
-    # @option options [Boolean] :github_flavored (true) Use GitHub-flavored markdown
-    # @return [String] The Markdown content
-    def convert(html, options = {})
+    # @return [String] The Markdown content with absolute URLs
+    def convert(html, base_url: nil, **options)
       return '' if html.nil? || html.empty?
 
       require_reverse_markdown
-      ReverseMarkdown.convert(html, default_options.merge(options))
+      markdown = ReverseMarkdown.convert(html, default_options.merge(options))
+      base_url ? resolve_relative_urls(markdown, base_url) : markdown
     rescue LoadError
       warn '[rubycrawl] reverse_markdown gem not installed. Add it to your Gemfile for markdown support.'
       ''
     end
 
+    # Resolve relative URLs in markdown to absolute URLs.
+    #
+    # @param markdown [String] The markdown content
+    # @param base_url [String] The base URL to resolve against
+    # @return [String] Markdown with absolute URLs
+    def resolve_relative_urls(markdown, base_url)
+      return markdown unless base_url
+
+      base_uri = URI.parse(base_url)
+      origin = "#{base_uri.scheme}://#{base_uri.host}"
+      origin += ":#{base_uri.port}" unless [80, 443].include?(base_uri.port)
+
+      MARKDOWN_URL_PATTERNS.reduce(markdown) do |md, pattern|
+        md.gsub(pattern) { "#{::Regexp.last_match(1)}(#{origin}#{::Regexp.last_match(2)})" }
+      end
+    rescue URI::InvalidURIError
+      markdown
+    end
+
     def require_reverse_markdown
       require 'reverse_markdown'
     end

data/lib/rubycrawl/result.rb

@@ -14,10 +14,18 @@ class RubyCrawl
     end
 
     # Returns markdown, converting from HTML lazily if needed.
+    # Relative URLs are resolved using the page's final_url.
     #
-    # @return [String] Markdown content
+    # @return [String] Markdown content with absolute URLs
     def markdown
-      @markdown ||= MarkdownConverter.convert(html)
+      @markdown ||= MarkdownConverter.convert(html, base_url: final_url)
+    end
+
+    # The final URL after redirects.
+    #
+    # @return [String, nil]
+    def final_url
+      metadata['final_url'] || metadata[:final_url]
     end
 
     # Check if markdown has been computed.

data/lib/rubycrawl/site_crawler.rb

@@ -17,9 +17,14 @@ class RubyCrawl
         @depth = depth
       end
 
-      # Lazy markdown conversion.
+      # Lazy markdown conversion with resolved URLs.
       def markdown
-        @markdown ||= MarkdownConverter.convert(html)
+        @markdown ||= MarkdownConverter.convert(html, base_url: final_url)
+      end
+
+      # The final URL after redirects.
+      def final_url
+        metadata['final_url'] || metadata[:final_url] || url
       end
     end
 

data/lib/rubycrawl/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class RubyCrawl
-  VERSION = '0.1.0'
+  VERSION = '0.1.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubycrawl
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - RubyCrawl contributors