rubycrawl 0.1.0

data/lib/rubycrawl/helpers.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+class RubyCrawl
+  # Helper methods for payloads, validation, and errors.
+  module Helpers
+    private
+
+    def validate_url!(url)
+      uri = URI.parse(url)
+
+      unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+        raise ConfigurationError, "Only HTTP(S) URLs are supported, got: #{url}"
+      end
+
+      if uri.host&.match?(/^(localhost|127\.|192\.168\.|10\.|172\.(1[6-9]|2[0-9]|3[01]))/)
+        warn '[rubycrawl] Warning: Crawling internal/private IP addresses'
+      end
+    rescue URI::InvalidURIError => e
+      raise ConfigurationError, "Invalid URL: #{e.message}"
+    end
+
+    def build_payload(url, wait_until, block_resources)
+      payload = { url: url }
+      payload[:wait_until] = wait_until if wait_until
+      payload[:block_resources] = block_resources unless block_resources.nil?
+      payload
+    end
+
+    def build_result(response)
+      Result.new(
+        text: response['text'].to_s,
+        html: response['html'].to_s,
+        links: Array(response['links']),
+        metadata: response['metadata'].is_a?(Hash) ? response['metadata'] : {}
+      )
+    end
+
+    def error_class_for(error_code)
+      case error_code
+      when 'navigation_timeout', 'crawl_timeout'
+        TimeoutError
+      when 'navigation_failed', 'crawl_failed'
+        NavigationError
+      when 'invalid_json', 'invalid_json_response'
+        ServiceError
+      else
+        Error
+      end
+    end
+
+    def error_message_for(error_code, error_message)
+      case error_code
+      when 'navigation_timeout', 'crawl_timeout'
+        "Crawl timeout: #{error_message}"
+      when 'navigation_failed', 'crawl_failed'
+        "Navigation failed: #{error_message}"
+      when 'invalid_json', 'invalid_json_response'
+        "Node service returned invalid JSON: #{error_message}"
+      else
+        "Crawl error [#{error_code}]: #{error_message}"
+      end
+    end
+  end
+end

data/lib/rubycrawl/markdown_converter.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class RubyCrawl
+  # Converts HTML to Markdown using reverse_markdown gem.
+  module MarkdownConverter
+    module_function
+
+    # Convert HTML to Markdown.
+    #
+    # @param html [String] The HTML content to convert
+    # @param options [Hash] Options for conversion
+    # @option options [Boolean] :unknown_tags (:bypass) How to handle unknown tags
+    # @option options [Boolean] :github_flavored (true) Use GitHub-flavored markdown
+    # @return [String] The Markdown content
+    def convert(html, options = {})
+      return '' if html.nil? || html.empty?
+
+      require_reverse_markdown
+      ReverseMarkdown.convert(html, default_options.merge(options))
+    rescue LoadError
+      warn '[rubycrawl] reverse_markdown gem not installed. Add it to your Gemfile for markdown support.'
+      ''
+    end
+
+    def require_reverse_markdown
+      require 'reverse_markdown'
+    end
+
+    def default_options
+      {
+        unknown_tags: :bypass,
+        github_flavored: true,
+        tag_border: ''
+      }
+    end
+  end
+end

data/lib/rubycrawl/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'rails/railtie'
+
+class RubyCrawl
+  # Railtie for loading rake tasks in Rails apps.
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path('tasks/install.rake', __dir__)
+    end
+  end
+end

data/lib/rubycrawl/result.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class RubyCrawl
+  # Result object with lazy markdown conversion.
+  class Result
+    attr_reader :text, :html, :links, :metadata
+
+    def initialize(text:, html:, links:, metadata:, markdown: nil)
+      @text = text
+      @html = html
+      @links = links
+      @metadata = metadata
+      @markdown = markdown unless markdown.to_s.empty?
+    end
+
+    # Returns markdown, converting from HTML lazily if needed.
+    #
+    # @return [String] Markdown content
+    def markdown
+      @markdown ||= MarkdownConverter.convert(html)
+    end
+
+    # Check if markdown has been computed.
+    #
+    # @return [Boolean]
+    def markdown?
+      !@markdown.nil?
+    end
+
+    def to_h
+      {
+        text: text,
+        html: html,
+        links: links,
+        metadata: metadata,
+        markdown: markdown
+      }
+    end
+  end
+end

data/lib/rubycrawl/service_client.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+class RubyCrawl
+  # Handles node service lifecycle and HTTP requests.
+  class ServiceClient
+    def initialize(host:, port:, node_dir:, node_bin:, node_log:)
+      @host = host
+      @port = Integer(port)
+      @node_dir = node_dir
+      @node_bin = node_bin
+      @node_log = node_log
+      @node_pid = nil
+    end
+
+    def ensure_running
+      return if healthy?
+
+      start_service
+      wait_until_healthy
+    end
+
+    def post_json(path, body)
+      uri = URI("http://#{@host}:#{@port}#{path}")
+      request = build_request(uri, body)
+      response = perform_request(uri, request)
+      JSON.parse(response.body)
+    rescue JSON::ParserError => e
+      raise ServiceError, "Node service returned invalid JSON: #{e.message}"
+    rescue Errno::ECONNREFUSED, Errno::ECONNRESET => e
+      raise ServiceError, "Cannot connect to node service at #{uri}: #{e.message}"
+    rescue Net::OpenTimeout, Net::ReadTimeout => e
+      raise TimeoutError, "Request to node service timed out: #{e.message}"
+    end
+
+    private
+
+    def build_request(uri, body)
+      request = Net::HTTP::Post.new(uri)
+      request['Content-Type'] = 'application/json'
+      request.body = JSON.generate(body)
+      request
+    end
+
+    def perform_request(uri, request)
+      Net::HTTP.start(uri.host, uri.port, open_timeout: 5, read_timeout: 30) do |http|
+        http.request(request)
+      end
+    end
+
+    def start_service
+      raise ServiceError, "rubycrawl node service directory not found: #{@node_dir}" unless Dir.exist?(@node_dir)
+
+      env = { 'RUBYCRAWL_NODE_PORT' => @port.to_s }
+      out = @node_log ? File.open(@node_log, 'a') : File::NULL
+      err = @node_log ? out : File::NULL
+      @node_pid = Process.spawn(env, @node_bin, 'src/index.js', chdir: @node_dir, out: out, err: err)
+      Process.detach(@node_pid)
+    end
+
+    def wait_until_healthy(timeout: 5)
+      deadline = Time.now + timeout
+      until Time.now > deadline
+        return true if healthy?
+
+        sleep 0.2
+      end
+
+      raise ServiceError, "rubycrawl node service failed to start within #{timeout}s. " \
+                          "Check logs at #{@node_log || 'RUBYCRAWL_NODE_LOG'}"
+    end
+
+    def healthy?
+      uri = URI("http://#{@host}:#{@port}/health")
+      response = Net::HTTP.start(uri.host, uri.port, open_timeout: 1, read_timeout: 1) do |http|
+        http.get(uri.request_uri)
+      end
+      response.is_a?(Net::HTTPSuccess)
+    rescue StandardError
+      false
+    end
+  end
+end

data/lib/rubycrawl/site_crawler.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'set'
+
+class RubyCrawl
+  # BFS crawler that follows links with deduplication.
+  class SiteCrawler
+    # Page result yielded to the block with lazy markdown.
+    class PageResult
+      attr_reader :url, :html, :links, :metadata, :depth
+
+      def initialize(url:, html:, links:, metadata:, depth:)
+        @url = url
+        @html = html
+        @links = links
+        @metadata = metadata
+        @depth = depth
+      end
+
+      # Lazy markdown conversion.
+      def markdown
+        @markdown ||= MarkdownConverter.convert(html)
+      end
+    end
+
+    def initialize(client, options = {})
+      @client = client
+      @max_pages = options.fetch(:max_pages, 50)
+      @max_depth = options.fetch(:max_depth, 3)
+      @same_host_only = options.fetch(:same_host_only, true)
+      @wait_until = options.fetch(:wait_until, nil)
+      @block_resources = options.fetch(:block_resources, nil)
+      @visited = Set.new
+      @queue = []
+    end
+
+    def crawl(start_url, &block)
+      raise ArgumentError, 'Block required for site crawl' unless block_given?
+
+      normalized = UrlNormalizer.normalize(start_url)
+      raise ConfigurationError, "Invalid start URL: #{start_url}" unless normalized
+
+      @base_url = normalized
+      enqueue(normalized, 0)
+      process_queue(&block)
+    end
+
+    private
+
+    def process_queue
+      pages_crawled = 0
+
+      while (item = @queue.shift) && pages_crawled < @max_pages
+        url, depth = item
+        next if @visited.include?(url)
+
+        result = process_page(url, depth)
+        next unless result
+
+        yield result
+        pages_crawled += 1
+      end
+
+      pages_crawled
+    end
+
+    def process_page(url, depth)
+      @visited.add(url)
+      result = crawl_page(url, depth)
+      enqueue_links(result.links, depth + 1) if result && depth < @max_depth
+      result
+    end
+
+    def crawl_page(url, depth)
+      result = @client.crawl(url, wait_until: @wait_until, block_resources: @block_resources)
+      build_page_result(url, depth, result)
+    rescue Error => e
+      warn "[rubycrawl] Failed to crawl #{url}: #{e.message}"
+      nil
+    end
+
+    def build_page_result(url, depth, result)
+      PageResult.new(
+        url: url,
+        html: result.html,
+        links: extract_urls(result.links),
+        metadata: result.metadata,
+        depth: depth
+      )
+    end
+
+    def extract_urls(links)
+      links.map { |link| link['url'] || link[:url] }.compact
+    end
+
+    def enqueue_links(links, depth)
+      links.each do |link|
+        normalized = UrlNormalizer.normalize(link, @base_url)
+        next unless normalized
+        next if @visited.include?(normalized)
+        next if @same_host_only && !UrlNormalizer.same_host?(normalized, @base_url)
+
+        enqueue(normalized, depth)
+      end
+    end
+
+    def enqueue(url, depth)
+      return if @visited.include?(url)
+
+      @queue.push([url, depth])
+    end
+  end
+end

data/lib/rubycrawl/tasks/install.rake

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/BlockLength
+namespace :rubycrawl do
+  desc 'Install Node dependencies and create initializer'
+  task :install do
+    require 'fileutils'
+
+    # Check Node.js is installed
+    unless system('node', '--version', out: File::NULL, err: File::NULL)
+      abort <<~MSG
+        [rubycrawl] ERROR: Node.js is not installed or not in PATH.
+
+        RubyCrawl requires Node.js (v18+ recommended) for browser automation.
+
+        Install Node.js:
+          - macOS:   brew install node
+          - Ubuntu:  curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - && sudo apt-get install -y nodejs
+          - Windows: https://nodejs.org/en/download/
+
+        After installing, run this task again:
+          bundle exec rake rubycrawl:install
+      MSG
+    end
+
+    gem_root = File.expand_path('../../../', __dir__)
+    node_dir = File.join(gem_root, 'node')
+
+    abort("[rubycrawl] ERROR: node directory not found at #{node_dir}") unless Dir.exist?(node_dir)
+
+    Dir.chdir(node_dir) do
+      puts('[rubycrawl] Installing Node dependencies...')
+      system('npm', 'install') || abort('[rubycrawl] ERROR: npm install failed')
+
+      puts('[rubycrawl] Installing Playwright browsers...')
+      system('npx', 'playwright', 'install') || abort('[rubycrawl] ERROR: playwright install failed')
+    end
+
+    if defined?(Rails)
+      initializer_path = Rails.root.join('config', 'initializers', 'rubycrawl.rb')
+      if File.exist?(initializer_path)
+        puts("[rubycrawl] Initializer already exists at #{initializer_path}")
+      else
+        content = <<~RUBY
+          # frozen_string_literal: true
+
+          # RubyCrawl Configuration
+          # =======================
+          # Uncomment and modify options as needed.
+
+          RubyCrawl.configure(
+            # wait_until - Page load strategy:
+            #   "load"             - Wait for load event (fastest, good for static sites)
+            #   "domcontentloaded" - Wait for DOM ready (medium speed)
+            #   "networkidle"      - Wait until no network requests for 500ms (best for SPAs)
+            # wait_until: "load",
+
+            # Block images, fonts, CSS, media for faster crawls (2-3x speedup)
+            # block_resources: true,
+
+            # Maximum retry attempts for transient failures (with exponential backoff)
+            # max_retries: 3,
+
+            # Node service settings (usually no need to change)
+            # host: "127.0.0.1",
+            # port: 3344,
+
+            # Custom Node.js binary path (if not in PATH)
+            # node_bin: "/usr/local/bin/node",
+
+            # Log file for Node service output (useful for debugging)
+            # node_log: Rails.root.join("log", "rubycrawl.log").to_s
+          )
+        RUBY
+
+        FileUtils.mkdir_p(File.dirname(initializer_path))
+        File.write(initializer_path, content)
+        puts("[rubycrawl] Created initializer at #{initializer_path}")
+      end
+    else
+      puts('[rubycrawl] Rails not detected. Skipping initializer creation.')
+    end
+  end
+end
+# rubocop:enable Metrics/BlockLength

data/lib/rubycrawl/url_normalizer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'set'
+
+class RubyCrawl
+  # Normalizes URLs for deduplication.
+  module UrlNormalizer
+    module_function
+
+    def normalize(url, base_url = nil)
+      uri = parse_uri(url, base_url)
+      return nil unless uri&.host
+
+      normalize_uri_parts(uri)
+      uri.to_s
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def normalize_uri_parts(uri)
+      uri.scheme = uri.scheme&.downcase
+      uri.host = uri.host&.downcase
+      uri.path = normalize_path(uri.path)
+      uri.fragment = nil
+      uri.query = normalize_query(uri.query)
+    end
+
+    def same_host?(url, base_url)
+      uri = URI.parse(url)
+      base_uri = URI.parse(base_url)
+      uri.host&.downcase == base_uri.host&.downcase
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def parse_uri(url, base_url)
+      uri = URI.parse(url)
+      return uri if uri.absolute?
+      return nil unless base_url
+
+      URI.join(base_url, url)
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def normalize_path(path)
+      return '/' if path.nil? || path.empty?
+
+      # Remove trailing slash except for root
+      path = path.chomp('/') if path.length > 1
+      path
+    end
+
+    def normalize_query(query)
+      return nil if query.nil? || query.empty?
+
+      # Remove tracking params
+      tracking_params = %w[utm_source utm_medium utm_campaign utm_term utm_content fbclid gclid]
+      params = URI.decode_www_form(query).reject { |k, _| tracking_params.include?(k.downcase) }
+      return nil if params.empty?
+
+      URI.encode_www_form(params.sort)
+    rescue ArgumentError
+      query
+    end
+  end
+end

data/lib/rubycrawl/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class RubyCrawl
+  VERSION = '0.1.0'
+end

data/lib/rubycrawl.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require_relative 'rubycrawl/version'
+require_relative 'rubycrawl/errors'
+require_relative 'rubycrawl/helpers'
+require_relative 'rubycrawl/service_client'
+require_relative 'rubycrawl/url_normalizer'
+require_relative 'rubycrawl/markdown_converter'
+require_relative 'rubycrawl/result'
+require_relative 'rubycrawl/site_crawler'
+require_relative 'rubycrawl/railtie' if defined?(Rails)
+
+# RubyCrawl provides a simple interface for crawling pages via a local Playwright service.
+class RubyCrawl
+  include Helpers
+
+  DEFAULT_HOST = '127.0.0.1'
+  DEFAULT_PORT = 3344
+
+  class << self
+    def client
+      @client ||= new
+    end
+
+    def crawl(url, **options)
+      client.crawl(url, **options)
+    end
+
+    # Crawl multiple pages starting from a URL, following links.
+    # Yields each page result to the block as it is crawled.
+    #
+    # @param url [String] The starting URL
+    # @param max_pages [Integer] Maximum number of pages to crawl (default: 50)
+    # @param max_depth [Integer] Maximum link depth from start URL (default: 3)
+    # @param same_host_only [Boolean] Only follow links on the same host (default: true)
+    # @yield [page] Yields each page result as it is crawled
+    # @yieldparam page [SiteCrawler::PageResult] The crawled page result
+    # @return [Integer] Number of pages crawled
+    #
+    # @example Save pages to database
+    #   RubyCrawl.crawl_site("https://example.com", max_pages: 100) do |page|
+    #     Page.create!(url: page.url, html: page.html, depth: page.depth)
+    #   end
+    def crawl_site(url, ...)
+      client.crawl_site(url, ...)
+    end
+
+    def configure(**options)
+      @client = new(**options)
+    end
+  end
+
+  def initialize(**options)
+    load_options(options)
+    build_service_client
+  end
+
+  def crawl(url, wait_until: @wait_until, block_resources: @block_resources, retries: @max_retries)
+    validate_url!(url)
+    @service_client.ensure_running
+    with_retries(retries) do
+      payload = build_payload(url, wait_until, block_resources)
+      response = @service_client.post_json('/crawl', payload)
+      raise_node_error!(response)
+      build_result(response)
+    end
+  end
+
+  # Crawl multiple pages starting from a URL, following links.
+  # @see RubyCrawl.crawl_site
+  def crawl_site(url, **options, &block)
+    @service_client.ensure_running
+    crawler_options = build_crawler_options(options)
+    crawler = SiteCrawler.new(self, crawler_options)
+    crawler.crawl(url, &block)
+  end
+
+  private
+
+  def raise_node_error!(response)
+    return unless response.is_a?(Hash) && response['error']
+
+    error_code = response['error']
+    error_message = response['message'] || error_code
+    raise error_class_for(error_code), error_message_for(error_code, error_message)
+  end
+
+  def with_retries(retries)
+    attempt = 0
+    begin
+      yield
+    rescue ServiceError, TimeoutError => e
+      attempt += 1
+      raise unless attempt < retries
+
+      retry_with_backoff(attempt, retries, e)
+      retry
+    end
+  end
+
+  def load_options(options)
+    @host = options.fetch(:host, DEFAULT_HOST)
+    @port = Integer(options.fetch(:port, DEFAULT_PORT))
+    @node_dir = options.fetch(:node_dir, default_node_dir)
+    @node_bin = options.fetch(:node_bin, ENV.fetch('RUBYCRAWL_NODE_BIN', nil)) || 'node'
+    @node_log = options.fetch(:node_log, ENV.fetch('RUBYCRAWL_NODE_LOG', nil))
+    @wait_until = options.fetch(:wait_until, nil)
+    @block_resources = options.fetch(:block_resources, nil)
+    @max_retries = options.fetch(:max_retries, 3)
+  end
+
+  def build_service_client
+    @service_client = ServiceClient.new(
+      host: @host,
+      port: @port,
+      node_dir: @node_dir,
+      node_bin: @node_bin,
+      node_log: @node_log
+    )
+  end
+
+  def retry_with_backoff(attempt, retries, error)
+    backoff_seconds = 2**attempt
+    warn "[rubycrawl] Retry #{attempt}/#{retries - 1} after #{backoff_seconds}s: #{error.message}"
+    sleep(backoff_seconds)
+  end
+
+  def build_crawler_options(options)
+    {
+      max_pages: options.fetch(:max_pages, 50),
+      max_depth: options.fetch(:max_depth, 3),
+      same_host_only: options.fetch(:same_host_only, true),
+      wait_until: options.fetch(:wait_until, @wait_until),
+      block_resources: options.fetch(:block_resources, @block_resources)
+    }
+  end
+
+  def default_node_dir
+    File.expand_path('../node', __dir__)
+  end
+end

data/node/.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+/.env

data/node/.npmrc

@@ -0,0 +1 @@
+fund=false

data/node/README.md

@@ -0,0 +1,19 @@
+# rubycrawl Node Service
+
+Local Playwright-backed HTTP service used by the Ruby gem.
+
+## Run
+
+```
+npm install
+npm start
+```
+
+## Environment
+
+Create a `.env` file (or copy from `.env.example`) if you need custom settings.
+
+## Endpoints
+
+- `POST /crawl` JSON body: `{ "url": "https://example.com" }`
+- `GET /health`

data/node/node_modules/.bin/playwright

@@ -0,0 +1 @@
+../playwright/cli.js

data/node/node_modules/.bin/playwright-core

@@ -0,0 +1 @@
+../playwright-core/cli.js