nous 0.2.0 → 0.4.0

data/lib/nous/crawler/single_page_fetcher.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/follow_redirects"
+
+module Nous
+  class Crawler < Command
+    class SinglePageFetcher < Command
+      class FetchError < StandardError; end
+
+      HTML_CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
+      MAX_REDIRECTS = 5
+
+      def initialize(url:, http_client: nil)
+        @url = url
+        @seed_host = Url.new(url).host
+        @connection = http_client || build_connection
+      end
+
+      def call
+        response = connection.get(url)
+        final_url = resolve_final_url(response)
+        content_type = response.headers["content-type"].to_s
+        redirected = final_url.to_s != url
+
+        record = build_record(
+          final_url: final_url.to_s,
+          pathname: final_url.path,
+          html: response.body,
+          content_type: content_type,
+          redirected: redirected
+        )
+
+        validate!(record)
+
+        success(payload: [record])
+      rescue FetchError => e
+        record = build_failed_record(error: e.message)
+        success(payload: [record])
+      rescue Faraday::Error => e
+        record = build_failed_record(error: e.message)
+        success(payload: [record])
+      end
+
+      private
+
+      attr_reader :url, :seed_host, :connection
+
+      def config
+        Nous.configuration
+      end
+
+      def build_record(final_url:, pathname:, html:, content_type:, redirected:)
+        FetchRecord.new(
+          requested_url: url,
+          final_url: final_url,
+          pathname: pathname,
+          html: html,
+          content_type: content_type,
+          ok: true,
+          error: nil,
+          redirected: redirected
+        )
+      end
+
+      def build_failed_record(error:)
+        FetchRecord.new(
+          requested_url: url,
+          final_url: nil,
+          pathname: Url.new(url).path,
+          html: nil,
+          content_type: nil,
+          ok: false,
+          error: error,
+          redirected: false
+        )
+      end
+
+      def validate!(record)
+        validate_host!(record.final_url)
+        validate_html!(record.content_type)
+      end
+
+      def resolve_final_url(response)
+        location = response.env.url.to_s
+        Url.new(location)
+      end
+
+      def validate_host!(final_url)
+        return if Url.new(final_url).host == seed_host
+
+        raise FetchError, "redirected to #{final_url} outside #{seed_host}"
+      end
+
+      def validate_html!(content_type)
+        return if HTML_CONTENT_TYPES.any? { |type| content_type.include?(type) }
+
+        raise FetchError, "non-html content: #{content_type}"
+      end
+
+      def build_connection
+        Faraday.new do |f|
+          f.response :follow_redirects, limit: MAX_REDIRECTS
+          f.response :raise_error
+
+          f.options.timeout = config.timeout
+          f.options.open_timeout = config.timeout
+        end
+      end
+    end
+  end
+end

data/lib/nous/crawler/url_filter.rb

@@ -5,10 +5,10 @@ module Nous
     class UrlFilter
       IGNORED_SCHEMES = %w[mailto: javascript: tel:].freeze
 
-      def initialize(config)
-        @host = config.seed.host
-        @match = config.match
-        @keep_query = config.keep_query
+      def initialize(seed_uri:)
+        @host = seed_uri.host
+        @match = Nous.configuration.match
+        @keep_query = Nous.configuration.keep_query
       end
 
       def canonicalize(uri)
@@ -25,8 +25,8 @@ module Nous
         IGNORED_SCHEMES.none? { |s| href.start_with?(s) }
       end
 
-      def same_host?(uri)
-        uri.is_a?(URI::HTTP) && uri.host == host
+      def same_host?(url)
+        url.http? && url.host == host
       end
 
       def matches_path?(path)

data/lib/nous/crawler.rb

@@ -1,91 +1,36 @@
 # frozen_string_literal: true
 
-require "async"
-require "async/http/internet"
-require "nokogiri"
-require "uri"
-
 module Nous
   class Crawler < Command
-    class Error < Command::Error; end
+    class CrawlError < StandardError; end
 
-    def initialize(seed_url:)
+    def initialize(seed_url:, http_client: nil)
       @seed_url = seed_url
+      @http_client = http_client
+      parse_seed!
     end
 
     def call
-      suppress_async_warnings unless config.verbose?
-
-      pages = []
-      queue = [url_filter.canonicalize(config.seed)]
-      seen = Set.new(queue)
-
-      Async do
-        client = Async::HTTP::Internet.new
-        begin
-          crawl(queue:, seen:, pages:, client:)
-        ensure
-          client.close
-        end
-      end.wait
-
-      success(payload: pages)
+      if config.recursive?
+        RecursivePageFetcher.call(seed_url:, http_client:)
+      else
+        SinglePageFetcher.call(url: seed_url, http_client:)
+      end
     end
 
     private
 
-    attr_reader :seed_url
+    attr_reader :seed_url, :http_client
 
     def config
       Nous.configuration
     end
 
-    def crawl(queue:, seen:, pages:, client:)
-      while queue.any? && pages.length < config.limit
-        batch = queue.shift(config.concurrency)
-        fetch_batch(batch, client).each do |page|
-          next unless page
-
-          pages << page
-          break if pages.length >= config.limit
-
-          link_extractor.extract(page[:url], page[:html]).each do |url|
-            next if seen.include?(url)
-
-            seen << url
-            queue << url
-          end
-        end
-      end
-    end
-
-    def fetch_batch(urls, client)
-      tasks = []
-
-      Async do |task|
-        urls.each do |url|
-          tasks << task.async { page_fetcher(client).fetch(url) }
-        end
-      end.wait
-
-      tasks.map(&:wait)
-    end
-
-    def url_filter
-      @url_filter ||= UrlFilter.new(config)
-    end
-
-    def link_extractor
-      @link_extractor ||= LinkExtractor.new(url_filter:)
-    end
-
-    def page_fetcher(client)
-      PageFetcher.new(client:)
-    end
-
-    def suppress_async_warnings
-      require "console"
-      Console.logger.level = :error
+    def parse_seed!
+      parsed = Url.new(seed_url)
+      raise CrawlError, "seed URL must be http or https" unless parsed.http?
+    rescue ArgumentError => e
+      raise CrawlError, "invalid seed URL: #{e.message}"
     end
   end
 end

data/lib/nous/extractor/default/client.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "readability"
+
+module Nous
+  module Extractor
+    class Default
+      class Client < Command
+        class ExtractionError < StandardError; end
+
+        NOISY_TAGS = %w[script style nav footer].freeze
+
+        def initialize(html:, selector: nil)
+          @html = html
+          @selector = selector
+        end
+
+        def call
+          readable = ::Readability::Document.new(prepared_html)
+
+          text = Nokogiri::HTML(readable.content).text.strip
+          return failure(ExtractionError.new("readability returned no content")) if text.empty?
+
+          title = resolve_title(readable)
+          success(payload: {title: title, content: readable.content})
+        end
+
+        private
+
+        attr_reader :html, :selector
+
+        def prepared_html
+          doc = Nokogiri::HTML(html)
+          original_title(doc)
+          doc = scope(doc, selector) if selector
+          strip_tags(doc)
+          doc.to_html
+        end
+
+        def original_title(doc)
+          @original_title ||= doc.at_css("title")&.text.to_s.strip
+        end
+
+        def scope(doc, selector)
+          scoped = doc.at_css(selector)
+          return doc unless scoped
+
+          Nokogiri::HTML.fragment(scoped.to_html)
+        end
+
+        def strip_tags(doc)
+          NOISY_TAGS.each { |tag| doc.css(tag).each(&:remove) }
+        end
+
+        def resolve_title(readable)
+          title = readable.title.to_s.strip
+          title = @original_title if title.empty?
+          title = title_from_content(readable.content) if title.empty?
+          title
+        end
+
+        def title_from_content(content)
+          Nokogiri::HTML(content).at_css("h1")&.text.to_s.strip
+        end
+      end
+    end
+  end
+end

data/lib/nous/extractor/default.rb

@@ -1,17 +1,21 @@
 # frozen_string_literal: true
 
 module Nous
-  class Extractor
-    class Default
+  module Extractor
+    class ExtractionError < StandardError; end
+
+    class Default < Command
       def initialize(selector: nil)
         @selector = selector
       end
 
-      def extract(page)
-        extracted = extract_content(page[:html])
+      def extract(record)
+        extracted = extract_content(record.html)
         markdown = convert_to_markdown(extracted[:content])
 
-        {title: extracted[:title], content: markdown}
+        success(payload: ExtractedContent.new(title: extracted[:title], content: markdown))
+      rescue Client::ExtractionError, Converter::ConversionError => e
+        failure(ExtractionError.new(e.message))
       end
 
       private
@@ -19,7 +23,7 @@ module Nous
       attr_reader :selector
 
       def extract_content(html)
-        result = Extractor.call(html:, selector:)
+        result = Client.call(html:, selector:)
         raise result.error if result.failure?
 
         result.payload

data/lib/nous/extractor/jina/client.rb

@@ -5,10 +5,10 @@ require "faraday/retry"
 require "json"
 
 module Nous
-  class Extractor
+  module Extractor
     class Jina
       class Client
-        class Error < Nous::Error; end
+        class RequestError < StandardError; end
 
         BASE_URL = "https://r.jina.ai"
         RETRYABLE_STATUSES = [429, 500, 502, 503, 504].freeze
@@ -22,7 +22,7 @@ module Nous
           response = connection.get("/#{url}")
           parse(response.body)
         rescue Faraday::Error => e
-          raise Error, e.message
+          raise RequestError, e.message
         end
 
         private
@@ -51,7 +51,7 @@ module Nous
         def parse(body)
           JSON.parse(body)
         rescue JSON::ParserError => e
-          raise Error, "invalid JSON: #{e.message}"
+          raise RequestError, "invalid JSON: #{e.message}"
         end
       end
     end

data/lib/nous/extractor/jina.rb

@@ -1,20 +1,21 @@
 # frozen_string_literal: true
 
 module Nous
-  class Extractor
-    class Jina
-      class Error < Nous::Error; end
-
+  module Extractor
+    class Jina < Command
       def initialize(api_key: nil, timeout: 30, **client_options)
         @client = Client.new(api_key: api_key || ENV["JINA_API_KEY"], timeout:, **client_options)
       end
 
-      def extract(page)
-        body = client.get(page[:url])
+      def extract(record)
+        body = client.get(record.final_url)
 
-        {title: body.dig("data", "title") || "", content: body.dig("data", "content") || ""}
-      rescue Client::Error => e
-        raise Error, e.message
+        success(payload: ExtractedContent.new(
+          title: body.dig("data", "title") || "",
+          content: body.dig("data", "content") || ""
+        ))
+      rescue Client::RequestError => e
+        failure(ExtractionError.new(e.message))
       end
 
       private

data/lib/nous/fetcher/extraction_runner.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Nous
+  class Fetcher < Command
+    class ExtractionRunner < Command
+      class ExtractionError < StandardError; end
+
+      def initialize(records:, extractor:)
+        @records = records
+        @extractor = extractor
+      end
+
+      def call
+        pages = records.each_slice(Nous.configuration.concurrency).each_with_object([]) do |batch, results|
+          threads = batch.map { |record| Thread.new { PageExtractor.call(extractor:, record:) } }
+
+          threads.each do |thread|
+            result = thread.value
+            results << result.payload if result.success?
+          end
+        end
+
+        success(payload: pages)
+      end
+
+      private
+
+      attr_reader :records, :extractor
+    end
+  end
+end

data/lib/nous/fetcher/page_extractor.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Nous
+  class Fetcher < Command
+    class PageExtractor < Command
+      def initialize(extractor:, record:)
+        @extractor = extractor
+        @record = record
+      end
+
+      def call
+        result = extractor.extract(record)
+
+        unless result.success?
+          warn("[nous] extract skip #{record.final_url}: #{result.error.message}") if Nous.configuration.debug?
+          return failure(result.error)
+        end
+
+        page = Page.new(
+          title: result.payload.title,
+          url: record.final_url,
+          pathname: record.pathname,
+          content: result.payload.content,
+          metadata: {
+            extractor: extractor.class.name,
+            requested_url: record.requested_url,
+            content_type: record.content_type,
+            redirected: record.redirected
+          }
+        )
+
+        success(payload: page)
+      end
+
+      private
+
+      attr_reader :extractor, :record
+    end
+  end
+end

data/lib/nous/fetcher.rb

@@ -2,35 +2,62 @@
 
 module Nous
   class Fetcher < Command
-    class Error < Command::Error; end
+    class FetchError < StandardError; end
 
-    def initialize(seed_url:, extractor: Extractor::Default.new)
+    def initialize(seed_url:, extractor: Extractor::Default.new, http_client: nil, details: false)
       @seed_url = seed_url
       @extractor = extractor
+      @http_client = http_client
+      @single_page = Nous.configuration.single_page?
+      @details = details
     end
 
     def call
-      raw_pages = crawl
-      pages = extract(raw_pages)
-      success(payload: pages)
+      records = crawl
+      successful_records, failed_records = records.partition(&:ok)
+
+      if single_page && !details && successful_records.empty?
+        raise FetchError, failed_records.first&.error || "fetch failed"
+      end
+
+      pages = extract(successful_records)
+
+      if details
+        success(payload: FetchResult.new(
+          pages: pages,
+          failures: build_failures(failed_records),
+          total_requested: records.length
+        ))
+      else
+        success(payload: pages)
+      end
     end
 
     private
 
-    attr_reader :seed_url, :extractor
+    attr_reader :seed_url, :extractor, :http_client, :single_page, :details
 
     def crawl
-      result = Crawler.call(seed_url:)
-      raise Error, result.error.message if result.failure?
+      result = Crawler.call(seed_url:, http_client:)
+      raise FetchError, result.error.message if result.failure?
 
       result.payload
     end
 
-    def extract(raw_pages)
-      result = ExtractionRunner.call(raw_pages:, extractor:)
-      raise Error, result.error.message if result.failure?
+    def extract(records)
+      result = ExtractionRunner.call(records:, extractor:)
+      raise FetchError, result.error.message if result.failure?
 
       result.payload
     end
+
+    def build_failures(records)
+      records.map do |record|
+        {
+          requested_url: record.requested_url,
+          error: record.error
+        }
+      end
+    end
   end
 end

data/lib/nous/primitives/configuration.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Nous
+  Configuration = Data.define(
+    :concurrency,
+    :match,
+    :limit,
+    :timeout,
+    :debug,
+    :keep_query,
+    :recursive
+  ) do
+    def debug? = debug
+    def recursive? = recursive
+    def single_page? = !recursive
+  end
+end

data/lib/nous/primitives/extracted_content.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nous
+  ExtractedContent = Data.define(:title, :content)
+end

data/lib/nous/primitives/fetch_record.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Nous
+  FetchRecord = Data.define(
+    :requested_url,
+    :final_url,
+    :pathname,
+    :html,
+    :content_type,
+    :ok,
+    :error,
+    :redirected
+  ) do
+    def initialize(
+      requested_url:,
+      pathname:, final_url: nil,
+      html: nil,
+      content_type: nil,
+      ok: true,
+      error: nil,
+      redirected: false
+    )
+      super
+    end
+  end
+end

data/lib/nous/primitives/fetch_result.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Nous
+  FetchResult = Data.define(:pages, :failures, :total_requested) do
+    def succeeded
+      pages.length
+    end
+
+    def failed
+      failures.length
+    end
+
+    def all_succeeded?
+      failures.empty?
+    end
+
+    def any_succeeded?
+      pages.any?
+    end
+  end
+end

data/lib/nous/primitives/page.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nous
+  Page = Data.define(:title, :url, :pathname, :content, :metadata)
+end

data/lib/nous/primitives/url.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "addressable/uri"
+
+module Nous
+  class Url
+    def initialize(raw)
+      @uri = Addressable::URI.parse(raw.to_s.strip).normalize
+      raise ArgumentError, "invalid URL: #{raw}" if uri.to_s.empty?
+    rescue Addressable::URI::InvalidURIError => e
+      raise ArgumentError, "invalid URL: #{e.message}"
+    end
+
+    def host
+      uri.host
+    end
+
+    def path
+      return "/" if uri.path.empty?
+
+      uri.path
+    end
+
+    def http?
+      %w[http https].include?(uri.scheme)
+    end
+
+    def to_s
+      uri.to_s
+    end
+
+    def ==(other)
+      other.is_a?(Url) && to_s == other.to_s
+    end
+    alias_method :eql?, :==
+
+    def hash
+      to_s.hash
+    end
+
+    private
+
+    attr_reader :uri
+  end
+end