nous 0.2.0 → 0.4.0

data/lib/nous/serializer.rb

@@ -4,7 +4,7 @@ require "json"
 
 module Nous
   class Serializer < Command
-    class Error < Command::Error; end
+    class SerializationError < StandardError; end
 
     FORMATS = %i[text json].freeze
 
@@ -24,7 +24,10 @@ module Nous
     attr_reader :pages, :format
 
     def validate_format!
-      raise Error, "unknown format: #{format}. Must be one of: #{FORMATS.join(", ")}" unless FORMATS.include?(format)
+      return if FORMATS.include?(format)
+
+      raise SerializationError,
+        "unknown format: #{format}. Must be one of: #{FORMATS.join(", ")}"
     end
 
     def serialize_text
@@ -40,6 +43,8 @@ module Nous
         <page>
           <title>#{page.title}</title>
           <url>#{page.url}</url>
+          <pathname>#{page.pathname}</pathname>
+          <extractor>#{page.metadata[:extractor]}</extractor>
           <content>
         #{page.content}
           </content>
@@ -48,7 +53,13 @@ module Nous
     end
 
     def json_page(page)
-      {title: page.title, url: page.url, content: page.content}
+      {
+        title: page.title,
+        url: page.url,
+        pathname: page.pathname,
+        content: page.content,
+        metadata: page.metadata
+      }
     end
   end
 end

data/lib/nous/url_resolver.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "addressable/uri"
+
+module Nous
+  class UrlResolver < Command
+    class ResolutionError < StandardError; end
+
+    def initialize(base_url:, href:)
+      @base_uri = Addressable::URI.parse(base_url.to_s)
+      @href = href.to_s.strip
+    end
+
+    def call
+      joined = base_uri.join(href)
+      success(payload: Url.new(joined))
+    rescue Addressable::URI::InvalidURIError => e
+      failure(ResolutionError.new("cannot resolve #{href.inspect}: #{e.message}"))
+    end
+
+    private
+
+    attr_reader :base_uri, :href
+  end
+end

data/lib/nous/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nous
-  VERSION = '0.2.0'
+  VERSION = "0.4.0"
 end

data/lib/nous.rb

@@ -3,24 +3,25 @@
 require "zeitwerk"
 
 loader = Zeitwerk::Loader.for_gem
+loader.collapse("#{__dir__}/nous/primitives")
 loader.setup
 
 module Nous
   class << self
     attr_reader :configuration
 
-    def configure(seed_url:, **options)
-      @configuration = Configuration.new(seed_url:, **options)
+    def configure(...)
+      @configuration = ConfigurationBuilder.call(...)
     end
 
     def reset_configuration!
       @configuration = nil
     end
 
-    def fetch(seed_url, extractor: Extractor::Default.new, **options)
-      configure(seed_url:, **options)
+    def fetch(seed_url, extractor: Extractor::Default.new, http_client: nil, details: false, **options)
+      configure(**options)
 
-      result = Fetcher.call(seed_url:, extractor:)
+      result = Fetcher.call(seed_url:, extractor:, http_client:, details:)
       raise result.error if result.failure?
 
       result.payload

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nous
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Dan Frenette
@@ -9,6 +9,20 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: async
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +65,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.12'
+- !ruby/object:Gem::Dependency
+  name: faraday-follow_redirects
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: faraday-retry
   requirement: !ruby/object:Gem::Requirement
@@ -197,22 +225,30 @@ files:
 - lib/nous.rb
 - lib/nous/cli.rb
 - lib/nous/command.rb
-- lib/nous/configuration.rb
+- lib/nous/configuration_builder.rb
 - lib/nous/converter.rb
 - lib/nous/crawler.rb
+- lib/nous/crawler/async_page_fetcher.rb
 - lib/nous/crawler/link_extractor.rb
-- lib/nous/crawler/page_fetcher.rb
+- lib/nous/crawler/recursive_page_fetcher.rb
+- lib/nous/crawler/redirect_follower.rb
+- lib/nous/crawler/single_page_fetcher.rb
 - lib/nous/crawler/url_filter.rb
-- lib/nous/error.rb
-- lib/nous/extraction_runner.rb
-- lib/nous/extraction_thread.rb
-- lib/nous/extractor.rb
 - lib/nous/extractor/default.rb
+- lib/nous/extractor/default/client.rb
 - lib/nous/extractor/jina.rb
 - lib/nous/extractor/jina/client.rb
 - lib/nous/fetcher.rb
-- lib/nous/page.rb
+- lib/nous/fetcher/extraction_runner.rb
+- lib/nous/fetcher/page_extractor.rb
+- lib/nous/primitives/configuration.rb
+- lib/nous/primitives/extracted_content.rb
+- lib/nous/primitives/fetch_record.rb
+- lib/nous/primitives/fetch_result.rb
+- lib/nous/primitives/page.rb
+- lib/nous/primitives/url.rb
 - lib/nous/serializer.rb
+- lib/nous/url_resolver.rb
 - lib/nous/version.rb
 - sig/nous.rbs
 homepage: https://github.com/danfrenette/nous

data/lib/nous/configuration.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module Nous
-  class Configuration
-    class Error < Nous::Error; end
-
-    attr_reader :seed, :concurrency, :match, :limit, :timeout, :keep_query
-
-    DEFAULT_CONCURRENCY = 3
-    DEFAULT_LIMIT = 100
-    DEFAULT_TIMEOUT = 15
-
-    def initialize(seed_url:, concurrency: DEFAULT_CONCURRENCY, match: [], limit: DEFAULT_LIMIT,
-      timeout: DEFAULT_TIMEOUT, verbose: false, keep_query: false)
-      @seed = parse_seed!(seed_url)
-      @concurrency = Integer(concurrency).clamp(1, 20)
-      @match = Array(match)
-      @limit = Integer(limit).clamp(1, 10_000)
-      @timeout = Integer(timeout)
-      @verbose = verbose
-      @keep_query = keep_query
-    end
-
-    def verbose? = @verbose
-
-    private
-
-    def parse_seed!(url)
-      uri = URI.parse(url)
-      raise Error, "seed URL must be http or https" unless uri.is_a?(URI::HTTP)
-
-      uri
-    rescue URI::InvalidURIError => e
-      raise Error, "invalid seed URL: #{e.message}"
-    end
-  end
-end

data/lib/nous/crawler/page_fetcher.rb

@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  class Crawler < Command
-    class PageFetcher
-      HTML_CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
-
-      def initialize(client:)
-        @client = client
-      end
-
-      def fetch(url)
-        Async::Task.current.with_timeout(config.timeout) do
-          response = client.get(url, {})
-          return skip(url, "status #{response.status}") unless response.status == 200
-          return skip(url, "non-html content") unless html?(response)
-
-          {url:, pathname: URI.parse(url).path, html: response.read}
-        ensure
-          response&.close
-        end
-      rescue Async::TimeoutError
-        skip(url, "timeout after #{config.timeout}s")
-      rescue IOError, SocketError, Errno::ECONNREFUSED => e
-        skip(url, e.message)
-      end
-
-      private
-
-      attr_reader :client
-
-      def config
-        Nous.configuration
-      end
-
-      def html?(response)
-        content_type = response.headers["content-type"].to_s
-        HTML_CONTENT_TYPES.any? { |type| content_type.include?(type) }
-      end
-
-      def skip(url, reason)
-        warn("[nous] skip #{url}: #{reason}") if config.verbose?
-        nil
-      end
-    end
-  end
-end

data/lib/nous/error.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  class Error < StandardError; end
-end

data/lib/nous/extraction_runner.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  class ExtractionRunner < Command
-    class Error < Command::Error; end
-
-    def initialize(raw_pages:, extractor:)
-      @raw_pages = raw_pages
-      @extractor = extractor
-    end
-
-    def call
-      pages = raw_pages.each_slice(Nous.configuration.concurrency).each_with_object([]) do |batch, results|
-        threads = batch.map { |raw| Thread.new { ExtractionThread.new(extractor:, raw_page: raw).call } }
-
-        threads.each do |thread|
-          result = thread.value
-          results << result if result
-        end
-      end
-
-      success(payload: pages)
-    end
-
-    private
-
-    attr_reader :raw_pages, :extractor
-  end
-end

data/lib/nous/extraction_thread.rb

@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  class ExtractionThread
-    def initialize(extractor:, raw_page:)
-      @extractor = extractor
-      @raw_page = raw_page
-    end
-
-    def call
-      extracted = extractor.extract(raw_page)
-
-      Page.new(
-        title: extracted[:title],
-        url: raw_page[:url],
-        pathname: raw_page[:pathname],
-        content: extracted[:content]
-      )
-    rescue Nous::Error => e
-      warn("[nous] extract skip #{raw_page[:url]}: #{e.message}") if Nous.configuration.verbose?
-      nil
-    end
-
-    private
-
-    attr_reader :extractor, :raw_page
-  end
-end

data/lib/nous/extractor.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require "readability"
-
-module Nous
-  class Extractor < Command
-    class Error < Command::Error; end
-
-    NOISY_TAGS = %w[script style link nav header footer img video svg].freeze
-
-    def initialize(html:, selector: nil)
-      @html = html
-      @selector = selector
-    end
-
-    def call
-      doc = Nokogiri::HTML(html)
-      doc = scope_to_selector(doc) if selector
-      strip_noisy_tags(doc)
-
-      readable = Readability::Document.new(doc.to_html)
-      text = Nokogiri::HTML(readable.content).text.strip
-
-      return failure(Error.new("readability returned no content")) if text.empty?
-
-      success(payload: {title: readable.title, content: readable.content})
-    end
-
-    private
-
-    attr_reader :html, :selector
-
-    def scope_to_selector(doc)
-      scoped = doc.at_css(selector)
-      return doc unless scoped
-
-      fragment = Nokogiri::HTML::Document.new
-      fragment.root = scoped
-      fragment
-    end
-
-    def strip_noisy_tags(doc)
-      NOISY_TAGS.each { |tag| doc.css(tag).each(&:remove) }
-    end
-  end
-end

data/lib/nous/page.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-module Nous
-  Page = Data.define(:title, :url, :pathname, :content)
-end