nous 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e781bd7a823a8022f4a6e5a4346c183aadde727a59306103be9ffa937b7029dc
-  data.tar.gz: 80bcb682c861204b589c3a548088d5e5806d23281d509e2556e27f9fa8ef3960
+  metadata.gz: 7636d207654dbf38a64aeec480164c0e57b3c8bf98ac8373e576f692896fb3a3
+  data.tar.gz: 62ae3b01ec837d71caf104710c42bde82df6d50e6c7acc50252f2902ef9b2046
 SHA512:
-  metadata.gz: 4cd9a3a161b7203689063820d9e5bc5fda4c04be288262ac5bdddeaae46bc283bd2a85a4029579b28329c5f1bcc6d324d3042ce4bde9a53763ab4d19830bfa28
-  data.tar.gz: 3b0a4e96b51060064f3494b7227aa43ffa0e487a9a01f36ce610be7c852c5a86570665bbd92be07a622fa3b25efd2eef1c772f1db56161d63c062d770a973e26
+  metadata.gz: af52f527a8720d46cd00f3a42814d432730d105aed05ebb2435f1546afb2140bd88fd1e2c6f4e75c0226afd5ef6c9072c6919518bae366047eb022f24b30ffcd
+  data.tar.gz: '049b133f406f694771617c34d3adfef2aa64ef3aa5608d89d98230b2f596e4cdef831e53f9ee039aa247a6c588900fc51cfd07a7b456dec3ee524e83abe08b93'

data/.ruby-version

@@ -0,0 +1 @@
+4.0.1

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 ## [Unreleased]
 
+## [0.3.0] - 2026-02-23
+
+- Remove `Nous::Error` base hierarchy; colocated errors inherit directly from `StandardError` with descriptive names
+- Move extraction pipeline under `Nous::Fetcher::*` namespace (`ExtractionRunner`, `ExtractionThread`)
+- Move readability command into `Nous::Extractor::Default::Client`, mirroring Jina structure
+- `Nous::Extractor` is now a module namespace (implicit via Zeitwerk), no longer a Command
+- Shared `Extractor::ExtractionError` contract: all extractor backends raise this on failure
+- Pull `seed_url` off `Configuration`; `Crawler` owns URL parsing and validation directly
+- Explicit rescue lists in CLI and extraction thread instead of broad `Nous::Error` rescue
+- Rename `--verbose`/`-v` to `--debug`/`-d`; `-v` is now `--version`
+- Add `Nous::Url`, `Nous::UrlResolver`, and `Crawler::RedirectFollower` to correctly handle redirects and path encoding (including spaces)
+- Add `-r`/`--recursive`; default mode now fetches only the seed page unless recursion is explicitly enabled
+- Split crawler fetchers by mode: `Crawler::AsyncPageFetcher`, `Crawler::RecursivePageFetcher`, and `Crawler::SinglePageFetcher`
+- Move configuration construction to `ConfigurationBuilder` and `Data.define`-based `Configuration` primitive
+- Add `faraday-follow_redirects` for single-page redirect handling and update integration/spec coverage for recursive and single-page flows
+
+## [0.2.0] - 2026-02-21
+
+- Promote Configuration to module-level singleton (`Nous.configure`, `Nous.configuration`)
+- Eliminate verbose/concurrency/timeout parameter drilling through pipeline classes
+- Promote ExtractionRunner to Command pattern
+- Suppress async-pool gardener ThreadError in non-verbose mode
+- Add CLI specs and full pipeline integration test
+- Replace boilerplate README with real documentation
+- Upgrade to Ruby 4.0.1
+
 ## [0.1.0] - 2026-02-21
 
 - Initial release

data/README.md

@@ -1,6 +1,6 @@
 # Nous
 
-Crawl websites and extract readable Markdown, optimized for LLM consumption.
+Crawl websites and extract readable Markdown, optimized for LLM consumption. Inspired by [sitefetch](https://github.com/egoist/sitefetch).
 
 Nous fetches same-host pages starting from a seed URL, extracts readable content, and outputs clean Markdown as XML-tagged text or JSON. It supports concurrent crawling, glob-based URL filtering, and two extraction backends: a local parser (ruby-readability) and the Jina Reader API for JS-rendered sites.
 
@@ -42,8 +42,8 @@ nous https://example.com -s "article.post"
 # Use Jina Reader API for JS-rendered sites (Next.js, SPAs)
 nous https://example.com --jina
 
-# Verbose logging
-nous https://example.com -v
+# Debug logging
+nous https://example.com -d
 ```
 
 ### Options
@@ -58,7 +58,9 @@ nous https://example.com -v
 | `-l`, `--limit N` | Maximum pages to fetch | `100` |
 | `--timeout N` | Per-request timeout in seconds | `15` |
 | `--jina` | Use Jina Reader API for extraction | off |
-| `-v`, `--verbose` | Verbose logging to stderr | off |
+| `-v`, `--version` | Print version and exit | off |
+| `-h`, `--help` | Print usage and exit | off |
+| `-d`, `--debug` | Debug logging to stderr | off |
 
 ## Ruby API
 
@@ -131,9 +133,10 @@ Extracted markdown content...
 ## Development
 
 ```bash
-bin/setup            # Install dependencies
-bundle exec rspec    # Run tests
+bin/setup               # Install dependencies
+bundle exec rspec       # Run tests
 bundle exec standardrb  # Lint
+bundle exec exe/nous    # Run the command line in-development
 ```
 
 ## License

data/lib/nous/cli.rb

@@ -4,7 +4,7 @@ require "optparse"
 
 module Nous
   class Cli
-    class Error < Nous::Error; end
+    class CliError < StandardError; end
 
     def initialize(argv)
       @argv = argv
@@ -18,7 +18,9 @@ module Nous
       pages = Nous.fetch(seed_url, **fetch_options)
       output = Nous.serialize(pages, format: options[:format])
       write_output(output)
-    rescue Nous::Error => e
+    rescue CliError,
+      Fetcher::FetchError,
+      Serializer::SerializationError => e
       warn("nous: #{e.message}")
       exit 1
     end
@@ -32,7 +34,7 @@ module Nous
     end
 
     def fetch_options
-      opts = options.slice(:concurrency, :match, :limit, :timeout, :verbose)
+      opts = options.slice(*Configuration.members)
       opts[:extractor] = extractor
       opts
     end
@@ -44,7 +46,7 @@ module Nous
     end
 
     def validate!
-      raise Error, "no URL provided. Usage: nous <url> [options]" unless seed_url
+      raise CliError, "no URL provided. Usage: nous <url> [options]" unless seed_url
     end
 
     def write_output(output)
@@ -58,7 +60,7 @@ module Nous
     def parse_options!
       parser.parse!(argv)
     rescue OptionParser::InvalidOption => e
-      raise Error, e.message
+      raise CliError, e.message
     end
 
     def parser
@@ -77,13 +79,14 @@ module Nous
         opts.on("-l", "--limit N", Integer, "Maximum pages to fetch") { |v| options[:limit] = v }
         opts.on("--timeout N", Integer, "Per-request timeout in seconds (default: 15)") { |v| options[:timeout] = v }
         opts.on("--jina", "Use Jina Reader API for extraction (handles JS-rendered sites)") { options[:jina] = true }
-        opts.on("-v", "--verbose", "Verbose logging to stderr") { options[:verbose] = true }
-        opts.on("-h", "--help", "Show help") do
-          $stdout.puts(opts)
+        opts.on("-r", "--recursive", "Follow same-host links recursively") { options[:recursive] = true }
+        opts.on("-d", "--debug", "Debug logging to stderr") { options[:debug] = true }
+        opts.on("-v", "--version", "Show version") do
+          $stdout.puts("nous #{Nous::VERSION}")
           exit
         end
-        opts.on("--version", "Show version") do
-          $stdout.puts("nous #{Nous::VERSION}")
+        opts.on("-h", "--help", "Show help") do
+          $stdout.puts(opts)
           exit
         end
       end

data/lib/nous/command.rb

@@ -2,7 +2,7 @@
 
 module Nous
   class Command
-    class Error < Nous::Error; end
+    class CommandError < StandardError; end
 
     class Result
       attr_reader :payload, :error, :metadata
@@ -27,7 +27,7 @@ module Nous
       command = new(...)
       command.call
     rescue => e
-      return command.failure(Error.new("unexpected: #{e.message}")) if command
+      return command.failure(CommandError.new("unexpected: #{e.message}")) if command
 
       Result.new(success: false, error: e)
     end

data/lib/nous/configuration_builder.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Nous
+  class ConfigurationBuilder
+    class UnknownOptionError < StandardError; end
+
+    DEFAULTS = {
+      concurrency: 3,
+      match: [],
+      limit: 100,
+      timeout: 15,
+      debug: false,
+      keep_query: false,
+      recursive: false
+    }.freeze
+
+    def self.call(**options)
+      new(options).call
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    def call
+      validate_keys!
+
+      Configuration.new(**coerced_options)
+    end
+
+    private
+
+    attr_reader :options
+
+    def validate_keys!
+      unknown = options.keys - Configuration.members
+      return if unknown.empty?
+
+      raise UnknownOptionError, "unknown option(s): #{unknown.join(", ")}"
+    end
+
+    def coerced_options
+      merged = DEFAULTS.merge(options)
+
+      {
+        concurrency: Integer(merged[:concurrency]).clamp(1, 20),
+        match: Array(merged[:match]),
+        limit: Integer(merged[:limit]).clamp(1, 10_000),
+        timeout: Integer(merged[:timeout]),
+        debug: !!merged[:debug],
+        keep_query: !!merged[:keep_query],
+        recursive: !!merged[:recursive]
+      }
+    end
+  end
+end

data/lib/nous/converter.rb

@@ -4,7 +4,7 @@ require "reverse_markdown"
 
 module Nous
   class Converter < Command
-    class Error < Command::Error; end
+    class ConversionError < StandardError; end
 
     def initialize(html:)
       @html = html

data/lib/nous/crawler/{page_fetcher.rb → async_page_fetcher.rb}

@@ -2,34 +2,40 @@
 
 module Nous
   class Crawler < Command
-    class PageFetcher
+    class AsyncPageFetcher
       HTML_CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
 
-      def initialize(client:, timeout:, verbose: false)
+      def initialize(client:, seed_host:)
         @client = client
-        @timeout = timeout
-        @verbose = verbose
+        @seed_host = seed_host
       end
 
       def fetch(url)
-        Async::Task.current.with_timeout(timeout) do
-          response = client.get(url, {})
+        Async::Task.current.with_timeout(config.timeout) do
+          result = RedirectFollower.call(client:, seed_host:, url:)
+          return skip(url, result.error.message) if result.failure?
+
+          response, final_url = result.payload
           return skip(url, "status #{response.status}") unless response.status == 200
           return skip(url, "non-html content") unless html?(response)
 
-          {url:, pathname: URI.parse(url).path, html: response.read}
+          RawPage.new(url: final_url.to_s, pathname: final_url.path, html: response.read)
         ensure
           response&.close
         end
       rescue Async::TimeoutError
-        skip(url, "timeout after #{timeout}s")
+        skip(url, "timeout after #{config.timeout}s")
       rescue IOError, SocketError, Errno::ECONNREFUSED => e
         skip(url, e.message)
       end
 
       private
 
-      attr_reader :client, :timeout, :verbose
+      attr_reader :client, :seed_host
+
+      def config
+        Nous.configuration
+      end
 
       def html?(response)
         content_type = response.headers["content-type"].to_s
@@ -37,7 +43,7 @@ module Nous
       end
 
       def skip(url, reason)
-        warn("[nous] skip #{url}: #{reason}") if verbose
+        warn("[nous] skip #{url}: #{reason}") if config.debug?
         nil
       end
     end

data/lib/nous/crawler/link_extractor.rb

@@ -1,40 +1,39 @@
 # frozen_string_literal: true
 
+require "nokogiri"
+
 module Nous
   class Crawler < Command
     class LinkExtractor
-      def initialize(url_filter:, verbose: false)
+      def initialize(url_filter:)
         @url_filter = url_filter
-        @verbose = verbose
       end
 
       def extract(current_url, html)
-        base_uri = URI.parse(current_url)
-
-        anchors(html).filter_map { |href| resolve(base_uri, href) }.uniq
+        anchors(html).filter_map { |href| resolve(current_url, href) }.uniq
       end
 
       private
 
-      attr_reader :url_filter, :verbose
+      attr_reader :url_filter
 
       def anchors(html)
         Nokogiri::HTML(html).css("a[href]").map { |node| node["href"] }
       end
 
-      def resolve(base_uri, href)
+      def resolve(current_url, href)
         return unless url_filter.allowed?(href)
 
-        uri = URI.join(base_uri, href)
-        return unless url_filter.same_host?(uri)
+        result = UrlResolver.call(base_url: current_url, href:)
+        return unless result.success?
+
+        url = result.payload
+        return unless url_filter.same_host?(url)
 
-        canonical = url_filter.canonicalize(uri)
-        return unless url_filter.matches_path?(URI.parse(canonical).path)
+        canonical = url_filter.canonicalize(url)
+        return unless url_filter.matches_path?(Url.new(canonical).path)
 
         canonical
-      rescue URI::InvalidURIError => e
-        warn("[nous] malformed href #{href.inspect}: #{e.message}") if verbose
-        nil
       end
     end
   end

data/lib/nous/crawler/recursive_page_fetcher.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/http/internet"
+
+module Nous
+  class Crawler < Command
+    class RecursivePageFetcher < Command
+      def initialize(seed_url:, http_client: nil)
+        @seed_uri = Url.new(seed_url)
+        @http_client = http_client
+        @pages = []
+        @queue = [url_filter.canonicalize(seed_uri)]
+        @seen = Set.new(queue)
+      end
+
+      def call
+        suppress_async_warnings unless config.debug?
+
+        open_connection do |client|
+          crawl(client)
+        end
+
+        success(payload: pages)
+      end
+
+      private
+
+      attr_reader :seed_uri, :http_client, :pages, :queue, :seen
+
+      def config
+        Nous.configuration
+      end
+
+      def crawl(client)
+        fetch_and_enqueue(queue.shift(config.concurrency), client) while queue.any? && within_limit?
+      end
+
+      def fetch_and_enqueue(batch, client)
+        fetch_batch(batch, client).each do |page|
+          next unless page
+          break unless within_limit?
+
+          pages << page
+          seen << page.url
+          enqueue_links(page)
+        end
+      end
+
+      def fetch_batch(urls, client)
+        tasks = []
+
+        Async do |task|
+          urls.each do |url|
+            tasks << task.async { page_fetcher(client).fetch(url) }
+          end
+        end.wait
+
+        tasks.map(&:wait)
+      end
+
+      def enqueue_links(page)
+        link_extractor.extract(page.url, page.html).each do |url|
+          next if seen.include?(url)
+
+          seen << url
+          queue << url
+        end
+      end
+
+      def within_limit?
+        pages.length < config.limit
+      end
+
+      def open_connection
+        client = http_client || Async::HTTP::Internet.new
+
+        Async do
+          yield client
+        ensure
+          client.close
+        end.wait
+      end
+
+      def page_fetcher(client)
+        AsyncPageFetcher.new(client:, seed_host: seed_uri.host)
+      end
+
+      def url_filter
+        @url_filter ||= UrlFilter.new(seed_uri:)
+      end
+
+      def link_extractor
+        @link_extractor ||= LinkExtractor.new(url_filter:)
+      end
+
+      def suppress_async_warnings
+        require "console"
+        Console.logger.level = :error
+      end
+    end
+  end
+end

data/lib/nous/crawler/redirect_follower.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Nous
+  class Crawler < Command
+    class RedirectFollower < Command
+      class RedirectError < StandardError; end
+
+      MAX_HOPS = 5
+
+      def initialize(client:, seed_host:, url:, hops_remaining: MAX_HOPS)
+        @client = client
+        @seed_host = seed_host
+        @url = url
+        @hops_remaining = hops_remaining
+      end
+
+      def call
+        response = client.get(url, {})
+
+        return success(payload: [response, Url.new(url)]) unless redirect?(response.status)
+
+        response.close
+        follow(response.headers["location"])
+      end
+
+      private
+
+      attr_reader :client, :seed_host, :url, :hops_remaining
+
+      def redirect?(status)
+        (300..399).cover?(status)
+      end
+
+      def follow(location)
+        target = resolve_target(location)
+        return target if target.failure?
+
+        self.class.call(client:, seed_host:, url: target.payload.to_s, hops_remaining: hops_remaining - 1)
+      end
+
+      def resolve_target(location)
+        return failure(RedirectError.new("redirect without location from #{url}")) unless location
+        return failure(RedirectError.new("too many redirects from #{url}")) if hops_remaining <= 0
+
+        result = UrlResolver.call(base_url: url, href: location)
+        return failure(RedirectError.new(result.error.message)) if result.failure?
+
+        unless safe?(result.payload)
+          return failure(RedirectError.new("redirect to #{result.payload} outside #{seed_host}"))
+        end
+
+        result
+      end
+
+      def safe?(target)
+        target.http? && target.host == seed_host
+      end
+    end
+  end
+end

data/lib/nous/crawler/single_page_fetcher.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/follow_redirects"
+
+module Nous
+  class Crawler < Command
+    class SinglePageFetcher < Command
+      class FetchError < StandardError; end
+
+      HTML_CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
+      MAX_REDIRECTS = 5
+
+      def initialize(url:, http_client: nil)
+        @url = url
+        @seed_host = Url.new(url).host
+        @connection = http_client || build_connection
+      end
+
+      def call
+        response = connection.get(url)
+        final_url = resolve_final_url(response)
+
+        validate_host!(final_url)
+        validate_html!(response)
+
+        raw_page = RawPage.new(url: final_url.to_s, pathname: final_url.path, html: response.body)
+        success(payload: [raw_page])
+      rescue FetchError => e
+        failure(e)
+      rescue Faraday::Error => e
+        failure(FetchError.new(e.message))
+      end
+
+      private
+
+      attr_reader :url, :seed_host, :connection
+
+      def config
+        Nous.configuration
+      end
+
+      def resolve_final_url(response)
+        location = response.env.url.to_s
+        Url.new(location)
+      end
+
+      def validate_host!(final_url)
+        return if final_url.host == seed_host
+
+        raise FetchError, "redirected to #{final_url} outside #{seed_host}"
+      end
+
+      def validate_html!(response)
+        content_type = response.headers["content-type"].to_s
+        return if HTML_CONTENT_TYPES.any? { |type| content_type.include?(type) }
+
+        raise FetchError, "non-html content: #{content_type}"
+      end
+
+      def build_connection
+        Faraday.new do |f|
+          f.response :follow_redirects, limit: MAX_REDIRECTS
+          f.response :raise_error
+
+          f.options.timeout = config.timeout
+          f.options.open_timeout = config.timeout
+        end
+      end
+    end
+  end
+end

data/lib/nous/crawler/url_filter.rb

@@ -5,10 +5,10 @@ module Nous
     class UrlFilter
       IGNORED_SCHEMES = %w[mailto: javascript: tel:].freeze
 
-      def initialize(config)
-        @host = config.seed.host
-        @match = config.match
-        @keep_query = config.keep_query
+      def initialize(seed_uri:)
+        @host = seed_uri.host
+        @match = Nous.configuration.match
+        @keep_query = Nous.configuration.keep_query
       end
 
       def canonicalize(uri)
@@ -25,8 +25,8 @@ module Nous
         IGNORED_SCHEMES.none? { |s| href.start_with?(s) }
       end
 
-      def same_host?(uri)
-        uri.is_a?(URI::HTTP) && uri.host == host
+      def same_host?(url)
+        url.http? && url.host == host
       end
 
       def matches_path?(path)