nous 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e781bd7a823a8022f4a6e5a4346c183aadde727a59306103be9ffa937b7029dc
-  data.tar.gz: 80bcb682c861204b589c3a548088d5e5806d23281d509e2556e27f9fa8ef3960
+  metadata.gz: c44bdc52070c6430739f9b0258ea53e3dafc1cff42d87814fd940c2e9e26ee94
+  data.tar.gz: e4b42ca9917d7e4656f8e8bc2d9b8b328781021c2ed02e9e1912bdb9ce8ac744
 SHA512:
-  metadata.gz: 4cd9a3a161b7203689063820d9e5bc5fda4c04be288262ac5bdddeaae46bc283bd2a85a4029579b28329c5f1bcc6d324d3042ce4bde9a53763ab4d19830bfa28
-  data.tar.gz: 3b0a4e96b51060064f3494b7227aa43ffa0e487a9a01f36ce610be7c852c5a86570665bbd92be07a622fa3b25efd2eef1c772f1db56161d63c062d770a973e26
+  metadata.gz: f55c5122dd9a53611c7045e648c34870f9e423afae6777d0004f0bc909c0b916fd5a8a0350168d286e2e63339be6ae393f9ee02cbe4703d1a392fceaee317fd0
+  data.tar.gz: fb6bdb6b9c283bc8350a4e697412869e9c1062af0659ad5b56aee6a0cdcad33983f8a15da9a94f3ae37b45b469a5d67a1c5e977d3d2c8b27a59e8f66eeedd59c

data/.ruby-version

@@ -0,0 +1 @@
+4.0.1

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 ## [Unreleased]
 
+## [0.2.0] - 2026-02-21
+
+- Promote Configuration to module-level singleton (`Nous.configure`, `Nous.configuration`)
+- Eliminate verbose/concurrency/timeout parameter drilling through pipeline classes
+- Promote ExtractionRunner to Command pattern
+- Suppress async-pool gardener ThreadError in non-verbose mode
+- Add CLI specs and full pipeline integration test
+- Replace boilerplate README with real documentation
+- Upgrade to Ruby 4.0.1
+
 ## [0.1.0] - 2026-02-21
 
 - Initial release

data/README.md

@@ -1,6 +1,6 @@
 # Nous
 
-Crawl websites and extract readable Markdown, optimized for LLM consumption.
+Crawl websites and extract readable Markdown, optimized for LLM consumption. Inspired by [sitefetch](https://github.com/egoist/sitefetch).
 
 Nous fetches same-host pages starting from a seed URL, extracts readable content, and outputs clean Markdown as XML-tagged text or JSON. It supports concurrent crawling, glob-based URL filtering, and two extraction backends: a local parser (ruby-readability) and the Jina Reader API for JS-rendered sites.
 
@@ -131,8 +131,8 @@ Extracted markdown content...
 ## Development
 
 ```bash
-bin/setup            # Install dependencies
-bundle exec rspec    # Run tests
+bin/setup               # Install dependencies
+bundle exec rspec       # Run tests
 bundle exec standardrb  # Lint
 ```
 

data/lib/nous/configuration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Nous
+  class Configuration
+    class Error < Nous::Error; end
+
+    attr_reader :seed, :concurrency, :match, :limit, :timeout, :keep_query
+
+    DEFAULT_CONCURRENCY = 3
+    DEFAULT_LIMIT = 100
+    DEFAULT_TIMEOUT = 15
+
+    def initialize(seed_url:, concurrency: DEFAULT_CONCURRENCY, match: [], limit: DEFAULT_LIMIT,
+      timeout: DEFAULT_TIMEOUT, verbose: false, keep_query: false)
+      @seed = parse_seed!(seed_url)
+      @concurrency = Integer(concurrency).clamp(1, 20)
+      @match = Array(match)
+      @limit = Integer(limit).clamp(1, 10_000)
+      @timeout = Integer(timeout)
+      @verbose = verbose
+      @keep_query = keep_query
+    end
+
+    def verbose? = @verbose
+
+    private
+
+    def parse_seed!(url)
+      uri = URI.parse(url)
+      raise Error, "seed URL must be http or https" unless uri.is_a?(URI::HTTP)
+
+      uri
+    rescue URI::InvalidURIError => e
+      raise Error, "invalid seed URL: #{e.message}"
+    end
+  end
+end

data/lib/nous/crawler/link_extractor.rb

@@ -3,9 +3,8 @@
 module Nous
   class Crawler < Command
     class LinkExtractor
-      def initialize(url_filter:, verbose: false)
+      def initialize(url_filter:)
         @url_filter = url_filter
-        @verbose = verbose
       end
 
       def extract(current_url, html)
@@ -16,7 +15,7 @@ module Nous
 
       private
 
-      attr_reader :url_filter, :verbose
+      attr_reader :url_filter
 
       def anchors(html)
         Nokogiri::HTML(html).css("a[href]").map { |node| node["href"] }
@@ -33,7 +32,7 @@ module Nous
 
         canonical
       rescue URI::InvalidURIError => e
-        warn("[nous] malformed href #{href.inspect}: #{e.message}") if verbose
+        warn("[nous] malformed href #{href.inspect}: #{e.message}") if Nous.configuration.verbose?
         nil
       end
     end

data/lib/nous/crawler/page_fetcher.rb

@@ -5,14 +5,12 @@ module Nous
     class PageFetcher
       HTML_CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
 
-      def initialize(client:, timeout:, verbose: false)
+      def initialize(client:)
         @client = client
-        @timeout = timeout
-        @verbose = verbose
       end
 
       def fetch(url)
-        Async::Task.current.with_timeout(timeout) do
+        Async::Task.current.with_timeout(config.timeout) do
           response = client.get(url, {})
           return skip(url, "status #{response.status}") unless response.status == 200
           return skip(url, "non-html content") unless html?(response)
@@ -22,14 +20,18 @@ module Nous
           response&.close
         end
       rescue Async::TimeoutError
-        skip(url, "timeout after #{timeout}s")
+        skip(url, "timeout after #{config.timeout}s")
       rescue IOError, SocketError, Errno::ECONNREFUSED => e
         skip(url, e.message)
       end
 
       private
 
-      attr_reader :client, :timeout, :verbose
+      attr_reader :client
+
+      def config
+        Nous.configuration
+      end
 
       def html?(response)
         content_type = response.headers["content-type"].to_s
@@ -37,7 +39,7 @@ module Nous
       end
 
       def skip(url, reason)
-        warn("[nous] skip #{url}: #{reason}") if verbose
+        warn("[nous] skip #{url}: #{reason}") if config.verbose?
         nil
       end
     end

data/lib/nous/crawler.rb

@@ -9,11 +9,13 @@ module Nous
   class Crawler < Command
     class Error < Command::Error; end
 
-    def initialize(seed_url:, **options)
-      @config = Configuration.new(seed_url:, **options)
+    def initialize(seed_url:)
+      @seed_url = seed_url
     end
 
     def call
+      suppress_async_warnings unless config.verbose?
+
       pages = []
       queue = [url_filter.canonicalize(config.seed)]
       seen = Set.new(queue)
@@ -32,7 +34,11 @@ module Nous
 
     private
 
-    attr_reader :config
+    attr_reader :seed_url
+
+    def config
+      Nous.configuration
+    end
 
     def crawl(queue:, seen:, pages:, client:)
       while queue.any? && pages.length < config.limit
@@ -70,11 +76,16 @@ module Nous
     end
 
     def link_extractor
-      @link_extractor ||= LinkExtractor.new(url_filter:, verbose: config.verbose)
+      @link_extractor ||= LinkExtractor.new(url_filter:)
     end
 
     def page_fetcher(client)
-      PageFetcher.new(client:, timeout: config.timeout, verbose: config.verbose)
+      PageFetcher.new(client:)
+    end
+
+    def suppress_async_warnings
+      require "console"
+      Console.logger.level = :error
     end
   end
 end

data/lib/nous/extraction_runner.rb

@@ -1,31 +1,29 @@
 # frozen_string_literal: true
 
 module Nous
-  class ExtractionRunner
-    def initialize(raw_pages:, extractor:, concurrency: 3, verbose: false)
+  class ExtractionRunner < Command
+    class Error < Command::Error; end
+
+    def initialize(raw_pages:, extractor:)
       @raw_pages = raw_pages
       @extractor = extractor
-      @concurrency = Integer(concurrency).clamp(1, 20)
-      @verbose = verbose
     end
 
     def call
-      raw_pages.each_slice(concurrency).each_with_object([]) do |batch, pages|
-        threads = batch.map { |raw| Thread.new { build_thread(raw).call } }
+      pages = raw_pages.each_slice(Nous.configuration.concurrency).each_with_object([]) do |batch, results|
+        threads = batch.map { |raw| Thread.new { ExtractionThread.new(extractor:, raw_page: raw).call } }
 
         threads.each do |thread|
           result = thread.value
-          pages << result if result
+          results << result if result
         end
       end
+
+      success(payload: pages)
     end
 
     private
 
-    attr_reader :raw_pages, :extractor, :concurrency, :verbose
-
-    def build_thread(raw_page)
-      ExtractionThread.new(extractor:, raw_page:, verbose:)
-    end
+    attr_reader :raw_pages, :extractor
   end
 end

data/lib/nous/extraction_thread.rb

@@ -2,10 +2,9 @@
 
 module Nous
   class ExtractionThread
-    def initialize(extractor:, raw_page:, verbose: false)
+    def initialize(extractor:, raw_page:)
       @extractor = extractor
       @raw_page = raw_page
-      @verbose = verbose
     end
 
     def call
@@ -18,12 +17,12 @@ module Nous
         content: extracted[:content]
       )
     rescue Nous::Error => e
-      warn("[nous] extract skip #{raw_page[:url]}: #{e.message}") if verbose
+      warn("[nous] extract skip #{raw_page[:url]}: #{e.message}") if Nous.configuration.verbose?
       nil
     end
 
     private
 
-    attr_reader :extractor, :raw_page, :verbose
+    attr_reader :extractor, :raw_page
   end
 end

data/lib/nous/fetcher.rb

@@ -4,10 +4,9 @@ module Nous
   class Fetcher < Command
     class Error < Command::Error; end
 
-    def initialize(seed_url:, extractor: Extractor::Default.new, **crawler_options)
+    def initialize(seed_url:, extractor: Extractor::Default.new)
       @seed_url = seed_url
       @extractor = extractor
-      @crawler_options = crawler_options
     end
 
     def call
@@ -18,22 +17,20 @@ module Nous
 
     private
 
-    attr_reader :seed_url, :extractor, :crawler_options
+    attr_reader :seed_url, :extractor
 
     def crawl
-      result = Crawler.call(seed_url:, **crawler_options)
+      result = Crawler.call(seed_url:)
       raise Error, result.error.message if result.failure?
 
       result.payload
     end
 
     def extract(raw_pages)
-      ExtractionRunner.new(
-        raw_pages:,
-        extractor:,
-        concurrency: crawler_options.fetch(:concurrency, 3),
-        verbose: crawler_options.fetch(:verbose, false)
-      ).call
+      result = ExtractionRunner.call(raw_pages:, extractor:)
+      raise Error, result.error.message if result.failure?
+
+      result.payload
     end
   end
 end

data/lib/nous/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nous
-  VERSION = "0.1.0"
+  VERSION = '0.2.0'
 end

data/lib/nous.rb

@@ -6,19 +6,31 @@ loader = Zeitwerk::Loader.for_gem
 loader.setup
 
 module Nous
-  module_function
+  class << self
+    attr_reader :configuration
 
-  def fetch(seed_url, **options)
-    result = Fetcher.call(seed_url:, **options)
-    raise result.error if result.failure?
+    def configure(seed_url:, **options)
+      @configuration = Configuration.new(seed_url:, **options)
+    end
 
-    result.payload
-  end
+    def reset_configuration!
+      @configuration = nil
+    end
+
+    def fetch(seed_url, extractor: Extractor::Default.new, **options)
+      configure(seed_url:, **options)
+
+      result = Fetcher.call(seed_url:, extractor:)
+      raise result.error if result.failure?
+
+      result.payload
+    end
 
-  def serialize(pages, format: :text)
-    result = Serializer.call(pages:, format:)
-    raise result.error if result.failure?
+    def serialize(pages, format: :text)
+      result = Serializer.call(pages:, format:)
+      raise result.error if result.failure?
 
-    result.payload
+      result.payload
+    end
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: nous
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Dan Frenette
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-21 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: async
@@ -188,6 +187,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".rspec"
+- ".ruby-version"
 - ".standard.yml"
 - CHANGELOG.md
 - LICENSE.txt
@@ -197,9 +197,9 @@ files:
 - lib/nous.rb
 - lib/nous/cli.rb
 - lib/nous/command.rb
+- lib/nous/configuration.rb
 - lib/nous/converter.rb
 - lib/nous/crawler.rb
-- lib/nous/crawler/configuration.rb
 - lib/nous/crawler/link_extractor.rb
 - lib/nous/crawler/page_fetcher.rb
 - lib/nous/crawler/url_filter.rb
@@ -222,7 +222,6 @@ metadata:
   homepage_uri: https://github.com/danfrenette/nous
   source_code_uri: https://github.com/danfrenette/nous
   changelog_uri: https://github.com/danfrenette/nous/blob/main/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -237,8 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.9
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Crawl websites and extract readable markdown for LLM workflows
 test_files: []

data/lib/nous/crawler/configuration.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require "uri"
-
-module Nous
-  class Crawler < Command
-    class Error < Command::Error; end
-
-    class Configuration
-      attr_reader :seed, :concurrency, :match, :limit, :timeout, :verbose, :keep_query
-
-      DEFAULT_CONCURRENCY = 3
-      DEFAULT_LIMIT = 100
-      DEFAULT_TIMEOUT = 15
-
-      def initialize(seed_url:, concurrency: DEFAULT_CONCURRENCY, match: [], limit: DEFAULT_LIMIT,
-        timeout: DEFAULT_TIMEOUT, verbose: false, keep_query: false)
-        @seed = parse_seed!(seed_url)
-        @concurrency = Integer(concurrency).clamp(1, 20)
-        @match = Array(match)
-        @limit = Integer(limit).clamp(1, 10_000)
-        @timeout = Integer(timeout)
-        @verbose = verbose
-        @keep_query = keep_query
-      end
-
-      private
-
-      def parse_seed!(url)
-        uri = URI.parse(url)
-        raise Error, "seed URL must be http or https" unless uri.is_a?(URI::HTTP)
-
-        uri
-      rescue URI::InvalidURIError => e
-        raise Error, "invalid seed URL: #{e.message}"
-      end
-    end
-  end
-end