nous 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e781bd7a823a8022f4a6e5a4346c183aadde727a59306103be9ffa937b7029dc
+  data.tar.gz: 80bcb682c861204b589c3a548088d5e5806d23281d509e2556e27f9fa8ef3960
+SHA512:
+  metadata.gz: 4cd9a3a161b7203689063820d9e5bc5fda4c04be288262ac5bdddeaae46bc283bd2a85a4029579b28329c5f1bcc6d324d3042ce4bde9a53763ab4d19830bfa28
+  data.tar.gz: 3b0a4e96b51060064f3494b7227aa43ffa0e487a9a01f36ce610be7c852c5a86570665bbd92be07a622fa3b25efd2eef1c772f1db56161d63c062d770a973e26

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/standardrb/standard
+ruby_version: 3.1

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2026-02-21
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2026 Dan Frenette
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+# Nous
+
+Crawl websites and extract readable Markdown, optimized for LLM consumption.
+
+Nous fetches same-host pages starting from a seed URL, extracts readable content, and outputs clean Markdown as XML-tagged text or JSON. It supports concurrent crawling, glob-based URL filtering, and two extraction backends: a local parser (ruby-readability) and the Jina Reader API for JS-rendered sites.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "nous"
+```
+
+Or install directly:
+
+```bash
+gem install nous
+```
+
+## CLI Usage
+
+```bash
+# Crawl a site and print extracted content to stdout
+nous https://example.com
+
+# Output as JSON
+nous https://example.com -f json
+
+# Write to a file
+nous https://example.com -o site.md
+
+# Limit pages and increase concurrency
+nous https://example.com -l 20 -c 5
+
+# Only crawl pages matching a glob pattern
+nous https://example.com -m "/blog/*"
+
+# Scope extraction to a CSS selector
+nous https://example.com -s "article.post"
+
+# Use Jina Reader API for JS-rendered sites (Next.js, SPAs)
+nous https://example.com --jina
+
+# Verbose logging
+nous https://example.com -v
+```
+
+### Options
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `-o`, `--output PATH` | Write output to file | stdout |
+| `-f`, `--format FORMAT` | Output format: `text` or `json` | `text` |
+| `-c`, `--concurrency N` | Concurrent requests | `3` |
+| `-m`, `--match PATTERN` | Glob filter for URLs (repeatable) | none |
+| `-s`, `--selector SELECTOR` | CSS selector to scope extraction | none |
+| `-l`, `--limit N` | Maximum pages to fetch | `100` |
+| `--timeout N` | Per-request timeout in seconds | `15` |
+| `--jina` | Use Jina Reader API for extraction | off |
+| `-v`, `--verbose` | Verbose logging to stderr | off |
+
+## Ruby API
+
+```ruby
+require "nous"
+
+# Fetch pages with the default extractor
+pages = Nous.fetch("https://example.com", limit: 10, concurrency: 3)
+
+# Each page is a Nous::Page with title, url, pathname, content
+pages.each do |page|
+  puts "#{page.title} (#{page.url})"
+  puts page.content
+end
+
+# Serialize to XML-tagged text
+text = Nous.serialize(pages, format: :text)
+
+# Serialize to JSON
+json = Nous.serialize(pages, format: :json)
+
+# Use the Jina extractor for JS-heavy sites
+pages = Nous.fetch("https://spa-site.com",
+  extractor: Nous::Extractor::Jina.new,
+  limit: 5
+)
+```
+
+## Extraction Backends
+
+### Default (ruby-readability)
+
+Parses static HTML using [ruby-readability](https://github.com/cantino/ruby-readability), strips noisy elements (nav, footer, script, header), and converts to Markdown via [reverse_markdown](https://github.com/xijo/reverse_markdown). Fast and requires no external services, but cannot extract content from JS-rendered pages.
+
+### Jina Reader API
+
+Uses the [Jina Reader API](https://jina.ai/reader/) which renders pages with headless Chrome. Handles Next.js App Router, React Server Components, SPAs, and other JS-heavy sites. Free tier allows 20 requests/minute without a key, or 500 RPM with a `JINA_API_KEY` environment variable.
+
+## Output Formats
+
+### Text (default)
+
+XML-tagged output designed for LLM context windows:
+
+```xml
+<page>
+<title>Page Title</title>
+<url>https://example.com/page</url>
+<content>
+# Heading
+
+Extracted markdown content...
+</content>
+</page>
+```
+
+### JSON
+
+```json
+[
+  {
+    "title": "Page Title",
+    "url": "https://example.com/page",
+    "pathname": "/page",
+    "content": "# Heading\n\nExtracted markdown content..."
+  }
+]
+```
+
+## Development
+
+```bash
+bin/setup            # Install dependencies
+bundle exec rspec    # Run tests
+bundle exec standardrb  # Lint
+```
+
+## License
+
+MIT License. See [LICENSE.txt](LICENSE.txt).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "standard/rake"
+
+task default: %i[spec standard]

data/exe/nous

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "nous"
+
+Nous::Cli.new(ARGV).run

data/lib/nous/cli.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Nous
+  class Cli
+    class Error < Nous::Error; end
+
+    def initialize(argv)
+      @argv = argv
+      @options = {format: :text, concurrency: 3, limit: 100, timeout: 15}
+    end
+
+    def run
+      parse_options!
+      validate!
+
+      pages = Nous.fetch(seed_url, **fetch_options)
+      output = Nous.serialize(pages, format: options[:format])
+      write_output(output)
+    rescue Nous::Error => e
+      warn("nous: #{e.message}")
+      exit 1
+    end
+
+    private
+
+    attr_reader :argv, :options
+
+    def seed_url
+      argv.first
+    end
+
+    def fetch_options
+      opts = options.slice(:concurrency, :match, :limit, :timeout, :verbose)
+      opts[:extractor] = extractor
+      opts
+    end
+
+    def extractor
+      return Extractor::Jina.new if options[:jina]
+
+      Extractor::Default.new(selector: options[:selector])
+    end
+
+    def validate!
+      raise Error, "no URL provided. Usage: nous <url> [options]" unless seed_url
+    end
+
+    def write_output(output)
+      if options[:output]
+        File.write(options[:output], output)
+      else
+        $stdout.puts(output)
+      end
+    end
+
+    def parse_options!
+      parser.parse!(argv)
+    rescue OptionParser::InvalidOption => e
+      raise Error, e.message
+    end
+
+    def parser
+      OptionParser.new do |opts|
+        opts.banner = "Usage: nous <url> [options]"
+
+        opts.on("-o", "--output PATH", "Write output to file (default: stdout)") { |v| options[:output] = v }
+        opts.on("-f", "--format FORMAT", "Output format: text or json (default: text)") do |v|
+          options[:format] = v.to_sym
+        end
+        opts.on("-c", "--concurrency N", Integer, "Concurrent requests (default: 3)") { |v| options[:concurrency] = v }
+        opts.on("-m", "--match PATTERN", "Only include pages matching glob (repeatable)") do |v|
+          (options[:match] ||= []) << v
+        end
+        opts.on("-s", "--selector SELECTOR", "CSS selector to scope extraction") { |v| options[:selector] = v }
+        opts.on("-l", "--limit N", Integer, "Maximum pages to fetch") { |v| options[:limit] = v }
+        opts.on("--timeout N", Integer, "Per-request timeout in seconds (default: 15)") { |v| options[:timeout] = v }
+        opts.on("--jina", "Use Jina Reader API for extraction (handles JS-rendered sites)") { options[:jina] = true }
+        opts.on("-v", "--verbose", "Verbose logging to stderr") { options[:verbose] = true }
+        opts.on("-h", "--help", "Show help") do
+          $stdout.puts(opts)
+          exit
+        end
+        opts.on("--version", "Show version") do
+          $stdout.puts("nous #{Nous::VERSION}")
+          exit
+        end
+      end
+    end
+  end
+end

data/lib/nous/command.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Nous
+  class Command
+    class Error < Nous::Error; end
+
+    class Result
+      attr_reader :payload, :error, :metadata
+
+      def initialize(success:, payload: nil, error: nil, metadata: {})
+        @success = success
+        @payload = payload
+        @error = error
+        @metadata = metadata
+      end
+
+      def success?
+        @success
+      end
+
+      def failure?
+        !@success
+      end
+    end
+
+    def self.call(...)
+      command = new(...)
+      command.call
+    rescue => e
+      return command.failure(Error.new("unexpected: #{e.message}")) if command
+
+      Result.new(success: false, error: e)
+    end
+
+    def success(payload:, metadata: {})
+      Result.new(success: true, payload:, metadata:)
+    end
+
+    def failure(error, metadata: {})
+      Result.new(success: false, error:, metadata:)
+    end
+  end
+end

data/lib/nous/converter.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "reverse_markdown"
+
+module Nous
+  class Converter < Command
+    class Error < Command::Error; end
+
+    def initialize(html:)
+      @html = html
+    end
+
+    def call
+      markdown = ReverseMarkdown.convert(html, github_flavored: true).strip
+      success(payload: markdown)
+    end
+
+    private
+
+    attr_reader :html
+  end
+end

data/lib/nous/crawler/configuration.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Nous
+  class Crawler < Command
+    class Error < Command::Error; end
+
+    class Configuration
+      attr_reader :seed, :concurrency, :match, :limit, :timeout, :verbose, :keep_query
+
+      DEFAULT_CONCURRENCY = 3
+      DEFAULT_LIMIT = 100
+      DEFAULT_TIMEOUT = 15
+
+      def initialize(seed_url:, concurrency: DEFAULT_CONCURRENCY, match: [], limit: DEFAULT_LIMIT,
+        timeout: DEFAULT_TIMEOUT, verbose: false, keep_query: false)
+        @seed = parse_seed!(seed_url)
+        @concurrency = Integer(concurrency).clamp(1, 20)
+        @match = Array(match)
+        @limit = Integer(limit).clamp(1, 10_000)
+        @timeout = Integer(timeout)
+        @verbose = verbose
+        @keep_query = keep_query
+      end
+
+      private
+
+      def parse_seed!(url)
+        uri = URI.parse(url)
+        raise Error, "seed URL must be http or https" unless uri.is_a?(URI::HTTP)
+
+        uri
+      rescue URI::InvalidURIError => e
+        raise Error, "invalid seed URL: #{e.message}"
+      end
+    end
+  end
+end

data/lib/nous/crawler/link_extractor.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Nous
+  class Crawler < Command
+    class LinkExtractor
+      def initialize(url_filter:, verbose: false)
+        @url_filter = url_filter
+        @verbose = verbose
+      end
+
+      def extract(current_url, html)
+        base_uri = URI.parse(current_url)
+
+        anchors(html).filter_map { |href| resolve(base_uri, href) }.uniq
+      end
+
+      private
+
+      attr_reader :url_filter, :verbose
+
+      def anchors(html)
+        Nokogiri::HTML(html).css("a[href]").map { |node| node["href"] }
+      end
+
+      def resolve(base_uri, href)
+        return unless url_filter.allowed?(href)
+
+        uri = URI.join(base_uri, href)
+        return unless url_filter.same_host?(uri)
+
+        canonical = url_filter.canonicalize(uri)
+        return unless url_filter.matches_path?(URI.parse(canonical).path)
+
+        canonical
+      rescue URI::InvalidURIError => e
+        warn("[nous] malformed href #{href.inspect}: #{e.message}") if verbose
+        nil
+      end
+    end
+  end
+end

data/lib/nous/crawler/page_fetcher.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Nous
+  class Crawler < Command
+    class PageFetcher
+      HTML_CONTENT_TYPES = %w[text/html application/xhtml+xml].freeze
+
+      def initialize(client:, timeout:, verbose: false)
+        @client = client
+        @timeout = timeout
+        @verbose = verbose
+      end
+
+      def fetch(url)
+        Async::Task.current.with_timeout(timeout) do
+          response = client.get(url, {})
+          return skip(url, "status #{response.status}") unless response.status == 200
+          return skip(url, "non-html content") unless html?(response)
+
+          {url:, pathname: URI.parse(url).path, html: response.read}
+        ensure
+          response&.close
+        end
+      rescue Async::TimeoutError
+        skip(url, "timeout after #{timeout}s")
+      rescue IOError, SocketError, Errno::ECONNREFUSED => e
+        skip(url, e.message)
+      end
+
+      private
+
+      attr_reader :client, :timeout, :verbose
+
+      def html?(response)
+        content_type = response.headers["content-type"].to_s
+        HTML_CONTENT_TYPES.any? { |type| content_type.include?(type) }
+      end
+
+      def skip(url, reason)
+        warn("[nous] skip #{url}: #{reason}") if verbose
+        nil
+      end
+    end
+  end
+end

data/lib/nous/crawler/url_filter.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Nous
+  class Crawler < Command
+    class UrlFilter
+      IGNORED_SCHEMES = %w[mailto: javascript: tel:].freeze
+
+      def initialize(config)
+        @host = config.seed.host
+        @match = config.match
+        @keep_query = config.keep_query
+      end
+
+      def canonicalize(uri)
+        uri = URI.parse(uri.to_s)
+        uri.fragment = nil
+        uri.query = nil unless keep_query
+        uri.path = "/" if uri.path.empty?
+        uri.to_s
+      end
+
+      def allowed?(href)
+        return false if href.strip.empty?
+
+        IGNORED_SCHEMES.none? { |s| href.start_with?(s) }
+      end
+
+      def same_host?(uri)
+        uri.is_a?(URI::HTTP) && uri.host == host
+      end
+
+      def matches_path?(path)
+        return true if match.empty?
+
+        match.any? { |pattern| File.fnmatch(pattern, path, File::FNM_PATHNAME | File::FNM_EXTGLOB) }
+      end
+
+      private
+
+      attr_reader :host, :match, :keep_query
+    end
+  end
+end

data/lib/nous/crawler.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "async"
+require "async/http/internet"
+require "nokogiri"
+require "uri"
+
+module Nous
+  class Crawler < Command
+    class Error < Command::Error; end
+
+    def initialize(seed_url:, **options)
+      @config = Configuration.new(seed_url:, **options)
+    end
+
+    def call
+      pages = []
+      queue = [url_filter.canonicalize(config.seed)]
+      seen = Set.new(queue)
+
+      Async do
+        client = Async::HTTP::Internet.new
+        begin
+          crawl(queue:, seen:, pages:, client:)
+        ensure
+          client.close
+        end
+      end.wait
+
+      success(payload: pages)
+    end
+
+    private
+
+    attr_reader :config
+
+    def crawl(queue:, seen:, pages:, client:)
+      while queue.any? && pages.length < config.limit
+        batch = queue.shift(config.concurrency)
+        fetch_batch(batch, client).each do |page|
+          next unless page
+
+          pages << page
+          break if pages.length >= config.limit
+
+          link_extractor.extract(page[:url], page[:html]).each do |url|
+            next if seen.include?(url)
+
+            seen << url
+            queue << url
+          end
+        end
+      end
+    end
+
+    def fetch_batch(urls, client)
+      tasks = []
+
+      Async do |task|
+        urls.each do |url|
+          tasks << task.async { page_fetcher(client).fetch(url) }
+        end
+      end.wait
+
+      tasks.map(&:wait)
+    end
+
+    def url_filter
+      @url_filter ||= UrlFilter.new(config)
+    end
+
+    def link_extractor
+      @link_extractor ||= LinkExtractor.new(url_filter:, verbose: config.verbose)
+    end
+
+    def page_fetcher(client)
+      PageFetcher.new(client:, timeout: config.timeout, verbose: config.verbose)
+    end
+  end
+end

data/lib/nous/error.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nous
+  class Error < StandardError; end
+end

data/lib/nous/extraction_runner.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Nous
+  class ExtractionRunner
+    def initialize(raw_pages:, extractor:, concurrency: 3, verbose: false)
+      @raw_pages = raw_pages
+      @extractor = extractor
+      @concurrency = Integer(concurrency).clamp(1, 20)
+      @verbose = verbose
+    end
+
+    def call
+      raw_pages.each_slice(concurrency).each_with_object([]) do |batch, pages|
+        threads = batch.map { |raw| Thread.new { build_thread(raw).call } }
+
+        threads.each do |thread|
+          result = thread.value
+          pages << result if result
+        end
+      end
+    end
+
+    private
+
+    attr_reader :raw_pages, :extractor, :concurrency, :verbose
+
+    def build_thread(raw_page)
+      ExtractionThread.new(extractor:, raw_page:, verbose:)
+    end
+  end
+end

data/lib/nous/extraction_thread.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Nous
+  class ExtractionThread
+    def initialize(extractor:, raw_page:, verbose: false)
+      @extractor = extractor
+      @raw_page = raw_page
+      @verbose = verbose
+    end
+
+    def call
+      extracted = extractor.extract(raw_page)
+
+      Page.new(
+        title: extracted[:title],
+        url: raw_page[:url],
+        pathname: raw_page[:pathname],
+        content: extracted[:content]
+      )
+    rescue Nous::Error => e
+      warn("[nous] extract skip #{raw_page[:url]}: #{e.message}") if verbose
+      nil
+    end
+
+    private
+
+    attr_reader :extractor, :raw_page, :verbose
+  end
+end

data/lib/nous/extractor/default.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Nous
+  class Extractor
+    class Default
+      def initialize(selector: nil)
+        @selector = selector
+      end
+
+      def extract(page)
+        extracted = extract_content(page[:html])
+        markdown = convert_to_markdown(extracted[:content])
+
+        {title: extracted[:title], content: markdown}
+      end
+
+      private
+
+      attr_reader :selector
+
+      def extract_content(html)
+        result = Extractor.call(html:, selector:)
+        raise result.error if result.failure?
+
+        result.payload
+      end
+
+      def convert_to_markdown(html)
+        result = Converter.call(html:)
+        raise result.error if result.failure?
+
+        result.payload
+      end
+    end
+  end
+end

data/lib/nous/extractor/jina/client.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "faraday/retry"
+require "json"
+
+module Nous
+  class Extractor
+    class Jina
+      class Client
+        class Error < Nous::Error; end
+
+        BASE_URL = "https://r.jina.ai"
+        RETRYABLE_STATUSES = [429, 500, 502, 503, 504].freeze
+        MAX_RETRIES = 3
+
+        def initialize(api_key: nil, timeout: 30, retry_interval: 1)
+          @connection = build_connection(api_key:, timeout:, retry_interval:)
+        end
+
+        def get(url)
+          response = connection.get("/#{url}")
+          parse(response.body)
+        rescue Faraday::Error => e
+          raise Error, e.message
+        end
+
+        private
+
+        attr_reader :connection
+
+        def build_connection(api_key:, timeout:, retry_interval:)
+          Faraday.new(url: BASE_URL) do |f|
+            f.response :raise_error
+
+            f.request :retry,
+              max: MAX_RETRIES,
+              interval: retry_interval,
+              backoff_factor: 2,
+              retry_statuses: RETRYABLE_STATUSES
+
+            f.headers["Accept"] = "application/json"
+            f.headers["X-No-Cache"] = "true"
+            f.headers["Authorization"] = "Bearer #{api_key}" if api_key
+
+            f.options.timeout = timeout
+            f.options.open_timeout = timeout
+          end
+        end
+
+        def parse(body)
+          JSON.parse(body)
+        rescue JSON::ParserError => e
+          raise Error, "invalid JSON: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/nous/extractor/jina.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Nous
+  class Extractor
+    class Jina
+      class Error < Nous::Error; end
+
+      def initialize(api_key: nil, timeout: 30, **client_options)
+        @client = Client.new(api_key: api_key || ENV["JINA_API_KEY"], timeout:, **client_options)
+      end
+
+      def extract(page)
+        body = client.get(page[:url])
+
+        {title: body.dig("data", "title") || "", content: body.dig("data", "content") || ""}
+      rescue Client::Error => e
+        raise Error, e.message
+      end
+
+      private
+
+      attr_reader :client
+    end
+  end
+end

data/lib/nous/extractor.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "readability"
+
+module Nous
+  class Extractor < Command
+    class Error < Command::Error; end
+
+    NOISY_TAGS = %w[script style link nav header footer img video svg].freeze
+
+    def initialize(html:, selector: nil)
+      @html = html
+      @selector = selector
+    end
+
+    def call
+      doc = Nokogiri::HTML(html)
+      doc = scope_to_selector(doc) if selector
+      strip_noisy_tags(doc)
+
+      readable = Readability::Document.new(doc.to_html)
+      text = Nokogiri::HTML(readable.content).text.strip
+
+      return failure(Error.new("readability returned no content")) if text.empty?
+
+      success(payload: {title: readable.title, content: readable.content})
+    end
+
+    private
+
+    attr_reader :html, :selector
+
+    def scope_to_selector(doc)
+      scoped = doc.at_css(selector)
+      return doc unless scoped
+
+      fragment = Nokogiri::HTML::Document.new
+      fragment.root = scoped
+      fragment
+    end
+
+    def strip_noisy_tags(doc)
+      NOISY_TAGS.each { |tag| doc.css(tag).each(&:remove) }
+    end
+  end
+end

data/lib/nous/fetcher.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Nous
+  class Fetcher < Command
+    class Error < Command::Error; end
+
+    def initialize(seed_url:, extractor: Extractor::Default.new, **crawler_options)
+      @seed_url = seed_url
+      @extractor = extractor
+      @crawler_options = crawler_options
+    end
+
+    def call
+      raw_pages = crawl
+      pages = extract(raw_pages)
+      success(payload: pages)
+    end
+
+    private
+
+    attr_reader :seed_url, :extractor, :crawler_options
+
+    def crawl
+      result = Crawler.call(seed_url:, **crawler_options)
+      raise Error, result.error.message if result.failure?
+
+      result.payload
+    end
+
+    def extract(raw_pages)
+      ExtractionRunner.new(
+        raw_pages:,
+        extractor:,
+        concurrency: crawler_options.fetch(:concurrency, 3),
+        verbose: crawler_options.fetch(:verbose, false)
+      ).call
+    end
+  end
+end

data/lib/nous/page.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nous
+  Page = Data.define(:title, :url, :pathname, :content)
+end

data/lib/nous/serializer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Nous
+  class Serializer < Command
+    class Error < Command::Error; end
+
+    FORMATS = %i[text json].freeze
+
+    def initialize(pages:, format: :text)
+      @pages = pages
+      @format = format.to_sym
+      validate_format!
+    end
+
+    def call
+      output = (format == :json) ? serialize_json : serialize_text
+      success(payload: output)
+    end
+
+    private
+
+    attr_reader :pages, :format
+
+    def validate_format!
+      raise Error, "unknown format: #{format}. Must be one of: #{FORMATS.join(", ")}" unless FORMATS.include?(format)
+    end
+
+    def serialize_text
+      pages.map { |page| text_page(page) }.join("\n\n")
+    end
+
+    def serialize_json
+      JSON.pretty_generate(pages.map { |page| json_page(page) })
+    end
+
+    def text_page(page)
+      <<~XML
+        <page>
+          <title>#{page.title}</title>
+          <url>#{page.url}</url>
+          <content>
+        #{page.content}
+          </content>
+        </page>
+      XML
+    end
+
+    def json_page(page)
+      {title: page.title, url: page.url, content: page.content}
+    end
+  end
+end

data/lib/nous/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Nous
+  VERSION = "0.1.0"
+end

data/lib/nous.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module Nous
+  module_function
+
+  def fetch(seed_url, **options)
+    result = Fetcher.call(seed_url:, **options)
+    raise result.error if result.failure?
+
+    result.payload
+  end
+
+  def serialize(pages, format: :text)
+    result = Serializer.call(pages:, format:)
+    raise result.error if result.failure?
+
+    result.payload
+  end
+end

data/sig/nous.rbs

@@ -0,0 +1,4 @@
+module Nous
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,244 @@
+--- !ruby/object:Gem::Specification
+name: nous
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dan Frenette
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
+- !ruby/object:Gem::Dependency
+  name: async-http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.88'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.88'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: reverse_markdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-readability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.42'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.42'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.25'
+description: Nous crawls same-host web pages, extracts readable content, and serializes
+  clean Markdown as text or JSON.
+email:
+- dan.r.frenette@gmail.com
+executables:
+- nous
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".standard.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/nous
+- lib/nous.rb
+- lib/nous/cli.rb
+- lib/nous/command.rb
+- lib/nous/converter.rb
+- lib/nous/crawler.rb
+- lib/nous/crawler/configuration.rb
+- lib/nous/crawler/link_extractor.rb
+- lib/nous/crawler/page_fetcher.rb
+- lib/nous/crawler/url_filter.rb
+- lib/nous/error.rb
+- lib/nous/extraction_runner.rb
+- lib/nous/extraction_thread.rb
+- lib/nous/extractor.rb
+- lib/nous/extractor/default.rb
+- lib/nous/extractor/jina.rb
+- lib/nous/extractor/jina/client.rb
+- lib/nous/fetcher.rb
+- lib/nous/page.rb
+- lib/nous/serializer.rb
+- lib/nous/version.rb
+- sig/nous.rbs
+homepage: https://github.com/danfrenette/nous
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/danfrenette/nous
+  source_code_uri: https://github.com/danfrenette/nous
+  changelog_uri: https://github.com/danfrenette/nous/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.9
+signing_key:
+specification_version: 4
+summary: Crawl websites and extract readable markdown for LLM workflows
+test_files: []