nous 0.3.0 → 0.5.0

data/lib/nous/extract_html.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+
+module Nous
+  class ExtractHtml < Command
+    class Error < StandardError; end
+    class InvalidLimits < ArgumentError; end
+    class InvalidContent < Error; end
+    class NoContent < Error; end
+    class UnsupportedContent < Error; end
+    class InputLimitExceeded < Error; end
+    class OutputLimitExceeded < Error; end
+
+    class UnexpectedFailure < Error
+      attr_reader :cause
+
+      def initialize(cause)
+        @cause = cause
+        super("HTML extraction failed unexpectedly")
+      end
+    end
+
+    def initialize(html:, source_url:, content_type: "text/html", format: :markdown,
+      max_input_bytes: 25 * 1024 * 1024, max_output_bytes: 32 * 1024 * 1024)
+      @html = html
+      @source_url = source_url
+      @content_type = content_type
+      @format = format
+      @max_input_bytes = max_input_bytes
+      @max_output_bytes = max_output_bytes
+    end
+
+    def call
+      verify_extraction_request
+      decode_retained_html
+      extract_readable_content
+      render_content
+      verify_extracted_content
+      report_extraction
+    rescue Error, InvalidLimits => error
+      failure(error)
+    rescue => error
+      failure(UnexpectedFailure.new(error))
+    end
+
+    private
+
+    attr_reader :html, :source_url, :content_type, :format, :max_input_bytes, :max_output_bytes
+
+    def verify_extraction_request
+      unless [max_input_bytes, max_output_bytes].all? { |limit| limit.is_a?(Integer) && limit.positive? }
+        raise InvalidLimits, "Byte limits must be positive integers"
+      end
+      raise InvalidContent, "HTML must be a byte string" unless html.is_a?(String)
+      raise InputLimitExceeded, "HTML exceeds input byte limit" if html.bytesize > max_input_bytes
+      raise UnsupportedContent, "Unsupported output format" unless [:markdown, :text].include?(format)
+      unless content_type.is_a?(String) && ["text/html", "application/xhtml+xml"].include?(content_type.split(";", 2).first.to_s.strip.downcase)
+        raise UnsupportedContent, "Unsupported HTML content type"
+      end
+    end
+
+    def decode_retained_html
+      result = DecodeHtml.new(html: html, content_type: content_type).call
+      raise result.error if result.failure?
+
+      @decoded = result.payload
+      @encoding = result.metadata.fetch(:encoding)
+    rescue DecodeHtml::InvalidContent => error
+      raise InvalidContent, error.message
+    rescue DecodeHtml::UnsupportedContent => error
+      raise UnsupportedContent, error.message
+    end
+
+    def extract_readable_content
+      result = Extractor::Defuddle::Client.new(html: @decoded, url: source_url).call
+      raise result.error if result.failure?
+
+      @readable = result.payload
+      @title = @readable[:title]
+    rescue Extractor::Defuddle::Client::NoContent => error
+      raise NoContent.new("HTML has no readable content"), cause: error
+    end
+
+    def render_content
+      @content = (format == :text) ? render_plain_text : @readable[:markdown]
+    end
+
+    def render_plain_text
+      Nokogiri::HTML.fragment(@readable[:content]).tap do |fragment|
+        fragment.css("p, div, h1, h2, h3, h4, h5, h6, li, tr, br").each do |node|
+          node.add_next_sibling(Nokogiri::XML::Text.new(" ", fragment.document))
+        end
+      end.text.gsub(/\s+/, " ").strip
+    end
+
+    def verify_extracted_content
+      raise NoContent, "HTML has no readable content" if @content.strip.empty?
+      if @content.bytesize + @title.bytesize > max_output_bytes
+        raise OutputLimitExceeded, "Extracted content exceeds output byte limit"
+      end
+    end
+
+    def report_extraction
+      success(payload: ExtractedContent.new(title: @title, content: @content),
+        metadata: {source_url: source_url, content_type: content_type, encoding: @encoding, format: format, limitations: ["static_html_only", "authentication_not_detected"]})
+    end
+  end
+end

data/lib/nous/extractor/default/client.rb

@@ -8,7 +8,7 @@ module Nous
       class Client < Command
         class ExtractionError < StandardError; end
 
-        NOISY_TAGS = %w[script style link nav header footer img video svg].freeze
+        NOISY_TAGS = %w[script style nav footer].freeze
 
         def initialize(html:, selector: nil)
           @html = html
@@ -16,34 +16,52 @@ module Nous
         end
 
         def call
-          doc = Nokogiri::HTML(html)
-          doc = scope_to_selector(doc) if selector
-          strip_noisy_tags(doc)
+          readable = ::Readability::Document.new(prepared_html)
 
-          readable = ::Readability::Document.new(doc.to_html)
           text = Nokogiri::HTML(readable.content).text.strip
-
           return failure(ExtractionError.new("readability returned no content")) if text.empty?
 
-          success(payload: {title: readable.title, content: readable.content})
+          title = resolve_title(readable)
+          success(payload: {title: title, content: readable.content})
         end
 
         private
 
         attr_reader :html, :selector
 
-        def scope_to_selector(doc)
+        def prepared_html
+          doc = Nokogiri::HTML(html)
+          original_title(doc)
+          doc = scope(doc, selector) if selector
+          strip_tags(doc)
+          doc.to_html
+        end
+
+        def original_title(doc)
+          @original_title ||= doc.at_css("title")&.text.to_s.strip
+        end
+
+        def scope(doc, selector)
           scoped = doc.at_css(selector)
           return doc unless scoped
 
-          fragment = Nokogiri::HTML::Document.new
-          fragment.root = scoped
-          fragment
+          Nokogiri::HTML.fragment(scoped.to_html)
         end
 
-        def strip_noisy_tags(doc)
+        def strip_tags(doc)
           NOISY_TAGS.each { |tag| doc.css(tag).each(&:remove) }
         end
+
+        def resolve_title(readable)
+          title = readable.title.to_s.strip
+          title = @original_title if title.empty?
+          title = title_from_content(readable.content) if title.empty?
+          title
+        end
+
+        def title_from_content(content)
+          Nokogiri::HTML(content).at_css("h1")&.text.to_s.strip
+        end
       end
     end
   end

data/lib/nous/extractor/default.rb

@@ -2,15 +2,13 @@
 
 module Nous
   module Extractor
-    class ExtractionError < StandardError; end
-
     class Default < Command
       def initialize(selector: nil)
         @selector = selector
       end
 
-      def extract(raw_page)
-        extracted = extract_content(raw_page.html)
+      def extract(record)
+        extracted = extract_content(record.html)
         markdown = convert_to_markdown(extracted[:content])
 
         success(payload: ExtractedContent.new(title: extracted[:title], content: markdown))

data/lib/nous/extractor/defuddle/client.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Nous
+  module Extractor
+    class Defuddle
+      class Client < Command
+        class NoContent < ExtractionError; end
+        ProcessError = RunProcess::Error
+
+        def initialize(html:, url:, selector: nil)
+          @html = html
+          @url = url
+          @selector = selector
+        end
+
+        def call
+          run_defuddle
+          validate_response
+          report_content
+        rescue NoContent, ProcessError => error
+          failure(error)
+        rescue JSON::JSONError, EncodingError
+          failure(ProcessError.new("Invalid Defuddle process response"))
+        end
+
+        private
+
+        def run_defuddle
+          request = JSON.generate(html: @html, url: @url, selector: @selector)
+          result = RunProcess.call(request: request)
+          raise result.error if result.failure?
+
+          @response = JSON.parse(result.payload, symbolize_names: true)
+        end
+
+        def validate_response
+          unless @response.is_a?(Hash) && [:title, :content, :markdown].all? { |key| valid_text?(@response[key]) }
+            raise ProcessError, "Invalid Defuddle response fields"
+          end
+          if @response[:markdown].strip.empty?
+            raise NoContent, "HTML has no readable content"
+          end
+        end
+
+        def valid_text?(value)
+          value.is_a?(String) && value.encoding == Encoding::UTF_8 && value.valid_encoding? && !value.include?("\0")
+        end
+
+        def report_content
+          success(payload: @response)
+        end
+      end
+    end
+  end
+end

data/lib/nous/extractor/defuddle/run_process.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "open3"
+require "timeout"
+
+module Nous
+  module Extractor
+    class Defuddle
+      class RunProcess < Command
+        class Error < ExtractionError; end
+
+        BRIDGE = File.expand_path("../../../../vendor/defuddle.mjs", __dir__)
+        OUTPUT_LIMIT = 64 * 1024 * 1024
+        TIMEOUT = 15
+
+        def initialize(request:)
+          @request = request
+        end
+
+        def call
+          execute_bridge
+          report_output
+        rescue Error => error
+          failure(error)
+        rescue Errno::ENOENT
+          failure(Error.new("Defuddle requires Node.js 22 or newer on PATH"))
+        rescue Timeout::Error
+          failure(Error.new("Defuddle extraction timed out"))
+        rescue IOError, SystemCallError
+          failure(Error.new("Invalid Defuddle process response"))
+        end
+
+        private
+
+        def execute_bridge
+          Open3.popen3("node", BRIDGE, pgroup: true) do |stdin, stdout, stderr, child|
+            workers = {}
+            begin
+              Timeout.timeout(TIMEOUT) do
+                @output = exchange_content(stdin, stdout, stderr, workers)
+                raise Error, "Defuddle extraction failed" unless child.value.success?
+              end
+            ensure
+              terminate(child.pid)
+              [stdin, stdout, stderr].each { |io| io.close unless io.closed? }
+              workers.each_value { |worker| finish_worker(worker) }
+            end
+          end
+        end
+
+        def exchange_content(stdin, stdout, stderr, workers)
+          workers[:input] = start_worker {
+            stdin.write(@request)
+            stdin.close
+          }
+          workers[:output] = start_worker { read_bounded(stdout, OUTPUT_LIMIT) }
+          workers[:diagnostics] = start_worker { read_bounded(stderr, 64 * 1024) }
+          collect_output(workers)
+        end
+
+        def collect_output(workers)
+          output = workers.fetch(:output).value
+          workers.fetch(:input).value
+          workers.fetch(:diagnostics).value
+          output
+        end
+
+        def start_worker(&operation)
+          Thread.new do
+            Thread.current.report_on_exception = false
+            operation.call
+          end
+        end
+
+        def finish_worker(worker)
+          worker.join
+        rescue
+          # Keep the original failure while joining every owned worker.
+          nil
+        end
+
+        def terminate(pid)
+          Process.kill("KILL", -pid)
+        rescue Errno::ESRCH
+          nil
+        end
+
+        def read_bounded(io, limit)
+          output = +""
+          loop do
+            output << io.readpartial(16 * 1024)
+            raise Error, "Defuddle process output exceeds byte limit" if output.bytesize > limit
+          end
+        rescue EOFError
+          output
+        end
+
+        def report_output
+          success(payload: @output)
+        end
+      end
+    end
+  end
+end

data/lib/nous/extractor/defuddle.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Nous
+  module Extractor
+    class Defuddle < Command
+      def initialize(selector: nil)
+        @selector = selector
+      end
+
+      def extract(record)
+        decoded = DecodeHtml.call(html: record.html, content_type: record.content_type)
+        raise decoded.error if decoded.failure?
+
+        result = Client.call(html: decoded.payload, url: record.final_url, selector: @selector)
+        return result if result.failure?
+
+        success(payload: ExtractedContent.new(title: result.payload[:title], content: result.payload[:markdown]))
+      rescue DecodeHtml::Error => error
+        failure(ExtractionError.new(error.message))
+      end
+    end
+  end
+end

data/lib/nous/extractor/extraction_error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Nous
+  module Extractor
+    class ExtractionError < StandardError; end
+  end
+end

data/lib/nous/fetcher/extraction_runner.rb

@@ -5,27 +5,42 @@ module Nous
     class ExtractionRunner < Command
       class ExtractionError < StandardError; end
 
-      def initialize(raw_pages:, extractor:)
-        @raw_pages = raw_pages
+      def initialize(records:, extractor:)
+        @records = records
         @extractor = extractor
       end
 
       def call
-        pages = raw_pages.each_slice(Nous.configuration.concurrency).each_with_object([]) do |batch, results|
-          threads = batch.map { |raw_page| Thread.new { PageExtractor.call(extractor:, raw_page:) } }
+        extract_records
+        report_extraction_outcomes
+      end
+
+      private
 
-          threads.each do |thread|
-            result = thread.value
-            results << result.payload if result.success?
+      attr_reader :records, :extractor, :pages, :failures
+
+      def extract_records
+        @pages = []
+        @failures = []
+        records.each_slice(Nous.configuration.concurrency) do |batch|
+          threads = batch.map { |record| Thread.new { PageExtractor.call(extractor:, record:) } }
+          batch.zip(threads).each do |record, thread|
+            retain_outcome(record, thread.value)
           end
         end
-
-        success(payload: pages)
       end
 
-      private
+      def retain_outcome(record, result)
+        if result.success?
+          pages << result.payload
+        else
+          failures << {requested_url: record.requested_url, error: result.error}
+        end
+      end
 
-      attr_reader :raw_pages, :extractor
+      def report_extraction_outcomes
+        success(payload: pages, metadata: {failures: failures})
+      end
     end
   end
 end

data/lib/nous/fetcher/page_extractor.rb

@@ -3,32 +3,47 @@
 module Nous
   class Fetcher < Command
     class PageExtractor < Command
-      def initialize(extractor:, raw_page:)
+      def initialize(extractor:, record:)
         @extractor = extractor
-        @raw_page = raw_page
+        @record = record
       end
 
       def call
-        result = extractor.extract(raw_page)
+        extract_content
+        report_page
+      rescue => error
+        failure(error)
+      end
+
+      private
+
+      attr_reader :extractor, :record
+
+      def extract_content
+        @extraction = extractor.extract(record)
+      end
 
+      def report_page
+        result = @extraction
         unless result.success?
-          warn("[nous] extract skip #{raw_page.url}: #{result.error.message}") if Nous.configuration.debug?
+          warn("[nous] extract skip #{record.final_url}: #{result.error.message}") if Nous.configuration.debug?
           return failure(result.error)
         end
 
         page = Page.new(
           title: result.payload.title,
-          url: raw_page.url,
-          pathname: raw_page.pathname,
-          content: result.payload.content
+          url: record.final_url,
+          pathname: record.pathname,
+          content: result.payload.content,
+          metadata: {
+            extractor: extractor.class.name,
+            requested_url: record.requested_url,
+            content_type: record.content_type,
+            redirected: record.redirected
+          }
         )
-
         success(payload: page)
       end
-
-      private
-
-      attr_reader :extractor, :raw_page
     end
   end
 end

data/lib/nous/fetcher.rb

@@ -4,34 +4,70 @@ module Nous
   class Fetcher < Command
     class FetchError < StandardError; end
 
-    def initialize(seed_url:, extractor: Extractor::Default.new, http_client: nil)
+    def initialize(seed_url:, extractor: Extractor::Defuddle.new, http_client: nil, details: false)
       @seed_url = seed_url
       @extractor = extractor
       @http_client = http_client
+      @details = details
     end
 
     def call
-      raw_pages = crawl
-      pages = extract(raw_pages)
-      success(payload: pages)
+      resolve_fetch_mode
+      acquire_records
+      verify_acquisition
+      extract_readable_pages
+      report_fetch_outcomes
     end
 
     private
 
-    attr_reader :seed_url, :extractor, :http_client
+    attr_reader :seed_url, :extractor, :http_client, :single_page, :details
+
+    def resolve_fetch_mode
+      @single_page = Nous.configuration.single_page?
+    end
+
+    def acquire_records
+      @records = crawl
+      @successful_records, @failed_records = @records.partition(&:ok)
+    end
+
+    def verify_acquisition
+      if single_page && !details && @successful_records.empty?
+        raise FetchError, @failed_records.first&.error || "fetch failed"
+      end
+    end
+
+    def extract_readable_pages
+      @extraction = ExtractionRunner.call(records: @successful_records, extractor:)
+      raise @extraction.error if @extraction.failure?
+    end
+
+    def report_fetch_outcomes
+      return success(payload: @extraction.payload) unless details
+
+      success(payload: FetchResult.new(
+        pages: @extraction.payload,
+        failures: build_failures(@failed_records) + @extraction.metadata.fetch(:failures),
+        total_requested: @total_requested
+      ))
+    end
 
     def crawl
       result = Crawler.call(seed_url:, http_client:)
       raise FetchError, result.error.message if result.failure?
 
+      @total_requested = result.metadata.fetch(:total_requested, result.payload.length)
       result.payload
     end
 
-    def extract(raw_pages)
-      result = ExtractionRunner.call(raw_pages:, extractor:)
-      raise FetchError, result.error.message if result.failure?
-
-      result.payload
+    def build_failures(records)
+      records.map do |record|
+        {
+          requested_url: record.requested_url,
+          error: record.error
+        }
+      end
     end
   end
 end

data/lib/nous/primitives/configuration.rb

@@ -12,5 +12,6 @@ module Nous
   ) do
     def debug? = debug
     def recursive? = recursive
+    def single_page? = !recursive
   end
 end

data/lib/nous/primitives/fetch_record.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Nous
+  FetchRecord = Data.define(
+    :requested_url,
+    :final_url,
+    :pathname,
+    :html,
+    :content_type,
+    :ok,
+    :error,
+    :redirected
+  ) do
+    def initialize(
+      requested_url:,
+      pathname:, final_url: nil,
+      html: nil,
+      content_type: nil,
+      ok: true,
+      error: nil,
+      redirected: false
+    )
+      super
+    end
+  end
+end

data/lib/nous/primitives/fetch_result.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Nous
+  FetchResult = Data.define(:pages, :failures, :total_requested) do
+    def succeeded
+      pages.length
+    end
+
+    def failed
+      failures.length
+    end
+
+    def all_succeeded?
+      failures.empty?
+    end
+
+    def any_succeeded?
+      pages.any?
+    end
+  end
+end

data/lib/nous/primitives/page.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nous
-  Page = Data.define(:title, :url, :pathname, :content)
+  Page = Data.define(:title, :url, :pathname, :content, :metadata)
 end

data/lib/nous/serializer.rb

@@ -43,6 +43,8 @@ module Nous
         <page>
           <title>#{page.title}</title>
           <url>#{page.url}</url>
+          <pathname>#{page.pathname}</pathname>
+          <extractor>#{page.metadata[:extractor]}</extractor>
           <content>
         #{page.content}
           </content>
@@ -51,7 +53,13 @@ module Nous
     end
 
     def json_page(page)
-      {title: page.title, url: page.url, content: page.content}
+      {
+        title: page.title,
+        url: page.url,
+        pathname: page.pathname,
+        content: page.content,
+        metadata: page.metadata
+      }
     end
   end
 end

data/lib/nous/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nous
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end