nous 0.4.0 → 0.5.0

data/lib/nous/fetcher/page_extractor.rb

@@ -9,8 +9,22 @@ module Nous
       end
 
       def call
-        result = extractor.extract(record)
+        extract_content
+        report_page
+      rescue => error
+        failure(error)
+      end
+
+      private
+
+      attr_reader :extractor, :record
+
+      def extract_content
+        @extraction = extractor.extract(record)
+      end
 
+      def report_page
+        result = @extraction
         unless result.success?
           warn("[nous] extract skip #{record.final_url}: #{result.error.message}") if Nous.configuration.debug?
           return failure(result.error)
@@ -28,13 +42,8 @@ module Nous
             redirected: record.redirected
           }
         )
-
         success(payload: page)
       end
-
-      private
-
-      attr_reader :extractor, :record
     end
   end
 end

data/lib/nous/fetcher.rb

@@ -4,50 +4,60 @@ module Nous
   class Fetcher < Command
     class FetchError < StandardError; end
 
-    def initialize(seed_url:, extractor: Extractor::Default.new, http_client: nil, details: false)
+    def initialize(seed_url:, extractor: Extractor::Defuddle.new, http_client: nil, details: false)
       @seed_url = seed_url
       @extractor = extractor
       @http_client = http_client
-      @single_page = Nous.configuration.single_page?
       @details = details
     end
 
     def call
-      records = crawl
-      successful_records, failed_records = records.partition(&:ok)
+      resolve_fetch_mode
+      acquire_records
+      verify_acquisition
+      extract_readable_pages
+      report_fetch_outcomes
+    end
 
-      if single_page && !details && successful_records.empty?
-        raise FetchError, failed_records.first&.error || "fetch failed"
-      end
+    private
 
-      pages = extract(successful_records)
+    attr_reader :seed_url, :extractor, :http_client, :single_page, :details
 
-      if details
-        success(payload: FetchResult.new(
-          pages: pages,
-          failures: build_failures(failed_records),
-          total_requested: records.length
-        ))
-      else
-        success(payload: pages)
-      end
+    def resolve_fetch_mode
+      @single_page = Nous.configuration.single_page?
     end
 
-    private
+    def acquire_records
+      @records = crawl
+      @successful_records, @failed_records = @records.partition(&:ok)
+    end
 
-    attr_reader :seed_url, :extractor, :http_client, :single_page, :details
+    def verify_acquisition
+      if single_page && !details && @successful_records.empty?
+        raise FetchError, @failed_records.first&.error || "fetch failed"
+      end
+    end
 
-    def crawl
-      result = Crawler.call(seed_url:, http_client:)
-      raise FetchError, result.error.message if result.failure?
+    def extract_readable_pages
+      @extraction = ExtractionRunner.call(records: @successful_records, extractor:)
+      raise @extraction.error if @extraction.failure?
+    end
 
-      result.payload
+    def report_fetch_outcomes
+      return success(payload: @extraction.payload) unless details
+
+      success(payload: FetchResult.new(
+        pages: @extraction.payload,
+        failures: build_failures(@failed_records) + @extraction.metadata.fetch(:failures),
+        total_requested: @total_requested
+      ))
     end
 
-    def extract(records)
-      result = ExtractionRunner.call(records:, extractor:)
+    def crawl
+      result = Crawler.call(seed_url:, http_client:)
       raise FetchError, result.error.message if result.failure?
 
+      @total_requested = result.metadata.fetch(:total_requested, result.payload.length)
       result.payload
     end
 

data/lib/nous/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Nous
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/nous.rb

@@ -18,7 +18,7 @@ module Nous
       @configuration = nil
     end
 
-    def fetch(seed_url, extractor: Extractor::Default.new, http_client: nil, details: false, **options)
+    def fetch(seed_url, extractor: Extractor::Defuddle.new, http_client: nil, details: false, **options)
       configure(**options)
 
       result = Fetcher.call(seed_url:, extractor:, http_client:, details:)