coelacanth 0.4.3 → 0.5.0

data/lib/coelacanth/extractor.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "extractor/preprocessor"
 require_relative "extractor/normalizer"
 require_relative "extractor/metadata_probe"
 require_relative "extractor/heuristic_probe"
@@ -8,6 +9,9 @@ require_relative "extractor/fallback_probe"
 require_relative "extractor/markdown_renderer"
 require_relative "extractor/image_collector"
 require_relative "extractor/markdown_listing_collector"
+require_relative "extractor/eyecatch_image_extractor"
+require_relative "extractor/morphological_analyzer"
+require_relative "extractor/utilities"
 
 module Coelacanth
   # High-level API for extracting articles without site-specific selectors.
@@ -22,8 +26,9 @@ module Coelacanth
       keyword_init: true
     )
 
-    def call(html:, url: nil)
-      document = Normalizer.new.call(html: html, base_url: url)
+    def call(html:, url: nil, response_metadata: nil)
+      preprocessed_html = Preprocessor.new.call(html: html, url: url)
+      document = Normalizer.new.call(html: preprocessed_html, base_url: url)
 
       [
         [MetadataProbe.new, 0.85],
@@ -34,34 +39,64 @@ module Coelacanth
         result = probe.call(doc: document, url: url)
         next unless result
 
-        return build_response(result, document:, url:) if result.confidence.to_f >= threshold
+        return build_response(result, document:, url:, response_metadata: response_metadata) if result.confidence.to_f >= threshold
       end
 
       build_response(
         PipelineResult.new(node: document, source_tag: :none, confidence: 0.0),
         document: document,
-        url: url
+        url: url,
+        response_metadata: response_metadata
       )
     end
 
     private
 
-    def build_response(result, document:, url:)
+    def build_response(result, document:, url:, response_metadata:)
       node = result.node
       body_markdown = MarkdownRenderer.render(node)
       body_markdown_list = body_markdown.to_s.split(/\n{2,}/).map { |segment| segment.strip }.reject(&:empty?)
+      body_morphemes = MorphologicalAnalyzer.new(config: Coelacanth.config).call(
+        node: node,
+        title: result.title,
+        markdown: body_markdown
+      )
+
+      site_name = extract_site_name(document)
+      body_text = extract_body_text(node)
 
       {
         title: result.title,
         body_markdown: body_markdown,
         body_markdown_list: body_markdown_list,
+        body_morphemes: body_morphemes,
         images: ImageCollector.new.call(node),
+        eyecatch_image: EyecatchImageExtractor.new.call(doc: document, base_url: url),
         published_at: result.published_at,
         byline: result.byline,
         source: result.source_tag,
         confidence: result.confidence,
-        listings: MarkdownListingCollector.new.call(markdown: body_markdown, base_url: url)
+        listings: MarkdownListingCollector.new.call(markdown: body_markdown, base_url: url),
+        site_name: site_name,
+        body_text: body_text,
+        response_metadata: response_metadata || {}
       }
     end
+
+    def extract_site_name(document)
+      Utilities.meta_content(
+        document,
+        "meta[property='og:site_name']",
+        "meta[name='application-name']",
+        "meta[name='apple-mobile-web-app-title']",
+        "meta[name='twitter:site']"
+      ) || document.at_css("title")&.text&.strip
+    end
+
+    def extract_body_text(node)
+      return if node.nil?
+
+      node.text.to_s.gsub(/\s+/, " ").strip
+    end
   end
 end

data/lib/coelacanth/http.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "delegate"
 require "net/http"
 require "open-uri"
 require "timeout"
@@ -14,7 +15,30 @@ module Coelacanth
     DEFAULT_READ_TIMEOUT = 10
     MAX_RETRIES = 2
 
-    ErrorResponse = Struct.new(:status, :meta, :base_uri, :body, keyword_init: true) do
+    Response = Class.new(SimpleDelegator) do
+      attr_reader :status_code, :headers, :final_uri
+
+      def initialize(response, final_uri: nil)
+        super(response)
+        @status_code = response.respond_to?(:code) ? response.code.to_i : nil
+        @headers = response.respond_to?(:each_header) ? response.each_header.to_h : {}
+        @final_uri = (response.respond_to?(:uri) ? response.uri : nil) || final_uri
+      end
+
+      def final_url
+        final_uri&.to_s
+      end
+
+      def is_a?(klass)
+        super || __getobj__.is_a?(klass)
+      end
+
+      def kind_of?(klass)
+        is_a?(klass)
+      end
+    end
+
+    ErrorResponse = Struct.new(:status, :meta, :base_uri, :final_uri, :body, keyword_init: true) do
       def string
         body.to_s
       end
@@ -26,7 +50,8 @@ module Coelacanth
 
     def get_response(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, retries: MAX_RETRIES)
       ensure_allowed!(uri)
-      raw_get_response(uri, open_timeout: open_timeout, read_timeout: read_timeout, retries: retries)
+      response = raw_get_response(uri, open_timeout: open_timeout, read_timeout: read_timeout, retries: retries)
+      Response.new(response, final_uri: uri)
     end
 
     def raw_get_response(uri, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, retries: MAX_RETRIES)
@@ -63,6 +88,7 @@ module Coelacanth
         status: [response.code, response.message],
         meta: response.each_header.to_h,
         base_uri: uri,
+        final_uri: response.respond_to?(:uri) ? response.uri : uri,
         body: response.body
       )
 

data/lib/coelacanth/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Coelacanth
-  VERSION = "0.4.3"
+  VERSION = "0.5.0"
 end

data/lib/coelacanth.rb

@@ -29,15 +29,21 @@ module Coelacanth
     rescue Coelacanth::TimeoutError
       nil
     end
+    response_metadata = {
+      status_code: response&.status_code,
+      headers: response&.headers || {},
+      final_url: response&.final_url || regular_url
+    }
     html = response&.body.to_s
     html = html.dup
     html = html.force_encoding(Encoding::UTF_8)
     html = html.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
-    extractor_result = Extractor.new.call(html: html, url: regular_url)
+    extractor_result = Extractor.new.call(html: html, url: regular_url, response_metadata: response_metadata)
     {
       dom: Dom.new.oga(regular_url, html: html),
       screenshot: @client.get_screenshot,
       extraction: extractor_result,
+      response: response_metadata
     }
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coelacanth
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Yusuke
@@ -37,13 +37,16 @@ files:
 - lib/coelacanth/configure.rb
 - lib/coelacanth/dom.rb
 - lib/coelacanth/extractor.rb
+- lib/coelacanth/extractor/eyecatch_image_extractor.rb
 - lib/coelacanth/extractor/fallback_probe.rb
 - lib/coelacanth/extractor/heuristic_probe.rb
 - lib/coelacanth/extractor/image_collector.rb
 - lib/coelacanth/extractor/markdown_listing_collector.rb
 - lib/coelacanth/extractor/markdown_renderer.rb
 - lib/coelacanth/extractor/metadata_probe.rb
+- lib/coelacanth/extractor/morphological_analyzer.rb
 - lib/coelacanth/extractor/normalizer.rb
+- lib/coelacanth/extractor/preprocessor.rb
 - lib/coelacanth/extractor/utilities.rb
 - lib/coelacanth/extractor/weak_ml_probe.rb
 - lib/coelacanth/http.rb