archaeo 0.2.11 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76a36571f0747712c2abda1a4aef93c7ade9a83b42590e23f0148b89138451b0
-  data.tar.gz: a9eed4768d084756fbb10eda17b1f2098246fd56a93cbe91b55f693850e5008a
+  metadata.gz: 5c83d2255fd16d4af732415985d1dcc48ca84711221e4b893838124252ffc062
+  data.tar.gz: '08c5a7e88bfcae8db40e47f7c1aeac2d4930f9354a94d50af2fd9d6da7e5846c'
 SHA512:
-  metadata.gz: fa8e01a6aa31aa678a17ce2fc4f59e324c4e8779716b7c41d876dbd366af06dda30296af446919eedc3136efe5bc2527abef60d5aa4274745e94ef7415a775fa
-  data.tar.gz: b3fd25ec4d3b10c759992226dd2d699276dbd9def9318ef343f632b69faa5c4fb0017f78ae7aa87b1b85c4fb48a642d4d667c3f43e0e768162486d63f1bf7be1
+  metadata.gz: 5c264ca7c5cfb256929599426c734927573f6ea8836fd43f1d7524765aa060aee8a7c0fec3a8dd3a9bbc01f8ab441d52428f4245ca66ff02a75c8e8ef1341e38
+  data.tar.gz: 35c17304038f91f8bd898a8076efd840faa5a35a46cac17e73c46bc40c44f5a4d04ca94093aef0226be27f53afb4deb5754a9ab50944ec76c80e9cc791c30830

data/lib/archaeo/archive_health_check.rb

@@ -1,20 +1,6 @@
 # frozen_string_literal: true
 
 module Archaeo
-  # Verifies that archived snapshots are still accessible.
-  #
-  # Checks each snapshot by performing HEAD requests to the
-  # archive URL and reporting accessibility status.
-  HealthReport = Struct.new(
-    :total, :accessible, :missing, :errors, :details,
-    keyword_init: true
-  )
-
-  HealthDetail = Struct.new(
-    :snapshot, :status, :error,
-    keyword_init: true
-  )
-
   class ArchiveHealthCheck
     def initialize(client: HttpClient.new, cdx_api: nil)
       @client = client

data/lib/archaeo/bulk_downloader.rb

@@ -245,9 +245,6 @@ module Archaeo
       page = fetch_page(snapshot)
       validate_page_status(page, snapshot)
       write_page_file(page, snapshot)
-    rescue StandardError
-      FileUtils.rm_f(tmp_path) if defined?(tmp_path)
-      raise
     end
 
     def fetch_page(snapshot)
@@ -272,6 +269,9 @@ module Archaeo
       File.binwrite(tmp_path, page.content)
       File.rename(tmp_path, filename)
       page.content
+    rescue StandardError
+      FileUtils.rm_f(tmp_path) if defined?(tmp_path)
+      raise
     end
 
     EXTENSION_MAP = {

data/lib/archaeo/coverage_analyzer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Archaeo
+  # Builds a CoverageReport from CDX snapshot data.
+  class CoverageAnalyzer
+    def initialize(cdx_api: nil)
+      @cdx_api = cdx_api
+    end
+
+    def analyze(url, from: nil, to: nil)
+      cdx = @cdx_api || CdxApi.new
+      snapshots = cdx.snapshots(url, from: from, to: to).to_a
+
+      unique_urls = snapshots.map(&:original_url).uniq
+      status_dist = compute_status_distribution(snapshots)
+      gaps = compute_temporal_gaps(snapshots)
+
+      CoverageReport.new(
+        url: url,
+        total_urls: unique_urls.size,
+        archived_urls: snapshots.count(&:success?),
+        status_distribution: status_dist,
+        temporal_gaps: gaps,
+      )
+    end
+
+    private
+
+    def compute_status_distribution(snapshots)
+      snapshots.each_with_object(Hash.new(0)) do |snap, counts|
+        counts[snap.status_code] += 1
+      end
+    end
+
+    def compute_temporal_gaps(snapshots)
+      return [] if snapshots.size < 2
+
+      sorted = snapshots.sort_by(&:timestamp)
+      gaps = []
+      sorted.each_cons(2) do |a, b|
+        diff_days = (b.timestamp.to_time - a.timestamp.to_time) / 86400
+        next unless diff_days > 30
+
+        gaps << { from: a.timestamp.to_s, to: b.timestamp.to_s,
+                  gap_days: diff_days.round }
+      end
+      gaps
+    end
+  end
+end

data/lib/archaeo/coverage_report.rb

@@ -51,51 +51,4 @@ module Archaeo
       to_h
     end
   end
-
-  # Builds a CoverageReport from CDX snapshot data.
-  class CoverageAnalyzer
-    def initialize(cdx_api: nil)
-      @cdx_api = cdx_api
-    end
-
-    def analyze(url, from: nil, to: nil)
-      cdx = @cdx_api || CdxApi.new
-      snapshots = cdx.snapshots(url, from: from, to: to).to_a
-
-      unique_urls = snapshots.map(&:original_url).uniq
-      status_dist = compute_status_distribution(snapshots)
-      gaps = compute_temporal_gaps(snapshots)
-
-      CoverageReport.new(
-        url: url,
-        total_urls: unique_urls.size,
-        archived_urls: snapshots.count(&:success?),
-        status_distribution: status_dist,
-        temporal_gaps: gaps,
-      )
-    end
-
-    private
-
-    def compute_status_distribution(snapshots)
-      snapshots.each_with_object(Hash.new(0)) do |snap, counts|
-        counts[snap.status_code] += 1
-      end
-    end
-
-    def compute_temporal_gaps(snapshots)
-      return [] if snapshots.size < 2
-
-      sorted = snapshots.sort_by(&:timestamp)
-      gaps = []
-      sorted.each_cons(2) do |a, b|
-        diff_days = (b.timestamp.to_time - a.timestamp.to_time) / 86400
-        next unless diff_days > 30
-
-        gaps << { from: a.timestamp.to_s, to: b.timestamp.to_s,
-                  gap_days: diff_days.round }
-      end
-      gaps
-    end
-  end
 end

data/lib/archaeo/download_state.rb

@@ -73,18 +73,15 @@ module Archaeo
     def stale_entries(base_dir: @output_dir)
       @mutex.synchronize do
         entries.reject do |e|
-          find_file(base_dir,
-                    e["ts"]) && File.exist?(find_file(base_dir, e["ts"]))
+          path = find_file(base_dir, e["ts"])
+          path && File.exist?(path)
         end
       end
     end
 
     def cleanup_stale(base_dir: @output_dir)
       @mutex.synchronize do
-        stale = entries.reject do |e|
-          path = find_file(base_dir, e["ts"])
-          path && File.exist?(path)
-        end
+        stale = stale_entries(base_dir: base_dir)
         @entries = entries - stale
         @entries_key = nil
         save

data/lib/archaeo/health_report.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Archaeo
+  HealthReport = Struct.new(
+    :total, :accessible, :missing, :errors, :details,
+    keyword_init: true
+  )
+
+  HealthDetail = Struct.new(
+    :snapshot, :status, :error,
+    keyword_init: true
+  )
+end

data/lib/archaeo/page.rb

@@ -59,21 +59,15 @@ module Archaeo
     end
 
     def title
-      @title ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        doc.at_css("title")&.text&.strip
-      rescue StandardError
-        nil
-      end
+      @title ||= html_doc.at_css("title")&.text&.strip
     end
 
     def links
       return [] unless html?
 
       @links ||= begin
-        doc = Nokogiri::HTML(@raw_content)
         base = @archive_url || @original_url
-        doc.css("a[href]").map do |anchor|
+        html_doc.css("a[href]").map do |anchor|
           href = resolve_page_url(anchor["href"], base)
           { href: href, text: anchor.text.strip,
             external: href && !href.include?(original_domain) }
@@ -85,9 +79,8 @@ module Archaeo
       return {} unless html?
 
       @meta_tags ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        result = extract_meta_entries(doc)
-        canonical = doc.at_css('link[rel="canonical"]')
+        result = extract_meta_entries(html_doc)
+        canonical = html_doc.at_css('link[rel="canonical"]')
         result["canonical"] = canonical["href"].to_s if canonical
         result
       end
@@ -96,46 +89,34 @@ module Archaeo
     def headings
       return [] unless html?
 
-      @headings ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        doc.css("h1, h2, h3, h4, h5, h6").map do |el|
-          { level: el.name[1].to_i, text: el.text.strip }
-        end
+      @headings ||= html_doc.css("h1, h2, h3, h4, h5, h6").map do |el|
+        { level: el.name[1].to_i, text: el.text.strip }
       end
     end
 
     def images
       return [] unless html?
 
-      @images ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        doc.css("img[src]").map do |el|
-          { src: el["src"], alt: el["alt"].to_s,
-            width: el["width"]&.to_i, height: el["height"]&.to_i }
-        end
+      @images ||= html_doc.css("img[src]").map do |el|
+        { src: el["src"], alt: el["alt"].to_s,
+          width: el["width"]&.to_i, height: el["height"]&.to_i }
       end
     end
 
     def forms
       return [] unless html?
 
-      @forms ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        doc.css("form").map do |form|
-          { action: form["action"].to_s, method: (form["method"] || "GET").upcase,
-            fields: extract_form_fields(form) }
-        end
+      @forms ||= html_doc.css("form").map do |form|
+        { action: form["action"].to_s, method: (form["method"] || "GET").upcase,
+          fields: extract_form_fields(form) }
       end
     end
 
     def scripts
       return [] unless html?
 
-      @scripts ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        doc.css("script").map do |el|
-          { src: el["src"].to_s, type: el["type"].to_s }
-        end
+      @scripts ||= html_doc.css("script").map do |el|
+        { src: el["src"].to_s, type: el["type"].to_s }
       end
     end
 
@@ -143,8 +124,7 @@ module Archaeo
       return [] unless html?
 
       @microposts ||= begin
-        doc = Nokogiri::HTML(@raw_content)
-        containers = find_article_containers(doc)
+        containers = find_article_containers(html_doc)
         containers.filter_map { |el| extract_micropost(el) }
       end
     end
@@ -162,15 +142,7 @@ module Archaeo
     end
 
     def as_json(*)
-      {
-        content_type: @content_type,
-        status_code: @status_code,
-        archive_url: @archive_url,
-        original_url: @original_url,
-        timestamp: @timestamp.to_s,
-        size: size,
-        encoding: encoding.to_s,
-      }
+      to_h.transform_values { |v| v.is_a?(Timestamp) ? v.to_s : v }
     end
 
     def inspect
@@ -179,6 +151,10 @@ module Archaeo
 
     private
 
+    def html_doc
+      @html_doc ||= Nokogiri::HTML(@raw_content)
+    end
+
     def detect_encoding
       charset = extract_charset(@content_type)
       return Encoding.find(charset) if charset
@@ -199,11 +175,10 @@ module Archaeo
     end
 
     def detect_html_charset
-      doc = Nokogiri::HTML(@raw_content)
-      node = doc.at_css("meta[charset]")
+      node = html_doc.at_css("meta[charset]")
       return node["charset"] if node
 
-      content = doc.at_css('meta[http-equiv="Content-Type"]')&.[]("content")
+      content = html_doc.at_css('meta[http-equiv="Content-Type"]')&.[]("content")
       return nil unless content
 
       match = content.match(/charset=([^\s;]+)/i)
@@ -250,8 +225,8 @@ module Archaeo
 
     def resolve_page_url(href, base)
       return href unless href
-      return href if href.start_with?("http", "//", "data:", "#",
-                                      "javascript:")
+      return href if href.start_with?("http:", "https:", "//", "data:",
+                                      "#", "javascript:")
       return nil unless base
 
       URI.join(base, href).to_s

data/lib/archaeo/snapshot_diff.rb

@@ -104,7 +104,6 @@ module Archaeo
     end
 
     def count_elements(page)
-      require "nokogiri"
       doc = Nokogiri::HTML(page.content)
       counts = Hash.new(0)
       doc.css("*").each { |el| counts[el.name] += 1 }

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.11"
+  VERSION = "0.2.12"
 end

data/lib/archaeo.rb

@@ -71,4 +71,7 @@ module Archaeo
   autoload :SearchResult, "archaeo/archive_search"
   autoload :LocalRewriter, "archaeo/local_rewriter"
   autoload :LocalRewriteSummary, "archaeo/local_rewriter"
+  autoload :CoverageAnalyzer, "archaeo/coverage_analyzer"
+  autoload :HealthReport, "archaeo/health_report"
+  autoload :HealthDetail, "archaeo/health_report"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.12
 platform: ruby
 authors:
 - Ribose Inc.
@@ -87,11 +87,13 @@ files:
 - lib/archaeo/color_output.rb
 - lib/archaeo/configuration.rb
 - lib/archaeo/content_tracker.rb
+- lib/archaeo/coverage_analyzer.rb
 - lib/archaeo/coverage_report.rb
 - lib/archaeo/download_scheduler.rb
 - lib/archaeo/download_state.rb
 - lib/archaeo/encoding_detector.rb
 - lib/archaeo/fetcher.rb
+- lib/archaeo/health_report.rb
 - lib/archaeo/http_client.rb
 - lib/archaeo/local_rewriter.rb
 - lib/archaeo/page.rb