archaeo 0.2.4 → 0.2.5

data/lib/archaeo/cli.rb

@@ -9,6 +9,10 @@ module Archaeo
   class Cli < Thor
     map %w[--version -v] => :version
 
+    def self.exit_on_failure?
+      true
+    end
+
     desc "version", "Show archaeo version"
     def version
       puts "archaeo #{VERSION}"
@@ -26,51 +30,64 @@ module Archaeo
     option :format, desc: "Output format (table, json, csv)",
                     default: "table"
     def snapshots(url)
-      cdx = CdxApi.new
-      opts = build_cdx_options(options)
-      snaps = cdx.snapshots(url, **opts).to_a
-      case options[:format]
-      when "json" then output_json(snaps)
-      when "csv" then output_csv(snaps)
-      else output_table(snaps)
+      fmt = validate_output_format
+      handle_errors do
+        snaps = fetch_snapshots(url)
+        output_formatted(snaps, fmt)
       end
     end
 
     desc "near URL TIMESTAMP",
          "Find the snapshot closest to a timestamp"
+    option :format, desc: "Output format (url, json)", default: "url"
     def near(url, timestamp)
-      snap = CdxApi.new.near(url, timestamp: timestamp)
-      puts snap.archive_url
+      handle_errors do
+        snap = CdxApi.new.near(url, timestamp: timestamp)
+        output_snapshot(snap)
+      end
     end
 
     desc "oldest URL", "Find the oldest snapshot of a URL"
+    option :format, desc: "Output format (url, json)", default: "url"
     def oldest(url)
-      snap = CdxApi.new.oldest(url)
-      puts snap.archive_url
+      handle_errors do
+        snap = CdxApi.new.oldest(url)
+        output_snapshot(snap)
+      end
     end
 
     desc "newest URL", "Find the newest snapshot of a URL"
+    option :format, desc: "Output format (url, json)", default: "url"
     def newest(url)
-      snap = CdxApi.new.newest(url)
-      puts snap.archive_url
+      handle_errors do
+        snap = CdxApi.new.newest(url)
+        output_snapshot(snap)
+      end
     end
 
     desc "available URL", "Check if a URL is archived"
+    option :timestamp, desc: "Check near this timestamp (YYYYMMDDHHmmss)"
     def available(url)
-      result = AvailabilityApi.new.near(url)
-      if result.available?
-        puts "Available: #{result.archive_url}"
-      else
-        puts "Not available"
-        exit 1
+      handle_errors do
+        result = AvailabilityApi.new.near(
+          url, timestamp: options[:timestamp]
+        )
+        if result.available?
+          puts "Available: #{result.archive_url}"
+        else
+          puts "Not available"
+          exit 1
+        end
       end
     end
 
     desc "save URL", "Save a URL to the Wayback Machine"
     def save(url)
-      result = SaveApi.new.save(url)
-      label = result.cached? ? "Cached" : "Saved"
-      puts "#{label}: #{result.archive_url}"
+      handle_errors do
+        result = SaveApi.new.save(url)
+        label = result.cached? ? "Cached" : "Saved"
+        puts "#{label}: #{result.archive_url}"
+      end
     end
 
     desc "fetch URL TIMESTAMP",
@@ -79,15 +96,12 @@ module Archaeo
                       desc: "Fetch raw (identity) content"
     option :output, desc: "Write content to file"
     def fetch(url, timestamp)
-      page = Fetcher.new.fetch(
-        url, timestamp: timestamp,
-             identity: options[:identity]
-      )
-
-      if options[:output]
-        write_output(options[:output], page.content)
-      else
-        $stdout.write(page.content)
+      handle_errors do
+        page = Fetcher.new.fetch(
+          url, timestamp: timestamp,
+               identity: options[:identity]
+        )
+        output_page(page)
       end
     end
 
@@ -97,38 +111,34 @@ module Archaeo
     option :to, desc: "End timestamp (YYYYMMDDHHmmss)"
     option :resume, type: :boolean, default: false,
                     desc: "Resume interrupted download"
+    option :concurrency, type: :numeric, default: 1,
+                         desc: "Number of parallel downloads"
     def download(url)
-      downloader = BulkDownloader.new(output_dir: options[:output])
-
-      downloader.download(
-        url,
-        from: options[:from],
-        to: options[:to],
-        resume: options[:resume],
-      ) do |current, total, snap|
-        warn "[#{current}/#{total}] " \
-             "#{snap.timestamp} #{snap.original_url}"
+      handle_errors do
+        downloader = BulkDownloader.new(
+          output_dir: options[:output],
+          concurrency: options[:concurrency],
+        )
+        download_with_progress(downloader, url)
       end
     end
 
     desc "known_urls DOMAIN",
          "List all known URLs for a domain"
     def known_urls(domain)
-      CdxApi.new.known_urls(domain).each do |u|
-        puts u
+      handle_errors do
+        CdxApi.new.known_urls(domain).each do |u|
+          puts u
+        end
       end
     end
 
     desc "num_pages URL",
          "Show number of CDX result pages for a URL"
     def num_pages(url)
-      puts CdxApi.new.num_pages(url)
-    rescue RateLimitError => e
-      warn "Error: #{e.message}"
-      exit 1
-    rescue Error => e
-      warn "Error: #{e.message}"
-      exit 1
+      handle_errors do
+        puts CdxApi.new.num_pages(url)
+      end
     end
 
     CDX_OPTION_MAP = {
@@ -143,6 +153,79 @@ module Archaeo
 
     private
 
+    def handle_errors
+      yield
+    rescue RateLimitError => e
+      warn "Rate limited: #{e.message}"
+      exit 1
+    rescue NoSnapshotFound => e
+      warn "Not found: #{e.message}"
+      exit 1
+    rescue BlockedSiteError => e
+      warn "Blocked: #{e.message}"
+      exit 1
+    rescue Error => e
+      warn "Error: #{e.message}"
+      exit 1
+    end
+
+    def validate_output_format
+      fmt = options[:format].to_s
+      fmt = "table" if fmt.empty?
+      unless %w[table json csv].include?(fmt)
+        warn "Unknown format '#{fmt}'. Use: table, json, csv"
+        exit 1
+      end
+      fmt
+    end
+
+    def fetch_snapshots(url)
+      cdx = CdxApi.new
+      opts = build_cdx_options(options)
+      cdx.snapshots(url, **opts).to_a
+    end
+
+    def output_formatted(snaps, fmt)
+      case fmt
+      when "json" then output_json(snaps)
+      when "csv" then output_csv(snaps)
+      else output_table(snaps)
+      end
+    end
+
+    def output_snapshot(snap)
+      case options[:format]
+      when "json"
+        puts JSON.generate(snap.as_json)
+      else
+        puts snap.archive_url
+      end
+    end
+
+    def output_page(page)
+      if options[:output]
+        write_output(options[:output], page.content)
+      elsif page.text? || page.json?
+        $stdout.write(page.content)
+      else
+        warn "Binary content (#{page.content_type}). " \
+             "Use --output FILE to save."
+        exit 1
+      end
+    end
+
+    def download_with_progress(downloader, url)
+      downloader.download(
+        url,
+        from: options[:from],
+        to: options[:to],
+        resume: options[:resume],
+      ) do |current, total, snap|
+        warn "[#{current}/#{total}] " \
+             "#{snap.timestamp} #{snap.original_url}"
+      end
+    end
+
     def build_cdx_options(opts)
       CDX_OPTION_MAP.each_with_object({}) do |(cli_key, api_key), result|
         value = opts[cli_key]
@@ -158,14 +241,7 @@ module Archaeo
     end
 
     def output_json(snaps)
-      data = snaps.map do |snap|
-        {
-          timestamp: snap.timestamp.to_s,
-          status_code: snap.status_code,
-          url: snap.original_url,
-          archive_url: snap.archive_url,
-        }
-      end
+      data = snaps.map(&:as_json)
       puts JSON.generate(data)
     end
 

data/lib/archaeo/download_state.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "set"
+
 module Archaeo
   # Tracks download progress for resume support.
   #
@@ -17,16 +19,21 @@ module Archaeo
     end
 
     def completed?(timestamp)
-      timestamps.include?(timestamp.to_s)
+      timestamps_set.include?(timestamp.to_s)
     end
 
     def mark_completed(timestamp)
-      timestamps << timestamp.to_s
+      ts = timestamp.to_s
+      return if timestamps_set.include?(ts)
+
+      timestamps << ts
+      @timestamps_set = nil
       save
     end
 
     def clear
       @timestamps = []
+      @timestamps_set = nil
       FileUtils.rm_f(@path)
     end
 
@@ -36,6 +43,10 @@ module Archaeo
       @timestamps ||= load_timestamps
     end
 
+    def timestamps_set
+      @timestamps_set ||= timestamps.to_set
+    end
+
     def load_timestamps
       return [] unless File.exist?(@path)
 
@@ -43,7 +54,10 @@ module Archaeo
     end
 
     def save
-      File.write(@path, "#{timestamps.uniq.sort.join("\n")}\n")
+      content = "#{timestamps.sort.join("\n")}\n"
+      tmp_path = "#{@path}.tmp"
+      File.write(tmp_path, content)
+      File.rename(tmp_path, @path)
     end
   end
 end

data/lib/archaeo/http_client.rb

@@ -15,6 +15,10 @@ module Archaeo
     DEFAULT_TIMEOUT = 30
     DEFAULT_MAX_RETRIES = 3
     DEFAULT_RETRY_DELAY = 2
+    MAX_POOL_SIZE = 8
+    MAX_IDLE_TIME = 60
+
+    RETRIABLE_STATUSES = [429, 502, 503, 504].freeze
 
     TRANSIENT_ERRORS = [
       Net::ReadTimeout,
@@ -64,7 +68,16 @@ module Archaeo
       @retry_delay = retry_delay
       @user_agent = user_agent
       @connections = {}
+      @last_used = {}
       @mutex = Mutex.new
+      @shutdown = false
+    end
+
+    def self.open(**opts)
+      client = new(**opts)
+      yield client
+    ensure
+      client&.shutdown
     end
 
     def get(url, headers: {})
@@ -81,6 +94,9 @@ module Archaeo
 
     def shutdown
       @mutex.synchronize do
+        return if @shutdown
+
+        @shutdown = true
         @connections.each_value do |http|
           http.finish
         rescue StandardError
@@ -103,16 +119,26 @@ module Archaeo
     def connection_for(uri)
       key = connection_key(uri)
       @mutex.synchronize do
+        evict_if_pool_full(key)
         http = @connections[key]
         if http && !http.active?
-          @connections.delete(key)
+          close_connection(key)
           http = nil
         end
         @connections[key] = build_connection(uri) unless http
+        @last_used[key] = Time.now
         @connections[key]
       end
     end
 
+    def evict_if_pool_full(key)
+      evict_stale_connections
+      return unless @connections.size >= MAX_POOL_SIZE &&
+        !@connections.key?(key)
+
+      evict_lru
+    end
+
     def build_connection(uri)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = uri.scheme == "https"
@@ -124,26 +150,82 @@ module Archaeo
 
     def invalidate_connection(uri)
       key = connection_key(uri)
-      @mutex.synchronize do
-        http = @connections.delete(key)
-        begin
-          http&.finish
-        rescue StandardError
-          nil
-        end
+      @mutex.synchronize { close_connection(key) }
+    end
+
+    def close_connection(key)
+      http = @connections.delete(key)
+      @last_used.delete(key)
+      begin
+        http&.finish
+      rescue StandardError
+        nil
+      end
+    end
+
+    def evict_stale_connections
+      now = Time.now
+      @connections.each_key do |key|
+        idle = now - (@last_used[key] || now)
+        close_connection(key) if idle > MAX_IDLE_TIME
+      end
+    end
+
+    def evict_lru
+      lru_key = @last_used.min_by { |_, t| t }&.first
+      close_connection(lru_key) if lru_key
+    end
+
+    # Internal error class for HTTP status retry signaling
+    class RetriableStatusError < StandardError
+      attr_reader :response
+
+      def initialize(response)
+        @response = response
+        super("Retriable HTTP status: #{response.status}")
       end
     end
 
     def attempt_with_retries(uri, headers, request_class)
       retries = 0
       begin
-        execute_with_connection(uri, headers, request_class)
+        execute_and_check(uri, headers, request_class)
+      rescue RetriableStatusError => e
+        retry_status(e, retries += 1) && retry
       rescue *TRANSIENT_ERRORS => e
-        retries += 1
-        raise_if_exhausted(retries, e)
-        invalidate_connection(uri)
-        sleep(@retry_delay * retries)
-        retry
+        retry_transient(e, uri, retries += 1) && retry
+      end
+    end
+
+    def retry_status(error, retries)
+      raise_if_exhausted(retries,
+                         RateLimitError.new("HTTP #{error.response.status}"))
+      sleep(extract_retry_after(error.response) || (@retry_delay * retries))
+    end
+
+    def retry_transient(error, uri, retries)
+      raise_if_exhausted(retries, error)
+      invalidate_connection(uri)
+      sleep(@retry_delay * retries)
+    end
+
+    def execute_and_check(uri, headers, request_class)
+      response = execute_with_connection(uri, headers, request_class)
+      if RETRIABLE_STATUSES.include?(response.status)
+        raise RetriableStatusError, response
+      end
+
+      response
+    end
+
+    def extract_retry_after(response)
+      value = response.headers["retry-after"]
+      return nil unless value
+
+      begin
+        Integer(value)
+      rescue StandardError
+        nil
       end
     end
 

data/lib/archaeo/page.rb

@@ -34,6 +34,35 @@ module Archaeo
       @encoding ||= detect_encoding
     end
 
+    def html?
+      @content_type&.include?("text/html")
+    end
+
+    def json?
+      @content_type&.include?("application/json")
+    end
+
+    def image?
+      @content_type&.start_with?("image/")
+    end
+
+    def text?
+      @content_type&.start_with?("text/")
+    end
+
+    def binary?
+      !(text? || json? || html?)
+    end
+
+    def title
+      @title ||= begin
+        doc = Nokogiri::HTML(@raw_content)
+        doc.at_css("title")&.text&.strip
+      rescue StandardError
+        nil
+      end
+    end
+
     private
 
     def detect_encoding

data/lib/archaeo/page_bundle.rb

@@ -6,11 +6,25 @@ module Archaeo
   # Bundles a Page with the AssetList discovered from its HTML,
   # providing a single object for complete page archival.
   class PageBundle
+    include Enumerable
+
     attr_reader :page, :assets
 
     def initialize(page:, assets:)
       @page = page
       @assets = assets
     end
+
+    def each(&block)
+      assets.each(&block)
+    end
+
+    def size
+      assets.size + 1
+    end
+
+    def asset_count
+      assets.size
+    end
   end
 end

data/lib/archaeo/save_api.rb

@@ -32,7 +32,7 @@ module Archaeo
         response = @client.get(save_url)
         check_response_errors!(response, url)
 
-        result = process_save_response(response, start_time)
+        result = process_save_response(response, start_time, url)
         return result if result
       end
 
@@ -40,13 +40,13 @@ module Archaeo
             "Failed to save #{url} after #{@max_tries} attempts"
     end
 
-    def process_save_response(response, start_time)
+    def process_save_response(response, start_time, url)
       archive_url = extract_archive_url(response)
       return nil unless archive_url
 
       ts = Timestamp.parse(extract_timestamp(archive_url))
       cached = ts.to_time < start_time - 2700
-      SaveResult.new(archive_url: archive_url,
+      SaveResult.new(url: url, archive_url: archive_url,
                      timestamp: ts, cached: cached)
     end
 

data/lib/archaeo/save_result.rb

@@ -6,9 +6,10 @@ module Archaeo
   # Contains the resulting archive URL, timestamp, and whether
   # the page was already cached in the archive.
   class SaveResult
-    attr_reader :archive_url, :timestamp
+    attr_reader :url, :archive_url, :timestamp
 
-    def initialize(archive_url:, timestamp:, cached:)
+    def initialize(url:, archive_url:, timestamp:, cached:)
+      @url = url
       @archive_url = archive_url
       @timestamp = Timestamp.coerce(timestamp)
       @cached = cached

data/lib/archaeo/snapshot.rb

@@ -37,6 +37,34 @@ module Archaeo
       @status_code == 200
     end
 
+    def redirect?
+      @status_code.between?(300, 399)
+    end
+
+    def client_error?
+      @status_code.between?(400, 499)
+    end
+
+    def server_error?
+      @status_code.between?(500, 599)
+    end
+
+    def error?
+      client_error? || server_error?
+    end
+
+    def fetch(client: HttpClient.new, identity: false)
+      Fetcher.new(client: client).fetch(
+        original_url, timestamp: @timestamp, identity: identity
+      )
+    end
+
+    def fetch_with_assets(client: HttpClient.new)
+      Fetcher.new(client: client).fetch_page_with_assets(
+        original_url, timestamp: @timestamp
+      )
+    end
+
     def to_a
       [@urlkey, @timestamp, @original_url, @mimetype,
        @status_code, @digest, @length]
@@ -54,6 +82,18 @@ module Archaeo
       }
     end
 
+    def as_json(*)
+      {
+        urlkey: @urlkey,
+        timestamp: @timestamp.to_s,
+        original_url: @original_url,
+        mimetype: @mimetype,
+        status_code: @status_code,
+        digest: @digest,
+        length: @length,
+      }
+    end
+
     def ==(other)
       other.is_a?(self.class) && to_a == other.to_a
     end

data/lib/archaeo/timestamp.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "date"
+
 module Archaeo
   # Value object representing a Wayback Machine timestamp (YYYYMMDDHHmmss).
   #
@@ -70,6 +72,26 @@ module Archaeo
       @to_time.to_i
     end
 
+    def to_iso8601
+      @to_time.iso8601
+    end
+
+    def to_rfc3339
+      @to_time.rfc3339
+    end
+
+    def +(seconds)
+      self.class.from_time(@to_time + seconds)
+    end
+
+    def -(other)
+      if other.is_a?(self.class)
+        @to_time - other.to_time
+      else
+        self.class.from_time(@to_time - other)
+      end
+    end
+
     def <=>(other)
       return nil unless other.is_a?(self.class)
 

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.4"
+  VERSION = "0.2.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Ribose Inc.