webarchive 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74bc28f7e8b343b9ef2c558991d22ea8560a9fc29a5501fc0ee0514078df4d14
-  data.tar.gz: 6ec0a915b37359d681caad36ddfdb6f2febce1cea43bfb8493b4702c611e67aa
+  metadata.gz: 16eba058e7574f02d9e444a6d72b2aefb41b55fb0acb9adc1b6eef492cef306d
+  data.tar.gz: 33411ba62ea808f72b930ef52a5f112f4df80b4febb1d2bd88c7017a62785550
 SHA512:
-  metadata.gz: 5666adba0ad58e9168874a205dde99288e407a3a2260543c9aa4c6ebc23c04b5705f61d4926dba7bb219d3cd5cab17406732aef45ec50b5d1fd2f289c58a272e
-  data.tar.gz: 7a5cf05ee6857a85e87508bd3c404be5a4e5cc7b3d1c8589cde4db8a374a3365f03c68fd52f0b9ac2e36301266f7f011c117a3ebd8d8539a04d9f1411e9ba766
+  metadata.gz: 8787995fe6bdc059e2275371ed33eb332a8f1c99aa099b995a20805258ed2d05983507a8f2958e104edcc31e7259bc3be68d22f838c917294687533718a799c5
+  data.tar.gz: f9e73ef39c3cb49b6bd25569cc2549292982aa47e71e1c31ce735eb350e8a1fe0882fe8cb0a3d9ee23d3b6075d33ec149a951f4d70f9095d4b8f03bc8c306e6c

data/README.md

@@ -1,9 +1,16 @@
 # Webarchive
 
-This is a CUI tool for sending URIs to public web archiving tools such as web.archive.org and archive.today.
+This is a CUI tool for sending URIs to public web archiving tools such
+as web.archive.org and archive.today.
 
 Requests are throttled.
 
+## Rationale
+
+This tool's motivation is simple - increased availability by redundancy. Your favorite web archiving service might be down at some point in time, or blocked by certain websites. Use 2 or more services to archive something, and your archive will be safe if at least one of them remains available.
+
+Browser extensions with similar functionalities might exist, but this tool might be for you when you need to archive a large number of URLs, and well, if you like CUI.
+
 ## Installation
 
 Use this line to install it:
@@ -22,25 +29,37 @@ If you have a list of URIs in a file, use pipe.
 
     $ cat list.txt | webarchive
 
+Note that, by default, this program logs all the URIs you enter into
+`~/.webarchive_history`.
+
 It has optional command-line parameters:
 
     $ webarchive -h
-    
+
     Usage: webarchive [options]
-        -w, --wait=SECS                  wait for SECS before sending a request
+        -w, --wait=SECONDS               wait for SECONDS between requests [default: 5.0]
+        -r, --retry=N                    retry for N times when failed [default: 5]
+        -t, --timeout=SECONDS            timeout after SECONDS [default: 60.0]
+            --[no-]history               record history [default: enabled]
         -d, --debug                      add debug output, implies verbose
             --verbose
-            --version
+        -h, --help                       show help
 
 ## Development
 
-After checking out the repo, run `bundle install` to install dependencies. Then, run `rake spec` to run the tests.
+After checking out the repo, run `bundle install` to install
+dependencies. Then, run `rake spec` to run the tests.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake
+install`. To release a new version, update the version number in
+`version.rb`, and then run `bundle exec rake release`, which will
+create a git tag for the version, push git commits and tags, and push
+the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome at https://gitlab.com/yusuke.matsubara/webarchive.
+Bug reports and pull requests are welcome at
+https://gitlab.com/yusuke.matsubara/webarchive.
 
 ## License
 

data/bin/webarchive

@@ -1,18 +1,39 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'webarchive'
 require 'webarchive/version'
 require 'optparse'
 
-wait = 4.0
+wait = 5.0
+retry_ = 5
+timeout = 60.0
 debug = false
 verbose = false
 history = true
 Version = WebArchive::VERSION
+
 OptionParser.new do |opt|
-  opt.on('-w', '--wait=SECS', 'wait for SECS before sending a request') { |v| wait = v.to_f }
-  opt.on('-d', '--debug', 'add debug output, implies verbose') { debug = true }
-  opt.on('--[no-]history', 'record history (enabled by default)') { |v| history = v }
+  opt.on('-w', '--wait=SECONDS',
+         "wait for SECONDS between requests [default: #{wait}]") do |v|
+    wait = v.to_f
+  end
+  opt.on('-r', '--retry=N',
+         "retry for N times when failed [default: #{retry_}]") do |v|
+    retry_ = v.to_i
+  end
+  opt.on('-t', '--timeout=SECONDS',
+         "timeout after SECONDS [default: #{timeout}]") do |v|
+    timeout = v.to_i
+  end
+  opt.on('--[no-]history',
+         "record history [default: #{history ? 'enabled' : 'disabled'}]") do |v|
+    history = v
+  end
+  opt.on('-d', '--debug', 'add debug output, implies verbose') do
+    debug = true
+    verbose = true
+  end
   opt.on('--verbose') { verbose = true }
   opt.on('-h', '--help', 'show help') do
     puts opt
@@ -20,5 +41,10 @@ OptionParser.new do |opt|
   end
 end.parse!(ARGV)
 
-warn "#{WebArchive} #{WebArchive::VERSION}" if verbose
-WebArchive.launch(wait_secs: wait, debug: debug, verbose: verbose, history: history)
+tagline = "#{WebArchive} #{WebArchive::VERSION}"
+tagline += ' (debug)' if debug
+warn tagline if verbose
+
+WebArchive.launch(wait_secs: wait, max_retry: retry_, history: history,
+                  read_timeout_secs: timeout,
+                  debug: debug, verbose: verbose)

data/lib/webarchive/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module WebArchive
-  VERSION = '0.1.3'
+  VERSION = '0.1.5'
 end

data/lib/webarchive.rb

@@ -1,6 +1,11 @@
 #! /usr/bin/env ruby
+# frozen_string_literal: true
 
-# (this file is also an executable - see the bottom)
+# This tool allows you to call multiple web archiving services. It
+# works from command-line interactively. In the inside, it maintains a
+# throttled queue for each service provider supported.
+
+# (This file itself is executable without installation - see the bottom.)
 
 require 'open-uri'
 require 'readline'
@@ -8,22 +13,40 @@ require 'tempfile'
 require 'simpleidn'
 require 'net/http'
 require 'addressable/uri'
+require 'digest/sha2'
 require 'mechanize'
 require 'trie'
+require 'concurrent'
 
 # classes and functions of webarchive package
 module WebArchive
+  Req = Struct.new("Req", :uri, :wait, :max_retry)
+
+  # error class for unexpected response from archiving service
+  class UnexpectedResponseError < StandardError
+    def initialize(cause = nil)
+      super(cause)
+    end
+  end
+
+  # error class for when redirect/canonical uri is not found
+  class NoAlternativeURIError < StandardError
+    def initialize(cause = nil)
+      super(cause)
+    end
+  end
+
   begin
     require 'libnotify'
-    def self.warn_archive_fail(uri, archiver, body)
-      warn "Not archived: #{uri} by #{archiver}; #{body}"
-      Libnotify.show(summary: "Not archived: #{uri} by #{archiver}",
+    def self.warn_archive_fail(req, archiver, body)
+      warn "Not archived: #{req} by #{archiver}; #{body}"
+      Libnotify.show(summary: "Not archived: #{req} by #{archiver}",
                      body: body, timeout: 3)
     end
   rescue LoadError
-    unless defined? warn_archive_fail
-      def self.warn_archive_fail(uri, archiver, body)
-        warn "Not archived: #{uri} by #{archiver}; #{body}"
+    unless respond_to? :warn_archive_fail
+      def self.warn_archive_fail(req, archiver, body)
+        warn "Not archived: #{req} by #{archiver}; #{body}"
       end
     end
   end
@@ -31,44 +54,183 @@ module WebArchive
   # Queue for sending URLs to a certain archiving web site
   # The block given to constructor will be executed for each '<<'
   class ArchiveQueue < Queue
-    def initialize(name, wait)
+    # Create a new instance of ArchiveQueue
+    # @param name [String]  name of the queue
+    # @param interval [Float]   length of the wait between requests
+    # @yield [String] URI that the queue receives
+    def initialize(name, interval)
       super()
       @name = name
+      @interval = interval
       @all_sent = false
-      @in_process = 0             # always <= 1
+      @in_process = Concurrent::AtomicFixnum.new(0)
+      last_request_time = Time.now - interval
       @consumer = Thread.new do
         loop do
-          uri = self.pop
-          @in_process += 1
+          req = self.deq        # deq blocks until non-empty
+          @in_process.value += 1
           begin
-            yield uri
+            sleep time_until_next_req(last_request_time, Time.now)
+            last_request_time = Time.now
+            yield req.uri
           rescue StandardError => e
-            WebArchive.warn_archive_fail(
-              uri, name, ([e.inspect] + e.backtrace).join("\n")
-            )
+            if retry?(e) && req.max_retry.positive?
+              buff = [].tap { |a| a << self.deq until self.empty? }
+              @in_process.value += buff.size + 1
+              Concurrent::ScheduledTask.execute(req.wait) do
+                @in_process.value -= buff.size + 1
+                self.enq Req.new(req.uri, req.wait * 2, req.max_retry - 1)
+                buff.each { |x| self.enq x }
+              end
+            else
+              WebArchive.warn_archive_fail(
+                req.uri, name, ([e.inspect] + e.backtrace).join("\n")
+              )
+            end
           ensure
-            @in_process -= 1
-            break if @all_sent && self.empty?
-
-            sleep wait
+            @in_process.value -= 1
+            break if @all_sent && self.remaining.zero?
           end
         end
       end
     end
 
+    def time_until_next_req(last_req, current)
+      elapsed = [current - last_req, 0].max
+      [@interval - elapsed, 0].max
+    end
+
+    # @param exc [Exception]
+    # @return [Boolean]
+    def retry?(exc)
+      [
+        Errno::ECONNRESET,
+        Errno::EHOSTUNREACH
+      ].include?(exc.class) ||
+        (exc.is_a?(OpenURI::HTTPError) && exc.message.start_with?('429 ')) ||
+        (exc.is_a?(OpenURI::HTTPError) && exc.message.start_with?('502 ')) ||
+        (exc.is_a?(OpenURI::HTTPError) && exc.message.start_with?('503 ')) ||
+        (exc.is_a?(Mechanize::ResponseCodeError) && exc.response_code == '503')
+    end
+
     # mark as 'sending done' and wait for items to be processed
+    # @return [Boolean]
     def done_sending
       @all_sent = true
-      @consumer.join if self.remaining > 0
+      @consumer.join if self.remaining.positive?
     end
 
     # number of queued items (including those being processed)
+    # @return [Integer]
     def remaining
-      self.size + @in_process
+      self.size + @in_process.value
     end
   end
 
-  def self.my_normalize(str)
+  # Client with multiple queues
+  class Client
+    def initialize(wait_secs: 1, max_retry: 3,
+                   redirect: false, canonical_uri: true)
+      @wait_secs = wait_secs
+      @max_retry = max_retry
+      @redirect = redirect
+      @canonical_uri = canonical_uri
+
+      @wait_secs = 0 if @wait_secs.negative?
+      @max_retry = 0 if @max_retry.negative?
+      @queues = []
+    end
+
+    # @param queue [ArchiveQueue]
+    def add_queue(queue)
+      @queues << queue
+    end
+
+    def queued_uris
+      @queues.map(&:remaining).inject(:+)
+    end
+
+    # @param uri [String]
+    # @return [Concurrent::Promises::Future] Gives the target URI if redirected
+    def with_redirect(uri)
+      Concurrent::Promises.future do
+        res = Net::HTTP.get_response(Addressable::URI.parse(uri))
+        raise NoAlternativeURIError, 'no redirect found' if
+          !res['location'] || res['location'] == uri
+
+        res['location']
+      end
+    end
+
+    def add_scheme(uri, scheme)
+      if uri.relative?
+        uri = uri.dup
+        uri.scheme = scheme
+      end
+      uri
+    end
+
+    def equivalent_uri?(uri, str)
+      uri = add_scheme(uri, Addressable::URI.parse(str).scheme)
+      uri.to_s == str
+    end
+
+    # @param uri [String]
+    # @return [Concurrent::Promises::Future] Gives the canonical URI if there is one
+    def with_canonical_uri(uri)
+      Concurrent::Promises.future do
+        agent = Mechanize.new
+        page = agent.get(uri)
+        ret = nil
+        raise NoAlternativeURIError, 'no canonical URI found' unless
+          page.canonical_uri &&
+          page.class == Mechanize::Page &&
+          page.canonical_uri != page.uri
+
+        if page.canonical_uri.relative?
+          u2 = URI.join(page.uri, page.canonical_uri)
+          ret = u2.to_s if !equivalent_uri?(u2, uri) &&
+                           !equivalent_uri?(u2, page.uri)
+        else
+          u1 = page.canonical_uri
+          u1 = add_scheme(u1, 'http') unless u1.scheme
+          ret = u1.to_s if !equivalent_uri?(u1, uri) &&
+                           !equivalent_uri?(u1, page.uri)
+        end
+
+        raise NoAlternativeURIError, 'no canonical URI found' unless ret
+
+        ret
+      end
+    end
+
+    # @param uri [String]
+    # @return [void]
+    def send_single_uri(uri)
+      @queues.each do |q|
+        q.enq Req.new(uri, @wait_secs, @max_retry)
+      end
+    end
+
+    # @param uri [String]
+    # @return [Concurrent::Promises::Future]
+    def send_uri(uri)
+      f0 = Concurrent::Promises.future{ send_single_uri(uri) }
+      f1 = with_canonical_uri(uri).then { |x| send_single_uri(x) } if @canonical_uri
+      f2 = with_redirect(uri).then { |x| send_single_uri(x) } if @redirect
+      f1 ||= Concurrent::Promises.future{}
+      f2 ||= Concurrent::Promises.future{}
+      f0.zip(f1).zip(f2)
+    end
+
+    def wait_for_queues
+      @queues.each(&:done_sending)
+    end
+  end
+
+  # @param str [String]
+  # @return [String]
+  def self.encode_non_ascii(str)
     if str =~ /[^[:ascii:]]/
       Addressable::URI.encode(str)
     else
@@ -76,51 +238,58 @@ module WebArchive
     end
   end
 
-  def self.to_ascii_uri(str)
-    uri = str.strip
-    if %r/\.[a-z]{2,4}(\/|$)/.match(uri) && !uri.include?('://') && !uri.start_with?('http')
-      uri = 'http://' + uri
+  # @param str [String]
+  # @return [String]
+  def self.prepend_http(uri)
+    if %r/\.[a-z]{2,4}(\/|$)/.match(uri) && %r{(^http|://)}.match(uri).nil?
+      'http://' + uri
+    else
+      uri
     end
+  end
 
+  # Encode non-ASCII components in the given string and make a URI instance from
+  # @param str [String]
+  # @return [Addressable::URI]
+  def self.to_ascii_uri(str)
+    uri = prepend_http(str.strip)
     u = Addressable::URI.parse(uri)
     u.host = SimpleIDN.to_ascii(u.host)
-    u.path = my_normalize(u.path)
-    u.query = my_normalize(u.query)
-    u.fragment = my_normalize(u.fragment)
+    u.path, u.query, u.fragment = [
+      u.path, u.query, u.fragment
+    ].map(&method(:encode_non_ascii))
     u
   end
 
-  def self.with_canonical_uri_and_redirect(uri, canonical, redirect)
-    if redirect
-      res = Net::HTTP.get_response(URI.parse(uri))
-      yield res['location'] if res['location'] &&
-                               res['location'] != uri
-    end
-    if canonical
-      agent = Mechanize.new
-      page = agent.get(uri)
-      yield page.canonical_uri.to_s if page.class == Mechanize::Page &&
-                                       page.canonical_uri &&
-                                       page.canonical_uri.to_s != uri &&
-                                       page.canonical_uri != page.uri
-    end
-  rescue Net::HTTPClientError, Mechanize::ResponseCodeError
-    # ignore since it will cause a warning later anyway
+  # Write log to a file
+  # @param source [String]
+  # @param content [String]
+  # @return [void]
+  def self.debug_output(source, uri, content)
+    ts = Time.now.strftime('%Y%m%d%H%M%S')
+    filename = "#{self}-#{source}-#{uri.gsub(/\W+/, '_')[0..30]}-"
+    filename += Digest::SHA256.hexdigest(uri + ts)[0..8]
+    Tempfile.open(filename) do |f|
+      f.puts content
+    end
   end
 
   # completer for URLs
   class Completer
+    # @param history_file [File]
     def initialize(history_file)
-      @file = File.expand_path(history_file)
+      @file = history_file
       @trie = Trie.new
-      self.reload
+      reload!
     end
 
-    def update
-      self.reload if File.stat(@file).mtime > @lastupdate
+    # @return [void]
+    def update!
+      reload! if File.stat(@file).mtime > @lastupdate
     end
 
-    def reload
+    # @return [void]
+    def reload!
       if File.exist? @file
         File.open(@file, encoding: 'utf-8').each_line do |x|
           @trie.add x.strip
@@ -131,13 +300,16 @@ module WebArchive
       @lastupdate = Time.now
     end
 
+    # @return [Proc]
     def to_proc
       proc do |s|
-        self.update
+        update!
         @trie.children(s)
       end
     end
 
+    # @param str [String]
+    # @return [void]
     def append_to_history(str)
       File.open(@file, mode: 'a', encoding: 'utf-8') do |f|
         f.puts str
@@ -147,65 +319,97 @@ module WebArchive
 
   HISTORY_FILE = '~/.webarchive_history'
 
-  def self.launch(wait_secs: 1, debug: false, verbose: false, redirect: false, canonical_uri: true, history: true)
-    verbose = true if debug
+  # Launch the CLI
+  # @return [Concurrent::Promises::Future]
+  def self.launch(wait_secs: 1, max_retry: 3,
+                  read_timeout_secs: 15,
+                  redirect: false, canonical_uri: true,
+                  history: true, debug: false, verbose: false)
+
+    args = method(__method__).parameters.map{ |k, v| [v, binding.local_variable_get(v)] }
+    puts "Arguments: #{args.inspect}" if verbose
+
+    wait_secs = 0 if wait_secs.negative?
+    max_retry = 0 if max_retry.negative?
+
     Thread.abort_on_exception = true
     completer = nil
     if history
-      completer = Completer.new(HISTORY_FILE)
+      completer = Completer.new(File.expand_path(HISTORY_FILE))
       Readline.completion_proc = completer.to_proc
       Readline.completion_append_character = ''
     end
 
-    queues = []
+    client = Client.new(wait_secs: wait_secs, max_retry: max_retry,
+                        redirect: redirect, canonical_uri: canonical_uri)
 
-    queues << ArchiveQueue.new('archive.org', wait_secs) do |uri|
-      URI.parse("https://web.archive.org/save/#{uri}").open do |f|
-        if f.meta['content-location'] && verbose
-          puts "<https://web.archive.org#{f.meta['content-location']}>"
-        elsif verbose
-          puts f.meta.inspect
+    # prepare queues
+    client.add_queue(
+      ArchiveQueue.new('archive.org (logged out)', wait_secs) do |uri|
+        u = URI.parse('https://web.archive.org/save/' + uri)
+        u.open(read_timeout: read_timeout_secs) do |f|
+          if f.meta['content-location'] && verbose
+            puts "<https://web.archive.org#{f.meta['content-location']}>"
+          elsif verbose
+            puts f.meta.inspect
+          end
         end
       end
-    end
+    )
 
-    queues << ArchiveQueue.new('megalodon.jp', wait_secs) do |uri|
-      agent = Mechanize.new
-      page = agent.get('https://megalodon.jp/pc/?' +
-                       Addressable::URI.form_encode(url: uri))
-      res = agent.submit(page.forms.first)
-      if debug
-        Tempfile.open("#{self}-#{uri.gsub(/\W/, '_')}") do |f|
-          f.puts res.body
+    client.add_queue(
+      ArchiveQueue.new('megalodon.jp', wait_secs) do |uri|
+        agent = Mechanize.new
+        agent.read_timeout = read_timeout_secs
+        page = agent.get('https://megalodon.jp/pc/?' +
+                         Addressable::URI.form_encode(url: uri))
+        form = page.forms.first
+        raise UnexpectedResponseError, page.inspect unless form
+
+        res = agent.submit(form)
+        if debug
+          debug_output('megalodonjp', uri, res.body)
         end
-      end
-      og = res.at('meta[property="og:url"]')
-      uri = if og
-              og[:content]
-            else
-              res.links.map(&:href).find(-> { res.uri.to_s }) do |x|
-                x =~ %r{megalodon\.jp/[\d-]+/}
+        og = res.at('meta[property="og:url"]')
+        uri = if og
+                og[:content]
+              else
+                res.links.map(&:href).find(-> { res.uri.to_s }) do |x|
+                  x =~ %r{megalodon\.jp/[\d-]+/}
+                end
               end
-            end
-      puts "<#{uri}>" if verbose
-      agent.shutdown
-    end
+        puts "<#{uri}>" if verbose
+        agent.shutdown
+      end
+    )
 
-    queues << ArchiveQueue.new('archive.today', wait_secs) do |uri|
-      agent = Mechanize.new
-      agent.follow_meta_refresh = true
+    client.add_queue(
+      ArchiveQueue.new('archive.today', wait_secs) do |uri|
+        agent = Mechanize.new
+        agent.read_timeout = read_timeout_secs
+        agent.follow_meta_refresh = true
 
-      page = agent.get('https://archive.today/')
-      form = page.form_with(id: 'submiturl')
-      form['anyway'] = '1'
-      form.field_with(name: 'url').value = uri
-      page = agent.submit(form)
-      puts "<#{page.uri}>" if verbose
-      agent.shutdown
-    end
+        page = agent.get('https://archive.is/')
+        form = page.form_with(id: 'submiturl')
+        if debug
+          debug_output('archivetoday', uri, page.inspect)
+        end
+        raise UnexpectedResponseError, page.inspect unless form
+
+        form['anyway'] = '1'
+        form.field_with(name: 'url').value = uri
+        sleep 5.0               # not submit too fast
+        page = agent.submit(form)
+        puts "<#{page.uri}>" if verbose
+        agent.shutdown
+      end
+    )
+
+    # main loop
 
     uri_regexp = URI::DEFAULT_PARSER.make_regexp
-    while line = Readline.readline("Q(#{queues.map(&:remaining).inject(:+)})> ", add_hist: true)
+    all = Concurrent::Promises.future{}
+    while line = Readline.readline("Q(#{client.queued_uris})> ", add_hist: true)
       uri = ''
       begin
         uri = to_ascii_uri(line).to_s
@@ -221,25 +425,22 @@ module WebArchive
         next
       end
 
-      queues.each do |q|
-        q << uri
-      end
-      begin
-        with_canonical_uri_and_redirect(uri, canonical_uri, redirect) do |x|
-          queues.each do |q|
-            q << x
-          end
-        end
-      rescue StandardError => e
-        warn "skipping canonical/redirect for #{uri}: #{e.message}"
-      end
-
-      completer.append_to_history(uri) if completer
+      f = client.send_uri(uri).then {
+        completer&.append_to_history(uri)
+      }.on_rejection { |reason, _|
+        warn "skipping canonical/redirect for #{uri}: #{reason}" if
+          !x.is_a?(NoAlternativeURIError)
+      }
+      all = all.zip(f)
     end
 
-    queues.each(&:done_sending)
+    all.wait
+    client.wait_for_queues
     # TODO: trap INT and ask for confirmation
+    all
   end
 end
 
-WebArchive.launch(wait_secs: 1.0, verbose: true, debug: true) if $PROGRAM_NAME == __FILE__
+if $PROGRAM_NAME == __FILE__
+  WebArchive.launch(wait_secs: 1.0, verbose: true, debug: true)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webarchive
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Yusuke Matsubara
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-30 00:00:00.000000000 Z
+date: 2025-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: fast_trie
   requirement: !ruby/object:Gem::Requirement
@@ -44,72 +58,114 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.6
+        version: 2.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.7.6
+        version: 2.14.0
 - !ruby/object:Gem::Dependency
   name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.0
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: simpleidn
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.70'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.70'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.24'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.24'
+- !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +179,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: pry-doc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -154,7 +210,7 @@ licenses:
 metadata:
   homepage_uri: https://rubygems.org/gems/webarchive
   source_code_uri: https://gitlab.com/yusuke.matsubara/webarchive
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -162,15 +218,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - "~>"
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '3.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key: 
+rubygems_version: 3.4.19
+signing_key:
 specification_version: 4
 summary: webarchive - CUI tool to archive URIs
 test_files: []