webarchive 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 90b93cbf35cd1c825adcac407f216e976c726b8e376074c3676411098a777ed4
+  data.tar.gz: c5015e92cb5eebaf598c1c9546197f82ff6362b0cd72213a5c62e5df7db8ffca
+SHA512:
+  metadata.gz: f1464df34ea47318fb724b3d1a31b169c2b60aa712e9f70615f0eb87bd33c0ef8c9079f62dad880782bd60274b5878037aad49ad835b1e880fccdf62aeddf2d3
+  data.tar.gz: b4d50991096eb19d66ef15229efe5fceb320eae76203ea03782651c385089fc3f9f12eeaf41e56d7c051db26ede108520aba6b50da4b6fbc7e70cdbcd7ccbbb0

data/bin/webarchive

@@ -0,0 +1,22 @@
+#! /usr/bin/env ruby
+
+require 'webarchive'
+require 'webarchive/version'
+require 'optparse'
+
+wait = 4.0
+debug = false
+verbose = false
+Version = WebArchive::VERSION
+OptionParser.new do |opt|
+  opt.on('-w', '--wait=SECS', 'wait for SECS before sending a request') { |v| wait = v.to_f }
+  opt.on('-d', '--debug', 'add debug output, implies verbose') { debug = true }
+  opt.on('--verbose') { verbose = true }
+  opt.on('--version') {
+    puts opt.ver
+    exit
+  }
+end.parse!(ARGV)
+
+warn "#{WebArchive} #{WebArchive::VERSION}" if verbose
+WebArchive.launch(wait_secs: wait, debug: debug, verbose: verbose)

data/lib/webarchive.rb

@@ -0,0 +1,236 @@
+#! /usr/bin/env ruby
+
+# (this file is also an executable - see the bottom)
+
+require 'open-uri'
+require 'readline'
+require 'tempfile'
+require 'simpleidn'
+require 'net/http'
+require 'addressable/uri'
+require 'mechanize'
+require 'trie'
+
+# classes and functions of webarchive package
+module WebArchive
+  begin
+    require 'libnotify'
+    def self.warn_archive_fail(uri, archiver, body)
+      warn "Not archived: #{uri} by #{archiver}; #{body}"
+      Libnotify.show(summary: "Not archived: #{uri} by #{archiver}",
+                     body: body, timeout: 3)
+    end
+  rescue LoadError
+    unless defined? warn_archive_fail
+      def self.warn_archive_fail(uri, archiver, body)
+        warn "Not archived: #{uri} by #{archiver}; #{body}"
+      end
+    end
+  end
+
+  # Queue for sending URLs to a certain archiving web site
+  # The block given to constructor will be executed for each '<<'
+  class ArchiveQueue < Queue
+    def initialize(name, wait)
+      super()
+      @name = name
+      @all_sent = false
+      @in_process = 0             # always <= 1
+      @consumer = Thread.new do
+        loop do
+          uri = self.pop
+          @in_process += 1
+          begin
+            yield uri
+          rescue StandardError => e
+            WebArchive.warn_archive_fail(uri, name, ([e.inspect] + e.backtrace).join("\n"))
+          ensure
+            @in_process -= 1
+            break if @all_sent && self.empty?
+
+            sleep wait
+          end
+        end
+      end
+    end
+
+    # mark as 'sending done' and wait for items to be processed
+    def done_sending
+      @all_sent = true
+      @consumer.join if self.remaining > 0
+    end
+
+    # number of queued items (including those being processed)
+    def remaining
+      self.size + @in_process
+    end
+  end
+
+  def self.my_normalize(str)
+    if str =~ /[^[:ascii:]]/
+      Addressable::URI.encode(str)
+    else
+      str
+    end
+  end
+
+  def self.to_ascii_uri(str)
+    uri = str.strip
+    if %r/\.[a-z]{2,4}(\/|$)/.match(uri) && !uri.include?('://') && !uri.start_with?('http')
+      uri = 'http://' + uri
+    end
+
+    u = Addressable::URI.parse(uri)
+    u.host = SimpleIDN.to_ascii(u.host)
+    u.path = my_normalize(u.path)
+    u.query = my_normalize(u.query)
+    u.fragment = my_normalize(u.fragment)
+    u
+  end
+
+  def self.with_canonical_uri_and_redirect(uri, canonical, redirect)
+    if redirect
+      res = Net::HTTP.get_response(URI.parse(uri))
+      yield res['location'] if res['location'] &&
+                               res['location'] != uri
+    end
+    if canonical
+      agent = Mechanize.new
+      page = agent.get(uri)
+      yield page.canonical_uri.to_s if page.class == Mechanize::Page &&
+                                       page.canonical_uri &&
+                                       page.canonical_uri.to_s != uri &&
+                                       page.canonical_uri != page.uri
+    end
+  rescue Net::HTTPClientError, Mechanize::ResponseCodeError
+    # ignore since it will cause a warning later anyway
+  end
+
+  # completer for URLs
+  class Completer
+    def initialize(history_file)
+      @file = File.expand_path(history_file)
+      @trie = Trie.new
+      self.reload
+    end
+
+    def update
+      self.reload if File.stat(@file).mtime > @lastupdate
+    end
+
+    def reload
+      if File.exist? @file
+        File.open(@file, encoding: 'utf-8').each_line do |x|
+          @trie.add x.strip
+        end
+      else
+        File.open(@file, 'w', encoding: 'utf-8') do |f|
+        end
+      end
+      @lastupdate = Time.now
+    end
+
+    def to_proc
+      proc do |s|
+        self.update
+        @trie.children(s)
+      end
+    end
+  end
+
+  HISTORY_FILE = '~/.webarchive.rb_history'
+
+  def self.launch(wait_secs: 1, debug: false, verbose: false, redirect: false, canonical_uri: true)
+    verbose = true if debug
+    Thread.abort_on_exception = true
+    Readline.completion_proc = Completer.new(HISTORY_FILE).to_proc
+    Readline.completion_append_character = ''
+
+    queues = []
+
+    queues << ArchiveQueue.new('archive.org', wait_secs) do |uri|
+      URI.parse("https://web.archive.org/save/#{uri}").open do |f|
+        if f.meta['content-location'] && verbose
+          puts "<https://web.archive.org#{f.meta['content-location']}>"
+        elsif verbose
+          puts f.meta.inspect
+        end
+      end
+    end
+
+    queues << ArchiveQueue.new('megalodon.jp', wait_secs) do |uri|
+      agent = Mechanize.new
+      page = agent.get('https://megalodon.jp/pc/?' +
+                       Addressable::URI.form_encode(url: uri))
+      res = agent.submit(page.forms.first)
+      if debug
+        Tempfile.open("#{self}-#{uri.gsub(/\W/, '_')}") do |f|
+          f.puts res.body
+        end
+      end
+      og = res.at('meta[property="og:url"]')
+      uri = if og
+              og[:content]
+            else
+              res.links.map(&:href).find(-> { res.uri.to_s }) do |x|
+                x =~ %r{megalodon\.jp/[\d-]+/}
+              end
+            end
+      puts "<#{uri}>" if verbose
+      agent.shutdown
+    end
+
+    queues << ArchiveQueue.new('archive.today', wait_secs) do |uri|
+      agent = Mechanize.new
+      agent.follow_meta_refresh = true
+
+      page = agent.get('https://archive.today/')
+      form = page.form_with(id: 'submiturl')
+      form['anyway'] = '1'
+      form.field_with(name: 'url').value = uri
+      page = agent.submit(form)
+      puts "<#{page.uri}>" if verbose
+      agent.shutdown
+    end
+
+    uri_regexp = URI::DEFAULT_PARSER.make_regexp
+    while line = Readline.readline("Q(#{queues.map(&:remaining).inject(:+)})> ", add_hist: true)
+      uri = ''
+      begin
+        uri = to_ascii_uri(line).to_s
+      rescue Addressable::URI::InvalidURIError => e
+        warn_archive_fail(line.strip, '<>', e.message)
+      end
+      next if uri == ''
+
+      puts uri if verbose
+
+      if uri !~ uri_regexp
+        warn "invalid; skipping '#{uri}'"
+        next
+      end
+
+      queues.each do |q|
+        q << uri
+      end
+      begin
+        with_canonical_uri_and_redirect(uri, canonical_uri, redirect) do |x|
+          queues.each do |q|
+            q << x
+          end
+        end
+      rescue StandardError => e
+        warn "skipping canonical/redirect for #{uri}: #{e.message}"
+      end
+
+      File.open(File.expand_path(HISTORY_FILE), mode: 'a', encoding: 'utf-8') do |f|
+        f.puts uri
+      end
+    end
+
+    queues.each(&:done_sending)
+    # TODO: trap INT and ask for confirmation
+  end
+end
+
+WebArchive.launch(wait_secs: 1.0, verbose: true, debug: true) if $PROGRAM_NAME == __FILE__

data/lib/webarchive/version.rb

@@ -0,0 +1,3 @@
+module WebArchive
+  VERSION = '0.1.0'
+end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: webarchive
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yusuke Matsubara
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-06-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+- !ruby/object:Gem::Dependency
+  name: fast_trie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.1
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+- !ruby/object:Gem::Dependency
+  name: simpleidn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: CUI tool to archive URIs using web.archive.org, archive.today, and others
+email: whym@whym.org
+executables:
+- webarchive
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/webarchive
+- lib/webarchive.rb
+- lib/webarchive/version.rb
+homepage: https://rubygems.org/gems/whym
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://rubygems.org/gems/whym
+  source_code_uri: https://gitlab.com/yusuke.matsubara/webarchive
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: webarchive - CUI tool to archive URIs
+test_files: []