wayback_machine_downloader 0.1.18 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a37e6ffa46aae434f606fd8d887c2017a0a06a1
-  data.tar.gz: c05f60c12fec76ec3709371a31fff7f5de17782f
+  metadata.gz: eccabaecb58ce40f79f3bfcbac3a50eea4c55142
+  data.tar.gz: fd5673c7ee5404f8f73b73de9bf94c5b1e2d08ea
 SHA512:
-  metadata.gz: 24a3096abfdc4506a873bb784380d169d1a0b45c39f5c1f91538f7c1f39d67d86b0a7d8f74103406b35d4718ab732a2ca442ee8bcbc27caf7bb3db2406530130
-  data.tar.gz: c19c30ccf76b8993e054aff2dabadb000e2b4fe90eff5ae61e93a170ef2aae79fc543b699e78b1df3df23c7b0ef1bc625b55da556943de9554aee3e732aa58a1
+  metadata.gz: 8db03676ee13a095e18d24ba98512852d00926d6142b4cba12b8053519384c9e96ca49ab542d45b310390248772db973b252a131937890a5ed3646c908a4d7a0
+  data.tar.gz: 8646cea23b376bfb80019ce670badeb5746d4cd2c8010e074f7157314e9112cece9845903a8c7ee00bb5f1bafaefbb5a05f700101245872f1070018f4ac65fd2

data/bin/wayback_machine_downloader

@@ -2,6 +2,7 @@
 
 require_relative '../lib/wayback_machine_downloader'
 require 'optparse'
+require 'pp'
 
 options = {}
 option_parser = OptionParser.new do |opts|
@@ -11,19 +12,23 @@ option_parser = OptionParser.new do |opts|
   opts.separator "Download any website from the Wayback Machine."
 
   opts.separator ""
-  opts.separator "Optional option:"
+  opts.separator "Optional options:"
 
   opts.on("-t", "--timestamp TIMESTAMP", Integer, "Only files on or before timestamp supplied (ie. 20150806225358)") do |t|
     options[:timestamp] = t
   end
 
+  opts.on("-o", "--only ONLY_FILTER", String, "Restrict downloading to file urls matching the only filter supplied (use // notation for the only filter to be treated as a regex)") do |t|
+    options[:only_filter] = t
+  end
+
   opts.on("-v", "--version", "Display version") do |t|
     options[:version] = t
   end
 end.parse!
 
-if base_url = ARGV[0]
-  wayback_machine_downloader = WaybackMachineDownloader.new base_url: base_url, timestamp: options[:timestamp]
+if base_url = ARGV[-1]
+  wayback_machine_downloader = WaybackMachineDownloader.new base_url: base_url, timestamp: options[:timestamp], only_filter: options[:only_filter]
   wayback_machine_downloader.download_files
 elsif options[:version]
   puts WaybackMachineDownloader::VERSION

data/lib/wayback_machine_downloader.rb

@@ -1,16 +1,20 @@
+# encoding: UTF-8
+
 require 'open-uri'
 require 'fileutils'
 require_relative 'wayback_machine_downloader/tidy_bytes'
+require_relative 'wayback_machine_downloader/to_regex'
 
 class WaybackMachineDownloader
 
-  VERSION = "0.1.18"
+  VERSION = "0.2.0"
 
-  attr_accessor :base_url, :timestamp
+  attr_accessor :base_url, :timestamp, :only_filter
 
   def initialize params
     @base_url = params[:base_url]
     @timestamp = params[:timestamp].to_i
+    @only_filter = params[:only_filter]
   end
 
   def backup_name
@@ -21,8 +25,21 @@ class WaybackMachineDownloader
     'websites/' + backup_name + '/'
   end
 
+  def match_only_filter file_url
+    if @only_filter
+      only_filter_regex = @only_filter.to_regex
+      if only_filter_regex
+        only_filter_regex =~ file_url
+      else
+        file_url.downcase.include? @only_filter.downcase
+      end
+    else
+      true
+    end
+  end
+
   def get_file_list_curated
-    index_file_list_raw =  open "http://web.archive.org/cdx/search/xd?url=#{@base_url}"
+    index_file_list_raw = open "http://web.archive.org/cdx/search/xd?url=#{@base_url}"
     all_file_list_raw = open "http://web.archive.org/cdx/search/xd?url=#{@base_url}/*"
     file_list_curated = Hash.new
     [index_file_list_raw, all_file_list_raw].each do |file|
@@ -36,7 +53,9 @@ class WaybackMachineDownloader
         if file_id.nil?
           puts "Malformed file url, ignoring: #{file_url}"
         elsif @timestamp == 0 or file_timestamp <= @timestamp
-          if file_list_curated[file_id]
+          if not match_only_filter(file_url)
+            puts "File url not in supplied only filter, ignoring: #{file_url}"
+          elsif file_list_curated[file_id]
             unless file_list_curated[file_id][:timestamp] > file_timestamp
               file_list_curated[file_id] = {file_url: file_url, timestamp: file_timestamp}
             end
@@ -62,6 +81,10 @@ class WaybackMachineDownloader
     puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine..."
     puts
     file_list_by_timestamp = get_file_list_by_timestamp
+    if file_list_by_timestamp.count == 0
+      puts "No files to download. Possible reasons:\n\t* Accept regex didn't let any files through (Accept Regex: \"#{@accept_regex.to_s}\")\n\t* Site is not in wayback machine."
+      return
+    end
     count = 0
     file_list_by_timestamp.each do |file_remote_info|
       count += 1

data/lib/wayback_machine_downloader/to_regex.rb

@@ -0,0 +1,81 @@
+module ToRegex
+  module StringMixin
+    class << self
+      def literal?(str)
+        REGEXP_DELIMITERS.none? { |s, e| str.start_with?(s) and str =~ /#{e}#{INLINE_OPTIONS}\z/ }
+      end
+    end
+
+    INLINE_OPTIONS = /[imxnesu]*/
+    REGEXP_DELIMITERS = {
+      '%r{' => '}',
+      '/' => '/',
+    }
+
+    # Get a regex back
+    #
+    # Without :literal or :detect, `"foo".to_regex` will return nil.
+    #
+    # @param [optional, Hash] options
+    # @option options [true,false] :literal Treat meta characters and other regexp codes as just text; always return a regexp
+    # @option options [true,false] :detect If string starts and ends with valid regexp delimiters, treat it as a regexp; otherwise, interpret it literally
+    # @option options [true,false] :ignore_case /foo/i
+    # @option options [true,false] :multiline /foo/m
+    # @option options [true,false] :extended /foo/x
+    # @option options [true,false] :lang /foo/[nesu]
+    def to_regex(options = {})
+      if args = as_regexp(options)
+        ::Regexp.new *args
+      end
+    end
+
+    # Return arguments that can be passed to `Regexp.new`
+    # @see to_regexp
+    def as_regexp(options = {})
+      unless options.is_a?(::Hash)
+        raise ::ArgumentError, "[to_regexp] Options must be a Hash"
+      end
+      str = self
+
+      return if options[:detect] and str == ''
+
+      if options[:literal] or (options[:detect] and ToRegexp::String.literal?(str))
+        content = ::Regexp.escape str
+      elsif delim_set = REGEXP_DELIMITERS.detect { |k, _| str.start_with?(k) }
+        delim_start, delim_end = delim_set
+        /\A#{delim_start}(.*)#{delim_end}(#{INLINE_OPTIONS})\z/u =~ str
+        content = $1
+        inline_options = $2
+        return unless content.is_a?(::String)
+        content.gsub! '\\/', '/'
+        if inline_options
+          options[:ignore_case] = true if inline_options.include?('i')
+          options[:multiline] = true if inline_options.include?('m')
+          options[:extended] = true if inline_options.include?('x')
+          # 'n', 'N' = none, 'e', 'E' = EUC, 's', 'S' = SJIS, 'u', 'U' = UTF-8
+          options[:lang] = inline_options.scan(/[nesu]/i).join.downcase
+        end
+      else
+        return
+      end
+
+      ignore_case = options[:ignore_case] ? ::Regexp::IGNORECASE : 0
+      multiline = options[:multiline] ? ::Regexp::MULTILINE : 0
+      extended = options[:extended] ? ::Regexp::EXTENDED : 0
+      lang = options[:lang] || ''
+      if ::RUBY_VERSION > '1.9' and lang.include?('u')
+        lang = lang.delete 'u'
+      end
+
+      if lang.empty?
+        [ content, (ignore_case|multiline|extended) ]
+      else
+        [ content, (ignore_case|multiline|extended), lang ]
+      end
+    end
+  end
+end
+
+class String
+  include ToRegex::StringMixin
+end

metadata

@@ -1,43 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.2.0
 platform: ruby
 authors:
 - hartator
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-05 00:00:00.000000000 Z
+date: 2015-11-19 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: pry-rescue
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.4'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.4'
-- !ruby/object:Gem::Dependency
-  name: pry-stack_explorer
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.4'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +50,7 @@ files:
 - bin/wayback_machine_downloader
 - lib/wayback_machine_downloader.rb
 - lib/wayback_machine_downloader/tidy_bytes.rb
+- lib/wayback_machine_downloader/to_regex.rb
 homepage: https://github.com/hartator/wayback-machine-downloader
 licenses:
 - MIT