wayback_machine_downloader 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f13c4b2fd341ad88d3962eb775435ab41ddabb43
-  data.tar.gz: 25b0d71d96dbeafd1e055a5d235544a1a23fd452
+  metadata.gz: f4dcc90f3f3f974aa5c894a6fda9e8158a7d8a2b
+  data.tar.gz: 560460143036a84357328e9b4dfba09a8b78e9d8
 SHA512:
-  metadata.gz: 3b97a0422f514ca4026fd1235bb60103316f943906187ce50578c033c4e350820d0eab24c19db09198d40c16e44a6713431178c28ff962d2627b30c0e8d317e8
-  data.tar.gz: 858cb6cbb51f32736d0e1f4496b555ae1d8fa15d97ce7d7f4cb3616115b6ee93cb0c2751f4b6d0cd1428c50109b461d2bbf503be4455bff99cef769f23d7b358
+  metadata.gz: 3808d953b979cc18a5efd0a0875b96e988649141b3591bfa3cf7eca96d6a06ed218c6d6418e8f3c20d12aff0de89fc10773d7381eb1a07f53aeea79816abe0aa
+  data.tar.gz: 1fd31782e408ae274546204033f7960accd8daeb79c5d09d62d53974298de717baf2c5c7e49f962e7a0101267c15a666c82941d7a9bd619bc7cd9d97e23e4343

data/bin/wayback_machine_downloader

@@ -14,6 +14,10 @@ option_parser = OptionParser.new do |opts|
   opts.separator ""
   opts.separator "Optional options:"
 
+  opts.on("-d", "--directory PATH", String, "Directory to save the downloaded files to. Default is ./websites/ plus the domain name.") do |t|
+    options[:directory] = t
+  end
+
   opts.on("-f", "--from TIMESTAMP", Integer, "Only files on or after timestamp supplied (ie. 20060716231334)") do |t|
     options[:from_timestamp] = t
   end
@@ -34,14 +38,14 @@ option_parser = OptionParser.new do |opts|
     options[:all] = true
   end
 
-  opts.on("-l", "--list", "Only list file urls in a JSON format with the archived timestamps. Won't download anything.") do |t|
-    options[:list] = true
-  end
-
   opts.on("-c", "--concurrency NUMBER", Integer, "Number of multiple files to dowload at a time. Default is one file at a time. (ie. 20)") do |t|
     options[:threads_count] = t
   end
 
+  opts.on("-l", "--list", "Only list file urls in a JSON format with the archived timestamps. Won't download anything.") do |t|
+    options[:list] = true
+  end
+
   opts.on("-v", "--version", "Display version") do |t|
     options[:version] = t
   end

data/lib/wayback_machine_downloader.rb

@@ -1,20 +1,25 @@
 # encoding: UTF-8
 
+require 'net/http'
 require 'open-uri'
 require 'fileutils'
 require 'cgi'
 require 'json'
 require_relative 'wayback_machine_downloader/tidy_bytes'
 require_relative 'wayback_machine_downloader/to_regex'
+require_relative 'wayback_machine_downloader/archive_api'
 
 class WaybackMachineDownloader
 
-  VERSION = "0.5.3"
+  include ArchiveAPI
 
-  attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all, :list, :threads_count
+  VERSION = "0.5.4"
+
+  attr_accessor :base_url, :directory, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all, :list, :threads_count
 
   def initialize params
     @base_url = params[:base_url]
+    @directory = params[:directory]
     @from_timestamp = params[:from_timestamp].to_i
     @to_timestamp = params[:to_timestamp].to_i
     @only_filter = params[:only_filter]
@@ -33,7 +38,15 @@ class WaybackMachineDownloader
   end
 
   def backup_path
-    'websites/' + backup_name + '/'
+    if @directory
+      if @directory[-1] == '/'
+        @directory
+      else
+        @directory + '/'
+      end
+    else
+      'websites/' + backup_name + '/'
+    end
   end
 
   def match_only_filter file_url
@@ -63,18 +76,8 @@ class WaybackMachineDownloader
   end
 
   def get_file_list_curated
-    parameters_for_wayback_machine_api = "&fl=timestamp,original&collapse=original"
-    unless @all
-      parameters_for_wayback_machine_api += "&filter=statuscode:200"
-    end
-    if @from_timestamp and @from_timestamp != 0
-      parameters_for_wayback_machine_api += "&from=" + @from_timestamp.to_s
-    end
-    if @to_timestamp and @to_timestamp != 0
-      parameters_for_wayback_machine_api += "&to=" + @to_timestamp.to_s
-    end
-    index_file_list_raw = open("http://web.archive.org/cdx/search/xd?url=#{@base_url}" + parameters_for_wayback_machine_api)
-    all_file_list_raw = open("http://web.archive.org/cdx/search/xd?url=#{@base_url}/*" + parameters_for_wayback_machine_api)
+    index_file_list_raw = get_raw_list_from_api(@base_url)
+    all_file_list_raw = get_raw_list_from_api(@base_url + '/*')
     file_list_curated = Hash.new
     [index_file_list_raw, all_file_list_raw].each do |file|
       file.each_line do |line|

data/lib/wayback_machine_downloader/archive_api.rb

@@ -0,0 +1,26 @@
+module ArchiveAPI
+
+	def get_raw_list_from_api url
+		request_url = "http://web.archive.org/cdx/search/xd?url="
+		request_url += url
+		request_url += parameters_for_api
+		request_uri = URI.parse request_url
+		response = Net::HTTP.get_response request_uri
+		response.body
+	end
+
+	def parameters_for_api
+		parameters = "&fl=timestamp,original&collapse=original&gzip=false"
+    unless @all
+      parameters += "&filter=statuscode:200"
+    end
+    if @from_timestamp and @from_timestamp != 0
+      parameters += "&from=" + @from_timestamp.to_s
+    end
+    if @to_timestamp and @to_timestamp != 0
+      parameters += "&to=" + @to_timestamp.to_s
+    end
+    parameters
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - hartator
@@ -50,6 +50,7 @@ extra_rdoc_files: []
 files:
 - bin/wayback_machine_downloader
 - lib/wayback_machine_downloader.rb
+- lib/wayback_machine_downloader/archive_api.rb
 - lib/wayback_machine_downloader/tidy_bytes.rb
 - lib/wayback_machine_downloader/to_regex.rb
 homepage: https://github.com/hartator/wayback-machine-downloader