wayback_machine_downloader 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/wayback_machine_downloader +9 -1
- data/lib/wayback_machine_downloader.rb +12 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f4788fe4e8cf006c42912383dae0bb64605827df
|
|
4
|
+
data.tar.gz: 485e59c366e796bfb06a85e440394d8504226bf6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b7b6e02f7f406f7be9294086fdec552082c297e2f7165bf03b33c84f463d09c4cdf4ade6d3c18a04d80d1214753a576a4a5eae9ac4e6bdd1f5a70ec8aa422c94
|
|
7
|
+
data.tar.gz: 1a1b634c00527d47bc52befa38ec38f4d58b64db4de4fcbb50d88fe2afac0476725563879ba596aa7d3fbab22d60f54276e4ad6d947c80cd04fa3ff377747042
|
|
@@ -34,6 +34,10 @@ option_parser = OptionParser.new do |opts|
|
|
|
34
34
|
options[:all] = true
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
opts.on("-l", "--list", "Only list file urls in a JSON format with the archived timestamps. Won't download anything.") do |t|
|
|
38
|
+
options[:list] = true
|
|
39
|
+
end
|
|
40
|
+
|
|
37
41
|
opts.on("-v", "--version", "Display version") do |t|
|
|
38
42
|
options[:version] = t
|
|
39
43
|
end
|
|
@@ -42,7 +46,11 @@ end.parse!
|
|
|
42
46
|
if (base_url = ARGV[-1])
|
|
43
47
|
options[:base_url] = base_url
|
|
44
48
|
wayback_machine_downloader = WaybackMachineDownloader.new options
|
|
45
|
-
wayback_machine_downloader.
|
|
49
|
+
if wayback_machine_downloader.list
|
|
50
|
+
wayback_machine_downloader.list_files
|
|
51
|
+
else
|
|
52
|
+
wayback_machine_downloader.download_files
|
|
53
|
+
end
|
|
46
54
|
elsif options[:version]
|
|
47
55
|
puts WaybackMachineDownloader::VERSION
|
|
48
56
|
else
|
|
@@ -3,14 +3,15 @@
|
|
|
3
3
|
require 'open-uri'
|
|
4
4
|
require 'fileutils'
|
|
5
5
|
require 'cgi'
|
|
6
|
+
require 'json'
|
|
6
7
|
require_relative 'wayback_machine_downloader/tidy_bytes'
|
|
7
8
|
require_relative 'wayback_machine_downloader/to_regex'
|
|
8
9
|
|
|
9
10
|
class WaybackMachineDownloader
|
|
10
11
|
|
|
11
|
-
VERSION = "0.4.
|
|
12
|
+
VERSION = "0.4.4"
|
|
12
13
|
|
|
13
|
-
attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all
|
|
14
|
+
attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all, :list
|
|
14
15
|
|
|
15
16
|
def initialize params
|
|
16
17
|
@base_url = params[:base_url]
|
|
@@ -19,6 +20,7 @@ class WaybackMachineDownloader
|
|
|
19
20
|
@only_filter = params[:only_filter]
|
|
20
21
|
@exclude_filter = params[:exclude_filter]
|
|
21
22
|
@all = params[:all]
|
|
23
|
+
@list = params[:list]
|
|
22
24
|
end
|
|
23
25
|
|
|
24
26
|
def backup_name
|
|
@@ -106,6 +108,14 @@ class WaybackMachineDownloader
|
|
|
106
108
|
end
|
|
107
109
|
end
|
|
108
110
|
|
|
111
|
+
def list_files
|
|
112
|
+
puts "["
|
|
113
|
+
get_file_list_by_timestamp.each do |file|
|
|
114
|
+
puts file.to_json + ","
|
|
115
|
+
end
|
|
116
|
+
puts "]"
|
|
117
|
+
end
|
|
118
|
+
|
|
109
119
|
def download_files
|
|
110
120
|
puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine..."
|
|
111
121
|
puts
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: wayback_machine_downloader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.4.
|
|
4
|
+
version: 0.4.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- hartator
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-08-
|
|
11
|
+
date: 2016-08-03 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|