epitools 0.5.98 → 0.5.99
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/epitools/autoloads.rb +2 -1
- data/lib/epitools/browser.rb +26 -25
- data/lib/epitools/browser/cache.rb +46 -46
- data/lib/epitools/clitools.rb +53 -37
- data/lib/epitools/core_ext/enumerable.rb +29 -0
- data/lib/epitools/core_ext/matrix.rb +3 -3
- data/lib/epitools/core_ext/numbers.rb +36 -32
- data/lib/epitools/minimal.rb +3 -0
- data/lib/epitools/path.rb +42 -7
- data/spec/browser_spec.rb +7 -6
- data/spec/clitools_spec.rb +36 -20
- data/spec/core_ext_spec.rb +7 -7
- data/spec/path_spec.rb +3 -4
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0cbf9aff2ad00720845a3bce7605343ff4b0a88e
|
4
|
+
data.tar.gz: cb8e34cb49a1bb091d60fcfecdfdf1b5006915ac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0fa9bcbf1d0275b21648d52ee0959250e517861e8b1a4b8acc9b44eba3b9e259d2ac3df83876af1736ae0ac8a564435662da7254bceb9c4f49340e611ac75e14
|
7
|
+
data.tar.gz: 6e51f33d5942756afaa8b90089a5657ed252940c59b4c95f6fb92245faed39ca59ebed6187064d0f287490fbb33099b76afb67481a5980efa61ce0aa7b76beb6
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.99
|
data/lib/epitools/autoloads.rb
CHANGED
@@ -17,8 +17,9 @@ autoload :Timeout, 'timeout'
|
|
17
17
|
autoload :Find, 'find'
|
18
18
|
autoload :Benchmark, 'benchmark'
|
19
19
|
autoload :Tracer, 'tracer'
|
20
|
-
autoload :CSV, 'csv'
|
21
20
|
autoload :Shellwords, 'shellwords'
|
21
|
+
autoload :PTY, 'pty'
|
22
|
+
autoload :CSV, 'csv'
|
22
23
|
autoload :SDBM, 'sdbm'
|
23
24
|
|
24
25
|
module Digest
|
data/lib/epitools/browser.rb
CHANGED
@@ -18,7 +18,7 @@ class Mechanize::File
|
|
18
18
|
response['content-type']
|
19
19
|
end
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
|
23
23
|
#
|
24
24
|
# A mechanize class that emulates a web-browser, with cache and everything.
|
@@ -43,8 +43,9 @@ class Browser
|
|
43
43
|
@use_cache = !!(options[:cache] || options[:cached] || options[:use_cache])
|
44
44
|
@use_logs = options[:logs] || false
|
45
45
|
@cookie_file = options[:cookiefile] || "cookies.txt"
|
46
|
-
|
47
|
-
|
46
|
+
@cache_file = options[:cache_file] || "browser-cache.db"
|
47
|
+
|
48
|
+
# TODO: @progress, @user_agent, @logfile, @cache_file (default location: ~/.epitools?)
|
48
49
|
|
49
50
|
if options[:proxy]
|
50
51
|
host, port = options[:proxy].split(':')
|
@@ -59,7 +60,7 @@ class Browser
|
|
59
60
|
def init_agent!
|
60
61
|
@agent = Mechanize.new do |a|
|
61
62
|
# ["Mechanize", "Mac Mozilla", "Linux Mozilla", "Windows IE 6", "iPhone", "Linux Konqueror", "Windows IE 7", "Mac FireFox", "Mac Safari", "Windows Mozilla"]
|
62
|
-
a.max_history = 10
|
63
|
+
a.max_history = 10
|
63
64
|
a.user_agent_alias = "Windows IE 7"
|
64
65
|
a.log = Logger.new "mechanize.log" if @use_logs
|
65
66
|
end
|
@@ -80,7 +81,7 @@ class Browser
|
|
80
81
|
|
81
82
|
def init_cache!
|
82
83
|
# TODO: Rescue "couldn't load" exception and disable caching
|
83
|
-
@cache = Cache.new(agent) if @use_cache
|
84
|
+
@cache = Cache.new(@cache_file, agent) if @use_cache
|
84
85
|
end
|
85
86
|
|
86
87
|
def load_cookies!
|
@@ -91,23 +92,23 @@ class Browser
|
|
91
92
|
false
|
92
93
|
end
|
93
94
|
end
|
94
|
-
|
95
|
+
|
95
96
|
def save_cookies!
|
96
97
|
agent.cookie_jar.save_as @cookie_file
|
97
98
|
true
|
98
99
|
end
|
99
|
-
|
100
|
+
|
100
101
|
def relative?(url)
|
101
102
|
not url[ %r{^https?://} ]
|
102
103
|
end
|
103
|
-
|
104
|
+
|
104
105
|
def cacheable?(page)
|
105
106
|
case page.content_type
|
106
107
|
when %r{^(text|application)}
|
107
108
|
true
|
108
109
|
end
|
109
|
-
end
|
110
|
-
|
110
|
+
end
|
111
|
+
|
111
112
|
def cache_put(page, url)
|
112
113
|
if cache.valid_page?(page)
|
113
114
|
if page.content_type =~ %r{(^text/|^application/javascript|javascript)}
|
@@ -117,9 +118,9 @@ class Browser
|
|
117
118
|
end
|
118
119
|
end
|
119
120
|
|
120
|
-
|
121
|
+
|
121
122
|
#
|
122
|
-
# Retrieve an URL, and return a Mechanize::Page instance (which acts a
|
123
|
+
# Retrieve an URL, and return a Mechanize::Page instance (which acts a
|
123
124
|
# bit like a Nokogiri::HTML::Document instance.)
|
124
125
|
#
|
125
126
|
# Options:
|
@@ -128,7 +129,7 @@ class Browser
|
|
128
129
|
def get(url, options={})
|
129
130
|
|
130
131
|
# TODO: Have a base-URL option
|
131
|
-
|
132
|
+
|
132
133
|
#if relative?(url)
|
133
134
|
# url = URI.join("http://base-url/", url).to_s
|
134
135
|
#end
|
@@ -140,13 +141,13 @@ class Browser
|
|
140
141
|
|
141
142
|
puts
|
142
143
|
puts "[ GET #{url} (using cache: #{!!use_cache}) ]"
|
143
|
-
|
144
|
+
|
144
145
|
delay unless cached_already
|
145
146
|
max_retries = 4
|
146
147
|
retries = 0
|
147
148
|
|
148
149
|
begin
|
149
|
-
|
150
|
+
|
150
151
|
if use_cache and page = cache.get(url)
|
151
152
|
puts " |_ cached (#{page.content_type})"
|
152
153
|
else
|
@@ -165,11 +166,11 @@ class Browser
|
|
165
166
|
|
166
167
|
puts " |_ ERROR: #{e.inspect} -- retrying"
|
167
168
|
delay(5)
|
168
|
-
retry
|
169
|
-
|
170
|
-
=begin
|
169
|
+
retry
|
170
|
+
|
171
|
+
=begin
|
171
172
|
rescue Mechanize::ResponseCodeError => e
|
172
|
-
|
173
|
+
|
173
174
|
case e.response_code
|
174
175
|
when "401" #=> Net::HTTPUnauthorized
|
175
176
|
p e
|
@@ -193,7 +194,7 @@ class Browser
|
|
193
194
|
page
|
194
195
|
end
|
195
196
|
|
196
|
-
|
197
|
+
|
197
198
|
#
|
198
199
|
# Delegate certain methods to @agent
|
199
200
|
#
|
@@ -202,7 +203,7 @@ class Browser
|
|
202
203
|
agent.send(meth, *args)
|
203
204
|
end
|
204
205
|
end
|
205
|
-
|
206
|
+
|
206
207
|
end
|
207
208
|
|
208
209
|
|
@@ -217,11 +218,11 @@ class BrowserOptions < OpenStruct
|
|
217
218
|
:use_logs => false,
|
218
219
|
:cookie_file => "cookies.txt"
|
219
220
|
}
|
220
|
-
|
221
|
+
|
221
222
|
def initialize(extra_opts)
|
222
|
-
|
223
|
+
|
223
224
|
@opts = DEFAULTS.dup
|
224
|
-
|
225
|
+
|
225
226
|
for key, val in opts
|
226
227
|
if key.in? DEFAULTS
|
227
228
|
@opts[key] = val
|
@@ -230,7 +231,7 @@ class BrowserOptions < OpenStruct
|
|
230
231
|
end
|
231
232
|
end
|
232
233
|
end
|
233
|
-
|
234
|
+
|
234
235
|
end
|
235
236
|
=end
|
236
237
|
|
@@ -2,51 +2,51 @@ require 'mechanize'
|
|
2
2
|
require 'sqlite3'
|
3
3
|
|
4
4
|
class Browser
|
5
|
-
|
5
|
+
|
6
6
|
#
|
7
7
|
# An SQLite3-backed browser cache (with gzip compressed pages)
|
8
8
|
#
|
9
9
|
class Cache
|
10
|
-
|
10
|
+
|
11
11
|
include Enumerable
|
12
|
-
|
12
|
+
|
13
13
|
attr_reader :db, :agent
|
14
|
-
|
15
|
-
def initialize(
|
14
|
+
|
15
|
+
def initialize(filename="browsercache.db", agent=nil)
|
16
16
|
@agent = agent
|
17
17
|
@filename = filename
|
18
|
-
|
18
|
+
|
19
19
|
@db = SQLite3::Database.new(filename)
|
20
20
|
@db.busy_timeout(50)
|
21
|
-
|
21
|
+
|
22
22
|
create_tables
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def inspect
|
26
26
|
"#<Browser::Cache filename=#{@filename.inspect}, count=#{count}, size=#{File.size @filename} bytes>"
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
def count
|
30
30
|
db.execute("SELECT COUNT(1) FROM cache").first.first.to_i
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
alias_method :size, :count
|
34
|
-
|
34
|
+
|
35
35
|
def valid_page?(page)
|
36
36
|
[:body, :content_type, :uri].all?{|m| page.respond_to? m }
|
37
37
|
end
|
38
|
-
|
39
|
-
|
38
|
+
|
39
|
+
|
40
40
|
def put(page, original_url=nil, options={})
|
41
41
|
dmsg [:put, original_url]
|
42
|
-
|
43
|
-
raise "Invalid page" unless valid_page?(page)
|
44
|
-
|
42
|
+
|
43
|
+
raise "Invalid page" unless valid_page?(page)
|
44
|
+
|
45
45
|
url = page.uri.to_s
|
46
|
-
|
46
|
+
|
47
47
|
dmsg [:page_uri, url]
|
48
48
|
dmsg [:original_url, url]
|
49
|
-
|
49
|
+
|
50
50
|
if url != original_url
|
51
51
|
# redirect original_url to url
|
52
52
|
expire(original_url) if options[:overwrite]
|
@@ -58,7 +58,7 @@ class Browser
|
|
58
58
|
url
|
59
59
|
)
|
60
60
|
end
|
61
|
-
|
61
|
+
|
62
62
|
#compressed_body = page.body
|
63
63
|
compressed_body = Zlib::Deflate.deflate(page.body)
|
64
64
|
|
@@ -70,23 +70,23 @@ class Browser
|
|
70
70
|
SQLite3::Blob.new( compressed_body ),
|
71
71
|
nil
|
72
72
|
)
|
73
|
-
|
73
|
+
|
74
74
|
true
|
75
|
-
|
75
|
+
|
76
76
|
rescue SQLite3::SQLException => e
|
77
77
|
p [:exception, e]
|
78
78
|
false
|
79
79
|
end
|
80
|
-
|
80
|
+
|
81
81
|
def row_to_page(row)
|
82
82
|
url, content_type, compressed_body, redirect = row
|
83
|
-
|
83
|
+
|
84
84
|
if redirect
|
85
85
|
get(redirect)
|
86
86
|
else
|
87
87
|
#body = compressed_body
|
88
88
|
body = Zlib::Inflate.inflate(compressed_body)
|
89
|
-
|
89
|
+
|
90
90
|
if content_type =~ %r{^(text/html|text/xml|application/xhtml\+xml)}i
|
91
91
|
Mechanize::Page.new(
|
92
92
|
#initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
|
@@ -105,10 +105,10 @@ class Browser
|
|
105
105
|
nil
|
106
106
|
)
|
107
107
|
end
|
108
|
-
|
108
|
+
|
109
109
|
end
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
def pages_via_sql(*args, &block)
|
113
113
|
dmsg [:pages_via_sql, args]
|
114
114
|
if block_given?
|
@@ -119,27 +119,27 @@ class Browser
|
|
119
119
|
db.execute(*args).map{|row| row_to_page(row) }
|
120
120
|
end
|
121
121
|
end
|
122
|
-
|
122
|
+
|
123
123
|
def grep(pattern, &block)
|
124
124
|
pages_via_sql("SELECT * FROM cache WHERE url like '%#{pattern}%'", &block)
|
125
125
|
end
|
126
|
-
|
126
|
+
|
127
127
|
def get(url)
|
128
128
|
pages = pages_via_sql("SELECT * FROM cache WHERE url = ?", url.to_s)
|
129
|
-
|
129
|
+
|
130
130
|
if pages.any?
|
131
131
|
pages.first
|
132
132
|
else
|
133
133
|
nil
|
134
134
|
end
|
135
135
|
end
|
136
|
-
|
136
|
+
|
137
137
|
def includes?(url)
|
138
138
|
db.execute("SELECT url FROM cache WHERE url = ?", url.to_s).any?
|
139
139
|
end
|
140
|
-
|
140
|
+
|
141
141
|
alias_method :include?, :includes?
|
142
|
-
|
142
|
+
|
143
143
|
def urls(pattern=nil)
|
144
144
|
if pattern
|
145
145
|
rows = db.execute("SELECT url FROM cache WHERE url LIKE '%#{pattern}%'")
|
@@ -148,54 +148,54 @@ class Browser
|
|
148
148
|
end
|
149
149
|
rows.map{|row| row.first}
|
150
150
|
end
|
151
|
-
|
151
|
+
|
152
152
|
def clear(pattern=nil)
|
153
153
|
if pattern
|
154
154
|
db.execute("DELETE FROM cache WHERE url LIKE '%#{pattern}%'")
|
155
155
|
else
|
156
|
-
db.execute("DELETE FROM cache")
|
156
|
+
db.execute("DELETE FROM cache")
|
157
157
|
end
|
158
158
|
end
|
159
|
-
|
159
|
+
|
160
160
|
def each(&block)
|
161
161
|
pages_via_sql("SELECT * FROM cache", &block)
|
162
162
|
end
|
163
|
-
|
163
|
+
|
164
164
|
def each_url
|
165
165
|
db.execute("SELECT url FROM cache") do |row|
|
166
166
|
yield row.first
|
167
167
|
end
|
168
168
|
end
|
169
|
-
|
169
|
+
|
170
170
|
def expire(url)
|
171
171
|
db.execute("DELETE FROM cache WHERE url = ?", url)
|
172
172
|
end
|
173
|
-
|
173
|
+
|
174
174
|
def recreate_tables
|
175
175
|
drop_tables rescue nil
|
176
176
|
create_tables
|
177
177
|
end
|
178
|
-
|
178
|
+
|
179
179
|
def delete!
|
180
180
|
db.close
|
181
|
-
File.unlink @filename
|
181
|
+
File.unlink @filename
|
182
182
|
end
|
183
|
-
|
183
|
+
|
184
184
|
private
|
185
|
-
|
185
|
+
|
186
186
|
def list_tables
|
187
187
|
db.execute("SELECT name FROM SQLITE_MASTER WHERE type='table'")
|
188
188
|
end
|
189
|
-
|
189
|
+
|
190
190
|
def create_tables
|
191
191
|
db.execute("CREATE TABLE IF NOT EXISTS cache ( url varchar(2048), content_type varchar(255), body blob, redirect varchar(2048) )")
|
192
192
|
db.execute("CREATE UNIQUE INDEX IF NOT EXISTS url_index ON cache ( url )")
|
193
193
|
end
|
194
|
-
|
194
|
+
|
195
195
|
def drop_tables
|
196
196
|
db.execute("DROP TABLE cache")
|
197
197
|
end
|
198
|
-
|
198
|
+
|
199
199
|
end
|
200
|
-
|
200
|
+
|
201
201
|
end
|
data/lib/epitools/clitools.rb
CHANGED
@@ -14,7 +14,7 @@
|
|
14
14
|
# * Allow colour
|
15
15
|
# * Don't wrap lines longer than the screen
|
16
16
|
# * Quit immediately (without paging) if there's less than one screen of text.
|
17
|
-
#
|
17
|
+
#
|
18
18
|
# You can change these options by passing a hash to `lesspipe`, like so:
|
19
19
|
#
|
20
20
|
# lesspipe(:wrap=>false) { |less| less.puts essay.to_s }
|
@@ -30,9 +30,9 @@ def lesspipe(*args)
|
|
30
30
|
else
|
31
31
|
options = {}
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
output = args.first if args.any?
|
35
|
-
|
35
|
+
|
36
36
|
params = []
|
37
37
|
params << "-R" unless options[:color] == false
|
38
38
|
params << "-S" unless options[:wrap] == true
|
@@ -42,7 +42,7 @@ def lesspipe(*args)
|
|
42
42
|
$stderr.puts "Seeking to end of stream..."
|
43
43
|
end
|
44
44
|
params << "-X"
|
45
|
-
|
45
|
+
|
46
46
|
IO.popen("less #{params * ' '}", "w") do |less|
|
47
47
|
if output
|
48
48
|
less.puts output
|
@@ -50,7 +50,7 @@ def lesspipe(*args)
|
|
50
50
|
yield less
|
51
51
|
end
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
rescue Errno::EPIPE, Interrupt
|
55
55
|
# less just quit -- eat the exception.
|
56
56
|
end
|
@@ -74,14 +74,14 @@ end
|
|
74
74
|
def cmd(*args)
|
75
75
|
|
76
76
|
cmd_args = []
|
77
|
-
|
77
|
+
|
78
78
|
for arg in args
|
79
|
-
|
79
|
+
|
80
80
|
case arg
|
81
|
-
|
81
|
+
|
82
82
|
when Array
|
83
83
|
cmd_literals = arg.shift.split(/\s+/)
|
84
|
-
|
84
|
+
|
85
85
|
for cmd_literal in cmd_literals
|
86
86
|
if cmd_literal == "?"
|
87
87
|
raise "Not enough substitution arguments" unless cmd_args.any?
|
@@ -90,22 +90,22 @@ def cmd(*args)
|
|
90
90
|
cmd_args << cmd_literal
|
91
91
|
end
|
92
92
|
end
|
93
|
-
|
93
|
+
|
94
94
|
raise "More parameters than ?'s in cmd string" if arg.any?
|
95
|
-
|
95
|
+
|
96
96
|
when String
|
97
97
|
cmd_args << arg
|
98
|
-
|
98
|
+
|
99
99
|
else
|
100
100
|
cmd_args << arg.to_s
|
101
|
-
|
101
|
+
|
102
102
|
end
|
103
|
-
|
104
|
-
end
|
103
|
+
|
104
|
+
end
|
105
105
|
|
106
106
|
p [:cmd_args, cmd_args] if $DEBUG
|
107
|
-
|
108
|
-
system(*cmd_args)
|
107
|
+
|
108
|
+
system(*cmd_args)
|
109
109
|
end
|
110
110
|
|
111
111
|
|
@@ -126,11 +126,11 @@ def prompt(message="Are you sure?", options="Yn")
|
|
126
126
|
default = defaults.first
|
127
127
|
|
128
128
|
loop do
|
129
|
-
|
129
|
+
|
130
130
|
print "#{message} (#{optstring}) "
|
131
|
-
|
131
|
+
|
132
132
|
response = STDIN.gets.strip.downcase
|
133
|
-
|
133
|
+
|
134
134
|
case response
|
135
135
|
when *opts
|
136
136
|
return response
|
@@ -139,13 +139,13 @@ def prompt(message="Are you sure?", options="Yn")
|
|
139
139
|
else
|
140
140
|
puts " |_ Invalid option: #{response.inspect}. Try again."
|
141
141
|
end
|
142
|
-
|
142
|
+
|
143
143
|
end
|
144
144
|
end
|
145
145
|
|
146
146
|
|
147
147
|
#
|
148
|
-
# Automatically install a required commandline tool using the platform's package manager (incomplete -- only supports debian :)
|
148
|
+
# Automatically install a required commandline tool using the platform's package manager (incomplete -- only supports debian :)
|
149
149
|
#
|
150
150
|
# * apt-get on debian/ubuntu
|
151
151
|
# * yum on fedora
|
@@ -154,7 +154,7 @@ end
|
|
154
154
|
#
|
155
155
|
def autoinstall(*packages)
|
156
156
|
all_packages_installed = packages.all? { |pkg| Path.which pkg }
|
157
|
-
|
157
|
+
|
158
158
|
unless all_packages_installed
|
159
159
|
cmd(["sudo apt-get install ?", packages.join(' ')])
|
160
160
|
end
|
@@ -170,24 +170,40 @@ def sudoifnotroot
|
|
170
170
|
end
|
171
171
|
end
|
172
172
|
|
173
|
+
#
|
174
|
+
# Lookup GeoIP information (city, state, country, etc.) for an IP address or hostname
|
175
|
+
#
|
176
|
+
# (Note: Must be able to find one of /usr/share/GeoIP/GeoIP{,City}.dat, or specified manually
|
177
|
+
# as (an) extra argument(s).)
|
178
|
+
#
|
179
|
+
def geoip(addr, city_data='/usr/share/GeoIP/GeoIPCity.dat', country_data='/usr/share/GeoIP/GeoIP.dat')
|
180
|
+
(
|
181
|
+
$geoip ||= begin
|
182
|
+
if city_data and File.exists? city_data
|
183
|
+
geo = GeoIP.new city_data
|
184
|
+
proc { |addr| geo.city(addr) }
|
173
185
|
|
174
|
-
|
175
|
-
|
186
|
+
elsif country_data and File.exists? country_data
|
187
|
+
geo = GeoIP.new country_data
|
188
|
+
proc { |addr| geo.country(addr) }
|
176
189
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
190
|
+
else
|
191
|
+
raise "Can't find GeoIP data files."
|
192
|
+
end
|
193
|
+
end
|
194
|
+
).call(addr)
|
195
|
+
end
|
182
196
|
|
183
|
-
elsif File.exists? GEOIP_COUNTRY_DATA
|
184
|
-
geo = GeoIP.new GEOIP_COUNTRY_DATA
|
185
|
-
proc { |addr| geo.country(addr) }
|
186
197
|
|
187
|
-
|
188
|
-
|
198
|
+
#
|
199
|
+
# Search the PATH environment variable for binaries, returning the first one that exists
|
200
|
+
#
|
201
|
+
def which(*bins)
|
202
|
+
bins.flatten.each do |bin|
|
203
|
+
ENV["PATH"].split(":").each do |dir|
|
204
|
+
full_path = File.join(dir, bin)
|
205
|
+
return full_path if File.exists? full_path
|
189
206
|
end
|
190
207
|
end
|
191
|
-
|
192
|
-
$geoip.call(addr)
|
208
|
+
nil
|
193
209
|
end
|
@@ -178,6 +178,35 @@ module Enumerable
|
|
178
178
|
|
179
179
|
alias_method :cut_between, :split_between
|
180
180
|
|
181
|
+
|
182
|
+
#
|
183
|
+
# Map elements of this Enumerable in parallel using a pool full of Threads
|
184
|
+
#
|
185
|
+
# eg: repos.parallel_map { |repo| system "git pull #{repo}" }
|
186
|
+
#
|
187
|
+
def parallel_map(num_workers=8, &block)
|
188
|
+
require 'thread'
|
189
|
+
|
190
|
+
queue = Queue.new
|
191
|
+
each { |e| queue.push e }
|
192
|
+
|
193
|
+
Enumerator.new do |y|
|
194
|
+
workers = (0...num_workers).map do
|
195
|
+
Thread.new do
|
196
|
+
begin
|
197
|
+
while e = queue.pop(true)
|
198
|
+
y << block.call(e)
|
199
|
+
end
|
200
|
+
rescue ThreadError
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
workers.map(&:join)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
|
181
210
|
#
|
182
211
|
# Sum the elements
|
183
212
|
#
|
@@ -51,15 +51,15 @@ class Matrix
|
|
51
51
|
alias_method :draw, :print
|
52
52
|
|
53
53
|
#
|
54
|
-
# Allow mathematical operations (*, /, +, -) with a
|
54
|
+
# Allow mathematical operations (*, /, +, -) with a scalar (integer or float) on the right side.
|
55
55
|
#
|
56
56
|
# eg: Matrix.zero(3) + 5
|
57
57
|
#
|
58
|
-
%
|
58
|
+
%i[* / + -].each do |op|
|
59
59
|
class_eval %{
|
60
60
|
def #{op}(other)
|
61
61
|
case other
|
62
|
-
when
|
62
|
+
when Numeric
|
63
63
|
map { |e| e #{op} other }
|
64
64
|
else
|
65
65
|
super(other)
|
@@ -30,13 +30,13 @@ class Numeric
|
|
30
30
|
#
|
31
31
|
# Examples:
|
32
32
|
# 234234234523.clamp(0..100) #=> 100
|
33
|
-
# 12.clamp(0..100) #=> 12
|
33
|
+
# 12.clamp(0..100) #=> 12
|
34
34
|
# -38817112.clamp(0..100) #=> 0
|
35
35
|
#
|
36
36
|
def clamp(range)
|
37
37
|
if self < range.first
|
38
38
|
range.first
|
39
|
-
elsif self >= range.last
|
39
|
+
elsif self >= range.last
|
40
40
|
if range.exclude_end?
|
41
41
|
range.last - 1
|
42
42
|
else
|
@@ -51,7 +51,7 @@ class Numeric
|
|
51
51
|
# Time methods
|
52
52
|
#
|
53
53
|
{
|
54
|
-
|
54
|
+
|
55
55
|
'second' => 1,
|
56
56
|
'minute' => 60,
|
57
57
|
'hour' => 60 * 60,
|
@@ -59,23 +59,23 @@ class Numeric
|
|
59
59
|
'week' => 60 * 60 * 24 * 7,
|
60
60
|
'month' => 60 * 60 * 24 * 30,
|
61
61
|
'year' => 60 * 60 * 24 * 365,
|
62
|
-
|
62
|
+
|
63
63
|
}.each do |unit, scale|
|
64
64
|
define_method(unit) { self * scale }
|
65
65
|
define_method(unit+'s') { self * scale }
|
66
66
|
end
|
67
|
-
|
67
|
+
|
68
68
|
def ago
|
69
69
|
Time.now - self
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def from_now
|
73
73
|
Time.now + self
|
74
74
|
end
|
75
75
|
|
76
76
|
#
|
77
|
-
# If `n.times` is like `each`, `n.things` is like `map`. Return
|
78
|
-
#
|
77
|
+
# If `n.times` is like `each`, `n.things` is like `map`. Return
|
78
|
+
#
|
79
79
|
def things(&block)
|
80
80
|
if block_given?
|
81
81
|
Array.new(self, &block)
|
@@ -255,25 +255,25 @@ class Integer
|
|
255
255
|
def to_hex
|
256
256
|
"%0.2x" % self
|
257
257
|
end
|
258
|
-
|
258
|
+
|
259
259
|
#
|
260
260
|
# Convert the number to an array of bits (least significant digit first, or little-endian).
|
261
261
|
#
|
262
262
|
def to_bits
|
263
|
-
# TODO: Why does
|
263
|
+
# TODO: Why does this go into an infinite loop in 1.8.7?
|
264
264
|
("%b" % self).chars.to_a.reverse.map(&:to_i)
|
265
265
|
end
|
266
266
|
alias_method :bits, :to_bits
|
267
|
-
|
267
|
+
|
268
268
|
#
|
269
269
|
# Cached constants for base62 encoding
|
270
270
|
#
|
271
|
-
BASE62_DIGITS = ['0'..'9', 'A'..'Z', 'a'..'z'].map(&:to_a).flatten
|
271
|
+
BASE62_DIGITS = ['0'..'9', 'A'..'Z', 'a'..'z'].map(&:to_a).flatten
|
272
272
|
BASE62_BASE = BASE62_DIGITS.size
|
273
273
|
|
274
274
|
#
|
275
275
|
# Convert a number to a string representation (in "base62" encoding).
|
276
|
-
#
|
276
|
+
#
|
277
277
|
# Base62 encoding represents the number using the characters: 0..9, A..Z, a..z
|
278
278
|
#
|
279
279
|
# It's the same scheme that url shorteners and YouTube uses for their
|
@@ -283,16 +283,16 @@ class Integer
|
|
283
283
|
result = []
|
284
284
|
remainder = self
|
285
285
|
max_power = ( Math.log(self) / Math.log(BASE62_BASE) ).floor
|
286
|
-
|
286
|
+
|
287
287
|
max_power.downto(0) do |power|
|
288
288
|
divisor = BASE62_BASE**power
|
289
|
-
#p [:div, divisor, :rem, remainder]
|
289
|
+
#p [:div, divisor, :rem, remainder]
|
290
290
|
digit, remainder = remainder.divmod(divisor)
|
291
291
|
result << digit
|
292
292
|
end
|
293
|
-
|
293
|
+
|
294
294
|
result << remainder if remainder > 0
|
295
|
-
|
295
|
+
|
296
296
|
result.map{|digit| BASE62_DIGITS[digit]}.join ''
|
297
297
|
end
|
298
298
|
|
@@ -312,7 +312,7 @@ class Integer
|
|
312
312
|
|
313
313
|
loop do
|
314
314
|
if current.prime?
|
315
|
-
result << current
|
315
|
+
result << current
|
316
316
|
return result if result.size >= self
|
317
317
|
end
|
318
318
|
current += 1
|
@@ -324,7 +324,7 @@ class Integer
|
|
324
324
|
#
|
325
325
|
def factors
|
326
326
|
Prime # autoload the prime module
|
327
|
-
prime_division.map { |n,count| [n]*count }.flatten
|
327
|
+
prime_division.map { |n,count| [n]*count }.flatten
|
328
328
|
end
|
329
329
|
|
330
330
|
#
|
@@ -352,32 +352,33 @@ class Integer
|
|
352
352
|
end
|
353
353
|
|
354
354
|
#
|
355
|
-
#
|
355
|
+
# Adds integer silcing (returning the bits) and raw-bytes
|
356
356
|
#
|
357
357
|
# (This is necessary because [] is defined directly on the classes, and a mixin
|
358
358
|
# module will still be overridden by Big/Fixnum's native [] method.)
|
359
359
|
#
|
360
|
-
[Bignum, Fixnum].each do |klass|
|
361
|
-
|
360
|
+
(RUBY_VERSION >= "2.4" ? [Integer] : [Bignum, Fixnum]).each do |klass|
|
362
361
|
klass.class_eval do
|
363
|
-
|
362
|
+
|
364
363
|
alias_method :bit, :"[]"
|
365
|
-
|
364
|
+
|
366
365
|
#
|
367
|
-
#
|
366
|
+
# Slice the bits of an integer by passing a range (eg: 1217389172842[0..5] #=> [0, 1, 0, 1, 0, 1])
|
368
367
|
#
|
369
368
|
def [](arg)
|
370
369
|
case arg
|
371
370
|
when Integer
|
372
|
-
|
371
|
+
bit(arg)
|
373
372
|
when Range
|
374
|
-
|
373
|
+
bits[arg]
|
375
374
|
end
|
376
375
|
end
|
377
|
-
|
376
|
+
|
378
377
|
#
|
379
378
|
# Convert the integer into its sequence of bytes (little endian format: lowest-order-byte first)
|
380
379
|
#
|
380
|
+
# TODO: This could be made much more efficient!
|
381
|
+
#
|
381
382
|
def bytes
|
382
383
|
nbytes = (bit_length / 8.0).ceil
|
383
384
|
|
@@ -386,6 +387,9 @@ end
|
|
386
387
|
end
|
387
388
|
end
|
388
389
|
|
390
|
+
def big_endian_bytes
|
391
|
+
bytes.reverse
|
392
|
+
end
|
389
393
|
end
|
390
394
|
end
|
391
395
|
|
@@ -415,7 +419,7 @@ class Time
|
|
415
419
|
def in_words
|
416
420
|
delta = (Time.now-self).to_i
|
417
421
|
a = delta.abs
|
418
|
-
|
422
|
+
|
419
423
|
amount = case a
|
420
424
|
when 0
|
421
425
|
'just now'
|
@@ -436,16 +440,16 @@ class Time
|
|
436
440
|
else
|
437
441
|
"year".amount(a/1.year)
|
438
442
|
end
|
439
|
-
|
443
|
+
|
440
444
|
if delta < 0
|
441
445
|
amount += " from now"
|
442
446
|
elsif delta > 0
|
443
447
|
amount += " ago"
|
444
448
|
end
|
445
|
-
|
449
|
+
|
446
450
|
amount
|
447
451
|
end
|
448
|
-
|
452
|
+
|
449
453
|
end
|
450
454
|
|
451
455
|
|
data/lib/epitools/minimal.rb
CHANGED
data/lib/epitools/path.rb
CHANGED
@@ -123,6 +123,14 @@ class Path
|
|
123
123
|
###############################################################################
|
124
124
|
|
125
125
|
def initialize(newpath, hints={})
|
126
|
+
if hints[:unlink_when_garbage_collected]
|
127
|
+
backup_path = path.dup
|
128
|
+
puts "unlinking #{backup_path} after gc!"
|
129
|
+
ObjectSpace.define_finalizer self do |object_id|
|
130
|
+
File.unlink backup_path
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
126
134
|
send("path=", newpath, hints)
|
127
135
|
end
|
128
136
|
|
@@ -583,6 +591,28 @@ class Path
|
|
583
591
|
end
|
584
592
|
end
|
585
593
|
|
594
|
+
def media_tags
|
595
|
+
require 'taglib'
|
596
|
+
|
597
|
+
tags = nil
|
598
|
+
|
599
|
+
TagLib::FileRef.open(path) do |fileref|
|
600
|
+
unless fileref.null?
|
601
|
+
tags = fileref.tag
|
602
|
+
result = {}
|
603
|
+
getters = tags.class.instance_methods(false).group_by {|m| m[/^.+[^=]/] }.map { |k,vs| vs.size == 2 ? k.to_sym : nil }.compact
|
604
|
+
getters.each do |getter|
|
605
|
+
tags[getter] = tags.public_send(getter)
|
606
|
+
end
|
607
|
+
end
|
608
|
+
end
|
609
|
+
|
610
|
+
tags
|
611
|
+
end
|
612
|
+
|
613
|
+
alias_method :id3, :media_tags
|
614
|
+
alias_method :id3tags, :media_tags
|
615
|
+
|
586
616
|
#
|
587
617
|
# Retrieve one of this file's xattrs
|
588
618
|
#
|
@@ -969,10 +999,12 @@ class Path
|
|
969
999
|
if dest.startswith("/")
|
970
1000
|
Path.ln_s(self, dest)
|
971
1001
|
else
|
972
|
-
Path.ln_s(self, self / dest
|
1002
|
+
Path.ln_s(self, self / dest)
|
973
1003
|
end
|
974
1004
|
end
|
975
1005
|
|
1006
|
+
alias_method :symlink_to, :ln_s
|
1007
|
+
|
976
1008
|
## Owners and permissions
|
977
1009
|
|
978
1010
|
def chmod(mode)
|
@@ -1054,7 +1086,7 @@ class Path
|
|
1054
1086
|
Zlib.deflate(read, level)
|
1055
1087
|
end
|
1056
1088
|
alias_method :gzip, :deflate
|
1057
|
-
|
1089
|
+
|
1058
1090
|
|
1059
1091
|
#
|
1060
1092
|
# gunzip the file, returning the result as a string
|
@@ -1270,21 +1302,24 @@ class Path
|
|
1270
1302
|
# TODO: Remove the tempfile when the Path object is garbage collected or freed.
|
1271
1303
|
#
|
1272
1304
|
def self.tmpfile(prefix="tmp")
|
1273
|
-
path = Path
|
1305
|
+
path = Path.new Tempfile.new(prefix).path, unlink_when_garbage_collected: true
|
1274
1306
|
yield path if block_given?
|
1275
1307
|
path
|
1276
1308
|
end
|
1277
1309
|
alias_class_method :tempfile, :tmpfile
|
1278
1310
|
alias_class_method :tmp, :tmpfile
|
1279
1311
|
|
1312
|
+
|
1313
|
+
#
|
1314
|
+
# Create a uniqely named directory in /tmp
|
1315
|
+
#
|
1280
1316
|
def self.tmpdir(prefix="tmp")
|
1281
1317
|
t = tmpfile
|
1282
|
-
|
1283
|
-
# FIXME: These operations should be atomic
|
1284
|
-
t.rm; t.mkdir
|
1285
|
-
|
1318
|
+
t.rm; t.mkdir # FIXME: These two operations should be made atomic
|
1286
1319
|
t
|
1287
1320
|
end
|
1321
|
+
alias_class_method :tempdir, :tmpdir
|
1322
|
+
|
1288
1323
|
|
1289
1324
|
def self.home
|
1290
1325
|
Path[ENV['HOME']]
|
data/spec/browser_spec.rb
CHANGED
@@ -11,7 +11,7 @@ end
|
|
11
11
|
describe Browser do
|
12
12
|
|
13
13
|
before :all do
|
14
|
-
@browser = Browser.new
|
14
|
+
@browser = Browser.new use_cache: true
|
15
15
|
end
|
16
16
|
|
17
17
|
after :all do
|
@@ -23,16 +23,16 @@ describe Browser do
|
|
23
23
|
page = @browser.get(url)
|
24
24
|
@browser.cache.get(url).should_not == nil
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
it "googles" do
|
28
28
|
page = @browser.get("http://google.com")
|
29
29
|
page.body["Feeling Lucky"].should_not be_empty
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
it "googles (cached)" do
|
33
33
|
lambda{ @browser.get("http://google.com").body }.should_not raise_error
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
it "delegates" do
|
37
37
|
lambda{ @browser.head("http://google.com").body }.should_not raise_error
|
38
38
|
@browser.respond_to?(:post).should == true
|
@@ -46,9 +46,10 @@ end
|
|
46
46
|
describe Browser::Cache do
|
47
47
|
|
48
48
|
before :all do
|
49
|
+
cache_file = "temp-cache.db"
|
49
50
|
@agent = Mechanize.new
|
50
|
-
Browser::Cache.new(@agent).delete!
|
51
|
-
@cache = Browser::Cache.new(@agent)
|
51
|
+
Browser::Cache.new(cache_file, @agent).delete!
|
52
|
+
@cache = Browser::Cache.new(cache_file, @agent)
|
52
53
|
end
|
53
54
|
|
54
55
|
after :all do
|
data/spec/clitools_spec.rb
CHANGED
@@ -1,30 +1,46 @@
|
|
1
|
+
require 'epitools/minimal'
|
1
2
|
require 'epitools/clitools'
|
2
3
|
|
3
|
-
describe
|
4
|
+
describe Object do
|
4
5
|
|
5
|
-
it "
|
6
|
-
|
7
|
-
|
6
|
+
# it "'highlight's" do
|
7
|
+
# color = :light_yellow
|
8
|
+
# highlighted = "xxx#{"match".send(color)}zzz"
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
10
|
+
# "xxxmatchzzz".highlight(/match/, color).should == highlighted
|
11
|
+
# "xxxmatchzzz".highlight("match", color).should == highlighted
|
12
|
+
# "xxxmatchzzz".highlight(/m.+h/, color).should == highlighted
|
13
|
+
# "xxxmatchzzz".highlight(/MATCH/i, color).should == highlighted
|
14
|
+
# end
|
14
15
|
|
15
|
-
it "
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
-
|
20
|
-
it "
|
16
|
+
# it "'highlight's with a block" do
|
17
|
+
# result = "xxxmatchxxx".highlight(/match/) { |match| "<8>#{match}</8>" }
|
18
|
+
# result.should == "xxx<8>match</8>xxx"
|
19
|
+
# end
|
20
|
+
|
21
|
+
it "'cmd's" do
|
21
22
|
cmd( ['test -f ?', __FILE__] ).should == true
|
22
23
|
cmd( ['test -d ?', __FILE__] ).should == false
|
23
|
-
cmd( "test", "-f", __FILE__ ).should == true
|
24
|
+
cmd( "test", "-f", __FILE__ ).should == true
|
24
25
|
cmd( "test", "-d", __FILE__ ).should == false
|
25
|
-
|
26
|
-
|
27
|
-
|
26
|
+
|
27
|
+
-> { cmd( ["test -f ? ?", __FILE__] ) }.should raise_error(TypeError) # more ?'s than args
|
28
|
+
-> { cmd( ["test -f", __FILE__] ) }.should raise_error(RuntimeError) # more args than ?'s
|
29
|
+
end
|
30
|
+
|
31
|
+
it "'which'es" do
|
32
|
+
which("totally nonexistant", "probably nonexistant", "ls", "df").should_not == nil
|
33
|
+
which("totally nonexistant", "probably nonexistant").should == nil
|
34
|
+
which("ls", "df").should =~ /\/ls$/
|
28
35
|
end
|
29
|
-
|
36
|
+
|
37
|
+
it "'geoip's" do
|
38
|
+
geoip("128.100.100.128").country_name.should == "Canada"
|
39
|
+
|
40
|
+
-> { geoip("butt"*20) }.should raise_error(SocketError)
|
41
|
+
|
42
|
+
$geoip = nil
|
43
|
+
-> { geoip("8.8.4.4", nil, nil) }.should raise_error(RuntimeError)
|
44
|
+
end
|
45
|
+
|
30
46
|
end
|
data/spec/core_ext_spec.rb
CHANGED
@@ -28,13 +28,13 @@ describe Object do
|
|
28
28
|
|
29
29
|
lambda {
|
30
30
|
time("time test") { raise "ERROR" }
|
31
|
-
}.should raise_error
|
31
|
+
}.should raise_error(RuntimeError)
|
32
32
|
end
|
33
33
|
|
34
34
|
it "benches" do
|
35
35
|
lambda { bench { rand } }.should_not raise_error
|
36
36
|
lambda { bench(20) { rand } }.should_not raise_error
|
37
|
-
lambda { bench }.should raise_error
|
37
|
+
lambda { bench }.should raise_error(RuntimeError)
|
38
38
|
lambda { bench(:rand => proc { rand }, :notrand => proc { 1 }) }.should_not raise_error
|
39
39
|
lambda { bench(200, :rand => proc { rand }, :notrand => proc { 1 }) }.should_not raise_error
|
40
40
|
end
|
@@ -49,15 +49,15 @@ describe Object do
|
|
49
49
|
s.try(:c).should == nil
|
50
50
|
|
51
51
|
lambda { s.try(:c) }.should_not raise_error
|
52
|
-
lambda { s.c }.should raise_error
|
52
|
+
lambda { s.c }.should raise_error(NoMethodError)
|
53
53
|
|
54
54
|
def s.test(a); a; end
|
55
55
|
|
56
|
-
s.test(1).should
|
57
|
-
s.try(:test, 1).should
|
56
|
+
s.test(1).should == 1
|
57
|
+
s.try(:test, 1).should == 1
|
58
58
|
|
59
|
-
lambda { s.test }.should raise_error
|
60
|
-
lambda { s.try(:test) }.should raise_error
|
59
|
+
lambda { s.test }.should raise_error(ArgumentError)
|
60
|
+
lambda { s.try(:test) }.should raise_error(ArgumentError)
|
61
61
|
|
62
62
|
def s.blocky; yield; end
|
63
63
|
|
data/spec/path_spec.rb
CHANGED
@@ -234,7 +234,7 @@ describe Path do
|
|
234
234
|
|
235
235
|
path.touch
|
236
236
|
lambda { path.rename(:ext=>".dat") }.should raise_error
|
237
|
-
|
237
|
+
|
238
238
|
dest.rm
|
239
239
|
path.rename!(:ext=>".dat")
|
240
240
|
path.to_s.should_not == old_name
|
@@ -483,16 +483,15 @@ describe Path do
|
|
483
483
|
end
|
484
484
|
|
485
485
|
it 'symlinks relative dirs' do
|
486
|
-
raise "Path.ln_s arguments are backwards. It needs to be something easier to remember, like 'Path#symlink_to'"
|
487
486
|
tmpdir = Path.tmpdir
|
488
487
|
|
489
488
|
symlink = (tmpdir/"a_new_link")
|
490
|
-
|
489
|
+
Path["../../etc/passwd"].ln_s symlink
|
491
490
|
|
492
491
|
symlink.symlink?.should == true
|
493
492
|
|
494
493
|
symlink.rm
|
495
|
-
|
494
|
+
tmpdir.rm
|
496
495
|
end
|
497
496
|
|
498
497
|
it 'swaps two files' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: epitools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.99
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- epitron
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
124
124
|
version: '0'
|
125
125
|
requirements: []
|
126
126
|
rubyforge_project:
|
127
|
-
rubygems_version: 2.
|
127
|
+
rubygems_version: 2.6.8
|
128
128
|
signing_key:
|
129
129
|
specification_version: 3
|
130
130
|
summary: Not utils... METILS!
|