epitools 0.5.98 → 0.5.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/lib/epitools/autoloads.rb +2 -1
- data/lib/epitools/browser.rb +26 -25
- data/lib/epitools/browser/cache.rb +46 -46
- data/lib/epitools/clitools.rb +53 -37
- data/lib/epitools/core_ext/enumerable.rb +29 -0
- data/lib/epitools/core_ext/matrix.rb +3 -3
- data/lib/epitools/core_ext/numbers.rb +36 -32
- data/lib/epitools/minimal.rb +3 -0
- data/lib/epitools/path.rb +42 -7
- data/spec/browser_spec.rb +7 -6
- data/spec/clitools_spec.rb +36 -20
- data/spec/core_ext_spec.rb +7 -7
- data/spec/path_spec.rb +3 -4
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0cbf9aff2ad00720845a3bce7605343ff4b0a88e
|
4
|
+
data.tar.gz: cb8e34cb49a1bb091d60fcfecdfdf1b5006915ac
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0fa9bcbf1d0275b21648d52ee0959250e517861e8b1a4b8acc9b44eba3b9e259d2ac3df83876af1736ae0ac8a564435662da7254bceb9c4f49340e611ac75e14
|
7
|
+
data.tar.gz: 6e51f33d5942756afaa8b90089a5657ed252940c59b4c95f6fb92245faed39ca59ebed6187064d0f287490fbb33099b76afb67481a5980efa61ce0aa7b76beb6
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.99
|
data/lib/epitools/autoloads.rb
CHANGED
@@ -17,8 +17,9 @@ autoload :Timeout, 'timeout'
|
|
17
17
|
autoload :Find, 'find'
|
18
18
|
autoload :Benchmark, 'benchmark'
|
19
19
|
autoload :Tracer, 'tracer'
|
20
|
-
autoload :CSV, 'csv'
|
21
20
|
autoload :Shellwords, 'shellwords'
|
21
|
+
autoload :PTY, 'pty'
|
22
|
+
autoload :CSV, 'csv'
|
22
23
|
autoload :SDBM, 'sdbm'
|
23
24
|
|
24
25
|
module Digest
|
data/lib/epitools/browser.rb
CHANGED
@@ -18,7 +18,7 @@ class Mechanize::File
|
|
18
18
|
response['content-type']
|
19
19
|
end
|
20
20
|
end
|
21
|
-
|
21
|
+
|
22
22
|
|
23
23
|
#
|
24
24
|
# A mechanize class that emulates a web-browser, with cache and everything.
|
@@ -43,8 +43,9 @@ class Browser
|
|
43
43
|
@use_cache = !!(options[:cache] || options[:cached] || options[:use_cache])
|
44
44
|
@use_logs = options[:logs] || false
|
45
45
|
@cookie_file = options[:cookiefile] || "cookies.txt"
|
46
|
-
|
47
|
-
|
46
|
+
@cache_file = options[:cache_file] || "browser-cache.db"
|
47
|
+
|
48
|
+
# TODO: @progress, @user_agent, @logfile, @cache_file (default location: ~/.epitools?)
|
48
49
|
|
49
50
|
if options[:proxy]
|
50
51
|
host, port = options[:proxy].split(':')
|
@@ -59,7 +60,7 @@ class Browser
|
|
59
60
|
def init_agent!
|
60
61
|
@agent = Mechanize.new do |a|
|
61
62
|
# ["Mechanize", "Mac Mozilla", "Linux Mozilla", "Windows IE 6", "iPhone", "Linux Konqueror", "Windows IE 7", "Mac FireFox", "Mac Safari", "Windows Mozilla"]
|
62
|
-
a.max_history = 10
|
63
|
+
a.max_history = 10
|
63
64
|
a.user_agent_alias = "Windows IE 7"
|
64
65
|
a.log = Logger.new "mechanize.log" if @use_logs
|
65
66
|
end
|
@@ -80,7 +81,7 @@ class Browser
|
|
80
81
|
|
81
82
|
def init_cache!
|
82
83
|
# TODO: Rescue "couldn't load" exception and disable caching
|
83
|
-
@cache = Cache.new(agent) if @use_cache
|
84
|
+
@cache = Cache.new(@cache_file, agent) if @use_cache
|
84
85
|
end
|
85
86
|
|
86
87
|
def load_cookies!
|
@@ -91,23 +92,23 @@ class Browser
|
|
91
92
|
false
|
92
93
|
end
|
93
94
|
end
|
94
|
-
|
95
|
+
|
95
96
|
def save_cookies!
|
96
97
|
agent.cookie_jar.save_as @cookie_file
|
97
98
|
true
|
98
99
|
end
|
99
|
-
|
100
|
+
|
100
101
|
def relative?(url)
|
101
102
|
not url[ %r{^https?://} ]
|
102
103
|
end
|
103
|
-
|
104
|
+
|
104
105
|
def cacheable?(page)
|
105
106
|
case page.content_type
|
106
107
|
when %r{^(text|application)}
|
107
108
|
true
|
108
109
|
end
|
109
|
-
end
|
110
|
-
|
110
|
+
end
|
111
|
+
|
111
112
|
def cache_put(page, url)
|
112
113
|
if cache.valid_page?(page)
|
113
114
|
if page.content_type =~ %r{(^text/|^application/javascript|javascript)}
|
@@ -117,9 +118,9 @@ class Browser
|
|
117
118
|
end
|
118
119
|
end
|
119
120
|
|
120
|
-
|
121
|
+
|
121
122
|
#
|
122
|
-
# Retrieve an URL, and return a Mechanize::Page instance (which acts a
|
123
|
+
# Retrieve an URL, and return a Mechanize::Page instance (which acts a
|
123
124
|
# bit like a Nokogiri::HTML::Document instance.)
|
124
125
|
#
|
125
126
|
# Options:
|
@@ -128,7 +129,7 @@ class Browser
|
|
128
129
|
def get(url, options={})
|
129
130
|
|
130
131
|
# TODO: Have a base-URL option
|
131
|
-
|
132
|
+
|
132
133
|
#if relative?(url)
|
133
134
|
# url = URI.join("http://base-url/", url).to_s
|
134
135
|
#end
|
@@ -140,13 +141,13 @@ class Browser
|
|
140
141
|
|
141
142
|
puts
|
142
143
|
puts "[ GET #{url} (using cache: #{!!use_cache}) ]"
|
143
|
-
|
144
|
+
|
144
145
|
delay unless cached_already
|
145
146
|
max_retries = 4
|
146
147
|
retries = 0
|
147
148
|
|
148
149
|
begin
|
149
|
-
|
150
|
+
|
150
151
|
if use_cache and page = cache.get(url)
|
151
152
|
puts " |_ cached (#{page.content_type})"
|
152
153
|
else
|
@@ -165,11 +166,11 @@ class Browser
|
|
165
166
|
|
166
167
|
puts " |_ ERROR: #{e.inspect} -- retrying"
|
167
168
|
delay(5)
|
168
|
-
retry
|
169
|
-
|
170
|
-
=begin
|
169
|
+
retry
|
170
|
+
|
171
|
+
=begin
|
171
172
|
rescue Mechanize::ResponseCodeError => e
|
172
|
-
|
173
|
+
|
173
174
|
case e.response_code
|
174
175
|
when "401" #=> Net::HTTPUnauthorized
|
175
176
|
p e
|
@@ -193,7 +194,7 @@ class Browser
|
|
193
194
|
page
|
194
195
|
end
|
195
196
|
|
196
|
-
|
197
|
+
|
197
198
|
#
|
198
199
|
# Delegate certain methods to @agent
|
199
200
|
#
|
@@ -202,7 +203,7 @@ class Browser
|
|
202
203
|
agent.send(meth, *args)
|
203
204
|
end
|
204
205
|
end
|
205
|
-
|
206
|
+
|
206
207
|
end
|
207
208
|
|
208
209
|
|
@@ -217,11 +218,11 @@ class BrowserOptions < OpenStruct
|
|
217
218
|
:use_logs => false,
|
218
219
|
:cookie_file => "cookies.txt"
|
219
220
|
}
|
220
|
-
|
221
|
+
|
221
222
|
def initialize(extra_opts)
|
222
|
-
|
223
|
+
|
223
224
|
@opts = DEFAULTS.dup
|
224
|
-
|
225
|
+
|
225
226
|
for key, val in opts
|
226
227
|
if key.in? DEFAULTS
|
227
228
|
@opts[key] = val
|
@@ -230,7 +231,7 @@ class BrowserOptions < OpenStruct
|
|
230
231
|
end
|
231
232
|
end
|
232
233
|
end
|
233
|
-
|
234
|
+
|
234
235
|
end
|
235
236
|
=end
|
236
237
|
|
@@ -2,51 +2,51 @@ require 'mechanize'
|
|
2
2
|
require 'sqlite3'
|
3
3
|
|
4
4
|
class Browser
|
5
|
-
|
5
|
+
|
6
6
|
#
|
7
7
|
# An SQLite3-backed browser cache (with gzip compressed pages)
|
8
8
|
#
|
9
9
|
class Cache
|
10
|
-
|
10
|
+
|
11
11
|
include Enumerable
|
12
|
-
|
12
|
+
|
13
13
|
attr_reader :db, :agent
|
14
|
-
|
15
|
-
def initialize(
|
14
|
+
|
15
|
+
def initialize(filename="browsercache.db", agent=nil)
|
16
16
|
@agent = agent
|
17
17
|
@filename = filename
|
18
|
-
|
18
|
+
|
19
19
|
@db = SQLite3::Database.new(filename)
|
20
20
|
@db.busy_timeout(50)
|
21
|
-
|
21
|
+
|
22
22
|
create_tables
|
23
23
|
end
|
24
|
-
|
24
|
+
|
25
25
|
def inspect
|
26
26
|
"#<Browser::Cache filename=#{@filename.inspect}, count=#{count}, size=#{File.size @filename} bytes>"
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
def count
|
30
30
|
db.execute("SELECT COUNT(1) FROM cache").first.first.to_i
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
alias_method :size, :count
|
34
|
-
|
34
|
+
|
35
35
|
def valid_page?(page)
|
36
36
|
[:body, :content_type, :uri].all?{|m| page.respond_to? m }
|
37
37
|
end
|
38
|
-
|
39
|
-
|
38
|
+
|
39
|
+
|
40
40
|
def put(page, original_url=nil, options={})
|
41
41
|
dmsg [:put, original_url]
|
42
|
-
|
43
|
-
raise "Invalid page" unless valid_page?(page)
|
44
|
-
|
42
|
+
|
43
|
+
raise "Invalid page" unless valid_page?(page)
|
44
|
+
|
45
45
|
url = page.uri.to_s
|
46
|
-
|
46
|
+
|
47
47
|
dmsg [:page_uri, url]
|
48
48
|
dmsg [:original_url, url]
|
49
|
-
|
49
|
+
|
50
50
|
if url != original_url
|
51
51
|
# redirect original_url to url
|
52
52
|
expire(original_url) if options[:overwrite]
|
@@ -58,7 +58,7 @@ class Browser
|
|
58
58
|
url
|
59
59
|
)
|
60
60
|
end
|
61
|
-
|
61
|
+
|
62
62
|
#compressed_body = page.body
|
63
63
|
compressed_body = Zlib::Deflate.deflate(page.body)
|
64
64
|
|
@@ -70,23 +70,23 @@ class Browser
|
|
70
70
|
SQLite3::Blob.new( compressed_body ),
|
71
71
|
nil
|
72
72
|
)
|
73
|
-
|
73
|
+
|
74
74
|
true
|
75
|
-
|
75
|
+
|
76
76
|
rescue SQLite3::SQLException => e
|
77
77
|
p [:exception, e]
|
78
78
|
false
|
79
79
|
end
|
80
|
-
|
80
|
+
|
81
81
|
def row_to_page(row)
|
82
82
|
url, content_type, compressed_body, redirect = row
|
83
|
-
|
83
|
+
|
84
84
|
if redirect
|
85
85
|
get(redirect)
|
86
86
|
else
|
87
87
|
#body = compressed_body
|
88
88
|
body = Zlib::Inflate.inflate(compressed_body)
|
89
|
-
|
89
|
+
|
90
90
|
if content_type =~ %r{^(text/html|text/xml|application/xhtml\+xml)}i
|
91
91
|
Mechanize::Page.new(
|
92
92
|
#initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
|
@@ -105,10 +105,10 @@ class Browser
|
|
105
105
|
nil
|
106
106
|
)
|
107
107
|
end
|
108
|
-
|
108
|
+
|
109
109
|
end
|
110
110
|
end
|
111
|
-
|
111
|
+
|
112
112
|
def pages_via_sql(*args, &block)
|
113
113
|
dmsg [:pages_via_sql, args]
|
114
114
|
if block_given?
|
@@ -119,27 +119,27 @@ class Browser
|
|
119
119
|
db.execute(*args).map{|row| row_to_page(row) }
|
120
120
|
end
|
121
121
|
end
|
122
|
-
|
122
|
+
|
123
123
|
def grep(pattern, &block)
|
124
124
|
pages_via_sql("SELECT * FROM cache WHERE url like '%#{pattern}%'", &block)
|
125
125
|
end
|
126
|
-
|
126
|
+
|
127
127
|
def get(url)
|
128
128
|
pages = pages_via_sql("SELECT * FROM cache WHERE url = ?", url.to_s)
|
129
|
-
|
129
|
+
|
130
130
|
if pages.any?
|
131
131
|
pages.first
|
132
132
|
else
|
133
133
|
nil
|
134
134
|
end
|
135
135
|
end
|
136
|
-
|
136
|
+
|
137
137
|
def includes?(url)
|
138
138
|
db.execute("SELECT url FROM cache WHERE url = ?", url.to_s).any?
|
139
139
|
end
|
140
|
-
|
140
|
+
|
141
141
|
alias_method :include?, :includes?
|
142
|
-
|
142
|
+
|
143
143
|
def urls(pattern=nil)
|
144
144
|
if pattern
|
145
145
|
rows = db.execute("SELECT url FROM cache WHERE url LIKE '%#{pattern}%'")
|
@@ -148,54 +148,54 @@ class Browser
|
|
148
148
|
end
|
149
149
|
rows.map{|row| row.first}
|
150
150
|
end
|
151
|
-
|
151
|
+
|
152
152
|
def clear(pattern=nil)
|
153
153
|
if pattern
|
154
154
|
db.execute("DELETE FROM cache WHERE url LIKE '%#{pattern}%'")
|
155
155
|
else
|
156
|
-
db.execute("DELETE FROM cache")
|
156
|
+
db.execute("DELETE FROM cache")
|
157
157
|
end
|
158
158
|
end
|
159
|
-
|
159
|
+
|
160
160
|
def each(&block)
|
161
161
|
pages_via_sql("SELECT * FROM cache", &block)
|
162
162
|
end
|
163
|
-
|
163
|
+
|
164
164
|
def each_url
|
165
165
|
db.execute("SELECT url FROM cache") do |row|
|
166
166
|
yield row.first
|
167
167
|
end
|
168
168
|
end
|
169
|
-
|
169
|
+
|
170
170
|
def expire(url)
|
171
171
|
db.execute("DELETE FROM cache WHERE url = ?", url)
|
172
172
|
end
|
173
|
-
|
173
|
+
|
174
174
|
def recreate_tables
|
175
175
|
drop_tables rescue nil
|
176
176
|
create_tables
|
177
177
|
end
|
178
|
-
|
178
|
+
|
179
179
|
def delete!
|
180
180
|
db.close
|
181
|
-
File.unlink @filename
|
181
|
+
File.unlink @filename
|
182
182
|
end
|
183
|
-
|
183
|
+
|
184
184
|
private
|
185
|
-
|
185
|
+
|
186
186
|
def list_tables
|
187
187
|
db.execute("SELECT name FROM SQLITE_MASTER WHERE type='table'")
|
188
188
|
end
|
189
|
-
|
189
|
+
|
190
190
|
def create_tables
|
191
191
|
db.execute("CREATE TABLE IF NOT EXISTS cache ( url varchar(2048), content_type varchar(255), body blob, redirect varchar(2048) )")
|
192
192
|
db.execute("CREATE UNIQUE INDEX IF NOT EXISTS url_index ON cache ( url )")
|
193
193
|
end
|
194
|
-
|
194
|
+
|
195
195
|
def drop_tables
|
196
196
|
db.execute("DROP TABLE cache")
|
197
197
|
end
|
198
|
-
|
198
|
+
|
199
199
|
end
|
200
|
-
|
200
|
+
|
201
201
|
end
|
data/lib/epitools/clitools.rb
CHANGED
@@ -14,7 +14,7 @@
|
|
14
14
|
# * Allow colour
|
15
15
|
# * Don't wrap lines longer than the screen
|
16
16
|
# * Quit immediately (without paging) if there's less than one screen of text.
|
17
|
-
#
|
17
|
+
#
|
18
18
|
# You can change these options by passing a hash to `lesspipe`, like so:
|
19
19
|
#
|
20
20
|
# lesspipe(:wrap=>false) { |less| less.puts essay.to_s }
|
@@ -30,9 +30,9 @@ def lesspipe(*args)
|
|
30
30
|
else
|
31
31
|
options = {}
|
32
32
|
end
|
33
|
-
|
33
|
+
|
34
34
|
output = args.first if args.any?
|
35
|
-
|
35
|
+
|
36
36
|
params = []
|
37
37
|
params << "-R" unless options[:color] == false
|
38
38
|
params << "-S" unless options[:wrap] == true
|
@@ -42,7 +42,7 @@ def lesspipe(*args)
|
|
42
42
|
$stderr.puts "Seeking to end of stream..."
|
43
43
|
end
|
44
44
|
params << "-X"
|
45
|
-
|
45
|
+
|
46
46
|
IO.popen("less #{params * ' '}", "w") do |less|
|
47
47
|
if output
|
48
48
|
less.puts output
|
@@ -50,7 +50,7 @@ def lesspipe(*args)
|
|
50
50
|
yield less
|
51
51
|
end
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
rescue Errno::EPIPE, Interrupt
|
55
55
|
# less just quit -- eat the exception.
|
56
56
|
end
|
@@ -74,14 +74,14 @@ end
|
|
74
74
|
def cmd(*args)
|
75
75
|
|
76
76
|
cmd_args = []
|
77
|
-
|
77
|
+
|
78
78
|
for arg in args
|
79
|
-
|
79
|
+
|
80
80
|
case arg
|
81
|
-
|
81
|
+
|
82
82
|
when Array
|
83
83
|
cmd_literals = arg.shift.split(/\s+/)
|
84
|
-
|
84
|
+
|
85
85
|
for cmd_literal in cmd_literals
|
86
86
|
if cmd_literal == "?"
|
87
87
|
raise "Not enough substitution arguments" unless cmd_args.any?
|
@@ -90,22 +90,22 @@ def cmd(*args)
|
|
90
90
|
cmd_args << cmd_literal
|
91
91
|
end
|
92
92
|
end
|
93
|
-
|
93
|
+
|
94
94
|
raise "More parameters than ?'s in cmd string" if arg.any?
|
95
|
-
|
95
|
+
|
96
96
|
when String
|
97
97
|
cmd_args << arg
|
98
|
-
|
98
|
+
|
99
99
|
else
|
100
100
|
cmd_args << arg.to_s
|
101
|
-
|
101
|
+
|
102
102
|
end
|
103
|
-
|
104
|
-
end
|
103
|
+
|
104
|
+
end
|
105
105
|
|
106
106
|
p [:cmd_args, cmd_args] if $DEBUG
|
107
|
-
|
108
|
-
system(*cmd_args)
|
107
|
+
|
108
|
+
system(*cmd_args)
|
109
109
|
end
|
110
110
|
|
111
111
|
|
@@ -126,11 +126,11 @@ def prompt(message="Are you sure?", options="Yn")
|
|
126
126
|
default = defaults.first
|
127
127
|
|
128
128
|
loop do
|
129
|
-
|
129
|
+
|
130
130
|
print "#{message} (#{optstring}) "
|
131
|
-
|
131
|
+
|
132
132
|
response = STDIN.gets.strip.downcase
|
133
|
-
|
133
|
+
|
134
134
|
case response
|
135
135
|
when *opts
|
136
136
|
return response
|
@@ -139,13 +139,13 @@ def prompt(message="Are you sure?", options="Yn")
|
|
139
139
|
else
|
140
140
|
puts " |_ Invalid option: #{response.inspect}. Try again."
|
141
141
|
end
|
142
|
-
|
142
|
+
|
143
143
|
end
|
144
144
|
end
|
145
145
|
|
146
146
|
|
147
147
|
#
|
148
|
-
# Automatically install a required commandline tool using the platform's package manager (incomplete -- only supports debian :)
|
148
|
+
# Automatically install a required commandline tool using the platform's package manager (incomplete -- only supports debian :)
|
149
149
|
#
|
150
150
|
# * apt-get on debian/ubuntu
|
151
151
|
# * yum on fedora
|
@@ -154,7 +154,7 @@ end
|
|
154
154
|
#
|
155
155
|
def autoinstall(*packages)
|
156
156
|
all_packages_installed = packages.all? { |pkg| Path.which pkg }
|
157
|
-
|
157
|
+
|
158
158
|
unless all_packages_installed
|
159
159
|
cmd(["sudo apt-get install ?", packages.join(' ')])
|
160
160
|
end
|
@@ -170,24 +170,40 @@ def sudoifnotroot
|
|
170
170
|
end
|
171
171
|
end
|
172
172
|
|
173
|
+
#
|
174
|
+
# Lookup GeoIP information (city, state, country, etc.) for an IP address or hostname
|
175
|
+
#
|
176
|
+
# (Note: Must be able to find one of /usr/share/GeoIP/GeoIP{,City}.dat, or specified manually
|
177
|
+
# as (an) extra argument(s).)
|
178
|
+
#
|
179
|
+
def geoip(addr, city_data='/usr/share/GeoIP/GeoIPCity.dat', country_data='/usr/share/GeoIP/GeoIP.dat')
|
180
|
+
(
|
181
|
+
$geoip ||= begin
|
182
|
+
if city_data and File.exists? city_data
|
183
|
+
geo = GeoIP.new city_data
|
184
|
+
proc { |addr| geo.city(addr) }
|
173
185
|
|
174
|
-
|
175
|
-
|
186
|
+
elsif country_data and File.exists? country_data
|
187
|
+
geo = GeoIP.new country_data
|
188
|
+
proc { |addr| geo.country(addr) }
|
176
189
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
190
|
+
else
|
191
|
+
raise "Can't find GeoIP data files."
|
192
|
+
end
|
193
|
+
end
|
194
|
+
).call(addr)
|
195
|
+
end
|
182
196
|
|
183
|
-
elsif File.exists? GEOIP_COUNTRY_DATA
|
184
|
-
geo = GeoIP.new GEOIP_COUNTRY_DATA
|
185
|
-
proc { |addr| geo.country(addr) }
|
186
197
|
|
187
|
-
|
188
|
-
|
198
|
+
#
|
199
|
+
# Search the PATH environment variable for binaries, returning the first one that exists
|
200
|
+
#
|
201
|
+
def which(*bins)
|
202
|
+
bins.flatten.each do |bin|
|
203
|
+
ENV["PATH"].split(":").each do |dir|
|
204
|
+
full_path = File.join(dir, bin)
|
205
|
+
return full_path if File.exists? full_path
|
189
206
|
end
|
190
207
|
end
|
191
|
-
|
192
|
-
$geoip.call(addr)
|
208
|
+
nil
|
193
209
|
end
|
@@ -178,6 +178,35 @@ module Enumerable
|
|
178
178
|
|
179
179
|
alias_method :cut_between, :split_between
|
180
180
|
|
181
|
+
|
182
|
+
#
|
183
|
+
# Map elements of this Enumerable in parallel using a pool full of Threads
|
184
|
+
#
|
185
|
+
# eg: repos.parallel_map { |repo| system "git pull #{repo}" }
|
186
|
+
#
|
187
|
+
def parallel_map(num_workers=8, &block)
|
188
|
+
require 'thread'
|
189
|
+
|
190
|
+
queue = Queue.new
|
191
|
+
each { |e| queue.push e }
|
192
|
+
|
193
|
+
Enumerator.new do |y|
|
194
|
+
workers = (0...num_workers).map do
|
195
|
+
Thread.new do
|
196
|
+
begin
|
197
|
+
while e = queue.pop(true)
|
198
|
+
y << block.call(e)
|
199
|
+
end
|
200
|
+
rescue ThreadError
|
201
|
+
end
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
workers.map(&:join)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
|
181
210
|
#
|
182
211
|
# Sum the elements
|
183
212
|
#
|
@@ -51,15 +51,15 @@ class Matrix
|
|
51
51
|
alias_method :draw, :print
|
52
52
|
|
53
53
|
#
|
54
|
-
# Allow mathematical operations (*, /, +, -) with a
|
54
|
+
# Allow mathematical operations (*, /, +, -) with a scalar (integer or float) on the right side.
|
55
55
|
#
|
56
56
|
# eg: Matrix.zero(3) + 5
|
57
57
|
#
|
58
|
-
%
|
58
|
+
%i[* / + -].each do |op|
|
59
59
|
class_eval %{
|
60
60
|
def #{op}(other)
|
61
61
|
case other
|
62
|
-
when
|
62
|
+
when Numeric
|
63
63
|
map { |e| e #{op} other }
|
64
64
|
else
|
65
65
|
super(other)
|
@@ -30,13 +30,13 @@ class Numeric
|
|
30
30
|
#
|
31
31
|
# Examples:
|
32
32
|
# 234234234523.clamp(0..100) #=> 100
|
33
|
-
# 12.clamp(0..100) #=> 12
|
33
|
+
# 12.clamp(0..100) #=> 12
|
34
34
|
# -38817112.clamp(0..100) #=> 0
|
35
35
|
#
|
36
36
|
def clamp(range)
|
37
37
|
if self < range.first
|
38
38
|
range.first
|
39
|
-
elsif self >= range.last
|
39
|
+
elsif self >= range.last
|
40
40
|
if range.exclude_end?
|
41
41
|
range.last - 1
|
42
42
|
else
|
@@ -51,7 +51,7 @@ class Numeric
|
|
51
51
|
# Time methods
|
52
52
|
#
|
53
53
|
{
|
54
|
-
|
54
|
+
|
55
55
|
'second' => 1,
|
56
56
|
'minute' => 60,
|
57
57
|
'hour' => 60 * 60,
|
@@ -59,23 +59,23 @@ class Numeric
|
|
59
59
|
'week' => 60 * 60 * 24 * 7,
|
60
60
|
'month' => 60 * 60 * 24 * 30,
|
61
61
|
'year' => 60 * 60 * 24 * 365,
|
62
|
-
|
62
|
+
|
63
63
|
}.each do |unit, scale|
|
64
64
|
define_method(unit) { self * scale }
|
65
65
|
define_method(unit+'s') { self * scale }
|
66
66
|
end
|
67
|
-
|
67
|
+
|
68
68
|
def ago
|
69
69
|
Time.now - self
|
70
70
|
end
|
71
|
-
|
71
|
+
|
72
72
|
def from_now
|
73
73
|
Time.now + self
|
74
74
|
end
|
75
75
|
|
76
76
|
#
|
77
|
-
# If `n.times` is like `each`, `n.things` is like `map`. Return
|
78
|
-
#
|
77
|
+
# If `n.times` is like `each`, `n.things` is like `map`. Return
|
78
|
+
#
|
79
79
|
def things(&block)
|
80
80
|
if block_given?
|
81
81
|
Array.new(self, &block)
|
@@ -255,25 +255,25 @@ class Integer
|
|
255
255
|
def to_hex
|
256
256
|
"%0.2x" % self
|
257
257
|
end
|
258
|
-
|
258
|
+
|
259
259
|
#
|
260
260
|
# Convert the number to an array of bits (least significant digit first, or little-endian).
|
261
261
|
#
|
262
262
|
def to_bits
|
263
|
-
# TODO: Why does
|
263
|
+
# TODO: Why does this go into an infinite loop in 1.8.7?
|
264
264
|
("%b" % self).chars.to_a.reverse.map(&:to_i)
|
265
265
|
end
|
266
266
|
alias_method :bits, :to_bits
|
267
|
-
|
267
|
+
|
268
268
|
#
|
269
269
|
# Cached constants for base62 encoding
|
270
270
|
#
|
271
|
-
BASE62_DIGITS = ['0'..'9', 'A'..'Z', 'a'..'z'].map(&:to_a).flatten
|
271
|
+
BASE62_DIGITS = ['0'..'9', 'A'..'Z', 'a'..'z'].map(&:to_a).flatten
|
272
272
|
BASE62_BASE = BASE62_DIGITS.size
|
273
273
|
|
274
274
|
#
|
275
275
|
# Convert a number to a string representation (in "base62" encoding).
|
276
|
-
#
|
276
|
+
#
|
277
277
|
# Base62 encoding represents the number using the characters: 0..9, A..Z, a..z
|
278
278
|
#
|
279
279
|
# It's the same scheme that url shorteners and YouTube uses for their
|
@@ -283,16 +283,16 @@ class Integer
|
|
283
283
|
result = []
|
284
284
|
remainder = self
|
285
285
|
max_power = ( Math.log(self) / Math.log(BASE62_BASE) ).floor
|
286
|
-
|
286
|
+
|
287
287
|
max_power.downto(0) do |power|
|
288
288
|
divisor = BASE62_BASE**power
|
289
|
-
#p [:div, divisor, :rem, remainder]
|
289
|
+
#p [:div, divisor, :rem, remainder]
|
290
290
|
digit, remainder = remainder.divmod(divisor)
|
291
291
|
result << digit
|
292
292
|
end
|
293
|
-
|
293
|
+
|
294
294
|
result << remainder if remainder > 0
|
295
|
-
|
295
|
+
|
296
296
|
result.map{|digit| BASE62_DIGITS[digit]}.join ''
|
297
297
|
end
|
298
298
|
|
@@ -312,7 +312,7 @@ class Integer
|
|
312
312
|
|
313
313
|
loop do
|
314
314
|
if current.prime?
|
315
|
-
result << current
|
315
|
+
result << current
|
316
316
|
return result if result.size >= self
|
317
317
|
end
|
318
318
|
current += 1
|
@@ -324,7 +324,7 @@ class Integer
|
|
324
324
|
#
|
325
325
|
def factors
|
326
326
|
Prime # autoload the prime module
|
327
|
-
prime_division.map { |n,count| [n]*count }.flatten
|
327
|
+
prime_division.map { |n,count| [n]*count }.flatten
|
328
328
|
end
|
329
329
|
|
330
330
|
#
|
@@ -352,32 +352,33 @@ class Integer
|
|
352
352
|
end
|
353
353
|
|
354
354
|
#
|
355
|
-
#
|
355
|
+
# Adds integer silcing (returning the bits) and raw-bytes
|
356
356
|
#
|
357
357
|
# (This is necessary because [] is defined directly on the classes, and a mixin
|
358
358
|
# module will still be overridden by Big/Fixnum's native [] method.)
|
359
359
|
#
|
360
|
-
[Bignum, Fixnum].each do |klass|
|
361
|
-
|
360
|
+
(RUBY_VERSION >= "2.4" ? [Integer] : [Bignum, Fixnum]).each do |klass|
|
362
361
|
klass.class_eval do
|
363
|
-
|
362
|
+
|
364
363
|
alias_method :bit, :"[]"
|
365
|
-
|
364
|
+
|
366
365
|
#
|
367
|
-
#
|
366
|
+
# Slice the bits of an integer by passing a range (eg: 1217389172842[0..5] #=> [0, 1, 0, 1, 0, 1])
|
368
367
|
#
|
369
368
|
def [](arg)
|
370
369
|
case arg
|
371
370
|
when Integer
|
372
|
-
|
371
|
+
bit(arg)
|
373
372
|
when Range
|
374
|
-
|
373
|
+
bits[arg]
|
375
374
|
end
|
376
375
|
end
|
377
|
-
|
376
|
+
|
378
377
|
#
|
379
378
|
# Convert the integer into its sequence of bytes (little endian format: lowest-order-byte first)
|
380
379
|
#
|
380
|
+
# TODO: This could be made much more efficient!
|
381
|
+
#
|
381
382
|
def bytes
|
382
383
|
nbytes = (bit_length / 8.0).ceil
|
383
384
|
|
@@ -386,6 +387,9 @@ end
|
|
386
387
|
end
|
387
388
|
end
|
388
389
|
|
390
|
+
def big_endian_bytes
|
391
|
+
bytes.reverse
|
392
|
+
end
|
389
393
|
end
|
390
394
|
end
|
391
395
|
|
@@ -415,7 +419,7 @@ class Time
|
|
415
419
|
def in_words
|
416
420
|
delta = (Time.now-self).to_i
|
417
421
|
a = delta.abs
|
418
|
-
|
422
|
+
|
419
423
|
amount = case a
|
420
424
|
when 0
|
421
425
|
'just now'
|
@@ -436,16 +440,16 @@ class Time
|
|
436
440
|
else
|
437
441
|
"year".amount(a/1.year)
|
438
442
|
end
|
439
|
-
|
443
|
+
|
440
444
|
if delta < 0
|
441
445
|
amount += " from now"
|
442
446
|
elsif delta > 0
|
443
447
|
amount += " ago"
|
444
448
|
end
|
445
|
-
|
449
|
+
|
446
450
|
amount
|
447
451
|
end
|
448
|
-
|
452
|
+
|
449
453
|
end
|
450
454
|
|
451
455
|
|
data/lib/epitools/minimal.rb
CHANGED
data/lib/epitools/path.rb
CHANGED
@@ -123,6 +123,14 @@ class Path
|
|
123
123
|
###############################################################################
|
124
124
|
|
125
125
|
def initialize(newpath, hints={})
|
126
|
+
if hints[:unlink_when_garbage_collected]
|
127
|
+
backup_path = path.dup
|
128
|
+
puts "unlinking #{backup_path} after gc!"
|
129
|
+
ObjectSpace.define_finalizer self do |object_id|
|
130
|
+
File.unlink backup_path
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
126
134
|
send("path=", newpath, hints)
|
127
135
|
end
|
128
136
|
|
@@ -583,6 +591,28 @@ class Path
|
|
583
591
|
end
|
584
592
|
end
|
585
593
|
|
594
|
+
def media_tags
|
595
|
+
require 'taglib'
|
596
|
+
|
597
|
+
tags = nil
|
598
|
+
|
599
|
+
TagLib::FileRef.open(path) do |fileref|
|
600
|
+
unless fileref.null?
|
601
|
+
tags = fileref.tag
|
602
|
+
result = {}
|
603
|
+
getters = tags.class.instance_methods(false).group_by {|m| m[/^.+[^=]/] }.map { |k,vs| vs.size == 2 ? k.to_sym : nil }.compact
|
604
|
+
getters.each do |getter|
|
605
|
+
tags[getter] = tags.public_send(getter)
|
606
|
+
end
|
607
|
+
end
|
608
|
+
end
|
609
|
+
|
610
|
+
tags
|
611
|
+
end
|
612
|
+
|
613
|
+
alias_method :id3, :media_tags
|
614
|
+
alias_method :id3tags, :media_tags
|
615
|
+
|
586
616
|
#
|
587
617
|
# Retrieve one of this file's xattrs
|
588
618
|
#
|
@@ -969,10 +999,12 @@ class Path
|
|
969
999
|
if dest.startswith("/")
|
970
1000
|
Path.ln_s(self, dest)
|
971
1001
|
else
|
972
|
-
Path.ln_s(self, self / dest
|
1002
|
+
Path.ln_s(self, self / dest)
|
973
1003
|
end
|
974
1004
|
end
|
975
1005
|
|
1006
|
+
alias_method :symlink_to, :ln_s
|
1007
|
+
|
976
1008
|
## Owners and permissions
|
977
1009
|
|
978
1010
|
def chmod(mode)
|
@@ -1054,7 +1086,7 @@ class Path
|
|
1054
1086
|
Zlib.deflate(read, level)
|
1055
1087
|
end
|
1056
1088
|
alias_method :gzip, :deflate
|
1057
|
-
|
1089
|
+
|
1058
1090
|
|
1059
1091
|
#
|
1060
1092
|
# gunzip the file, returning the result as a string
|
@@ -1270,21 +1302,24 @@ class Path
|
|
1270
1302
|
# TODO: Remove the tempfile when the Path object is garbage collected or freed.
|
1271
1303
|
#
|
1272
1304
|
def self.tmpfile(prefix="tmp")
|
1273
|
-
path = Path
|
1305
|
+
path = Path.new Tempfile.new(prefix).path, unlink_when_garbage_collected: true
|
1274
1306
|
yield path if block_given?
|
1275
1307
|
path
|
1276
1308
|
end
|
1277
1309
|
alias_class_method :tempfile, :tmpfile
|
1278
1310
|
alias_class_method :tmp, :tmpfile
|
1279
1311
|
|
1312
|
+
|
1313
|
+
#
|
1314
|
+
# Create a uniqely named directory in /tmp
|
1315
|
+
#
|
1280
1316
|
def self.tmpdir(prefix="tmp")
|
1281
1317
|
t = tmpfile
|
1282
|
-
|
1283
|
-
# FIXME: These operations should be atomic
|
1284
|
-
t.rm; t.mkdir
|
1285
|
-
|
1318
|
+
t.rm; t.mkdir # FIXME: These two operations should be made atomic
|
1286
1319
|
t
|
1287
1320
|
end
|
1321
|
+
alias_class_method :tempdir, :tmpdir
|
1322
|
+
|
1288
1323
|
|
1289
1324
|
def self.home
|
1290
1325
|
Path[ENV['HOME']]
|
data/spec/browser_spec.rb
CHANGED
@@ -11,7 +11,7 @@ end
|
|
11
11
|
describe Browser do
|
12
12
|
|
13
13
|
before :all do
|
14
|
-
@browser = Browser.new
|
14
|
+
@browser = Browser.new use_cache: true
|
15
15
|
end
|
16
16
|
|
17
17
|
after :all do
|
@@ -23,16 +23,16 @@ describe Browser do
|
|
23
23
|
page = @browser.get(url)
|
24
24
|
@browser.cache.get(url).should_not == nil
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
it "googles" do
|
28
28
|
page = @browser.get("http://google.com")
|
29
29
|
page.body["Feeling Lucky"].should_not be_empty
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
it "googles (cached)" do
|
33
33
|
lambda{ @browser.get("http://google.com").body }.should_not raise_error
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
it "delegates" do
|
37
37
|
lambda{ @browser.head("http://google.com").body }.should_not raise_error
|
38
38
|
@browser.respond_to?(:post).should == true
|
@@ -46,9 +46,10 @@ end
|
|
46
46
|
describe Browser::Cache do
|
47
47
|
|
48
48
|
before :all do
|
49
|
+
cache_file = "temp-cache.db"
|
49
50
|
@agent = Mechanize.new
|
50
|
-
Browser::Cache.new(@agent).delete!
|
51
|
-
@cache = Browser::Cache.new(@agent)
|
51
|
+
Browser::Cache.new(cache_file, @agent).delete!
|
52
|
+
@cache = Browser::Cache.new(cache_file, @agent)
|
52
53
|
end
|
53
54
|
|
54
55
|
after :all do
|
data/spec/clitools_spec.rb
CHANGED
@@ -1,30 +1,46 @@
|
|
1
|
+
require 'epitools/minimal'
|
1
2
|
require 'epitools/clitools'
|
2
3
|
|
3
|
-
describe
|
4
|
+
describe Object do
|
4
5
|
|
5
|
-
it "
|
6
|
-
|
7
|
-
|
6
|
+
# it "'highlight's" do
|
7
|
+
# color = :light_yellow
|
8
|
+
# highlighted = "xxx#{"match".send(color)}zzz"
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
end
|
10
|
+
# "xxxmatchzzz".highlight(/match/, color).should == highlighted
|
11
|
+
# "xxxmatchzzz".highlight("match", color).should == highlighted
|
12
|
+
# "xxxmatchzzz".highlight(/m.+h/, color).should == highlighted
|
13
|
+
# "xxxmatchzzz".highlight(/MATCH/i, color).should == highlighted
|
14
|
+
# end
|
14
15
|
|
15
|
-
it "
|
16
|
-
|
17
|
-
|
18
|
-
end
|
19
|
-
|
20
|
-
it "
|
16
|
+
# it "'highlight's with a block" do
|
17
|
+
# result = "xxxmatchxxx".highlight(/match/) { |match| "<8>#{match}</8>" }
|
18
|
+
# result.should == "xxx<8>match</8>xxx"
|
19
|
+
# end
|
20
|
+
|
21
|
+
it "'cmd's" do
|
21
22
|
cmd( ['test -f ?', __FILE__] ).should == true
|
22
23
|
cmd( ['test -d ?', __FILE__] ).should == false
|
23
|
-
cmd( "test", "-f", __FILE__ ).should == true
|
24
|
+
cmd( "test", "-f", __FILE__ ).should == true
|
24
25
|
cmd( "test", "-d", __FILE__ ).should == false
|
25
|
-
|
26
|
-
|
27
|
-
|
26
|
+
|
27
|
+
-> { cmd( ["test -f ? ?", __FILE__] ) }.should raise_error(TypeError) # more ?'s than args
|
28
|
+
-> { cmd( ["test -f", __FILE__] ) }.should raise_error(RuntimeError) # more args than ?'s
|
29
|
+
end
|
30
|
+
|
31
|
+
it "'which'es" do
|
32
|
+
which("totally nonexistant", "probably nonexistant", "ls", "df").should_not == nil
|
33
|
+
which("totally nonexistant", "probably nonexistant").should == nil
|
34
|
+
which("ls", "df").should =~ /\/ls$/
|
28
35
|
end
|
29
|
-
|
36
|
+
|
37
|
+
it "'geoip's" do
|
38
|
+
geoip("128.100.100.128").country_name.should == "Canada"
|
39
|
+
|
40
|
+
-> { geoip("butt"*20) }.should raise_error(SocketError)
|
41
|
+
|
42
|
+
$geoip = nil
|
43
|
+
-> { geoip("8.8.4.4", nil, nil) }.should raise_error(RuntimeError)
|
44
|
+
end
|
45
|
+
|
30
46
|
end
|
data/spec/core_ext_spec.rb
CHANGED
@@ -28,13 +28,13 @@ describe Object do
|
|
28
28
|
|
29
29
|
lambda {
|
30
30
|
time("time test") { raise "ERROR" }
|
31
|
-
}.should raise_error
|
31
|
+
}.should raise_error(RuntimeError)
|
32
32
|
end
|
33
33
|
|
34
34
|
it "benches" do
|
35
35
|
lambda { bench { rand } }.should_not raise_error
|
36
36
|
lambda { bench(20) { rand } }.should_not raise_error
|
37
|
-
lambda { bench }.should raise_error
|
37
|
+
lambda { bench }.should raise_error(RuntimeError)
|
38
38
|
lambda { bench(:rand => proc { rand }, :notrand => proc { 1 }) }.should_not raise_error
|
39
39
|
lambda { bench(200, :rand => proc { rand }, :notrand => proc { 1 }) }.should_not raise_error
|
40
40
|
end
|
@@ -49,15 +49,15 @@ describe Object do
|
|
49
49
|
s.try(:c).should == nil
|
50
50
|
|
51
51
|
lambda { s.try(:c) }.should_not raise_error
|
52
|
-
lambda { s.c }.should raise_error
|
52
|
+
lambda { s.c }.should raise_error(NoMethodError)
|
53
53
|
|
54
54
|
def s.test(a); a; end
|
55
55
|
|
56
|
-
s.test(1).should
|
57
|
-
s.try(:test, 1).should
|
56
|
+
s.test(1).should == 1
|
57
|
+
s.try(:test, 1).should == 1
|
58
58
|
|
59
|
-
lambda { s.test }.should raise_error
|
60
|
-
lambda { s.try(:test) }.should raise_error
|
59
|
+
lambda { s.test }.should raise_error(ArgumentError)
|
60
|
+
lambda { s.try(:test) }.should raise_error(ArgumentError)
|
61
61
|
|
62
62
|
def s.blocky; yield; end
|
63
63
|
|
data/spec/path_spec.rb
CHANGED
@@ -234,7 +234,7 @@ describe Path do
|
|
234
234
|
|
235
235
|
path.touch
|
236
236
|
lambda { path.rename(:ext=>".dat") }.should raise_error
|
237
|
-
|
237
|
+
|
238
238
|
dest.rm
|
239
239
|
path.rename!(:ext=>".dat")
|
240
240
|
path.to_s.should_not == old_name
|
@@ -483,16 +483,15 @@ describe Path do
|
|
483
483
|
end
|
484
484
|
|
485
485
|
it 'symlinks relative dirs' do
|
486
|
-
raise "Path.ln_s arguments are backwards. It needs to be something easier to remember, like 'Path#symlink_to'"
|
487
486
|
tmpdir = Path.tmpdir
|
488
487
|
|
489
488
|
symlink = (tmpdir/"a_new_link")
|
490
|
-
|
489
|
+
Path["../../etc/passwd"].ln_s symlink
|
491
490
|
|
492
491
|
symlink.symlink?.should == true
|
493
492
|
|
494
493
|
symlink.rm
|
495
|
-
|
494
|
+
tmpdir.rm
|
496
495
|
end
|
497
496
|
|
498
497
|
it 'swaps two files' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: epitools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.99
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- epitron
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
124
124
|
version: '0'
|
125
125
|
requirements: []
|
126
126
|
rubyforge_project:
|
127
|
-
rubygems_version: 2.
|
127
|
+
rubygems_version: 2.6.8
|
128
128
|
signing_key:
|
129
129
|
specification_version: 3
|
130
130
|
summary: Not utils... METILS!
|