tansaku 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -2
- data/lib/tansaku/cli.rb +2 -0
- data/lib/tansaku/crawler.rb +4 -10
- data/lib/tansaku/fixtures/paths.txt +1450 -1449
- data/lib/tansaku/version.rb +1 -1
- data/tansaku.gemspec +1 -1
- metadata +8 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e1df6b0d432754c0859695d783db4cbd51523593c787aa5e42cb8553003d1d9d
|
4
|
+
data.tar.gz: 48c390edf8d05445e2165845953852eee656a4e1e9d55aae005296e37e2d3f89
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bfa5f53b9e860efe887ad22d1bd419c97466bd9a42e589b6c8ef5540b9d87e518254eedca346c31b72e19d182b0180784a49e4ada9960919e173101e59deceee
|
7
|
+
data.tar.gz: 11af402ceafdad601111261d733a6a47c1377a8d1cbd2c2227918405580f23d7d3404c196227701af7c171518c1c6adb21dce3ebe30ddb246342d39a26733df8
|
data/.travis.yml
CHANGED
data/lib/tansaku/cli.rb
CHANGED
data/lib/tansaku/crawler.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
require "cgi"
|
4
4
|
require "net/http"
|
5
|
-
require "
|
5
|
+
require "parallel"
|
6
6
|
require "uri"
|
7
7
|
|
8
8
|
module Tansaku
|
@@ -31,16 +31,10 @@ module Tansaku
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def crawl
|
34
|
-
|
35
|
-
|
36
|
-
urls.each do |url|
|
37
|
-
pool.process do
|
38
|
-
results << url if online?(url)
|
39
|
-
end
|
34
|
+
results = Parallel.map(urls, in_threads: threads) do |url|
|
35
|
+
url if online?(url)
|
40
36
|
end
|
41
|
-
|
42
|
-
|
43
|
-
results
|
37
|
+
results.compact
|
44
38
|
end
|
45
39
|
|
46
40
|
private
|