scrapey 0.0.19 → 0.0.22
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/scrapey.rb +7 -5
- data/lib/scrapey/cache.rb +6 -0
- data/lib/scrapey/cache/disk.rb +4 -2
- data/lib/scrapey/constants.rb +1 -1
- data/lib/scrapey/core.rb +0 -8
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
---
|
|
2
2
|
!binary "U0hBMQ==":
|
|
3
3
|
metadata.gz: !binary |-
|
|
4
|
-
|
|
4
|
+
ZjczYTJlODBiMjJkMjk1ZThiZjZmYTFiOWIxM2RiZDUzNGI2N2Y3Nw==
|
|
5
5
|
data.tar.gz: !binary |-
|
|
6
|
-
|
|
6
|
+
ZmI2ZTcwYTcxMDgxZDA4MzU5MTAzMGUwZmM5MmJmZmJiYzZjZWVmYQ==
|
|
7
7
|
SHA512:
|
|
8
8
|
metadata.gz: !binary |-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
YWM2NTBiOTQ2OTU4YTNjY2Q3ZmExODVhZGEzM2E5ODViYWVjM2I3YWY5MDM2
|
|
10
|
+
NjIxOWYwOGQ1Yjk3YjViZDQzYTAwM2I1NTRlZDdmMTM4MGYyMjM1MTVmZTA0
|
|
11
|
+
Zjc1NGI4YzMyYTY1ZTc4NzUyYzAyNGZhNzQ1NWQ2ZTRkM2U3M2I=
|
|
12
12
|
data.tar.gz: !binary |-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
MTA1ZTQ4NTg1NjJkYmZkZmVlYzEzNWQwYmFmNjk2ZWNlMDhjYmIwM2ZmNjM2
|
|
14
|
+
OWY0Njk3YmVmZjVmMjdlYTU0NTM0NjRjZmY3ZjUxYTAxNjk3ZmYzMTNlN2Ji
|
|
15
|
+
MTYxMjE0OGE5MWRkZDUxMTJiYzNlYWQ4NmUxM2EwM2QyNTk2NjA=
|
data/lib/scrapey.rb
CHANGED
|
@@ -11,14 +11,16 @@ require "scrapey/database"
|
|
|
11
11
|
require "scrapey/multi"
|
|
12
12
|
require "scrapey/tee"
|
|
13
13
|
|
|
14
|
-
require 'addressable/uri'
|
|
15
14
|
|
|
16
15
|
EMAIL_REGEX = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}\b/i
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
unless ENV['USE_URI']
|
|
18
|
+
require 'addressable/uri'
|
|
19
|
+
class URI::Parser
|
|
20
|
+
def split url
|
|
21
|
+
a = Addressable::URI::parse url
|
|
22
|
+
[a.scheme, a.userinfo, a.host, a.port, nil, a.path, nil, a.query, a.fragment]
|
|
23
|
+
end
|
|
22
24
|
end
|
|
23
25
|
end
|
|
24
26
|
|
data/lib/scrapey/cache.rb
CHANGED
|
@@ -8,6 +8,12 @@ module Scrapey
|
|
|
8
8
|
require 'scrapey/cache/disk'
|
|
9
9
|
@config['cache_dir'] ||= "#{BASEDIR}/cache"
|
|
10
10
|
FileUtils.mkdir_p @config['cache_dir']
|
|
11
|
+
["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"].each do |l1|
|
|
12
|
+
["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"].each do |l2|
|
|
13
|
+
FileUtils.mkdir_p "#{@config['cache_dir']}/#{l1}/#{l2}"
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
11
17
|
end
|
|
12
18
|
end
|
|
13
19
|
|
data/lib/scrapey/cache/disk.rb
CHANGED
|
@@ -3,7 +3,7 @@ require 'fileutils'
|
|
|
3
3
|
module Scrapey
|
|
4
4
|
|
|
5
5
|
def cache_filename url
|
|
6
|
-
@config['cache_dir'] + "/" + Digest::MD5.hexdigest(url) + ".cache"
|
|
6
|
+
@config['cache_dir'] + "/" + Digest::MD5.hexdigest(url).sub(/(.)(.)/, '\1/\2/\1\2') + ".cache"
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
def is_cached? url
|
|
@@ -35,6 +35,8 @@ module Scrapey
|
|
|
35
35
|
Mechanize::Page.new URI.parse(url), [], Marshal.load(Zlib::Inflate.inflate(File.open(filename, "rb"){|f| f.read})), nil, @agent
|
|
36
36
|
rescue Exception => e
|
|
37
37
|
puts e.message
|
|
38
|
+
# delete_cache url
|
|
39
|
+
# Mechanize::Page.new URI.parse(url), [], '<html></html>', nil, @agent
|
|
38
40
|
end
|
|
39
41
|
end
|
|
40
42
|
|
|
@@ -47,4 +49,4 @@ module Scrapey
|
|
|
47
49
|
FileUtils.rm(cache_filename(url)) rescue nil
|
|
48
50
|
end
|
|
49
51
|
|
|
50
|
-
end
|
|
52
|
+
end
|
data/lib/scrapey/constants.rb
CHANGED
data/lib/scrapey/core.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: scrapey
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.22
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- P Guardiario
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-03-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: mechanize
|