ero_getter 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +12 -0
- data/Gemfile +5 -0
- data/VERSION +1 -1
- data/ero_getter.gemspec +78 -0
- data/lib/ero_getter/downloader/base.rb +28 -1
- data/lib/ero_getter.rb +4 -1
- data/spec/downloader/base_spec.rb +19 -7
- data/spec/ero_getter_spec.rb +4 -0
- data/spec/samples/sample.html +34 -0
- data/spec/spec_helper.rb +6 -1
- metadata +37 -2
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.1
|
data/ero_getter.gemspec
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "ero_getter"
|
8
|
+
s.version = "0.1.1"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["masarakki"]
|
12
|
+
s.date = "2012-05-12"
|
13
|
+
s.description = "ero getter"
|
14
|
+
s.email = "masaki@hisme.net"
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE.txt",
|
17
|
+
"README.rdoc"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
".rspec",
|
22
|
+
".rvmrc",
|
23
|
+
".travis.yml",
|
24
|
+
"Gemfile",
|
25
|
+
"LICENSE.txt",
|
26
|
+
"README.rdoc",
|
27
|
+
"Rakefile",
|
28
|
+
"VERSION",
|
29
|
+
"ero_getter.gemspec",
|
30
|
+
"lib/ero_getter.rb",
|
31
|
+
"lib/ero_getter/downloader.rb",
|
32
|
+
"lib/ero_getter/downloader/base.rb",
|
33
|
+
"spec/downloader/base_spec.rb",
|
34
|
+
"spec/downloader_spec.rb",
|
35
|
+
"spec/ero_getter_spec.rb",
|
36
|
+
"spec/samples/sample.html",
|
37
|
+
"spec/spec_helper.rb"
|
38
|
+
]
|
39
|
+
s.homepage = "http://github.com/masarakki/ero_getter"
|
40
|
+
s.licenses = ["MIT"]
|
41
|
+
s.require_paths = ["lib"]
|
42
|
+
s.rubygems_version = "1.8.24"
|
43
|
+
s.summary = "ero getter"
|
44
|
+
|
45
|
+
if s.respond_to? :specification_version then
|
46
|
+
s.specification_version = 3
|
47
|
+
|
48
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
49
|
+
s.add_runtime_dependency(%q<activesupport>, [">= 0"])
|
50
|
+
s.add_runtime_dependency(%q<httpclient>, [">= 0"])
|
51
|
+
s.add_runtime_dependency(%q<rubyzip>, [">= 0"])
|
52
|
+
s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
|
53
|
+
s.add_runtime_dependency(%q<jruby-openssl>, [">= 0"])
|
54
|
+
s.add_development_dependency(%q<rspec>, [">= 0"])
|
55
|
+
s.add_development_dependency(%q<bundler>, [">= 0"])
|
56
|
+
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
57
|
+
else
|
58
|
+
s.add_dependency(%q<activesupport>, [">= 0"])
|
59
|
+
s.add_dependency(%q<httpclient>, [">= 0"])
|
60
|
+
s.add_dependency(%q<rubyzip>, [">= 0"])
|
61
|
+
s.add_dependency(%q<nokogiri>, [">= 0"])
|
62
|
+
s.add_dependency(%q<jruby-openssl>, [">= 0"])
|
63
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
64
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
65
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
66
|
+
end
|
67
|
+
else
|
68
|
+
s.add_dependency(%q<activesupport>, [">= 0"])
|
69
|
+
s.add_dependency(%q<httpclient>, [">= 0"])
|
70
|
+
s.add_dependency(%q<rubyzip>, [">= 0"])
|
71
|
+
s.add_dependency(%q<nokogiri>, [">= 0"])
|
72
|
+
s.add_dependency(%q<jruby-openssl>, [">= 0"])
|
73
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
74
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
75
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
@@ -1,10 +1,12 @@
|
|
1
1
|
require 'active_support/inflector'
|
2
2
|
require 'httpclient'
|
3
|
+
require 'nokogiri'
|
3
4
|
|
4
5
|
class EroGetter::Downloader::Base
|
5
|
-
def initialize(url)
|
6
|
+
def initialize(url, direction = 0)
|
6
7
|
raise unless url.match url_regex
|
7
8
|
@url = url
|
9
|
+
@direction = direction
|
8
10
|
end
|
9
11
|
|
10
12
|
def base_dir
|
@@ -15,6 +17,18 @@ class EroGetter::Downloader::Base
|
|
15
17
|
@http_client ||= HTTPClient.new
|
16
18
|
end
|
17
19
|
|
20
|
+
def url
|
21
|
+
@url
|
22
|
+
end
|
23
|
+
|
24
|
+
def document
|
25
|
+
@document ||= Nokogiri::HTML(open(url).read)
|
26
|
+
end
|
27
|
+
|
28
|
+
def title
|
29
|
+
@title ||= document.title
|
30
|
+
end
|
31
|
+
|
18
32
|
class << self
|
19
33
|
def name(site_name)
|
20
34
|
define_method(:name) do
|
@@ -28,5 +42,18 @@ class EroGetter::Downloader::Base
|
|
28
42
|
end
|
29
43
|
EroGetter.add_mapping(regex, self)
|
30
44
|
end
|
45
|
+
|
46
|
+
def target(css_selector, &block)
|
47
|
+
define_method(:targets) do
|
48
|
+
unless instance_variable_defined?(:@targets)
|
49
|
+
items = document.css(css_selector).map do |elm|
|
50
|
+
yield(elm)
|
51
|
+
end
|
52
|
+
instance_variable_set(:@targets, items.compact)
|
53
|
+
end
|
54
|
+
instance_variable_get(:@targets)
|
55
|
+
end
|
56
|
+
end
|
31
57
|
end
|
58
|
+
|
32
59
|
end
|
data/lib/ero_getter.rb
CHANGED
@@ -1,33 +1,45 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe EroGetter::Downloader::Base do
|
4
|
-
let
|
5
|
-
before
|
4
|
+
let(:regex) { %r{http://example.net/\d+.html} }
|
5
|
+
before do
|
6
6
|
_regex = regex
|
7
|
-
|
7
|
+
@klazz = Class.new(EroGetter::Downloader::Base) do
|
8
8
|
name 'NijiEro BBS'
|
9
9
|
url _regex
|
10
|
+
target "ul#sources li a" do |elm|
|
11
|
+
elm[:href]
|
12
|
+
end
|
10
13
|
end
|
14
|
+
@klazz.stub(:to_s).and_return('TestClass')
|
11
15
|
end
|
12
16
|
|
13
17
|
describe "assign url_mapping" do
|
14
18
|
it { EroGetter.url_mapping.should have_key regex }
|
15
|
-
it { EroGetter.url_mapping[regex].should ==
|
19
|
+
it { EroGetter.url_mapping[regex].should == @klazz }
|
16
20
|
end
|
17
21
|
|
18
22
|
describe :instance_methods do
|
19
|
-
subject { @
|
23
|
+
subject { @dl }
|
20
24
|
context :good do
|
21
|
-
before
|
25
|
+
before do
|
26
|
+
@dl = @klazz.new('http://example.net/10101010.html')
|
27
|
+
@dl.stub(:open).and_return(File.open(sample_path('sample.html')))
|
28
|
+
end
|
22
29
|
its(:name) { should == 'NijiEro BBS' }
|
23
30
|
its(:url_regex) { should == regex }
|
24
31
|
its(:base_dir) { should == 'test_class' }
|
25
32
|
its(:http_client) { should be_a HTTPClient }
|
33
|
+
its(:document) { should be_a Nokogiri::HTML::Document }
|
34
|
+
its(:title) { should == 'EroGetter Server' }
|
35
|
+
its(:url) { should == 'http://example.net/10101010.html' }
|
36
|
+
its(:targets) { should == ['https://github.com/masarakki/ero_getter_server',
|
37
|
+
'https://github.com/masarakki/ero_getter_chrome_extension'] }
|
26
38
|
end
|
27
39
|
context :url_mismatch do
|
28
40
|
it {
|
29
41
|
lambda {
|
30
|
-
|
42
|
+
@klazz.new('http://example.com/10101010.html')
|
31
43
|
}.should raise_error
|
32
44
|
}
|
33
45
|
end
|
data/spec/ero_getter_spec.rb
CHANGED
@@ -0,0 +1,34 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
<title>EroGetter Server</title>
|
3
|
+
<link href='/bootstrap/css/bootstrap.css' rel='stylesheet' type='text/css' />
|
4
|
+
<script src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js' type='text/javascript'></script>
|
5
|
+
<div class='container'>
|
6
|
+
<h1>EroGetter Server</h1>
|
7
|
+
<div class='row'>
|
8
|
+
<div class='span6'>
|
9
|
+
<h2>Usage</h2>
|
10
|
+
<dl>
|
11
|
+
<dt>GET /</dt>
|
12
|
+
<dd>show usage and queues (this page)</dd>
|
13
|
+
<dt>POST /</dt>
|
14
|
+
<dd>add url to queue</dd>
|
15
|
+
</dl>
|
16
|
+
<h2>Source</h2>
|
17
|
+
<ul id='sources'>
|
18
|
+
<li>
|
19
|
+
<a href='https://github.com/masarakki/ero_getter_server'>EroGetter Server</a>
|
20
|
+
</li>
|
21
|
+
<li>
|
22
|
+
<a href='https://github.com/masarakki/ero_getter_chrome_extension'>EroGetter Chrome Extension</a>
|
23
|
+
</li>
|
24
|
+
</ul>
|
25
|
+
</div>
|
26
|
+
<div class='span6'>
|
27
|
+
<h1>process</h1>
|
28
|
+
<h2>pid</h2>
|
29
|
+
<p>7016</p>
|
30
|
+
<h2>queues</h2>
|
31
|
+
<p>empty</p>
|
32
|
+
</div>
|
33
|
+
</div>
|
34
|
+
</div>
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ero_getter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -59,6 +59,38 @@ dependencies:
|
|
59
59
|
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: nokogiri
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: jruby-openssl
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
62
94
|
- !ruby/object:Gem::Dependency
|
63
95
|
name: rspec
|
64
96
|
requirement: !ruby/object:Gem::Requirement
|
@@ -118,17 +150,20 @@ files:
|
|
118
150
|
- .document
|
119
151
|
- .rspec
|
120
152
|
- .rvmrc
|
153
|
+
- .travis.yml
|
121
154
|
- Gemfile
|
122
155
|
- LICENSE.txt
|
123
156
|
- README.rdoc
|
124
157
|
- Rakefile
|
125
158
|
- VERSION
|
159
|
+
- ero_getter.gemspec
|
126
160
|
- lib/ero_getter.rb
|
127
161
|
- lib/ero_getter/downloader.rb
|
128
162
|
- lib/ero_getter/downloader/base.rb
|
129
163
|
- spec/downloader/base_spec.rb
|
130
164
|
- spec/downloader_spec.rb
|
131
165
|
- spec/ero_getter_spec.rb
|
166
|
+
- spec/samples/sample.html
|
132
167
|
- spec/spec_helper.rb
|
133
168
|
homepage: http://github.com/masarakki/ero_getter
|
134
169
|
licenses:
|
@@ -145,7 +180,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
145
180
|
version: '0'
|
146
181
|
segments:
|
147
182
|
- 0
|
148
|
-
hash:
|
183
|
+
hash: -3104763422106775519
|
149
184
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
150
185
|
none: false
|
151
186
|
requirements:
|