bookscan 0.5.11 → 0.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +1 -0
- data/Rakefile +2 -1
- data/VERSION +1 -1
- data/bookscan.gemspec +6 -3
- data/lib/bookscan/agent.rb +14 -8
- data/lib/bookscan/book.rb +1 -1
- data/lib/bookscan/groups.rb +7 -4
- metadata +58 -44
data/Gemfile
CHANGED
data/Rakefile
CHANGED
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
0.5.
|
|
1
|
+
0.5.12
|
data/bookscan.gemspec
CHANGED
|
@@ -5,11 +5,11 @@
|
|
|
5
5
|
|
|
6
6
|
Gem::Specification.new do |s|
|
|
7
7
|
s.name = "bookscan"
|
|
8
|
-
s.version = "0.5.
|
|
8
|
+
s.version = "0.5.12"
|
|
9
9
|
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
11
11
|
s.authors = ["Yoshihiro TAKAHARA"]
|
|
12
|
-
s.date = "
|
|
12
|
+
s.date = "2012-02-11"
|
|
13
13
|
s.description = "This is a scraper of Bookscan (http://www.bookscan.co.jp) Service.This is *NOT* a official software of Bookscan."
|
|
14
14
|
s.email = "y.takahara@gmail.com"
|
|
15
15
|
s.executables = ["bookscan"]
|
|
@@ -39,7 +39,7 @@ Gem::Specification.new do |s|
|
|
|
39
39
|
s.homepage = "http://github.com/tumf/bookscan"
|
|
40
40
|
s.licenses = ["MIT"]
|
|
41
41
|
s.require_paths = ["lib"]
|
|
42
|
-
s.rubygems_version = "1.8.
|
|
42
|
+
s.rubygems_version = "1.8.12"
|
|
43
43
|
s.summary = "BookScan Scraper"
|
|
44
44
|
s.test_files = [
|
|
45
45
|
"test/helper.rb",
|
|
@@ -56,6 +56,7 @@ Gem::Specification.new do |s|
|
|
|
56
56
|
s.add_runtime_dependency(%q<highline>, ["> 1.6"])
|
|
57
57
|
s.add_runtime_dependency(%q<progressbar>, [">= 0.9.0"])
|
|
58
58
|
s.add_runtime_dependency(%q<httpclient>, [">= 2.1.6.1"])
|
|
59
|
+
s.add_runtime_dependency(%q<rdoc>, [">= 0"])
|
|
59
60
|
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
|
60
61
|
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
|
61
62
|
s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
|
|
@@ -67,6 +68,7 @@ Gem::Specification.new do |s|
|
|
|
67
68
|
s.add_dependency(%q<highline>, ["> 1.6"])
|
|
68
69
|
s.add_dependency(%q<progressbar>, [">= 0.9.0"])
|
|
69
70
|
s.add_dependency(%q<httpclient>, [">= 2.1.6.1"])
|
|
71
|
+
s.add_dependency(%q<rdoc>, [">= 0"])
|
|
70
72
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
|
71
73
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
|
72
74
|
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
|
@@ -79,6 +81,7 @@ Gem::Specification.new do |s|
|
|
|
79
81
|
s.add_dependency(%q<highline>, ["> 1.6"])
|
|
80
82
|
s.add_dependency(%q<progressbar>, [">= 0.9.0"])
|
|
81
83
|
s.add_dependency(%q<httpclient>, [">= 2.1.6.1"])
|
|
84
|
+
s.add_dependency(%q<rdoc>, [">= 0"])
|
|
82
85
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
|
83
86
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
|
84
87
|
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
data/lib/bookscan/agent.rb
CHANGED
|
@@ -136,7 +136,7 @@ module Bookscan
|
|
|
136
136
|
u.text == "変換"
|
|
137
137
|
book = Book.new
|
|
138
138
|
book.title = u.text.to_s
|
|
139
|
-
book.url = u.attributes["href"].value.to_s
|
|
139
|
+
book.url = "/" + u.attributes["href"].value.to_s
|
|
140
140
|
book.group_url = group.url
|
|
141
141
|
bs << book
|
|
142
142
|
end
|
|
@@ -152,7 +152,7 @@ module Bookscan
|
|
|
152
152
|
u.text == "変換"
|
|
153
153
|
book = Book.new
|
|
154
154
|
book.title = u.text.to_s
|
|
155
|
-
book.url = u.attributes["href"].value.to_s
|
|
155
|
+
book.url = "/" + u.attributes["href"].value.to_s
|
|
156
156
|
book.group_url = url
|
|
157
157
|
books[book.book_id] = book
|
|
158
158
|
end
|
|
@@ -162,19 +162,25 @@ module Bookscan
|
|
|
162
162
|
|
|
163
163
|
def download(url,path)
|
|
164
164
|
url = URI.parse(url)
|
|
165
|
+
url.scheme = "https"
|
|
166
|
+
url.host = "system.bookscan.co.jp"
|
|
167
|
+
|
|
165
168
|
cli = HTTPClient.new
|
|
166
|
-
|
|
167
|
-
@cookie_jar && @cookie_jar.cookies(url).each do |cookie|
|
|
169
|
+
cookie_jar && cookie_jar.cookies(url).each do |cookie|
|
|
168
170
|
cli.cookie_manager.parse(cookie.to_s,url)
|
|
169
171
|
end
|
|
170
172
|
|
|
171
173
|
length = 0;total = 0
|
|
172
|
-
res = cli.head(url)
|
|
173
|
-
|
|
174
|
+
res = cli.head(url.to_s)
|
|
175
|
+
|
|
176
|
+
if res.status == 302
|
|
177
|
+
url = URI.parse(res.header["Location"].to_s)
|
|
178
|
+
res = cli.head(url.to_s)
|
|
179
|
+
end
|
|
174
180
|
|
|
175
|
-
total =
|
|
181
|
+
total = res.header["Content-Length"].to_s.to_i
|
|
176
182
|
t = Thread.new {
|
|
177
|
-
conn = cli.get_async(url)
|
|
183
|
+
conn = cli.get_async(url.to_s)
|
|
178
184
|
io = conn.pop.content
|
|
179
185
|
::File::open(path, "wb") { |f|
|
|
180
186
|
while str = io.read(40)
|
data/lib/bookscan/book.rb
CHANGED
data/lib/bookscan/groups.rb
CHANGED
|
@@ -36,16 +36,19 @@ module Bookscan
|
|
|
36
36
|
column
|
|
37
37
|
end
|
|
38
38
|
# table << ["#","Date","num","price","status"]
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
|
|
40
|
+
sort{ |a,b| a.date <=> b.date}.each do |group|
|
|
41
|
+
table << group.to_a
|
|
41
42
|
end
|
|
42
43
|
table.to_s if length > 0
|
|
43
44
|
end
|
|
44
45
|
|
|
45
46
|
def book(book_id)
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
g = find do |g|
|
|
48
|
+
g.books.has?(book_id)
|
|
48
49
|
end
|
|
50
|
+
return nil unless g
|
|
51
|
+
g.books.by_id(book_id)
|
|
49
52
|
end
|
|
50
53
|
|
|
51
54
|
end
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: bookscan
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
hash:
|
|
4
|
+
hash: 19
|
|
5
5
|
prerelease:
|
|
6
6
|
segments:
|
|
7
7
|
- 0
|
|
8
8
|
- 5
|
|
9
|
-
-
|
|
10
|
-
version: 0.5.
|
|
9
|
+
- 12
|
|
10
|
+
version: 0.5.12
|
|
11
11
|
platform: ruby
|
|
12
12
|
authors:
|
|
13
13
|
- Yoshihiro TAKAHARA
|
|
@@ -15,10 +15,12 @@ autorequire:
|
|
|
15
15
|
bindir: bin
|
|
16
16
|
cert_chain: []
|
|
17
17
|
|
|
18
|
-
date:
|
|
18
|
+
date: 2012-02-11 00:00:00 Z
|
|
19
19
|
dependencies:
|
|
20
20
|
- !ruby/object:Gem::Dependency
|
|
21
|
-
|
|
21
|
+
prerelease: false
|
|
22
|
+
type: :runtime
|
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
|
22
24
|
none: false
|
|
23
25
|
requirements:
|
|
24
26
|
- - ">="
|
|
@@ -27,12 +29,12 @@ dependencies:
|
|
|
27
29
|
segments:
|
|
28
30
|
- 0
|
|
29
31
|
version: "0"
|
|
30
|
-
|
|
31
|
-
prerelease: false
|
|
32
|
+
version_requirements: *id001
|
|
32
33
|
name: mutter
|
|
33
|
-
type: :runtime
|
|
34
34
|
- !ruby/object:Gem::Dependency
|
|
35
|
-
|
|
35
|
+
prerelease: false
|
|
36
|
+
type: :runtime
|
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
|
36
38
|
none: false
|
|
37
39
|
requirements:
|
|
38
40
|
- - ">="
|
|
@@ -43,12 +45,12 @@ dependencies:
|
|
|
43
45
|
- 4
|
|
44
46
|
- 7
|
|
45
47
|
version: 0.4.7
|
|
46
|
-
|
|
47
|
-
prerelease: false
|
|
48
|
+
version_requirements: *id002
|
|
48
49
|
name: keystorage
|
|
49
|
-
type: :runtime
|
|
50
50
|
- !ruby/object:Gem::Dependency
|
|
51
|
-
|
|
51
|
+
prerelease: false
|
|
52
|
+
type: :runtime
|
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
|
52
54
|
none: false
|
|
53
55
|
requirements:
|
|
54
56
|
- - ">="
|
|
@@ -59,12 +61,12 @@ dependencies:
|
|
|
59
61
|
- 0
|
|
60
62
|
- 1
|
|
61
63
|
version: 2.0.1
|
|
62
|
-
|
|
63
|
-
prerelease: false
|
|
64
|
+
version_requirements: *id003
|
|
64
65
|
name: mechanize
|
|
65
|
-
type: :runtime
|
|
66
66
|
- !ruby/object:Gem::Dependency
|
|
67
|
-
|
|
67
|
+
prerelease: false
|
|
68
|
+
type: :runtime
|
|
69
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
|
68
70
|
none: false
|
|
69
71
|
requirements:
|
|
70
72
|
- - ">"
|
|
@@ -74,12 +76,12 @@ dependencies:
|
|
|
74
76
|
- 1
|
|
75
77
|
- 6
|
|
76
78
|
version: "1.6"
|
|
77
|
-
|
|
78
|
-
prerelease: false
|
|
79
|
+
version_requirements: *id004
|
|
79
80
|
name: highline
|
|
80
|
-
type: :runtime
|
|
81
81
|
- !ruby/object:Gem::Dependency
|
|
82
|
-
|
|
82
|
+
prerelease: false
|
|
83
|
+
type: :runtime
|
|
84
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
|
83
85
|
none: false
|
|
84
86
|
requirements:
|
|
85
87
|
- - ">="
|
|
@@ -90,12 +92,12 @@ dependencies:
|
|
|
90
92
|
- 9
|
|
91
93
|
- 0
|
|
92
94
|
version: 0.9.0
|
|
93
|
-
|
|
94
|
-
prerelease: false
|
|
95
|
+
version_requirements: *id005
|
|
95
96
|
name: progressbar
|
|
96
|
-
type: :runtime
|
|
97
97
|
- !ruby/object:Gem::Dependency
|
|
98
|
-
|
|
98
|
+
prerelease: false
|
|
99
|
+
type: :runtime
|
|
100
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
|
99
101
|
none: false
|
|
100
102
|
requirements:
|
|
101
103
|
- - ">="
|
|
@@ -107,12 +109,12 @@ dependencies:
|
|
|
107
109
|
- 6
|
|
108
110
|
- 1
|
|
109
111
|
version: 2.1.6.1
|
|
110
|
-
|
|
111
|
-
prerelease: false
|
|
112
|
+
version_requirements: *id006
|
|
112
113
|
name: httpclient
|
|
113
|
-
type: :runtime
|
|
114
114
|
- !ruby/object:Gem::Dependency
|
|
115
|
-
|
|
115
|
+
prerelease: false
|
|
116
|
+
type: :runtime
|
|
117
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
|
116
118
|
none: false
|
|
117
119
|
requirements:
|
|
118
120
|
- - ">="
|
|
@@ -121,12 +123,26 @@ dependencies:
|
|
|
121
123
|
segments:
|
|
122
124
|
- 0
|
|
123
125
|
version: "0"
|
|
124
|
-
|
|
126
|
+
version_requirements: *id007
|
|
127
|
+
name: rdoc
|
|
128
|
+
- !ruby/object:Gem::Dependency
|
|
125
129
|
prerelease: false
|
|
126
|
-
name: shoulda
|
|
127
130
|
type: :development
|
|
131
|
+
requirement: &id008 !ruby/object:Gem::Requirement
|
|
132
|
+
none: false
|
|
133
|
+
requirements:
|
|
134
|
+
- - ">="
|
|
135
|
+
- !ruby/object:Gem::Version
|
|
136
|
+
hash: 3
|
|
137
|
+
segments:
|
|
138
|
+
- 0
|
|
139
|
+
version: "0"
|
|
140
|
+
version_requirements: *id008
|
|
141
|
+
name: shoulda
|
|
128
142
|
- !ruby/object:Gem::Dependency
|
|
129
|
-
|
|
143
|
+
prerelease: false
|
|
144
|
+
type: :development
|
|
145
|
+
requirement: &id009 !ruby/object:Gem::Requirement
|
|
130
146
|
none: false
|
|
131
147
|
requirements:
|
|
132
148
|
- - ~>
|
|
@@ -137,12 +153,12 @@ dependencies:
|
|
|
137
153
|
- 0
|
|
138
154
|
- 0
|
|
139
155
|
version: 1.0.0
|
|
140
|
-
|
|
141
|
-
prerelease: false
|
|
156
|
+
version_requirements: *id009
|
|
142
157
|
name: bundler
|
|
143
|
-
type: :development
|
|
144
158
|
- !ruby/object:Gem::Dependency
|
|
145
|
-
|
|
159
|
+
prerelease: false
|
|
160
|
+
type: :development
|
|
161
|
+
requirement: &id010 !ruby/object:Gem::Requirement
|
|
146
162
|
none: false
|
|
147
163
|
requirements:
|
|
148
164
|
- - ~>
|
|
@@ -153,12 +169,12 @@ dependencies:
|
|
|
153
169
|
- 5
|
|
154
170
|
- 2
|
|
155
171
|
version: 1.5.2
|
|
156
|
-
|
|
157
|
-
prerelease: false
|
|
172
|
+
version_requirements: *id010
|
|
158
173
|
name: jeweler
|
|
159
|
-
type: :development
|
|
160
174
|
- !ruby/object:Gem::Dependency
|
|
161
|
-
|
|
175
|
+
prerelease: false
|
|
176
|
+
type: :development
|
|
177
|
+
requirement: &id011 !ruby/object:Gem::Requirement
|
|
162
178
|
none: false
|
|
163
179
|
requirements:
|
|
164
180
|
- - ">="
|
|
@@ -167,10 +183,8 @@ dependencies:
|
|
|
167
183
|
segments:
|
|
168
184
|
- 0
|
|
169
185
|
version: "0"
|
|
170
|
-
|
|
171
|
-
prerelease: false
|
|
186
|
+
version_requirements: *id011
|
|
172
187
|
name: rcov
|
|
173
|
-
type: :development
|
|
174
188
|
description: This is a scraper of Bookscan (http://www.bookscan.co.jp) Service.This is *NOT* a official software of Bookscan.
|
|
175
189
|
email: y.takahara@gmail.com
|
|
176
190
|
executables:
|
|
@@ -227,7 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
227
241
|
requirements: []
|
|
228
242
|
|
|
229
243
|
rubyforge_project:
|
|
230
|
-
rubygems_version: 1.8.
|
|
244
|
+
rubygems_version: 1.8.12
|
|
231
245
|
signing_key:
|
|
232
246
|
specification_version: 3
|
|
233
247
|
summary: BookScan Scraper
|