nicoscraper 0.2.15 → 0.2.15.2
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +1 -1
- data/Gemfile.lock +2 -2
- data/README.md +3 -3
- data/VERSION +1 -1
- data/lib/classes/parser.rb +1 -1
- data/lib/classes/searcher.rb +2 -2
- data/lib/config/header.rb +8 -0
- data/lib/nicoscraper.rb +2 -1
- data/nicoscraper.gemspec +6 -6
- metadata +5 -5
- data/lib/classes/header.rb +0 -9
data/Gemfile
CHANGED
@@ -5,7 +5,7 @@ gem "libxml-ruby", ">= 2.2.2"
|
|
5
5
|
# Add dependencies to develop your gem here.
|
6
6
|
# Include everything needed to run rake, tests, features, etc.
|
7
7
|
group :development do
|
8
|
-
gem "rake", "0.
|
8
|
+
gem "rake", "0.9.2"
|
9
9
|
gem "shoulda", ">= 0"
|
10
10
|
gem "bundler", "~> 1.0.0"
|
11
11
|
gem "jeweler", "~> 1.6.4"
|
data/Gemfile.lock
CHANGED
@@ -7,7 +7,7 @@ GEM
|
|
7
7
|
git (>= 1.2.5)
|
8
8
|
rake
|
9
9
|
libxml-ruby (2.2.2)
|
10
|
-
rake (0.
|
10
|
+
rake (0.9.2)
|
11
11
|
rcov (0.9.10)
|
12
12
|
shoulda (2.11.3)
|
13
13
|
|
@@ -18,6 +18,6 @@ DEPENDENCIES
|
|
18
18
|
bundler (~> 1.0.0)
|
19
19
|
jeweler (~> 1.6.4)
|
20
20
|
libxml-ruby (>= 2.2.2)
|
21
|
-
rake (= 0.
|
21
|
+
rake (= 0.9.2)
|
22
22
|
rcov
|
23
23
|
shoulda
|
data/README.md
CHANGED
@@ -9,8 +9,8 @@ NicoScraper
|
|
9
9
|
**Author:** Masami Yonehara
|
10
10
|
**Copyright:** 2011
|
11
11
|
**License:** MIT License
|
12
|
-
**Latest Version:** 0.2.15
|
13
|
-
**Release Date:**
|
12
|
+
**Latest Version:** 0.2.15.2
|
13
|
+
**Release Date:** Nov 12th 2011
|
14
14
|
|
15
15
|
|
16
16
|
何をするライブラリなの?
|
@@ -169,7 +169,7 @@ NicoScraper
|
|
169
169
|
|
170
170
|
ブロック内の第1引数には取得結果に基づく動画インスタンスが与えられるのですが、これは32個分の配列です。なぜ32個のセットなのかと言うと、ご存知のようにニコニコ動画の検索画面はページで区切られており、Searcherモジュールの各メソッドはページ毎に情報を取得し、ページ単位でブロックをコールするからです。HTMLから取得するにしろAtomフィードから取得するにしろ、1ページに32個の動画情報が含まれています。第2引数にはそれまでのアクセス成否情報等が渡されます。
|
171
171
|
|
172
|
-
そして、**ブロック内でシンボル`:continue
|
172
|
+
そして、**ブロック内でシンボル`:continue`(もしくは"continue"文字列)を返すことによりスクレイプが継続します。**つまり、`:continue`を返し続けるロジックを組み込まないと、1ページ目を読んだ時点で処理が終了します。これは意図せざる過剰アクセスを防ぐための措置です。
|
173
173
|
|
174
174
|
上の例では、取得した動画の日付を調べ、3日前の0時0分より前の動画に到達すればそこでループを終える設計です。ループを継続するために取得情報を使うかどうかは任意なので、例えば10分間の制限で取得出来るだけ取得するということも可能でしょう。
|
175
175
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.15
|
1
|
+
0.2.15.2
|
data/lib/classes/parser.rb
CHANGED
data/lib/classes/searcher.rb
CHANGED
@@ -56,7 +56,7 @@ module Nicos
|
|
56
56
|
|
57
57
|
def loop(tag, sort, method, &block)
|
58
58
|
@page = 1
|
59
|
-
order
|
59
|
+
order = nil
|
60
60
|
|
61
61
|
begin
|
62
62
|
movieObjAry = []
|
@@ -82,7 +82,7 @@ module Nicos
|
|
82
82
|
status = { :page => @page, :results => @connector.result}
|
83
83
|
order = block.call(movieObjAry, status)
|
84
84
|
@page += 1
|
85
|
-
end until order != "continue"
|
85
|
+
end until order != "continue" && order != :continue
|
86
86
|
end
|
87
87
|
|
88
88
|
public
|
data/lib/nicoscraper.rb
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
module Nicos
|
3
|
-
VERSION = "0.2.
|
3
|
+
VERSION = "0.2.15.2"
|
4
4
|
REPOSITORY =
|
5
5
|
"http://github.com/hdemon/nicoscraper/"
|
6
6
|
AUTHOR = "Masami Yonehara"
|
7
|
+
MAIL = "hdemon7@gmail.com"
|
7
8
|
|
8
9
|
# nicos.rbが存在する絶対パスを取得
|
9
10
|
ROOT = File.expand_path(File.dirname(__FILE__))
|
data/nicoscraper.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{nicoscraper}
|
8
|
-
s.version = "0.2.15"
|
8
|
+
s.version = "0.2.15.2"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = [%q{Masami Yonehara}]
|
12
|
-
s.date = %q{2011-11-
|
12
|
+
s.date = %q{2011-11-12}
|
13
13
|
s.description = %q{It scrape movies and mylists of Niconico douga.
|
14
14
|
}
|
15
15
|
s.email = %q{zeitdiebe@gmail.com}
|
@@ -28,12 +28,12 @@ Gem::Specification.new do |s|
|
|
28
28
|
"index.html",
|
29
29
|
"lib/classes/connector.rb",
|
30
30
|
"lib/classes/converter.rb",
|
31
|
-
"lib/classes/header.rb",
|
32
31
|
"lib/classes/movie.rb",
|
33
32
|
"lib/classes/mylist.rb",
|
34
33
|
"lib/classes/parser.rb",
|
35
34
|
"lib/classes/searcher.rb",
|
36
35
|
"lib/classes/tools.rb",
|
36
|
+
"lib/config/header.rb",
|
37
37
|
"lib/config/mylist.rb",
|
38
38
|
"lib/config/wait.rb",
|
39
39
|
"lib/nicoscraper.rb",
|
@@ -54,14 +54,14 @@ Gem::Specification.new do |s|
|
|
54
54
|
|
55
55
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
56
56
|
s.add_runtime_dependency(%q<libxml-ruby>, [">= 2.2.2"])
|
57
|
-
s.add_development_dependency(%q<rake>, ["= 0.
|
57
|
+
s.add_development_dependency(%q<rake>, ["= 0.9.2"])
|
58
58
|
s.add_development_dependency(%q<shoulda>, [">= 0"])
|
59
59
|
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
60
60
|
s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
|
61
61
|
s.add_development_dependency(%q<rcov>, [">= 0"])
|
62
62
|
else
|
63
63
|
s.add_dependency(%q<libxml-ruby>, [">= 2.2.2"])
|
64
|
-
s.add_dependency(%q<rake>, ["= 0.
|
64
|
+
s.add_dependency(%q<rake>, ["= 0.9.2"])
|
65
65
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
66
66
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
67
67
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
@@ -69,7 +69,7 @@ Gem::Specification.new do |s|
|
|
69
69
|
end
|
70
70
|
else
|
71
71
|
s.add_dependency(%q<libxml-ruby>, [">= 2.2.2"])
|
72
|
-
s.add_dependency(%q<rake>, ["= 0.
|
72
|
+
s.add_dependency(%q<rake>, ["= 0.9.2"])
|
73
73
|
s.add_dependency(%q<shoulda>, [">= 0"])
|
74
74
|
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
75
75
|
s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: nicoscraper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.2.15
|
5
|
+
version: 0.2.15.2
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Masami Yonehara
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2011-11-
|
13
|
+
date: 2011-11-12 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: libxml-ruby
|
@@ -30,7 +30,7 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "="
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.
|
33
|
+
version: 0.9.2
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: *id002
|
@@ -98,12 +98,12 @@ files:
|
|
98
98
|
- index.html
|
99
99
|
- lib/classes/connector.rb
|
100
100
|
- lib/classes/converter.rb
|
101
|
-
- lib/classes/header.rb
|
102
101
|
- lib/classes/movie.rb
|
103
102
|
- lib/classes/mylist.rb
|
104
103
|
- lib/classes/parser.rb
|
105
104
|
- lib/classes/searcher.rb
|
106
105
|
- lib/classes/tools.rb
|
106
|
+
- lib/config/header.rb
|
107
107
|
- lib/config/mylist.rb
|
108
108
|
- lib/config/wait.rb
|
109
109
|
- lib/nicoscraper.rb
|
@@ -125,7 +125,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
125
125
|
requirements:
|
126
126
|
- - ">="
|
127
127
|
- !ruby/object:Gem::Version
|
128
|
-
hash:
|
128
|
+
hash: -2813718495648355555
|
129
129
|
segments:
|
130
130
|
- 0
|
131
131
|
version: "0"
|