my-lib 0.0.1.4 → 0.0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/lib/lib/job.rb +58 -53
- data/lib/lib/machine.rb +135 -0
- data/lib/my-lib.rb +1 -1
- data/my-lib.gemspec +2 -3
- metadata +23 -24
- data/lib/lib/anisoku.rb +0 -290
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.1.
|
1
|
+
0.0.1.5
|
data/lib/lib/job.rb
CHANGED
@@ -29,6 +29,10 @@ class MyJobAnisoku
|
|
29
29
|
@a[:url] = URI.parse @a[:url] unless @a[:url].class == URI::HTTP
|
30
30
|
@agent = Mechanize.new
|
31
31
|
@a[:status] ||= :new
|
32
|
+
@a[:recent] ||= 7
|
33
|
+
@a[:limit] ||= 3
|
34
|
+
# make md5 with magicword '_gGddgPfeaf_gzyr'
|
35
|
+
@FC2magick = @a[:fc2magick] ||='_gGddgPfeaf_gzyr' #updated FC2 2011.7
|
32
36
|
raise "job have no machine error" unless @a[:machine]
|
33
37
|
end
|
34
38
|
|
@@ -36,18 +40,18 @@ class MyJobAnisoku
|
|
36
40
|
def tokkakari
|
37
41
|
print "Tokkakari".yellow
|
38
42
|
@agent.get @a[:url]
|
39
|
-
links_kousins = @agent.page.links_with(:text => /#{"更新状況"
|
43
|
+
links_kousins = @agent.page.links_with(:text => /#{"更新状況"}/)
|
40
44
|
targs = []
|
41
45
|
links_kousins.each do |link|
|
42
46
|
targs << link.uri
|
43
47
|
end
|
44
48
|
targs.each_with_index do |link,i|
|
45
|
-
break if i
|
49
|
+
break if i >= @a[:recent]
|
46
50
|
job = MyJobAnisoku.new(
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
+
@a.merge({
|
52
|
+
:url => link,
|
53
|
+
:status => :second
|
54
|
+
}) )
|
51
55
|
@a[:machine].retry job
|
52
56
|
end
|
53
57
|
|
@@ -67,10 +71,10 @@ class MyJobAnisoku
|
|
67
71
|
# make job for each links_kobetu
|
68
72
|
links_kobetu.each do |link|
|
69
73
|
job = MyJobAnisoku.new(
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
)
|
74
|
+
@a.merge({
|
75
|
+
:url => link,
|
76
|
+
:status => :kobetu
|
77
|
+
} ))
|
74
78
|
@a[:machine].retry job
|
75
79
|
end
|
76
80
|
end
|
@@ -79,32 +83,33 @@ class MyJobAnisoku
|
|
79
83
|
def kobetu
|
80
84
|
print "Kobetu".yellow
|
81
85
|
@agent.get @a[:url]
|
82
|
-
|
83
|
-
limit = 5
|
84
|
-
urls = []
|
86
|
+
title = @agent.page.title.gsub(' ★ You Tube アニ速 ★','')
|
85
87
|
# acume url
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
88
|
+
htmlA = @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/div[@class='kijisub']"
|
89
|
+
require 'pp'
|
90
|
+
targsHTMLs = htmlA.inner_html.toutf8.split(/ランキング/)[0].split(/\n第/).reverse!
|
91
|
+
#http://posterous.com/getfile/files.posterous.com/temp-2011-08-21/eolunzlwwwFopCnhszaBwJlFEJEnHcloqkoyaFuhdezmdgipcyyiyzdpqcpG/cro08nyoutube.doc
|
92
|
+
require 'digest'
|
93
|
+
targsHTMLs.each_with_index do |html,i|
|
94
|
+
break if i >= @a[:limit]
|
95
|
+
key = title + html.to_s
|
96
|
+
unless @a[:machine].episode_exists?( Digest::MD5.hexdigest(key) )
|
97
|
+
puts "NOW 2 PROCEED FETCH".green.bold + html[0..20].yellow.bold
|
98
|
+
indi = Nokogiri::HTML.fragment(html).css("a")
|
99
|
+
indi.each do |va|
|
100
|
+
if va[:href] =~ /(http:\/\/say-move\.org\/comeplay\.php.*)/
|
101
|
+
job = MyJobAnisoku.new(
|
102
|
+
@a.merge({
|
103
|
+
:url => $1,
|
104
|
+
:title => title + '第' + html.split('<').first.gsub(' ','').gsub(' ',''),
|
105
|
+
:status => :third}))
|
106
|
+
@a[:machine].retry job
|
107
|
+
end
|
108
|
+
end
|
109
|
+
else
|
110
|
+
puts "CANCELL FETCH".cyan.bold + html[0..20].yellow.bold
|
111
|
+
end
|
112
|
+
key = nil
|
108
113
|
end
|
109
114
|
end
|
110
115
|
|
@@ -125,12 +130,12 @@ class MyJobAnisoku
|
|
125
130
|
fc2 = set[0].value.split('&')[1].split('=')[1]
|
126
131
|
unless fc2.nil?
|
127
132
|
job = MyJobAnisoku.new(
|
133
|
+
@a.merge({
|
128
134
|
:url => sm[:url],
|
129
135
|
:fc2 => fc2,
|
130
136
|
:title => sm[:title],
|
131
|
-
:status => :fc2
|
132
|
-
|
133
|
-
)
|
137
|
+
:status => :fc2
|
138
|
+
}))
|
134
139
|
@a[:machine].retry job
|
135
140
|
return
|
136
141
|
else
|
@@ -138,28 +143,26 @@ class MyJobAnisoku
|
|
138
143
|
end
|
139
144
|
|
140
145
|
job = MyJobAnisoku.new(
|
146
|
+
@a.merge({
|
141
147
|
:url => sm[:videourl],
|
142
148
|
:title => sm[:title],
|
143
|
-
:status => :video
|
144
|
-
|
145
|
-
)
|
149
|
+
:status => :video
|
150
|
+
}))
|
146
151
|
@a[:machine].retry job
|
147
152
|
end
|
148
153
|
|
149
154
|
def fc2
|
150
155
|
print "fc2".yellow
|
151
156
|
require 'digest'
|
152
|
-
|
153
|
-
url = "http://video.fc2.com/ginfo.php?mimi=#{Digest::MD5.hexdigest(@a[:fc2] + '_gGddgPfeaf_gzyr')}&v=#{@a[:fc2]}&upid=#{@a[:fc2]}&otag=1"
|
157
|
+
url = "http://video.fc2.com/ginfo.php?mimi=#{Digest::MD5.hexdigest(@a[:fc2] + @FC2magick)}&v=#{@a[:fc2]}&upid=#{@a[:fc2]}&otag=1"
|
154
158
|
url = `curl -# -L -R "#{url}"`
|
155
159
|
url = url.split('&')[0].split('=')[1] + '?' + url.split('&')[1]
|
156
160
|
puts url.red.bold
|
157
161
|
job = MyJobAnisoku.new(
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
)
|
162
|
+
@a.merge({
|
163
|
+
:url => url,
|
164
|
+
:status => :video
|
165
|
+
}))
|
163
166
|
@a[:machine].retry job
|
164
167
|
end
|
165
168
|
|
@@ -169,7 +172,7 @@ class MyJobAnisoku
|
|
169
172
|
# save video directory is supplied by machine.
|
170
173
|
savedir = @a[:machine].savedir
|
171
174
|
Dir.chdir savedir
|
172
|
-
filename = "#{@a[:title]}.mp4"
|
175
|
+
filename = "#{@a[:title].gsub(' ','').gsub(' ','')}.mp4"
|
173
176
|
savepath = "#{savedir}/#{filename}"
|
174
177
|
# check fetch candidate had been already saved?
|
175
178
|
if File.exist?(savepath) && File.size(savepath) > 1024 * 1024 * 3
|
@@ -179,13 +182,15 @@ class MyJobAnisoku
|
|
179
182
|
puts "Fetching ".green.bold + savepath
|
180
183
|
MyLogger.ln "Fetch Attempt Start ".green.bold + savepath
|
181
184
|
end
|
182
|
-
|
185
|
+
|
183
186
|
# use curl command
|
184
187
|
# no need UA...
|
185
|
-
|
186
|
-
|
188
|
+
uri = "http://#{@a[:url].host}#{@a[:url].path}"
|
189
|
+
uri += "?#{@a[:url].query}" if @a[:url].query
|
190
|
+
command = "curl -# -L -R -o '#{filename}' '#{uri}' "
|
191
|
+
command += "&& growlnotify -t '#{filename}' -m '#{uri}' "
|
187
192
|
|
188
|
-
|
193
|
+
puts command
|
189
194
|
system command
|
190
195
|
end
|
191
196
|
|
data/lib/lib/machine.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# -*- coding:utf-8 -*-
|
1
2
|
|
2
3
|
#
|
3
4
|
# Module of Machine
|
@@ -156,3 +157,137 @@ class MyEventMachineDojin
|
|
156
157
|
|
157
158
|
end
|
158
159
|
|
160
|
+
#
|
161
|
+
# Class of Machine by EventMachine.
|
162
|
+
# this class controll jobs for Anisoku
|
163
|
+
# @example
|
164
|
+
# machine = MyMachineAnisoku.new("YourVideoSaveDir")
|
165
|
+
# machine.setup
|
166
|
+
# machine.go
|
167
|
+
#
|
168
|
+
class MyMachineAnisoku
|
169
|
+
include MyMachine
|
170
|
+
|
171
|
+
Version = "0.0.1"
|
172
|
+
|
173
|
+
# directory of save video files default "#{ENV['HOME']}/Desktop/video"
|
174
|
+
attr_accessor :savedir
|
175
|
+
|
176
|
+
# set video save dir
|
177
|
+
# @param [Hash] args
|
178
|
+
# @option args [String] :savedir save dir
|
179
|
+
# default "#{ENV['HOME']}/Desktop/video"
|
180
|
+
def initialize(args={ })
|
181
|
+
super()
|
182
|
+
args[:savedir] ||= "#{ENV['HOME']}/Desktop/video"
|
183
|
+
@savedir = args[:savedir]
|
184
|
+
begin
|
185
|
+
Dir::mkdir(@savedir, 0777)
|
186
|
+
rescue => ex
|
187
|
+
warn ex
|
188
|
+
end
|
189
|
+
require 'rubygems'
|
190
|
+
require 'eventmachine'
|
191
|
+
# @gaman controll eventmachine end
|
192
|
+
@gaman = 0
|
193
|
+
@checklist = {}
|
194
|
+
@filelist = {}
|
195
|
+
@dellist = []
|
196
|
+
require 'pp'
|
197
|
+
make_filelist
|
198
|
+
make_dellist
|
199
|
+
end
|
200
|
+
|
201
|
+
def make_filelist
|
202
|
+
Dir.new(@savedir).each do |entry|
|
203
|
+
if File.file?("#{@savedir}/#{entry}")
|
204
|
+
header = make_header(entry)
|
205
|
+
filesize = File.size("#{@savedir}/#{entry}")
|
206
|
+
@filelist[header] = [] unless chk_header(header)
|
207
|
+
@filelist[header] << {:size => filesize,:name => "#{@savedir}/#{entry}" }
|
208
|
+
end
|
209
|
+
end
|
210
|
+
pp @filelist
|
211
|
+
end
|
212
|
+
|
213
|
+
def chk_header(string)
|
214
|
+
@filelist[string]
|
215
|
+
end
|
216
|
+
|
217
|
+
def make_header(string)
|
218
|
+
header = string.scan(/^.*?\d{1,3}話/).first.gsub(' ','').gsub(' ','')
|
219
|
+
p header
|
220
|
+
return header
|
221
|
+
end
|
222
|
+
|
223
|
+
def make_dellist
|
224
|
+
@filelist.each do |k,v|
|
225
|
+
p k
|
226
|
+
max_size = v.map { |e| e[:size] }.max
|
227
|
+
p max_size
|
228
|
+
v.each do |vv|
|
229
|
+
if vv[:size] < max_size || vv[:size] < 1024 * 1024 * 2
|
230
|
+
@dellist << vv[:name]
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
234
|
+
pp @dellist
|
235
|
+
end
|
236
|
+
|
237
|
+
def del_small_files
|
238
|
+
@dellist.each do |e|
|
239
|
+
p e
|
240
|
+
File.delete("#{e}")
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def episode_exists?(key)
|
245
|
+
if @checklist[key].nil?
|
246
|
+
# puts "MACHINE NOE FIRST CHECK THIS EPISODE!!".red.bold
|
247
|
+
@checklist[key] = "checked"
|
248
|
+
return false
|
249
|
+
else
|
250
|
+
return true
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# machine go to run eventmachine
|
255
|
+
def go
|
256
|
+
EM.run do
|
257
|
+
EM.add_periodic_timer(0.00001) do
|
258
|
+
# print "loop".green
|
259
|
+
if should_stop_machine?
|
260
|
+
finalize_files
|
261
|
+
EM.stop
|
262
|
+
end
|
263
|
+
@queue.pop.run unless @queue.empty?
|
264
|
+
end
|
265
|
+
end
|
266
|
+
puts "End of fetch".green.bold
|
267
|
+
end
|
268
|
+
|
269
|
+
|
270
|
+
# delete tiny fail files
|
271
|
+
def finalize_files
|
272
|
+
del_small_files
|
273
|
+
command = "find #{@savedir} -size -1000k -type f -print0| xargs -0 rm -v "
|
274
|
+
exec(command)
|
275
|
+
end
|
276
|
+
|
277
|
+
private
|
278
|
+
|
279
|
+
# setup jobs
|
280
|
+
def setupjobs
|
281
|
+
ajob = MyJobAnisoku.new(
|
282
|
+
:machine => self
|
283
|
+
)
|
284
|
+
@queue.push ajob
|
285
|
+
end
|
286
|
+
|
287
|
+
# should stop machine or not
|
288
|
+
def should_stop_machine?
|
289
|
+
@gaman += 1 if @queue.empty?
|
290
|
+
print @gaman
|
291
|
+
return @queue.empty? && @gaman > 1500
|
292
|
+
end
|
293
|
+
end
|
data/lib/my-lib.rb
CHANGED
data/my-lib.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{my-lib}
|
8
|
-
s.version = "0.0.1.
|
8
|
+
s.version = "0.0.1.5"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = [%q{modeverv}]
|
12
|
-
s.date = %q{2011-09-
|
12
|
+
s.date = %q{2011-09-05}
|
13
13
|
s.description = %q{longer description of your gem}
|
14
14
|
s.email = %q{modeverv@gmail.com}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -27,7 +27,6 @@ Gem::Specification.new do |s|
|
|
27
27
|
"TAGS",
|
28
28
|
"VERSION",
|
29
29
|
"lib/1.rb",
|
30
|
-
"lib/lib/anisoku.rb",
|
31
30
|
"lib/lib/job.rb",
|
32
31
|
"lib/lib/machine.rb",
|
33
32
|
"lib/lib/myat.rb",
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: my-lib
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.1.
|
4
|
+
version: 0.0.1.5
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-09-
|
12
|
+
date: 2011-09-05 00:00:00.000000000Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: term-ansicolor
|
16
|
-
requirement: &
|
16
|
+
requirement: &2161208760 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *2161208760
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: eventmachine
|
27
|
-
requirement: &
|
27
|
+
requirement: &2161207180 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *2161207180
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: mechanize
|
38
|
-
requirement: &
|
38
|
+
requirement: &2161205420 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *2161205420
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: rspec
|
49
|
-
requirement: &
|
49
|
+
requirement: &2161202020 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>'
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 2.3.0
|
55
55
|
type: :development
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *2161202020
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: yard
|
60
|
-
requirement: &
|
60
|
+
requirement: &2161199560 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>'
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 0.6.0
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *2161199560
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: bundler
|
71
|
-
requirement: &
|
71
|
+
requirement: &2161197180 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ! '>'
|
@@ -76,10 +76,10 @@ dependencies:
|
|
76
76
|
version: 1.0.0
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *2161197180
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
81
|
name: jeweler
|
82
|
-
requirement: &
|
82
|
+
requirement: &2161196360 !ruby/object:Gem::Requirement
|
83
83
|
none: false
|
84
84
|
requirements:
|
85
85
|
- - ! '>='
|
@@ -87,10 +87,10 @@ dependencies:
|
|
87
87
|
version: '0'
|
88
88
|
type: :development
|
89
89
|
prerelease: false
|
90
|
-
version_requirements: *
|
90
|
+
version_requirements: *2161196360
|
91
91
|
- !ruby/object:Gem::Dependency
|
92
92
|
name: rcov
|
93
|
-
requirement: &
|
93
|
+
requirement: &2161195600 !ruby/object:Gem::Requirement
|
94
94
|
none: false
|
95
95
|
requirements:
|
96
96
|
- - ! '>='
|
@@ -98,10 +98,10 @@ dependencies:
|
|
98
98
|
version: '0'
|
99
99
|
type: :development
|
100
100
|
prerelease: false
|
101
|
-
version_requirements: *
|
101
|
+
version_requirements: *2161195600
|
102
102
|
- !ruby/object:Gem::Dependency
|
103
103
|
name: guard
|
104
|
-
requirement: &
|
104
|
+
requirement: &2161194860 !ruby/object:Gem::Requirement
|
105
105
|
none: false
|
106
106
|
requirements:
|
107
107
|
- - ! '>='
|
@@ -109,10 +109,10 @@ dependencies:
|
|
109
109
|
version: '0'
|
110
110
|
type: :development
|
111
111
|
prerelease: false
|
112
|
-
version_requirements: *
|
112
|
+
version_requirements: *2161194860
|
113
113
|
- !ruby/object:Gem::Dependency
|
114
114
|
name: guard-spork
|
115
|
-
requirement: &
|
115
|
+
requirement: &2161194120 !ruby/object:Gem::Requirement
|
116
116
|
none: false
|
117
117
|
requirements:
|
118
118
|
- - ! '>='
|
@@ -120,7 +120,7 @@ dependencies:
|
|
120
120
|
version: '0'
|
121
121
|
type: :development
|
122
122
|
prerelease: false
|
123
|
-
version_requirements: *
|
123
|
+
version_requirements: *2161194120
|
124
124
|
description: longer description of your gem
|
125
125
|
email: modeverv@gmail.com
|
126
126
|
executables: []
|
@@ -139,7 +139,6 @@ files:
|
|
139
139
|
- TAGS
|
140
140
|
- VERSION
|
141
141
|
- lib/1.rb
|
142
|
-
- lib/lib/anisoku.rb
|
143
142
|
- lib/lib/job.rb
|
144
143
|
- lib/lib/machine.rb
|
145
144
|
- lib/lib/myat.rb
|
@@ -169,7 +168,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
169
168
|
version: '0'
|
170
169
|
segments:
|
171
170
|
- 0
|
172
|
-
hash:
|
171
|
+
hash: 1152943948087336518
|
173
172
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
174
173
|
none: false
|
175
174
|
requirements:
|
data/lib/lib/anisoku.rb
DELETED
@@ -1,290 +0,0 @@
|
|
1
|
-
#-*- coding:utf-8 -*-
|
2
|
-
|
3
|
-
#
|
4
|
-
# Job Class for Fetch Anisoku
|
5
|
-
# Function:
|
6
|
-
# access "youtubeanisoku1.blog106.fc2.com" and crawl web site.
|
7
|
-
# find a link to "say-move.org" and access "say-move.org".
|
8
|
-
# finary find video link of Animetion,then fetch video file to your local.
|
9
|
-
# save video directory is supplied by machine.
|
10
|
-
# Notice:
|
11
|
-
# job is automatically generated on after another.
|
12
|
-
# This Class Needs to be handle by Machine Class
|
13
|
-
# @example
|
14
|
-
# # inside machine class
|
15
|
-
# job = MyJobAnisoku.new(
|
16
|
-
# :machine => self
|
17
|
-
# )
|
18
|
-
# job.run
|
19
|
-
#
|
20
|
-
|
21
|
-
class MyJobAnisoku
|
22
|
-
|
23
|
-
def initialize(args = { })
|
24
|
-
require 'rubygems'
|
25
|
-
require 'kconv'
|
26
|
-
require 'mechanize'
|
27
|
-
require 'net/http'
|
28
|
-
@a = args
|
29
|
-
@a[:url] ||= 'http://youtubeanisoku1.blog106.fc2.com/'
|
30
|
-
@a[:url] = URI.parse @a[:url] unless @a[:url].class == URI::HTTP
|
31
|
-
@agent = Mechanize.new
|
32
|
-
@a[:status] ||= :new
|
33
|
-
raise "job have no machine error" unless @a[:machine]
|
34
|
-
end
|
35
|
-
|
36
|
-
# check kousin page
|
37
|
-
def tokkakari
|
38
|
-
print "Tokkakari".yellow
|
39
|
-
@agent.get @a[:url]
|
40
|
-
links_kousins = @agent.page.links_with(:text => /#{"更新状況".toutf8}/)
|
41
|
-
targs = []
|
42
|
-
links_kousins.each do |link|
|
43
|
-
targs << link.uri
|
44
|
-
end
|
45
|
-
targs.each_with_index do |link,i|
|
46
|
-
break if i > 6
|
47
|
-
job = MyJobAnisoku.new(
|
48
|
-
:url => link,
|
49
|
-
:status => :second,
|
50
|
-
:machine => @a[:machine]
|
51
|
-
)
|
52
|
-
@a[:machine].retry job
|
53
|
-
end
|
54
|
-
|
55
|
-
end
|
56
|
-
|
57
|
-
# check shoukai page
|
58
|
-
def second
|
59
|
-
print "Second".yellow
|
60
|
-
@agent.get @a[:url]
|
61
|
-
links_kousin = @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/ul/li/a/@href"
|
62
|
-
# links_kobetu
|
63
|
-
links_kobetu = []
|
64
|
-
links_kousin.each do |link|
|
65
|
-
links_kobetu << $1 if link.value =~ /(http:\/\/youtubeanisoku.*)/
|
66
|
-
end
|
67
|
-
|
68
|
-
# make job for each links_kobetu
|
69
|
-
links_kobetu.each do |link|
|
70
|
-
job = MyJobAnisoku.new(
|
71
|
-
:url => link,
|
72
|
-
:status => :kobetu,
|
73
|
-
:machine => @a[:machine]
|
74
|
-
)
|
75
|
-
@a[:machine].retry job
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
#access say-move and make video job
|
80
|
-
def kobetu
|
81
|
-
print "Kobetu".yellow
|
82
|
-
@agent.get @a[:url]
|
83
|
-
_tt = @agent.page.title.gsub(' ★ You Tube アニ速 ★','')
|
84
|
-
limit = 5
|
85
|
-
urls = []
|
86
|
-
# acume url
|
87
|
-
nodeset_vs = @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/div[2]/a/@href"
|
88
|
-
_dd = []
|
89
|
-
|
90
|
-
nodeset_vs.each do |va|
|
91
|
-
_dd << $1 if va.value =~ /(http:\/\/say-move\.org\/comeplay\.php.*)/
|
92
|
-
end
|
93
|
-
_dd.reverse!
|
94
|
-
|
95
|
-
#hard coding for adjust fetch limit
|
96
|
-
_dd.each_with_index do |url,i|
|
97
|
-
break if i > limit
|
98
|
-
urls << url
|
99
|
-
end
|
100
|
-
|
101
|
-
urls.each_with_index do |url,i|
|
102
|
-
job = MyJobAnisoku.new(
|
103
|
-
:url => url,
|
104
|
-
:title => _tt,
|
105
|
-
:status => :third,
|
106
|
-
:machine => @a[:machine]
|
107
|
-
)
|
108
|
-
@a[:machine].retry job
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
#access say-move and make video job
|
113
|
-
def third
|
114
|
-
print "Third".yellow
|
115
|
-
#sm has title and url
|
116
|
-
|
117
|
-
sm = { :title => @a[:title],:url => @a[:url]}
|
118
|
-
# debug fc2 video sm[:url] = "http://say-move.org/comeplay.php?comeid=217953"
|
119
|
-
@agent.get(sm[:url])
|
120
|
-
sm[:title] += @agent.page.title.gsub!('FC2 SayMove!','')
|
121
|
-
set = @agent.page/"/html/body/div/div[2]/div[7]/div[2]/input/@value"
|
122
|
-
if !set.empty?
|
123
|
-
sm[:videourl] = set[0].value
|
124
|
-
else
|
125
|
-
set = @agent.page/"/html/body/div/div[2]/div[3]/object/param[5]/@value"
|
126
|
-
fc2 = set[0].value.split('&')[1].split('=')[1]
|
127
|
-
unless fc2.nil?
|
128
|
-
job = MyJobAnisoku.new(
|
129
|
-
:url => sm[:url],
|
130
|
-
:fc2 => fc2,
|
131
|
-
:title => sm[:title],
|
132
|
-
:status => :fc2,
|
133
|
-
:machine => @a[:machine]
|
134
|
-
)
|
135
|
-
@a[:machine].retry job
|
136
|
-
return
|
137
|
-
else
|
138
|
-
end
|
139
|
-
end
|
140
|
-
|
141
|
-
job = MyJobAnisoku.new(
|
142
|
-
:url => sm[:videourl],
|
143
|
-
:title => sm[:title],
|
144
|
-
:status => :video,
|
145
|
-
:machine => @a[:machine]
|
146
|
-
)
|
147
|
-
@a[:machine].retry job
|
148
|
-
end
|
149
|
-
|
150
|
-
def fc2
|
151
|
-
print "fc2".yellow
|
152
|
-
require 'digest'
|
153
|
-
# make md5 with magicword '_gGddgPfeaf_gzyr'
|
154
|
-
url = "http://video.fc2.com/ginfo.php?mimi=#{Digest::MD5.hexdigest(@a[:fc2] + '_gGddgPfeaf_gzyr')}&v=#{@a[:fc2]}&upid=#{@a[:fc2]}&otag=1"
|
155
|
-
url = `curl -# -L -R "#{url}"`
|
156
|
-
url = url.split('&')[0].split('=')[1] + '?' + url.split('&')[1]
|
157
|
-
puts url.red.bold
|
158
|
-
job = MyJobAnisoku.new(
|
159
|
-
:url => url,
|
160
|
-
:title => @a[:title],
|
161
|
-
:status => :video,
|
162
|
-
:machine => @a[:machine]
|
163
|
-
)
|
164
|
-
@a[:machine].retry job
|
165
|
-
end
|
166
|
-
|
167
|
-
#fetch video
|
168
|
-
def video
|
169
|
-
print "video".yellow
|
170
|
-
# save video directory is supplied by machine.
|
171
|
-
savedir = @a[:machine].savedir
|
172
|
-
Dir.chdir savedir
|
173
|
-
filename = "#{@a[:title]}.mp4"
|
174
|
-
savepath = "#{savedir}/#{filename}"
|
175
|
-
# check fetch candidate had been already saved?
|
176
|
-
if File.exist?(savepath) && File.size(savepath) > 1024 * 1024 * 3
|
177
|
-
puts "File Already Saved ".yellow.bold + savepath
|
178
|
-
return
|
179
|
-
else
|
180
|
-
puts "Fetching ".green.bold + savepath
|
181
|
-
MyLogger.ln "Fetch Attempt Start ".green.bold + savepath
|
182
|
-
end
|
183
|
-
|
184
|
-
# use curl command
|
185
|
-
# no need UA...
|
186
|
-
# command = "curl -# -L -R -o '#{filename}' 'http://#{@a[:url].host}#{@a[:url].path}?#{@a[:url].query}' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0a1) Gecko/20110901 Firefox/9.0a1'"
|
187
|
-
command = "curl -# -L -R -o '#{filename}' 'http://#{@a[:url].host}#{@a[:url].path}?#{@a[:url].query}' "
|
188
|
-
|
189
|
-
p command
|
190
|
-
system command
|
191
|
-
end
|
192
|
-
|
193
|
-
# run in thread
|
194
|
-
def run
|
195
|
-
t = Thread.new do
|
196
|
-
case @a[:status]
|
197
|
-
when :new then
|
198
|
-
tokkakari
|
199
|
-
when :second then
|
200
|
-
second
|
201
|
-
when :kobetu then
|
202
|
-
kobetu
|
203
|
-
when :third then
|
204
|
-
third
|
205
|
-
when :fc2 then
|
206
|
-
fc2
|
207
|
-
when :video then
|
208
|
-
video
|
209
|
-
end
|
210
|
-
end
|
211
|
-
end
|
212
|
-
|
213
|
-
end
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
#
|
218
|
-
# Class of Machine by EventMachine.
|
219
|
-
# this class controll jobs for Anisoku
|
220
|
-
# @example
|
221
|
-
# machine = MyMachineAnisoku.new("YourVideoSaveDir")
|
222
|
-
# machine.setup
|
223
|
-
# machine.go
|
224
|
-
#
|
225
|
-
class MyMachineAnisoku
|
226
|
-
include MyMachine
|
227
|
-
|
228
|
-
Version = "0.0.1"
|
229
|
-
|
230
|
-
# directory of save video files default "#{ENV['HOME']}/Desktop/video"
|
231
|
-
attr_accessor :savedir
|
232
|
-
|
233
|
-
# set video save dir
|
234
|
-
# @param [Hash] args
|
235
|
-
# @option args [String] :savedir save dir
|
236
|
-
# default "#{ENV['HOME']}/Desktop/video"
|
237
|
-
def initialize(args={ })
|
238
|
-
super()
|
239
|
-
args[:savedir] ||= "#{ENV['HOME']}/Desktop/video2"
|
240
|
-
@savedir = args[:savedir]
|
241
|
-
begin
|
242
|
-
Dir::mkdir(@savedir, 0777)
|
243
|
-
rescue => ex
|
244
|
-
warn ex
|
245
|
-
end
|
246
|
-
require 'rubygems'
|
247
|
-
require 'eventmachine'
|
248
|
-
# @gaman controll eventmachine end
|
249
|
-
@gaman = 0
|
250
|
-
end
|
251
|
-
|
252
|
-
# machine go to run eventmachine
|
253
|
-
def go
|
254
|
-
EM.run do
|
255
|
-
EM.add_periodic_timer(0.00001) do
|
256
|
-
# print "loop".green
|
257
|
-
if should_stop_machine?
|
258
|
-
finalize_files
|
259
|
-
EM.stop
|
260
|
-
end
|
261
|
-
@queue.pop.run unless @queue.empty?
|
262
|
-
end
|
263
|
-
end
|
264
|
-
puts "End of fetch".green.bold
|
265
|
-
end
|
266
|
-
|
267
|
-
|
268
|
-
# delete tiny fail files
|
269
|
-
def finalize_files
|
270
|
-
command = "find #{@savedir} -size -1000k -type f -print0| xargs -0 rm -v "
|
271
|
-
exec(command)
|
272
|
-
end
|
273
|
-
|
274
|
-
private
|
275
|
-
|
276
|
-
# setup jobs
|
277
|
-
def setupjobs
|
278
|
-
ajob = MyJobAnisoku.new(
|
279
|
-
:machine => self
|
280
|
-
)
|
281
|
-
@queue.push ajob
|
282
|
-
end
|
283
|
-
|
284
|
-
# should stop machine or not
|
285
|
-
def should_stop_machine?
|
286
|
-
@gaman += 1 if @queue.empty?
|
287
|
-
print @gaman
|
288
|
-
return @queue.empty? && @gaman > 1500
|
289
|
-
end
|
290
|
-
end
|