javlibrary 0.1.8 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/javlibrary.rb +8 -6
- data/lib/javlibrary/database.rb +0 -16
- data/lib/javlibrary/downloader.rb +0 -116
- data/lib/javlibrary/info_hash.rb +0 -29
- data/lib/javlibrary/init.rb +0 -41
- data/lib/javlibrary/name.rb +0 -52
- data/lib/javlibrary/version.rb +1 -1
- data/lib/javlibrary/video.rb +0 -80
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f3c002736fab9c81e80d622d7a42a3f68bfce89d
|
|
4
|
+
data.tar.gz: 36f6a4597cc1f5ce070ecdd7d9c2e42853aaaf64
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 250193cf08ac4a162fbadbab501b9523ffc0533ca2922c2457883357fc294087ed3d37cafb0fad69339695e4e5040b24401859cd0ce7ad8e99a95a669584e1a8
|
|
7
|
+
data.tar.gz: 22fa27762291548035b8ca9ccbd3a19b7418e23897061da0e6f2f5098dae934f5e6c19bbacb6186792c2ac727b6d9742ec8850ac34a187d2dfe5fb2d2b1a53c1
|
data/lib/javlibrary.rb
CHANGED
|
@@ -15,13 +15,15 @@ module Javlibrary
|
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def Javlibrary.downloader(identifer)
|
|
18
|
-
baseurl = "http://www.
|
|
18
|
+
baseurl = "http://www.javlibrary.com/cn/?v=#{identifer}"
|
|
19
19
|
response = Mechanize.new
|
|
20
20
|
response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
|
|
21
21
|
begin
|
|
22
22
|
response.get baseurl
|
|
23
|
-
rescue
|
|
23
|
+
rescue Timeout::Error
|
|
24
24
|
retry
|
|
25
|
+
rescue
|
|
26
|
+
return
|
|
25
27
|
end
|
|
26
28
|
|
|
27
29
|
doc = Nokogiri::HTML(response.page.body)
|
|
@@ -134,7 +136,7 @@ module Javlibrary
|
|
|
134
136
|
def Javlibrary.genres
|
|
135
137
|
response = Mechanize.new; genres = Array.new
|
|
136
138
|
begin
|
|
137
|
-
response.get "http://www.
|
|
139
|
+
response.get "http://www.javlibrary.com/cn/genres.php"
|
|
138
140
|
rescue
|
|
139
141
|
retry
|
|
140
142
|
end
|
|
@@ -168,7 +170,7 @@ module Javlibrary
|
|
|
168
170
|
end
|
|
169
171
|
|
|
170
172
|
def get_all_actor
|
|
171
|
-
firsturl = "http://www.
|
|
173
|
+
firsturl = "http://www.javlibrary.com/cn/star_list.php?prefix="
|
|
172
174
|
|
|
173
175
|
client = Javlibrary.client
|
|
174
176
|
'A'.upto('Z') do |alphabet|
|
|
@@ -209,8 +211,8 @@ module Javlibrary
|
|
|
209
211
|
end
|
|
210
212
|
|
|
211
213
|
def Javlibrary.download_video_label(actor_id)
|
|
212
|
-
firsturl = "http://www.
|
|
213
|
-
baseurl = "http://www.
|
|
214
|
+
firsturl = "http://www.javlibrary.com/ja/vl_star.php?s=#{actor_id}"
|
|
215
|
+
baseurl = "http://www.javlibrary.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="
|
|
214
216
|
|
|
215
217
|
begin
|
|
216
218
|
response = RestClient.get firsturl
|
data/lib/javlibrary/database.rb
CHANGED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
require 'mysql2'
|
|
4
|
-
|
|
5
|
-
module Javlibrary
|
|
6
|
-
def client
|
|
7
|
-
client = Mysql2::Client.new(:host => "127.0.0.1",
|
|
8
|
-
:username => "root",
|
|
9
|
-
:password => "XuHefeng",
|
|
10
|
-
:database => "javlibrary_new")
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
module_function :client
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
|
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
|
-
|
|
3
|
-
require 'mysql2'
|
|
4
|
-
require 'mechanize'
|
|
5
|
-
require 'mechanize'
|
|
6
|
-
require 'pp'
|
|
7
|
-
|
|
8
|
-
require_relative 'database'
|
|
9
|
-
require_relative 'info_hash'
|
|
10
|
-
|
|
11
|
-
module Javlibrary
|
|
12
|
-
def Javlibrary.downloader(identifer)
|
|
13
|
-
baseurl = "http://www.jav11b.com/cn/?v=#{identifer}"
|
|
14
|
-
response = Mechanize.new
|
|
15
|
-
response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
|
|
16
|
-
begin
|
|
17
|
-
response.get baseurl
|
|
18
|
-
rescue
|
|
19
|
-
retry
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
doc = Nokogiri::HTML(response.page.body)
|
|
23
|
-
|
|
24
|
-
video_title, details, video_genres, video_jacket_img = String.new, Array.new, String.new, String.new
|
|
25
|
-
|
|
26
|
-
video_title = doc.search('div[@id="video_title"]/h3/a').children.text
|
|
27
|
-
doc.search('//div[@id="video_info"]/div[@class="item"]/table/tr/td[@class="text"]').map do |row|
|
|
28
|
-
details << row.children.text
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
doc.search('//div[@id="video_genres"]/table/tr/td[@class="text"]/span[@class="genre"]/a').each do |row|
|
|
32
|
-
video_genres << row.children.text << " "
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
doc.search('//img[@id="video_jacket_img"]').each do |row|
|
|
36
|
-
video_jacket_img = row['src']
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
# return data format: title$id$date$director$maker$label$cast$genres$img_url
|
|
40
|
-
"#{video_title}$#{details[0]}$#{details[1]}$#{details[2]}$#{details[3]}$#{details[4]}$#{details[-1]}$#{video_genres}$#{video_jacket_img}"
|
|
41
|
-
#result = Hash.new
|
|
42
|
-
#result["title"] = video_title; result["id"] = details[0]; result["date"] = details[1]
|
|
43
|
-
#result["director"] = details[2]; result["maker"] = details[3]; result["label"] = details[4]
|
|
44
|
-
#result["cast"] = details[-1]; result["genres"] = video_genres; result["img_url"] = video_jacket_img
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def Javlibrary.video_info_insert(client, index, identifer, actor_hash, genres_hash)
|
|
48
|
-
result = downloader(identifer)
|
|
49
|
-
title, id, date, director, maker, label, cast_tmp, genres_tmp, img_url = downloader(identifer).split('$')
|
|
50
|
-
cast = cast_tmp.split.reject(&:empty?)
|
|
51
|
-
genres = genres_tmp.split.reject(&:empty?)
|
|
52
|
-
begin
|
|
53
|
-
client.query("INSERT INTO video (video_id,video_name,license,url,director,label,date,maker)
|
|
54
|
-
VALUES (#{index},'#{title}','#{id}','#{img_url}','#{director}','#{label}','#{date}','#{maker}')")
|
|
55
|
-
rescue
|
|
56
|
-
return nil
|
|
57
|
-
end
|
|
58
|
-
cast.each do |a|
|
|
59
|
-
a_tmp = actor_hash[a]
|
|
60
|
-
next if a_tmp == nil
|
|
61
|
-
client.query("INSERT INTO v2a (v2a_fk_video,v2a_fk_actor) VALUES(#{index}, #{a_tmp.to_i})")
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
genres.each do |g|
|
|
65
|
-
g_tmp = genres_hash[g]
|
|
66
|
-
next if g_tmp == nil
|
|
67
|
-
client.query("INSERT INTO v2c (v2c_fk_video,v2c_fk_category) VALUES(#{index}, #{g_tmp.to_i})")
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
client.query("UPDATE label SET video_download=1 WHERE video_num=#{index}")
|
|
71
|
-
return nil
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def download_all_video
|
|
75
|
-
client = Javlibrary.client
|
|
76
|
-
result = client.query("SELECT video_num, video_label FROM label WHERE video_download=0")
|
|
77
|
-
client.close
|
|
78
|
-
|
|
79
|
-
video_array = Array.new
|
|
80
|
-
result.each do |item|
|
|
81
|
-
video_array << item
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
video_array = video_array.each_slice(5000).to_a
|
|
85
|
-
actor_hash = Javlibrary::actor_hash
|
|
86
|
-
genre_hash = Javlibrary::genre_hash
|
|
87
|
-
thread_pool = Array::new
|
|
88
|
-
|
|
89
|
-
video_array.each do |group|
|
|
90
|
-
# Create a download thread
|
|
91
|
-
thread_temp = Thread.new {
|
|
92
|
-
client = Javlibrary.client
|
|
93
|
-
group.each do |item|
|
|
94
|
-
begin
|
|
95
|
-
video_info_insert(client, item['video_num'], item['video_label'],
|
|
96
|
-
actor_hash, genre_hash)
|
|
97
|
-
rescue
|
|
98
|
-
next
|
|
99
|
-
end
|
|
100
|
-
end
|
|
101
|
-
client.close
|
|
102
|
-
}
|
|
103
|
-
thread_pool << thread_temp
|
|
104
|
-
end
|
|
105
|
-
thread_pool.map(&:join)
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def test
|
|
109
|
-
pp downloader('javlia322m')
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
module_function :test
|
|
113
|
-
module_function :download_all_video
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
Javlibrary::download_all_video
|
data/lib/javlibrary/info_hash.rb
CHANGED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
|
-
|
|
3
|
-
require 'mysql2'
|
|
4
|
-
|
|
5
|
-
module Javlibrary
|
|
6
|
-
def actor_hash
|
|
7
|
-
client = Javlibrary.client
|
|
8
|
-
actor_hash = Hash.new
|
|
9
|
-
client.query("SELECT * FROM actor").each do |item|
|
|
10
|
-
actor_hash["#{item['actor_name']}"] = item['actor_id']
|
|
11
|
-
end
|
|
12
|
-
client.close
|
|
13
|
-
|
|
14
|
-
actor_hash
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def genre_hash
|
|
18
|
-
client = Javlibrary.client
|
|
19
|
-
category_hash = Hash.new
|
|
20
|
-
client.query("SELECT * FROM category").each do |item|
|
|
21
|
-
category_hash["#{item['category_name']}"] = item['category_id']
|
|
22
|
-
end
|
|
23
|
-
client.close
|
|
24
|
-
|
|
25
|
-
category_hash
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
module_function :actor_hash, :genre_hash
|
|
29
|
-
end
|
data/lib/javlibrary/init.rb
CHANGED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
|
-
|
|
3
|
-
require 'mechanize'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'mysql2'
|
|
6
|
-
|
|
7
|
-
require_relative 'database'
|
|
8
|
-
|
|
9
|
-
module Javlibrary
|
|
10
|
-
def Javlibrary.genres
|
|
11
|
-
response = Mechanize.new; genres = Array.new
|
|
12
|
-
begin
|
|
13
|
-
response.get "http://www.jav11b.com/cn/genres.php"
|
|
14
|
-
rescue
|
|
15
|
-
retry
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
Nokogiri::HTML(response.page.body).search('//div[@class="genreitem"]/a').each do |row|
|
|
19
|
-
genres << row.children.text
|
|
20
|
-
end
|
|
21
|
-
genres.uniq
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def genres_insert
|
|
25
|
-
client = Javlibrary.client
|
|
26
|
-
genres = genres()
|
|
27
|
-
genres.each do |e|
|
|
28
|
-
begin
|
|
29
|
-
client.query("INSERT INTO category (category_name) VALUES ('#{e}')")
|
|
30
|
-
rescue
|
|
31
|
-
next
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
client.close
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
module_function :genres_insert
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
Javlibrary::genres_insert
|
data/lib/javlibrary/name.rb
CHANGED
|
@@ -1,53 +1 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
1
|
|
|
3
|
-
require 'rest-client'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'mysql2'
|
|
6
|
-
|
|
7
|
-
require_relative 'database.rb'
|
|
8
|
-
|
|
9
|
-
module Javlibrary
|
|
10
|
-
def Javlibrary.author_page_num(nokogiri_doc)
|
|
11
|
-
last_page = 1
|
|
12
|
-
nokogiri_doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
|
|
13
|
-
last_page = row['href'].split("=")[-1].to_i
|
|
14
|
-
end
|
|
15
|
-
last_page
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def get_all_actor
|
|
19
|
-
firsturl = "http://www.jav11b.com/cn/star_list.php?prefix="
|
|
20
|
-
|
|
21
|
-
client = Javlibrary.client
|
|
22
|
-
'A'.upto('Z') do |alphabet|
|
|
23
|
-
tempurl = firsturl + alphabet
|
|
24
|
-
response = RestClient.get tempurl
|
|
25
|
-
|
|
26
|
-
doc = Nokogiri::HTML(response.body)
|
|
27
|
-
last_page = author_page_num(doc)
|
|
28
|
-
|
|
29
|
-
1.upto(last_page) do |page_num|
|
|
30
|
-
temp_page_url = tempurl + "&page=#{page_num.to_s}"
|
|
31
|
-
response_page = RestClient.get temp_page_url
|
|
32
|
-
doc_page = Nokogiri::HTML(response_page.body)
|
|
33
|
-
doc_page.search('//div[@class="starbox"]/div[@class="searchitem"]/a').each do |row|
|
|
34
|
-
# row.text Actor.name
|
|
35
|
-
# row['href'].split("=")[-1] Actor.label
|
|
36
|
-
name = row.text; label = row['href'].split("=")[-1]
|
|
37
|
-
begin
|
|
38
|
-
client.query("INSERT INTO actor (actor_name, actor_label, type)
|
|
39
|
-
VALUES ('#{name}', '#{label}', '#{alphabet}')")
|
|
40
|
-
rescue
|
|
41
|
-
next
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
client.close
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
module_function :get_all_actor
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
Javlibrary::get_all_actor
|
data/lib/javlibrary/version.rb
CHANGED
data/lib/javlibrary/video.rb
CHANGED
|
@@ -1,81 +1 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
1
|
|
|
3
|
-
require 'rest-client'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'mysql2'
|
|
6
|
-
require 'redis'
|
|
7
|
-
require 'pp'
|
|
8
|
-
|
|
9
|
-
require_relative 'database'
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
module Javlibrary
|
|
13
|
-
def Javlibrary.download_video_label(actor_id)
|
|
14
|
-
firsturl = "http://www.jav11b.com/ja/vl_star.php?s=#{actor_id}"
|
|
15
|
-
baseurl = "http://www.jav11b.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="
|
|
16
|
-
|
|
17
|
-
begin
|
|
18
|
-
response = RestClient.get firsturl
|
|
19
|
-
rescue
|
|
20
|
-
retry
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
doc = Nokogiri::HTML(response.body)
|
|
24
|
-
last_page = 1
|
|
25
|
-
doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
|
|
26
|
-
last_page = row['href'].split("=")[-1].to_i
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
result = []
|
|
30
|
-
1.upto(last_page) do |page|
|
|
31
|
-
tempurl = baseurl + page.to_s
|
|
32
|
-
response = RestClient.get tempurl
|
|
33
|
-
Nokogiri::HTML(response.body).search('//div[@class="video"]/a').each do |row|
|
|
34
|
-
# Data:
|
|
35
|
-
# Video_label: row['href'].split("=")[-1]
|
|
36
|
-
# Video_title: row['title']
|
|
37
|
-
# client.query("INSERT INTO label (lable) VALUES ('#{row['href'].split("=")[-1]}')")
|
|
38
|
-
result << row['href'].split("=")[-1]
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
client = Javlibrary.client
|
|
43
|
-
result.each do |e|
|
|
44
|
-
begin
|
|
45
|
-
client.query("INSERT INTO label (video_label, video_download) VALUES ('#{e}', '0')")
|
|
46
|
-
rescue
|
|
47
|
-
next
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
client.close
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def Javlibrary.select_actor(type)
|
|
54
|
-
client = Javlibrary.client
|
|
55
|
-
result = client.query("SELECT actor_label FROM actor WHERE type='#{type}'")
|
|
56
|
-
client.close
|
|
57
|
-
|
|
58
|
-
result.each do |e|
|
|
59
|
-
download_video_label(e["actor_label"])
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def download_all_video_label
|
|
64
|
-
thread_pool =[]
|
|
65
|
-
'A'.upto('Z').each do |alphabet|
|
|
66
|
-
thread_temp = Thread.new{
|
|
67
|
-
select_actor(alphabet)
|
|
68
|
-
}
|
|
69
|
-
thread_pool << thread_temp
|
|
70
|
-
end
|
|
71
|
-
thread_pool.map(&:join)
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def download_all_video_info
|
|
75
|
-
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
module_function :download_all_video_label
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
Javlibrary::download_all_video_label
|