javlibrary 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/javlibrary.rb +8 -6
- data/lib/javlibrary/database.rb +0 -16
- data/lib/javlibrary/downloader.rb +0 -116
- data/lib/javlibrary/info_hash.rb +0 -29
- data/lib/javlibrary/init.rb +0 -41
- data/lib/javlibrary/name.rb +0 -52
- data/lib/javlibrary/version.rb +1 -1
- data/lib/javlibrary/video.rb +0 -80
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f3c002736fab9c81e80d622d7a42a3f68bfce89d
|
|
4
|
+
data.tar.gz: 36f6a4597cc1f5ce070ecdd7d9c2e42853aaaf64
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 250193cf08ac4a162fbadbab501b9523ffc0533ca2922c2457883357fc294087ed3d37cafb0fad69339695e4e5040b24401859cd0ce7ad8e99a95a669584e1a8
|
|
7
|
+
data.tar.gz: 22fa27762291548035b8ca9ccbd3a19b7418e23897061da0e6f2f5098dae934f5e6c19bbacb6186792c2ac727b6d9742ec8850ac34a187d2dfe5fb2d2b1a53c1
|
data/lib/javlibrary.rb
CHANGED
|
@@ -15,13 +15,15 @@ module Javlibrary
|
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def Javlibrary.downloader(identifer)
|
|
18
|
-
baseurl = "http://www.
|
|
18
|
+
baseurl = "http://www.javlibrary.com/cn/?v=#{identifer}"
|
|
19
19
|
response = Mechanize.new
|
|
20
20
|
response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
|
|
21
21
|
begin
|
|
22
22
|
response.get baseurl
|
|
23
|
-
rescue
|
|
23
|
+
rescue Timeout::Error
|
|
24
24
|
retry
|
|
25
|
+
rescue
|
|
26
|
+
return
|
|
25
27
|
end
|
|
26
28
|
|
|
27
29
|
doc = Nokogiri::HTML(response.page.body)
|
|
@@ -134,7 +136,7 @@ module Javlibrary
|
|
|
134
136
|
def Javlibrary.genres
|
|
135
137
|
response = Mechanize.new; genres = Array.new
|
|
136
138
|
begin
|
|
137
|
-
response.get "http://www.
|
|
139
|
+
response.get "http://www.javlibrary.com/cn/genres.php"
|
|
138
140
|
rescue
|
|
139
141
|
retry
|
|
140
142
|
end
|
|
@@ -168,7 +170,7 @@ module Javlibrary
|
|
|
168
170
|
end
|
|
169
171
|
|
|
170
172
|
def get_all_actor
|
|
171
|
-
firsturl = "http://www.
|
|
173
|
+
firsturl = "http://www.javlibrary.com/cn/star_list.php?prefix="
|
|
172
174
|
|
|
173
175
|
client = Javlibrary.client
|
|
174
176
|
'A'.upto('Z') do |alphabet|
|
|
@@ -209,8 +211,8 @@ module Javlibrary
|
|
|
209
211
|
end
|
|
210
212
|
|
|
211
213
|
def Javlibrary.download_video_label(actor_id)
|
|
212
|
-
firsturl = "http://www.
|
|
213
|
-
baseurl = "http://www.
|
|
214
|
+
firsturl = "http://www.javlibrary.com/ja/vl_star.php?s=#{actor_id}"
|
|
215
|
+
baseurl = "http://www.javlibrary.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="
|
|
214
216
|
|
|
215
217
|
begin
|
|
216
218
|
response = RestClient.get firsturl
|
data/lib/javlibrary/database.rb
CHANGED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
# encoding: utf-8
|
|
2
|
-
|
|
3
|
-
require 'mysql2'
|
|
4
|
-
|
|
5
|
-
module Javlibrary
|
|
6
|
-
def client
|
|
7
|
-
client = Mysql2::Client.new(:host => "127.0.0.1",
|
|
8
|
-
:username => "root",
|
|
9
|
-
:password => "XuHefeng",
|
|
10
|
-
:database => "javlibrary_new")
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
module_function :client
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
|
|
@@ -1,116 +0,0 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
|
-
|
|
3
|
-
require 'mysql2'
|
|
4
|
-
require 'mechanize'
|
|
5
|
-
require 'mechanize'
|
|
6
|
-
require 'pp'
|
|
7
|
-
|
|
8
|
-
require_relative 'database'
|
|
9
|
-
require_relative 'info_hash'
|
|
10
|
-
|
|
11
|
-
module Javlibrary
|
|
12
|
-
def Javlibrary.downloader(identifer)
|
|
13
|
-
baseurl = "http://www.jav11b.com/cn/?v=#{identifer}"
|
|
14
|
-
response = Mechanize.new
|
|
15
|
-
response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
|
|
16
|
-
begin
|
|
17
|
-
response.get baseurl
|
|
18
|
-
rescue
|
|
19
|
-
retry
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
doc = Nokogiri::HTML(response.page.body)
|
|
23
|
-
|
|
24
|
-
video_title, details, video_genres, video_jacket_img = String.new, Array.new, String.new, String.new
|
|
25
|
-
|
|
26
|
-
video_title = doc.search('div[@id="video_title"]/h3/a').children.text
|
|
27
|
-
doc.search('//div[@id="video_info"]/div[@class="item"]/table/tr/td[@class="text"]').map do |row|
|
|
28
|
-
details << row.children.text
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
doc.search('//div[@id="video_genres"]/table/tr/td[@class="text"]/span[@class="genre"]/a').each do |row|
|
|
32
|
-
video_genres << row.children.text << " "
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
doc.search('//img[@id="video_jacket_img"]').each do |row|
|
|
36
|
-
video_jacket_img = row['src']
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
# return data format: title$id$date$director$maker$label$cast$genres$img_url
|
|
40
|
-
"#{video_title}$#{details[0]}$#{details[1]}$#{details[2]}$#{details[3]}$#{details[4]}$#{details[-1]}$#{video_genres}$#{video_jacket_img}"
|
|
41
|
-
#result = Hash.new
|
|
42
|
-
#result["title"] = video_title; result["id"] = details[0]; result["date"] = details[1]
|
|
43
|
-
#result["director"] = details[2]; result["maker"] = details[3]; result["label"] = details[4]
|
|
44
|
-
#result["cast"] = details[-1]; result["genres"] = video_genres; result["img_url"] = video_jacket_img
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def Javlibrary.video_info_insert(client, index, identifer, actor_hash, genres_hash)
|
|
48
|
-
result = downloader(identifer)
|
|
49
|
-
title, id, date, director, maker, label, cast_tmp, genres_tmp, img_url = downloader(identifer).split('$')
|
|
50
|
-
cast = cast_tmp.split.reject(&:empty?)
|
|
51
|
-
genres = genres_tmp.split.reject(&:empty?)
|
|
52
|
-
begin
|
|
53
|
-
client.query("INSERT INTO video (video_id,video_name,license,url,director,label,date,maker)
|
|
54
|
-
VALUES (#{index},'#{title}','#{id}','#{img_url}','#{director}','#{label}','#{date}','#{maker}')")
|
|
55
|
-
rescue
|
|
56
|
-
return nil
|
|
57
|
-
end
|
|
58
|
-
cast.each do |a|
|
|
59
|
-
a_tmp = actor_hash[a]
|
|
60
|
-
next if a_tmp == nil
|
|
61
|
-
client.query("INSERT INTO v2a (v2a_fk_video,v2a_fk_actor) VALUES(#{index}, #{a_tmp.to_i})")
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
genres.each do |g|
|
|
65
|
-
g_tmp = genres_hash[g]
|
|
66
|
-
next if g_tmp == nil
|
|
67
|
-
client.query("INSERT INTO v2c (v2c_fk_video,v2c_fk_category) VALUES(#{index}, #{g_tmp.to_i})")
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
client.query("UPDATE label SET video_download=1 WHERE video_num=#{index}")
|
|
71
|
-
return nil
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def download_all_video
|
|
75
|
-
client = Javlibrary.client
|
|
76
|
-
result = client.query("SELECT video_num, video_label FROM label WHERE video_download=0")
|
|
77
|
-
client.close
|
|
78
|
-
|
|
79
|
-
video_array = Array.new
|
|
80
|
-
result.each do |item|
|
|
81
|
-
video_array << item
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
video_array = video_array.each_slice(5000).to_a
|
|
85
|
-
actor_hash = Javlibrary::actor_hash
|
|
86
|
-
genre_hash = Javlibrary::genre_hash
|
|
87
|
-
thread_pool = Array::new
|
|
88
|
-
|
|
89
|
-
video_array.each do |group|
|
|
90
|
-
# Create a download thread
|
|
91
|
-
thread_temp = Thread.new {
|
|
92
|
-
client = Javlibrary.client
|
|
93
|
-
group.each do |item|
|
|
94
|
-
begin
|
|
95
|
-
video_info_insert(client, item['video_num'], item['video_label'],
|
|
96
|
-
actor_hash, genre_hash)
|
|
97
|
-
rescue
|
|
98
|
-
next
|
|
99
|
-
end
|
|
100
|
-
end
|
|
101
|
-
client.close
|
|
102
|
-
}
|
|
103
|
-
thread_pool << thread_temp
|
|
104
|
-
end
|
|
105
|
-
thread_pool.map(&:join)
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
def test
|
|
109
|
-
pp downloader('javlia322m')
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
module_function :test
|
|
113
|
-
module_function :download_all_video
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
Javlibrary::download_all_video
|
data/lib/javlibrary/info_hash.rb
CHANGED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
|
-
|
|
3
|
-
require 'mysql2'
|
|
4
|
-
|
|
5
|
-
module Javlibrary
|
|
6
|
-
def actor_hash
|
|
7
|
-
client = Javlibrary.client
|
|
8
|
-
actor_hash = Hash.new
|
|
9
|
-
client.query("SELECT * FROM actor").each do |item|
|
|
10
|
-
actor_hash["#{item['actor_name']}"] = item['actor_id']
|
|
11
|
-
end
|
|
12
|
-
client.close
|
|
13
|
-
|
|
14
|
-
actor_hash
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def genre_hash
|
|
18
|
-
client = Javlibrary.client
|
|
19
|
-
category_hash = Hash.new
|
|
20
|
-
client.query("SELECT * FROM category").each do |item|
|
|
21
|
-
category_hash["#{item['category_name']}"] = item['category_id']
|
|
22
|
-
end
|
|
23
|
-
client.close
|
|
24
|
-
|
|
25
|
-
category_hash
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
module_function :actor_hash, :genre_hash
|
|
29
|
-
end
|
data/lib/javlibrary/init.rb
CHANGED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
|
-
|
|
3
|
-
require 'mechanize'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'mysql2'
|
|
6
|
-
|
|
7
|
-
require_relative 'database'
|
|
8
|
-
|
|
9
|
-
module Javlibrary
|
|
10
|
-
def Javlibrary.genres
|
|
11
|
-
response = Mechanize.new; genres = Array.new
|
|
12
|
-
begin
|
|
13
|
-
response.get "http://www.jav11b.com/cn/genres.php"
|
|
14
|
-
rescue
|
|
15
|
-
retry
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
Nokogiri::HTML(response.page.body).search('//div[@class="genreitem"]/a').each do |row|
|
|
19
|
-
genres << row.children.text
|
|
20
|
-
end
|
|
21
|
-
genres.uniq
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def genres_insert
|
|
25
|
-
client = Javlibrary.client
|
|
26
|
-
genres = genres()
|
|
27
|
-
genres.each do |e|
|
|
28
|
-
begin
|
|
29
|
-
client.query("INSERT INTO category (category_name) VALUES ('#{e}')")
|
|
30
|
-
rescue
|
|
31
|
-
next
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
client.close
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
module_function :genres_insert
|
|
39
|
-
end
|
|
40
|
-
|
|
41
|
-
Javlibrary::genres_insert
|
data/lib/javlibrary/name.rb
CHANGED
|
@@ -1,53 +1 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
1
|
|
|
3
|
-
require 'rest-client'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'mysql2'
|
|
6
|
-
|
|
7
|
-
require_relative 'database.rb'
|
|
8
|
-
|
|
9
|
-
module Javlibrary
|
|
10
|
-
def Javlibrary.author_page_num(nokogiri_doc)
|
|
11
|
-
last_page = 1
|
|
12
|
-
nokogiri_doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
|
|
13
|
-
last_page = row['href'].split("=")[-1].to_i
|
|
14
|
-
end
|
|
15
|
-
last_page
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def get_all_actor
|
|
19
|
-
firsturl = "http://www.jav11b.com/cn/star_list.php?prefix="
|
|
20
|
-
|
|
21
|
-
client = Javlibrary.client
|
|
22
|
-
'A'.upto('Z') do |alphabet|
|
|
23
|
-
tempurl = firsturl + alphabet
|
|
24
|
-
response = RestClient.get tempurl
|
|
25
|
-
|
|
26
|
-
doc = Nokogiri::HTML(response.body)
|
|
27
|
-
last_page = author_page_num(doc)
|
|
28
|
-
|
|
29
|
-
1.upto(last_page) do |page_num|
|
|
30
|
-
temp_page_url = tempurl + "&page=#{page_num.to_s}"
|
|
31
|
-
response_page = RestClient.get temp_page_url
|
|
32
|
-
doc_page = Nokogiri::HTML(response_page.body)
|
|
33
|
-
doc_page.search('//div[@class="starbox"]/div[@class="searchitem"]/a').each do |row|
|
|
34
|
-
# row.text Actor.name
|
|
35
|
-
# row['href'].split("=")[-1] Actor.label
|
|
36
|
-
name = row.text; label = row['href'].split("=")[-1]
|
|
37
|
-
begin
|
|
38
|
-
client.query("INSERT INTO actor (actor_name, actor_label, type)
|
|
39
|
-
VALUES ('#{name}', '#{label}', '#{alphabet}')")
|
|
40
|
-
rescue
|
|
41
|
-
next
|
|
42
|
-
end
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
client.close
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
module_function :get_all_actor
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
Javlibrary::get_all_actor
|
data/lib/javlibrary/version.rb
CHANGED
data/lib/javlibrary/video.rb
CHANGED
|
@@ -1,81 +1 @@
|
|
|
1
|
-
# encoding:utf-8
|
|
2
1
|
|
|
3
|
-
require 'rest-client'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'mysql2'
|
|
6
|
-
require 'redis'
|
|
7
|
-
require 'pp'
|
|
8
|
-
|
|
9
|
-
require_relative 'database'
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
module Javlibrary
|
|
13
|
-
def Javlibrary.download_video_label(actor_id)
|
|
14
|
-
firsturl = "http://www.jav11b.com/ja/vl_star.php?s=#{actor_id}"
|
|
15
|
-
baseurl = "http://www.jav11b.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="
|
|
16
|
-
|
|
17
|
-
begin
|
|
18
|
-
response = RestClient.get firsturl
|
|
19
|
-
rescue
|
|
20
|
-
retry
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
doc = Nokogiri::HTML(response.body)
|
|
24
|
-
last_page = 1
|
|
25
|
-
doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
|
|
26
|
-
last_page = row['href'].split("=")[-1].to_i
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
result = []
|
|
30
|
-
1.upto(last_page) do |page|
|
|
31
|
-
tempurl = baseurl + page.to_s
|
|
32
|
-
response = RestClient.get tempurl
|
|
33
|
-
Nokogiri::HTML(response.body).search('//div[@class="video"]/a').each do |row|
|
|
34
|
-
# Data:
|
|
35
|
-
# Video_label: row['href'].split("=")[-1]
|
|
36
|
-
# Video_title: row['title']
|
|
37
|
-
# client.query("INSERT INTO label (lable) VALUES ('#{row['href'].split("=")[-1]}')")
|
|
38
|
-
result << row['href'].split("=")[-1]
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
client = Javlibrary.client
|
|
43
|
-
result.each do |e|
|
|
44
|
-
begin
|
|
45
|
-
client.query("INSERT INTO label (video_label, video_download) VALUES ('#{e}', '0')")
|
|
46
|
-
rescue
|
|
47
|
-
next
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
client.close
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def Javlibrary.select_actor(type)
|
|
54
|
-
client = Javlibrary.client
|
|
55
|
-
result = client.query("SELECT actor_label FROM actor WHERE type='#{type}'")
|
|
56
|
-
client.close
|
|
57
|
-
|
|
58
|
-
result.each do |e|
|
|
59
|
-
download_video_label(e["actor_label"])
|
|
60
|
-
end
|
|
61
|
-
end
|
|
62
|
-
|
|
63
|
-
def download_all_video_label
|
|
64
|
-
thread_pool =[]
|
|
65
|
-
'A'.upto('Z').each do |alphabet|
|
|
66
|
-
thread_temp = Thread.new{
|
|
67
|
-
select_actor(alphabet)
|
|
68
|
-
}
|
|
69
|
-
thread_pool << thread_temp
|
|
70
|
-
end
|
|
71
|
-
thread_pool.map(&:join)
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def download_all_video_info
|
|
75
|
-
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
module_function :download_all_video_label
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
Javlibrary::download_all_video_label
|