javlibrary 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8f45d8547442b57271158fda6f5f4532a95bf515
-  data.tar.gz: 75e66027075c77f07385afb7cf3e5e5957187fc6
+  metadata.gz: a9f057e97a5ad367420e48abd74ce00766698321
+  data.tar.gz: ff8adfe4c7e0cc48c27df322f0a87185eaf419ca
 SHA512:
-  metadata.gz: 407078027c7cb548bd5d3f0ef652d3f82e4b513709796beb76e478786a4a673f18a0a34de368d77212908d92bd3666c1d7cdd17fa159a18dabcb616f7f2936c9
-  data.tar.gz: 32a281b11d4094bf9025683175200d0921f9630e16745a1069755b897bcef92fd305762fa62d6cfd28518f10a62197290c33990775ed9bbb4cf076574f393e25
+  metadata.gz: cf3bfbf99a201d5110e4228b041a3de500ab0047d4208b22787afbd375fe5ff55eab8331ced816e25dcf7199126d885d140aff610ec3ecff0cbdc2a4584f357e
+  data.tar.gz: 8a64494d675a6242c1e01b75d8176065d68b4a521b1b971c59b38a28ce4725cb481205ef0882e618388970c04cb52740b648dd1823d7f4b544a730c988eb2d15

data/lib/javlibrary.rb

@@ -6,13 +6,265 @@ require 'nokogiri'
 require 'mysql2'
 require 'pp'
 
-require_relative 'javlibrary/init'
-require_relative 'javlibrary/database'
-require_relative 'javlibrary/downloader'
-require_relative 'javlibrary/info_hash'
-require_relative 'javlibrary/name'
-require_relative 'javlibrary/video'
-
 module Javlibrary
+    def client
+        client = Mysql2::Client.new(:host => "127.0.0.1",
+                                    :username => "root",
+                                    :password => "XuHefeng",
+                                    :database => "javlibrary_new")
+    end
+
+    def Javlibrary.downloader(identifer)
+        baseurl = "http://www.jav11b.com/cn/?v=#{identifer}"
+        response = Mechanize.new
+        response.user_agent = Mechanize::AGENT_ALIASES.values[rand(21)]
+        begin
+            response.get baseurl
+        rescue
+            retry
+        end
+
+        doc = Nokogiri::HTML(response.page.body)
+
+        video_title, details, video_genres, video_jacket_img = String.new, Array.new, String.new, String.new
+
+        video_title = doc.search('div[@id="video_title"]/h3/a').children.text
+        doc.search('//div[@id="video_info"]/div[@class="item"]/table/tr/td[@class="text"]').map do |row|
+            details << row.children.text
+        end
+
+        doc.search('//div[@id="video_genres"]/table/tr/td[@class="text"]/span[@class="genre"]/a').each do |row|
+            video_genres << row.children.text << " "
+        end
+
+        doc.search('//img[@id="video_jacket_img"]').each do |row|
+            video_jacket_img = row['src']
+        end
+
+        # return data format: title$id$date$director$maker$label$cast$genres$img_url
+        "#{video_title}$#{details[0]}$#{details[1]}$#{details[2]}$#{details[3]}$#{details[4]}$#{details[-1]}$#{video_genres}$#{video_jacket_img}"
+        #result = Hash.new
+        #result["title"] = video_title; result["id"] = details[0]; result["date"] = details[1]
+        #result["director"] = details[2]; result["maker"] = details[3]; result["label"] = details[4]
+        #result["cast"] = details[-1]; result["genres"] = video_genres; result["img_url"] = video_jacket_img
+    end
+
+    def Javlibrary.video_info_insert(client, index, identifer, actor_hash, genres_hash)
+        result = downloader(identifer)
+        title, id, date, director, maker, label, cast_tmp, genres_tmp, img_url = downloader(identifer).split('$')
+        cast = cast_tmp.split.reject(&:empty?)
+        genres = genres_tmp.split.reject(&:empty?)
+        begin
+            client.query("INSERT INTO video (video_id,video_name,license,url,director,label,date,maker)
+            VALUES (#{index},'#{title}','#{id}','#{img_url}','#{director}','#{label}','#{date}','#{maker}')")
+        rescue
+            return
+        end
+        cast.each do |a|
+            a_tmp = actor_hash[a]
+            next if a_tmp == nil
+            client.query("INSERT INTO v2a (v2a_fk_video,v2a_fk_actor) VALUES(#{index}, #{a_tmp.to_i})")
+        end
+
+        genres.each do |g|
+            g_tmp = genres_hash[g]
+            next if g_tmp == nil
+            client.query("INSERT INTO v2c (v2c_fk_video,v2c_fk_category) VALUES(#{index}, #{g_tmp.to_i})")
+        end
+
+        client.query("UPDATE label SET video_download=1 WHERE video_num=#{index}")
+        return nil
+    end
+
+    def download_all_video
+        client = Javlibrary.client
+        result = client.query("SELECT video_num, video_label FROM label WHERE video_download=0")
+        client.close
+
+        video_array = Array.new
+        result.each do |item|
+            video_array << item
+        end
+
+        video_array = video_array.each_slice(5000).to_a
+        actor_hash = Javlibrary::actor_hash
+        genre_hash = Javlibrary::genre_hash
+        thread_pool = Array::new
+
+        video_array.each do |group|
+            # Create a download thread
+            thread_temp = Thread.new {
+                client = Javlibrary.client
+                group.each do |item|
+                    begin
+                        video_info_insert(client, item['video_num'], item['video_label'],
+                            actor_hash, genre_hash)
+                    rescue
+                        next
+                    end
+                end
+                client.close
+            }
+            thread_pool << thread_temp
+        end
+        thread_pool.map(&:join)
+    end
+
+    def actor_hash
+        client = Javlibrary.client
+        actor_hash = Hash.new
+        client.query("SELECT * FROM actor").each do |item|
+            actor_hash["#{item['actor_name']}"] = item['actor_id']
+        end
+        client.close
+
+        actor_hash
+    end
+
+    def genre_hash
+        client = Javlibrary.client
+        category_hash = Hash.new
+        client.query("SELECT * FROM category").each do |item|
+            category_hash["#{item['category_name']}"] = item['category_id']
+        end
+        client.close
+
+        category_hash
+    end
+
+    def Javlibrary.genres
+        response = Mechanize.new; genres = Array.new
+        begin
+            response.get "http://www.jav11b.com/cn/genres.php"
+        rescue
+            retry
+        end
+
+        Nokogiri::HTML(response.page.body).search('//div[@class="genreitem"]/a').each do |row|
+            genres << row.children.text
+        end
+        genres.uniq
+    end
+
+    def genres_insert
+        client = Javlibrary.client
+        genres = genres()
+        genres.each do |e|
+            begin
+                client.query("INSERT INTO category (category_name) VALUES ('#{e}')")
+            rescue
+                next
+            end
+        end
+
+        client.close
+    end
+
+    def Javlibrary.author_page_num(nokogiri_doc)
+        last_page = 1
+        nokogiri_doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
+            last_page = row['href'].split("=")[-1].to_i
+        end
+        last_page
+    end
+
+    def get_all_actor
+        firsturl = "http://www.jav11b.com/cn/star_list.php?prefix="
+
+        client = Javlibrary.client
+        'A'.upto('Z') do |alphabet|
+            tempurl = firsturl + alphabet
+            response = RestClient.get tempurl
+
+            doc = Nokogiri::HTML(response.body)
+            last_page = author_page_num(doc)
+
+            1.upto(last_page) do |page_num|
+                temp_page_url = tempurl + "&page=#{page_num.to_s}"
+                response_page = RestClient.get temp_page_url
+                doc_page = Nokogiri::HTML(response_page.body)
+                doc_page.search('//div[@class="starbox"]/div[@class="searchitem"]/a').each do |row|
+                    # row.text Actor.name
+                    # row['href'].split("=")[-1] Actor.label
+                    name = row.text; label = row['href'].split("=")[-1]
+                    begin
+                        client.query("INSERT INTO actor (actor_name, actor_label, type)
+                            VALUES ('#{name}', '#{label}', '#{alphabet}')")
+                    rescue
+                        next
+                    end
+                end
+            end
+        end
+
+        client.close
+    end
+
+    def Javlibrary.download_video_label(actor_id)
+        firsturl = "http://www.jav11b.com/ja/vl_star.php?s=#{actor_id}"
+        baseurl = "http://www.jav11b.com/ja/vl_star.php?&mode=&s=#{actor_id}&page="
+
+        begin
+            response = RestClient.get firsturl
+        rescue
+            retry
+        end
+
+        doc = Nokogiri::HTML(response.body)
+        last_page = 1
+        doc.search('//div[@class="page_selector"]/a[@class="page last"]').each do |row|
+            last_page = row['href'].split("=")[-1].to_i
+        end
+
+        result = []
+        1.upto(last_page) do |page|
+            tempurl = baseurl + page.to_s
+            response = RestClient.get tempurl
+            Nokogiri::HTML(response.body).search('//div[@class="video"]/a').each do |row|
+                # Data:
+                # Video_label: row['href'].split("=")[-1]
+                # Video_title: row['title']
+                # client.query("INSERT INTO label (lable) VALUES ('#{row['href'].split("=")[-1]}')")
+                result << row['href'].split("=")[-1]
+            end
+        end
+
+        client = Javlibrary.client
+        result.each do |e|
+            begin
+                client.query("INSERT INTO label (video_label, video_download) VALUES ('#{e}', '0')")
+            rescue
+                next
+            end
+        end
+        client.close
+    end
+
+    def Javlibrary.select_actor(type)
+        client = Javlibrary.client
+        result = client.query("SELECT actor_label FROM actor WHERE type='#{type}'")
+        client.close
+
+        result.each do |e|
+            download_video_label(e["actor_label"])
+        end
+    end
+
+    def download_all_video_label
+        thread_pool =[]
+        'A'.upto('Z').each do |alphabet|
+            thread_temp = Thread.new{
+                select_actor(alphabet)
+            }
+            thread_pool << thread_temp
+        end
+        thread_pool.map(&:join)
+    end
 
+    module_function :client
+    module_function :download_all_video
+    module_function :actor_hash, :genre_hash
+    module_function :genres_insert
+    module_function :get_all_actor
+    module_function :download_all_video_label
 end

data/lib/javlibrary/database.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+require 'mysql2'
+
 module Javlibrary
     def client
         client = Mysql2::Client.new(:host => "127.0.0.1",
@@ -10,3 +12,5 @@ module Javlibrary
 
     module_function :client
 end
+
+

data/lib/javlibrary/downloader.rb

@@ -1,5 +1,13 @@
 # encoding:utf-8
 
+require 'mysql2'
+require 'mechanize'
+require 'mechanize'
+require 'pp'
+
+require_relative 'database'
+require_relative 'info_hash'
+
 module Javlibrary
     def Javlibrary.downloader(identifer)
         baseurl = "http://www.jav11b.com/cn/?v=#{identifer}"
@@ -97,6 +105,11 @@ module Javlibrary
         thread_pool.map(&:join)
     end
 
+    def test
+        pp downloader('javlia322m')
+    end
+
+    module_function :test
     module_function :download_all_video
 end
 

data/lib/javlibrary/info_hash.rb

@@ -1,5 +1,7 @@
 # encoding:utf-8
 
+require 'mysql2'
+
 module Javlibrary
     def actor_hash
         client = Javlibrary.client

data/lib/javlibrary/init.rb

@@ -1,5 +1,11 @@
 # encoding:utf-8
 
+require 'mechanize'
+require 'nokogiri'
+require 'mysql2'
+
+require_relative 'database'
+
 module Javlibrary
     def Javlibrary.genres
         response = Mechanize.new; genres = Array.new

data/lib/javlibrary/name.rb

@@ -1,5 +1,11 @@
 # encoding:utf-8
 
+require 'rest-client'
+require 'nokogiri'
+require 'mysql2'
+
+require_relative 'database.rb'
+
 module Javlibrary
     def Javlibrary.author_page_num(nokogiri_doc)
         last_page = 1

data/lib/javlibrary/version.rb

@@ -1,3 +1,3 @@
 module Javlibrary
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/lib/javlibrary/video.rb

@@ -1,5 +1,14 @@
 # encoding:utf-8
 
+require 'rest-client'
+require 'nokogiri'
+require 'mysql2'
+require 'redis'
+require 'pp'
+
+require_relative 'database'
+
+
 module Javlibrary
     def Javlibrary.download_video_label(actor_id)
         firsturl = "http://www.jav11b.com/ja/vl_star.php?s=#{actor_id}"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: javlibrary
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Yuanhao Sun