video_chat_get 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c3d71c5521116c13bf9c41874427297cfc11eae66110549640e63f26127db18a
-  data.tar.gz: e004bba054b4f8d024b8de7bf1599d5842db6fe853805a2d0f83927c18749583
+  metadata.gz: e4c548904d898fbcd56d725e6b7a7dd45df9f8d1a6133f279e0e7edb8a679bac
+  data.tar.gz: 29a668bf9086627306e46d436e93a4d1391cea3467e34791092f6f2332802c03
 SHA512:
-  metadata.gz: 22423a1cad39a3820f57e071564ebc7e6a747436089d02d5f578a7c2ccd2d0cdb8b1e10eaba646581f615d5ab83ebab90bf23d88831259f90e34675fc79e4a73
-  data.tar.gz: d9126eff4ae9a97398e754239d035425c33149a166b8e6161b48b29d6fceb5c5a87525df553379f904f72520e62791f52f36efb1e4a9eaa73893b377ebf28000
+  metadata.gz: 2a0123fff8d7773855e61ce3a7ff5aaf4dc94a0b361902ee7c3133ddc5f32897ea2a6092d920f85fffdd67ca129a77860b1a8e07f4232751ab447a4bd53fe6d4
+  data.tar.gz: 9b9e4a2071cd9f3674c66982f378d205d38f6dd814280455444cf521f48dd9fa3a401b5c4269ef2883fa617344bd5214404d8f1ef7e629d2e4dc1507c4fff8d3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    video_chat_get (0.1.5)
+    video_chat_get (0.1.6)
       httpclient (~> 2.8.3)
       json (~> 2.3.0)
       nokogiri (~> 1.10.9)

data/lib/file_operats/file_operat_chatdata.rb

@@ -1,15 +1,17 @@
 
 def file_write(data_list,log_flag,log_path)
 
-    if log_flag==true then
-        File.open(log_path,"w") do |file|
-            data_list.each do |chat|
-                file.puts chat
-            end
-        end
-        puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
-    else
+    if log_flag==false 
         puts "Scraping finished!! , log_flag false , chat log path is (no_path) "
+        return
     end
-
+    
+    File.open(log_path,"w") do |file|
+        data_list.each do |chat|
+            file.puts chat
+        end
+    end
+    puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
+    
+    return
 end

data/lib/requests/request.rb

@@ -5,16 +5,15 @@ require 'open-uri'
 
 
 def request_json_parse(url)
-    body_dic={}
 
     client=HTTPClient.new()
     response=client.get(url)
 
     if response.status==200
-     body_dic=JSON.parse(response.body)
+        return JSON.parse(response.body),response.status
     end
 
-    return body_dic,response.status
+    return nil,response.status
  end
 
 
@@ -28,7 +27,10 @@ def request_json_parse(url)
         f.read
     end
 
-    response_body=Nokogiri::HTML.parse(response,nil,charset)
-    return response_body,status
+    if status[0]=="200"
+        return Nokogiri::HTML.parse(response,nil,charset),status
+    end
+
+    return nil,status
  end
  

data/lib/site_list/mildom_analyze.rb

@@ -15,47 +15,38 @@ class Mildom_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail?v_id="
         @CHAT_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/chat/replay?video_id="
         @CHAT_REQEST_PARAMETER="&time_offset_ms="
-        @CHAT_STARTTIME="0"
 
         @video_url=url
-        @video_id=videoid_get(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     end
 
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("=")[1].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[5].split("&")[0]
     end
 
 
     def chat_nextpage_get(time_key)
-        time_key=time_key.to_s
-        chat_request_url=@CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key
-        return chat_request_url
+        return @CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key.to_s
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        next_url=chat_nextpage_get(@CHAT_STARTTIME)
-        chat_body,chat_status=request_json_parse(next_url)
-        
-        time_length=@videoinfo["body"]["playback"]["video_length"]
-        next_time=0
         chat_list=[]
-        
+        next_time=0
+        time_length=@videoinfo["body"]["playback"]["video_length"]
 
         while next_time<=time_length do
-                
+
+            chat_body=chat_body_get(next_time)
             chat_body["body"]["models"][0]["detail"][0..-1].each do |chat|
                 chat_list.push chat
             end
 
             next_time=chat_body["body"]["models"][0]["summary"]["end_offset_ms"]
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
             progressbar(next_time,time_length)
             sleep(1)
         end
@@ -64,7 +55,14 @@ class Mildom_analyze<Video_analyze
         return chat_list
     end
 
+
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
+
     public :chat_scrape
-    private :videoid_get, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
 end

data/lib/site_list/openrec_analyze.rb

@@ -17,16 +17,15 @@ class Openrec_analyze<Video_analyze
         @CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
 
         @video_url=url
-        @video_id=videoid_get(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     
     end
 
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("/")[4].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[4].split("&")[0]
     end
 
 
@@ -37,33 +36,28 @@ class Openrec_analyze<Video_analyze
     def chat_nextpage_get(time_key)
         #datatime型→time型→iso8601型
         time_key=DateTime.parse(time_key).to_time.utc.iso8601
-        chat_request_url=@VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
-
-        return chat_request_url
+        return @VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        start_time=@videoinfo["started_at"]
-        end_time=@videoinfo["ended_at"]
-        next_url=chat_nextpage_get(start_time)
-        chat_body,chat_status=request_json_parse(next_url)
-
         chat_list=[]
-        next_time=""
         head=0
+        next_time=@videoinfo["started_at"]
+        chat_body=chat_body_get(next_time)
         
         while !(chat_body[head..-1].empty?) do
+
             chat_body[head..-1].each do |chat|
                 chat_list.push chat
                 next_time=chat["posted_at"]
             end
-            
+
+            chat_body=chat_body_get(next_time)
             head=1
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
-            progressbar(next_time,end_time)
+
+            progressbar(next_time,@videoinfo["ended_at"])
             sleep(1)
         end
 
@@ -71,7 +65,14 @@ class Openrec_analyze<Video_analyze
         return chat_list
     end
 
+
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
+
     public :chat_scrape
-    private :videoid_get, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
 end

data/lib/site_list/twitcasting_analyze.rb

@@ -24,66 +24,74 @@ class Twitcasting_analyze<Video_analyze
 
 
     def videoid_get()
-        videoid=@video_url.split("/")[5].split("&")[0]
-        return videoid
+        return @video_url.split("/")[5].split("&")[0]
     end
 
 
     def userid_get()
-        userid=@video_url.split("/")[3]
-        return userid
+        return @video_url.split("/")[3]
     end
 
 
     def videoinfo_get()
 
         videoinfo={}
-        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.gsub(" ","")
+        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.strip
         videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
+        videoinfo["video_time"]=@videoinfo_request_body.at_css(".tw-player-duration-time").text.strip
+        videoinfo["video_start_time"]=@videoinfo_request_body.at_css(".tw-player-meta__status_item > time")[:datetime]
 
         videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
+        i=0
         videoinfo_polymer.each do |fact|
-            fact=fact.text
-            fact=fact.gsub(/ {2,}|\n/, "")
-            fact=fact.split(":",2)
-            videoinfo[fact[0]]=fact[1]
+            if i==1 then
+                videoinfo["total_view"]=fact.text.strip.split(":",2)[1]
+                return videoinfo
+            end
+            i+=1
         end
-        
+
         return videoinfo
     end
 
 
     def chat_page_range()
-        
         size=@chat_request_body.css(".paging").css("a").size()
         range=@chat_request_body.css(".paging").css("a")[size-1].text
-        range=range.to_i
+        return range.to_i
+    end
+
+
+    def chat_date_get(chatinfo_body)
+
+        chat_list=[]
+        chat_fact_dic={}
+
+        chatinfo_body.css(".tw-comment-history-item").each do |chat|
+            chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
+            chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
+            chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
+            chat_list.push(chat_fact_dic)
+            chat_fact_dic={}
+        end
         
-        return range
+        return chat_list
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        chat_fact_dic={}
         chat_list=[]
-        next_url=""
         chatinfo_body=@chat_request_body
-        page_count=0
         page_range=chat_page_range()
+        page_count=0
 
         while page_count<=page_range do
             begin
-                chatinfo_body.css(".tw-comment-history-item").each do |chat|
-                    chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
-                    chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
-                    chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
-                    chat_list.push(chat_fact_dic)
-                    chat_fact_dic={}
-                end
+                chat_list+=chat_date_get(chatinfo_body)
                 page_count+=1
                 next_url=@chat_request_url+"-"+"#{page_count}"
-                chatinfo_body,status=request_html_parse(next_url,{})
+                chatinfo_body,_=request_html_parse(next_url,{})
                 progressbar(page_count,page_range)
                 sleep(1)
 
@@ -99,6 +107,6 @@ class Twitcasting_analyze<Video_analyze
     end
 
     public :chat_scrape
-    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range
+    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range, :chat_date_get
 
 end

data/lib/site_list/whowatch_analyze.rb

@@ -16,16 +16,15 @@ class Whowatch_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://api.whowatch.tv/lives/"
         
         @video_url=url
-        @video_id=videoid_get(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
 
     end
 
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("/")[4]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[4]
     end
 
 
@@ -37,15 +36,14 @@ class Whowatch_analyze<Video_analyze
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
-        
-        comment_count=@videoinfo["live"]["comment_count"]
+
         chat_list=[]
         count=0
         
         @chat_body[0..-1].each do |chat|
             chat_list.push chat
             count+=1
-            progressbar(count,comment_count)
+            progressbar(count,@videoinfo["live"]["comment_count"])
         end
         
         file_write(chat_list,log_flag,log_path)

data/lib/site_list/youtubelive_analyze.rb

@@ -15,27 +15,21 @@ class Youtubelive_analyze<Video_analyze
         @USER_AGENT='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
 
         @video_url=url
-        @video_id=videoid_get(@video_url)
-        @videoinfo_body,@videoinfo_request_status=videoinfo_get(@video_url)
+        @video_id=videoid_get()
+        @videoinfo_body,@videoinfo_request_status=videoinfo_get()
         @videoinfo=videoinfo_extraction()
         @chatlog_filepath="./"+@video_id+".txt"
     end
     
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("=")[1].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("=")[1].split("&")[0]
     end
 
 
-    def videoinfo_get(url=@video_url)
-        
-        opt={}
-        videoinfo_body=""
-
-        videoinfo_respons,videoinfo_status=request_html_parse(url,opt)
-        videoinfo_body=htmlpage_script_parse(videoinfo_respons,method(:videoinfo_script_cleanup))
-        return videoinfo_body,videoinfo_status
+    def videoinfo_get()
+        videoinfo_respons,videoinfo_status=request_html_parse(@video_url,{})
+        return htmlpage_script_parse(videoinfo_respons,method(:videoinfo_script_cleanup)),videoinfo_status
     end
     
 
@@ -55,54 +49,51 @@ class Youtubelive_analyze<Video_analyze
     
 
     def videoinfo_script_cleanup(script_date)
-        script_body=""
-        script_body=script_date.split("=",2)[1].split("\n",2)[0].strip.chomp(";")
-        return script_body
+        return script_date.split("=",2)[1].split("\n",2)[0].strip.chomp(";")
     end
 
 
     def chatinfo_script_cleanup(script_date)
-        script_body=""
-        script_body=script_date.split("=",2)[1].chomp("</script>").strip.chomp(";")
-        return script_body
+        return script_date.split("=",2)[1].chomp("</script>").strip.chomp(";")
     end
 
 
     def htmlpage_script_parse(respons,callback)
-        script_body=""
+
         respons.search('script').each do |script|
-            script=script.to_s
-            if script.include?("window[\"ytInitialData\"]") then
-                script_body=callback.call(script)
+            if script.to_s.include?("window[\"ytInitialData\"]") then
+                script_body=callback.call(script.to_s)
                 script_body=JSON.parse(script_body)
+                return script_body
             end
         end
-        return script_body
+        return ""
     end
 
 
+    def chat_body_get(next_url, opt={'User-Agent' => @USER_AGENT})
+        chat_respons,chat_status=request_html_parse(next_url,opt)
+        return htmlpage_script_parse(chat_respons,method(:chatinfo_script_cleanup))
+    end
+    
+
+
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        chat_body={}
         chat_list=[]
         chat_count=0
-        opt={'User-Agent' => @USER_AGENT}
-        continue_parameter=@videoinfo_body["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
-        next_url=@CHAT_REQUEST_URL+continue_parameter
+        next_url=@CHAT_REQUEST_URL + @videoinfo_body["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
 
         while true do
             begin
-                chat_respons,chat_status=request_html_parse(next_url,opt)
-                chat_body=htmlpage_script_parse(chat_respons,method(:chatinfo_script_cleanup))
-                continue_parameter = chat_body["continuationContents"]["liveChatContinuation"]["continuations"][0]["liveChatReplayContinuationData"]["continuation"]
-                next_url=@CHAT_REQUEST_URL+continue_parameter
-
+                chat_body=chat_body_get(next_url)
                 chat_body["continuationContents"]["liveChatContinuation"]["actions"][1..-1].each do |chat|
                     chat_count+=1
                     chat_list.push chat
                 end
-
                 progressbar(chat_count,"chat_count_inf")
+
+                next_url=@CHAT_REQUEST_URL + chat_body["continuationContents"]["liveChatContinuation"]["continuations"][0]["liveChatReplayContinuationData"]["continuation"]
                 sleep(1)
 
             rescue
@@ -117,6 +108,6 @@ class Youtubelive_analyze<Video_analyze
 
 
     public :chat_scrape
-    private :videoid_get, :videoinfo_get, :videoinfo_extraction, :htmlpage_script_parse, :videoinfo_script_cleanup, :chatinfo_script_cleanup
+    private :videoid_get, :videoinfo_get, :videoinfo_extraction, :htmlpage_script_parse, :videoinfo_script_cleanup, :chatinfo_script_cleanup, :chat_body_get
 
 end