video_chat_get 0.1.4 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 490967e4b44f6a5a5f765df1b2add73f53013f42c56f0cb3c3e30f4563d3aad6
-  data.tar.gz: 495f2cd0df05ca4ce6bcfd7a54e5000f6cd5f255c2cc51da039a0a6db89195e4
+  metadata.gz: 94f8d90857170ebe5524126c1c3c0d6d819eff1816d535ad0fd2156ee5d08c8e
+  data.tar.gz: 344dc8d57be6c1124dd13a3cdb5bb6ebe85c2ee1060cdb7e48da41363353daeb
 SHA512:
-  metadata.gz: 295f625e0d125c4c8d54377af795bd739d60b74ee6681f2d60012ce16d499dcaa75ab069137ca8e45475557499494075b536f7eaab8672acdcd066a65e11c68a
-  data.tar.gz: d94c0bdee710e761b12cd31a100d500ae66daa6428c18f6531269d730b848effd79175373cc4b067fa4ca90e6915401ec95eb10dd88c75d03cfb185550a35174
+  metadata.gz: aaa5bc65908f18d7450093ff7304f5e01a45fcfe4d027b1110c226f27ff03f85607870691e44c5c94bea533dc5e958d8732bfae1c58a26e096687e2d77cebfb3
+  data.tar.gz: 135ae6f0bc1ddff52b10e34c83f104cc0c82e3627eb0dda580dc78fa2da854fd9e657438d5fe7934843589c8142238668817d51a7beb064f27a5386b1aaab0fd

data/.circleci/config.yml

@@ -1,4 +1,8 @@
 version: 2.1
+
+orbs:
+  slack: circleci/slack@3.3.0
+
 executors:
    my-executor:
     docker:
@@ -27,13 +31,25 @@ jobs:
       - run: 
            name: bundle exec rake spec
            command: bundle exec rake spec
+      
+      - slack/status:
+          success_message: ':circleci-pass: $CIRCLE_BRANCH のテストが完了しました\n:github_octocat: User：$CIRCLE_USERNAME'
+          failure_message: ':circleci-fail: $CIRCLE_BRANCH のテストが失敗しました\n:github_octocat: User：$CIRCLE_USERNAME'
+          webhook: '${SLACK_WEBHOOK}'
 
 workflows:
   version: 2
-  test-:
+  test:
     jobs:
-      - test:
-         filters:
+      - test
+
+  test_cron: 
+    triggers:
+      - schedule:
+          cron: "0 21 * * 0" 
+          filters:
             branches:
               only:
-                - master
+                - master
+    jobs:
+      - test

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/vendor/
 
 # rspec failure tracking
 .rspec_status

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    video_chat_get (0.1.4)
+    video_chat_get (0.2.1)
       httpclient (~> 2.8.3)
       json (~> 2.3.0)
       nokogiri (~> 1.10.9)
@@ -11,9 +11,9 @@ GEM
   specs:
     diff-lcs (1.3)
     httpclient (2.8.3)
-    json (2.3.0)
+    json (2.3.1)
     mini_portile2 (2.4.0)
-    nokogiri (1.10.9)
+    nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     rake (10.5.0)
     rspec (3.9.0)

data/README.md

@@ -8,7 +8,7 @@ This is a ruby ​​package that gets a list of chats from the broadcasts of vi
 |mildom |◯ |
 |niconico | |
 |openrec |◯ |
-|twitcast | |
+|twitcasting |◯ |
 |twitch | |
 |whowatch |◯ |
 |youtubelive |◯ |
@@ -50,6 +50,11 @@ obj=Openrec_analyze.new(url)
 video_info=obj.videoinfo
 chat_list=obj.chat_scrape()
 
+#twitcasting
+obj=Twitcasting_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
 #whowatch
 obj=Whowatch_analyze.new(url)
 video_info=obj.videoinfo

data/lib/file_operats/file_operat_chatdata.rb

@@ -1,15 +1,17 @@
 
 def file_write(data_list,log_flag,log_path)
 
-    if log_flag==true then
-        File.open(log_path,"w") do |file|
-            data_list.each do |chat|
-                file.puts chat
-            end
-        end
-        puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
-    else
+    if log_flag==false 
         puts "Scraping finished!! , log_flag false , chat log path is (no_path) "
+        return
     end
-
+    
+    File.open(log_path,"w") do |file|
+        data_list.each do |chat|
+            file.puts chat
+        end
+    end
+    puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
+    
+    return
 end

data/lib/requests/request.rb

@@ -5,16 +5,15 @@ require 'open-uri'
 
 
 def request_json_parse(url)
-    body_dic={}
 
     client=HTTPClient.new()
     response=client.get(url)
 
     if response.status==200
-     body_dic=JSON.parse(response.body)
+        return JSON.parse(response.body),response.status
     end
 
-    return body_dic,response.status
+    return nil,response.status
  end
 
 
@@ -22,13 +21,16 @@ def request_json_parse(url)
 
     charset=nil
     status=[]
-    response=open(url,opt) do |f|
+    response=URI.open(url,opt) do |f|
         charset=f.charset
         status=f.status
         f.read
     end
 
-    response_body=Nokogiri::HTML.parse(response,nil,charset)
-    return response_body,status
+    if status[0]=="200"
+        return Nokogiri::HTML.parse(response,nil,charset),status
+    end
+
+    return nil,status
  end
  

data/lib/site_list/mildom_analyze.rb

@@ -13,57 +13,57 @@ class Mildom_analyze<Video_analyze
 
     def initialize(url)
         @VIDEOINFO_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail?v_id="
+        @VIDEOINFO_REQEST_PARAMETER="&__platform=web"
         @CHAT_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/chat/replay?video_id="
         @CHAT_REQEST_PARAMETER="&time_offset_ms="
-        @CHAT_STARTTIME="0"
 
         @video_url=url
-        @video_id=videoid_get(@video_url)
-        @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
+        @video_id=videoid_get()
+        @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id+@VIDEOINFO_REQEST_PARAMETER)
         @chatlog_filepath="./"+@video_id+".txt"
     end
 
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("=")[1].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[5].split("&")[0]
     end
 
 
     def chat_nextpage_get(time_key)
-        time_key=time_key.to_s
-        chat_request_url=@CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key
-        return chat_request_url
+        return @CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key.to_s
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        next_url=chat_nextpage_get(@CHAT_STARTTIME)
-        chat_body,chat_status=request_json_parse(next_url)
-        
-        time_length=@videoinfo["body"]["playback"]["video_length"]
-        next_time=0
         chat_list=[]
-        
+        next_time=0
+        time_length=@videoinfo["body"]["playback"]["video_length"]
 
         while next_time<=time_length do
-                
+
+            chat_body=chat_body_get(next_time)
             chat_body["body"]["models"][0]["detail"][0..-1].each do |chat|
                 chat_list.push chat
             end
 
             next_time=chat_body["body"]["models"][0]["summary"]["end_offset_ms"]
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
             progressbar(next_time,time_length)
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
         return chat_list
     end
 
+
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
+
     public :chat_scrape
-    private :videoid_get, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
 end

data/lib/site_list/openrec_analyze.rb

@@ -17,16 +17,15 @@ class Openrec_analyze<Video_analyze
         @CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
 
         @video_url=url
-        @video_id=videoid_get(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     
     end
 
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("/")[4].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[4].split("&")[0]
     end
 
 
@@ -37,40 +36,43 @@ class Openrec_analyze<Video_analyze
     def chat_nextpage_get(time_key)
         #datatime型→time型→iso8601型
         time_key=DateTime.parse(time_key).to_time.utc.iso8601
-        chat_request_url=@VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
-
-        return chat_request_url
+        return @VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        start_time=@videoinfo["started_at"]
-        end_time=@videoinfo["ended_at"]
-        next_url=chat_nextpage_get(start_time)
-        chat_body,chat_status=request_json_parse(next_url)
-
         chat_list=[]
-        next_time=""
         head=0
+        next_time=@videoinfo["started_at"]
+        chat_body=chat_body_get(next_time)
         
         while !(chat_body[head..-1].empty?) do
+
             chat_body[head..-1].each do |chat|
                 chat_list.push chat
                 next_time=chat["posted_at"]
             end
-            
+
+            chat_body=chat_body_get(next_time)
             head=1
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
-            progressbar(next_time,end_time)
+
+            progressbar(next_time,@videoinfo["ended_at"])
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
         return chat_list
     end
 
+
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
+
     public :chat_scrape
-    private :videoid_get, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
 end

data/lib/site_list/twitcasting_analyze.rb

@@ -24,67 +24,76 @@ class Twitcasting_analyze<Video_analyze
 
 
     def videoid_get()
-        videoid=@video_url.split("/")[5].split("&")[0]
-        return videoid
+        return @video_url.split("/")[5].split("&")[0]
     end
 
 
     def userid_get()
-        userid=@video_url.split("/")[3]
-        return userid
+        return @video_url.split("/")[3]
     end
 
 
     def videoinfo_get()
 
         videoinfo={}
-        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.gsub(" ","")
+        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.strip
         videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
+        videoinfo["video_time"]=@videoinfo_request_body.at_css(".tw-player-duration-time").text.strip
+        videoinfo["video_start_time"]=@videoinfo_request_body.at_css(".tw-player-meta__status_item > time")[:datetime]
 
         videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
+        i=0
         videoinfo_polymer.each do |fact|
-            fact=fact.text
-            fact=fact.gsub(/ {2,}|\n/, "")
-            fact=fact.split(":",2)
-            videoinfo[fact[0]]=fact[1]
+            if i==1 then
+                videoinfo["total_view"]=fact.text.strip.split(":",2)[1]
+                return videoinfo
+            end
+            i+=1
         end
-        
+
         return videoinfo
     end
 
 
     def chat_page_range()
+        size=@chat_request_body.css(".tw-pager").css("a").size()
+        range=@chat_request_body.css(".tw-pager").css("a")[size-1].text
+        return range.to_i
+    end
+
+
+    def chat_date_get(chatinfo_body)
+
+        chat_list=[]
+        chat_fact_dic={}
+
+        chatinfo_body.css(".tw-comment-history-item").each do |chat|
+            chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
+            chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
+            chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
+            chat_list.push(chat_fact_dic)
+            chat_fact_dic={}
+        end
         
-        size=@chat_request_body.css(".paging").css("a").size()
-        range=@chat_request_body.css(".paging").css("a")[size-1].text
-        range=range.to_i
-        
-        return range
+        return chat_list
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        chat_fact_dic={}
         chat_list=[]
-        next_url=""
         chatinfo_body=@chat_request_body
-        page_count=0
         page_range=chat_page_range()
+        page_count=0
 
         while page_count<=page_range do
             begin
-                chatinfo_body.css(".tw-comment-history-item").each do |chat|
-                    chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
-                    chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
-                    chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
-                    chat_list.push(chat_fact_dic)
-                    chat_fact_dic={}
-                end
+                chat_list+=chat_date_get(chatinfo_body)
                 page_count+=1
                 next_url=@chat_request_url+"-"+"#{page_count}"
-                chatinfo_body,status=request_html_parse(next_url,{})
+                chatinfo_body,_=request_html_parse(next_url,{})
                 progressbar(page_count,page_range)
+                sleep(1)
 
             rescue
                 break
@@ -98,6 +107,6 @@ class Twitcasting_analyze<Video_analyze
     end
 
     public :chat_scrape
-    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range
+    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range, :chat_date_get
 
 end

data/lib/site_list/whowatch_analyze.rb

@@ -16,16 +16,15 @@ class Whowatch_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://api.whowatch.tv/lives/"
         
         @video_url=url
-        @video_id=videoid_get(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
 
     end
 
 
-    def videoid_get(url=@video_url)
-        videoid=url.split("/")[4]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[4]
     end
 
 
@@ -37,15 +36,14 @@ class Whowatch_analyze<Video_analyze
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
-        
-        comment_count=@videoinfo["live"]["comment_count"]
+
         chat_list=[]
         count=0
         
         @chat_body[0..-1].each do |chat|
             chat_list.push chat
             count+=1
-            progressbar(count,comment_count)
+            progressbar(count,@videoinfo["live"]["comment_count"])
         end
         
         file_write(chat_list,log_flag,log_path)