video_chat_get 0.1.2 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe6d8d4965ef4236416987dd4eb0e0c953a29d8a5e0dbc723c2d70a063046668
-  data.tar.gz: 5ebe81abde7347cde06f88be947865a665cb69e6e5422b8663ebb3f379194c52
+  metadata.gz: 72c2d47a8f701af1d42780d9261be02ca29d743800971982bf42b9438ee4bdba
+  data.tar.gz: 90d62278f6cfee916570aad6ebe63acfb4122d9d19eb2f7dba4ef4b153561090
 SHA512:
-  metadata.gz: 2d0ba1f3b315a8006361207336e6a9c9a766660d3635c21d3a43044b006b50d113c84c5bf591378a72f3f423a1e77db9c3b4bd262845fbfe51fcaa1fcd65c626
-  data.tar.gz: 7bb4fcebe188bc33c89e45d979bca5f0a3cd886a40d2d44ae53d28061f307aafbafc9d99eade6f4911dc51f8e777a3dd2d6bea257cca3ab9002e642c29ce5816
+  metadata.gz: 45ed4d022d5b3b40da4ee41103fa589ad4eed996c1ece2afffbfcedfaed0621acac7ffe1555d9b6237d048fe2e5bd5fea06ed128913acedbe665cc1025d6a472
+  data.tar.gz: bafe20e7dbc9b3e67ac1f7d41aebebbde6bb43930e5cd5a300ce8b3babbfc79b408db9428834e0d5de9eaa2e62420f10440d83d518fe662d3e3f4a514957ffe9

data/.circleci/config.yml

@@ -0,0 +1,55 @@
+version: 2.1
+
+orbs:
+  slack: circleci/slack@3.3.0
+
+executors:
+   my-executor:
+    docker:
+      - image: circleci/ruby:2.5.8-node-browsers
+
+jobs:
+  test:
+    executor: my-executor
+    steps: 
+      - checkout 
+
+      - restore_cache:
+          name: cache lord
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+
+      - run: 
+         name: bundle install  
+         command: bundle check --path=vendor/bundle || bundle install --path vendor/bundle --clean --jobs 4 --retry 3
+
+      - save_cache:
+          name: save cache
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+          paths:
+            - ./vendor/bundle
+
+      - run: 
+           name: bundle exec rake spec
+           command: bundle exec rake spec
+      
+      - slack/status:
+          success_message: ':circleci-pass: $CIRCLE_BRANCH のテストが完了しました\n:github_octocat: User：$CIRCLE_USERNAME'
+          failure_message: ':circleci-fail: $CIRCLE_BRANCH のテストが失敗しました\n:github_octocat: User：$CIRCLE_USERNAME'
+          webhook: '${SLACK_WEBHOOK}'
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - test
+
+  test_cron: 
+    triggers:
+      - schedule:
+          cron: "0 21 * * 0" 
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - test

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    video_chat_get (0.1.2)
+    video_chat_get (0.1.9)
       httpclient (~> 2.8.3)
       json (~> 2.3.0)
       nokogiri (~> 1.10.9)

data/README.md

@@ -8,7 +8,7 @@ This is a ruby ​​package that gets a list of chats from the broadcasts of vi
 |mildom |◯ |
 |niconico | |
 |openrec |◯ |
-|twitcast | |
+|twitcasting |◯ |
 |twitch | |
 |whowatch |◯ |
 |youtubelive |◯ |
@@ -42,19 +42,28 @@ require 'video_chat_get'
 
 #mildom
 obj=Mildom_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 #openrec
 obj=Openrec_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#twitcasting
+obj=Twitcasting_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 #whowatch
 obj=Whowatch_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 #youtubelive
 obj=Youtubelive_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 ```
 
@@ -63,7 +72,7 @@ Basically use instance method
 
 ↓　Example of use
 ```ruby
-obj=Mildom_analyze.new(url)
+obj=Youtubelive_analyze.new(url)
 video_info=obj.videoinfo
 chat_list=obj.chat_scrape()
 ```

data/lib/file_operats/file_operat_chatdata.rb

@@ -1,15 +1,17 @@
 
 def file_write(data_list,log_flag,log_path)
 
-    if log_flag==true then
-        File.open(log_path,"w") do |file|
-            data_list.each do |chat|
-                file.puts chat
-            end
-        end
-        puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
-    else
+    if log_flag==false 
         puts "Scraping finished!! , log_flag false , chat log path is (no_path) "
+        return
     end
-
+    
+    File.open(log_path,"w") do |file|
+        data_list.each do |chat|
+            file.puts chat
+        end
+    end
+    puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
+    
+    return
 end

data/lib/requests/request.rb

@@ -5,16 +5,15 @@ require 'open-uri'
 
 
 def request_json_parse(url)
-    body_dic={}
 
     client=HTTPClient.new()
     response=client.get(url)
 
     if response.status==200
-     body_dic=JSON.parse(response.body)
+        return JSON.parse(response.body),response.status
     end
 
-    return body_dic,response.status
+    return nil,response.status
  end
 
 
@@ -22,13 +21,16 @@ def request_json_parse(url)
 
     charset=nil
     status=[]
-    response=open(url,opt) do |f|
+    response=URI.open(url,opt) do |f|
         charset=f.charset
         status=f.status
         f.read
     end
 
-    response_body=Nokogiri::HTML.parse(response,nil,charset)
-    return response_body,status
+    if status[0]=="200"
+        return Nokogiri::HTML.parse(response,nil,charset),status
+    end
+
+    return nil,status
  end
  

data/lib/site_list/mildom_analyze.rb

@@ -15,57 +15,54 @@ class Mildom_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail?v_id="
         @CHAT_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/chat/replay?video_id="
         @CHAT_REQEST_PARAMETER="&time_offset_ms="
-        @CHAT_STARTTIME="0"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     end
 
 
-    def videoid_get!(url)
-        videoid=url.split("=")[1].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[5].split("&")[0]
     end
 
 
     def chat_nextpage_get(time_key)
-        time_key=time_key.to_s
-        chat_request_url=@CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key
-        return chat_request_url
+        return @CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key.to_s
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        next_url=chat_nextpage_get(@CHAT_STARTTIME)
-        chat_body,chat_status=request_json_parse(next_url)
-        
-        time_length=@videoinfo["body"]["playback"]["video_length"]
-        next_time=0
         chat_list=[]
-        
+        next_time=0
+        time_length=@videoinfo["body"]["playback"]["video_length"]
 
         while next_time<=time_length do
-                
+
+            chat_body=chat_body_get(next_time)
             chat_body["body"]["models"][0]["detail"][0..-1].each do |chat|
                 chat_list.push chat
             end
 
             next_time=chat_body["body"]["models"][0]["summary"]["end_offset_ms"]
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
             progressbar(next_time,time_length)
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
         return chat_list
     end
 
-    public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
 
-end
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
 
+    public :chat_scrape
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
+end

data/lib/site_list/openrec_analyze.rb

@@ -17,16 +17,15 @@ class Openrec_analyze<Video_analyze
         @CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     
     end
 
 
-    def videoid_get!(url)
-        videoid=url.split("/")[4].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[4].split("&")[0]
     end
 
 
@@ -37,40 +36,43 @@ class Openrec_analyze<Video_analyze
     def chat_nextpage_get(time_key)
         #datatime型→time型→iso8601型
         time_key=DateTime.parse(time_key).to_time.utc.iso8601
-        chat_request_url=@VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
-
-        return chat_request_url
+        return @VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        start_time=@videoinfo["started_at"]
-        end_time=@videoinfo["ended_at"]
-        next_url=chat_nextpage_get(start_time)
-        chat_body,chat_status=request_json_parse(next_url)
-
         chat_list=[]
-        next_time=""
         head=0
+        next_time=@videoinfo["started_at"]
+        chat_body=chat_body_get(next_time)
         
         while !(chat_body[head..-1].empty?) do
+
             chat_body[head..-1].each do |chat|
                 chat_list.push chat
                 next_time=chat["posted_at"]
             end
-            
+
+            chat_body=chat_body_get(next_time)
             head=1
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
-            progressbar(next_time,end_time)
+
+            progressbar(next_time,@videoinfo["ended_at"])
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
         return chat_list
     end
 
+
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
+
     public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
 end

data/lib/site_list/twitcasting_analyze.rb

@@ -0,0 +1,112 @@
+require 'site_list/video_analyze'
+require 'requests/request'
+require 'progressbars/progressbar'
+require 'file_operats/file_operat_chatdata'
+
+class Twitcasting_analyze<Video_analyze
+
+    attr_reader :video_id, :user_id, :videoinfo, :videoinfo_request_status
+
+    def initialize(url)
+
+        @video_url=url
+        @video_id=videoid_get()
+        @user_id=userid_get()
+
+        @videoinfo_request_body,@videoinfo_request_status=request_html_parse(@video_url,{})
+        @videoinfo=videoinfo_get()
+
+        @chat_request_url="https://twitcasting.tv/"+@user_id+"/moviecomment/"+@video_id
+        @chat_request_body,@chat_request_status=request_html_parse(@chat_request_url,{})
+
+        @chatlog_filepath="./"+@video_id+".txt"
+    end
+
+
+    def videoid_get()
+        return @video_url.split("/")[5].split("&")[0]
+    end
+
+
+    def userid_get()
+        return @video_url.split("/")[3]
+    end
+
+
+    def videoinfo_get()
+
+        videoinfo={}
+        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.strip
+        videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
+        videoinfo["video_time"]=@videoinfo_request_body.at_css(".tw-player-duration-time").text.strip
+        videoinfo["video_start_time"]=@videoinfo_request_body.at_css(".tw-player-meta__status_item > time")[:datetime]
+
+        videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
+        i=0
+        videoinfo_polymer.each do |fact|
+            if i==1 then
+                videoinfo["total_view"]=fact.text.strip.split(":",2)[1]
+                return videoinfo
+            end
+            i+=1
+        end
+
+        return videoinfo
+    end
+
+
+    def chat_page_range()
+        size=@chat_request_body.css(".tw-pager").css("a").size()
+        range=@chat_request_body.css(".tw-pager").css("a")[size-1].text
+        return range.to_i
+    end
+
+
+    def chat_date_get(chatinfo_body)
+
+        chat_list=[]
+        chat_fact_dic={}
+
+        chatinfo_body.css(".tw-comment-history-item").each do |chat|
+            chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
+            chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
+            chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
+            chat_list.push(chat_fact_dic)
+            chat_fact_dic={}
+        end
+        
+        return chat_list
+    end
+
+
+    def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
+
+        chat_list=[]
+        chatinfo_body=@chat_request_body
+        page_range=chat_page_range()
+        page_count=0
+
+        while page_count<=page_range do
+            begin
+                chat_list+=chat_date_get(chatinfo_body)
+                page_count+=1
+                next_url=@chat_request_url+"-"+"#{page_count}"
+                chatinfo_body,_=request_html_parse(next_url,{})
+                progressbar(page_count,page_range)
+                sleep(1)
+
+            rescue
+                break
+            end
+
+        end
+
+        chat_list.reverse!
+        file_write(chat_list,log_flag,log_path)
+        return chat_list
+    end
+
+    public :chat_scrape
+    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range, :chat_date_get
+
+end