video_chat_get 0.1.1 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5face6a3d0ab762647671b20730566d92e5b9fc4b519b13aa7c59f6cab2e8d43
-  data.tar.gz: 01edaee8d261c536e2b29602f877a59cb10847684cfdb001d54072164f3d51fe
+  metadata.gz: e4c548904d898fbcd56d725e6b7a7dd45df9f8d1a6133f279e0e7edb8a679bac
+  data.tar.gz: 29a668bf9086627306e46d436e93a4d1391cea3467e34791092f6f2332802c03
 SHA512:
-  metadata.gz: 12c16efdabf9b58691c2b2d28574311cdd2823567dfe72e1dfec63709b86c4806277772201531b16761910afc68bb3d618d3ef1892b30f4a23bd58acb18e41a0
-  data.tar.gz: 5dfabfc662a3f62721c178632f8cfe6ea156d6c3cec0e946b09db285cb8a150177ecde8b0f824722cbd2dd1f9819af3b08b7b6e11ed0c019778fbce686b840d9
+  metadata.gz: 2a0123fff8d7773855e61ce3a7ff5aaf4dc94a0b361902ee7c3133ddc5f32897ea2a6092d920f85fffdd67ca129a77860b1a8e07f4232751ab447a4bd53fe6d4
+  data.tar.gz: 9b9e4a2071cd9f3674c66982f378d205d38f6dd814280455444cf521f48dd9fa3a401b5c4269ef2883fa617344bd5214404d8f1ef7e629d2e4dc1507c4fff8d3

data/.circleci/config.yml

@@ -0,0 +1,39 @@
+version: 2.1
+executors:
+   my-executor:
+    docker:
+      - image: circleci/ruby:2.5.8-node-browsers
+
+jobs:
+  test:
+    executor: my-executor
+    steps: 
+      - checkout 
+
+      - restore_cache:
+          name: cache lord
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+
+      - run: 
+         name: bundle install  
+         command: bundle check --path=vendor/bundle || bundle install --path vendor/bundle --clean --jobs 4 --retry 3
+
+      - save_cache:
+          name: save cache
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+          paths:
+            - ./vendor/bundle
+
+      - run: 
+           name: bundle exec rake spec
+           command: bundle exec rake spec
+
+workflows:
+  version: 2
+  test-:
+    jobs:
+      - test:
+         filters:
+            branches:
+              only:
+                - master

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    video_chat_get (0.1.1)
+    video_chat_get (0.1.6)
       httpclient (~> 2.8.3)
       json (~> 2.3.0)
       nokogiri (~> 1.10.9)

data/README.md

@@ -1,8 +1,18 @@
 # VideoChatGet
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/video_chat_get`. To experiment with that code, run `bin/console` for an interactive prompt.
+This is a ruby ​​package that gets a list of chats from the broadcasts of video distribution sites.
+
+## Support sites
+|site_name |status |
+|:---|:---:|
+|mildom |◯ |
+|niconico | |
+|openrec |◯ |
+|twitcasting |◯ |
+|twitch | |
+|whowatch |◯ |
+|youtubelive |◯ |
 
-TODO: Delete this and the text above, and describe your gem
 
 ## Installation
 
@@ -28,27 +38,67 @@ use CLI:
 
 ### In code
 ```ruby
-obj=Youtubelive_analyze.new(videourl)
-chat_list=obj.chat_scrape
+require 'video_chat_get'
+
+#mildom
+obj=Mildom_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#openrec
+obj=Openrec_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#twitcasting
+obj=Twitcasting_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#whowatch
+obj=Whowatch_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#youtubelive
+obj=Youtubelive_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+```
+
+## Document
+Basically use instance method
+
+↓　Example of use
+```ruby
+obj=Youtubelive_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 ```
 
-## Support sites
-|site_name |status |
-|:---|:---:|
-|mildom |◯ |
-|niconico | |
-|openrec |◯ |
-|twitcast | |
-|twitch | |
-|whowatch |◯ |
-|youtubelive |◯ |
+### instance variable
+@videoinfo
+
+・Returns: hash  
+   We can acquire broadcast frame information.  
 
+### instance method  
+#chat_scrape(log_flag=true,log_path="./videoid.txt")  
+  
+Parameters:  
+   ・log_flag:boolean (default=true)  
+   　Whether to write chat list to file, write when true.  
 
-## Development
+   ・log_path:string (default="./videoid.txt")  
+   　File path to write chat list.  
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Returns:  
+   ・chat_list:array  
+   　Chat data list. Chat information is stored as dictionary data,  
+   　and this dictionary data exists for the number of chats.  
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+    chat_list=[hash,hash,...]
 
 ## Contributing
 

data/lib/file_operats/file_operat_chatdata.rb

@@ -1,15 +1,17 @@
 
 def file_write(data_list,log_flag,log_path)
 
-    if log_flag==true then
-        File.open(log_path,"w") do |file|
-            data_list.each do |chat|
-                file.puts chat
-            end
-        end
-        puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
-    else
+    if log_flag==false 
         puts "Scraping finished!! , log_flag false , chat log path is (no_path) "
+        return
     end
-
+    
+    File.open(log_path,"w") do |file|
+        data_list.each do |chat|
+            file.puts chat
+        end
+    end
+    puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
+    
+    return
 end

data/lib/requests/request.rb

@@ -5,16 +5,15 @@ require 'open-uri'
 
 
 def request_json_parse(url)
-    body_dic={}
 
     client=HTTPClient.new()
     response=client.get(url)
 
     if response.status==200
-     body_dic=JSON.parse(response.body)
+        return JSON.parse(response.body),response.status
     end
 
-    return body_dic,response.status
+    return nil,response.status
  end
 
 
@@ -22,13 +21,16 @@ def request_json_parse(url)
 
     charset=nil
     status=[]
-    response=open(url,opt) do |f|
+    response=URI.open(url,opt) do |f|
         charset=f.charset
         status=f.status
         f.read
     end
 
-    response_body=Nokogiri::HTML.parse(response,nil,charset)
-    return response_body,status
+    if status[0]=="200"
+        return Nokogiri::HTML.parse(response,nil,charset),status
+    end
+
+    return nil,status
  end
  

data/lib/site_list/mildom_analyze.rb

@@ -15,57 +15,54 @@ class Mildom_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail?v_id="
         @CHAT_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/chat/replay?video_id="
         @CHAT_REQEST_PARAMETER="&time_offset_ms="
-        @CHAT_STARTTIME="0"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     end
 
 
-    def videoid_get!(url)
-        videoid=url.split("=")[1].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[5].split("&")[0]
     end
 
 
     def chat_nextpage_get(time_key)
-        time_key=time_key.to_s
-        chat_request_url=@CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key
-        return chat_request_url
+        return @CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key.to_s
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        next_url=chat_nextpage_get(@CHAT_STARTTIME)
-        chat_body,chat_status=request_json_parse(next_url)
-        
-        time_length=@videoinfo["body"]["playback"]["video_length"]
-        next_time=0
         chat_list=[]
-        
+        next_time=0
+        time_length=@videoinfo["body"]["playback"]["video_length"]
 
         while next_time<=time_length do
-                
+
+            chat_body=chat_body_get(next_time)
             chat_body["body"]["models"][0]["detail"][0..-1].each do |chat|
                 chat_list.push chat
             end
 
             next_time=chat_body["body"]["models"][0]["summary"]["end_offset_ms"]
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
             progressbar(next_time,time_length)
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
         return chat_list
     end
 
-    public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
 
-end
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
 
+    public :chat_scrape
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
+end

data/lib/site_list/openrec_analyze.rb

@@ -3,36 +3,7 @@ require 'requests/request'
 require 'progressbars/progressbar'
 require 'file_operats/file_operat_chatdata'
 
-
 """
-Openrec_analyze
-
-・Explanation
-get chat data from openrec
-
-・use
-obj=Openrec_analyze.new(url)
-obj.chat_scrape
-
-・instance variable
-video_id（String）:video identifier
-videoinfo（Hash）:Video information
-videoinfo_request_status(Integer):Videoinformation request statuscode
-
-・method
-private method:
-videoid_get!(url)
-chat_nextpage_get(time_key)
-
-public method:
-chat_scrape()
-chatinfo_request_status()
-
-・test
-Check if the request statuscode is 200
-
-videoinfo_request_status==200
-chatinfo_request_status==200
 """
 
 class Openrec_analyze<Video_analyze
@@ -46,16 +17,15 @@ class Openrec_analyze<Video_analyze
         @CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get()
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     
     end
 
 
-    def videoid_get!(url)
-        videoid=url.split("/")[4].split("&")[0]
-        return videoid
+    def videoid_get()
+        return @video_url.split("/")[4].split("&")[0]
     end
 
 
@@ -66,40 +36,43 @@ class Openrec_analyze<Video_analyze
     def chat_nextpage_get(time_key)
         #datatime型→time型→iso8601型
         time_key=DateTime.parse(time_key).to_time.utc.iso8601
-        chat_request_url=@VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
-
-        return chat_request_url
+        return @VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
     end
 
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
 
-        start_time=@videoinfo["started_at"]
-        end_time=@videoinfo["ended_at"]
-        next_url=chat_nextpage_get(start_time)
-        chat_body,chat_status=request_json_parse(next_url)
-
         chat_list=[]
-        next_time=""
         head=0
+        next_time=@videoinfo["started_at"]
+        chat_body=chat_body_get(next_time)
         
         while !(chat_body[head..-1].empty?) do
+
             chat_body[head..-1].each do |chat|
                 chat_list.push chat
                 next_time=chat["posted_at"]
             end
-            
+
+            chat_body=chat_body_get(next_time)
             head=1
-            next_url=chat_nextpage_get(next_time)
-            chat_body,chat_status=request_json_parse(next_url)
-            progressbar(next_time,end_time)
+
+            progressbar(next_time,@videoinfo["ended_at"])
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
         return chat_list
     end
 
+
+    def chat_body_get(next_time)
+        next_url=chat_nextpage_get(next_time)
+        chat_body,_=request_json_parse(next_url)
+        return chat_body
+    end
+
     public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get, :chat_body_get
 
 end

data/lib/site_list/twitcasting_analyze.rb

@@ -0,0 +1,112 @@
+require 'site_list/video_analyze'
+require 'requests/request'
+require 'progressbars/progressbar'
+require 'file_operats/file_operat_chatdata'
+
+class Twitcasting_analyze<Video_analyze
+
+    attr_reader :video_id, :user_id, :videoinfo, :videoinfo_request_status
+
+    def initialize(url)
+
+        @video_url=url
+        @video_id=videoid_get()
+        @user_id=userid_get()
+
+        @videoinfo_request_body,@videoinfo_request_status=request_html_parse(@video_url,{})
+        @videoinfo=videoinfo_get()
+
+        @chat_request_url="https://twitcasting.tv/"+@user_id+"/moviecomment/"+@video_id
+        @chat_request_body,@chat_request_status=request_html_parse(@chat_request_url,{})
+
+        @chatlog_filepath="./"+@video_id+".txt"
+    end
+
+
+    def videoid_get()
+        return @video_url.split("/")[5].split("&")[0]
+    end
+
+
+    def userid_get()
+        return @video_url.split("/")[3]
+    end
+
+
+    def videoinfo_get()
+
+        videoinfo={}
+        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.strip
+        videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
+        videoinfo["video_time"]=@videoinfo_request_body.at_css(".tw-player-duration-time").text.strip
+        videoinfo["video_start_time"]=@videoinfo_request_body.at_css(".tw-player-meta__status_item > time")[:datetime]
+
+        videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
+        i=0
+        videoinfo_polymer.each do |fact|
+            if i==1 then
+                videoinfo["total_view"]=fact.text.strip.split(":",2)[1]
+                return videoinfo
+            end
+            i+=1
+        end
+
+        return videoinfo
+    end
+
+
+    def chat_page_range()
+        size=@chat_request_body.css(".paging").css("a").size()
+        range=@chat_request_body.css(".paging").css("a")[size-1].text
+        return range.to_i
+    end
+
+
+    def chat_date_get(chatinfo_body)
+
+        chat_list=[]
+        chat_fact_dic={}
+
+        chatinfo_body.css(".tw-comment-history-item").each do |chat|
+            chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
+            chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
+            chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
+            chat_list.push(chat_fact_dic)
+            chat_fact_dic={}
+        end
+        
+        return chat_list
+    end
+
+
+    def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
+
+        chat_list=[]
+        chatinfo_body=@chat_request_body
+        page_range=chat_page_range()
+        page_count=0
+
+        while page_count<=page_range do
+            begin
+                chat_list+=chat_date_get(chatinfo_body)
+                page_count+=1
+                next_url=@chat_request_url+"-"+"#{page_count}"
+                chatinfo_body,_=request_html_parse(next_url,{})
+                progressbar(page_count,page_range)
+                sleep(1)
+
+            rescue
+                break
+            end
+
+        end
+
+        chat_list.reverse!
+        file_write(chat_list,log_flag,log_path)
+        return chat_list
+    end
+
+    public :chat_scrape
+    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range, :chat_date_get
+
+end