video_chat_get 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e9afebc85e7857a3c61009be068b3b4ef20c47e2b4366cb935d040f70f11d09c
-  data.tar.gz: bba2adab791ae346447df6cacaec680534863b947c8ee6a70cf3fa8be170466e
+  metadata.gz: c3d71c5521116c13bf9c41874427297cfc11eae66110549640e63f26127db18a
+  data.tar.gz: e004bba054b4f8d024b8de7bf1599d5842db6fe853805a2d0f83927c18749583
 SHA512:
-  metadata.gz: ca33354ed333b12e86d9f26296dd9b2b20ba67cb43b8b07fc1a2a3d70ab1eac42b01d101f954eda70843593d93b664475f3274ac71af85e7b87ff2bb99e815b9
-  data.tar.gz: 9860a64d72062602415fcedfb0431e883da5228d005d14cd0c714806452396bc3d18390b73504dfd93db5f6e142ead5172e6497b59b48fc64e3b19d969dd4310
+  metadata.gz: 22423a1cad39a3820f57e071564ebc7e6a747436089d02d5f578a7c2ccd2d0cdb8b1e10eaba646581f615d5ab83ebab90bf23d88831259f90e34675fc79e4a73
+  data.tar.gz: d9126eff4ae9a97398e754239d035425c33149a166b8e6161b48b29d6fceb5c5a87525df553379f904f72520e62791f52f36efb1e4a9eaa73893b377ebf28000

data/.circleci/config.yml

@@ -0,0 +1,39 @@
+version: 2.1
+executors:
+   my-executor:
+    docker:
+      - image: circleci/ruby:2.5.8-node-browsers
+
+jobs:
+  test:
+    executor: my-executor
+    steps: 
+      - checkout 
+
+      - restore_cache:
+          name: cache lord
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+
+      - run: 
+         name: bundle install  
+         command: bundle check --path=vendor/bundle || bundle install --path vendor/bundle --clean --jobs 4 --retry 3
+
+      - save_cache:
+          name: save cache
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+          paths:
+            - ./vendor/bundle
+
+      - run: 
+           name: bundle exec rake spec
+           command: bundle exec rake spec
+
+workflows:
+  version: 2
+  test-:
+    jobs:
+      - test:
+         filters:
+            branches:
+              only:
+                - master

data/Gemfile

@@ -4,6 +4,6 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in video_chat_get.gemspec
 gemspec
-gem 'httpclient'
-gem 'json'
-gem 'nokogiri'
+#gem 'httpclient'
+#gem 'json'
+#gem 'nokogiri'

data/Gemfile.lock

@@ -1,7 +1,10 @@
 PATH
   remote: .
   specs:
-    video_chat_get (0.1.0)
+    video_chat_get (0.1.5)
+      httpclient (~> 2.8.3)
+      json (~> 2.3.0)
+      nokogiri (~> 1.10.9)
 
 GEM
   remote: https://rubygems.org/
@@ -32,9 +35,6 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.16)
-  httpclient
-  json
-  nokogiri
   rake (~> 10.0)
   rspec (~> 3.0)
   video_chat_get!

data/README.md

@@ -1,8 +1,18 @@
 # VideoChatGet
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/video_chat_get`. To experiment with that code, run `bin/console` for an interactive prompt.
+This is a ruby ​​package that gets a list of chats from the broadcasts of video distribution sites.
+
+## Support sites
+|site_name |status |
+|:---|:---:|
+|mildom |◯ |
+|niconico | |
+|openrec |◯ |
+|twitcasting |◯ |
+|twitch | |
+|whowatch |◯ |
+|youtubelive |◯ |
 
-TODO: Delete this and the text above, and describe your gem
 
 ## Installation
 
@@ -28,27 +38,67 @@ use CLI:
 
 ### In code
 ```ruby
-obj=Youtubelive_analyze.new(videourl)
-chat_list=obj.chat_scrape
+require 'video_chat_get'
+
+#mildom
+obj=Mildom_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#openrec
+obj=Openrec_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#twitcasting
+obj=Twitcasting_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#whowatch
+obj=Whowatch_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+#youtubelive
+obj=Youtubelive_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
+
+```
+
+## Document
+Basically use instance method
+
+↓　Example of use
+```ruby
+obj=Youtubelive_analyze.new(url)
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 ```
 
-## Support sites
-|site_name |status |
-|:---|:---:|
-|mildom |◯ |
-|niconico | |
-|openrec |◯ |
-|twitcast | |
-|twitch | |
-|whowatch |◯ |
-|youtubelive |◯ |
+### instance variable
+@videoinfo
+
+・Returns: hash  
+   We can acquire broadcast frame information.  
 
+### instance method  
+#chat_scrape(log_flag=true,log_path="./videoid.txt")  
+  
+Parameters:  
+   ・log_flag:boolean (default=true)  
+   　Whether to write chat list to file, write when true.  
 
-## Development
+   ・log_path:string (default="./videoid.txt")  
+   　File path to write chat list.  
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Returns:  
+   ・chat_list:array  
+   　Chat data list. Chat information is stored as dictionary data,  
+   　and this dictionary data exists for the number of chats.  
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+    chat_list=[hash,hash,...]
 
 ## Contributing
 

data/lib/requests/request.rb

@@ -22,7 +22,7 @@ def request_json_parse(url)
 
     charset=nil
     status=[]
-    response=open(url,opt) do |f|
+    response=URI.open(url,opt) do |f|
         charset=f.charset
         status=f.status
         f.read

data/lib/site_list/mildom_analyze.rb

@@ -18,13 +18,13 @@ class Mildom_analyze<Video_analyze
         @CHAT_STARTTIME="0"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get(@video_url)
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     end
 
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("=")[1].split("&")[0]
         return videoid
     end
@@ -57,6 +57,7 @@ class Mildom_analyze<Video_analyze
             next_url=chat_nextpage_get(next_time)
             chat_body,chat_status=request_json_parse(next_url)
             progressbar(next_time,time_length)
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
@@ -64,8 +65,6 @@ class Mildom_analyze<Video_analyze
     end
 
     public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
-
-end
-
+    private :videoid_get, :chat_nextpage_get
 
+end

data/lib/site_list/openrec_analyze.rb

@@ -3,36 +3,7 @@ require 'requests/request'
 require 'progressbars/progressbar'
 require 'file_operats/file_operat_chatdata'
 
-
 """
-Openrec_analyze
-
-・Explanation
-get chat data from openrec
-
-・use
-obj=Openrec_analyze.new(url)
-obj.chat_scrape
-
-・instance variable
-video_id（String）:video identifier
-videoinfo（Hash）:Video information
-videoinfo_request_status(Integer):Videoinformation request statuscode
-
-・method
-private method:
-videoid_get!(url)
-chat_nextpage_get(time_key)
-
-public method:
-chat_scrape()
-chatinfo_request_status()
-
-・test
-Check if the request statuscode is 200
-
-videoinfo_request_status==200
-chatinfo_request_status==200
 """
 
 class Openrec_analyze<Video_analyze
@@ -46,14 +17,14 @@ class Openrec_analyze<Video_analyze
         @CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get(@video_url)
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     
     end
 
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("/")[4].split("&")[0]
         return videoid
     end
@@ -93,6 +64,7 @@ class Openrec_analyze<Video_analyze
             next_url=chat_nextpage_get(next_time)
             chat_body,chat_status=request_json_parse(next_url)
             progressbar(next_time,end_time)
+            sleep(1)
         end
 
         file_write(chat_list,log_flag,log_path)
@@ -100,6 +72,6 @@ class Openrec_analyze<Video_analyze
     end
 
     public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get
 
 end

data/lib/site_list/twitcasting_analyze.rb

@@ -0,0 +1,104 @@
+require 'site_list/video_analyze'
+require 'requests/request'
+require 'progressbars/progressbar'
+require 'file_operats/file_operat_chatdata'
+
+class Twitcasting_analyze<Video_analyze
+
+    attr_reader :video_id, :user_id, :videoinfo, :videoinfo_request_status
+
+    def initialize(url)
+
+        @video_url=url
+        @video_id=videoid_get()
+        @user_id=userid_get()
+
+        @videoinfo_request_body,@videoinfo_request_status=request_html_parse(@video_url,{})
+        @videoinfo=videoinfo_get()
+
+        @chat_request_url="https://twitcasting.tv/"+@user_id+"/moviecomment/"+@video_id
+        @chat_request_body,@chat_request_status=request_html_parse(@chat_request_url,{})
+
+        @chatlog_filepath="./"+@video_id+".txt"
+    end
+
+
+    def videoid_get()
+        videoid=@video_url.split("/")[5].split("&")[0]
+        return videoid
+    end
+
+
+    def userid_get()
+        userid=@video_url.split("/")[3]
+        return userid
+    end
+
+
+    def videoinfo_get()
+
+        videoinfo={}
+        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.gsub(" ","")
+        videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
+
+        videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
+        videoinfo_polymer.each do |fact|
+            fact=fact.text
+            fact=fact.gsub(/ {2,}|\n/, "")
+            fact=fact.split(":",2)
+            videoinfo[fact[0]]=fact[1]
+        end
+        
+        return videoinfo
+    end
+
+
+    def chat_page_range()
+        
+        size=@chat_request_body.css(".paging").css("a").size()
+        range=@chat_request_body.css(".paging").css("a")[size-1].text
+        range=range.to_i
+        
+        return range
+    end
+
+
+    def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
+
+        chat_fact_dic={}
+        chat_list=[]
+        next_url=""
+        chatinfo_body=@chat_request_body
+        page_count=0
+        page_range=chat_page_range()
+
+        while page_count<=page_range do
+            begin
+                chatinfo_body.css(".tw-comment-history-item").each do |chat|
+                    chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
+                    chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
+                    chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
+                    chat_list.push(chat_fact_dic)
+                    chat_fact_dic={}
+                end
+                page_count+=1
+                next_url=@chat_request_url+"-"+"#{page_count}"
+                chatinfo_body,status=request_html_parse(next_url,{})
+                progressbar(page_count,page_range)
+                sleep(1)
+
+            rescue
+                break
+            end
+
+        end
+
+        chat_list.reverse!
+        file_write(chat_list,log_flag,log_path)
+        return chat_list
+    end
+
+    public :chat_scrape
+    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range
+
+end

data/lib/site_list/whowatch_analyze.rb

@@ -16,20 +16,20 @@ class Whowatch_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://api.whowatch.tv/lives/"
         
         @video_url=url
-        @video_id=videoid_get!(@video_url)
-        @videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get!(@VIDEOINFO_REQEST_URL+@video_id)
+        @video_id=videoid_get(@video_url)
+        @videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
 
     end
 
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("/")[4]
         return videoid
     end
 
 
-    def videoinfo_get!(url)
+    def videoinfo_get(url)
         body,status=request_json_parse(url)
         chat_body=body.delete("comments")
         return body,chat_body,status
@@ -53,6 +53,6 @@ class Whowatch_analyze<Video_analyze
      end
 
     public :chat_scrape
-    private :videoid_get!, :videoinfo_get!
+    private :videoid_get, :videoinfo_get
 
 end

data/lib/site_list/youtubelive_analyze.rb

@@ -8,122 +8,115 @@ require 'nokogiri'
 
 class Youtubelive_analyze<Video_analyze
 
-    attr_reader :video_id
+    attr_reader :video_id, :videoinfo, :videoinfo_request_status
 
     def initialize(url)
         @CHAT_REQUEST_URL="https://www.youtube.com/live_chat_replay?continuation="
         @USER_AGENT='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get(@video_url)
+        @videoinfo_body,@videoinfo_request_status=videoinfo_get(@video_url)
+        @videoinfo=videoinfo_extraction()
         @chatlog_filepath="./"+@video_id+".txt"
     end
     
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("=")[1].split("&")[0]
         return videoid
     end
 
-    
-    def videoinfo_get()
-    end
 
+    def videoinfo_get(url=@video_url)
+        
+        opt={}
+        videoinfo_body=""
 
-    """
-    chat url get from iframe 
-    202077end
-    """
-    def chat_url_get_iframe_extraction(url=@video_url)
+        videoinfo_respons,videoinfo_status=request_html_parse(url,opt)
+        videoinfo_body=htmlpage_script_parse(videoinfo_respons,method(:videoinfo_script_cleanup))
+        return videoinfo_body,videoinfo_status
+    end
+    
 
-        next_url=""
-        body_status=""
-        iframe_count=0
-        opt={}
+    def videoinfo_extraction()
 
-        while next_url.empty?
-            sleep(1)
-            iframe_count+=1
-            first_respons, body_status=request_html_parse(url,opt)
-            first_respons.search('iframe').each do |info|
-                puts info
-                if info["src"].include?("live_chat_replay") then
-                    next_url=info["src"]
-                end
-            end
-            progressbar(iframe_count,"iframe_count_inf")
-        end
+        videoinfo={}
+        common_hash=@videoinfo_body["contents"]["twoColumnWatchNextResults"]["results"]["results"]["contents"]
 
-        puts "iframe search finished!! iframe acquisition count  #{iframe_count} "
-        return next_url,body_status
+        videoinfo["ch"]=common_hash[1]["videoSecondaryInfoRenderer"]["owner"]["videoOwnerRenderer"]["title"]["runs"][0]["text"]
+        videoinfo["title"]=common_hash[0]["videoPrimaryInfoRenderer"]["title"]["runs"][0]["text"]
+        videoinfo["starttime"]=common_hash[0]["videoPrimaryInfoRenderer"]["dateText"]["simpleText"]
+        videoinfo["videocount"]=common_hash[0]["videoPrimaryInfoRenderer"]["viewCount"]["videoViewCountRenderer"]["viewCount"]["simpleText"]
+        videoinfo["good"]=common_hash[0]["videoPrimaryInfoRenderer"]["videoActions"]["menuRenderer"]["topLevelButtons"][0]["toggleButtonRenderer"]["defaultText"]["simpleText"]
+        videoinfo["bad"]=common_hash[0]["videoPrimaryInfoRenderer"]["videoActions"]["menuRenderer"]["topLevelButtons"][1]["toggleButtonRenderer"]["defaultText"]["simpleText"]
+        return videoinfo
     end
+    
 
+    def videoinfo_script_cleanup(script_date)
+        script_body=""
+        script_body=script_date.split("=",2)[1].split("\n",2)[0].strip.chomp(";")
+        return script_body
+    end
 
-    """
-    chat url get from htmljs
-    202077start
-    """
-    def chat_url_get_js_extraction(url=@video_url)
-        
-        opt={}
-        next_url_polymer=""
-        next_url=""
 
-        chat_respons,chat_status=request_html_parse(url,opt)
-        chat_respons.search('script').each do |script|
+    def chatinfo_script_cleanup(script_date)
+        script_body=""
+        script_body=script_date.split("=",2)[1].chomp("</script>").strip.chomp(";")
+        return script_body
+    end
+
+
+    def htmlpage_script_parse(respons,callback)
+        script_body=""
+        respons.search('script').each do |script|
             script=script.to_s
             if script.include?("window[\"ytInitialData\"]") then
-                next_url_polymer=script.split("=",2)[1].split("\n",2)[0]
-                next_url_polymer=next_url_polymer.strip.chomp(";")
-                next_url_polymer=JSON.parse(next_url_polymer)
+                script_body=callback.call(script)
+                script_body=JSON.parse(script_body)
             end
         end
-
-        next_url=next_url_polymer["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
-        next_url=@CHAT_REQUEST_URL+next_url
-        return next_url,chat_status
+        return script_body
     end
-    
+
 
     def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
-    
-    chat_body={}
-    chat_list=[]
-    chat_count=0
-    opt={'User-Agent' => @USER_AGENT}
-    next_url,status=chat_url_get_js_extraction(@video_url)
-
-    while true do
-
-        begin
-            chat_respons,chat_status=request_html_parse(next_url,opt)
-
-            chat_respons.search('script').each do |script|
-                script=script.to_s
-
-                if script.include?("window[\"ytInitialData\"]") then
-                    chat_body=script.split("=",2)[1].chomp("</script>").strip.chomp(";")
-                    chat_body=JSON.parse(chat_body)
-                    continue_url = chat_body["continuationContents"]["liveChatContinuation"]["continuations"][0]["liveChatReplayContinuationData"]["continuation"]
-                    next_url=@CHAT_REQUEST_URL+continue_url
-
-                    chat_body["continuationContents"]["liveChatContinuation"]["actions"][1..-1].each do |chat|
-                        chat_count+=1
-                        chat_list.push chat
-                        end
-                    progressbar(chat_count,"chat_count_inf")
+
+        chat_body={}
+        chat_list=[]
+        chat_count=0
+        opt={'User-Agent' => @USER_AGENT}
+        continue_parameter=@videoinfo_body["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
+        next_url=@CHAT_REQUEST_URL+continue_parameter
+
+        while true do
+            begin
+                chat_respons,chat_status=request_html_parse(next_url,opt)
+                chat_body=htmlpage_script_parse(chat_respons,method(:chatinfo_script_cleanup))
+                continue_parameter = chat_body["continuationContents"]["liveChatContinuation"]["continuations"][0]["liveChatReplayContinuationData"]["continuation"]
+                next_url=@CHAT_REQUEST_URL+continue_parameter
+
+                chat_body["continuationContents"]["liveChatContinuation"]["actions"][1..-1].each do |chat|
+                    chat_count+=1
+                    chat_list.push chat
                 end
+
+                progressbar(chat_count,"chat_count_inf")
+                sleep(1)
+
+            rescue
+                break
             end
-        rescue
-            break
+
         end
-    end
 
-    file_write(chat_list,log_flag,log_path)
-    return chat_list
+        file_write(chat_list,log_flag,log_path)
+        return chat_list
     end
 
-    public :chat_scrape, :chat_url_get_iframe_extraction, :chat_url_get_js_extraction
-    private :videoid_get!, :videoinfo_get
 
-end
+    public :chat_scrape
+    private :videoid_get, :videoinfo_get, :videoinfo_extraction, :htmlpage_script_parse, :videoinfo_script_cleanup, :chatinfo_script_cleanup
+
+end