video_chat_get 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 518a37dc694766450b8b7492ade79c354e260c9b2970fb49ad3dec3d4c060563
-  data.tar.gz: aa53b6f56bfc39853e8233e52ea6239a66c56c6c1a76e2585f04c099f60838ab
+  metadata.gz: 490967e4b44f6a5a5f765df1b2add73f53013f42c56f0cb3c3e30f4563d3aad6
+  data.tar.gz: 495f2cd0df05ca4ce6bcfd7a54e5000f6cd5f255c2cc51da039a0a6db89195e4
 SHA512:
-  metadata.gz: 9a2eaa81eeac2250b299a62d66181f366ea5d08a581d95cfaa7ecd39157761bf45a40537559749f68a575b304dee645e5cf8b6ccc41afba1475a2a3f3711cfd9
-  data.tar.gz: 04e540629458d2bf3a64f388588384ea4df8db2a1a37eec28218cff65b73dd116f03e055df5e6c952b603ffa0fe0879a9b62e5b454537b7abc8ab035d316d5f8
+  metadata.gz: 295f625e0d125c4c8d54377af795bd739d60b74ee6681f2d60012ce16d499dcaa75ab069137ca8e45475557499494075b536f7eaab8672acdcd066a65e11c68a
+  data.tar.gz: d94c0bdee710e761b12cd31a100d500ae66daa6428c18f6531269d730b848effd79175373cc4b067fa4ca90e6915401ec95eb10dd88c75d03cfb185550a35174

data/.circleci/config.yml

@@ -0,0 +1,39 @@
+version: 2.1
+executors:
+   my-executor:
+    docker:
+      - image: circleci/ruby:2.5.8-node-browsers
+
+jobs:
+  test:
+    executor: my-executor
+    steps: 
+      - checkout 
+
+      - restore_cache:
+          name: cache lord
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+
+      - run: 
+         name: bundle install  
+         command: bundle check --path=vendor/bundle || bundle install --path vendor/bundle --clean --jobs 4 --retry 3
+
+      - save_cache:
+          name: save cache
+          key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
+          paths:
+            - ./vendor/bundle
+
+      - run: 
+           name: bundle exec rake spec
+           command: bundle exec rake spec
+
+workflows:
+  version: 2
+  test-:
+    jobs:
+      - test:
+         filters:
+            branches:
+              only:
+                - master

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    video_chat_get (0.1.3)
+    video_chat_get (0.1.4)
       httpclient (~> 2.8.3)
       json (~> 2.3.0)
       nokogiri (~> 1.10.9)

data/README.md

@@ -42,19 +42,23 @@ require 'video_chat_get'
 
 #mildom
 obj=Mildom_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 #openrec
 obj=Openrec_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 #whowatch
 obj=Whowatch_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 #youtubelive
 obj=Youtubelive_analyze.new(url)
-chat_list=obj.chat_scrape
+video_info=obj.videoinfo
+chat_list=obj.chat_scrape()
 
 ```
 
@@ -63,7 +67,7 @@ Basically use instance method
 
 ↓　Example of use
 ```ruby
-obj=Mildom_analyze.new(url)
+obj=Youtubelive_analyze.new(url)
 video_info=obj.videoinfo
 chat_list=obj.chat_scrape()
 ```

data/lib/site_list/mildom_analyze.rb

@@ -18,13 +18,13 @@ class Mildom_analyze<Video_analyze
         @CHAT_STARTTIME="0"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get(@video_url)
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     end
 
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("=")[1].split("&")[0]
         return videoid
     end
@@ -64,6 +64,6 @@ class Mildom_analyze<Video_analyze
     end
 
     public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get
 
 end

data/lib/site_list/openrec_analyze.rb

@@ -17,14 +17,14 @@ class Openrec_analyze<Video_analyze
         @CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get(@video_url)
         @videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
     
     end
 
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("/")[4].split("&")[0]
         return videoid
     end
@@ -71,6 +71,6 @@ class Openrec_analyze<Video_analyze
     end
 
     public :chat_scrape
-    private :videoid_get!, :chat_nextpage_get
+    private :videoid_get, :chat_nextpage_get
 
 end

data/lib/site_list/twitcasting_analyze.rb

@@ -0,0 +1,103 @@
+require 'site_list/video_analyze'
+require 'requests/request'
+require 'progressbars/progressbar'
+require 'file_operats/file_operat_chatdata'
+
+class Twitcasting_analyze<Video_analyze
+
+    attr_reader :video_id, :user_id, :videoinfo, :videoinfo_request_status
+
+    def initialize(url)
+
+        @video_url=url
+        @video_id=videoid_get()
+        @user_id=userid_get()
+
+        @videoinfo_request_body,@videoinfo_request_status=request_html_parse(@video_url,{})
+        @videoinfo=videoinfo_get()
+
+        @chat_request_url="https://twitcasting.tv/"+@user_id+"/moviecomment/"+@video_id
+        @chat_request_body,@chat_request_status=request_html_parse(@chat_request_url,{})
+
+        @chatlog_filepath="./"+@video_id+".txt"
+    end
+
+
+    def videoid_get()
+        videoid=@video_url.split("/")[5].split("&")[0]
+        return videoid
+    end
+
+
+    def userid_get()
+        userid=@video_url.split("/")[3]
+        return userid
+    end
+
+
+    def videoinfo_get()
+
+        videoinfo={}
+        videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.gsub(" ","")
+        videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
+
+        videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
+        videoinfo_polymer.each do |fact|
+            fact=fact.text
+            fact=fact.gsub(/ {2,}|\n/, "")
+            fact=fact.split(":",2)
+            videoinfo[fact[0]]=fact[1]
+        end
+        
+        return videoinfo
+    end
+
+
+    def chat_page_range()
+        
+        size=@chat_request_body.css(".paging").css("a").size()
+        range=@chat_request_body.css(".paging").css("a")[size-1].text
+        range=range.to_i
+        
+        return range
+    end
+
+
+    def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
+
+        chat_fact_dic={}
+        chat_list=[]
+        next_url=""
+        chatinfo_body=@chat_request_body
+        page_count=0
+        page_range=chat_page_range()
+
+        while page_count<=page_range do
+            begin
+                chatinfo_body.css(".tw-comment-history-item").each do |chat|
+                    chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
+                    chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
+                    chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
+                    chat_list.push(chat_fact_dic)
+                    chat_fact_dic={}
+                end
+                page_count+=1
+                next_url=@chat_request_url+"-"+"#{page_count}"
+                chatinfo_body,status=request_html_parse(next_url,{})
+                progressbar(page_count,page_range)
+
+            rescue
+                break
+            end
+
+        end
+
+        chat_list.reverse!
+        file_write(chat_list,log_flag,log_path)
+        return chat_list
+    end
+
+    public :chat_scrape
+    private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range
+
+end

data/lib/site_list/whowatch_analyze.rb

@@ -16,20 +16,20 @@ class Whowatch_analyze<Video_analyze
         @VIDEOINFO_REQEST_URL="https://api.whowatch.tv/lives/"
         
         @video_url=url
-        @video_id=videoid_get!(@video_url)
-        @videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get!(@VIDEOINFO_REQEST_URL+@video_id)
+        @video_id=videoid_get(@video_url)
+        @videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get(@VIDEOINFO_REQEST_URL+@video_id)
         @chatlog_filepath="./"+@video_id+".txt"
 
     end
 
 
-    def videoid_get!(url)
+    def videoid_get(url=@video_url)
         videoid=url.split("/")[4]
         return videoid
     end
 
 
-    def videoinfo_get!(url)
+    def videoinfo_get(url)
         body,status=request_json_parse(url)
         chat_body=body.delete("comments")
         return body,chat_body,status
@@ -53,6 +53,6 @@ class Whowatch_analyze<Video_analyze
      end
 
     public :chat_scrape
-    private :videoid_get!, :videoinfo_get!
+    private :videoid_get, :videoinfo_get
 
 end

data/lib/site_list/youtubelive_analyze.rb

@@ -15,14 +15,14 @@ class Youtubelive_analyze<Video_analyze
         @USER_AGENT='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
 
         @video_url=url
-        @video_id=videoid_get!(@video_url)
+        @video_id=videoid_get(@video_url)
         @videoinfo_polymer,@videoinfo_request_status=videoinfo_get(@video_url)
         @videoinfo=videoinfo_extraction()
         @chatlog_filepath="./"+@video_id+".txt"
     end
     
 
-    def videoid_get!(url=@video_url)
+    def videoid_get(url=@video_url)
         videoid=url.split("=")[1].split("&")[0]
         return videoid
     end
@@ -70,17 +70,17 @@ class Youtubelive_analyze<Video_analyze
     chat_list=[]
     chat_count=0
     opt={'User-Agent' => @USER_AGENT}
-    next_url=@videoinfo_polymer["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
-    next_url=@CHAT_REQUEST_URL+next_url
+    continue_url=@videoinfo_polymer["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
+    next_url=@CHAT_REQUEST_URL+continue_url
 
-    while true do
 
+    while true do
         begin
             chat_respons,chat_status=request_html_parse(next_url,opt)
 
             chat_respons.search('script').each do |script|
                 script=script.to_s
-
+                
                 if script.include?("window[\"ytInitialData\"]") then
                     chat_body=script.split("=",2)[1].chomp("</script>").strip.chomp(";")
                     chat_body=JSON.parse(chat_body)
@@ -90,21 +90,25 @@ class Youtubelive_analyze<Video_analyze
                     chat_body["continuationContents"]["liveChatContinuation"]["actions"][1..-1].each do |chat|
                         chat_count+=1
                         chat_list.push chat
-                        end
+                    end
+
                     progressbar(chat_count,"chat_count_inf")
                 end
             end
+
         rescue
             break
         end
+
     end
 
+
     file_write(chat_list,log_flag,log_path)
     return chat_list
     end
 
 
     public :chat_scrape
-    private :videoid_get!, :videoinfo_get, :videoinfo_extraction
+    private :videoid_get, :videoinfo_get, :videoinfo_extraction
 
 end

data/lib/video_chat_get.rb

@@ -1,9 +1,5 @@
 require 'video_chat_get/version'
-require 'site_list/video_analyze'
-require 'site_list/openrec_analyze'
-require 'site_list/mildom_analyze'
-require 'site_list/whowatch_analyze'
-require 'site_list/youtubelive_analyze'
+Dir[File.expand_path('../site_list',__FILE__)<<'/*.rb'].each do |file| require file end
 
    """
     ・argument
@@ -11,6 +7,7 @@ require 'site_list/youtubelive_analyze'
     @@MILDOM_IDENTFIER:
     @@WHOWATCH_IDENTFIER:
     @@YOUTUBELIVE_IDENTFIER:
+    @@TWITCASTING_IDENTFIER:
 
 　　 ・method
     url_route():
@@ -22,6 +19,7 @@ module VideoChatGet
   @@MILDOM_IDENTFIER="https://www.mildom.com/playback/"
   @@WHOWATCH_IDENTFIER="https://whowatch.tv/archives/"
   @@YOUTUBELIVE_IDENTFIER="https://www.youtube.com/watch?"
+  @@TWITCASTING_IDENTFIER="https://twitcasting.tv/"
 
   def self.test
     "hello"
@@ -45,6 +43,10 @@ module VideoChatGet
         puts "identifier:youtube videourl:#{url}"
         obj=Youtubelive_analyze.new(url)
         obj.chat_scrape
+    elsif url.include?(@@TWITCASTING_IDENTFIER) then
+        puts "identifier:twitcasting videourl:#{url}"
+        obj=Twitcasting_analyze.new(url)
+        obj.chat_scrape
     else
         puts "urlerr"
     end

data/lib/video_chat_get/version.rb

@@ -1,3 +1,3 @@
 module VideoChatGet
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: video_chat_get
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - tf0101
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-14 00:00:00.000000000 Z
+date: 2020-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,6 +102,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -119,6 +120,7 @@ files:
 - lib/requests/request.rb
 - lib/site_list/mildom_analyze.rb
 - lib/site_list/openrec_analyze.rb
+- lib/site_list/twitcasting_analyze.rb
 - lib/site_list/video_analyze.rb
 - lib/site_list/whowatch_analyze.rb
 - lib/site_list/youtubelive_analyze.rb