video_chat_get 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +39 -0
- data/Gemfile.lock +1 -1
- data/README.md +9 -5
- data/lib/site_list/mildom_analyze.rb +3 -3
- data/lib/site_list/openrec_analyze.rb +3 -3
- data/lib/site_list/twitcasting_analyze.rb +103 -0
- data/lib/site_list/whowatch_analyze.rb +5 -5
- data/lib/site_list/youtubelive_analyze.rb +12 -8
- data/lib/video_chat_get.rb +7 -5
- data/lib/video_chat_get/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 490967e4b44f6a5a5f765df1b2add73f53013f42c56f0cb3c3e30f4563d3aad6
|
4
|
+
data.tar.gz: 495f2cd0df05ca4ce6bcfd7a54e5000f6cd5f255c2cc51da039a0a6db89195e4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 295f625e0d125c4c8d54377af795bd739d60b74ee6681f2d60012ce16d499dcaa75ab069137ca8e45475557499494075b536f7eaab8672acdcd066a65e11c68a
|
7
|
+
data.tar.gz: d94c0bdee710e761b12cd31a100d500ae66daa6428c18f6531269d730b848effd79175373cc4b067fa4ca90e6915401ec95eb10dd88c75d03cfb185550a35174
|
@@ -0,0 +1,39 @@
|
|
1
|
+
version: 2.1
|
2
|
+
executors:
|
3
|
+
my-executor:
|
4
|
+
docker:
|
5
|
+
- image: circleci/ruby:2.5.8-node-browsers
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
test:
|
9
|
+
executor: my-executor
|
10
|
+
steps:
|
11
|
+
- checkout
|
12
|
+
|
13
|
+
- restore_cache:
|
14
|
+
name: cache lord
|
15
|
+
key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
|
16
|
+
|
17
|
+
- run:
|
18
|
+
name: bundle install
|
19
|
+
command: bundle check --path=vendor/bundle || bundle install --path vendor/bundle --clean --jobs 4 --retry 3
|
20
|
+
|
21
|
+
- save_cache:
|
22
|
+
name: save cache
|
23
|
+
key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
|
24
|
+
paths:
|
25
|
+
- ./vendor/bundle
|
26
|
+
|
27
|
+
- run:
|
28
|
+
name: bundle exec rake spec
|
29
|
+
command: bundle exec rake spec
|
30
|
+
|
31
|
+
workflows:
|
32
|
+
version: 2
|
33
|
+
test-:
|
34
|
+
jobs:
|
35
|
+
- test:
|
36
|
+
filters:
|
37
|
+
branches:
|
38
|
+
only:
|
39
|
+
- master
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -42,19 +42,23 @@ require 'video_chat_get'
|
|
42
42
|
|
43
43
|
#mildom
|
44
44
|
obj=Mildom_analyze.new(url)
|
45
|
-
|
45
|
+
video_info=obj.videoinfo
|
46
|
+
chat_list=obj.chat_scrape()
|
46
47
|
|
47
48
|
#openrec
|
48
49
|
obj=Openrec_analyze.new(url)
|
49
|
-
|
50
|
+
video_info=obj.videoinfo
|
51
|
+
chat_list=obj.chat_scrape()
|
50
52
|
|
51
53
|
#whowatch
|
52
54
|
obj=Whowatch_analyze.new(url)
|
53
|
-
|
55
|
+
video_info=obj.videoinfo
|
56
|
+
chat_list=obj.chat_scrape()
|
54
57
|
|
55
58
|
#youtubelive
|
56
59
|
obj=Youtubelive_analyze.new(url)
|
57
|
-
|
60
|
+
video_info=obj.videoinfo
|
61
|
+
chat_list=obj.chat_scrape()
|
58
62
|
|
59
63
|
```
|
60
64
|
|
@@ -63,7 +67,7 @@ Basically use instance method
|
|
63
67
|
|
64
68
|
↓ Example of use
|
65
69
|
```ruby
|
66
|
-
obj=
|
70
|
+
obj=Youtubelive_analyze.new(url)
|
67
71
|
video_info=obj.videoinfo
|
68
72
|
chat_list=obj.chat_scrape()
|
69
73
|
```
|
@@ -18,13 +18,13 @@ class Mildom_analyze<Video_analyze
|
|
18
18
|
@CHAT_STARTTIME="0"
|
19
19
|
|
20
20
|
@video_url=url
|
21
|
-
@video_id=videoid_get
|
21
|
+
@video_id=videoid_get(@video_url)
|
22
22
|
@videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
|
23
23
|
@chatlog_filepath="./"+@video_id+".txt"
|
24
24
|
end
|
25
25
|
|
26
26
|
|
27
|
-
def videoid_get
|
27
|
+
def videoid_get(url=@video_url)
|
28
28
|
videoid=url.split("=")[1].split("&")[0]
|
29
29
|
return videoid
|
30
30
|
end
|
@@ -64,6 +64,6 @@ class Mildom_analyze<Video_analyze
|
|
64
64
|
end
|
65
65
|
|
66
66
|
public :chat_scrape
|
67
|
-
private :videoid_get
|
67
|
+
private :videoid_get, :chat_nextpage_get
|
68
68
|
|
69
69
|
end
|
@@ -17,14 +17,14 @@ class Openrec_analyze<Video_analyze
|
|
17
17
|
@CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
|
18
18
|
|
19
19
|
@video_url=url
|
20
|
-
@video_id=videoid_get
|
20
|
+
@video_id=videoid_get(@video_url)
|
21
21
|
@videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
|
22
22
|
@chatlog_filepath="./"+@video_id+".txt"
|
23
23
|
|
24
24
|
end
|
25
25
|
|
26
26
|
|
27
|
-
def videoid_get
|
27
|
+
def videoid_get(url=@video_url)
|
28
28
|
videoid=url.split("/")[4].split("&")[0]
|
29
29
|
return videoid
|
30
30
|
end
|
@@ -71,6 +71,6 @@ class Openrec_analyze<Video_analyze
|
|
71
71
|
end
|
72
72
|
|
73
73
|
public :chat_scrape
|
74
|
-
private :videoid_get
|
74
|
+
private :videoid_get, :chat_nextpage_get
|
75
75
|
|
76
76
|
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
require 'site_list/video_analyze'
|
2
|
+
require 'requests/request'
|
3
|
+
require 'progressbars/progressbar'
|
4
|
+
require 'file_operats/file_operat_chatdata'
|
5
|
+
|
6
|
+
class Twitcasting_analyze<Video_analyze
|
7
|
+
|
8
|
+
attr_reader :video_id, :user_id, :videoinfo, :videoinfo_request_status
|
9
|
+
|
10
|
+
def initialize(url)
|
11
|
+
|
12
|
+
@video_url=url
|
13
|
+
@video_id=videoid_get()
|
14
|
+
@user_id=userid_get()
|
15
|
+
|
16
|
+
@videoinfo_request_body,@videoinfo_request_status=request_html_parse(@video_url,{})
|
17
|
+
@videoinfo=videoinfo_get()
|
18
|
+
|
19
|
+
@chat_request_url="https://twitcasting.tv/"+@user_id+"/moviecomment/"+@video_id
|
20
|
+
@chat_request_body,@chat_request_status=request_html_parse(@chat_request_url,{})
|
21
|
+
|
22
|
+
@chatlog_filepath="./"+@video_id+".txt"
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def videoid_get()
|
27
|
+
videoid=@video_url.split("/")[5].split("&")[0]
|
28
|
+
return videoid
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def userid_get()
|
33
|
+
userid=@video_url.split("/")[3]
|
34
|
+
return userid
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
def videoinfo_get()
|
39
|
+
|
40
|
+
videoinfo={}
|
41
|
+
videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.gsub(" ","")
|
42
|
+
videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
|
43
|
+
|
44
|
+
videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
|
45
|
+
videoinfo_polymer.each do |fact|
|
46
|
+
fact=fact.text
|
47
|
+
fact=fact.gsub(/ {2,}|\n/, "")
|
48
|
+
fact=fact.split(":",2)
|
49
|
+
videoinfo[fact[0]]=fact[1]
|
50
|
+
end
|
51
|
+
|
52
|
+
return videoinfo
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
def chat_page_range()
|
57
|
+
|
58
|
+
size=@chat_request_body.css(".paging").css("a").size()
|
59
|
+
range=@chat_request_body.css(".paging").css("a")[size-1].text
|
60
|
+
range=range.to_i
|
61
|
+
|
62
|
+
return range
|
63
|
+
end
|
64
|
+
|
65
|
+
|
66
|
+
def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
|
67
|
+
|
68
|
+
chat_fact_dic={}
|
69
|
+
chat_list=[]
|
70
|
+
next_url=""
|
71
|
+
chatinfo_body=@chat_request_body
|
72
|
+
page_count=0
|
73
|
+
page_range=chat_page_range()
|
74
|
+
|
75
|
+
while page_count<=page_range do
|
76
|
+
begin
|
77
|
+
chatinfo_body.css(".tw-comment-history-item").each do |chat|
|
78
|
+
chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
|
79
|
+
chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
|
80
|
+
chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
|
81
|
+
chat_list.push(chat_fact_dic)
|
82
|
+
chat_fact_dic={}
|
83
|
+
end
|
84
|
+
page_count+=1
|
85
|
+
next_url=@chat_request_url+"-"+"#{page_count}"
|
86
|
+
chatinfo_body,status=request_html_parse(next_url,{})
|
87
|
+
progressbar(page_count,page_range)
|
88
|
+
|
89
|
+
rescue
|
90
|
+
break
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
chat_list.reverse!
|
96
|
+
file_write(chat_list,log_flag,log_path)
|
97
|
+
return chat_list
|
98
|
+
end
|
99
|
+
|
100
|
+
public :chat_scrape
|
101
|
+
private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range
|
102
|
+
|
103
|
+
end
|
@@ -16,20 +16,20 @@ class Whowatch_analyze<Video_analyze
|
|
16
16
|
@VIDEOINFO_REQEST_URL="https://api.whowatch.tv/lives/"
|
17
17
|
|
18
18
|
@video_url=url
|
19
|
-
@video_id=videoid_get
|
20
|
-
@videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get
|
19
|
+
@video_id=videoid_get(@video_url)
|
20
|
+
@videoinfo,@chat_body,@videoandchat_info_request_status=videoinfo_get(@VIDEOINFO_REQEST_URL+@video_id)
|
21
21
|
@chatlog_filepath="./"+@video_id+".txt"
|
22
22
|
|
23
23
|
end
|
24
24
|
|
25
25
|
|
26
|
-
def videoid_get
|
26
|
+
def videoid_get(url=@video_url)
|
27
27
|
videoid=url.split("/")[4]
|
28
28
|
return videoid
|
29
29
|
end
|
30
30
|
|
31
31
|
|
32
|
-
def videoinfo_get
|
32
|
+
def videoinfo_get(url)
|
33
33
|
body,status=request_json_parse(url)
|
34
34
|
chat_body=body.delete("comments")
|
35
35
|
return body,chat_body,status
|
@@ -53,6 +53,6 @@ class Whowatch_analyze<Video_analyze
|
|
53
53
|
end
|
54
54
|
|
55
55
|
public :chat_scrape
|
56
|
-
private :videoid_get
|
56
|
+
private :videoid_get, :videoinfo_get
|
57
57
|
|
58
58
|
end
|
@@ -15,14 +15,14 @@ class Youtubelive_analyze<Video_analyze
|
|
15
15
|
@USER_AGENT='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
|
16
16
|
|
17
17
|
@video_url=url
|
18
|
-
@video_id=videoid_get
|
18
|
+
@video_id=videoid_get(@video_url)
|
19
19
|
@videoinfo_polymer,@videoinfo_request_status=videoinfo_get(@video_url)
|
20
20
|
@videoinfo=videoinfo_extraction()
|
21
21
|
@chatlog_filepath="./"+@video_id+".txt"
|
22
22
|
end
|
23
23
|
|
24
24
|
|
25
|
-
def videoid_get
|
25
|
+
def videoid_get(url=@video_url)
|
26
26
|
videoid=url.split("=")[1].split("&")[0]
|
27
27
|
return videoid
|
28
28
|
end
|
@@ -70,17 +70,17 @@ class Youtubelive_analyze<Video_analyze
|
|
70
70
|
chat_list=[]
|
71
71
|
chat_count=0
|
72
72
|
opt={'User-Agent' => @USER_AGENT}
|
73
|
-
|
74
|
-
next_url=@CHAT_REQUEST_URL+
|
73
|
+
continue_url=@videoinfo_polymer["contents"]["twoColumnWatchNextResults"]["conversationBar"]["liveChatRenderer"]["continuations"][0]["reloadContinuationData"]["continuation"]
|
74
|
+
next_url=@CHAT_REQUEST_URL+continue_url
|
75
75
|
|
76
|
-
while true do
|
77
76
|
|
77
|
+
while true do
|
78
78
|
begin
|
79
79
|
chat_respons,chat_status=request_html_parse(next_url,opt)
|
80
80
|
|
81
81
|
chat_respons.search('script').each do |script|
|
82
82
|
script=script.to_s
|
83
|
-
|
83
|
+
|
84
84
|
if script.include?("window[\"ytInitialData\"]") then
|
85
85
|
chat_body=script.split("=",2)[1].chomp("</script>").strip.chomp(";")
|
86
86
|
chat_body=JSON.parse(chat_body)
|
@@ -90,21 +90,25 @@ class Youtubelive_analyze<Video_analyze
|
|
90
90
|
chat_body["continuationContents"]["liveChatContinuation"]["actions"][1..-1].each do |chat|
|
91
91
|
chat_count+=1
|
92
92
|
chat_list.push chat
|
93
|
-
|
93
|
+
end
|
94
|
+
|
94
95
|
progressbar(chat_count,"chat_count_inf")
|
95
96
|
end
|
96
97
|
end
|
98
|
+
|
97
99
|
rescue
|
98
100
|
break
|
99
101
|
end
|
102
|
+
|
100
103
|
end
|
101
104
|
|
105
|
+
|
102
106
|
file_write(chat_list,log_flag,log_path)
|
103
107
|
return chat_list
|
104
108
|
end
|
105
109
|
|
106
110
|
|
107
111
|
public :chat_scrape
|
108
|
-
private :videoid_get
|
112
|
+
private :videoid_get, :videoinfo_get, :videoinfo_extraction
|
109
113
|
|
110
114
|
end
|
data/lib/video_chat_get.rb
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
require 'video_chat_get/version'
|
2
|
-
|
3
|
-
require 'site_list/openrec_analyze'
|
4
|
-
require 'site_list/mildom_analyze'
|
5
|
-
require 'site_list/whowatch_analyze'
|
6
|
-
require 'site_list/youtubelive_analyze'
|
2
|
+
Dir[File.expand_path('../site_list',__FILE__)<<'/*.rb'].each do |file| require file end
|
7
3
|
|
8
4
|
"""
|
9
5
|
・argument
|
@@ -11,6 +7,7 @@ require 'site_list/youtubelive_analyze'
|
|
11
7
|
@@MILDOM_IDENTFIER:
|
12
8
|
@@WHOWATCH_IDENTFIER:
|
13
9
|
@@YOUTUBELIVE_IDENTFIER:
|
10
|
+
@@TWITCASTING_IDENTFIER:
|
14
11
|
|
15
12
|
・method
|
16
13
|
url_route():
|
@@ -22,6 +19,7 @@ module VideoChatGet
|
|
22
19
|
@@MILDOM_IDENTFIER="https://www.mildom.com/playback/"
|
23
20
|
@@WHOWATCH_IDENTFIER="https://whowatch.tv/archives/"
|
24
21
|
@@YOUTUBELIVE_IDENTFIER="https://www.youtube.com/watch?"
|
22
|
+
@@TWITCASTING_IDENTFIER="https://twitcasting.tv/"
|
25
23
|
|
26
24
|
def self.test
|
27
25
|
"hello"
|
@@ -45,6 +43,10 @@ module VideoChatGet
|
|
45
43
|
puts "identifier:youtube videourl:#{url}"
|
46
44
|
obj=Youtubelive_analyze.new(url)
|
47
45
|
obj.chat_scrape
|
46
|
+
elsif url.include?(@@TWITCASTING_IDENTFIER) then
|
47
|
+
puts "identifier:twitcasting videourl:#{url}"
|
48
|
+
obj=Twitcasting_analyze.new(url)
|
49
|
+
obj.chat_scrape
|
48
50
|
else
|
49
51
|
puts "urlerr"
|
50
52
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: video_chat_get
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- tf0101
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-07-
|
11
|
+
date: 2020-07-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -102,6 +102,7 @@ executables:
|
|
102
102
|
extensions: []
|
103
103
|
extra_rdoc_files: []
|
104
104
|
files:
|
105
|
+
- ".circleci/config.yml"
|
105
106
|
- ".gitignore"
|
106
107
|
- ".rspec"
|
107
108
|
- ".travis.yml"
|
@@ -119,6 +120,7 @@ files:
|
|
119
120
|
- lib/requests/request.rb
|
120
121
|
- lib/site_list/mildom_analyze.rb
|
121
122
|
- lib/site_list/openrec_analyze.rb
|
123
|
+
- lib/site_list/twitcasting_analyze.rb
|
122
124
|
- lib/site_list/video_analyze.rb
|
123
125
|
- lib/site_list/whowatch_analyze.rb
|
124
126
|
- lib/site_list/youtubelive_analyze.rb
|