video_chat_get 0.1.2 → 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +55 -0
- data/Gemfile.lock +1 -1
- data/README.md +15 -6
- data/lib/file_operats/file_operat_chatdata.rb +11 -9
- data/lib/requests/request.rb +8 -6
- data/lib/site_list/mildom_analyze.rb +17 -20
- data/lib/site_list/openrec_analyze.rb +20 -18
- data/lib/site_list/twitcasting_analyze.rb +112 -0
- data/lib/site_list/whowatch_analyze.rb +8 -10
- data/lib/site_list/youtubelive_analyze.rb +88 -82
- data/lib/video_chat_get.rb +7 -5
- data/lib/video_chat_get/version.rb +1 -1
- data/vendor/bundle/ruby/2.5.0/bin/htmldiff +1 -1
- data/vendor/bundle/ruby/2.5.0/bin/ldiff +1 -1
- data/vendor/bundle/ruby/2.5.0/extensions/{x86_64-darwin-16 → x86_64-darwin-18}/2.5.0/json-2.3.0/gem.build_complete +0 -0
- data/vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-18/2.5.0/json-2.3.0/gem_make.out +13 -0
- data/vendor/bundle/ruby/2.5.0/extensions/{x86_64-darwin-16 → x86_64-darwin-18}/2.5.0/json-2.3.0/json/ext/generator.bundle +0 -0
- data/vendor/bundle/ruby/2.5.0/extensions/{x86_64-darwin-16 → x86_64-darwin-18}/2.5.0/json-2.3.0/json/ext/parser.bundle +0 -0
- data/vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-18/2.5.0/json-2.3.0/mkmf.log +60 -0
- data/vendor/bundle/ruby/2.5.0/extensions/{x86_64-darwin-16 → x86_64-darwin-18}/2.5.0/nokogiri-1.10.9/gem.build_complete +0 -0
- data/vendor/bundle/ruby/2.5.0/extensions/{x86_64-darwin-16 → x86_64-darwin-18}/2.5.0/nokogiri-1.10.9/gem_make.out +21 -21
- data/vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-18/2.5.0/nokogiri-1.10.9/mkmf.log +542 -0
- data/vendor/bundle/ruby/2.5.0/extensions/{x86_64-darwin-16 → x86_64-darwin-18}/2.5.0/nokogiri-1.10.9/nokogiri/nokogiri.bundle +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/Makefile +16 -15
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/html_document.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/html_element_description.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/html_entity_lookup.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/html_sax_parser_context.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/html_sax_push_parser.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/nokogiri.bundle +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/nokogiri.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_attr.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_attribute_decl.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_cdata.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_comment.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_document.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_document_fragment.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_dtd.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_element_content.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_element_decl.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_encoding_handler.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_entity_decl.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_entity_reference.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_io.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_namespace.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_node.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_node_set.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_processing_instruction.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_reader.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_relax_ng.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_sax_parser.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_sax_parser_context.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_sax_push_parser.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_schema.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_syntax_error.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_text.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xml_xpath_context.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/ext/nokogiri/xslt_stylesheet.o +0 -0
- data/vendor/bundle/ruby/2.5.0/gems/nokogiri-1.10.9/lib/nokogiri/nokogiri.bundle +0 -0
- data/vendor/bundle/ruby/2.5.0/specifications/diff-lcs-1.3.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/httpclient-2.8.3.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/json-2.3.0.gemspec +0 -0
- data/vendor/bundle/ruby/2.5.0/specifications/mini_portile2-2.4.0.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/nokogiri-1.10.9.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/rake-10.5.0.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/rspec-3.9.0.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/rspec-core-3.9.2.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/rspec-expectations-3.9.2.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/rspec-mocks-3.9.1.gemspec +2 -2
- data/vendor/bundle/ruby/2.5.0/specifications/rspec-support-3.9.3.gemspec +2 -2
- metadata +14 -12
- data/vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0/json-2.3.0/gem_make.out +0 -13
- data/vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0/json-2.3.0/mkmf.log +0 -60
- data/vendor/bundle/ruby/2.5.0/extensions/x86_64-darwin-16/2.5.0/nokogiri-1.10.9/mkmf.log +0 -542
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72c2d47a8f701af1d42780d9261be02ca29d743800971982bf42b9438ee4bdba
|
4
|
+
data.tar.gz: 90d62278f6cfee916570aad6ebe63acfb4122d9d19eb2f7dba4ef4b153561090
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 45ed4d022d5b3b40da4ee41103fa589ad4eed996c1ece2afffbfcedfaed0621acac7ffe1555d9b6237d048fe2e5bd5fea06ed128913acedbe665cc1025d6a472
|
7
|
+
data.tar.gz: bafe20e7dbc9b3e67ac1f7d41aebebbde6bb43930e5cd5a300ce8b3babbfc79b408db9428834e0d5de9eaa2e62420f10440d83d518fe662d3e3f4a514957ffe9
|
@@ -0,0 +1,55 @@
|
|
1
|
+
version: 2.1
|
2
|
+
|
3
|
+
orbs:
|
4
|
+
slack: circleci/slack@3.3.0
|
5
|
+
|
6
|
+
executors:
|
7
|
+
my-executor:
|
8
|
+
docker:
|
9
|
+
- image: circleci/ruby:2.5.8-node-browsers
|
10
|
+
|
11
|
+
jobs:
|
12
|
+
test:
|
13
|
+
executor: my-executor
|
14
|
+
steps:
|
15
|
+
- checkout
|
16
|
+
|
17
|
+
- restore_cache:
|
18
|
+
name: cache lord
|
19
|
+
key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
|
20
|
+
|
21
|
+
- run:
|
22
|
+
name: bundle install
|
23
|
+
command: bundle check --path=vendor/bundle || bundle install --path vendor/bundle --clean --jobs 4 --retry 3
|
24
|
+
|
25
|
+
- save_cache:
|
26
|
+
name: save cache
|
27
|
+
key: gem-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Gemfile.lock" }}
|
28
|
+
paths:
|
29
|
+
- ./vendor/bundle
|
30
|
+
|
31
|
+
- run:
|
32
|
+
name: bundle exec rake spec
|
33
|
+
command: bundle exec rake spec
|
34
|
+
|
35
|
+
- slack/status:
|
36
|
+
success_message: ':circleci-pass: $CIRCLE_BRANCH のテストが完了しました\n:github_octocat: User:$CIRCLE_USERNAME'
|
37
|
+
failure_message: ':circleci-fail: $CIRCLE_BRANCH のテストが失敗しました\n:github_octocat: User:$CIRCLE_USERNAME'
|
38
|
+
webhook: '${SLACK_WEBHOOK}'
|
39
|
+
|
40
|
+
workflows:
|
41
|
+
version: 2
|
42
|
+
test:
|
43
|
+
jobs:
|
44
|
+
- test
|
45
|
+
|
46
|
+
test_cron:
|
47
|
+
triggers:
|
48
|
+
- schedule:
|
49
|
+
cron: "0 21 * * 0"
|
50
|
+
filters:
|
51
|
+
branches:
|
52
|
+
only:
|
53
|
+
- master
|
54
|
+
jobs:
|
55
|
+
- test
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -8,7 +8,7 @@ This is a ruby package that gets a list of chats from the broadcasts of vi
|
|
8
8
|
|mildom |◯ |
|
9
9
|
|niconico | |
|
10
10
|
|openrec |◯ |
|
11
|
-
|
|
11
|
+
|twitcasting |◯ |
|
12
12
|
|twitch | |
|
13
13
|
|whowatch |◯ |
|
14
14
|
|youtubelive |◯ |
|
@@ -42,19 +42,28 @@ require 'video_chat_get'
|
|
42
42
|
|
43
43
|
#mildom
|
44
44
|
obj=Mildom_analyze.new(url)
|
45
|
-
|
45
|
+
video_info=obj.videoinfo
|
46
|
+
chat_list=obj.chat_scrape()
|
46
47
|
|
47
48
|
#openrec
|
48
49
|
obj=Openrec_analyze.new(url)
|
49
|
-
|
50
|
+
video_info=obj.videoinfo
|
51
|
+
chat_list=obj.chat_scrape()
|
52
|
+
|
53
|
+
#twitcasting
|
54
|
+
obj=Twitcasting_analyze.new(url)
|
55
|
+
video_info=obj.videoinfo
|
56
|
+
chat_list=obj.chat_scrape()
|
50
57
|
|
51
58
|
#whowatch
|
52
59
|
obj=Whowatch_analyze.new(url)
|
53
|
-
|
60
|
+
video_info=obj.videoinfo
|
61
|
+
chat_list=obj.chat_scrape()
|
54
62
|
|
55
63
|
#youtubelive
|
56
64
|
obj=Youtubelive_analyze.new(url)
|
57
|
-
|
65
|
+
video_info=obj.videoinfo
|
66
|
+
chat_list=obj.chat_scrape()
|
58
67
|
|
59
68
|
```
|
60
69
|
|
@@ -63,7 +72,7 @@ Basically use instance method
|
|
63
72
|
|
64
73
|
↓ Example of use
|
65
74
|
```ruby
|
66
|
-
obj=
|
75
|
+
obj=Youtubelive_analyze.new(url)
|
67
76
|
video_info=obj.videoinfo
|
68
77
|
chat_list=obj.chat_scrape()
|
69
78
|
```
|
@@ -1,15 +1,17 @@
|
|
1
1
|
|
2
2
|
def file_write(data_list,log_flag,log_path)
|
3
3
|
|
4
|
-
if log_flag==
|
5
|
-
File.open(log_path,"w") do |file|
|
6
|
-
data_list.each do |chat|
|
7
|
-
file.puts chat
|
8
|
-
end
|
9
|
-
end
|
10
|
-
puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
|
11
|
-
else
|
4
|
+
if log_flag==false
|
12
5
|
puts "Scraping finished!! , log_flag false , chat log path is (no_path) "
|
6
|
+
return
|
13
7
|
end
|
14
|
-
|
8
|
+
|
9
|
+
File.open(log_path,"w") do |file|
|
10
|
+
data_list.each do |chat|
|
11
|
+
file.puts chat
|
12
|
+
end
|
13
|
+
end
|
14
|
+
puts "Scraping finished!! , log_flag true , chat log path is (#{log_path}) "
|
15
|
+
|
16
|
+
return
|
15
17
|
end
|
data/lib/requests/request.rb
CHANGED
@@ -5,16 +5,15 @@ require 'open-uri'
|
|
5
5
|
|
6
6
|
|
7
7
|
def request_json_parse(url)
|
8
|
-
body_dic={}
|
9
8
|
|
10
9
|
client=HTTPClient.new()
|
11
10
|
response=client.get(url)
|
12
11
|
|
13
12
|
if response.status==200
|
14
|
-
|
13
|
+
return JSON.parse(response.body),response.status
|
15
14
|
end
|
16
15
|
|
17
|
-
return
|
16
|
+
return nil,response.status
|
18
17
|
end
|
19
18
|
|
20
19
|
|
@@ -22,13 +21,16 @@ def request_json_parse(url)
|
|
22
21
|
|
23
22
|
charset=nil
|
24
23
|
status=[]
|
25
|
-
response=open(url,opt) do |f|
|
24
|
+
response=URI.open(url,opt) do |f|
|
26
25
|
charset=f.charset
|
27
26
|
status=f.status
|
28
27
|
f.read
|
29
28
|
end
|
30
29
|
|
31
|
-
|
32
|
-
|
30
|
+
if status[0]=="200"
|
31
|
+
return Nokogiri::HTML.parse(response,nil,charset),status
|
32
|
+
end
|
33
|
+
|
34
|
+
return nil,status
|
33
35
|
end
|
34
36
|
|
@@ -15,57 +15,54 @@ class Mildom_analyze<Video_analyze
|
|
15
15
|
@VIDEOINFO_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/playback/getPlaybackDetail?v_id="
|
16
16
|
@CHAT_REQEST_URL="https://cloudac.mildom.com/nonolive/videocontent/chat/replay?video_id="
|
17
17
|
@CHAT_REQEST_PARAMETER="&time_offset_ms="
|
18
|
-
@CHAT_STARTTIME="0"
|
19
18
|
|
20
19
|
@video_url=url
|
21
|
-
@video_id=videoid_get
|
20
|
+
@video_id=videoid_get()
|
22
21
|
@videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
|
23
22
|
@chatlog_filepath="./"+@video_id+".txt"
|
24
23
|
end
|
25
24
|
|
26
25
|
|
27
|
-
def videoid_get
|
28
|
-
|
29
|
-
return videoid
|
26
|
+
def videoid_get()
|
27
|
+
return @video_url.split("/")[5].split("&")[0]
|
30
28
|
end
|
31
29
|
|
32
30
|
|
33
31
|
def chat_nextpage_get(time_key)
|
34
|
-
time_key
|
35
|
-
chat_request_url=@CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key
|
36
|
-
return chat_request_url
|
32
|
+
return @CHAT_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER+time_key.to_s
|
37
33
|
end
|
38
34
|
|
39
35
|
|
40
36
|
def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
|
41
37
|
|
42
|
-
next_url=chat_nextpage_get(@CHAT_STARTTIME)
|
43
|
-
chat_body,chat_status=request_json_parse(next_url)
|
44
|
-
|
45
|
-
time_length=@videoinfo["body"]["playback"]["video_length"]
|
46
|
-
next_time=0
|
47
38
|
chat_list=[]
|
48
|
-
|
39
|
+
next_time=0
|
40
|
+
time_length=@videoinfo["body"]["playback"]["video_length"]
|
49
41
|
|
50
42
|
while next_time<=time_length do
|
51
|
-
|
43
|
+
|
44
|
+
chat_body=chat_body_get(next_time)
|
52
45
|
chat_body["body"]["models"][0]["detail"][0..-1].each do |chat|
|
53
46
|
chat_list.push chat
|
54
47
|
end
|
55
48
|
|
56
49
|
next_time=chat_body["body"]["models"][0]["summary"]["end_offset_ms"]
|
57
|
-
next_url=chat_nextpage_get(next_time)
|
58
|
-
chat_body,chat_status=request_json_parse(next_url)
|
59
50
|
progressbar(next_time,time_length)
|
51
|
+
sleep(1)
|
60
52
|
end
|
61
53
|
|
62
54
|
file_write(chat_list,log_flag,log_path)
|
63
55
|
return chat_list
|
64
56
|
end
|
65
57
|
|
66
|
-
public :chat_scrape
|
67
|
-
private :videoid_get!, :chat_nextpage_get
|
68
58
|
|
69
|
-
|
59
|
+
def chat_body_get(next_time)
|
60
|
+
next_url=chat_nextpage_get(next_time)
|
61
|
+
chat_body,_=request_json_parse(next_url)
|
62
|
+
return chat_body
|
63
|
+
end
|
70
64
|
|
65
|
+
public :chat_scrape
|
66
|
+
private :videoid_get, :chat_nextpage_get, :chat_body_get
|
71
67
|
|
68
|
+
end
|
@@ -17,16 +17,15 @@ class Openrec_analyze<Video_analyze
|
|
17
17
|
@CHAT_REQEST_PARAMETER2="&is_including_system_message=false"
|
18
18
|
|
19
19
|
@video_url=url
|
20
|
-
@video_id=videoid_get
|
20
|
+
@video_id=videoid_get()
|
21
21
|
@videoinfo,@videoinfo_request_status=request_json_parse(@VIDEOINFO_REQEST_URL+@video_id)
|
22
22
|
@chatlog_filepath="./"+@video_id+".txt"
|
23
23
|
|
24
24
|
end
|
25
25
|
|
26
26
|
|
27
|
-
def videoid_get
|
28
|
-
|
29
|
-
return videoid
|
27
|
+
def videoid_get()
|
28
|
+
return @video_url.split("/")[4].split("&")[0]
|
30
29
|
end
|
31
30
|
|
32
31
|
|
@@ -37,40 +36,43 @@ class Openrec_analyze<Video_analyze
|
|
37
36
|
def chat_nextpage_get(time_key)
|
38
37
|
#datatime型→time型→iso8601型
|
39
38
|
time_key=DateTime.parse(time_key).to_time.utc.iso8601
|
40
|
-
|
41
|
-
|
42
|
-
return chat_request_url
|
39
|
+
return @VIDEOINFO_REQEST_URL+@video_id+@CHAT_REQEST_PARAMETER1+time_key+@CHAT_REQEST_PARAMETER2
|
43
40
|
end
|
44
41
|
|
45
42
|
|
46
43
|
def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
|
47
44
|
|
48
|
-
start_time=@videoinfo["started_at"]
|
49
|
-
end_time=@videoinfo["ended_at"]
|
50
|
-
next_url=chat_nextpage_get(start_time)
|
51
|
-
chat_body,chat_status=request_json_parse(next_url)
|
52
|
-
|
53
45
|
chat_list=[]
|
54
|
-
next_time=""
|
55
46
|
head=0
|
47
|
+
next_time=@videoinfo["started_at"]
|
48
|
+
chat_body=chat_body_get(next_time)
|
56
49
|
|
57
50
|
while !(chat_body[head..-1].empty?) do
|
51
|
+
|
58
52
|
chat_body[head..-1].each do |chat|
|
59
53
|
chat_list.push chat
|
60
54
|
next_time=chat["posted_at"]
|
61
55
|
end
|
62
|
-
|
56
|
+
|
57
|
+
chat_body=chat_body_get(next_time)
|
63
58
|
head=1
|
64
|
-
|
65
|
-
|
66
|
-
|
59
|
+
|
60
|
+
progressbar(next_time,@videoinfo["ended_at"])
|
61
|
+
sleep(1)
|
67
62
|
end
|
68
63
|
|
69
64
|
file_write(chat_list,log_flag,log_path)
|
70
65
|
return chat_list
|
71
66
|
end
|
72
67
|
|
68
|
+
|
69
|
+
def chat_body_get(next_time)
|
70
|
+
next_url=chat_nextpage_get(next_time)
|
71
|
+
chat_body,_=request_json_parse(next_url)
|
72
|
+
return chat_body
|
73
|
+
end
|
74
|
+
|
73
75
|
public :chat_scrape
|
74
|
-
private :videoid_get
|
76
|
+
private :videoid_get, :chat_nextpage_get, :chat_body_get
|
75
77
|
|
76
78
|
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'site_list/video_analyze'
|
2
|
+
require 'requests/request'
|
3
|
+
require 'progressbars/progressbar'
|
4
|
+
require 'file_operats/file_operat_chatdata'
|
5
|
+
|
6
|
+
class Twitcasting_analyze<Video_analyze
|
7
|
+
|
8
|
+
attr_reader :video_id, :user_id, :videoinfo, :videoinfo_request_status
|
9
|
+
|
10
|
+
def initialize(url)
|
11
|
+
|
12
|
+
@video_url=url
|
13
|
+
@video_id=videoid_get()
|
14
|
+
@user_id=userid_get()
|
15
|
+
|
16
|
+
@videoinfo_request_body,@videoinfo_request_status=request_html_parse(@video_url,{})
|
17
|
+
@videoinfo=videoinfo_get()
|
18
|
+
|
19
|
+
@chat_request_url="https://twitcasting.tv/"+@user_id+"/moviecomment/"+@video_id
|
20
|
+
@chat_request_body,@chat_request_status=request_html_parse(@chat_request_url,{})
|
21
|
+
|
22
|
+
@chatlog_filepath="./"+@video_id+".txt"
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def videoid_get()
|
27
|
+
return @video_url.split("/")[5].split("&")[0]
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
def userid_get()
|
32
|
+
return @video_url.split("/")[3]
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
def videoinfo_get()
|
37
|
+
|
38
|
+
videoinfo={}
|
39
|
+
videoinfo["user_name"]=@videoinfo_request_body.at_css(".tw-user-nav-name").text.strip
|
40
|
+
videoinfo["video_title"]=@videoinfo_request_body.at_css("#movie_title_content").text
|
41
|
+
videoinfo["video_time"]=@videoinfo_request_body.at_css(".tw-player-duration-time").text.strip
|
42
|
+
videoinfo["video_start_time"]=@videoinfo_request_body.at_css(".tw-player-meta__status_item > time")[:datetime]
|
43
|
+
|
44
|
+
videoinfo_polymer=@videoinfo_request_body.css(".tw-player-meta__status").css(".tw-player-meta__status_item")
|
45
|
+
i=0
|
46
|
+
videoinfo_polymer.each do |fact|
|
47
|
+
if i==1 then
|
48
|
+
videoinfo["total_view"]=fact.text.strip.split(":",2)[1]
|
49
|
+
return videoinfo
|
50
|
+
end
|
51
|
+
i+=1
|
52
|
+
end
|
53
|
+
|
54
|
+
return videoinfo
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
def chat_page_range()
|
59
|
+
size=@chat_request_body.css(".tw-pager").css("a").size()
|
60
|
+
range=@chat_request_body.css(".tw-pager").css("a")[size-1].text
|
61
|
+
return range.to_i
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
def chat_date_get(chatinfo_body)
|
66
|
+
|
67
|
+
chat_list=[]
|
68
|
+
chat_fact_dic={}
|
69
|
+
|
70
|
+
chatinfo_body.css(".tw-comment-history-item").each do |chat|
|
71
|
+
chat_fact_dic["comment"]=chat.at_css(".tw-comment-history-item__content__text").text.strip
|
72
|
+
chat_fact_dic["user_name"]=chat.at_css(".tw-comment-history-item__details__user-link").text.strip
|
73
|
+
chat_fact_dic["time"]=chat.at_css(".tw-comment-history-item__info__date")[:datetime]
|
74
|
+
chat_list.push(chat_fact_dic)
|
75
|
+
chat_fact_dic={}
|
76
|
+
end
|
77
|
+
|
78
|
+
return chat_list
|
79
|
+
end
|
80
|
+
|
81
|
+
|
82
|
+
def chat_scrape(log_flag=true,log_path=@chatlog_filepath)
|
83
|
+
|
84
|
+
chat_list=[]
|
85
|
+
chatinfo_body=@chat_request_body
|
86
|
+
page_range=chat_page_range()
|
87
|
+
page_count=0
|
88
|
+
|
89
|
+
while page_count<=page_range do
|
90
|
+
begin
|
91
|
+
chat_list+=chat_date_get(chatinfo_body)
|
92
|
+
page_count+=1
|
93
|
+
next_url=@chat_request_url+"-"+"#{page_count}"
|
94
|
+
chatinfo_body,_=request_html_parse(next_url,{})
|
95
|
+
progressbar(page_count,page_range)
|
96
|
+
sleep(1)
|
97
|
+
|
98
|
+
rescue
|
99
|
+
break
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
|
104
|
+
chat_list.reverse!
|
105
|
+
file_write(chat_list,log_flag,log_path)
|
106
|
+
return chat_list
|
107
|
+
end
|
108
|
+
|
109
|
+
public :chat_scrape
|
110
|
+
private :videoid_get, :userid_get, :videoinfo_get, :chat_page_range, :chat_date_get
|
111
|
+
|
112
|
+
end
|