rget 4.9.1 → 4.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.devcontainer/Dockerfile +14 -0
- data/.devcontainer/devcontainer.json +33 -0
- data/.travis.yml +1 -1
- data/README.md +3 -2
- data/bin/hls-dl +11 -96
- data/lib/hibiki.rb +3 -1
- data/lib/hls.rb +10 -28
- data/lib/onsen.rb +9 -25
- data/lib/standfm.rb +62 -0
- data/lib/webradio.rb +10 -4
- data/lib/youtube.rb +121 -0
- data/rget.gemspec +3 -3
- data/rget.yaml +12 -0
- metadata +9 -7
- data/lib/freshlive.rb +0 -113
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 313f0120701854057599c157f85971584cd70eed1dd45f6d8cc5ea1444cc714c
|
4
|
+
data.tar.gz: 0f998f200f67b6a28186d3070b4b3f03e50b76d14e9c6def56a39afbdfda7a52
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ce23bc480247bcf370528ac3f08191846516d5746f0a075862f21eb7e863caf2b6e89ecb297766018b248e8ddcb76b0ce477cd98ec5e58939a15f38da27930d
|
7
|
+
data.tar.gz: 7cdfbc35c482db64f26899523d0338d69ca5faa82b70b84f187d4824766c97e2dc86f06d89565c9f4d54586a8449453a9aa3384cad17465a1446e251f8c19e28
|
@@ -0,0 +1,14 @@
|
|
1
|
+
FROM ruby:2
|
2
|
+
ENV USER vscode
|
3
|
+
LABEL maintainer "@tdtds <t@tdtds.jp>"
|
4
|
+
RUN apt-get -y update && apt-get -y install ffmpeg && \
|
5
|
+
curl -sLo /usr/local/bin/youtube-dl http://www.yt-dl.org/downloads/latest/youtube-dl && \
|
6
|
+
chmod +x /usr/local/bin/youtube-dl && \
|
7
|
+
useradd -u 1000 -m $USER && chsh -s /bin/bash $USER
|
8
|
+
USER $USER
|
9
|
+
RUN bundle config set path vendor/bundle && \
|
10
|
+
bundle config set with development:test && \
|
11
|
+
echo 'git config --global --unset core.editor' >> /home/$USER/.bashrc && \
|
12
|
+
echo 'git config --global --unset core.sshCommand' >> /home/$USER/.bashrc && \
|
13
|
+
echo 'git ls-remote -q > /dev/null' >> /home/$USER/.bashrc
|
14
|
+
CMD [ "sleep", "infinity" ]
|
@@ -0,0 +1,33 @@
|
|
1
|
+
// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at:
|
2
|
+
// https://github.com/microsoft/vscode-dev-containers/tree/v0.122.1/containers/docker-from-docker-compose
|
3
|
+
// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
|
4
|
+
{
|
5
|
+
"name": "rget",
|
6
|
+
"context": "..",
|
7
|
+
"dockerFile": "Dockerfile",
|
8
|
+
|
9
|
+
// Set *default* container specific settings.json values on container create.
|
10
|
+
"settings": {
|
11
|
+
"terminal.integrated.shell.linux": "/bin/bash"
|
12
|
+
},
|
13
|
+
|
14
|
+
// Add the IDs of extensions you want installed when the container is created.
|
15
|
+
"extensions": [
|
16
|
+
"rebornix.ruby"
|
17
|
+
,"github.vscode-pull-request-github"
|
18
|
+
,"eamodio.gitlens"
|
19
|
+
,"donjayamanne.githistory"
|
20
|
+
],
|
21
|
+
|
22
|
+
// Uncomment the next line if you want start specific services in your Docker Compose config.
|
23
|
+
// "runServices": [],
|
24
|
+
|
25
|
+
// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
|
26
|
+
// "shutdownAction": "none",
|
27
|
+
|
28
|
+
// Use 'postCreateCommand' to run commands after the container is created.
|
29
|
+
// "postCreateCommand": "docker --version",
|
30
|
+
|
31
|
+
// Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root.
|
32
|
+
// "remoteUser": "vscode"
|
33
|
+
}
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -5,12 +5,13 @@ Downloading newest radio programs on the web. Supported radio stations are:
|
|
5
5
|
* hibiki
|
6
6
|
* onsen
|
7
7
|
* niconico
|
8
|
-
* freshlive.tv
|
9
8
|
* himalaya.fm
|
10
9
|
* Asobi Store
|
10
|
+
* stand.fm
|
11
|
+
* YouTube (playlist or channel)
|
11
12
|
|
12
13
|
If you want to save files as MP3, needs `ffmpeg` command.
|
13
|
-
To download niconico video, also needs latest `youtube-dl` command (you can get it from `https://yt-dl.org/`), then specify
|
14
|
+
To download niconico video or YouTube, also needs latest `youtube-dl` command (you can get it from `https://yt-dl.org/`), then specify user ID and password to `~/.netrc` for niconico as:
|
14
15
|
|
15
16
|
```
|
16
17
|
machine niconico
|
data/bin/hls-dl
CHANGED
@@ -1,103 +1,18 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
require "
|
3
|
-
require "mechanize"
|
4
|
-
require 'open-uri'
|
5
|
-
require 'thread'
|
2
|
+
require "open3"
|
6
3
|
|
7
4
|
if ARGV.size != 2
|
8
|
-
$stderr.puts "
|
9
|
-
$stderr.puts "hls-dl <playlist.m3u8> <output(.
|
5
|
+
$stderr.puts "HLS downloader wrapping ffmpeg"
|
6
|
+
$stderr.puts "hls-dl <playlist.m3u8> <output(.mp4,mp3,...)>"
|
10
7
|
exit 1
|
11
8
|
end
|
12
9
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
21
|
-
agent.get(m3u8)
|
22
|
-
body = agent.page.body
|
23
|
-
rescue ArgumentError
|
24
|
-
body = URI.open(m3u8, &:read)
|
25
|
-
end
|
26
|
-
tses = body.scan(/.*\.ts.*/)
|
27
|
-
key_url = body.scan(/URI="(.*)"/).flatten.first
|
28
|
-
|
29
|
-
if key_url
|
30
|
-
key = agent.get_file(key_url)
|
31
|
-
decoder = OpenSSL::Cipher.new('aes-128-cbc')
|
32
|
-
decoder.key = key
|
33
|
-
decoder.decrypt
|
34
|
-
else
|
35
|
-
decoder = ''
|
36
|
-
def decoder.update(s); return s; end
|
37
|
-
def decoder.final(); return ''; end
|
38
|
-
end
|
39
|
-
|
40
|
-
q = Queue.new
|
41
|
-
tses.each_with_index{|ts_url, block_no| q.push([ts_url, block_no])}
|
42
|
-
q.close
|
43
|
-
results = [nil] * q.size
|
44
|
-
|
45
|
-
msgs = Queue.new
|
46
|
-
stats = [0] * 3
|
47
|
-
stats.size.times do |thread_no|
|
48
|
-
Thread.start(thread_no) do |thread_no|
|
49
|
-
while (ts_url, block_no = q.pop)
|
50
|
-
begin
|
51
|
-
sleep(rand(0.1) * 3)
|
52
|
-
results[block_no] = agent.get_file(ts_url)
|
53
|
-
stats[thread_no] = block_no
|
54
|
-
rescue Mechanize::ResponseCodeError => e
|
55
|
-
case e.response_code
|
56
|
-
when "403" # Forbidden
|
57
|
-
msgs.push "T#{thread_no}: 403 Forbidden"
|
58
|
-
retry
|
59
|
-
when "429" # TooManyRequests
|
60
|
-
msgs.push "T#{thread_no}: 429 TooManyRequests"
|
61
|
-
retry
|
62
|
-
else
|
63
|
-
msgs.push "T#{thread_no}: #{e}"
|
64
|
-
retry # TODO: catch other status code
|
65
|
-
end
|
66
|
-
rescue
|
67
|
-
msgs.push "T#{thread_no}: #{e.class} #{e}"
|
68
|
-
break
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
def print_status(block, tses, stats, msgs)
|
75
|
-
$stderr.puts msgs.pop while !msgs.empty?
|
76
|
-
$stderr.print "#{block+1}/#{tses.size} saved --"
|
77
|
-
stats.each_with_index{|block_no, thread_no| $stderr.print " T#{thread_no+1}:#{block_no+1}"}
|
78
|
-
$stderr.print "\r"
|
79
|
-
end
|
80
|
-
|
81
|
-
def save(fd, results, tses, stats, msgs, decoder)
|
82
|
-
results.each_with_index do |data, i|
|
83
|
-
print_status(i, tses, stats, msgs)
|
84
|
-
loop do
|
85
|
-
if data
|
86
|
-
fd.write(decoder.update(data))
|
87
|
-
results[i] = nil # be saved data want to GC
|
88
|
-
break
|
89
|
-
else
|
90
|
-
sleep(1.0) # wait for complete of chunk download finished
|
91
|
-
data = results[i]
|
92
|
-
end
|
93
|
-
end
|
94
|
-
end
|
95
|
-
fd.write(decoder.final)
|
96
|
-
puts
|
97
|
-
end
|
98
|
-
|
99
|
-
if ts_file
|
100
|
-
open(ts_file, 'wb:ASCII-8BIT'){|f|save(f, results, tses, stats, msgs, decoder)}
|
101
|
-
else
|
102
|
-
save($stdout, results, tses, stats, msgs, decoder)
|
10
|
+
ffmpeg = %Q[ffmpeg -loglevel error -protocol_whitelist file,http,https,tcp,tls,crypto -n -i "#{ARGV[0]}" #{ARGV[1]}]
|
11
|
+
result = Open3.capture3(ffmpeg)
|
12
|
+
unless result[2].to_i == 0
|
13
|
+
p result
|
14
|
+
File.delete(ARGV[1]) if File.exist?(ARGV[1])
|
15
|
+
$stderr.puts result[1]
|
16
|
+
exit(1)
|
103
17
|
end
|
18
|
+
exit(0)
|
data/lib/hibiki.rb
CHANGED
@@ -57,7 +57,9 @@ private
|
|
57
57
|
playlist_url = video_info[:playlist_url]
|
58
58
|
m3u8_url = URI(agent.get(playlist_url).body.scan(/http.*/).flatten.first)
|
59
59
|
|
60
|
-
|
60
|
+
# choice the max size element from multi-part m3u8 file
|
61
|
+
m3u8 = agent.get(m3u8_url).body.split("EXT-X-KEY").sort{|m,n|m.size <=> n.size}.last
|
62
|
+
|
61
63
|
key_url = m3u8.scan(/URI="(.*)"/).flatten.first
|
62
64
|
|
63
65
|
tses = m3u8.scan(/^.*ts_.*\.ts/)
|
data/lib/hls.rb
CHANGED
@@ -1,35 +1,17 @@
|
|
1
1
|
require 'webradio'
|
2
|
+
require 'open3'
|
2
3
|
|
3
4
|
class HLS < WebRadio
|
4
5
|
private
|
5
|
-
def hls_download(name,
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
m3u_meta1 = URI.open(m3u_meta2, &:read)
|
15
|
-
m3u = m3u_meta1.scan(/^[^#].*/).first
|
16
|
-
save_m4a(URI(m3u), @m4a_file)
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def get_m4a(uri_playlist)
|
21
|
-
URI.open(uri_playlist).each_line do |l|
|
22
|
-
next if /^#/ =~ l
|
23
|
-
l.chomp!
|
24
|
-
print "."
|
25
|
-
yield URI.open(uri_playlist + l, 'r:ASCII-8BIT', &:read)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def save_m4a(uri_playlist, m4a_file)
|
30
|
-
open(m4a_file, 'wb:ASCII-8BIT') do |m4a|
|
31
|
-
get_m4a(uri_playlist) do |part|
|
32
|
-
m4a.write part
|
6
|
+
def hls_download(name, serial, m3u_meta)
|
7
|
+
mp4_file = "#{name}##{serial}.mp4"
|
8
|
+
mp3_file = "#{name}##{serial}.mp3"
|
9
|
+
m3u = URI.open(m3u_meta, &:read).scan(/^[^#].*/).first
|
10
|
+
m3u = Pathname(m3u_meta).dirname.join(m3u) if Pathname(m3u).relative?
|
11
|
+
mp3nize(mp4_file, mp3_file) do
|
12
|
+
result = Open3.capture3(%Q[ffmpeg -loglevel error -protocol_whitelist file,http,https,tcp,tls,crypto -i "#{m3u}" "#{mp4_file}"])
|
13
|
+
unless result[2].to_i == 0
|
14
|
+
raise WebRadio::DownloadError.new("failed download the radio program (#{@label}).")
|
33
15
|
end
|
34
16
|
end
|
35
17
|
end
|
data/lib/onsen.rb
CHANGED
@@ -1,22 +1,26 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require '
|
3
|
+
require 'hls'
|
4
4
|
require 'nokogiri'
|
5
|
+
require 'json'
|
5
6
|
|
6
|
-
class Onsen <
|
7
|
+
class Onsen < HLS
|
7
8
|
def initialize(params, options)
|
8
9
|
super
|
9
|
-
@cover = "//*[@
|
10
|
+
@cover = "//*[@class='newest-content--left']//img[1]/@src" unless @cover
|
10
11
|
end
|
11
12
|
|
12
13
|
def download
|
13
|
-
|
14
|
+
html = URI.open(@url, &:read)
|
15
|
+
serial = Nokogiri(html).css('.play-video-info td')[0].text.scan(/\d+/)[0].to_i
|
16
|
+
m3u8 = JSON.parse(html.scan(%r|streaming_url:("https:.*?.m3u8")|).flatten.sort.last)
|
17
|
+
hls_download(@label, serial, m3u8)
|
14
18
|
end
|
15
19
|
|
16
20
|
def dump
|
17
21
|
tag = Pathname(@url).basename.to_s.gsub(%r|[-/]|, '_')
|
18
22
|
html = Nokogiri(URI.open(@url, &:read))
|
19
|
-
title = html.css('
|
23
|
+
title = html.css('h3')[0].text
|
20
24
|
return {
|
21
25
|
tag => {
|
22
26
|
'desc' => title,
|
@@ -25,24 +29,4 @@ class Onsen < WebRadio
|
|
25
29
|
}
|
26
30
|
}
|
27
31
|
end
|
28
|
-
|
29
|
-
private
|
30
|
-
def onsen_download(name, program_id)
|
31
|
-
html = Nokogiri(URI.open('http://onsen.ag/', 'User-Agent' => 'iPhone', &:read))
|
32
|
-
begin
|
33
|
-
serial = html.css("##{program_id}").text.scan(/#(\d+)/).flatten.first
|
34
|
-
mp3_url = html.css('form[target=_self]').select {|form|
|
35
|
-
form.attr('action') =~ %r|/#{program_id}\w+\.mp[34]|
|
36
|
-
}.first.attr('action')
|
37
|
-
rescue NoMethodError
|
38
|
-
raise NotFoundError.new("no radio program in #{program_id}.")
|
39
|
-
end
|
40
|
-
src_file = "#{name}##{serial}#{mp3_url.scan(/\.mp[34]$/).first}"
|
41
|
-
mp3_file = "#{name}##{serial}.mp3"
|
42
|
-
mp3nize(src_file, mp3_file, false) do
|
43
|
-
open(src_file, 'wb:ASCII-8BIT') do |mp3|
|
44
|
-
mp3.write URI.open(mp3_url, 'rb:ASCII-8BIT', &:read)
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
32
|
end
|
data/lib/standfm.rb
ADDED
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'mechanize'
|
4
|
+
require 'json'
|
5
|
+
|
6
|
+
class StandFm < WebRadio
|
7
|
+
def initialize(params, options)
|
8
|
+
super
|
9
|
+
@offset = 0
|
10
|
+
end
|
11
|
+
|
12
|
+
def download
|
13
|
+
uri = URI(@url)
|
14
|
+
html = Nokogiri(uri.open.read)
|
15
|
+
episode = uri + html.css('#root a[href^="/episodes/"]').map{|e|e.attr('href')}.uniq[@offset]
|
16
|
+
|
17
|
+
html = episode.open.read
|
18
|
+
json = JSON.parse(html.scan(%r[<script>window.__SERVER_STATE__=(.*)</script>]).flatten[0])
|
19
|
+
m4a = json['topics'].find{|k,v|v['episodeId'] == File.basename(episode.path)}.last['downloadUrl']
|
20
|
+
|
21
|
+
doc = Nokogiri(html)
|
22
|
+
serial = doc.title.scan(/#(\d+)/).flatten.first.to_i
|
23
|
+
@cover = doc.css('meta[property="og:image"]').attr('content').text unless @cover
|
24
|
+
m4a_file = "#{@label}##{serial}.m4a"
|
25
|
+
mp3_file = "#{@label}##{serial}.mp3"
|
26
|
+
|
27
|
+
mp3nize(m4a_file, mp3_file) do
|
28
|
+
open(m4a_file, 'wb:ASCII-8BIT') do |w|
|
29
|
+
w.write(URI(m4a).open.read)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def dump
|
35
|
+
uri = URI(@url)
|
36
|
+
tag = File.basename(uri.path)
|
37
|
+
html = Nokogiri(uri.open.read)
|
38
|
+
label, = html.css('title').text.split(/ \| /)
|
39
|
+
cover = html.css('meta[property="og:image"]').attr('content').text
|
40
|
+
return {
|
41
|
+
tag => {
|
42
|
+
'desc' => label,
|
43
|
+
'url' => @url,
|
44
|
+
'label' => label,
|
45
|
+
'cover' => cover
|
46
|
+
}
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
def find_player(url)
|
52
|
+
programs = Nokogiri(URI.open(url).read)
|
53
|
+
programs.css('.list-main-product a.wrap').each do |program|
|
54
|
+
begin
|
55
|
+
return Nokogiri(URI.open("https://asobistore.jp#{program.attr('href')}").read).css('iframe').last.attr('src')
|
56
|
+
rescue # access denied because only access by premium members
|
57
|
+
next
|
58
|
+
end
|
59
|
+
end
|
60
|
+
raise StandardError.new('movie not found.')
|
61
|
+
end
|
62
|
+
end
|
data/lib/webradio.rb
CHANGED
@@ -14,21 +14,27 @@ class WebRadio
|
|
14
14
|
when %r[^https?://hibiki-radio\.jp/]
|
15
15
|
require 'hibiki'
|
16
16
|
Hibiki.new(params, options)
|
17
|
-
when %r[^
|
17
|
+
when %r[^https?://(www\.)?onsen\.ag/program/]
|
18
18
|
require 'onsen'
|
19
19
|
Onsen.new(params, options)
|
20
20
|
when %r[nicovideo\.jp]
|
21
21
|
require 'nicovideo'
|
22
22
|
Nicovideo.new(params, options)
|
23
|
-
when %r[^https://freshlive\.tv/]
|
24
|
-
require 'freshlive'
|
25
|
-
FreshLive.new(params, options)
|
26
23
|
when %r[^http://m\.himalaya\.fm/]
|
27
24
|
require 'himalaya'
|
28
25
|
Himalaya.new(params, options)
|
29
26
|
when %r[^https://asobistore\.jp/]
|
30
27
|
require 'asobistore'
|
31
28
|
AsobiStore.new(params, options)
|
29
|
+
when %r[^https://stand\.fm/channels/]
|
30
|
+
require 'standfm'
|
31
|
+
StandFm.new(params, options)
|
32
|
+
when %r[^https://www\.youtube\.com/playlist]
|
33
|
+
require 'youtube'
|
34
|
+
YoutubePlaylist.new(params, options)
|
35
|
+
when %r[^https://www\.youtube\.com/c/]
|
36
|
+
require 'youtube'
|
37
|
+
YoutubeChannel.new(params, options)
|
32
38
|
else
|
33
39
|
raise UnsupportError, "unsupported url: #{params['url']}"
|
34
40
|
end
|
data/lib/youtube.rb
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
require 'json'
|
3
|
+
require 'nokogiri'
|
4
|
+
|
5
|
+
class Youtube < WebRadio
|
6
|
+
def initialize(params, options)
|
7
|
+
super
|
8
|
+
@offset = 0
|
9
|
+
@target_content = []
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def youtube_download(mp4_url, mp4_file, mp3_file)
|
15
|
+
begin
|
16
|
+
mp3nize(mp4_file, mp3_file) do
|
17
|
+
loop do
|
18
|
+
print '.'
|
19
|
+
_, err, status = Open3.capture3("youtube-dl -f mp4 -o '#{mp4_file}' --netrc '#{mp4_url}'")
|
20
|
+
break if status == 0
|
21
|
+
next if err =~ /403: Forbidden/
|
22
|
+
raise ForbiddenError.new("Could not access to #{mp4_url}") if err =~ /TypeError|AssertionError/
|
23
|
+
raise DownloadError.new(err)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
rescue ForbiddenError
|
27
|
+
puts "#{$!.message}, try next."
|
28
|
+
@offset += 1
|
29
|
+
retry
|
30
|
+
rescue NotFoundError
|
31
|
+
raise DownloadError.new('video not found')
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def first_video(html)
|
36
|
+
json_str = html.scan(/ytInitialData = (.*);<\/script>/).flatten.first
|
37
|
+
json = JSON.parse(json_str)
|
38
|
+
contents = json.dig(*@target_content)
|
39
|
+
yield contents.first
|
40
|
+
end
|
41
|
+
|
42
|
+
def serial_file(title, label, serial, ext)
|
43
|
+
if serial > 0
|
44
|
+
"#{@label}##{serial}.#{ext}"
|
45
|
+
else
|
46
|
+
"#{title}.#{ext}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class YoutubePlaylist < Youtube
|
52
|
+
|
53
|
+
def initialize(params, options)
|
54
|
+
super
|
55
|
+
@target_content = ["contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "playlistVideoListRenderer", "contents"].freeze
|
56
|
+
end
|
57
|
+
|
58
|
+
def download
|
59
|
+
first_video(URI.open(@url).read) do |content|
|
60
|
+
item = content['playlistVideoRenderer']
|
61
|
+
@cover = item['thumbnail']['thumbnails'].last['url']
|
62
|
+
title = item['title']['runs'][0]['text']
|
63
|
+
serial = title.scan(/\d+/).first.to_i
|
64
|
+
url = "https://www.youtube.com#{item["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]}"
|
65
|
+
|
66
|
+
mp4_file = serial_file title, @label, serial, 'mp4'
|
67
|
+
mp3_file = serial_file title, @label, serial, 'mp3'
|
68
|
+
|
69
|
+
youtube_download url, mp4_file, mp3_file
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def dump
|
74
|
+
uri = URI(@url)
|
75
|
+
tag = Hash[URI.decode_www_form(File.basename(uri.query))]['list']
|
76
|
+
html = Nokogiri(uri.open.read)
|
77
|
+
label, = html.css('title').text.split(/ - .*\Z/)
|
78
|
+
return {
|
79
|
+
tag => {
|
80
|
+
'desc' => label,
|
81
|
+
'url' => @url,
|
82
|
+
'label' => label
|
83
|
+
}
|
84
|
+
}
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
class YoutubeChannel < Youtube
|
89
|
+
def initialize(params, options)
|
90
|
+
super
|
91
|
+
@target_content = ['contents', 'twoColumnBrowseResultsRenderer', 'tabs', 1, 'tabRenderer', 'content', 'sectionListRenderer', 'contents', 0, 'itemSectionRenderer', 'contents', 0, 'gridRenderer', 'items'].freeze
|
92
|
+
end
|
93
|
+
|
94
|
+
def download
|
95
|
+
first_video(URI.open(@url).read) do |content|
|
96
|
+
item = content['gridVideoRenderer']
|
97
|
+
@cover = item['thumbnail']['thumbnails'].last['url']
|
98
|
+
title = item['title']['runs'][0]['text']
|
99
|
+
serial = title.scan(/\d+/).first.to_i
|
100
|
+
url = "https://www.youtube.com#{item['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']}"
|
101
|
+
|
102
|
+
mp4_file = serial_file title, @label, serial, 'mp4'
|
103
|
+
mp3_file = serial_file title, @label, serial, 'mp3'
|
104
|
+
|
105
|
+
youtube_download url, mp4_file, mp3_file
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def dump
|
110
|
+
tag = @url.scan(%r|/c/([^/]*)|).flatten.first
|
111
|
+
html = Nokogiri(URI(@url).open.read)
|
112
|
+
label = html.css('meta[property="og:title"]').attr('content').text
|
113
|
+
return {
|
114
|
+
tag => {
|
115
|
+
'desc' => label,
|
116
|
+
'url' => "#{@url.scan(%r|(.*?/c/[^/]*)|).flatten.first}/videos?view=0&sort=dd&flow=grid",
|
117
|
+
'label' => label
|
118
|
+
}
|
119
|
+
}
|
120
|
+
end
|
121
|
+
end
|
data/rget.gemspec
CHANGED
@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "rget"
|
7
|
-
spec.version = "4.
|
8
|
-
spec.authors = ["
|
7
|
+
spec.version = "4.12.0"
|
8
|
+
spec.authors = ["Tada, Tadashi"]
|
9
9
|
spec.email = ["t@tdtds.jp"]
|
10
|
-
spec.description = %q{Downloading newest radio programs on the web. Supported radio stations are hibiki, onsen, niconico,
|
10
|
+
spec.description = %q{Downloading newest radio programs on the web. Supported radio stations are hibiki, onsen, niconico, himalaya, asobi store, stand.fm and youtube.}
|
11
11
|
spec.summary = %q{Downloading newest radio programs on the web.}
|
12
12
|
spec.homepage = "https://github.com/wasamas/rget"
|
13
13
|
spec.license = "GPL"
|
data/rget.yaml
CHANGED
@@ -23,3 +23,15 @@ programs:
|
|
23
23
|
desc: ル美子さん
|
24
24
|
url: https://ch.nicovideo.jp/luminesan/video?rss=2.0
|
25
25
|
label: ル美子さん
|
26
|
+
ranacha:
|
27
|
+
desc: らなちゃのひとりごと
|
28
|
+
url: https://stand.fm/channels/5f7d9011f04555115d2170ef
|
29
|
+
label: らなちゃのひとりごと
|
30
|
+
active:
|
31
|
+
desc: 愛美とはるかの2年A組青春アクティ部!
|
32
|
+
url: https://www.youtube.com/playlist?list=PLI5zma0g_n8_y5Xk5eFwX8osz4Q2a3cqU
|
33
|
+
label: アクティ部
|
34
|
+
shimotsukin:
|
35
|
+
desc: 霜月はるかのしもつきんチャンネル
|
36
|
+
url: https://www.youtube.com/c/shimotsukinchannel/videos?view=0&sort=dd&flow=grid
|
37
|
+
label: しもつきん
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rget
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.12.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- Tada, Tadashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -151,7 +151,7 @@ dependencies:
|
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0'
|
153
153
|
description: Downloading newest radio programs on the web. Supported radio stations
|
154
|
-
are hibiki, onsen, niconico,
|
154
|
+
are hibiki, onsen, niconico, himalaya, asobi store, stand.fm and youtube.
|
155
155
|
email:
|
156
156
|
- t@tdtds.jp
|
157
157
|
executables:
|
@@ -161,6 +161,8 @@ executables:
|
|
161
161
|
extensions: []
|
162
162
|
extra_rdoc_files: []
|
163
163
|
files:
|
164
|
+
- ".devcontainer/Dockerfile"
|
165
|
+
- ".devcontainer/devcontainer.json"
|
164
166
|
- ".gitignore"
|
165
167
|
- ".travis.yml"
|
166
168
|
- Gemfile
|
@@ -172,14 +174,15 @@ files:
|
|
172
174
|
- bin/rget
|
173
175
|
- lib/asobistore.rb
|
174
176
|
- lib/dropbox.rb
|
175
|
-
- lib/freshlive.rb
|
176
177
|
- lib/hibiki.rb
|
177
178
|
- lib/himalaya.rb
|
178
179
|
- lib/hls.rb
|
179
180
|
- lib/nicovideo.rb
|
180
181
|
- lib/onsen.rb
|
181
182
|
- lib/podcast.rb
|
183
|
+
- lib/standfm.rb
|
182
184
|
- lib/webradio.rb
|
185
|
+
- lib/youtube.rb
|
183
186
|
- rget.gemspec
|
184
187
|
- rget.yaml
|
185
188
|
homepage: https://github.com/wasamas/rget
|
@@ -201,8 +204,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
201
204
|
- !ruby/object:Gem::Version
|
202
205
|
version: '0'
|
203
206
|
requirements: []
|
204
|
-
|
205
|
-
rubygems_version: 2.7.7
|
207
|
+
rubygems_version: 3.0.8
|
206
208
|
signing_key:
|
207
209
|
specification_version: 4
|
208
210
|
summary: Downloading newest radio programs on the web.
|
data/lib/freshlive.rb
DELETED
@@ -1,113 +0,0 @@
|
|
1
|
-
require 'webradio'
|
2
|
-
require 'nokogiri'
|
3
|
-
require 'json'
|
4
|
-
|
5
|
-
class FreshLive < WebRadio
|
6
|
-
def download
|
7
|
-
if URI(@url).path =~ %r|/search/|
|
8
|
-
archive = @url
|
9
|
-
else
|
10
|
-
archive = URI(File.join(@url + '/programs/archive'))
|
11
|
-
end
|
12
|
-
|
13
|
-
each_programs(Nokogiri(URI.open(archive).read)) do |meta|
|
14
|
-
begin
|
15
|
-
serial = meta['data']['title'].scan(/\d+$/).first.to_i
|
16
|
-
src = "#{@label}##{'%02d' % serial}.ts"
|
17
|
-
dst = src.sub(/\.ts$/, @options.mp3 ? '.mp3' : '.mp4')
|
18
|
-
if exist?(dst)
|
19
|
-
puts "#{dst} is existent, skipped."
|
20
|
-
return
|
21
|
-
end
|
22
|
-
unless exist?(src)
|
23
|
-
open(src, 'wb') do |w|
|
24
|
-
print "getting #{src}..."
|
25
|
-
ts_list(meta['data']['archiveStreamUrl']).each_with_index do |u, i|
|
26
|
-
print '.' if i % 50 == 0
|
27
|
-
w.write(URI.open(u, 'rb').read)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
if @options.mp3
|
32
|
-
dst = to_mp3(src)
|
33
|
-
else
|
34
|
-
dst = to_mp4(src)
|
35
|
-
end
|
36
|
-
puts 'done.'
|
37
|
-
move(dst)
|
38
|
-
return
|
39
|
-
rescue OpenURI::HTTPError
|
40
|
-
puts 'try next.'
|
41
|
-
next
|
42
|
-
rescue
|
43
|
-
puts 'fail.'
|
44
|
-
$stderr.puts 'faild to convert .ts => .mp4'
|
45
|
-
return
|
46
|
-
end
|
47
|
-
end
|
48
|
-
puts 'fail.'
|
49
|
-
$stderr.puts 'free program not found.'
|
50
|
-
end
|
51
|
-
|
52
|
-
def dump
|
53
|
-
u = URI(@url)
|
54
|
-
if u.path =~ %r|/search/|
|
55
|
-
desc = URI.decode_www_form_component(Pathname(u.path).basename.to_s)
|
56
|
-
return {
|
57
|
-
'freshlive_search' => {
|
58
|
-
'desc' => desc,
|
59
|
-
'url' => @url,
|
60
|
-
'label' => desc
|
61
|
-
}
|
62
|
-
}
|
63
|
-
else
|
64
|
-
tag = Pathname(u.path).basename.to_s
|
65
|
-
meta = JSON.parse(Nokogiri(URI.open(@url, &:read)).css('script').first)
|
66
|
-
return {
|
67
|
-
tag => {
|
68
|
-
'desc' => meta['name'],
|
69
|
-
'url' => @url,
|
70
|
-
'label' => tag
|
71
|
-
}
|
72
|
-
}
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
private
|
77
|
-
def each_programs(html)
|
78
|
-
x = "//section[descendant::h1[contains(text(),'アーカイブ')]]//*[contains(@class,'ProgramTitle')]/a/@href"
|
79
|
-
html.xpath(x).each do |href|
|
80
|
-
id = Pathname(href.value).basename.to_s
|
81
|
-
yield JSON.parse(URI.open("https://freshlive.tv/proxy/Programs;id=#{id}", &:read))
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
def ts_list(rate_m3u8)
|
86
|
-
ts_m3u8 = URI.open(rate_m3u8).read.each_line.grep_v(/^#/)[1].chomp
|
87
|
-
URI.open(URI(rate_m3u8) + ts_m3u8).read.each_line.grep_v(/^#/).map{|u|URI(rate_m3u8) + u.chomp}
|
88
|
-
end
|
89
|
-
|
90
|
-
def to_mp3(src)
|
91
|
-
dst = src.sub(/ts$/, 'mp3')
|
92
|
-
command = "ffmpeg -i '#{src}' -vn -ab 64k '#{dst}'"
|
93
|
-
result = Open3.capture3(command)
|
94
|
-
if result[2].to_i == 0
|
95
|
-
File.delete(src)
|
96
|
-
else
|
97
|
-
File.delete(dst) if File.exist?(dst)
|
98
|
-
end
|
99
|
-
return dst
|
100
|
-
end
|
101
|
-
|
102
|
-
def to_mp4(src)
|
103
|
-
dst = src.sub(/ts$/, 'mp4')
|
104
|
-
command = "ffmpeg -i '#{src}' -vcodec copy -strict -2 '#{dst}'"
|
105
|
-
result = Open3.capture3(command)
|
106
|
-
if result[2].to_i == 0
|
107
|
-
File.delete(src)
|
108
|
-
else
|
109
|
-
File.delete(dst) if File.exist?(dst)
|
110
|
-
end
|
111
|
-
return dst
|
112
|
-
end
|
113
|
-
end
|