rget 4.9.2 → 4.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce4daabfbe713595cc6bf6343c237e9fff9599f962fe25045025f6e54679c560
-  data.tar.gz: 4eec27c11a58ab8b97caa2974d0de31852c4e54b7cf46c3b7c9125a0ae659dcb
+  metadata.gz: 222b83cd5f8c3cff4f1b3372ec7488cc2bc4643c66469be1259d123c54e53643
+  data.tar.gz: d2eda18075858e1e78fe0a862140161edfab2ad886075ba80e8b1a450f0ec617
 SHA512:
-  metadata.gz: b0cddcb3fec37d9b3a570a340254887552c5f6634a4b5142a6b905fd81b625a78544f4779d7483dcb3de550f63e6fd723f4846dbd502163db1cea3c976d6d914
-  data.tar.gz: 8e59ad5545d4ebf4997245a5f25145d09a91c96d08d5bbac6ad73ff8ded7b1432c56f1620873d37d4c23fa20a3323858ce15982ee1bf6544037b85f8e1b563ab
+  metadata.gz: 54d7bcead0ea3ddbb7a88afad7e7b95bde817c7272d0c2c4c567925bc41cc5ce35dff780624be93b92a49ec0e74ce9ac6cacf22f4c61c109543d73dc6d320728
+  data.tar.gz: f722aabcdaa8767d0fecaebedc400c26312ad44c5b8894f1cd3567c0eabbadee13aaeefbcc8a7211779b8427a386725e1a1406b788ae2c791183b2b07727e15a

data/.devcontainer/Dockerfile

@@ -0,0 +1,14 @@
+FROM ruby:2
+ENV USER vscode
+LABEL maintainer "@tdtds <t@tdtds.jp>"
+RUN apt-get -y update && apt-get -y install ffmpeg && \
+    curl -sLo /usr/local/bin/youtube-dl http://www.yt-dl.org/downloads/latest/youtube-dl && \
+    chmod +x /usr/local/bin/youtube-dl && \
+    useradd -u 1000 -m $USER && chsh -s /bin/bash $USER
+USER $USER
+RUN bundle config set path vendor/bundle && \
+    bundle config set with development:test && \
+    echo 'git config --global --unset core.editor' >> /home/$USER/.bashrc && \
+    echo 'git config --global --unset core.sshCommand' >> /home/$USER/.bashrc && \
+    echo 'git ls-remote -q > /dev/null' >> /home/$USER/.bashrc
+CMD [ "sleep", "infinity" ]

data/.devcontainer/devcontainer.json

@@ -0,0 +1,30 @@
+// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.122.1/containers/docker-from-docker-compose
+// If you want to run as a non-root user in the container, see .devcontainer/docker-compose.yml.
+{
+  "name": "rget",
+  "context": "..",
+  "dockerFile": "Dockerfile",
+  
+  // Set *default* container specific settings.json values on container create.
+  "settings": { 
+    "terminal.integrated.shell.linux": "/bin/bash"
+  },
+
+  // Add the IDs of extensions you want installed when the container is created.
+  "extensions": [
+		"rebornix.ruby"
+  ],
+
+  // Uncomment the next line if you want start specific services in your Docker Compose config.
+  // "runServices": [],
+
+  // Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+  // "shutdownAction": "none",
+
+  // Use 'postCreateCommand' to run commands after the container is created.
+  // "postCreateCommand": "docker --version",
+
+  // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root.
+  // "remoteUser": "vscode"
+}

data/bin/hls-dl

@@ -1,103 +1,18 @@
 #!/usr/bin/env ruby
-require "uri"
-require "mechanize"
-require 'open-uri'
-require 'thread'
+require "open3"
 
 if ARGV.size != 2
-	$stderr.puts "high speed video downloader of Http Live Streaming (HLS)"
-	$stderr.puts "hls-dl <playlist.m3u8> <output(.ts)>"
+	$stderr.puts "HLS downloader wrapping ffmpeg"
+	$stderr.puts "hls-dl <playlist.m3u8> <output(.mp4,mp3,...)>"
 	exit 1
 end
 
-m3u8 = ARGV.shift
-ts_file = "#{ARGV.shift}.ts"
-ts_file = nil if File.basename(ts_file) == "-.ts"
-
-begin
-	agent = Mechanize.new
-	agent.user_agent_alias = 'Windows Chrome'
-	agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
-	agent.get(m3u8)
-	body = agent.page.body
-rescue ArgumentError
-	body = URI.open(m3u8, &:read)
-end
-tses = body.scan(/.*\.ts.*/)
-key_url = body.scan(/URI="(.*)"/).flatten.first
-
-if key_url
-	key = agent.get_file(key_url)
-	decoder = OpenSSL::Cipher.new('aes-128-cbc')
-	decoder.key = key
-	decoder.decrypt
-else
-	decoder = ''
-	def decoder.update(s); return s; end
-	def decoder.final(); return ''; end
-end
-
-q = Queue.new
-tses.each_with_index{|ts_url, block_no| q.push([ts_url, block_no])}
-q.close
-results = [nil] * q.size
-
-msgs = Queue.new
-stats = [0] * 3
-stats.size.times do |thread_no|
-	Thread.start(thread_no) do |thread_no|
-		while (ts_url, block_no = q.pop)
-			begin
-				sleep(rand(0.1) * 3)
-				results[block_no] = agent.get_file(ts_url)
-				stats[thread_no] = block_no
-			rescue Mechanize::ResponseCodeError => e
-				case e.response_code
-				when "403" # Forbidden
-					msgs.push "T#{thread_no}: 403 Forbidden"
-					retry
-				when "429" # TooManyRequests
-					msgs.push "T#{thread_no}: 429 TooManyRequests"
-					retry
-				else
-					msgs.push "T#{thread_no}: #{e}"
-					retry # TODO: catch other status code
-				end
-			rescue
-				msgs.push "T#{thread_no}: #{e.class} #{e}"
-				break
-			end
-		end
-	end
-end
-
-def print_status(block, tses, stats, msgs)
-	$stderr.puts msgs.pop while !msgs.empty?
-	$stderr.print "#{block+1}/#{tses.size} saved --"
-	stats.each_with_index{|block_no, thread_no| $stderr.print " T#{thread_no+1}:#{block_no+1}"}
-	$stderr.print "\r"
-end
-
-def save(fd, results, tses, stats, msgs, decoder)
-	results.each_with_index do |data, i|
-		print_status(i, tses, stats, msgs)
-		loop do
-			if data
-				fd.write(decoder.update(data))
-				results[i] = nil # be saved data want to GC
-				break
-			else
-				sleep(1.0) # wait for complete of chunk download finished
-				data = results[i]
-			end
-		end
-	end
-	fd.write(decoder.final)
-	puts
-end
-
-if ts_file
-	open(ts_file, 'wb:ASCII-8BIT'){|f|save(f, results, tses, stats, msgs, decoder)}
-else
-	save($stdout, results, tses, stats, msgs, decoder)
+ffmpeg = %Q[ffmpeg -loglevel error -protocol_whitelist file,http,https,tcp,tls,crypto -n -i "#{ARGV[0]}" #{ARGV[1]}]
+result = Open3.capture3(ffmpeg)
+unless result[2].to_i == 0
+	p result
+	File.delete(ARGV[1]) if File.exist?(ARGV[1])
+	$stderr.puts result[1]
+	exit(1)
 end
+exit(0)

data/lib/hibiki.rb

@@ -58,9 +58,7 @@ private
 			m3u8_url = URI(agent.get(playlist_url).body.scan(/http.*/).flatten.first)
 
 			# choice the max size element from multi-part m3u8 file
-			m3u8s = agent.get(m3u8_url).body.split("EXT-X-KEY")
-			m3u8_sizes = m3u8s.map(&:size)
-			m3u8 = m3u8s[m3u8_sizes.index(m3u8_sizes.max)]
+			m3u8 = agent.get(m3u8_url).body.split("EXT-X-KEY").sort{|m,n|m.size <=> n.size}.last
 
 			key_url = m3u8.scan(/URI="(.*)"/).flatten.first
 

data/lib/hls.rb

@@ -1,35 +1,17 @@
 require 'webradio'
+require 'open3'
 
 class HLS < WebRadio
 private
-	def hls_download(name, html, serial_pattern, m3u_pattern)
-		serial = html.scan(serial_pattern).flatten.sort{|a,b| a.to_i <=> b.to_i}.last
-		@m4a_file = "#{name}##{serial}.m4a"
-		@mp3_file = @m4a_file.sub(/\.m4a$/, '.mp3')
-		mp3nize(@m4a_file, @mp3_file) do
-			m3u_meta2 = html.scan(m3u_pattern).flatten.sort.last
-			unless m3u_meta2
-				raise WebRadio::DownloadError.new("recent radio program not found.")
-			end
-			m3u_meta1 = URI.open(m3u_meta2, &:read)
-			m3u = m3u_meta1.scan(/^[^#].*/).first
-			save_m4a(URI(m3u), @m4a_file)
-		end
-	end
-
-	def get_m4a(uri_playlist)
-		URI.open(uri_playlist).each_line do |l|
-			next if /^#/ =~ l
-			l.chomp!
-			print "."
-			yield URI.open(uri_playlist + l, 'r:ASCII-8BIT', &:read)
-		end
-	end
-	
-	def save_m4a(uri_playlist, m4a_file)
-		open(m4a_file, 'wb:ASCII-8BIT') do |m4a|
-			get_m4a(uri_playlist) do |part|
-				m4a.write part
+	def hls_download(name, serial, m3u_meta)
+		mp4_file = "#{name}##{serial}.mp4"
+		mp3_file = "#{name}##{serial}.mp3"
+		m3u = URI.open(m3u_meta, &:read).scan(/^[^#].*/).first
+		m3u = Pathname(m3u_meta).dirname.join(m3u) if Pathname(m3u).relative?
+		mp3nize(mp4_file, mp3_file) do
+			result = Open3.capture3(%Q[ffmpeg -loglevel error -protocol_whitelist file,http,https,tcp,tls,crypto -i "#{m3u}" "#{mp4_file}"])
+			unless result[2].to_i == 0
+				raise WebRadio::DownloadError.new("failed download the radio program (#{@label}).")
 			end
 		end
 	end

data/lib/onsen.rb

@@ -1,22 +1,26 @@
 # encoding: utf-8
 
-require 'webradio'
+require 'hls'
 require 'nokogiri'
+require 'json'
 
-class Onsen < WebRadio
+class Onsen < HLS
 	def initialize(params, options)
 		super
-		@cover = "//*[@id='newProgramWrap']//img[1]/@src" unless @cover
+		@cover = "//*[@class='newest-content--left']//img[1]/@src" unless @cover
 	end
 
 	def download
-		onsen_download(@label, @url.scan(%r|/([^/]*)/$|).flatten.first)
+		html = URI.open(@url, &:read)
+		serial = Nokogiri(html).css('.play-video-info td')[0].text.scan(/\d+/)[0].to_i
+		m3u8 = JSON.parse(html.scan(%r|streaming_url:("https:.*?.m3u8")|).flatten.sort.last)
+		hls_download(@label, serial, m3u8)
 	end
 
 	def dump
 		tag = Pathname(@url).basename.to_s.gsub(%r|[-/]|, '_')
 		html = Nokogiri(URI.open(@url, &:read))
-		title = html.css('#outLineWrap h1').text
+		title = html.css('h3')[0].text
 		return {
 			tag => {
 				'desc' => title,
@@ -25,24 +29,4 @@ class Onsen < WebRadio
 			}
 		}
 	end
-
-private
-	def onsen_download(name, program_id)
-		html = Nokogiri(URI.open('http://onsen.ag/', 'User-Agent' => 'iPhone', &:read))
-		begin
-			serial = html.css("##{program_id}").text.scan(/#(\d+)/).flatten.first
-			mp3_url = html.css('form[target=_self]').select {|form|
-				form.attr('action') =~ %r|/#{program_id}\w+\.mp[34]|
-			}.first.attr('action')
-		rescue NoMethodError
-			raise NotFoundError.new("no radio program in #{program_id}.")
-		end
-		src_file = "#{name}##{serial}#{mp3_url.scan(/\.mp[34]$/).first}"
-		mp3_file = "#{name}##{serial}.mp3"
-		mp3nize(src_file, mp3_file, false) do
-			open(src_file, 'wb:ASCII-8BIT') do |mp3|
-				mp3.write URI.open(mp3_url, 'rb:ASCII-8BIT', &:read)
-			end
-		end
-	end
 end

data/lib/webradio.rb

@@ -14,7 +14,7 @@ class WebRadio
 		when %r[^https?://hibiki-radio\.jp/]
 			require 'hibiki'
 			Hibiki.new(params, options)
-		when %r[^http://(www\.)?onsen\.ag/program/]
+		when %r[^https?://(www\.)?onsen\.ag/program/]
 			require 'onsen'
 			Onsen.new(params, options)
 		when %r[nicovideo\.jp]

data/rget.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "rget"
-  spec.version       = "4.9.2"
+  spec.version       = "4.9.3"
   spec.authors       = ["TADA Tadashi"]
   spec.email         = ["t@tdtds.jp"]
   spec.description   = %q{Downloading newest radio programs on the web. Supported radio stations are hibiki, onsen, niconico, freshlive, himalaya and asobi store.}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rget
 version: !ruby/object:Gem::Version
-  version: 4.9.2
+  version: 4.9.3
 platform: ruby
 authors:
 - TADA Tadashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-10 00:00:00.000000000 Z
+date: 2020-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -161,6 +161,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".devcontainer/Dockerfile"
+- ".devcontainer/devcontainer.json"
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
@@ -201,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Downloading newest radio programs on the web.