rget 4.11.0 → 4.13.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 399f9407dcfa7a2c99884c4c7c1cac60fabaae386622bac006896d877e2f1c71
-  data.tar.gz: 5ba2047c907946753452437b51ecf3a55f3e7fa97e247c6ffab04f10badda4fc
+  metadata.gz: bef9d92d8791dfd9e2e4cf6852a30d07bf4bcd8a46a492a443d1d7223ac7877d
+  data.tar.gz: d03997be575932a0cc303ba44848c5178bd3802efcccfa223507eb0e3b417165
 SHA512:
-  metadata.gz: f0d57aff2683b09e53fbd663a11db02e862430d51ed97588c317e1432aa0b5d13625fc6747a8c83b210b1b394a726c48662af5c2b22ba24bbd08d8ddebf08487
-  data.tar.gz: de34fca3119bbb1a78704131e0a825fe3f2d0b77d1426dbe08bb0cf94eed751f6961b1c34e37a55f2907dfafed803877f692aeafffe780f9513b1b239270ba03
+  metadata.gz: 208553638033e5d34e9367c7a44da64620bd8f3306b91e70d831a087607a22d5e3467a6ae769678e59ca687b7fb55539ff6b24720e0c2944976b86a2110c09b1
+  data.tar.gz: d7e6f651fb7c91df44f5fe0947f627bbcef767019719e4405c288bcad668eb3262178526100cb11b49a303720fb0873456c2964cb7cdd2a879ff2be1e6ba6d0c

data/.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2
+FROM ruby:3
 ENV USER vscode
 LABEL maintainer "@tdtds <t@tdtds.jp>"
 RUN apt-get -y update && apt-get -y install ffmpeg && \

data/.devcontainer/devcontainer.json

@@ -13,7 +13,10 @@
 
   // Add the IDs of extensions you want installed when the container is created.
   "extensions": [
-		"rebornix.ruby"
+    "rebornix.ruby"
+    ,"github.vscode-pull-request-github"
+    ,"eamodio.gitlens"
+    ,"donjayamanne.githistory"
   ],
 
   // Uncomment the next line if you want start specific services in your Docker Compose config.

data/README.md

@@ -8,7 +8,7 @@ Downloading newest radio programs on the web. Supported radio stations are:
 * himalaya.fm
 * Asobi Store
 * stand.fm
-* YouTube (playlist only)
+* YouTube (playlist or  channel)
 
 If you want to save files as MP3, needs `ffmpeg` command.
 To download niconico video or YouTube, also needs latest `youtube-dl` command (you can get it from `https://yt-dl.org/`), then specify user ID and password to `~/.netrc` for niconico as:

data/lib/hls.rb

@@ -3,13 +3,14 @@ require 'open3'
 
 class HLS < WebRadio
 private
-	def hls_download(name, serial, m3u_meta)
+	def hls_download(name, serial, m3u_meta, headers = {})
 		mp4_file = "#{name}##{serial}.mp4"
 		mp3_file = "#{name}##{serial}.mp3"
-		m3u = URI.open(m3u_meta, &:read).scan(/^[^#].*/).first
+		m3u = URI.open(m3u_meta, headers, &:read).scan(/^[^#].*/).first
 		m3u = Pathname(m3u_meta).dirname.join(m3u) if Pathname(m3u).relative?
+		ffmpeg_headers = headers.empty? ? '' : "-headers '" + headers.map{|k, v| "#{k}: #{v}"}.join("\r\n") + "'"
 		mp3nize(mp4_file, mp3_file) do
-			result = Open3.capture3(%Q[ffmpeg -loglevel error -protocol_whitelist file,http,https,tcp,tls,crypto -i "#{m3u}" "#{mp4_file}"])
+			result = Open3.capture3(%Q[ffmpeg -loglevel error -protocol_whitelist file,http,https,tcp,tls,crypto #{ffmpeg_headers} -i "#{m3u}" "#{mp4_file}"])
 			unless result[2].to_i == 0
 				raise WebRadio::DownloadError.new("failed download the radio program (#{@label}).")
 			end

data/lib/nicovideo.rb

@@ -31,7 +31,7 @@ class Nicovideo < WebRadio
 				serial = tmp if tmp > 0
 			end
 			appendix = title =~ /おまけ|アフタートーク/ ? 'a' : ''
-			@file = "#{@label}##{'%02d' % serial}#{appendix}.#{video.type}"
+			@file = "#{@label}##{'%02d' % serial}#{appendix}.#{video.type || 'mp4'}"
 			@mp3_file = @file.sub(/\....$/, '.mp3')
 			mp3nize(@file, @mp3_file) do
 				loop do
@@ -39,7 +39,7 @@ class Nicovideo < WebRadio
 					_, err, status = Open3.capture3("youtube-dl -f mp4 -o #{@file} --netrc #{video.url}")
 					break if status == 0
 					next if err =~ /403: Forbidden/
-					raise ForbiddenError.new("Could not access to #{video.url}") if err =~ /TypeError|AssertionError/
+					raise ForbiddenError.new("Could not access to #{video.url}") if err =~ /TypeError|AssertionError|The video can't be downloaded/
 					raise DownloadError.new(err) 
 				end
 			end

data/lib/onsen.rb

@@ -5,21 +5,25 @@ require 'nokogiri'
 require 'json'
 
 class Onsen < HLS
+	HEADERS = {
+		"Referer" => 'https://www.onsen.ag/'
+	}
+
 	def initialize(params, options)
 		super
 		@cover = "//*[@class='newest-content--left']//img[1]/@src" unless @cover
 	end
 
 	def download
-		html = URI.open(@url, &:read)
+		html = URI.open(@url, HEADERS, &:read)
 		serial = Nokogiri(html).css('.play-video-info td')[0].text.scan(/\d+/)[0].to_i
 		m3u8 = JSON.parse(html.scan(%r|streaming_url:("https:.*?.m3u8")|).flatten.sort.last)
-		hls_download(@label, serial, m3u8)
+		hls_download(@label, serial, m3u8, HEADERS)
 	end
 
 	def dump
 		tag = Pathname(@url).basename.to_s.gsub(%r|[-/]|, '_')
-		html = Nokogiri(URI.open(@url, &:read))
+		html = Nokogiri(URI.open(@url, HEADERS, &:read))
 		title = html.css('h3')[0].text
 		return {
 			tag => {

data/lib/webradio.rb

@@ -20,9 +20,6 @@ class WebRadio
 		when %r[nicovideo\.jp]
 			require 'nicovideo'
 			Nicovideo.new(params, options)
-		when %r[^https://freshlive\.tv/]
-			require 'freshlive'
-			FreshLive.new(params, options)
 		when %r[^http://m\.himalaya\.fm/]
 			require 'himalaya'
 			Himalaya.new(params, options)
@@ -34,7 +31,10 @@ class WebRadio
 			StandFm.new(params, options)
 		when %r[^https://www\.youtube\.com/playlist]
 			require 'youtube'
-			Youtube.new(params, options)
+			YoutubePlaylist.new(params, options)
+		when %r[^https://www\.youtube\.com/c/]
+			require 'youtube'
+			YoutubeChannel.new(params, options)
 		else
 			raise UnsupportError, "unsupported url: #{params['url']}"
 		end

data/lib/youtube.rb

@@ -1,40 +1,121 @@
 require 'open-uri'
 require 'json'
+require 'nokogiri'
 
 class Youtube < WebRadio
-	def initialize(params, options)
-		super
-		@offset = 0
-	end
+    def initialize(params, options)
+    super
+        @offset = 0
+        @target_content = []
+  end
 
-    def download
-        html = URI.open(@url).read
-        json_str = html.scan(/ytInitialData = (.*);<\/script>/).flatten.first
-        json = JSON.parse(json_str)
-        title = json["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"][0]["playlistVideoRenderer"]["title"]["runs"][0]["text"]
-        @cover = json["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"][0]["playlistVideoRenderer"]["thumbnail"]["thumbnails"].last["url"]
-        mp4_url = "https://www.youtube.com#{json["contents"]["twoColumnBrowseResultsRenderer"]["tabs"][0]["tabRenderer"]["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["playlistVideoListRenderer"]["contents"][0]["playlistVideoRenderer"]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]}"
-        serial = title.scan(/[0-9]+/).first
+    private
 
-        mp4_file = "#{@label}##{serial}.mp4"
-        mp3_file = "#{@label}##{serial}.mp3"
+    def youtube_download(mp4_url, mp4_file, mp3_file)
         begin
             mp3nize(mp4_file, mp3_file) do
                 loop do
                     print '.'
-                    _, err, status = Open3.capture3("youtube-dl -f mp4 -o #{mp4_file} --netrc '#{mp4_url}'")
+                    _, err, status = Open3.capture3("youtube-dl -f mp4 -o '#{mp4_file}' --netrc '#{mp4_url}'")
                     break if status == 0
                     next if err =~ /403: Forbidden/
                     raise ForbiddenError.new("Could not access to #{mp4_url}") if err =~ /TypeError|AssertionError/
                     raise DownloadError.new(err)
                 end
             end
-		rescue ForbiddenError
-			puts "#{$!.message}, try next."
-			@offset += 1
-			retry
-		rescue NotFoundError
-			raise DownloadError.new('video not found')
+    rescue ForbiddenError
+      puts "#{$!.message}, try next."
+      @offset += 1
+      retry
+    rescue NotFoundError
+      raise DownloadError.new('video not found')
+        end
+    end
+
+    def first_video(html)
+        json_str = html.scan(/ytInitialData = (.*);<\/script>/).flatten.first
+        json = JSON.parse(json_str)
+        contents = json.dig(*@target_content)
+        yield contents.first
+    end
+
+    def serial_file(title, label, serial, ext)
+        if serial > 0
+            "#{@label}##{serial}.#{ext}"
+        else
+            "#{title}.#{ext}"
         end
     end
 end
+
+class YoutubePlaylist < Youtube
+
+    def initialize(params, options)
+    super
+    @target_content = ["contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "playlistVideoListRenderer", "contents"].freeze
+  end
+
+    def download
+        first_video(URI.open(@url).read) do |content|
+            item = content['playlistVideoRenderer']
+            @cover = item['thumbnail']['thumbnails'].last['url']
+            title = item['title']['runs'][0]['text']
+            serial = title.scan(/\d+/).first.to_i
+            url = "https://www.youtube.com/watch?v=#{item["videoId"]}"
+
+            mp4_file = serial_file title, @label, serial, 'mp4'
+            mp3_file = serial_file title, @label, serial, 'mp3'
+
+            youtube_download url, mp4_file, mp3_file
+        end
+    end
+
+    def dump
+      uri = URI(@url)
+      tag = Hash[URI.decode_www_form(File.basename(uri.query))]['list']
+      html = Nokogiri(uri.open.read)
+      label, = html.css('title').text.split(/ - .*\Z/)
+      return {
+        tag => {
+          'desc' => label,
+          'url' => @url,
+          'label' => label
+        }
+      }
+     end
+end
+
+class YoutubeChannel < Youtube
+  def initialize(params, options)
+        super
+        @target_content = ['contents', 'twoColumnBrowseResultsRenderer', 'tabs', 1, 'tabRenderer', 'content', 'sectionListRenderer', 'contents', 0, 'itemSectionRenderer', 'contents', 0, 'gridRenderer', 'items'].freeze
+    end
+
+    def download
+        first_video(URI.open(@url).read) do |content|
+            item = content['gridVideoRenderer']
+            @cover = item['thumbnail']['thumbnails'].last['url']
+            title = item['title']['runs'][0]['text']
+            serial = title.scan(/\d+/).first.to_i
+            url = "https://www.youtube.com#{item['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url']}"
+
+            mp4_file = serial_file title, @label, serial, 'mp4'
+            mp3_file = serial_file title, @label, serial, 'mp3'
+
+            youtube_download url, mp4_file, mp3_file
+        end
+    end
+
+    def dump
+      tag = @url.scan(%r|/c/([^/]*)|).flatten.first
+      html = Nokogiri(URI(@url).open.read)
+      label = html.css('meta[property="og:title"]').attr('content').text
+      return {
+        tag => {
+          'desc' => label,
+          'url' => "#{@url.scan(%r|(.*?/c/[^/]*)|).flatten.first}/videos?view=0&sort=dd&flow=grid",
+          'label' => label
+        }
+      }
+    end
+end

data/rget.gemspec

@@ -4,10 +4,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "rget"
-  spec.version       = "4.11.0"
+  spec.version       = "4.13.2"
   spec.authors       = ["Tada, Tadashi"]
   spec.email         = ["t@tdtds.jp"]
-  spec.description   = %q{Downloading newest radio programs on the web. Supported radio stations are hibiki, onsen, niconico, himalaya, asobi store, stand.fm and youtube playlist.}
+  spec.description   = %q{Downloading newest radio programs on the web. Supported radio stations are hibiki, onsen, niconico, himalaya, asobi store, stand.fm and youtube.}
   spec.summary       = %q{Downloading newest radio programs on the web.}
   spec.homepage      = "https://github.com/wasamas/rget"
   spec.license       = "GPL"
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "pit"
   spec.add_runtime_dependency "dropbox_api"
   spec.add_runtime_dependency "mp3info"
+  spec.add_runtime_dependency "rss"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"

data/rget.yaml

@@ -9,7 +9,6 @@ programs:
     desc: ラジオ「松井恵理子のにじらじっ！」
     url: http://www.onsen.ag/program/matsui/
     label: 松井恵理子のにじらじ
-    cover: //*[@id='newProgramWrap']//img[1]/@src
   suzakinishi:
     desc: 洲崎西
     url: http://ch.nicovideo.jp/search/%E6%B4%B2%E5%B4%8E%E8%A5%BF?channel_id=ch2589908&mode=s&sort=f&order=d&type=video
@@ -30,4 +29,8 @@ programs:
   active:
     desc: 愛美とはるかの2年A組青春アクティ部!
     url: https://www.youtube.com/playlist?list=PLI5zma0g_n8_y5Xk5eFwX8osz4Q2a3cqU
-    label: アクティ部
+    label: アクティ部
+  shimotsukin:
+    desc: 霜月はるかのしもつきんチャンネル
+    url: https://www.youtube.com/c/shimotsukinchannel/videos?view=0&sort=dd&flow=grid
+    label: しもつきん

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rget
 version: !ruby/object:Gem::Version
-  version: 4.11.0
+  version: 4.13.2
 platform: ruby
 authors:
 - Tada, Tadashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-03 00:00:00.000000000 Z
+date: 2022-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rss
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -151,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Downloading newest radio programs on the web. Supported radio stations
-  are hibiki, onsen, niconico, himalaya, asobi store, stand.fm and youtube playlist.
+  are hibiki, onsen, niconico, himalaya, asobi store, stand.fm and youtube.
 email:
 - t@tdtds.jp
 executables:
@@ -174,7 +188,6 @@ files:
 - bin/rget
 - lib/asobistore.rb
 - lib/dropbox.rb
-- lib/freshlive.rb
 - lib/hibiki.rb
 - lib/himalaya.rb
 - lib/hls.rb
@@ -205,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.3.4
 signing_key: 
 specification_version: 4
 summary: Downloading newest radio programs on the web.

data/lib/freshlive.rb

@@ -1,113 +0,0 @@
-require 'webradio'
-require 'nokogiri'
-require 'json'
-
-class FreshLive < WebRadio
-	def download
-		if URI(@url).path =~ %r|/search/|
-			archive = @url
-		else
-			archive = URI(File.join(@url + '/programs/archive'))
-		end
-
-		each_programs(Nokogiri(URI.open(archive).read)) do |meta|
-			begin
-				serial = meta['data']['title'].scan(/\d+$/).first.to_i
-				src = "#{@label}##{'%02d' % serial}.ts"
-				dst = src.sub(/\.ts$/, @options.mp3 ? '.mp3' : '.mp4')
-				if exist?(dst)
-					puts "#{dst} is existent, skipped."
-					return
-				end
-				unless exist?(src)
-					open(src, 'wb') do |w|
-						print "getting #{src}..."
-						ts_list(meta['data']['archiveStreamUrl']).each_with_index do |u, i|
-							print '.' if i % 50 == 0
-							w.write(URI.open(u, 'rb').read)
-						end
-					end
-				end
-				if @options.mp3
-					dst = to_mp3(src)
-				else
-					dst = to_mp4(src)
-				end
-				puts 'done.'
-				move(dst)
-				return
-			rescue OpenURI::HTTPError
-				puts 'try next.'
-				next
-			rescue
-				puts 'fail.'
-				$stderr.puts 'faild to convert .ts => .mp4'
-				return
-			end
-		end
-		puts 'fail.'
-		$stderr.puts 'free program not found.'
-	end
-
-	def dump
-		u = URI(@url)
-		if u.path =~ %r|/search/|
-			desc = URI.decode_www_form_component(Pathname(u.path).basename.to_s)
-			return {
-				'freshlive_search' => {
-					'desc' => desc,
-					'url' => @url,
-					'label' => desc
-				}
-			}
-		else
-			tag = Pathname(u.path).basename.to_s
-			meta = JSON.parse(Nokogiri(URI.open(@url, &:read)).css('script').first)
-			return {
-				tag => {
-					'desc' => meta['name'],
-					'url' => @url,
-					'label' => tag
-				}
-			}
-		end
-	end
-
-private
-	def each_programs(html)
-		x = "//section[descendant::h1[contains(text(),'アーカイブ')]]//*[contains(@class,'ProgramTitle')]/a/@href"
-		html.xpath(x).each do |href|
-			id = Pathname(href.value).basename.to_s
-			yield JSON.parse(URI.open("https://freshlive.tv/proxy/Programs;id=#{id}", &:read))
-		end
-	end
-
-	def ts_list(rate_m3u8)
-		ts_m3u8 = URI.open(rate_m3u8).read.each_line.grep_v(/^#/)[1].chomp
-		URI.open(URI(rate_m3u8) + ts_m3u8).read.each_line.grep_v(/^#/).map{|u|URI(rate_m3u8) + u.chomp}
-	end
-
-	def to_mp3(src)
-		dst = src.sub(/ts$/, 'mp3')
-		command = "ffmpeg -i '#{src}' -vn -ab 64k '#{dst}'"
-		result = Open3.capture3(command)
-		if result[2].to_i == 0
-			File.delete(src)
-		else
-			File.delete(dst) if File.exist?(dst)
-		end
-		return dst
-	end
-
-	def to_mp4(src)
-		dst = src.sub(/ts$/, 'mp4')
-		command = "ffmpeg -i '#{src}' -vcodec copy -strict -2 '#{dst}'"
-		result = Open3.capture3(command)
-		if result[2].to_i == 0
-			File.delete(src)
-		else
-			File.delete(dst) if File.exist?(dst)
-		end
-		return dst
-	end
-end