rget 4.7.6 → 4.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0fd38e56606b4cab53195c0fdfbc513bb9c111059a8f2d4b8901635f05fd4ef
-  data.tar.gz: 91557b05c0841059bbacf5293a8ca7adb63bae5a384c27094e5564e5615aceec
+  metadata.gz: 67ae7e0feff45509f86be682672f93b2bc85458e686cf3f77b188549daf7447f
+  data.tar.gz: 5b63864b67d2c93337f4b48d5a253295b1d72ea3863c03216ee315d9e66916e6
 SHA512:
-  metadata.gz: bf5b31cbea8c7f77224820fb345f344e0573c4758db41756908d295b8f627c19b51faacb51bac08fee3ba233fea239758d408d2a879613061c2c0e94aa821ca9
-  data.tar.gz: 2ae8d69829a71ada3b512ea861ce3ba2156e1dfe038c9474cbe3736c57b44a6937276611f5766118395a39f5820246af0e7e1ed47d37cc90b2255ba472d6667c
+  metadata.gz: 1e3009d6cbe8c14cc0d6cd42fdfa62077b4be96ee17825df1a8e65f0a4d3cc08610151ec6b37a0b25bbd0c4b88b4fc9c79b7ffdfb70c5da0be7c3861b3d16fd1
+  data.tar.gz: 558ecd9a332a374f03f3092dea3499ef69516da359662b1c1f912af7f3fffd1122772f965000c9af962a1dd244c87d90d2606c8a931935932001c2ff5d4fe72c

data/bin/ulizadl

@@ -2,6 +2,7 @@
 require "uri"
 require "mechanize"
 require 'open-uri'
+require 'thread'
 
 if ARGV.size != 2
 	$stderr.puts "download TS file decoded by Uliza style video streaming"
@@ -11,6 +12,7 @@ end
 
 m3u8 = ARGV.shift
 ts_file = "#{ARGV.shift}.ts"
+ts_file = nil if File.basename(ts_file) == "-.ts"
 
 begin
 	agent = Mechanize.new
@@ -35,11 +37,67 @@ else
 	def decoder.final(); return ''; end
 end
 
-open(ts_file, 'wb:ASCII-8BIT') do |ts|
-	tses.each_with_index do |file, count|
-		print "#{count+1}/#{tses.size}\r"
-		ts.write(decoder.update(agent.get_file(file)))
+q = Queue.new
+tses.each_with_index{|ts_url, block_no| q.push([ts_url, block_no])}
+q.close
+results = [nil] * q.size
+
+msgs = Queue.new
+stats = [0] * 3
+stats.size.times do |thread_no|
+	Thread.start(thread_no) do |thread_no|
+		while (ts_url, block_no = q.pop)
+			begin
+				sleep(rand(0.1) * 3)
+				results[block_no] = agent.get_file(ts_url)
+				stats[thread_no] = block_no
+			rescue Mechanize::ResponseCodeError => e
+				case e.response_code
+				when "403" # Forbidden
+					msgs.push "T#{thread_no}: 403 Forbidden"
+					retry
+				when "429" # TooManyRequests
+					msgs.push "T#{thread_no}: 429 TooManyRequests"
+					retry
+				else
+					msgs.push "T#{thread_no}: #{e}"
+					retry # TODO: catch other status code
+				end
+			rescue
+				msgs.push "T#{thread_no}: #{e.class} #{e}"
+				break
+			end
+		end
 	end
-	ts.write(decoder.final)
+end
+
+def print_status(block, tses, stats, msgs)
+	$stderr.puts msgs.pop while !msgs.empty?
+	$stderr.print "#{block+1}/#{tses.size} saved --"
+	stats.each_with_index{|block_no, thread_no| $stderr.print " T#{thread_no+1}:#{block_no+1}"}
+	$stderr.print "\r"
+end
+
+def save(fd, results, tses, stats, msgs, decoder)
+	results.each_with_index do |data, i|
+		print_status(i, tses, stats, msgs)
+		loop do
+			if data
+				fd.write(decoder.update(data))
+				results[i] = nil # be saved data want to GC
+				break
+			else
+				sleep(1.0) # wait for complete of chunk download finished
+				data = results[i]
+			end
+		end
+	end
+	fd.write(decoder.final)
 	puts
 end
+
+if ts_file
+	open(ts_file, 'wb:ASCII-8BIT'){|f|save(f, results, tses, stats, msgs, decoder)}
+else
+	save($stdout, results, tses, stats, msgs, decoder)
+end

data/rget.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "rget"
-  spec.version       = "4.7.6"
+  spec.version       = "4.7.7"
   spec.authors       = ["TADA Tadashi"]
   spec.email         = ["t@tdtds.jp"]
   spec.description   = %q{Downloading newest radio programs on the web. Supported radio stations are hibiki, onsen, niconico, freshlive, himalaya and asobi store.}
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "dropbox_api"
   spec.add_runtime_dependency "mp3info"
 
-  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "travis"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rget
 version: !ruby/object:Gem::Version
-  version: 4.7.6
+  version: 4.7.7
 platform: ruby
 authors:
 - TADA Tadashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-09 00:00:00.000000000 Z
+date: 2019-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -98,16 +98,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -203,8 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Downloading newest radio programs on the web.