rubytube 0.5.0 → 1.2.0

data/lib/rubytube/stream.rb

@@ -1,95 +1,126 @@
-module RubyTube
-  class Stream
-    attr_accessor(
-      :monostate,
-      :url,
-      :itag,
-      :mime_type,
-      :codecs,
-      :type,
-      :subtype,
-      :file_size,
-      :is_otf,
-      :bitrate
-    )
-
-    def initialize(stream, monostate)
-      self.monostate = monostate
-
-      self.url = stream["url"]
-      self.itag = stream["itag"].to_i
-
-      self.mime_type, self.codecs = Extractor.mime_type_codec(stream["mimeType"])
-      self.type, self.subtype = mime_type.split("/")
-
-      self.is_otf = stream["is_otf"]
-      self.bitrate = stream["bitrate"]
-
-      self.file_size = stream.fetch("contentLength", 0).to_i
-    end
-
-    def download(filename: nil, output_dir: nil)
-      file_path = get_file_path(filename, output_dir)
-
-      return file_path if File.exist?(file_path)
-
-      bytes_remaining = file_size
-
-      File.open(file_path, "wb") do |f|
-        Request.stream(url) do |chunk|
-          bytes_remaining -= chunk.bytesize
-          f.write(chunk)
-        end
-      rescue HTTPError => e
-        raise e if e.code != 404
-      end
-
-      file_path
-    end
-
-    def is_audio?
-      type == "audio"
-    end
-
-    def is_video?
-      type == "video"
-    end
-
-    def is_adaptive?
-      codecs.size % 2 == 1
-    end
-
-    def is_progressive?
-      !is_adaptive?
-    end
-
-    def title
-      monostate.title
-    end
-
-    def resolution
-      stream_format.resolution
-    end
-
-    private
-
-    def get_file_path(filename, output_dir, prefix = "")
-      filename ||= default_filename
-
-      if prefix
-        filename = "#{prefix}#{filename}"
-      end
-
-      output_path = Utils.target_directory(output_dir)
-      File.join(output_path, filename)
-    end
-
-    def default_filename
-      "#{monostate.title}.#{subtype}"
-    end
-
-    def stream_format
-      @stream_format ||= StreamFormat.new(itag)
-    end
-  end
-end
+require "net/http"
+require "uri"
+require "fileutils"
+
+module RubyTube
+  class Stream
+    CHUNK_SIZE = 9 * 1024 * 1024
+
+    attr_reader :itag, :url, :mime_type, :type, :subtype, :codecs, :bitrate,
+                :fps, :width, :height, :resolution, :audio_sample_rate, :duration_ms
+
+    def initialize(format, yt)
+      @yt = yt
+      @itag = format["itag"]
+      @url = format["url"]
+      @mime_type, @codecs = parse_mime_type(format["mimeType"])
+      @type, @subtype = @mime_type.split("/")
+      @bitrate = format["bitrate"]
+      @fps = format["fps"]
+      @width = format["width"]
+      @height = format["height"]
+      @resolution = format["qualityLabel"] || (@height && "#{@height}p")
+      @audio_sample_rate = format["audioSampleRate"]&.to_i
+      @duration_ms = format["approxDurationMs"]&.to_i
+      @content_length = format["contentLength"]&.to_i&.nonzero?
+      @otf = format["type"] == "FORMAT_STREAM_TYPE_OTF"
+    end
+
+    alias container subtype
+
+    def video? = type == "video"
+    def audio? = type == "audio"
+    def otf? = @otf
+
+    def video_codec = video? ? codecs.first : nil
+    def audio_codec = audio? ? codecs.first : nil
+
+    def abr = bitrate && "#{bitrate / 1000}kbps"
+
+    def filesize
+      @content_length ||= fetch_filesize
+    end
+
+    def default_filename
+      name = (@yt.title || video_id_or_itag).gsub(%r{[/\\:*?"<>|]}, "").strip
+      "#{name}.#{audio? && container == "mp4" ? "m4a" : container}"
+    end
+
+    def download(output_path: nil, filename: nil, filename_prefix: nil,
+                 skip_existing: true, max_retries: 0)
+      raise Error, "OTF streams are not supported" if otf?
+
+      target = File.join(File.expand_path(output_path || Dir.pwd),
+                         "#{filename_prefix}#{filename || default_filename}")
+      return target if skip_existing && File.size?(target)
+
+      FileUtils.mkdir_p(File.dirname(target))
+      total = filesize
+      downloaded = 0
+      File.open(target, "wb") do |io|
+        while downloaded < total
+          chunk = fetch_range(downloaded, [downloaded + CHUNK_SIZE, total].min - 1, max_retries)
+          io.write(chunk)
+          downloaded += chunk.bytesize
+          yield chunk, total - downloaded if block_given?
+        end
+      end
+      target
+    end
+
+    def to_s
+      attrs = { itag:, mime_type:, resolution:, fps:, abr: }.compact
+      "#<RubyTube::Stream #{attrs.map { |k, v| "#{k}=#{v}" }.join(' ')}>"
+    end
+    alias inspect to_s
+
+    private
+
+    def video_id_or_itag = @yt.respond_to?(:video_id) ? @yt.video_id : itag.to_s
+
+    def parse_mime_type(raw)
+      match = raw&.match(%r{([\w-]+/[\w-]+);\s*codecs="([^"]+)"}) or
+        raise ExtractError, "unparseable mimeType: #{raw.inspect}"
+      [match[1], match[2].split(",").map(&:strip)]
+    end
+
+    def fetch_range(start, stop, max_retries)
+      attempts = 0
+      begin
+        response = http.get(request_path, request_headers("bytes=#{start}-#{stop}"))
+        raise Error, "HTTP #{response.code} while downloading itag #{itag}" unless response.is_a?(Net::HTTPSuccess)
+
+        response.body
+      rescue Error, SystemCallError, IOError, Net::OpenTimeout, Net::ReadTimeout => e
+        @http&.finish rescue nil
+        @http = nil
+        attempts += 1
+        retry if attempts <= max_retries
+        raise e
+      end
+    end
+
+    def fetch_filesize
+      response = http.get(request_path, request_headers("bytes=0-0"))
+      total = response["Content-Range"]&.split("/")&.last&.to_i
+      raise ExtractError, "cannot determine filesize for itag #{itag} (HTTP #{response.code})" unless total&.positive?
+
+      total
+    end
+
+    # googlevideo applies the range= query param and the Range header in sequence, so sending
+    # both slices an already-sliced body and 416s on any nonzero offset. Use the header alone.
+    def request_path = @request_path ||= URI(url).request_uri
+
+    def request_headers(range)
+      { "Range" => range, "User-Agent" => InnerTube::USER_AGENT }
+    end
+
+    def http
+      @http ||= begin
+        uri = URI(url)
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true)
+      end
+    end
+  end
+end

data/lib/rubytube/stream_query.rb

@@ -1,61 +1,56 @@
-module RubyTube
-  class StreamQuery < Array
-    attr_reader :streams
-
-    def initialize(fmt_streams)
-      super
-      @streams = fmt_streams
-    end
-
-    def filter(file_extension: nil, only_audio: false, only_video: false, resolution: nil, progressive: false, adaptive: false)
-      filters = []
-
-      filters << ->(stream) { stream.subtype == file_extension } if file_extension
-      filters << ->(stream) { stream.is_audio? } if only_audio
-      filters << ->(stream) { stream.is_video? } if only_video
-      filters << ->(stream) { stream.resolution == resolution } if resolution
-      filters << ->(stream) { stream.is_progressive? } if progressive
-      filters << ->(stream) { stream.is_adaptive? } if adaptive
-
-      r = streams
-      filters.each do |f|
-        r = r.select(&f)
-      end
-
-      StreamQuery.new(r)
-    end
-
-    def get_by_itag(itag)
-      streams.find { |s| s.itag == itag }
-    end
-
-    def get_by_resolution(resolution)
-      streams.find { |s| s.resolution == resolution }
-    end
-
-    def get_highest_resolution
-      order(resolution: :desc).first
-    end
-
-    def order(arg)
-      case arg
-      when Symbol
-        field = arg
-        dir = :asc
-      when Hash
-        field = arg.keys.first
-        dir = (arg[field] == :desc) ? :desc : :asc
-      end
-
-      allowed_fields = [:file_size, :itag, :resolution]
-      raise InvalidArgumentError unless allowed_fields.include? field
-
-      r = streams
-      r.sort! { |a, b| a.send(field).to_i <=> b.send(field).to_i }
-
-      r.reverse! if dir == :desc
-
-      StreamQuery.new(r)
-    end
-  end
-end
+module RubyTube
+  class StreamQuery
+    include Enumerable
+
+    def initialize(streams)
+      @streams = streams
+    end
+
+    def each(&) = @streams.each(&)
+    def size = @streams.size
+    def [](index) = @streams[index]
+    def last = @streams.last
+    def empty? = @streams.empty?
+
+    def filter(itag: nil, res: nil, fps: nil, mime_type: nil, type: nil, container: nil, abr: nil, &block)
+      selected = @streams
+      selected = selected.select { |s| s.itag == itag } if itag
+      selected = selected.select { |s| s.resolution == res } if res
+      selected = selected.select { |s| s.fps == fps } if fps
+      selected = selected.select { |s| s.mime_type == mime_type } if mime_type
+      selected = selected.select { |s| s.type == type } if type
+      selected = selected.select { |s| s.container == container } if container
+      selected = selected.select { |s| s.abr == abr } if abr
+      selected = selected.select(&block) if block
+      StreamQuery.new(selected)
+    end
+
+    def order_by(attribute)
+      StreamQuery.new(
+        @streams.reject { |s| s.public_send(attribute).nil? }
+                .sort_by { |s| sort_key(s.public_send(attribute)) }
+      )
+    end
+
+    def get_by_itag(itag) = @streams.find { |s| s.itag == itag }
+    def video_streams = filter(type: "video")
+    def audio_streams = filter(type: "audio")
+
+    def best_video(container: "mp4", max_resolution: nil)
+      candidates = video_streams.filter(container:)
+      candidates = candidates.filter { |s| s.resolution.to_i <= max_resolution } if max_resolution
+      candidates.max_by { |s| [s.resolution.to_i, s.fps.to_i, s.video_codec.start_with?("avc1") ? 1 : 0, s.bitrate.to_i] }
+    end
+
+    def best_audio(container: "mp4")
+      audio_streams.filter(container:).max_by { |s| s.bitrate.to_i }
+    end
+
+    def to_s = "#<RubyTube::StreamQuery #{@streams.size} streams>"
+    alias inspect to_s
+
+    private
+
+    def sort_key(value) = value.is_a?(String) ? value[/\d+/].to_i : value
+  end
+end

data/lib/rubytube/video_item.rb

@@ -0,0 +1,28 @@
+module RubyTube
+  # One entry of a video listing (channel tab, playlist). Built from the listing only: no per-video request is made
+  # until #to_youtube. view_count_text / published_text are YouTube's rounded English strings.
+  VideoItem = Data.define(:video_id, :title, :length, :view_count_text, :published_text, :thumbnail_url) do
+    def url = "https://www.youtube.com/watch?v=#{video_id}"
+    def to_youtube = YouTube.new(video_id)
+
+    def self.from_lockup(lockup)
+      meta = lockup.dig("metadata", "lockupMetadataViewModel")
+      # Usually one row ["16K views", "2 days ago"]; collaborations prepend a row of channel names.
+      rows = Array(meta&.dig("metadata", "contentMetadataViewModel", "metadataRows"))
+             .map { |r| Array(r["metadataParts"]).filter_map { |p| p.dig("text", "content") } }
+      parts = rows.find { |r| r.any? { |t| t.include?("view") } } || rows.last || []
+      views = parts.find { |t| t.include?("view") }
+      badge = Array(lockup.dig("contentImage", "thumbnailViewModel", "overlays"))
+              .flat_map { |o| Array(o.dig("thumbnailBottomOverlayViewModel", "badges")) }
+              .filter_map { |b| b.dig("thumbnailBadgeViewModel", "text") }
+              .find { |t| t.match?(/\A\d+(:\d\d)+\z/) }
+      new(video_id: lockup.fetch("contentId"),
+          title: meta&.dig("title", "content"),
+          length: badge&.split(":")&.reduce(0) { |acc, n| acc * 60 + n.to_i },
+          view_count_text: views,
+          published_text: (parts - [views]).first,
+          thumbnail_url: Array(lockup.dig("contentImage", "thumbnailViewModel", "image", "sources"))
+                           .max_by { |s| s["width"].to_i }&.dig("url"))
+    end
+  end
+end

data/lib/rubytube/youtube.rb

@@ -0,0 +1,95 @@
+module RubyTube
+  class YouTube
+    VIDEO_ID_PATTERNS = [
+      %r{youtu\.be/([0-9A-Za-z_-]{11})},
+      %r{youtube\.com/(?:watch\?(?:.*&)?v=|shorts/|embed/|live/)([0-9A-Za-z_-]{11})},
+      /\A([0-9A-Za-z_-]{11})\z/
+    ].freeze
+
+    attr_reader :video_id
+
+    def initialize(url)
+      @video_id = self.class.extract_video_id(url)
+      @innertube = InnerTube.new
+    end
+
+    def self.extract_video_id(url)
+      VIDEO_ID_PATTERNS.each do |pattern|
+        match = pattern.match(url)
+        return match[1] if match
+      end
+      raise ExtractError, "could not extract a video id from #{url.inspect}"
+    end
+
+    def player_response
+      @player_response ||= @innertube.player(video_id).tap { |data| check_availability(data) }
+    end
+
+    def streams
+      @streams ||= begin
+        streaming_data = player_response["streamingData"] or
+          raise ExtractError, "no streamingData in player response (live stream?)"
+        formats = Array(streaming_data["adaptiveFormats"]).select { |f| f["url"] }
+        StreamQuery.new(formats.map { |f| Stream.new(f, self) })
+      end
+    end
+
+    def download(output_path: nil, filename: nil, container: "mp4", max_resolution: nil,
+                 audio_only: false, skip_existing: true, max_retries: 0, &progress)
+      audio = streams.best_audio(container:) or raise ExtractError, "no #{container} audio stream for #{video_id}"
+      return audio.download(output_path:, filename:, skip_existing:, max_retries:, &progress) if audio_only
+
+      video = streams.best_video(container:, max_resolution:) or
+        raise ExtractError, "no #{container} video stream for #{video_id}"
+      target = File.join(File.expand_path(output_path || Dir.pwd), filename || video.default_filename)
+      return target if skip_existing && File.size?(target)
+
+      remaining = video.filesize + audio.filesize
+      track = lambda do |chunk, _|
+        remaining -= chunk.bytesize
+        progress&.call(chunk, remaining)
+      end
+      parts = [video, audio].map do |s|
+        s.download(output_path: File.dirname(target), filename: "#{File.basename(target)}.#{s.itag}.part",
+                   skip_existing: false, max_retries:, &track)
+      end
+      mux(parts, target)
+      target
+    ensure
+      parts&.each { |part| File.delete(part) if File.exist?(part) }
+    end
+
+    def video_details = player_response.fetch("videoDetails", {})
+    def title = video_details["title"]
+    def author = video_details["author"]
+    def channel_id = video_details["channelId"]
+    def channel = Channel.new(channel_id)
+    def length = video_details["lengthSeconds"]&.to_i
+    def views = video_details["viewCount"]&.to_i
+    def description = video_details["shortDescription"]
+    def keywords = video_details["keywords"] || []
+
+    def thumbnail_url
+      video_details.dig("thumbnail", "thumbnails")&.max_by { |t| t["width"].to_i }&.dig("url")
+    end
+
+    private
+
+    def mux(inputs, target)
+      ok = system("ffmpeg", "-y", "-loglevel", "error", *inputs.flat_map { |i| ["-i", i] }, "-c", "copy", target)
+      return if ok
+
+      File.delete(target) if File.exist?(target)
+      raise Error, ok.nil? ? "ffmpeg not found: install it, or fetch tracks separately via streams.best_video / best_audio" :
+                             "ffmpeg failed to mux #{target}"
+    end
+
+    def check_availability(data)
+      status = data.dig("playabilityStatus", "status")
+      return if status == "OK"
+
+      reason = data.dig("playabilityStatus", "reason") || "unknown reason"
+      raise VideoUnavailable.new(status || "NO_STATUS", reason)
+    end
+  end
+end

data/lib/rubytube.rb

@@ -1,27 +1,25 @@
-# frozen_string_literal: true
-
-require "faraday"
-require "faraday/follow_redirects"
-
-require_relative "rubytube/version"
-
-require_relative "rubytube/cipher"
-require_relative "rubytube/client"
-require_relative "rubytube/error"
-require_relative "rubytube/extractor"
-require_relative "rubytube/innertube"
-require_relative "rubytube/monostate"
-require_relative "rubytube/parser"
-require_relative "rubytube/request"
-require_relative "rubytube/stream_format"
-require_relative "rubytube/stream_query"
-require_relative "rubytube/stream"
-require_relative "rubytube/utils"
-
-module RubyTube
-  class << self
-    def new(url)
-      Client.new(url)
-    end
-  end
-end
+module RubyTube
+  VERSION = "1.2.0"
+
+  class Error < StandardError; end
+
+  class ExtractError < Error; end
+
+  class VideoUnavailable < Error
+    attr_reader :status, :reason
+
+    def initialize(status, reason)
+      @status = status
+      @reason = reason
+      super("#{status}: #{reason}")
+    end
+  end
+end
+
+require_relative "rubytube/innertube"
+require_relative "rubytube/stream"
+require_relative "rubytube/stream_query"
+require_relative "rubytube/youtube"
+require_relative "rubytube/video_item"
+require_relative "rubytube/channel"
+require_relative "rubytube/playlist"

metadata

@@ -1,81 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: rubytube
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.2.0
 platform: ruby
 authors:
 - nightswinger
-autorequire:
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2023-10-16 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: faraday
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.0.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.0.1
-- !ruby/object:Gem::Dependency
-  name: faraday-follow_redirects
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description:
+date: 2026-09-16 00:00:00.000000000 Z
+dependencies: []
 email:
 - stardustkids83@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rspec"
-- ".standard.yml"
-- CHANGELOG.md
-- CODE_OF_CONDUCT.md
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
 - lib/rubytube.rb
-- lib/rubytube/cipher.rb
-- lib/rubytube/client.rb
-- lib/rubytube/error.rb
-- lib/rubytube/extractor.rb
+- lib/rubytube/channel.rb
 - lib/rubytube/innertube.rb
-- lib/rubytube/monostate.rb
-- lib/rubytube/parser.rb
-- lib/rubytube/request.rb
+- lib/rubytube/playlist.rb
 - lib/rubytube/stream.rb
-- lib/rubytube/stream_format.rb
 - lib/rubytube/stream_query.rb
-- lib/rubytube/utils.rb
-- lib/rubytube/version.rb
-- sig/rubytube.rbs
+- lib/rubytube/video_item.rb
+- lib/rubytube/youtube.rb
 homepage: https://github.com/nightswinger/rubytube
 licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/nightswinger/rubytube
   source_code_uri: https://github.com/nightswinger/rubytube
-  changelog_uri: https://github.com/nightswinger/rubytube
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -83,16 +38,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: '3.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
-summary: RubyTube is a Ruby adaptation of pytube, enabling simplified downloading
-  and streaming of YouTube videos in a Ruby-friendly manner.
+summary: 'Download YouTube videos from Ruby: picks the best video and audio tracks
+  and muxes them with ffmpeg.'
 test_files: []

data/.rspec

@@ -1,3 +0,0 @@
---format documentation
---color
---require spec_helper

data/.standard.yml

@@ -1,3 +0,0 @@
-# For available configuration options, see:
-#   https://github.com/testdouble/standard
-ruby_version: 2.6

data/CHANGELOG.md

@@ -1,5 +0,0 @@
-## [Unreleased]
-
-## [0.1.0] - 2023-09-15
-
-- Initial release