youtube-data 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9d7d85c28c1258272db9500f0aaf57b66cd831b8d8134b411a0e25ced0cd4912
+  data.tar.gz: 2de11bacc9a8ebd4b075d23c6da2b6b2f2e0f8898e8ee1f70ae2ebac30216a21
+SHA512:
+  metadata.gz: b85bf4e5f90c9bf857f2853fb971c8d61b53f9fff978d318c02f70f4458bbf1bda169954c2de6f22d2a6aa42f6c25d114cba51b760b9fd472c7dee24aa5cd96a
+  data.tar.gz: 9d2fb4892bda2a428c0da64fc27def5cf243b39fba5fec8245f178da8937c05a16f2afadcbce621ea72c271c748ca3bc604847e8b799063b02567b1748a47e7a

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in youtube.gemspec
+gemspec
+
+gem "rake", "~> 13.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 https://github.com/boddz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# Youtube
+
+A ruby gem for extracting youtube video data.
+
+Currently a work in progress. This is just a mirror at the moment so support will be limited because of this for now,
+and I am still new to Ruby, so some things will be changed a lot in the future most likely.
+
+
+## Installation
+
+Written and tested with and `ruby 3.0.2` on `Ubuntu 22.04.3 LTS`.
+
+Install required gems:
+
+```bash
+bundle install
+```
+
+Build and install gem file locally:
+
+```bash
+gem build && gem install youtube-data-[version].gem
+```
+
+Or install from gem server:
+
+```bash
+gem install youtube-data
+```
+
+
+## Usage
+
+TODO: Write usage instructions here...
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
+version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+Running tests:
+
+```bash
+cd tests  # These commands must be run in ./tests for relative path reasons.
+ruby test_all.rb  # To run all tests, or you can just call an individual test instead.
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on [GitHub](https://github.com/boddz/youtube).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/TODO.md

@@ -0,0 +1,12 @@
+# A List of To-dos
+
+## Little Things
+
+- Good tests (with mocks for all requests & sessions).
+- Some basic examples.
+
+
+## Bigger Things
+
+- Docs at some point.
+- Youtube video source URL sig decipher sub module.

data/lib/youtube/extractor.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Youtube
+
+  class InitExtractorError < RuntimeError
+  end
+
+  class InvalidVideoIDError < StandardError
+  end
+
+  class InvalidPathError < StandardError
+  end
+
+  # For extracting the data needed from a video in a native format.
+  #
+  # @param video_id [String] The video ID in which to scrape data from.
+  #
+  # Options
+  # =======
+  #
+  #   :mock_session => [MockSession] The mock session to use when testing.
+  #
+  class DataExtractor
+
+    HOMEPAGE = URI('https://www.youtube.com')
+
+    def initialize(video_id, opts = {})
+      @video_id = video_id
+      @video_path = "/watch?v=#{@video_id}"
+
+      # The session to use for the extractor allows information to persist during requests/session.
+      @session = Net::HTTP.start(HOMEPAGE.hostname, {'use_ssl': true})
+      if opts.include?(:mock_session)
+        @session = opts[:mock_session]
+      end
+
+      # Cache variables for raw data, saves overhead and less requests from the client session.
+      # Required requests.
+      @video_html = get_raw_html
+      # Required non-requests.
+      @video_json_raw = find_raw_json_in_html(@video_html)
+      @video_player_path = find_player_base_js_path_in_html(@video_html)
+    end
+
+    # Full url to the video's html page returned/ yielded as a `URI::HTTPS` instance.
+    def video_uri
+      uri = URI.join(HOMEPAGE, video_path)
+      return uri unless block_given?
+      yield uri
+    end
+
+    # Full url to the `base.js` video player script returned/ yielded as a `URI::HTTPS` instance.
+    def player_uri
+      uri = URI.join(HOMEPAGE, @video_player_path)
+      return uri unless block_given?
+      yield uri
+    end
+
+    # Send a simple get request using the extractor session. This should be how the module sends all further requests
+    # to the `youtube.com` hostname outside of this class also.
+    #
+    # @param path [String] Any valid path on the server, prefixed with `/`.
+    # @return [Net::HTTPResponse] The untouched response sent back from the request.
+    # @yield [Net::HTTPResponse] Same as return but yields to block if present.
+    #
+    def get_raw(path)
+      res = get_request_path(path)
+      return res unless block_given?
+      yield res
+    end
+
+    def video_raw_html
+      return @video_html unless block_given?
+      yield @video_html
+    end
+
+    # This is the json data for the video that is not yet been altered in terms of it's underlying structure, and of
+    # which is in it's purest form, but has been processed in a way that makes it easy to work with either through a
+    # file or a hash that is returned to the caller.
+    #
+    # @param dump_file [String] File path to write to (don't open if not provided).
+    # @param opt [Hash, String] `:pretty` if a dump file is specified, when set to true then format pretty else raw.
+    #
+    # @return [Hash] A hash representing the untouched json data parsed from the raw json html data of a video.
+    # @yield [Hash] Same as return but yields to block if provided.
+    #
+    def video_json_untouched(dump_file = nil, opt = {:pretty => true})
+      parsed_json = JSON.parse(@video_json_raw)
+
+      if dump_file.nil? == false
+        File.open("#{dump_file}", 'w') do |json_file|
+          if opt.include?(:pretty) and opt[:pretty] == true
+            json_file.write(JSON.pretty_generate(parsed_json))
+          else
+            JSON.dump(parsed_json, io=json_file)
+          end
+        end
+      end
+
+      return parsed_json unless block_given?
+      yield parsed_json
+    end
+
+    # Path from the homepage to the video's page.
+    private def video_path
+      if @video_id.length != 11  # 11 is the fixed length of a video ID on youtube.
+        raise InvalidVideoIDError, "The video id `#{@video_id}' is not valid (too short)"
+      end
+      return @video_path
+    end
+
+    # Sends a GET request to a specified path on the server using the request handler's session.
+    private def get_request_path(path)
+      if path.class != "".class
+        raise InvalidPathError, 'Path on server must be type `String\''
+      end
+      if path.empty? == false and path[0] != "/"
+        raise InvalidPathError, 'Path must be prefixed with `/\''
+      end
+      return @session.get(path)
+    end
+
+    private def get_raw_html
+      return get_request_path(video_path).body
+    end
+
+    private def find_raw_json_in_html(html)
+      var = html[/ytInitialPlayerResponse.*=.*\{.*\};/]  # Regex match containing var.
+      return var[/\{.*\}/]  # From matched var, extract the valid js object.
+    end
+
+    private def find_player_base_js_path_in_html(html)
+      return html[/([A-Za-z0-9]+(\/[A-Za-z0-9]+)+)_[A-Za-z0-9]+\.[A-Za-z0-9]+\/[A-Za-z0-9]+_[A-Za-z0-9]+\/base\.js/]
+    end
+
+  end
+
+end

data/lib/youtube/thumbnail.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Youtube
+
+  # Used for extracting thumbnails/ images data and bytes stored on `i.ytimg.com`.
+  #
+  # @param extractor [Youtube::DataExtractor] The extractor to use when getting initial required video data.
+  #
+  class Thumbnail
+    BASEHOST = URI('https://i.ytimg.com/')
+
+    def initialize(extractor)
+      @extractor = extractor
+
+      @session = Net::HTTP.start(BASEHOST.hostname, {'use_ssl': true})
+
+      @thumb_json_array = jump_to_in_json
+      @thumb_default_json = @thumb_json_array[0]
+    end
+
+    # An array of thumbnail(s) json data (stored as a hash).
+    #
+    # @return [Array] The array of hashes about the thumbnail(s).
+    # @yield [Array] The same as return, but yields to block if given.
+    #
+    def thumbnails_json
+      return @thumb_json_array unless block_given?
+      yield @thumb_json_array
+    end
+
+    # The json data for the default thumbnail (stored as a hash).
+    #
+    # @return [Hash] The hash form of the default thumbnail's json.
+    # @yield [Hash] The same as return, but yields to block if given.
+    #
+    def default_json
+      return @thumb_default_json unless block_given?
+      yield @thumb_default_json
+    end
+
+    # Destination URL to default thumbnail file on the server which is returned/ yielded as a `URI::HTTPS` instance.
+    def default_url
+      return URI(@thumb_default_json['url']) unless block_given?
+      yield URI(@thumb_default_json['url'])
+    end
+
+    # The file name for the default thumbnail which is returned/ yielded as `String` instance.
+    def default_filename
+      filename = default_url.path[/[A-Za-z0-9]+\.[A-Za-z0-9]+/]
+      return filename unless block_given?
+      yield filename
+    end
+
+    # The bytes contained in the default thumbnail file stored on the server returned/ yielded as `String` instance.
+    def default_bytes
+      res = get_raw(default_url.path)
+      return res.body unless block_given?
+      yield res.body
+    end
+
+    # Send a simple get request using the thumbnail session. This should be how the module sends all further requests
+    # to the `i.ytimg.com` hostname outside of this class also.
+    #
+    # @param path [String] Any valid path on the server, prefixed with `/`.
+    # @return [Net::HTTPResponse] The untouched response sent back from the request.
+    # @yield [Net::HTTPResponse] Same as return but yielded to block.
+    #
+    def get_raw(path)
+      res = get_request_path(path)
+      return res unless block_given?
+      yield res
+    end
+
+    # Use `get_raw` method it's the same, but is shorter and can yield.
+    private def get_request_path(path)
+      if path.class != "".class
+        raise InvalidPathError, 'Path on server must be type `String\''
+      end
+      if path.empty? == false and path[0] != "/"
+        raise InvalidPathError, 'Path must be prefixed with `/\''
+      end
+      return @session.get(path)
+    end
+
+    private def jump_to_in_json
+      return @extractor.video_json_untouched['microformat']['playerMicroformatRenderer']['thumbnail']['thumbnails']
+    end
+
+  end
+
+end

data/lib/youtube/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Youtube
+  VERSION = "0.1.0"
+end

data/lib/youtube-data.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'net/http'
+require 'json'
+
+require_relative 'youtube/version'
+require_relative 'youtube/extractor'
+require_relative 'youtube/thumbnail'
+
+module Youtube
+  class Error < StandardError; end
+end

data/sig/youtube.rbs

@@ -0,0 +1,4 @@
+module Youtube
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end