youtube_parser 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37de03ca4cf583c72ed032b32f6a69afa8e67562a85cadd753a73e8cff2b3754
-  data.tar.gz: 4c2dc703e0cf7c954bf48b2e3d5413ed15b693a9282517b7381de11f22092e52
+  metadata.gz: 6d80fe7f501434dd15d6eb673f3ceb0eee764c81ff5d73a2916890400544bdf4
+  data.tar.gz: 423630f4671d05371789bdd06eab593f3eb30bf7d25fff7d67855fa6d2c65888
 SHA512:
-  metadata.gz: c0c46db11b78fdc4ac1346ea44e67fbdc05ab2417b536e1e588177c3f3d5b4e30fdf57cf523b32b554960b6d966ec8addec3d2e0dd97b35c12dcbc9b855d0a68
-  data.tar.gz: 304fb644a213c32686c5ce13162a0a5c8dd8c510cefb9d9a331bde8dd1725cdf9ac5535d9db1f0fa8d7c2dd4ad22c63c83c40bdae8c053be1dee5a280fdb3d45
+  metadata.gz: 8c90751d67f0ecb0493bb8a204dac01dade93b3deb62e726c4ce6597fa956e9c95478e04ab260c84f20a3d121c1c145f5696cbc2874229a394d636fe9244c14c
+  data.tar.gz: 115d0da5107925986761490852856e452f3e519070a68223538de049e7f4c469aea668f580fcc4ef20da0db62dbc0347ba6dca7494fa830adad82a7d48be7646

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    youtube_parser (0.1.2)
+    youtube_parser (0.1.3)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -29,11 +29,23 @@ options = { channel_url: 'https://www.youtube.com/channel/channel_id',
 YoutubeParser::Channel.new(options).info
 ```
 
-## Development
+To search channels by query:
+```ruby
+require 'youtube_parser'
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 '\
+    '(KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
+options = { search_query: 'search query',
+            user_agent: user_agent }
+
+# Search with pagination
+YoutubeParser::Search::Channels.new(options).for_each_channels do |channel|
+  channel
+end
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+# First page channels
+YoutubeParser::Search::Channels.new(options).info
+```
 
 ## Contributing
 

data/lib/youtube_parser.rb

@@ -15,6 +15,7 @@ require 'youtube_parser/base_parser'
 require 'youtube_parser/channel'
 require 'youtube_parser/channels/about_section'
 require 'youtube_parser/channels/videos_section'
+require 'youtube_parser/search/channels'
 
 module YoutubeParser
   def self.root

data/lib/youtube_parser/channel.rb

@@ -12,6 +12,7 @@ module YoutubeParser
 
     def collect_channel_info
       data = about_section_info.merge(video_section_info)
+      data[:channel_url] = channel_url
       data.select! { |_, v| v.present? }
     end
 
@@ -23,6 +24,15 @@ module YoutubeParser
       @video_section_info ||= section(:videos).info
     end
 
+    def channel_url
+      uri = URI(client.class::BASE_URL)
+      uri.path = options.channel_url
+
+      uri.to_s
+    rescue URI::InvalidComponentError
+      options.channel_url
+    end
+
     def section(title)
       constant_name = "YoutubeParser::Channels::#{title.capitalize}Section"
       opts = { channel_url: options.channel_url, client: client }

data/lib/youtube_parser/channels/videos_section.rb

@@ -18,13 +18,16 @@ module YoutubeParser
         return @video_ids if @video_ids&.any?
 
         sections.each do |section|
-          videos = section.dig(*keys.video_section_tabs)
+          videos = section.dig(*keys.video_section_tabs) ||
+            section.dig(*keys.second_video_section)
           next unless videos.is_a? Array
 
-          @video_ids = videos.map { |v| v.dig(*keys.video_ids) }.compact
+          @video_ids = scrape_video_ids videos
 
           return @video_ids if @video_ids.any?
         end
+
+        []
       end
 
       def playlist_id
@@ -34,6 +37,10 @@ module YoutubeParser
         @playlist_id = section&.dig(*keys.playlist_id)
       end
 
+      def scrape_video_ids(videos)
+        videos.map { |video| video.dig(*keys.video_ids) }.compact
+      end
+
       def sections
         @sections ||= json.dig(*keys.section_tabs) || []
       end

data/lib/youtube_parser/client.rb

@@ -13,6 +13,8 @@ module YoutubeParser
 
     def get(endpoint, options = {})
       response = client.get(endpoint, options)
+      return {} unless response.status.eql?(200)
+
       json_str = response.body[DATA_REGEX, 1]
 
       get_json json_str

data/lib/youtube_parser/config/keys.yml

@@ -11,6 +11,9 @@ playlist_id: ['tabRenderer', 'content', 'sectionListRenderer', 'subMenu',
   'channelSubMenuRenderer', 'playAllButton', 'buttonRenderer',
   'navigationEndpoint', 'watchPlaylistEndpoint', 'playlistId']
 video_ids: ['gridVideoRenderer', 'videoId']
+second_video_section: ['tabRenderer', 'content', 'sectionListRenderer', 'contents',
+  0, 'itemSectionRenderer', 'contents', 0, 'shelfRenderer', 'content',
+  'horizontalListRenderer', 'items']
 country: ['country', 'simpleText']
 description_first: ['description', 'simpleText']
 description_second: ['artistBio', 'simpleText']
@@ -21,4 +24,12 @@ views: ['viewCountText', 'runs']
 keywords: ['metadata', 'channelMetadataRenderer', 'keywords']
 tab_renderer: ['tabRenderer']
 title: ['title', 'simpleText']
-statistics: ['networkStatistics', 'statistics']
+statistics: ['networkStatistics', 'statistics']
+channels_page: ['contents', 'twoColumnSearchResultsRenderer', 'primaryContents',
+  'sectionListRenderer', 'contents']
+continuation_contents: ['continuationContents', 'itemSectionContinuation']
+channels_contents: ['itemSectionRenderer', 'contents']
+channel_renderer: ['channelRenderer']
+continuations: ['continuations']
+renderer_continuations: ['itemSectionRenderer', 'continuations']
+continuation: ['nextContinuationData', 'continuation']

data/lib/youtube_parser/resource.rb

@@ -13,6 +13,10 @@ module YoutubeParser
       @attributes ||= @hash.transform_keys(&:to_sym)
     end
 
+    def keys
+      @keys ||= attributes.keys
+    end
+
     def method_missing(method, *args, &block)
       super_method = super
       return super_method if super_method

data/lib/youtube_parser/search/channels.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module YoutubeParser
+  module Search
+    class Channels < BaseParser
+      DEFAULT_PARAMS = { sp: 'EgIQAg%3D%3D' }.freeze
+
+      options :search_query
+
+      def info
+        (channels(search)[:channels] || []).map { |c| channel_info c }
+      end
+
+      def for_each_channel
+        for_each_page do |channels|
+          channels.each { |channel| yield channel_info channel }
+        end
+      end
+
+      private
+
+      def for_each_page
+        channels = channels(search)
+        return if channels.empty? || channels[:channels].empty?
+
+        loop do
+          yield channels[:channels]
+
+          break unless channels[:continuation]
+
+          channels = channels(search(ctoken: channels[:continuation]))
+
+          break if channels.empty?
+        end
+      end
+
+      def search(opts = {})
+        opts = opts.merge(params)
+        client.get('results', opts)
+      end
+
+      def channels(json)
+        page_contents = json.dig(*keys.channels_page)
+        continuation_contents = json.dig(*keys.continuation_contents)
+        return {} if page_contents.nil? && continuation_contents.nil?
+
+        contents = (page_contents || [continuation_contents&.dig('contents')])
+        contents.each do |content|
+          channels = scrape_channels content
+          channels_hash = {
+            channels: channels,
+            continuation: continuation(continuation_contents, content)
+          }
+
+          return channels_hash if channels.any?
+        end
+
+        {}
+      end
+
+      def continuation(contents, content)
+        continuations = content.dig(*keys.renderer_continuations)
+        continuations = contents&.dig(*keys.continuations) if contents
+        continuations&.map do |cont|
+          cont.dig(*keys.continuation)
+        end&.compact&.first
+      end
+
+      def scrape_channels(content)
+        contents = content
+        contents = content.dig(*keys.channels_contents) if content.is_a? Hash
+        contents&.map { |c| c.dig(*keys.channel_renderer) }&.compact || []
+      end
+
+      def channel_info(channel)
+        channel_url = channel.dig(*keys.channel_url)
+        opts = { channel_url: channel_url, client: client }
+
+        YoutubeParser::Channel.new(opts).info
+      end
+
+      def params
+        options.attributes.slice(*required_options).merge(DEFAULT_PARAMS)
+      end
+    end
+  end
+end

data/lib/youtube_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module YoutubeParser
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: youtube_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - o.vykhor
@@ -107,6 +107,7 @@ files:
 - lib/youtube_parser/client.rb
 - lib/youtube_parser/config/keys.yml
 - lib/youtube_parser/resource.rb
+- lib/youtube_parser/search/channels.rb
 - lib/youtube_parser/version.rb
 - youtube_parser.gemspec
 homepage: https://github.com/oleksiivykhor/youtube_parser