youtube_parser 0.1.1 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2787d7682c42086c3586dd86a774760b0ee78f314ab297cc06e669cd1c5b8167
-  data.tar.gz: 537a9cdf529e9b5760661aa75188bfc0990d82402de42776f9ebd6786394bc8c
+  metadata.gz: f903b865d02d0e7cd331a737f268a5203669cdce1b1b64ce39c31ea82f941385
+  data.tar.gz: e363de69b04ce8a78f172b71dd5e9c2c033bebae040dc7e8100b8278a6f42a51
 SHA512:
-  metadata.gz: c131bee117f67f0ad2735d6b5ef82ccd68265db53564fc98f1cbb6f090c954983c7de7470628992757be9ab50b76fa6549d511157441e689322477d7d2d4dbd2
-  data.tar.gz: b31abce91b7e2a2773f162143aabf96dd71563c24f5e4350e40750e95089491ca4f496c7d44e2a0bea85d4fd0ec51b1088a38ce472864e5d6beaec226cb63b4a
+  metadata.gz: e0860723613a734b3e84efeea7315fa59b8a778517f421c365af4381e8120f37d059529656a717339be14f491eb3b85991a8d23432f772529a95fadd19cf9c77
+  data.tar.gz: 7690928cbbbed44014abea0ed85743f5614b58d64143800efa76b614e9f6963f2d55d1599b9e7008676caf75cedabd9544edaa630dc35857491449e79f793bb3

data/Gemfile

@@ -5,4 +5,4 @@ gemspec
 
 gem 'faraday'
 gem 'oj'
-gem 'activesupport'
+gem 'activesupport'

data/Gemfile.lock

@@ -1,7 +1,10 @@
 PATH
   remote: .
   specs:
-    youtube_parser (0.1.1)
+    youtube_parser (0.1.7)
+      activesupport
+      faraday
+      oj
 
 GEM
   remote: https://rubygems.org/
@@ -12,8 +15,8 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2)
-    addressable (2.6.0)
-      public_suffix (>= 2.0.2, < 4.0)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     byebug (11.0.1)
     concurrent-ruby (1.1.5)
     crack (0.4.3)
@@ -21,13 +24,13 @@ GEM
     diff-lcs (1.3)
     faraday (0.17.1)
       multipart-post (>= 1.2, < 3)
-    hashdiff (0.3.7)
+    hashdiff (1.0.0)
     i18n (1.7.0)
       concurrent-ruby (~> 1.0)
     minitest (5.13.0)
     multipart-post (2.1.1)
     oj (3.9.2)
-    public_suffix (3.0.3)
+    public_suffix (4.0.2)
     rake (10.5.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -42,14 +45,14 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.0)
-    safe_yaml (1.0.4)
+    safe_yaml (1.0.5)
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    webmock (3.0.1)
+    webmock (3.7.6)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
-      hashdiff
+      hashdiff (>= 0.4.0, < 2.0.0)
     zeitwerk (2.2.2)
 
 PLATFORMS

data/README.md

@@ -29,11 +29,23 @@ options = { channel_url: 'https://www.youtube.com/channel/channel_id',
 YoutubeParser::Channel.new(options).info
 ```
 
-## Development
+To search channels by query:
+```ruby
+require 'youtube_parser'
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 '\
+    '(KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
+options = { search_query: 'search query',
+            user_agent: user_agent }
+
+# Search with pagination
+YoutubeParser::Search::Channels.new(options).for_each_channel do |channel|
+  channel
+end
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+# First page channels
+YoutubeParser::Search::Channels.new(options).info
+```
 
 ## Contributing
 

data/lib/youtube_parser.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext'
-require 'uri'
 require 'oj'
+require 'uri'
 require 'faraday'
 require 'ostruct'
 require 'yaml'
 require 'pathname'
+require 'active_support'
+require 'active_support/core_ext'
 
 require 'youtube_parser/version'
 require 'youtube_parser/resource'
@@ -15,9 +16,11 @@ require 'youtube_parser/base_parser'
 require 'youtube_parser/channel'
 require 'youtube_parser/channels/about_section'
 require 'youtube_parser/channels/videos_section'
+require 'youtube_parser/search/channels'
 
 module YoutubeParser
   def self.root
-    Pathname.new(Dir.pwd)
+    path = Gem::Specification.find_by_name('youtube_parser').gem_dir
+    Pathname.new(path)
   end
 end

data/lib/youtube_parser/channel.rb

@@ -12,6 +12,7 @@ module YoutubeParser
 
     def collect_channel_info
       data = about_section_info.merge(video_section_info)
+      data[:channel_url] = channel_url
       data.select! { |_, v| v.present? }
     end
 
@@ -23,6 +24,15 @@ module YoutubeParser
       @video_section_info ||= section(:videos).info
     end
 
+    def channel_url
+      uri = URI(client.class::BASE_URL)
+      uri.path = options.channel_url
+
+      uri.to_s
+    rescue URI::InvalidComponentError
+      options.channel_url
+    end
+
     def section(title)
       constant_name = "YoutubeParser::Channels::#{title.capitalize}Section"
       opts = { channel_url: options.channel_url, client: client }

data/lib/youtube_parser/channels/about_section.rb

@@ -8,6 +8,7 @@ module YoutubeParser
       def info
         {
           title: title,
+          email: email,
           description: description,
           country: country,
           tags: tags,

data/lib/youtube_parser/channels/videos_section.rb

@@ -18,13 +18,16 @@ module YoutubeParser
         return @video_ids if @video_ids&.any?
 
         sections.each do |section|
-          videos = section.dig(*keys.video_section_tabs)
+          videos = section.dig(*keys.video_section_tabs) ||
+            section.dig(*keys.second_video_section)
           next unless videos.is_a? Array
 
-          @video_ids = videos.map { |v| v.dig(*keys.video_ids) }.compact
+          @video_ids = scrape_video_ids videos
 
           return @video_ids if @video_ids.any?
         end
+
+        []
       end
 
       def playlist_id
@@ -34,6 +37,10 @@ module YoutubeParser
         @playlist_id = section&.dig(*keys.playlist_id)
       end
 
+      def scrape_video_ids(videos)
+        videos.map { |video| video.dig(*keys.video_ids) }.compact
+      end
+
       def sections
         @sections ||= json.dig(*keys.section_tabs) || []
       end

data/lib/youtube_parser/client.rb

@@ -13,6 +13,8 @@ module YoutubeParser
 
     def get(endpoint, options = {})
       response = client.get(endpoint, options)
+      return {} unless response.status.eql?(200)
+
       json_str = response.body[DATA_REGEX, 1]
 
       get_json json_str
@@ -24,6 +26,7 @@ module YoutubeParser
       opts = { ssl: { verify: false }, request: { timeout: 10 } }
       @client ||= Faraday.new(BASE_URL, opts) do |request|
         request.adapter Faraday.default_adapter
+        request.headers['Accept-Language'] = 'en-US'
         request.headers['User-Agent'] = options.user_agent if options.user_agent
         request.proxy = proxy if options.proxy
       end

data/lib/youtube_parser/config/keys.yml

@@ -11,6 +11,9 @@ playlist_id: ['tabRenderer', 'content', 'sectionListRenderer', 'subMenu',
   'channelSubMenuRenderer', 'playAllButton', 'buttonRenderer',
   'navigationEndpoint', 'watchPlaylistEndpoint', 'playlistId']
 video_ids: ['gridVideoRenderer', 'videoId']
+second_video_section: ['tabRenderer', 'content', 'sectionListRenderer', 'contents',
+  0, 'itemSectionRenderer', 'contents', 0, 'shelfRenderer', 'content',
+  'horizontalListRenderer', 'items']
 country: ['country', 'simpleText']
 description_first: ['description', 'simpleText']
 description_second: ['artistBio', 'simpleText']
@@ -21,4 +24,12 @@ views: ['viewCountText', 'runs']
 keywords: ['metadata', 'channelMetadataRenderer', 'keywords']
 tab_renderer: ['tabRenderer']
 title: ['title', 'simpleText']
-statistics: ['networkStatistics', 'statistics']
+statistics: ['networkStatistics', 'statistics']
+channels_page: ['contents', 'twoColumnSearchResultsRenderer', 'primaryContents',
+  'sectionListRenderer', 'contents']
+continuation_contents: ['continuationContents', 'itemSectionContinuation']
+channels_contents: ['itemSectionRenderer', 'contents']
+channel_renderer: ['channelRenderer']
+continuations: ['continuations']
+renderer_continuations: ['itemSectionRenderer', 'continuations']
+continuation: ['nextContinuationData', 'continuation']

data/lib/youtube_parser/resource.rb

@@ -13,6 +13,10 @@ module YoutubeParser
       @attributes ||= @hash.transform_keys(&:to_sym)
     end
 
+    def keys
+      @keys ||= attributes.keys
+    end
+
     def method_missing(method, *args, &block)
       super_method = super
       return super_method if super_method

data/lib/youtube_parser/search/channels.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module YoutubeParser
+  module Search
+    class Channels < BaseParser
+      DEFAULT_PARAMS = { sp: 'EgIQAg%3D%3D' }.freeze
+
+      options :search_query
+
+      def info
+        (channels(search)[:channels] || []).map { |c| channel_info c }
+      end
+
+      def for_each_channel
+        for_each_page do |channels|
+          channels.each { |channel| yield channel_info channel }
+        end
+      end
+
+      private
+
+      def for_each_page
+        channels = channels(search)
+        return if channels.empty? || channels[:channels].empty?
+
+        loop do
+          yield channels[:channels]
+
+          break unless channels[:continuation]
+
+          channels = channels(search(ctoken: channels[:continuation]))
+
+          break if channels.empty?
+        end
+      end
+
+      def search(opts = {})
+        opts = opts.merge(params)
+        client.get('results', opts)
+      end
+
+      def channels(json)
+        page_contents = json.dig(*keys.channels_page)
+        continuation_contents = json.dig(*keys.continuation_contents)
+        return {} if page_contents.nil? && continuation_contents.nil?
+
+        contents = (page_contents || [continuation_contents&.dig('contents')])
+        contents.each do |content|
+          channels = scrape_channels content
+          channels_hash = {
+            channels: channels,
+            continuation: continuation(continuation_contents, content)
+          }
+
+          return channels_hash if channels.any?
+        end
+
+        {}
+      end
+
+      def continuation(contents, content)
+        continuations = content.dig(*keys.renderer_continuations)
+        continuations = contents&.dig(*keys.continuations) if contents
+        continuations&.map do |cont|
+          cont.dig(*keys.continuation)
+        end&.compact&.first
+      end
+
+      def scrape_channels(content)
+        contents = content
+        contents = content.dig(*keys.channels_contents) if content.is_a? Hash
+        contents&.map { |c| c.dig(*keys.channel_renderer) }&.compact || []
+      end
+
+      def channel_info(channel)
+        channel_url = channel.dig(*keys.channel_url)
+        opts = { channel_url: channel_url, client: client }
+
+        YoutubeParser::Channel.new(opts).info
+      end
+
+      def params
+        options.attributes.slice(*required_options).merge(DEFAULT_PARAMS)
+      end
+    end
+  end
+end

data/lib/youtube_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module YoutubeParser
-  VERSION = '0.1.1'
+  VERSION = '0.1.7'
 end

data/youtube_parser.gemspec

@@ -27,4 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'webmock'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'faraday'
+  spec.add_dependency 'oj'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: youtube_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.7
 platform: ruby
 authors:
 - o.vykhor
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-01 00:00:00.000000000 Z
+date: 2020-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - o.vykhor@ukr.net
@@ -107,6 +149,7 @@ files:
 - lib/youtube_parser/client.rb
 - lib/youtube_parser/config/keys.yml
 - lib/youtube_parser/resource.rb
+- lib/youtube_parser/search/channels.rb
 - lib/youtube_parser/version.rb
 - youtube_parser.gemspec
 homepage: https://github.com/oleksiivykhor/youtube_parser