youtube_parser 0.1.2 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37de03ca4cf583c72ed032b32f6a69afa8e67562a85cadd753a73e8cff2b3754
-  data.tar.gz: 4c2dc703e0cf7c954bf48b2e3d5413ed15b693a9282517b7381de11f22092e52
+  metadata.gz: 6e7896f15e739e7d88e6de6acf67f07bc543897deffebe0c4e723a3746d84e32
+  data.tar.gz: bff5544c0e250bd76c9265f74eaee6d743423c7748030415d0983175b3c263ab
 SHA512:
-  metadata.gz: c0c46db11b78fdc4ac1346ea44e67fbdc05ab2417b536e1e588177c3f3d5b4e30fdf57cf523b32b554960b6d966ec8addec3d2e0dd97b35c12dcbc9b855d0a68
-  data.tar.gz: 304fb644a213c32686c5ce13162a0a5c8dd8c510cefb9d9a331bde8dd1725cdf9ac5535d9db1f0fa8d7c2dd4ad22c63c83c40bdae8c053be1dee5a280fdb3d45
+  metadata.gz: 7b6ae178f21036ff1f620982349975cd88973aa51443a59caf8efa81a0da5a4b1bbd135f00c383f0e3c672a1edd63c74d12219c346f364066bbc952f8f59a525
+  data.tar.gz: 2b30490336766753b5b18aa7f0bdc5f61eea7ac6dd13c97777466c2ec5f2f25c8df3bde616152243811882b8c2c656625a50c68989cb80f51c41ff1a2939428e

data/Gemfile

@@ -5,4 +5,4 @@ gemspec
 
 gem 'faraday'
 gem 'oj'
-gem 'activesupport'
+gem 'activesupport'

data/Gemfile.lock

@@ -1,7 +1,10 @@
 PATH
   remote: .
   specs:
-    youtube_parser (0.1.2)
+    youtube_parser (0.1.8)
+      activesupport
+      faraday
+      oj
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -29,11 +29,23 @@ options = { channel_url: 'https://www.youtube.com/channel/channel_id',
 YoutubeParser::Channel.new(options).info
 ```
 
-## Development
+To search channels by query:
+```ruby
+require 'youtube_parser'
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 '\
+    '(KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
+options = { search_query: 'search query',
+            user_agent: user_agent }
+
+# Search with pagination
+YoutubeParser::Search::Channels.new(options).for_each_channel do |channel|
+  channel
+end
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+# First page channels
+YoutubeParser::Search::Channels.new(options).info
+```
 
 ## Contributing
 

data/lib/youtube_parser.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
-require 'active_support/core_ext'
-require 'uri'
 require 'oj'
+require 'uri'
 require 'faraday'
 require 'ostruct'
 require 'yaml'
 require 'pathname'
+require 'active_support'
+require 'active_support/core_ext'
 
 require 'youtube_parser/version'
 require 'youtube_parser/resource'
@@ -15,9 +16,11 @@ require 'youtube_parser/base_parser'
 require 'youtube_parser/channel'
 require 'youtube_parser/channels/about_section'
 require 'youtube_parser/channels/videos_section'
+require 'youtube_parser/search/channels'
 
 module YoutubeParser
   def self.root
-    Pathname.new(Dir.pwd)
+    path = Gem::Specification.find_by_name('youtube_parser').gem_dir
+    Pathname.new(path)
   end
 end

data/lib/youtube_parser/channel.rb

@@ -12,6 +12,7 @@ module YoutubeParser
 
     def collect_channel_info
       data = about_section_info.merge(video_section_info)
+      data[:channel_url] = channel_url
       data.select! { |_, v| v.present? }
     end
 
@@ -23,6 +24,15 @@ module YoutubeParser
       @video_section_info ||= section(:videos).info
     end
 
+    def channel_url
+      uri = URI(client.class::BASE_URL)
+      uri.path = options.channel_url
+
+      uri.to_s
+    rescue URI::InvalidComponentError
+      options.channel_url
+    end
+
     def section(title)
       constant_name = "YoutubeParser::Channels::#{title.capitalize}Section"
       opts = { channel_url: options.channel_url, client: client }

data/lib/youtube_parser/channels/about_section.rb

@@ -41,11 +41,11 @@ module YoutubeParser
       end
 
       def tags
-        section.dig(*keys.keywords)
+        json.dig(*keys.keywords).to_s.split('"').select(&:present?)
       end
 
       def avatar_url
-        results = section.dig(*keys.avatar) || section.dig(*keys.meta_avatar)
+        results = json.dig(*keys.meta_avatar) || section.dig(*keys.avatar)
         return if results.nil? || results.empty?
 
         results.detect { |t| t&.dig('url') }&.dig('url')
@@ -59,7 +59,8 @@ module YoutubeParser
       end
 
       def followers_count
-        statistics :followers_count
+        count = scrape_subscribers_count
+        count.present? ? count : statistics(:followers_count)
       end
 
       def videos_count
@@ -73,6 +74,29 @@ module YoutubeParser
         stats[indexes[title.to_sym].to_i].to_s.gsub(/\D+/, '')
       end
 
+      def scrape_subscribers_count
+        return unless subscribers_match
+
+        number = subscribers_match[:number].gsub(',', '.').to_f
+        return number.to_i unless subscribers_match[:units].present?
+
+        subscribers_with_units number
+      end
+
+      def subscribers_with_units(number)
+        (number * units[subscribers_match[:units].downcase.to_sym]).to_i
+      end
+
+      def subscribers_match
+        regex = /(?<number>[\d.,]+)(?<units>\w?)/
+        subscribers_text = json.dig(*keys.subscribers_count_text).to_s
+        subscribers_text.match(regex)
+      end
+
+      def units
+        { m: 1_000_000, k: 1_000 }
+      end
+
       def section
         return @section if @section
 

data/lib/youtube_parser/channels/videos_section.rb

@@ -18,13 +18,16 @@ module YoutubeParser
         return @video_ids if @video_ids&.any?
 
         sections.each do |section|
-          videos = section.dig(*keys.video_section_tabs)
+          videos = section.dig(*keys.video_section_tabs) ||
+            section.dig(*keys.second_video_section)
           next unless videos.is_a? Array
 
-          @video_ids = videos.map { |v| v.dig(*keys.video_ids) }.compact
+          @video_ids = scrape_video_ids videos
 
           return @video_ids if @video_ids.any?
         end
+
+        []
       end
 
       def playlist_id
@@ -34,6 +37,10 @@ module YoutubeParser
         @playlist_id = section&.dig(*keys.playlist_id)
       end
 
+      def scrape_video_ids(videos)
+        videos.map { |video| video.dig(*keys.video_ids) }.compact
+      end
+
       def sections
         @sections ||= json.dig(*keys.section_tabs) || []
       end

data/lib/youtube_parser/client.rb

@@ -13,6 +13,8 @@ module YoutubeParser
 
     def get(endpoint, options = {})
       response = client.get(endpoint, options)
+      return {} unless response.status.eql?(200)
+
       json_str = response.body[DATA_REGEX, 1]
 
       get_json json_str

data/lib/youtube_parser/config/keys.yml

@@ -1,6 +1,8 @@
 channel_url: ['navigationEndpoint', 'browseEndpoint', 'canonicalBaseUrl']
 channel_title: ['title', 'simpleText']
 subscribers_count: ['subscriberCountText', 'simpleText']
+subscribers_count_text: ['header', 'c4TabbedHeaderRenderer',
+  'subscriberCountText']
 section_tabs: ['contents', 'twoColumnBrowseResultsRenderer', 'tabs']
 about_section_tab: ['tabRenderer', 'content', 'sectionListRenderer',
   'contents', 0, 'itemSectionRenderer', 'contents', 0,
@@ -11,6 +13,9 @@ playlist_id: ['tabRenderer', 'content', 'sectionListRenderer', 'subMenu',
   'channelSubMenuRenderer', 'playAllButton', 'buttonRenderer',
   'navigationEndpoint', 'watchPlaylistEndpoint', 'playlistId']
 video_ids: ['gridVideoRenderer', 'videoId']
+second_video_section: ['tabRenderer', 'content', 'sectionListRenderer', 'contents',
+  0, 'itemSectionRenderer', 'contents', 0, 'shelfRenderer', 'content',
+  'horizontalListRenderer', 'items']
 country: ['country', 'simpleText']
 description_first: ['description', 'simpleText']
 description_second: ['artistBio', 'simpleText']
@@ -21,4 +26,12 @@ views: ['viewCountText', 'runs']
 keywords: ['metadata', 'channelMetadataRenderer', 'keywords']
 tab_renderer: ['tabRenderer']
 title: ['title', 'simpleText']
-statistics: ['networkStatistics', 'statistics']
+statistics: ['networkStatistics', 'statistics']
+channels_page: ['contents', 'twoColumnSearchResultsRenderer', 'primaryContents',
+  'sectionListRenderer', 'contents']
+continuation_contents: ['continuationContents', 'itemSectionContinuation']
+channels_contents: ['itemSectionRenderer', 'contents']
+channel_renderer: ['channelRenderer']
+continuations: ['continuations']
+renderer_continuations: ['itemSectionRenderer', 'continuations']
+continuation: ['nextContinuationData', 'continuation']

data/lib/youtube_parser/resource.rb

@@ -13,6 +13,10 @@ module YoutubeParser
       @attributes ||= @hash.transform_keys(&:to_sym)
     end
 
+    def keys
+      @keys ||= attributes.keys
+    end
+
     def method_missing(method, *args, &block)
       super_method = super
       return super_method if super_method

data/lib/youtube_parser/search/channels.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module YoutubeParser
+  module Search
+    class Channels < BaseParser
+      DEFAULT_PARAMS = { sp: 'EgIQAg%3D%3D' }.freeze
+
+      options :search_query
+
+      def info
+        (channels(search)[:channels] || []).map { |c| channel_info c }
+      end
+
+      def for_each_channel
+        for_each_page do |channels|
+          channels.each { |channel| yield channel_info channel }
+        end
+      end
+
+      private
+
+      def for_each_page
+        channels = channels(search)
+        return if channels.empty? || channels[:channels].empty?
+
+        loop do
+          yield channels[:channels]
+
+          break unless channels[:continuation]
+
+          channels = channels(search(ctoken: channels[:continuation]))
+
+          break if channels.empty?
+        end
+      end
+
+      def search(opts = {})
+        opts = opts.merge(params)
+        client.get('results', opts)
+      end
+
+      def channels(json)
+        page_contents = json.dig(*keys.channels_page)
+        continuation_contents = json.dig(*keys.continuation_contents)
+        return {} if page_contents.nil? && continuation_contents.nil?
+
+        contents = (page_contents || [continuation_contents&.dig('contents')])
+        contents.each do |content|
+          channels = scrape_channels content
+          channels_hash = {
+            channels: channels,
+            continuation: continuation(continuation_contents, content)
+          }
+
+          return channels_hash if channels.any?
+        end
+
+        {}
+      end
+
+      def continuation(contents, content)
+        continuations = content.dig(*keys.renderer_continuations)
+        continuations = contents&.dig(*keys.continuations) if contents
+        continuations&.map do |cont|
+          cont.dig(*keys.continuation)
+        end&.compact&.first
+      end
+
+      def scrape_channels(content)
+        contents = content
+        contents = content.dig(*keys.channels_contents) if content.is_a? Hash
+        contents&.map { |c| c.dig(*keys.channel_renderer) }&.compact || []
+      end
+
+      def channel_info(channel)
+        channel_url = channel.dig(*keys.channel_url)
+        opts = { channel_url: channel_url, client: client }
+
+        YoutubeParser::Channel.new(opts).info
+      end
+
+      def params
+        options.attributes.slice(*required_options).merge(DEFAULT_PARAMS)
+      end
+    end
+  end
+end

data/lib/youtube_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module YoutubeParser
-  VERSION = '0.1.2'
+  VERSION = '0.1.8'
 end

data/youtube_parser.gemspec

@@ -27,4 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'webmock'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'faraday'
+  spec.add_dependency 'oj'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: youtube_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.8
 platform: ruby
 authors:
 - o.vykhor
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-01 00:00:00.000000000 Z
+date: 2020-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - o.vykhor@ukr.net
@@ -107,6 +149,7 @@ files:
 - lib/youtube_parser/client.rb
 - lib/youtube_parser/config/keys.yml
 - lib/youtube_parser/resource.rb
+- lib/youtube_parser/search/channels.rb
 - lib/youtube_parser/version.rb
 - youtube_parser.gemspec
 homepage: https://github.com/oleksiivykhor/youtube_parser