scrape_creators 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1b0ec47242c7f1e3f0d551233d27f9e11be3874cc3b78a78181eaa40a21b84f
-  data.tar.gz: 5a8827a29aa9b0ae06a890bc979d9e52e8a9f94ba2b717d38ebb305b7459fcad
+  metadata.gz: c214fa7c8b7b56a2483d3432420003745f73747e8391100312168466f9f03c7b
+  data.tar.gz: 0ec20deecab206e3ba0f54707abbfee7e624485d5f7954311bdb34552b207f40
 SHA512:
-  metadata.gz: '08f33db279a55d7b432d7cc29c0b7b1ebec5b8d5938bab8c00e13484871f4fcdfe7bba2989146f0253e4c3cfe1dc86dc8110ba732cfd05ac6a0e2b08184c9108'
-  data.tar.gz: 92519f692c9d01b014c933f3a333a5494ebc86f0cc3ad0c98247b6799cb2d457220304cd11f71c7f8df3c5f277ee1ea66aaf75f05a025a27e080640ada04378e
+  metadata.gz: f15f5722980fca1ccb48a32e1d62927324ae944c81132cc3210f2a35d12209033ef244804c988a9f4b3924c28f78c9ddca63bf66e29dd0e24e9477195ad95aed
+  data.tar.gz: 20aadfbd40d3362a8d6fbbe52223fb1bb4baafdaec99406e9a9bbe477e8ab1fd40b29bf7279fdfc813a88c3126237a5bd598bf7ccf2db5c0adb133e7c79d4c66

data/README.md

@@ -50,6 +50,16 @@ info = scraper.post("https://www.instagram.com/p/DKSMEpKRd6h/", download_media:
 
 ```ruby
 comments = scraper.comments("https://www.instagram.com/p/DKSMEpKRd6h/")
+
+page = scraper.comments_page("https://www.instagram.com/p/DKSMEpKRd6h/")
+page[:comments]
+page[:cursor]
+page[:has_more]
+
+next_page = scraper.comments_page(
+  "https://www.instagram.com/p/DKSMEpKRd6h/",
+  cursor: page[:cursor]
+)
 ```
 
 ## License

data/lib/scrape_creators/client.rb

@@ -10,6 +10,7 @@ module ScrapeCreators
   class Client
     TIKTOK = 'tiktok'
     INSTAGRAM = 'instagram'
+    YOUTUBE = 'youtube'
 
     attr_reader :config, :api_key
 
@@ -19,61 +20,84 @@ module ScrapeCreators
     end
 
     def posts_url_for(source:)
-      case source
+      case source.to_s
       when TIKTOK then '/v3/tiktok/profile/videos'
       when INSTAGRAM then '/v2/instagram/user/posts'
+      when YOUTUBE then '/v1/youtube/channel/shorts'
       else '/404'
       end
     end
 
     def post_url_for(source:)
-      case source
+      case source.to_s
       when TIKTOK then '/v2/tiktok/video'
       when INSTAGRAM then '/v1/instagram/post'
+      when YOUTUBE then '/v1/youtube/video'
       else '/404'
       end
     end
 
     def comments_url_for(source:)
-      case source
+      case source.to_s
       when TIKTOK then '/v1/tiktok/video/comments'
       when INSTAGRAM then '/v2/instagram/post/comments'
+      when YOUTUBE then '/v1/youtube/video/comments'
       else '/404'
       end
     end
 
     def posts_key_for(source:)
-      case source
+      case source.to_s
       when TIKTOK then 'aweme_list'
       when INSTAGRAM then 'items'
+      when YOUTUBE then 'shorts'
       else 'items'
       end
     end
 
     def posts(handle, options = {})
-      params = { handle: }.merge(options)
-      source = options.dig(:source) || 'instagram'
-      posts_url = posts_url_for(source:)
+      source = (options[:source] || 'instagram').to_s
+      params = { handle: handle }.merge(options.reject { |k, _| k.to_s == 'source' })
+      posts_url = posts_url_for(source: source)
       res = get(posts_url, params)
-      key = posts_key_for(source:)
+      key = posts_key_for(source: source)
       res.is_a?(Hash) ? (res[key] || []) : []
     end
 
     def post(url_or_code, options = {})
+      source = (options[:source] || 'instagram').to_s
       url = normalize_url(url_or_code)
-      source = options.dig(:source) || 'instagram'
-      post_url = post_url_for(source:)
-      params = { url: url }.merge(options)
+      post_url = post_url_for(source: source)
+      params = { url: url }.merge(options.reject { |k, _| k.to_s == 'source' })
       get(post_url, params)
     end
 
     def comments(url_or_code, options = {})
+      comments_page(url_or_code, options)[:comments]
+    end
+
+    def comments_page(url_or_code, options = {})
+      source = (options[:source] || 'instagram').to_s
       url = normalize_url(url_or_code)
-      params = { url: url }.merge(options)
-      source = options.dig(:source) || 'instagram'
-      comments_url = comments_url_for(source:)
-      res = get(comments_url, params)
-      res.is_a?(Hash) ? (res["comments"] || []) : []
+      extra = options.reject { |k, _| %w[source cursor continuationToken continuation_token].include?(k.to_s) }
+      params = { url: url }.merge(extra)
+      cursor = options[:cursor] || options['cursor'] || options[:continuationToken] || options['continuationToken'] || options[:continuation_token]
+      if cursor && !cursor.to_s.empty?
+        if source == YOUTUBE
+          params[:continuationToken] = cursor
+        else
+          params[:cursor] = cursor
+        end
+      end
+
+      res = get(comments_url_for(source: source), params)
+      comments = res.is_a?(Hash) ? (res['comments'] || []) : []
+      next_cursor = next_comments_cursor(res, source)
+      {
+        comments: comments,
+        cursor: next_cursor,
+        has_more: comments_has_more?(res, source, next_cursor)
+      }
     end
 
     def get(endpoint, params = {}, options = {})
@@ -116,6 +140,29 @@ module ScrapeCreators
 
     private
 
+    def next_comments_cursor(res, source)
+      return nil unless res.is_a?(Hash)
+
+      cursor = if source == YOUTUBE
+                 res['continuationToken']
+               else
+                 res['cursor']
+               end
+      return nil if cursor.nil? || cursor.to_s.empty?
+
+      cursor
+    end
+
+    def comments_has_more?(res, source, next_cursor)
+      return false unless res.is_a?(Hash)
+
+      if source == TIKTOK
+        res['has_more'].to_i == 1
+      else
+        !next_cursor.nil?
+      end
+    end
+
     def normalize_url(url_or_code)
       str = url_or_code.to_s.strip
       if str.start_with?('http://', 'https://')

data/lib/scrape_creators/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScrapeCreators
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrape_creators
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - swlkr