scrape_creators 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36ed3616d4e59159e3241d2a07b743ad39aa63ac4adf93e63c7f51fbd0e8f173
-  data.tar.gz: 946bcdd67f4f3e9d5b53819c5c15ace2a56a148871ed58844479349507d8ac09
+  metadata.gz: 81c09defede693eaa2d2a5096d7d9be96179e236e48ff2f49c4bce865be49773
+  data.tar.gz: d1978cf6dc4d1d3feaf1fbdb128be6aca6ed45d7269aa1b6991615b25207d509
 SHA512:
-  metadata.gz: 27340bf2224cccfc9c5b10041e844e2f202765f34faa5d632f9fd04e7e00b1ea22ccbdcba49ee367af7fad54200618fc108c9ae9b202ac14df93713892c34574
-  data.tar.gz: 8b4fa602579a360b37af6f80847fe923fa3afb6fb165af8862afd2a81d796ebec4bff0068c2ba2c2781b71d26314669b04ddc4d0d29065fc3068b0e424d9d09a
+  metadata.gz: 2db5b1d62794c71c70292bb4225e651ae84cd7558adcb8bb7db4be79b66324096defc3274f9d69c9db4fc86d6299a8b74d16c139bc125c47bca7c78a75f8dca4
+  data.tar.gz: ece660c23bb218af965016f85ca918b08c964939ecb3012ac8fb0a7b23bccf7383631975812d35230bef3170139442fd2f25be05b7a4f020c044f5a4cfb1cb97

data/README.md

@@ -7,7 +7,7 @@ A zero-dependency Ruby client for the [Scrape Creators API](https://docs.scrapec
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'scrape_creators', '0.6.0'
+gem 'scrape_creators', '0.8.0'
 ```
 
 And then execute:
@@ -85,6 +85,54 @@ info = scraper.post("https://www.instagram.com/p/DKSMEpKRd6h/", download_media:
 comments = scraper.comments("https://www.instagram.com/p/DKSMEpKRd6h/")
 ```
 
+### Comment pagination
+
+`comments` returns the first page by default. Pass `pages:` to collect several pages:
+
+```ruby
+comments = scraper.comments(video_url, source: 'instagram', pages: 5)
+```
+
+Use `comments_page` for an initial batch followed by a Load more button. It returns
+`{ comments:, cursor:, has_more: }` for Instagram, TikTok, and YouTube:
+
+```ruby
+page = scraper.comments_page(video_url, source: 'instagram')
+
+# When the user asks for more:
+if page[:has_more]
+  page = scraper.comments_page(video_url, source: 'instagram', cursor: page[:cursor])
+end
+```
+
+Pass the returned cursor unchanged. The client translates it to YouTube's
+`continuationToken` parameter and uses `cursor` for Instagram and TikTok. TikTok's
+`has_more` flag is respected; missing or unchanged cursors stop pagination.
+Multi-page collection also stops if a cursor repeats. Failed API responses raise
+`ScrapeCreators::APIError` so callers can retry without losing their place.
+
+API documentation: [Instagram](https://docs.scrapecreators.com/v2/instagram/post/comments/),
+[TikTok](https://docs.scrapecreators.com/v1/tiktok/video/comments/),
+[YouTube](https://docs.scrapecreators.com/v1/youtube/video/comments/).
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Account search
+
+```ruby
+client.search_users("sean walker", source: "instagram")
+client.search_users("sean walker", source: "tiktok")
+client.search_users("sean walker", source: "youtube")
+# => [{ handle: "seanwalker", name: "Sean Walker", avatar_url: "https://..." }]
+```
+
+Returns one page of accounts, deduplicated by handle. YouTube searches channels,
+whose handles can be passed to `posts_page` to fetch Shorts. Results without a
+usable handle are omitted. Blank queries and unsupported sources raise
+`ArgumentError`; unsuccessful API responses raise `ScrapeCreators::APIError`.
+
+Endpoints: [Instagram](https://docs.scrapecreators.com/v1/instagram/search/),
+[TikTok](https://docs.scrapecreators.com/v1/tiktok/search/users/),
+[YouTube](https://docs.scrapecreators.com/v1/youtube/search/).

data/lib/scrape_creators/client.rb

@@ -55,6 +55,38 @@ module ScrapeCreators
       end
     end
 
+    def search_users(query, source: INSTAGRAM)
+      query = query.to_s.strip
+      raise ArgumentError, 'Search query is required' if query.empty?
+
+      users = case source.to_s
+              when INSTAGRAM
+                response = get('/v1/instagram/search', query: query)
+                search_results(response).dig('data', 'users').to_a.map do |user|
+                  { handle: user['username'], name: user['full_name'], avatar_url: user['profile_pic_url'] }
+                end
+              when TIKTOK
+                response = get('/v1/tiktok/search/users', query: query, trim: true)
+                search_results(response).fetch('users', []).map do |user|
+                  { handle: user['unique_id'], name: user['nickname'], avatar_url: user.dig('avatar_medium', 'url_list', 0) }
+                end
+              when YOUTUBE
+                response = get('/v1/youtube/search', query: query, type: 'channels')
+                search_results(response).fetch('channels', []).map do |channel|
+                  { handle: channel['handle'], name: channel['title'], avatar_url: channel['thumbnail'] }
+                end
+              else
+                raise ArgumentError, "Unsupported search source: #{source}"
+              end
+
+      users.filter_map do |user|
+        handle = user[:handle].to_s.strip.delete_prefix('@')
+        next if handle.empty? || handle.match?(%r{[[:space:]/]})
+
+        user.merge(handle: handle)
+      end.uniq { |user| user[:handle].downcase }
+    end
+
     def posts(handle, options = {})
       source = (options[:source] || 'instagram').to_s
       pages = options[:pages] || options[:max_pages] || 1
@@ -105,12 +137,33 @@ module ScrapeCreators
     end
 
     def comments(url_or_code, options = {})
-      url = normalize_url(url_or_code)
-      params = { url: url }.merge(options)
-      source = options.dig(:source) || 'instagram'
-      comments_url = comments_url_for(source:)
-      res = get(comments_url, params)
-      res.is_a?(Hash) ? (res["comments"] || []) : []
+      pages = options[:pages] || options[:max_pages] || 1
+      cursor = options[:cursor]
+      seen_cursors = {}
+      all_comments = []
+
+      pages.to_i.times do
+        break if seen_cursors[cursor.to_s]
+
+        seen_cursors[cursor.to_s] = true
+        page = comments_page(url_or_code, options.merge(cursor: cursor))
+        all_comments.concat(page[:comments])
+        break unless page[:has_more]
+
+        cursor = page[:cursor]
+      end
+
+      all_comments
+    end
+
+    def comments_page(url_or_code, options = {})
+      source = (options[:source] || INSTAGRAM).to_s
+      raise ArgumentError, "Unsupported comments source: #{source}" unless [INSTAGRAM, TIKTOK, YOUTUBE].include?(source)
+
+      cursor = options[:cursor] || options['cursor']
+      params = comments_params(url_or_code, source, cursor, options)
+      response = get(comments_url_for(source: source), params)
+      comments_result(response, source, cursor)
     end
 
     def get(endpoint, params = {}, options = {})
@@ -153,6 +206,37 @@ module ScrapeCreators
 
     private
 
+    def comments_cursor_key(source)
+      source == YOUTUBE ? 'continuationToken' : 'cursor'
+    end
+
+    def comments_params(url_or_code, source, cursor, options)
+      params = options.reject { |key, _| %w[source cursor pages max_pages].include?(key.to_s) }
+      params[:url] = normalize_url(url_or_code)
+      params[comments_cursor_key(source)] = cursor unless cursor.nil? || cursor.to_s.empty?
+      params
+    end
+
+    def comments_result(response, source, cursor)
+      unless response.is_a?(Hash) && response['success'] != false && response['comments'].is_a?(Array)
+        raise APIError, 'Unable to fetch comments'
+      end
+
+      next_cursor = response[comments_cursor_key(source)]&.to_s
+      next_cursor = nil if next_cursor.to_s.empty? || next_cursor == '0'
+      has_more = !next_cursor.nil? && next_cursor != cursor.to_s
+      has_more &&= [true, 1, '1'].include?(response['has_more']) if source == TIKTOK
+      { comments: response['comments'], cursor: next_cursor, has_more: has_more }
+    end
+
+    def search_results(response)
+      unless response.is_a?(Hash) && response['success'] != false
+        raise APIError, 'Account search failed'
+      end
+
+      response
+    end
+
     def posts_cursor_param_for(source:)
       case source.to_s
       when TIKTOK then 'max_cursor'

data/lib/scrape_creators/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScrapeCreators
-  VERSION = "0.6.0"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrape_creators
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - swlkr