scrape_creators 0.5.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c214fa7c8b7b56a2483d3432420003745f73747e8391100312168466f9f03c7b
-  data.tar.gz: 0ec20deecab206e3ba0f54707abbfee7e624485d5f7954311bdb34552b207f40
+  metadata.gz: 385c61c807a5dd2f2f1d7106b9430991d97ba68d5e894b6675d58d324243e1c3
+  data.tar.gz: 6293f02f4f4724cc2e6b1a11447e66c38ac2440db19f2b31b362a1a260757d81
 SHA512:
-  metadata.gz: f15f5722980fca1ccb48a32e1d62927324ae944c81132cc3210f2a35d12209033ef244804c988a9f4b3924c28f78c9ddca63bf66e29dd0e24e9477195ad95aed
-  data.tar.gz: 20aadfbd40d3362a8d6fbbe52223fb1bb4baafdaec99406e9a9bbe477e8ab1fd40b29bf7279fdfc813a88c3126237a5bd598bf7ccf2db5c0adb133e7c79d4c66
+  metadata.gz: 16846f8a031d0629aece90c2b59ae7f59026a4de1507f0e17a35d036b7c18efb185cdc619babf04bbbe1b81dc4b792ce186d576e58dc7cbd8bea8e33098e6d59
+  data.tar.gz: e7d80c9c3d21e7b1482776421c9f70ab6a33759ff3e8ec20d674e2b638dd146084917cb4966158b9b2451a8077ea72981f0500b767a9c8a333b407a58ea9f5c7

data/README.md

@@ -7,7 +7,7 @@ A zero-dependency Ruby client for the [Scrape Creators API](https://docs.scrapec
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'scrape_creators', '0.2.0'
+gem 'scrape_creators', '0.6.0'
 ```
 
 And then execute:
@@ -40,6 +40,39 @@ scraper = ScrapeCreators::Client.new
 posts = scraper.posts("zuck")
 ```
 
+### Fetch TikTok Profile Videos
+
+```ruby
+posts = scraper.posts("zuck", source: 'tiktok')
+```
+
+### Fetch YouTube Channel Shorts
+
+```ruby
+posts = scraper.posts("zuck", source: 'youtube')
+```
+
+### Cursor-based pagination
+
+All three platforms support cursor-based pagination. By default `posts` returns the first page only. Pass `pages:` to fetch more (e.g. `5` for the first five pages of a creator's feed):
+
+```ruby
+posts = scraper.posts("zuck", pages: 5)                    # Instagram
+posts = scraper.posts("zuck", source: 'tiktok', pages: 5)   # TikTok
+posts = scraper.posts("zuck", source: 'youtube', pages: 5) # YouTube Shorts
+```
+
+If you need finer control, `posts_page` returns `{ items:, cursor:, has_more: }` so you can page through the full timeline yourself:
+
+```ruby
+page = scraper.posts_page("zuck", source: 'tiktok')
+loop do
+  page[:items].each { |item| puts item['desc'] }
+  break unless page[:has_more] && page[:cursor]
+  page = scraper.posts_page("zuck", source: 'tiktok', cursor: page[:cursor])
+end
+```
+
 ### Fetch Instagram Post / Reel Info
 
 ```ruby
@@ -50,18 +83,26 @@ info = scraper.post("https://www.instagram.com/p/DKSMEpKRd6h/", download_media:
 
 ```ruby
 comments = scraper.comments("https://www.instagram.com/p/DKSMEpKRd6h/")
-
-page = scraper.comments_page("https://www.instagram.com/p/DKSMEpKRd6h/")
-page[:comments]
-page[:cursor]
-page[:has_more]
-
-next_page = scraper.comments_page(
-  "https://www.instagram.com/p/DKSMEpKRd6h/",
-  cursor: page[:cursor]
-)
 ```
 
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Account search
+
+```ruby
+client.search_users("sean walker", source: "instagram")
+client.search_users("sean walker", source: "tiktok")
+client.search_users("sean walker", source: "youtube")
+# => [{ handle: "seanwalker", name: "Sean Walker", avatar_url: "https://..." }]
+```
+
+Returns one page of accounts, deduplicated by handle. YouTube searches channels,
+whose handles can be passed to `posts_page` to fetch Shorts. Results without a
+usable handle are omitted. Blank queries and unsupported sources raise
+`ArgumentError`; unsuccessful API responses raise `ScrapeCreators::APIError`.
+
+Endpoints: [Instagram](https://docs.scrapecreators.com/v1/instagram/search/),
+[TikTok](https://docs.scrapecreators.com/v1/tiktok/search/users/),
+[YouTube](https://docs.scrapecreators.com/v1/youtube/search/).

data/lib/scrape_creators/client.rb

@@ -55,51 +55,96 @@ module ScrapeCreators
       end
     end
 
-    def posts(handle, options = {})
-      source = (options[:source] || 'instagram').to_s
-      params = { handle: handle }.merge(options.reject { |k, _| k.to_s == 'source' })
-      posts_url = posts_url_for(source: source)
-      res = get(posts_url, params)
-      key = posts_key_for(source: source)
-      res.is_a?(Hash) ? (res[key] || []) : []
+    def search_users(query, source: INSTAGRAM)
+      query = query.to_s.strip
+      raise ArgumentError, 'Search query is required' if query.empty?
+
+      users = case source.to_s
+              when INSTAGRAM
+                response = get('/v1/instagram/search', query: query)
+                search_results(response).dig('data', 'users').to_a.map do |user|
+                  { handle: user['username'], name: user['full_name'], avatar_url: user['profile_pic_url'] }
+                end
+              when TIKTOK
+                response = get('/v1/tiktok/search/users', query: query, trim: true)
+                search_results(response).fetch('users', []).map do |user|
+                  { handle: user['unique_id'], name: user['nickname'], avatar_url: user.dig('avatar_medium', 'url_list', 0) }
+                end
+              when YOUTUBE
+                response = get('/v1/youtube/search', query: query, type: 'channels')
+                search_results(response).fetch('channels', []).map do |channel|
+                  { handle: channel['handle'], name: channel['title'], avatar_url: channel['thumbnail'] }
+                end
+              else
+                raise ArgumentError, "Unsupported search source: #{source}"
+              end
+
+      users.filter_map do |user|
+        handle = user[:handle].to_s.strip.delete_prefix('@')
+        next if handle.empty? || handle.match?(%r{[[:space:]/]})
+
+        user.merge(handle: handle)
+      end.uniq { |user| user[:handle].downcase }
     end
 
-    def post(url_or_code, options = {})
+    def posts(handle, options = {})
       source = (options[:source] || 'instagram').to_s
-      url = normalize_url(url_or_code)
-      post_url = post_url_for(source: source)
-      params = { url: url }.merge(options.reject { |k, _| k.to_s == 'source' })
-      get(post_url, params)
-    end
+      pages = options[:pages] || options[:max_pages] || 1
+      all_items = []
+      cursor = nil
+      has_more = true
+
+      pages.to_i.times do
+        break unless has_more
+
+        page = posts_page(handle, options.merge(cursor: cursor))
+        items = Array(page[:items])
+        all_items.concat(items)
+        cursor = page[:cursor]
+        has_more = page[:has_more] && !cursor.to_s.empty? && !items.empty?
+      end
 
-    def comments(url_or_code, options = {})
-      comments_page(url_or_code, options)[:comments]
+      all_items
     end
 
-    def comments_page(url_or_code, options = {})
+    def posts_page(handle, options = {})
       source = (options[:source] || 'instagram').to_s
-      url = normalize_url(url_or_code)
-      extra = options.reject { |k, _| %w[source cursor continuationToken continuation_token].include?(k.to_s) }
-      params = { url: url }.merge(extra)
-      cursor = options[:cursor] || options['cursor'] || options[:continuationToken] || options['continuationToken'] || options[:continuation_token]
+      params = { handle: handle }
+      cursor = options[:cursor] || options['cursor']
       if cursor && !cursor.to_s.empty?
-        if source == YOUTUBE
-          params[:continuationToken] = cursor
-        else
-          params[:cursor] = cursor
-        end
+        params[posts_cursor_param_for(source:)] = cursor
       end
+      extra = options.reject { |k, _| %w[source cursor max_pages pages].include?(k.to_s) }
+      params.merge!(extra)
 
-      res = get(comments_url_for(source: source), params)
-      comments = res.is_a?(Hash) ? (res['comments'] || []) : []
-      next_cursor = next_comments_cursor(res, source)
+      res = get(posts_url_for(source:), params)
+      key = posts_key_for(source:)
+      items = res.is_a?(Hash) ? (res[key] || []) : []
+      next_cursor = next_posts_cursor(res, source)
       {
-        comments: comments,
+        items: items,
         cursor: next_cursor,
-        has_more: comments_has_more?(res, source, next_cursor)
+        has_more: posts_has_more?(res, source, next_cursor)
       }
     end
 
+    def post(url_or_code, options = {})
+      url = normalize_url(url_or_code)
+      source = options.dig(:source) || 'instagram'
+      post_url = post_url_for(source:)
+      params = { url: url }.merge(options)
+      get(post_url, params)
+    end
+
+    def comments(url_or_code, options = {})
+      url = normalize_url(url_or_code)
+      params = { url: url }.merge(options)
+      source = options.dig(:source) || 'instagram'
+      comments_url = comments_url_for(source:)
+      res = get(comments_url, params)
+      res.is_a?(Hash) ? (res["comments"] || []) : []
+    end
+
     def get(endpoint, params = {}, options = {})
       request(endpoint, method: :get, params: params, options: options)
     end
@@ -140,23 +185,41 @@ module ScrapeCreators
 
     private
 
-    def next_comments_cursor(res, source)
+    def search_results(response)
+      unless response.is_a?(Hash) && response['success'] != false
+        raise APIError, 'Account search failed'
+      end
+
+      response
+    end
+
+    def posts_cursor_param_for(source:)
+      case source.to_s
+      when TIKTOK then 'max_cursor'
+      when INSTAGRAM then 'next_max_id'
+      when YOUTUBE then 'continuationToken'
+      else 'cursor'
+      end
+    end
+
+    def next_posts_cursor(res, source)
       return nil unless res.is_a?(Hash)
 
-      cursor = if source == YOUTUBE
-                 res['continuationToken']
-               else
-                 res['cursor']
+      cursor = case source.to_s
+               when TIKTOK then res['max_cursor']
+               when INSTAGRAM then res['next_max_id']
+               when YOUTUBE then res['continuationToken'] || res['continuation_token']
+               else res['cursor']
                end
-      return nil if cursor.nil? || cursor.to_s.empty?
+      return nil if cursor.nil? || cursor.to_s.empty? || cursor.to_s == '0'
 
       cursor
     end
 
-    def comments_has_more?(res, source, next_cursor)
+    def posts_has_more?(res, source, next_cursor)
       return false unless res.is_a?(Hash)
 
-      if source == TIKTOK
+      if source.to_s == TIKTOK
         res['has_more'].to_i == 1
       else
         !next_cursor.nil?

data/lib/scrape_creators/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScrapeCreators
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrape_creators
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.0
 platform: ruby
 authors:
 - swlkr