scrape_creators 0.4.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41069945ebf877cbd439c8bfbb318ba0114d4e9df6275aabacef16867417a7b4
-  data.tar.gz: 24d8743a67a698cfc97f974015a8ff30ab33254b8dd155e96f1d4b0283e9d9c0
+  metadata.gz: 36ed3616d4e59159e3241d2a07b743ad39aa63ac4adf93e63c7f51fbd0e8f173
+  data.tar.gz: 946bcdd67f4f3e9d5b53819c5c15ace2a56a148871ed58844479349507d8ac09
 SHA512:
-  metadata.gz: fdab6ab8877d97093743a3e52d3daa17bf33ac1c4b3a461b6ea85957e83bff213b9e27adbaa878b36e9af15858f014334aceda898beae6cb9d498d15e56e2d1f
-  data.tar.gz: ddeaf8dc9f781648485b348a3ccaeb660a2dfe47a611e37a6c0cd9283fae66179a91521c11be731faf698b8cf22683cf5a8b8b3862b198d1c148f20641011bda
+  metadata.gz: 27340bf2224cccfc9c5b10041e844e2f202765f34faa5d632f9fd04e7e00b1ea22ccbdcba49ee367af7fad54200618fc108c9ae9b202ac14df93713892c34574
+  data.tar.gz: 8b4fa602579a360b37af6f80847fe923fa3afb6fb165af8862afd2a81d796ebec4bff0068c2ba2c2781b71d26314669b04ddc4d0d29065fc3068b0e424d9d09a

data/README.md

@@ -7,7 +7,7 @@ A zero-dependency Ruby client for the [Scrape Creators API](https://docs.scrapec
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'scrape_creators', '0.2.0'
+gem 'scrape_creators', '0.6.0'
 ```
 
 And then execute:
@@ -40,6 +40,39 @@ scraper = ScrapeCreators::Client.new
 posts = scraper.posts("zuck")
 ```
 
+### Fetch TikTok Profile Videos
+
+```ruby
+posts = scraper.posts("zuck", source: 'tiktok')
+```
+
+### Fetch YouTube Channel Shorts
+
+```ruby
+posts = scraper.posts("zuck", source: 'youtube')
+```
+
+### Cursor-based pagination
+
+All three platforms support cursor-based pagination. By default `posts` returns the first page only. Pass `pages:` to fetch more (e.g. `5` for the first five pages of a creator's feed):
+
+```ruby
+posts = scraper.posts("zuck", pages: 5)                    # Instagram
+posts = scraper.posts("zuck", source: 'tiktok', pages: 5)   # TikTok
+posts = scraper.posts("zuck", source: 'youtube', pages: 5) # YouTube Shorts
+```
+
+If you need finer control, `posts_page` returns `{ items:, cursor:, has_more: }` so you can page through the full timeline yourself:
+
+```ruby
+page = scraper.posts_page("zuck", source: 'tiktok')
+loop do
+  page[:items].each { |item| puts item['desc'] }
+  break unless page[:has_more] && page[:cursor]
+  page = scraper.posts_page("zuck", source: 'tiktok', cursor: page[:cursor])
+end
+```
+
 ### Fetch Instagram Post / Reel Info
 
 ```ruby

data/lib/scrape_creators/client.rb

@@ -57,26 +57,58 @@ module ScrapeCreators
 
     def posts(handle, options = {})
       source = (options[:source] || 'instagram').to_s
-      params = { handle: handle }.merge(options.reject { |k, _| k.to_s == 'source' })
-      posts_url = posts_url_for(source: source)
-      res = get(posts_url, params)
-      key = posts_key_for(source: source)
-      res.is_a?(Hash) ? (res[key] || []) : []
+      pages = options[:pages] || options[:max_pages] || 1
+      all_items = []
+      cursor = nil
+      has_more = true
+
+      pages.to_i.times do
+        break unless has_more
+
+        page = posts_page(handle, options.merge(cursor: cursor))
+        items = Array(page[:items])
+        all_items.concat(items)
+        cursor = page[:cursor]
+        has_more = page[:has_more] && !cursor.to_s.empty? && !items.empty?
+      end
+
+      all_items
     end
 
-    def post(url_or_code, options = {})
+    def posts_page(handle, options = {})
       source = (options[:source] || 'instagram').to_s
+      params = { handle: handle }
+      cursor = options[:cursor] || options['cursor']
+      if cursor && !cursor.to_s.empty?
+        params[posts_cursor_param_for(source:)] = cursor
+      end
+      extra = options.reject { |k, _| %w[source cursor max_pages pages].include?(k.to_s) }
+      params.merge!(extra)
+
+      res = get(posts_url_for(source:), params)
+      key = posts_key_for(source:)
+      items = res.is_a?(Hash) ? (res[key] || []) : []
+      next_cursor = next_posts_cursor(res, source)
+      {
+        items: items,
+        cursor: next_cursor,
+        has_more: posts_has_more?(res, source, next_cursor)
+      }
+    end
+
+    def post(url_or_code, options = {})
       url = normalize_url(url_or_code)
-      post_url = post_url_for(source: source)
-      params = { url: url }.merge(options.reject { |k, _| k.to_s == 'source' })
+      source = options.dig(:source) || 'instagram'
+      post_url = post_url_for(source:)
+      params = { url: url }.merge(options)
       get(post_url, params)
     end
 
     def comments(url_or_code, options = {})
-      source = (options[:source] || 'instagram').to_s
       url = normalize_url(url_or_code)
-      params = { url: url }.merge(options.reject { |k, _| k.to_s == 'source' })
-      comments_url = comments_url_for(source: source)
+      params = { url: url }.merge(options)
+      source = options.dig(:source) || 'instagram'
+      comments_url = comments_url_for(source:)
       res = get(comments_url, params)
       res.is_a?(Hash) ? (res["comments"] || []) : []
     end
@@ -121,6 +153,39 @@ module ScrapeCreators
 
     private
 
+    def posts_cursor_param_for(source:)
+      case source.to_s
+      when TIKTOK then 'max_cursor'
+      when INSTAGRAM then 'next_max_id'
+      when YOUTUBE then 'continuationToken'
+      else 'cursor'
+      end
+    end
+
+    def next_posts_cursor(res, source)
+      return nil unless res.is_a?(Hash)
+
+      cursor = case source.to_s
+               when TIKTOK then res['max_cursor']
+               when INSTAGRAM then res['next_max_id']
+               when YOUTUBE then res['continuationToken'] || res['continuation_token']
+               else res['cursor']
+               end
+      return nil if cursor.nil? || cursor.to_s.empty? || cursor.to_s == '0'
+
+      cursor
+    end
+
+    def posts_has_more?(res, source, next_cursor)
+      return false unless res.is_a?(Hash)
+
+      if source.to_s == TIKTOK
+        res['has_more'].to_i == 1
+      else
+        !next_cursor.nil?
+      end
+    end
+
     def normalize_url(url_or_code)
       str = url_or_code.to_s.strip
       if str.start_with?('http://', 'https://')

data/lib/scrape_creators/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScrapeCreators
-  VERSION = "0.4.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrape_creators
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.0
 platform: ruby
 authors:
 - swlkr