pinterest-dl 1.0.1 → 2.0.0

data/lib/pinterest_dl/board.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+module PinterestDL
+  # Handles Pinterest's internal search resource, including pagination
+  # via the `bookmarks` cursor Pinterest returns with each page.
+  class Search
+    RESOURCE_URL = 'https://www.pinterest.com/resource/BaseSearchResource/get/'
+
+    def initialize(client: PinterestDL::Client.new, config: PinterestDL.config)
+      @client = client
+      @config = config
+    end
+
+    # @param query [String]
+    # @param limit [Integer] max number of pins to return in total
+    # @param pages [Integer] max number of pages to fetch (pagination safety valve)
+    # @return [Array<Hash>] pin metadata hashes
+    def call(query, limit: 25, pages: 5)
+      results = []
+      bookmarks = nil
+      pages_fetched = 0
+
+      loop do
+        page = fetch_page(query, bookmarks: bookmarks)
+        pins = page[:pins]
+        results.concat(pins)
+        bookmarks = page[:bookmarks]
+        pages_fetched += 1
+
+        break if results.size >= limit
+        break if bookmarks.nil? || bookmarks.empty? || bookmarks == ['-end-']
+        break if pages_fetched >= pages
+        break if pins.empty?
+      end
+
+      results.first(limit)
+    end
+
+    private
+
+    def fetch_page(query, bookmarks: nil)
+      source_path = "/search/pins/?q=#{URI.encode_www_form_component(query)}"
+      home_uri = URI("https://www.pinterest.com#{source_path}")
+
+      home_response = @client.raw_get(home_uri.to_s)
+      session_cookies = (home_response.get_fields('set-cookie') || []).map { |c| c.split(';').first }
+      cookie_header = [@config.cookies, session_cookies.join('; ')].compact.reject(&:empty?).join('; ')
+      csrf = session_cookies.find { |c| c.start_with?('csrftoken=') }&.split('=', 2)&.last || 'undefined'
+
+      options = { query: query, scope: 'pins', page_size: 25 }
+      options[:bookmarks] = bookmarks if bookmarks && !bookmarks.empty?
+      data = { options: options, context: {} }.to_json
+
+      res_uri = URI(RESOURCE_URL)
+      res_uri.query = URI.encode_www_form('source_url' => source_path, 'data' => data)
+
+      body = @client.get(res_uri.to_s, use_cache: false, headers: {
+                           'Accept' => 'application/json, text/javascript, */*, q=0.01',
+                           'X-Requested-With' => 'XMLHttpRequest',
+                           'X-Pinterest-PWS-Handler' => 'www/search/[scope].js',
+                           'X-CSRFToken' => csrf,
+                           'Referer' => home_uri.to_s,
+                           'Cookie' => cookie_header
+                         })
+
+      json = safe_parse(body)
+      raise PinterestDL::NotFoundError, "No search results for \"#{query}\"" unless json
+
+      data_node = json.dig('resource_response', 'data') || {}
+      raw_results = data_node['results'] || []
+      next_bookmarks = json.dig('resource_response', 'bookmark') ||
+                       json.dig('resource', 'options', 'bookmarks')
+
+      { pins: raw_results.filter_map { |r| build_pin(r) }, bookmarks: Array(next_bookmarks) }
+    end
+
+    def build_pin(raw)
+      id = raw['id']
+      return nil unless id
+
+      orig_image = raw.dig('images', 'orig', 'url')
+      fallback_image = raw['images']&.values&.map { |v| v.is_a?(Hash) ? v['url'] : nil }&.compact&.last
+      video_url = raw.dig('videos', 'video_list')&.values&.first&.dig('url')
+
+      {
+        id: id,
+        pin_url: "https://www.pinterest.com/pin/#{id}/",
+        title: raw['title'] || raw['grid_title'],
+        description: raw['description'],
+        creator: raw.dig('pinner', 'username') || raw.dig('pinner', 'full_name'),
+        image_url: orig_image || fallback_image,
+        video_url: video_url
+      }
+    end
+
+    def safe_parse(body)
+      JSON.parse(body)
+    rescue JSON::ParserError, TypeError
+      nil
+    end
+  end
+end

data/lib/pinterest_dl/client.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# Monji024
+require 'net/http'
+require 'uri'
+require 'openssl'
+
+module PinterestDL
+  class Client
+    def initialize(config: PinterestDL.config)
+      @config = config
+      @cache = {}
+      @last_request_at = nil
+    end
+
+    def get(url, headers: {}, use_cache: true)
+      cache_key = [url, headers].hash
+      return @cache[cache_key] if use_cache && @cache.key?(cache_key)
+
+      response = request_with_retries(url, headers: headers)
+      body = handle_response(response, url)
+      @cache[cache_key] = body if use_cache
+      body
+    end
+
+    def raw_get(url, headers: {})
+      response = request_with_retries(url, headers: headers)
+      handle_response(response, url, raw: true)
+      response
+    end
+
+    def clear_cache
+      @cache.clear
+    end
+
+    private
+
+    def request_with_retries(url, headers: {})
+      attempts = 0
+      begin
+        throttle!
+        uri = URI(url)
+        http = build_http(uri)
+        req = Net::HTTP::Get.new(uri, default_headers.merge(headers))
+        http.request(req)
+      rescue Timeout::Error, Errno::ECONNRESET, Errno::ECONNREFUSED,
+             SocketError, OpenSSL::SSL::SSLError, EOFError => e
+        attempts += 1
+        if attempts <= @config.max_retries
+          @config.logger.warn("request to #{url} failed (#{e.class}), retrying (#{attempts}/#{@config.max_retries})")
+          sleep(@config.retry_wait * attempts)
+          retry
+        end
+        raise PinterestDL::NetworkError.new("Failed to reach #{url}: #{e.message}", cause_error: e)
+      end
+    end
+
+    def handle_response(response, url, raw: false)
+      case response
+      when Net::HTTPSuccess
+        raw ? response : response.body
+      when Net::HTTPTooManyRequests
+        raise PinterestDL::RateLimitError.new(
+          "Rate limited while requesting #{url}",
+          retry_after: response['Retry-After']
+        )
+      when Net::HTTPNotFound
+        raise PinterestDL::NotFoundError, "Nothing found at #{url}"
+      when Net::HTTPRedirection
+        location = response['location']
+        raise PinterestDL::NotFoundError, "Redirected without a location header (#{url})" unless location
+
+        location = URI.join(url, location).to_s if location.start_with?('/')
+        get_or_raw(location, raw: raw)
+      when Net::HTTPForbidden
+        raise PinterestDL::RateLimitError, "Request to #{url} was blocked (HTTP 403)"
+      else
+        raise PinterestDL::NetworkError, "Unexpected response #{response.code} from #{url}"
+      end
+    end
+
+    def get_or_raw(url, raw:)
+      raw ? raw_get(url) : get(url)
+    end
+
+    def build_http(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == 'https')
+      http.open_timeout = @config.open_timeout
+      http.read_timeout = @config.read_timeout
+      http
+    end
+
+    def default_headers
+      headers = { 'User-Agent' => @config.user_agent }
+      headers['Cookie'] = @config.cookies if @config.authenticated?
+      headers
+    end
+
+    def throttle!
+      return if @config.rate_limit_interval <= 0
+      return unless @last_request_at
+
+      elapsed = Time.now - @last_request_at
+      wait = @config.rate_limit_interval - elapsed
+      sleep(wait) if wait.positive?
+    ensure
+      @last_request_at = Time.now
+    end
+  end
+end

data/lib/pinterest_dl/configuration.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module PinterestDL
+  class Configuration
+    DEFAULT_USER_AGENT =
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' \
+      '(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
+
+    VALID_QUALITIES = %i[originals 736x 474x 236x].freeze
+
+    attr_accessor :cookies, :user_agent, :open_timeout, :read_timeout,
+                  :max_retries, :retry_wait, :logger,
+                  :rate_limit_interval
+
+    attr_reader :quality
+
+    def initialize
+      @user_agent = DEFAULT_USER_AGENT
+      @cookies = nil
+      @open_timeout = 10
+      @read_timeout = 20
+      @max_retries = 3
+      @retry_wait = 1.0
+      @quality = :originals
+      @rate_limit_interval = 0.0
+      @logger = Logger.new($stdout).tap do |log|
+        log.level = Logger::WARN
+        log.formatter = proc { |severity, _time, _progname, msg| "[pinterest-dl] #{severity}: #{msg}\n" }
+      end
+    end
+
+    def quality=(value)
+      value = value.to_sym
+      valid = VALID_QUALITIES.join(', ')
+      raise ArgumentError, "quality must be one of: #{valid}" unless VALID_QUALITIES.include?(value)
+
+      @quality = value
+    end
+
+    def authenticated?
+      !cookies.nil? && !cookies.to_s.empty?
+    end
+  end
+end

data/lib/pinterest_dl/downloader.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module PinterestDL
+  class Downloader
+    def initialize(client: PinterestDL::Client.new, config: PinterestDL.config)
+      @client = client
+      @config = config
+    end
+
+    def download(url, path:)
+      raise ArgumentError, 'url must not be nil/empty' if url.to_s.empty?
+
+      FileUtils.mkdir_p(File.dirname(path))
+      response = @client.raw_get(url)
+      File.binwrite(path, response.body)
+      path
+    rescue PinterestDL::Error
+      raise
+    rescue StandardError => e
+      raise PinterestDL::DownloadError, "Could not download #{url} to #{path}: #{e.message}"
+    end
+
+    def download_batch(urls, dir:, filename: nil, &progress)
+      FileUtils.mkdir_p(dir)
+      succeeded = []
+      failed = []
+      total = urls.size
+
+      urls.each_with_index do |url, index|
+        name = filename ? filename.call(url, index) : default_filename(url, index)
+        path = File.join(dir, name)
+        begin
+          download(url, path: path)
+          succeeded << path
+        rescue PinterestDL::Error => e
+          @config.logger.warn("failed to download #{url}: #{e.message}")
+          failed << { url: url, error: e }
+        end
+        progress&.call(index + 1, total, url)
+      end
+
+      { succeeded: succeeded, failed: failed }
+    end
+
+    private
+
+    def default_filename(url, index)
+      ext = File.extname(URI(url).path)
+      ext = '.jpg' if ext.nil? || ext.empty?
+      format('%<index>04d%<ext>s', index: index + 1, ext: ext)
+    rescue URI::InvalidURIError
+      format('%<index>04d.jpg', index: index + 1)
+    end
+  end
+end

data/lib/pinterest_dl/errors.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  class Error < StandardError
+  end
+
+  class NetworkError < Error
+    attr_reader :cause_error
+
+    def initialize(message = 'A network error occurred', cause_error: nil)
+      @cause_error = cause_error
+      super(message)
+    end
+  end
+
+  class NotFoundError < Error
+    def initialize(message = 'The requested resource was not found')
+      super
+    end
+  end
+
+  class RateLimitError < Error
+    attr_reader :retry_after
+
+    def initialize(message = 'Rate limited by Pinterest', retry_after: nil)
+      @retry_after = retry_after
+      super(message)
+    end
+  end
+
+  class AuthenticationError < Error
+    def initialize(message = 'Authentication required (set PinterestDL.configure { |c| c.cookies = ... })')
+      super
+    end
+  end
+
+  class InvalidURLError < Error
+    def initialize(message = 'The given URL does not look like a valid Pinterest URL')
+      super
+    end
+  end
+
+  class DownloadError < Error
+    def initialize(message = 'Failed to download file')
+      super
+    end
+  end
+end

data/lib/pinterest_dl/extractors/image.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  module Extractors
+    module Image
+      QUALITY_PATH = {
+        originals: 'originals',
+        '736x': '736x',
+        '474x': '474x',
+        '236x': '236x'
+      }.freeze
+
+      module_function
+
+      def extract(html, quality: :originals)
+        url = from_json_ld(html) || from_open_graph(html) || from_raw_pattern(html)
+        return nil unless url
+
+        apply_quality(url, quality)
+      end
+
+      def from_json_ld(html)
+        PinterestDL::Extractors::JsonLd.content_url(html, 'ImageObject')
+      end
+
+      def from_open_graph(html)
+        match = html.match(/<meta[^>]*property=["']og:image["'][^>]*content=["']([^"']+)["']/)
+        match && match[1]
+      end
+
+      def from_raw_pattern(html)
+        match = html.match(%r{https://i\.pinimg\.com/(?:originals|736x|474x|236x)/[^"'\s\\]+})
+        match && match[0]
+      end
+
+      def apply_quality(url, quality)
+        return url unless url.include?('i.pinimg.com')
+
+        target = QUALITY_PATH[quality.to_sym]
+        return url unless target
+
+        url.sub(%r{/(?:originals|736x|474x|236x)/}, "/#{target}/")
+      end
+    end
+  end
+end

data/lib/pinterest_dl/extractors/video.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  module Extractors
+    # Extracts the direct video URL for a pin from raw HTML.
+    module Video
+      module_function
+
+      def extract(html)
+        url = from_json_ld(html) || from_open_graph(html) || from_content_url(html)
+        return nil unless url
+
+        unescape(url)
+      end
+
+      def from_json_ld(html)
+        PinterestDL::Extractors::JsonLd.content_url(html, 'VideoObject')
+      end
+
+      def from_open_graph(html)
+        match = html.match(/<meta[^>]*property=["']og:video["'][^>]*content=["']([^"']+)["']/)
+        match && match[1]
+      end
+
+      def from_content_url(html)
+        match = html.match(/"contentUrl"\s*:\s*"([^"]+\.mp4[^"]*)"/)
+        match && match[1]
+      end
+
+      def unescape(url)
+        url.gsub('\\u002F', '/').gsub('\\', '')
+      end
+    end
+
+    # Shared helper for both extractors: pulls a `contentUrl` out of an
+    # `application/ld+json` <script> tag matching the given @type.
+    module JsonLd
+      module_function
+
+      def content_url(html, type)
+        html.to_s.scan(%r{<script[^>]*type=["']application/ld\+json["'][^>]*>(.*?)</script>}m).each do |(raw)|
+          data = safe_parse(raw)
+          next unless data
+
+          data = data[0] if data.is_a?(Array) && !data.empty?
+          next unless data.is_a?(Hash)
+
+          return data['contentUrl'] if data['@type'] == type && data['contentUrl']
+        end
+        nil
+      end
+
+      def safe_parse(raw)
+        JSON.parse(raw)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/pinterest_dl/progress_bar.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  class ProgressBar
+    def initialize(total:, width: 30, out: $stdout)
+      @total = total.zero? ? 1 : total
+      @width = width
+      @out = out
+    end
+
+    def update(done)
+      ratio = done.to_f / @total
+      filled = (@width * ratio).round
+      bar = ('#' * filled) + ('-' * (@width - filled))
+      @out.print "\r[#{bar}] #{done}/#{@total} (#{(ratio * 100).round}%)"
+      @out.print "\n" if done >= @total
+      @out.flush
+    end
+  end
+end

data/lib/pinterest_dl/search.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+module PinterestDL
+  class Search
+    RESOURCE_URL = 'https://www.pinterest.com/resource/BaseSearchResource/get/'
+
+    def initialize(client: PinterestDL::Client.new, config: PinterestDL.config)
+      @client = client
+      @config = config
+    end
+
+    def call(query, limit: 25, pages: 5)
+      results = []
+      bookmarks = nil
+      pages_fetched = 0
+
+      loop do
+        page = fetch_page(query, bookmarks: bookmarks)
+        pins = page[:pins]
+        results.concat(pins)
+        bookmarks = page[:bookmarks]
+        pages_fetched += 1
+
+        break if results.size >= limit
+        break if bookmarks.nil? || bookmarks.empty? || bookmarks == ['-end-']
+        break if pages_fetched >= pages
+        break if pins.empty?
+      end
+
+      results.first(limit)
+    end
+
+    private
+
+    def fetch_page(query, bookmarks: nil)
+      session = establish_session(query)
+      body = @client.get(
+        build_search_url(session[:source_path], query, bookmarks),
+        use_cache: false,
+        headers: search_headers(session)
+      )
+
+      json = safe_parse(body)
+      raise PinterestDL::NotFoundError, "No search results for \"#{query}\"" unless json
+
+      parse_search_response(json)
+    end
+
+    def establish_session(query)
+      source_path = "/search/pins/?q=#{URI.encode_www_form_component(query)}"
+      home_uri = URI("https://www.pinterest.com#{source_path}")
+      home_response = @client.raw_get(home_uri.to_s)
+      cookies = (home_response.get_fields('set-cookie') || []).map { |c| c.split(';').first }
+
+      { source_path: source_path, home_uri: home_uri, cookies: cookies, csrf: csrf_token(cookies) }
+    end
+
+    def csrf_token(cookies)
+      token_cookie = cookies.find { |c| c.start_with?('csrftoken=') }
+      token_cookie ? token_cookie.split('=', 2).last : 'undefined'
+    end
+
+    def build_search_url(source_path, query, bookmarks)
+      options = { query: query, scope: 'pins', page_size: 25 }
+      options[:bookmarks] = bookmarks if bookmarks && !bookmarks.empty?
+      data = { options: options, context: {} }.to_json
+
+      res_uri = URI(RESOURCE_URL)
+      res_uri.query = URI.encode_www_form('source_url' => source_path, 'data' => data)
+      res_uri.to_s
+    end
+
+    def search_headers(session)
+      cookie_header = [@config.cookies, session[:cookies].join('; ')].compact.reject(&:empty?).join('; ')
+
+      {
+        'Accept' => 'application/json, text/javascript, */*, q=0.01',
+        'X-Requested-With' => 'XMLHttpRequest',
+        'X-Pinterest-PWS-Handler' => 'www/search/[scope].js',
+        'X-CSRFToken' => session[:csrf],
+        'Referer' => session[:home_uri].to_s,
+        'Cookie' => cookie_header
+      }
+    end
+
+    def parse_search_response(json)
+      data_node = json.dig('resource_response', 'data') || {}
+      raw_results = data_node['results'] || []
+      next_bookmarks = json.dig('resource_response', 'bookmark') ||
+                       json.dig('resource', 'options', 'bookmarks')
+
+      { pins: raw_results.filter_map { |r| build_pin(r) }, bookmarks: Array(next_bookmarks) }
+    end
+
+    def build_pin(raw)
+      id = raw['id']
+      return nil unless id
+
+      {
+        id: id,
+        pin_url: "https://www.pinterest.com/pin/#{id}/",
+        title: raw['title'] || raw['grid_title'],
+        description: raw['description'],
+        creator: pin_creator(raw),
+        image_url: pin_image_url(raw),
+        video_url: first_video_url(raw)
+      }
+    end
+
+    def pin_creator(raw)
+      raw.dig('pinner', 'username') || raw.dig('pinner', 'full_name')
+    end
+
+    def pin_image_url(raw)
+      raw.dig('images', 'orig', 'url') || fallback_image_url(raw)
+    end
+
+    def fallback_image_url(raw)
+      images = raw['images']
+      return nil unless images
+
+      images.values.filter_map { |v| v['url'] if v.is_a?(Hash) }.last
+    end
+
+    def first_video_url(raw)
+      video_list = raw.dig('videos', 'video_list')
+      return nil unless video_list
+
+      video_list.values.first&.dig('url')
+    end
+
+    def safe_parse(body)
+      JSON.parse(body)
+    rescue JSON::ParserError, TypeError
+      nil
+    end
+  end
+end

data/lib/pinterest_dl/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  VERSION = '2.0.0'
+end