pinterest-dl 1.0.3 → 2.0.0

data/lib/pinterest_dl/api.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  PIN_URL_PATTERN = %r{pinterest\.[a-z.]+/pin/}i.freeze
+
+  class << self
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield config
+      config
+    end
+
+    def logger
+      config.logger
+    end
+
+    def get_image_url(pin_url, quality: config.quality, client: Client.new(config: config))
+      validate_pin_url!(pin_url)
+      html = client.get(pin_url)
+      url = Extractors::Image.extract(html, quality: quality)
+      raise NotFoundError, "No image found at #{pin_url}" unless url
+
+      url
+    end
+
+    def get_video_url(pin_url, client: Client.new(config: config))
+      validate_pin_url!(pin_url)
+      html = client.get(pin_url)
+      url = Extractors::Video.extract(html)
+      raise NotFoundError, "No video found at #{pin_url}" unless url
+
+      url
+    end
+
+    def get_media(pin_url, quality: config.quality)
+      client = Client.new(config: config)
+      html = client.get(pin_url)
+      {
+        image_url: Extractors::Image.extract(html, quality: quality),
+        video_url: Extractors::Video.extract(html)
+      }
+    end
+
+    def download(url, path:)
+      Downloader.new(config: config).download(url, path: path)
+    end
+
+    def download_batch(urls, dir:, filename: nil, &progress)
+      Downloader.new(config: config).download_batch(urls, dir: dir, filename: filename, &progress)
+    end
+
+    def search(query, limit: 25, pages: 5)
+      Search.new(config: config).call(query, limit: limit, pages: pages)
+    end
+
+    def search_json(query, limit: 25)
+      JSON.pretty_generate(search(query, limit: limit))
+    end
+
+    def search_images(query, limit: 25)
+      search(query, limit: limit).filter_map { |p| p[:image_url] }
+    end
+
+    def search_videos(query, limit: 25)
+      search(query, limit: limit).filter_map { |p| p[:video_url] }
+    end
+
+    def board(board_url, limit: 50, pages: 10)
+      Board.new(config: config).call(board_url, limit: limit, pages: pages)
+    end
+
+    private
+
+    def validate_pin_url!(pin_url)
+      return if pin_url.to_s.match?(PIN_URL_PATTERN)
+
+      raise InvalidURLError, "Not a Pinterest pin URL: #{pin_url.inspect}"
+    end
+  end
+end

data/lib/pinterest_dl/board.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+module PinterestDL
+  # Handles Pinterest's internal search resource, including pagination
+  # via the `bookmarks` cursor Pinterest returns with each page.
+  class Search
+    RESOURCE_URL = 'https://www.pinterest.com/resource/BaseSearchResource/get/'
+
+    def initialize(client: PinterestDL::Client.new, config: PinterestDL.config)
+      @client = client
+      @config = config
+    end
+
+    # @param query [String]
+    # @param limit [Integer] max number of pins to return in total
+    # @param pages [Integer] max number of pages to fetch (pagination safety valve)
+    # @return [Array<Hash>] pin metadata hashes
+    def call(query, limit: 25, pages: 5)
+      results = []
+      bookmarks = nil
+      pages_fetched = 0
+
+      loop do
+        page = fetch_page(query, bookmarks: bookmarks)
+        pins = page[:pins]
+        results.concat(pins)
+        bookmarks = page[:bookmarks]
+        pages_fetched += 1
+
+        break if results.size >= limit
+        break if bookmarks.nil? || bookmarks.empty? || bookmarks == ['-end-']
+        break if pages_fetched >= pages
+        break if pins.empty?
+      end
+
+      results.first(limit)
+    end
+
+    private
+
+    def fetch_page(query, bookmarks: nil)
+      source_path = "/search/pins/?q=#{URI.encode_www_form_component(query)}"
+      home_uri = URI("https://www.pinterest.com#{source_path}")
+
+      home_response = @client.raw_get(home_uri.to_s)
+      session_cookies = (home_response.get_fields('set-cookie') || []).map { |c| c.split(';').first }
+      cookie_header = [@config.cookies, session_cookies.join('; ')].compact.reject(&:empty?).join('; ')
+      csrf = session_cookies.find { |c| c.start_with?('csrftoken=') }&.split('=', 2)&.last || 'undefined'
+
+      options = { query: query, scope: 'pins', page_size: 25 }
+      options[:bookmarks] = bookmarks if bookmarks && !bookmarks.empty?
+      data = { options: options, context: {} }.to_json
+
+      res_uri = URI(RESOURCE_URL)
+      res_uri.query = URI.encode_www_form('source_url' => source_path, 'data' => data)
+
+      body = @client.get(res_uri.to_s, use_cache: false, headers: {
+                           'Accept' => 'application/json, text/javascript, */*, q=0.01',
+                           'X-Requested-With' => 'XMLHttpRequest',
+                           'X-Pinterest-PWS-Handler' => 'www/search/[scope].js',
+                           'X-CSRFToken' => csrf,
+                           'Referer' => home_uri.to_s,
+                           'Cookie' => cookie_header
+                         })
+
+      json = safe_parse(body)
+      raise PinterestDL::NotFoundError, "No search results for \"#{query}\"" unless json
+
+      data_node = json.dig('resource_response', 'data') || {}
+      raw_results = data_node['results'] || []
+      next_bookmarks = json.dig('resource_response', 'bookmark') ||
+                       json.dig('resource', 'options', 'bookmarks')
+
+      { pins: raw_results.filter_map { |r| build_pin(r) }, bookmarks: Array(next_bookmarks) }
+    end
+
+    def build_pin(raw)
+      id = raw['id']
+      return nil unless id
+
+      orig_image = raw.dig('images', 'orig', 'url')
+      fallback_image = raw['images']&.values&.map { |v| v.is_a?(Hash) ? v['url'] : nil }&.compact&.last
+      video_url = raw.dig('videos', 'video_list')&.values&.first&.dig('url')
+
+      {
+        id: id,
+        pin_url: "https://www.pinterest.com/pin/#{id}/",
+        title: raw['title'] || raw['grid_title'],
+        description: raw['description'],
+        creator: raw.dig('pinner', 'username') || raw.dig('pinner', 'full_name'),
+        image_url: orig_image || fallback_image,
+        video_url: video_url
+      }
+    end
+
+    def safe_parse(body)
+      JSON.parse(body)
+    rescue JSON::ParserError, TypeError
+      nil
+    end
+  end
+end

data/lib/pinterest_dl/client.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+# Monji024
+require 'net/http'
+require 'uri'
+require 'openssl'
+
+module PinterestDL
+  class Client
+    def initialize(config: PinterestDL.config)
+      @config = config
+      @cache = {}
+      @last_request_at = nil
+    end
+
+    def get(url, headers: {}, use_cache: true)
+      cache_key = [url, headers].hash
+      return @cache[cache_key] if use_cache && @cache.key?(cache_key)
+
+      response = request_with_retries(url, headers: headers)
+      body = handle_response(response, url)
+      @cache[cache_key] = body if use_cache
+      body
+    end
+
+    def raw_get(url, headers: {})
+      response = request_with_retries(url, headers: headers)
+      handle_response(response, url, raw: true)
+      response
+    end
+
+    def clear_cache
+      @cache.clear
+    end
+
+    private
+
+    def request_with_retries(url, headers: {})
+      attempts = 0
+      begin
+        throttle!
+        uri = URI(url)
+        http = build_http(uri)
+        req = Net::HTTP::Get.new(uri, default_headers.merge(headers))
+        http.request(req)
+      rescue Timeout::Error, Errno::ECONNRESET, Errno::ECONNREFUSED,
+             SocketError, OpenSSL::SSL::SSLError, EOFError => e
+        attempts += 1
+        if attempts <= @config.max_retries
+          @config.logger.warn("request to #{url} failed (#{e.class}), retrying (#{attempts}/#{@config.max_retries})")
+          sleep(@config.retry_wait * attempts)
+          retry
+        end
+        raise PinterestDL::NetworkError.new("Failed to reach #{url}: #{e.message}", cause_error: e)
+      end
+    end
+
+    def handle_response(response, url, raw: false)
+      case response
+      when Net::HTTPSuccess
+        raw ? response : response.body
+      when Net::HTTPTooManyRequests
+        raise PinterestDL::RateLimitError.new(
+          "Rate limited while requesting #{url}",
+          retry_after: response['Retry-After']
+        )
+      when Net::HTTPNotFound
+        raise PinterestDL::NotFoundError, "Nothing found at #{url}"
+      when Net::HTTPRedirection
+        location = response['location']
+        raise PinterestDL::NotFoundError, "Redirected without a location header (#{url})" unless location
+
+        location = URI.join(url, location).to_s if location.start_with?('/')
+        get_or_raw(location, raw: raw)
+      when Net::HTTPForbidden
+        raise PinterestDL::RateLimitError, "Request to #{url} was blocked (HTTP 403)"
+      else
+        raise PinterestDL::NetworkError, "Unexpected response #{response.code} from #{url}"
+      end
+    end
+
+    def get_or_raw(url, raw:)
+      raw ? raw_get(url) : get(url)
+    end
+
+    def build_http(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == 'https')
+      http.open_timeout = @config.open_timeout
+      http.read_timeout = @config.read_timeout
+      http
+    end
+
+    def default_headers
+      headers = { 'User-Agent' => @config.user_agent }
+      headers['Cookie'] = @config.cookies if @config.authenticated?
+      headers
+    end
+
+    def throttle!
+      return if @config.rate_limit_interval <= 0
+      return unless @last_request_at
+
+      elapsed = Time.now - @last_request_at
+      wait = @config.rate_limit_interval - elapsed
+      sleep(wait) if wait.positive?
+    ensure
+      @last_request_at = Time.now
+    end
+  end
+end

data/lib/pinterest_dl/configuration.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module PinterestDL
+  class Configuration
+    DEFAULT_USER_AGENT =
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' \
+      '(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
+
+    VALID_QUALITIES = %i[originals 736x 474x 236x].freeze
+
+    attr_accessor :cookies, :user_agent, :open_timeout, :read_timeout,
+                  :max_retries, :retry_wait, :logger,
+                  :rate_limit_interval
+
+    attr_reader :quality
+
+    def initialize
+      @user_agent = DEFAULT_USER_AGENT
+      @cookies = nil
+      @open_timeout = 10
+      @read_timeout = 20
+      @max_retries = 3
+      @retry_wait = 1.0
+      @quality = :originals
+      @rate_limit_interval = 0.0
+      @logger = Logger.new($stdout).tap do |log|
+        log.level = Logger::WARN
+        log.formatter = proc { |severity, _time, _progname, msg| "[pinterest-dl] #{severity}: #{msg}\n" }
+      end
+    end
+
+    def quality=(value)
+      value = value.to_sym
+      valid = VALID_QUALITIES.join(', ')
+      raise ArgumentError, "quality must be one of: #{valid}" unless VALID_QUALITIES.include?(value)
+
+      @quality = value
+    end
+
+    def authenticated?
+      !cookies.nil? && !cookies.to_s.empty?
+    end
+  end
+end

data/lib/pinterest_dl/downloader.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module PinterestDL
+  class Downloader
+    def initialize(client: PinterestDL::Client.new, config: PinterestDL.config)
+      @client = client
+      @config = config
+    end
+
+    def download(url, path:)
+      raise ArgumentError, 'url must not be nil/empty' if url.to_s.empty?
+
+      FileUtils.mkdir_p(File.dirname(path))
+      response = @client.raw_get(url)
+      File.binwrite(path, response.body)
+      path
+    rescue PinterestDL::Error
+      raise
+    rescue StandardError => e
+      raise PinterestDL::DownloadError, "Could not download #{url} to #{path}: #{e.message}"
+    end
+
+    def download_batch(urls, dir:, filename: nil, &progress)
+      FileUtils.mkdir_p(dir)
+      succeeded = []
+      failed = []
+      total = urls.size
+
+      urls.each_with_index do |url, index|
+        name = filename ? filename.call(url, index) : default_filename(url, index)
+        path = File.join(dir, name)
+        begin
+          download(url, path: path)
+          succeeded << path
+        rescue PinterestDL::Error => e
+          @config.logger.warn("failed to download #{url}: #{e.message}")
+          failed << { url: url, error: e }
+        end
+        progress&.call(index + 1, total, url)
+      end
+
+      { succeeded: succeeded, failed: failed }
+    end
+
+    private
+
+    def default_filename(url, index)
+      ext = File.extname(URI(url).path)
+      ext = '.jpg' if ext.nil? || ext.empty?
+      format('%<index>04d%<ext>s', index: index + 1, ext: ext)
+    rescue URI::InvalidURIError
+      format('%<index>04d.jpg', index: index + 1)
+    end
+  end
+end

data/lib/pinterest_dl/errors.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  class Error < StandardError
+  end
+
+  class NetworkError < Error
+    attr_reader :cause_error
+
+    def initialize(message = 'A network error occurred', cause_error: nil)
+      @cause_error = cause_error
+      super(message)
+    end
+  end
+
+  class NotFoundError < Error
+    def initialize(message = 'The requested resource was not found')
+      super
+    end
+  end
+
+  class RateLimitError < Error
+    attr_reader :retry_after
+
+    def initialize(message = 'Rate limited by Pinterest', retry_after: nil)
+      @retry_after = retry_after
+      super(message)
+    end
+  end
+
+  class AuthenticationError < Error
+    def initialize(message = 'Authentication required (set PinterestDL.configure { |c| c.cookies = ... })')
+      super
+    end
+  end
+
+  class InvalidURLError < Error
+    def initialize(message = 'The given URL does not look like a valid Pinterest URL')
+      super
+    end
+  end
+
+  class DownloadError < Error
+    def initialize(message = 'Failed to download file')
+      super
+    end
+  end
+end

data/lib/pinterest_dl/extractors/image.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  module Extractors
+    module Image
+      QUALITY_PATH = {
+        originals: 'originals',
+        '736x': '736x',
+        '474x': '474x',
+        '236x': '236x'
+      }.freeze
+
+      module_function
+
+      def extract(html, quality: :originals)
+        url = from_json_ld(html) || from_open_graph(html) || from_raw_pattern(html)
+        return nil unless url
+
+        apply_quality(url, quality)
+      end
+
+      def from_json_ld(html)
+        PinterestDL::Extractors::JsonLd.content_url(html, 'ImageObject')
+      end
+
+      def from_open_graph(html)
+        match = html.match(/<meta[^>]*property=["']og:image["'][^>]*content=["']([^"']+)["']/)
+        match && match[1]
+      end
+
+      def from_raw_pattern(html)
+        match = html.match(%r{https://i\.pinimg\.com/(?:originals|736x|474x|236x)/[^"'\s\\]+})
+        match && match[0]
+      end
+
+      def apply_quality(url, quality)
+        return url unless url.include?('i.pinimg.com')
+
+        target = QUALITY_PATH[quality.to_sym]
+        return url unless target
+
+        url.sub(%r{/(?:originals|736x|474x|236x)/}, "/#{target}/")
+      end
+    end
+  end
+end

data/lib/pinterest_dl/extractors/video.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  module Extractors
+    # Extracts the direct video URL for a pin from raw HTML.
+    module Video
+      module_function
+
+      def extract(html)
+        url = from_json_ld(html) || from_open_graph(html) || from_content_url(html)
+        return nil unless url
+
+        unescape(url)
+      end
+
+      def from_json_ld(html)
+        PinterestDL::Extractors::JsonLd.content_url(html, 'VideoObject')
+      end
+
+      def from_open_graph(html)
+        match = html.match(/<meta[^>]*property=["']og:video["'][^>]*content=["']([^"']+)["']/)
+        match && match[1]
+      end
+
+      def from_content_url(html)
+        match = html.match(/"contentUrl"\s*:\s*"([^"]+\.mp4[^"]*)"/)
+        match && match[1]
+      end
+
+      def unescape(url)
+        url.gsub('\\u002F', '/').gsub('\\', '')
+      end
+    end
+
+    # Shared helper for both extractors: pulls a `contentUrl` out of an
+    # `application/ld+json` <script> tag matching the given @type.
+    module JsonLd
+      module_function
+
+      def content_url(html, type)
+        html.to_s.scan(%r{<script[^>]*type=["']application/ld\+json["'][^>]*>(.*?)</script>}m).each do |(raw)|
+          data = safe_parse(raw)
+          next unless data
+
+          data = data[0] if data.is_a?(Array) && !data.empty?
+          next unless data.is_a?(Hash)
+
+          return data['contentUrl'] if data['@type'] == type && data['contentUrl']
+        end
+        nil
+      end
+
+      def safe_parse(raw)
+        JSON.parse(raw)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/pinterest_dl/progress_bar.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# Monji024
+module PinterestDL
+  class ProgressBar
+    def initialize(total:, width: 30, out: $stdout)
+      @total = total.zero? ? 1 : total
+      @width = width
+      @out = out
+    end
+
+    def update(done)
+      ratio = done.to_f / @total
+      filled = (@width * ratio).round
+      bar = ('#' * filled) + ('-' * (@width - filled))
+      @out.print "\r[#{bar}] #{done}/#{@total} (#{(ratio * 100).round}%)"
+      @out.print "\n" if done >= @total
+      @out.flush
+    end
+  end
+end