keeper_secrets_manager 17.2.1

data/lib/keeper_secrets_manager/core.rb

@@ -0,0 +1,1781 @@
+require 'net/http'
+require 'uri'
+require 'json'
+require 'logger'
+require 'openssl'
+
+module KeeperSecretsManager
+  module Core
+    class SecretsManager
+      attr_reader :config, :hostname, :verify_ssl_certs
+
+      NOTATION_PREFIX = 'keeper'.freeze
+      DEFAULT_KEY_ID = '7'.freeze
+
+      # Throttle retry (KSM-876 / KSM-883). The backend throttles HTTP 403 {"error":"throttled"}
+      # per clientId+endpoint (100 requests / 10s window; memcached TTL 10s that resets on every
+      # request, so the counter only clears after 10s of silence).
+      MAX_THROTTLE_RETRIES = 5
+      BASE_THROTTLE_DELAY_SEC = 11  # 1s safety margin over the backend's 10s memcached TTL
+      MAX_THROTTLE_DELAY_SEC = 176  # cap on server-supplied retry_after (BASE * 2**4)
+
+      # Field types that can be inflated
+      INFLATE_REF_TYPES = {
+        'addressRef' => ['address'],
+        'cardRef' => %w[paymentCard text pinCode addressRef]
+      }.freeze
+
+      def initialize(options = {})
+        # Check Ruby version
+        raise Error, 'KSM SDK requires Ruby 3.1 or greater' if RUBY_VERSION < '3.1'
+
+        # Check AES-GCM support
+        begin
+          OpenSSL::Cipher.new('AES-256-GCM')
+        rescue RuntimeError => e
+          if e.message.include?('unsupported cipher')
+            raise Error,
+                  "KSM SDK requires AES-GCM support. Your Ruby/OpenSSL version (#{OpenSSL::OPENSSL_LIBRARY_VERSION}) does not support AES-256-GCM. Please upgrade to Ruby 2.7+ or use a Ruby compiled with OpenSSL 1.1.0+"
+          end
+
+          raise e
+        end
+
+        @token = nil
+        @hostname = nil
+        @verify_ssl_certs = options.fetch(:verify_ssl_certs, true)
+        @custom_post_function = options[:custom_post_function]
+
+        # optional custom server public key overrides (isolated deployments).
+        # Precedence, highest first: these programmatic params > OTT segments > pre-existing config.
+        @server_public_key_override    = options[:server_public_key]
+        @server_public_key_id_override = options[:server_public_key_id]
+
+        # Set up proxy configuration
+        # Priority: explicit proxy_url parameter > HTTPS_PROXY env var > no proxy
+        @proxy_url = options[:proxy_url] || ENV['HTTPS_PROXY'] || ENV['https_proxy']
+
+        if @proxy_url
+          begin
+            proxy_uri = URI.parse(@proxy_url)
+            unless proxy_uri.is_a?(URI::HTTP) && !proxy_uri.host.to_s.empty?
+              raise ArgumentError,
+                    "Invalid proxy_url '#{@proxy_url}': must be a valid http or https URL with a host (e.g., http://proxy.example.com:8080)"
+            end
+          rescue URI::InvalidURIError => e
+            raise ArgumentError, "Invalid proxy_url '#{@proxy_url}': #{e.message}"
+          end
+        end
+
+        # Set up logging
+        @logger = options[:logger] || Logger.new(STDOUT)
+        @logger.level = options[:log_level] || Logger::WARN
+
+        @logger.debug("Proxy configuration: #{@proxy_url ? @proxy_url : 'none'}") if @proxy_url
+
+        # Handle configuration
+        config = options[:config]
+        token = options[:token]
+
+        # Check environment variable if no config provided
+        config = Storage::InMemoryStorage.new(ENV['KSM_CONFIG']) if config.nil? && ENV['KSM_CONFIG']
+
+        # If we have config, check if it's already initialized
+        if config
+          @config = config
+          # Check if already bound (has client ID and app key)
+          if @config.get_string(ConfigKeys::KEY_CLIENT_ID) && @config.get_bytes(ConfigKeys::KEY_APP_KEY)
+            @logger.debug('Using existing credentials from config')
+          elsif token
+            # Config exists but not bound, use token to bind
+            @logger.debug('Config provided but not bound, using token to initialize')
+            process_token_binding(token, options[:hostname])
+          else
+            @logger.warn('Config provided but no credentials found and no token provided')
+          end
+        elsif token
+          # No config provided, create new one with token
+          @logger.debug('No config provided, creating new one with token')
+          process_token_binding(token, options[:hostname])
+          @config ||= Storage::InMemoryStorage.new
+        else
+          # No config and no token
+          raise Error, 'Either token or initialized config must be provided'
+        end
+
+        # programmatic custom key wins over token/config (applied once @config is set).
+        @config.save_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY, @server_public_key_override) if @server_public_key_override && !@server_public_key_override.empty?
+        @config.save_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY_ID, @server_public_key_id_override.to_s) if @server_public_key_id_override
+
+        # if the configured key id is not in the built-in table AND no custom key backs it,
+        # fall back to the default (mirrors Python). A persisted custom serverPublicKey is preserved.
+        current_key_id = @config.get_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY_ID)
+        custom_server_key = @config.get_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY)
+        if current_key_id && !KeeperGlobals::KEEPER_PUBLIC_KEYS.key?(current_key_id) && (custom_server_key.nil? || custom_server_key.empty?)
+          @logger.debug("Public key id #{current_key_id} unknown and no custom key present; using default #{DEFAULT_KEY_ID}")
+          @config.save_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY_ID, DEFAULT_KEY_ID)
+        end
+
+        # Override hostname if provided
+        if options[:hostname]
+          @hostname = options[:hostname]
+          @config.save_string(ConfigKeys::KEY_HOSTNAME, @hostname)
+        else
+          @hostname = @config.get_string(ConfigKeys::KEY_HOSTNAME) || KeeperGlobals::DEFAULT_SERVER
+        end
+
+      end
+
+      # Get secrets with optional filtering
+      def get_secrets(uids = nil, full_response: false, request_links: false)
+        uids = [uids] if uids.is_a?(String)
+
+        query_options = Dto::QueryOptions.new(records: uids, folders: nil, request_links: request_links)
+        get_secrets_with_options(query_options, full_response: full_response)
+      end
+
+      # Get secrets with query options
+      def get_secrets_with_options(query_options = nil, full_response: false)
+        records_resp = fetch_and_decrypt_secrets(query_options)
+
+        # If just bound, fetch again
+        records_resp = fetch_and_decrypt_secrets(query_options) if records_resp.just_bound
+
+        # Log warnings
+        records_resp.warnings&.each { |warning| @logger.warn(warning) }
+
+        # Log bad records/folders
+        if records_resp.errors&.any?
+          records_resp.errors.each do |error|
+            @logger.error("Error: #{error}")
+          end
+        end
+
+        full_response ? records_resp : (records_resp.records || [])
+      end
+
+      # Get all folders
+      def get_folders
+        fetch_and_decrypt_folders
+      end
+
+      # Fetch and decrypt folders from dedicated endpoint
+      def fetch_and_decrypt_folders
+        # Prepare payload for get_folders endpoint (no filters)
+        payload = prepare_get_payload(nil)
+
+        # Make request to get_folders endpoint
+        response_json = post_query('get_folders', payload)
+        response_dict = JSON.parse(response_json)
+
+        # Get app key for decryption
+        app_key_str = @config.get_string(ConfigKeys::KEY_APP_KEY)
+
+        # If we have app key directly (one-time token binding), use it
+        if app_key_str && !app_key_str.empty?
+          app_key = Utils.base64_to_bytes(app_key_str)
+        else
+          # Otherwise decrypt it using client key
+          app_key_encrypted = Utils.base64_to_bytes(@config.get_string(ConfigKeys::KEY_ENCRYPTED_APP_KEY))
+          client_key = Utils.base64_to_bytes(@config.get_string(ConfigKeys::KEY_CLIENT_KEY))
+          app_key = Crypto.decrypt_aes_gcm(app_key_encrypted, client_key)
+        end
+
+        # Decrypt folders - need to handle them in order for shared folder keys
+        folders = []
+        response_folders = response_dict['folders'] || []
+
+        response_folders.each do |encrypted_folder|
+          folder_uid = encrypted_folder['folderUid']
+          folder_parent = encrypted_folder['parent']
+
+          # Decrypt folder key based on whether it has a parent
+          if !folder_parent || folder_parent.empty?
+            # Root folder - decrypt with app key
+            folder_key_encrypted = Utils.base64_to_bytes(encrypted_folder['folderKey'])
+            folder_key = Crypto.decrypt_aes_gcm(folder_key_encrypted, app_key)
+          else
+            # Child folder - decrypt with parent's shared folder key
+            shared_folder_key = get_shared_folder_key(folders, response_folders, folder_parent)
+            unless shared_folder_key
+              @logger.error("Cannot find shared folder key for parent #{folder_parent}")
+              next
+            end
+            folder_key_encrypted = Utils.base64_to_bytes(encrypted_folder['folderKey'])
+            folder_key = Crypto.decrypt_aes_cbc(folder_key_encrypted, shared_folder_key)
+          end
+
+          # Decrypt folder data if present
+          folder_name = ''
+          if encrypted_folder['data'] && !encrypted_folder['data'].empty?
+            data_encrypted = Utils.base64_to_bytes(encrypted_folder['data'])
+            data_json = Crypto.decrypt_aes_cbc(data_encrypted, folder_key)
+            data = JSON.parse(data_json)
+            folder_name = data['name'] || ''
+          end
+
+          # Create folder object
+          folder = Dto::KeeperFolder.new(
+            'folderUid' => folder_uid,
+            'name' => folder_name,
+            'folderKey' => folder_key,
+            'parent' => folder_parent,
+            'records' => []
+          )
+
+          folders << folder
+        rescue StandardError => e
+          @logger.error("Failed to decrypt folder #{encrypted_folder['folderUid']}: #{e.message}")
+        end
+
+        folders
+      end
+
+      # Get secrets by title
+      def get_secrets_by_title(title)
+        records = get_secrets
+        records.select { |r| r.title == title }
+      end
+
+      # Get first secret by title
+      def get_secret_by_title(title)
+        get_secrets_by_title(title).first
+      end
+
+      def create_secret_with_options(create_options, record_data, folders: nil)
+        raise ArgumentError, 'folder_uid is required to create a record' unless create_options&.folder_uid
+
+        folders ||= get_folders
+
+        folder = folders.find { |f| f.uid == create_options.folder_uid }
+        raise Error, "Folder #{create_options.folder_uid} not found or not accessible" unless folder
+
+        folder_key = folder.folder_key
+        raise Error, "Unable to create record - folder key for #{create_options.folder_uid} is missing" unless folder_key
+
+        record_uid = Utils.generate_uid
+        record_key = Crypto.generate_encryption_key_bytes
+
+        record = record_data.is_a?(Dto::KeeperRecord) ? record_data.to_h : record_data
+
+        encrypted_data = Crypto.encrypt_aes_gcm(Utils.dict_to_json(record), record_key)
+
+        payload = prepare_create_payload(
+          record_uid:    record_uid,
+          record_key:    record_key,
+          folder_uid:    create_options.folder_uid,
+          folder_key:    folder_key,
+          data:          encrypted_data,
+          subfolder_uid: create_options.subfolder_uid
+        )
+
+        post_query('create_secret', payload)
+        record_uid
+      end
+
+      def create_secret(record_data, options = nil)
+        create_options = if options.is_a?(Dto::CreateOptions)
+                           options
+                         elsif options.is_a?(String)
+                           Dto::CreateOptions.new(folder_uid: options)
+                         else
+                           Dto::CreateOptions.new
+                         end
+        create_secret_with_options(create_options, record_data)
+      end
+
+      # Update existing secret with UpdateOptions
+      def update_secret_with_options(record, update_options = nil)
+        # Handle both record object and hash
+        if record.is_a?(Dto::KeeperRecord)
+          record_uid = record.uid
+          record_data = record.to_h
+        else
+          record_uid = record['uid'] || record[:uid]
+          record_data = record
+        end
+
+        raise ArgumentError, 'Record UID is required' unless record_uid
+
+        # Get existing record to get the key
+        existing = get_secrets([record_uid]).first
+        raise RecordNotFoundError, "Record #{record_uid} not found" unless existing
+
+        # Get record key for encryption
+        record_key = existing.record_key
+        raise Error, "Record key not available for #{record_uid}" unless record_key
+
+        # Prepare payload (handles UpdateOptions internally)
+        payload = prepare_update_payload(
+          record_uid: record_uid,
+          record_data: record_data,
+          record_key: record_key,
+          revision: existing.revision,
+          update_options: update_options
+        )
+
+        # Send request
+        post_query('update_secret', payload)
+      end
+
+      # Update existing secret (convenience wrapper)
+      def update_secret(record, transaction_type: 'general')
+        update_options = Dto::UpdateOptions.new(transaction_type: transaction_type)
+        update_secret_with_options(record, update_options)
+
+        uid = record.is_a?(Dto::KeeperRecord) ? record.uid : (record['uid'] || record[:uid])
+        complete_transaction(uid) if uid
+
+        if record.is_a?(Dto::KeeperRecord)
+          updated_record = get_secrets([record.uid]).first
+          if updated_record
+            record.revision = updated_record.revision
+            @logger&.debug("update_secret: updated local revision to #{record.revision}")
+          end
+        end
+
+        true
+      end
+
+      def save_with_options(record, update_options = nil)
+        if record.is_a?(Dto::KeeperRecord)
+          record_uid  = record.uid
+          record_data = record.to_h
+          record_key  = record.record_key
+          revision    = record.revision
+        else
+          record_uid  = record['uid'] || record[:uid]
+          record_data = record
+          record_key  = record['record_key'] || record[:record_key]
+          revision    = record['revision']   || record[:revision] || 0
+        end
+
+        raise ArgumentError, 'Record UID is required' unless record_uid
+        raise Error, "Record key not available for #{record_uid} - record must be obtained via get_secrets" unless record_key
+
+        payload = prepare_update_payload(
+          record_uid:     record_uid,
+          record_data:    record_data,
+          record_key:     record_key,
+          revision:       revision,
+          update_options: update_options
+        )
+
+        post_query('update_secret', payload)
+        true
+      end
+
+      def save(record, transaction_type: nil, links_to_remove: nil)
+        update_options = Dto::UpdateOptions.new(transaction_type: transaction_type, links_to_remove: links_to_remove)
+        save_with_options(record, update_options)
+      end
+
+      # Complete transaction - commit or rollback
+      # Used after update_secret with transaction_type to finalize PAM rotation
+      def complete_transaction(record_uid, rollback: false)
+        @logger.debug("Completing transaction for record #{record_uid}, rollback: #{rollback}")
+
+        # Prepare payload
+        payload = prepare_complete_transaction_payload(record_uid)
+
+        # Route to different endpoints based on rollback parameter
+        endpoint = rollback ? 'rollback_secret_update' : 'finalize_secret_update'
+
+        # Send request
+        post_query(endpoint, payload)
+
+        true
+      end
+
+      # Delete secrets
+      def delete_secret(record_uids)
+        record_uids = [record_uids] if record_uids.is_a?(String)
+
+        payload = prepare_delete_payload(record_uids)
+        response = post_query('delete_secret', payload)
+
+        result = JSON.parse(response)
+        records = result['records'] || []
+        records.each do |r|
+          next if r['responseCode'] == 'ok'
+          @logger.error("Failed to delete record #{r['recordUid']}: #{r['responseCode']} #{r['errorMessage']}")
+        end
+        records
+      end
+
+      def get_inflate_ref_types(field_type)
+        INFLATE_REF_TYPES.fetch(field_type, [])
+      end
+
+      def inflate_field_value(uids, replace_fields)
+        records = get_secrets(uids)
+        lookup  = records.each_with_object({}) { |r, h| h[r.uid] = r }
+
+        uids.filter_map do |uid|
+          record    = lookup[uid]
+          next nil unless record
+
+          new_value = nil
+          replace_fields.each do |field_type|
+            field = record.get_field(field_type)
+            next unless field
+
+            raw_values = field['value'] || []
+            next if raw_values.empty?
+
+            real_value = raw_values.first
+
+            if INFLATE_REF_TYPES.key?(field_type)
+              inflated   = inflate_field_value([real_value], INFLATE_REF_TYPES[field_type])
+              real_value = inflated.first unless inflated.empty?
+            end
+
+            label = field['label'] || field_type
+            if new_value.nil?
+              new_value = real_value
+            elsif new_value.is_a?(Hash) && real_value.is_a?(Hash)
+              new_value[label] = real_value
+            else
+              new_value = { label => new_value, field_type => real_value }
+            end
+          end
+
+          new_value
+        end
+      end
+
+      # Get notation value
+      def get_notation(notation_uri)
+        parser = Notation::Parser.new(self)
+        parser.parse(notation_uri)
+      end
+
+      def get_notation_results(notation_uri)
+        Notation::Parser.new(self).get_notation_results(notation_uri)
+      end
+
+      # Like get_notation_results but never raises; logs errors and returns [].
+      def try_get_notation_results(notation_uri)
+        get_notation_results(notation_uri)
+      rescue NotationError, RecordNotFoundError, StandardError => e
+        @logger.error("try_get_notation_results failed for '#{notation_uri}': #{e.message}")
+        []
+      end
+
+      # Get notation value without raising exceptions (convenience method)
+      # Returns empty array if notation is invalid or record not found
+      def try_get_notation(notation_uri)
+        parser = Notation::Parser.new(self)
+        Array(parser.parse(notation_uri))
+      rescue NotationError, RecordNotFoundError, StandardError => e
+        @logger.debug("try_get_notation failed for '#{notation_uri}': #{e.message}")
+        []
+      end
+
+      # Create folder
+      def create_folder(folder_name, parent_uid: nil)
+        raise ArgumentError, 'parent_uid is required to create a folder' unless parent_uid
+
+        # Get folders to find parent's shared folder key
+        folders = get_folders
+
+        # Find parent folder
+        parent_folder = folders.find { |f| f.uid == parent_uid }
+        raise Error, "Parent folder #{parent_uid} not found" unless parent_folder
+
+        # Determine if parent is a shared folder root (no parent_uid)
+        is_shared_root = parent_folder.parent_uid.nil? || parent_folder.parent_uid.empty?
+
+        # Find the shared folder root by traversing up the hierarchy
+        if is_shared_root
+          # Parent is the shared root, so new folder is at root level
+          shared_folder_uid = parent_uid
+          actual_parent_uid = nil  # nil for root-level folders
+        else
+          # Parent is a subfolder, traverse up to find shared root
+          shared_folder_uid = parent_uid
+          current_folder = parent_folder
+
+          while current_folder.parent_uid && !current_folder.parent_uid.empty?
+            parent = folders.find { |f| f.uid == current_folder.parent_uid }
+            break unless parent
+
+            shared_folder_uid = current_folder.parent_uid
+            current_folder = parent
+          end
+
+          actual_parent_uid = parent_uid  # Subfolder creation
+        end
+
+        # Get shared folder's key (the root folder's key)
+        shared_folder = folders.find { |f| f.uid == shared_folder_uid }
+        raise Error, "Shared folder #{shared_folder_uid} not found" unless shared_folder
+
+        shared_folder_key = shared_folder.folder_key
+        raise Error, "Shared folder key missing for #{shared_folder_uid}" unless shared_folder_key
+
+        # Generate new folder UID and key
+        folder_uid = Utils.generate_uid
+        folder_key = Crypto.generate_encryption_key_bytes
+
+        # Prepare folder data
+        folder_data = {
+          'name' => folder_name
+        }
+
+        # Encrypt folder data with NEW folder's key using AES-CBC
+        encrypted_data = Crypto.encrypt_aes_cbc(
+          Utils.dict_to_json(folder_data),
+          folder_key
+        )
+
+        # Encrypt folder key with SHARED folder's key using AES-CBC
+        encrypted_folder_key = Crypto.encrypt_aes_cbc(folder_key, shared_folder_key)
+
+        # Prepare payload
+        payload = prepare_create_folder_payload(
+          folder_uid: folder_uid,
+          shared_folder_uid: shared_folder_uid,
+          encrypted_folder_key: encrypted_folder_key,
+          data: encrypted_data,
+          parent_uid: actual_parent_uid  # nil for root, subfolder UID for nested
+        )
+
+        post_query('create_folder', payload)
+        folder_uid
+      end
+
+      # Update folder
+      def update_folder(folder_uid, folder_name)
+        # Get folders to find the folder's key
+        folders = get_folders
+        folder = folders.find { |f| f.uid == folder_uid }
+        raise Error, "Folder #{folder_uid} not found" unless folder
+
+        folder_key = folder.folder_key
+        raise Error, "Folder key missing for #{folder_uid}" unless folder_key
+
+        # Prepare folder data
+        folder_data = {
+          'name' => folder_name
+        }
+
+        # Encrypt folder data with folder's key using AES-CBC
+        encrypted_data = Crypto.encrypt_aes_cbc(
+          Utils.dict_to_json(folder_data),
+          folder_key
+        )
+
+        payload = prepare_update_folder_payload(
+          folder_uid: folder_uid,
+          data: encrypted_data
+        )
+
+        post_query('update_folder', payload)
+        true
+      end
+
+      # Delete folders
+      def delete_folder(folder_uids, force: false)
+        folder_uids = [folder_uids] if folder_uids.is_a?(String)
+
+        payload = prepare_delete_folder_payload(folder_uids, force)
+        response = post_query('delete_folder', payload)
+
+        result = JSON.parse(response)
+        folders = result['folders'] || []
+        folders.each do |f|
+          next if f['responseCode'] == 'ok'
+          @logger.error("Failed to delete folder #{f['folderUid']}: #{f['responseCode']} #{f['errorMessage']}")
+        end
+        folders
+      end
+
+      # Get folder hierarchy manager
+      def folder_manager
+        folders = get_folders
+        FolderManager.new(folders)
+      end
+
+      # Get folder path (convenience method)
+      def get_folder_path(folder_uid)
+        folder_manager.get_folder_path(folder_uid)
+      end
+
+      # Find folder by name (convenience method)
+      def find_folder_by_name(name, parent_uid: nil)
+        folder_manager.find_folder_by_name(name, parent_uid: parent_uid)
+      end
+
+      # Upload file
+      def upload_file(owner_record_uid, file_data, file_name, file_title = nil)
+        file_title ||= file_name
+
+        # Fetch the owner record (decrypted) to get current state
+        owner_records = get_secrets([owner_record_uid])
+        raise Error, "Owner record #{owner_record_uid} not found" if owner_records.empty?
+
+        owner_record = owner_records.first
+        owner_revision = owner_record.revision
+
+        # Get owner record data as hash for manipulation
+        owner_record_data = owner_record.to_h
+
+        # Get the record_key (stored during decryption)
+        owner_record_key = owner_record.record_key
+        raise Error, "Record key not available for owner record #{owner_record_uid}" unless owner_record_key
+
+        # Get owner record's public key from storage (app owner public key)
+        owner_public_key = @config.get_string(ConfigKeys::KEY_OWNER_PUBLIC_KEY)
+        raise Error, "Owner public key not found in config - application may need re-binding" unless owner_public_key
+
+        owner_public_key_bytes = Utils.url_safe_str_to_bytes(owner_public_key)
+
+        # Generate file record UID and key
+        file_uid = Utils.generate_uid
+        file_key = Crypto.generate_encryption_key_bytes
+
+        # Encrypt file data with file key
+        encrypted_file = Crypto.encrypt_aes_gcm(file_data, file_key)
+
+        # Create file record metadata
+        file_record = {
+          'name' => file_name,
+          'size' => file_data.bytesize,
+          'title' => file_title,
+          'lastModified' => (Time.now.to_f * 1000).to_i,
+          'type' => 'application/octet-stream'
+        }
+
+        # Encrypt file record metadata with file key
+        file_record_json = Utils.dict_to_json(file_record)
+        file_record_bytes = file_record_json.bytes
+        encrypted_file_record = Crypto.encrypt_aes_gcm(file_record_bytes.pack('C*'), file_key)
+
+        # Encrypt file record key with owner's public key (ECIES)
+        encrypted_file_record_key = Crypto.encrypt_ec(file_key, owner_public_key_bytes)
+
+        # Encrypt file record key with owner record key (for linkKey)
+        encrypted_link_key = Crypto.encrypt_aes_gcm(file_key, owner_record_key)
+
+        # Add fileRef to owner record's fields
+        fields = owner_record_data['fields'] || []
+
+        file_ref_field = fields.find { |f| f['type'] == 'fileRef' }
+        if file_ref_field
+          file_ref_field['value'] ||= []
+          file_ref_field['value'] << file_uid
+        else
+          fields << { 'type' => 'fileRef', 'value' => [file_uid] }
+        end
+
+        # Update owner record data
+        owner_record_data['fields'] = fields
+        owner_record_json = Utils.dict_to_json(owner_record_data)
+        owner_record_bytes = owner_record_json.bytes.pack('C*')
+
+        # Encrypt updated owner record with its record key
+        encrypted_owner_record_data = Crypto.encrypt_aes_gcm(owner_record_bytes, owner_record_key)
+
+        # Prepare payload
+        payload = prepare_file_upload_payload(
+          file_record_uid: file_uid,
+          file_record_key: encrypted_file_record_key,
+          file_record_data: encrypted_file_record,
+          owner_record_uid: owner_record_uid,
+          owner_record_data: encrypted_owner_record_data,
+          owner_record_revision: owner_revision,
+          link_key: encrypted_link_key,
+          file_size: encrypted_file.bytesize
+        )
+
+        # Get upload URL
+        response = post_query('add_file', payload)
+        upload_result = JSON.parse(response)
+
+        # Upload file
+        upload_file_function(
+          upload_result['url'],
+          upload_result['parameters'],
+          encrypted_file
+        )
+
+        file_uid
+      end
+
+      # Upload file from disk path (convenience method)
+      # Reads file from disk and uploads to specified record
+      def upload_file_from_path(owner_record_uid, file_path, file_title: nil)
+        raise ArgumentError, "File not found: #{file_path}" unless File.exist?(file_path)
+        raise ArgumentError, "Path is a directory: #{file_path}" if File.directory?(file_path)
+
+        # Read file data
+        file_data = File.binread(file_path)
+
+        # Extract filename from path
+        file_name = File.basename(file_path)
+
+        # Use file_title if provided, otherwise use filename
+        file_title ||= file_name
+
+        @logger.debug("Uploading file from path: #{file_path} (#{file_data.bytesize} bytes)")
+
+        # Delegate to existing upload_file method
+        upload_file(owner_record_uid, file_data, file_name, file_title)
+      end
+
+      # Download file from record's file data
+      def download_file(file_data)
+        # Extract file metadata (already decrypted)
+        file_uid = file_data['fileUid']
+        file_url = file_data['url']
+        file_name = file_data['name'] || file_data['title'] || 'unnamed'
+
+        raise Error, "No download URL available for file #{file_uid}" unless file_url
+
+        # The file key should already be decrypted (base64 encoded)
+        file_key = Utils.base64_to_bytes(file_data['fileKey'])
+
+        # Download the encrypted file content
+        encrypted_content = download_encrypted_file(file_url)
+
+        # Decrypt the file content with the file key
+        decrypted_content = Crypto.decrypt_aes_gcm(encrypted_content, file_key)
+
+        # Return file info and data
+        {
+          'name' => file_name,
+          'title' => file_data['title'] || file_name,
+          'type' => file_data['type'],
+          'size' => file_data['size'] || decrypted_content.bytesize,
+          'data' => decrypted_content
+        }
+      end
+
+      # Download file thumbnail
+      def download_thumbnail(file_data)
+        if file_data.is_a?(Dto::KeeperFile)
+          file_uid      = file_data.uid
+          thumbnail_url = file_data.thumbnail_url
+          file_key_str  = file_data.file_key
+        else
+          file_uid      = file_data['fileUid'] || file_data['uid']
+          thumbnail_url = file_data['thumbnailUrl'] || file_data['thumbnail_url']
+          file_key_str  = file_data['fileKey'] || file_data['file_key']
+        end
+
+        raise ArgumentError, 'File UID is required' unless file_uid
+        raise Error, "No thumbnail URL available for file #{file_uid}" unless thumbnail_url
+        raise Error, "File key not available for #{file_uid}" unless file_key_str
+
+        file_key = Utils.base64_to_bytes(file_key_str)
+        encrypted_content = download_encrypted_file(thumbnail_url)
+        decrypted_content = Crypto.decrypt_aes_gcm(encrypted_content, file_key)
+
+        {
+          'file_uid' => file_uid,
+          'data' => decrypted_content,
+          'size' => decrypted_content.bytesize
+        }
+      end
+
+      # Get file metadata from server
+      def get_file_data(file_uid)
+        payload = prepare_get_payload(nil)
+        payload.file_uids = [file_uid]
+
+        response = post_query('get_files', payload)
+        response_dict = JSON.parse(response)
+
+        if response_dict['files'] && !response_dict['files'].empty?
+          file_data = response_dict['files'].first
+
+          # Decrypt file metadata
+          # Get app key for decryption
+          app_key_str = @config.get_string(ConfigKeys::KEY_APP_KEY)
+          if app_key_str && !app_key_str.empty?
+            app_key = Utils.base64_to_bytes(app_key_str)
+          else
+            # Decrypt app key with client key
+            app_key_encrypted = Utils.base64_to_bytes(@config.get_string(ConfigKeys::KEY_ENCRYPTED_APP_KEY))
+            client_key = get_client_key
+            app_key = Crypto.decrypt_aes_gcm(app_key_encrypted, client_key)
+          end
+
+          encrypted_data = Utils.base64_to_bytes(file_data['data'])
+          decrypted_json = Crypto.decrypt_aes_gcm(encrypted_data, app_key)
+
+          JSON.parse(decrypted_json).merge('fileKey' => file_data['fileKey'])
+        else
+          raise Error, "File not found: #{file_uid}"
+        end
+      end
+
+      # Download encrypted file from URL
+      def download_encrypted_file(url)
+        uri = URI(url)
+
+        @logger.debug("Downloading file from URL: #{url}")
+
+        request = Net::HTTP::Get.new(uri)
+
+        http = create_http_client(uri)
+        configure_http_ssl(http)
+
+        response = http.request(request)
+
+        @logger.debug("Download response status: #{response.code}")
+
+        if response.code == '200'
+          response.body
+        else
+          raise Error, "Failed to download file: #{response.code} #{response.message}"
+        end
+      end
+
+      private
+
+      # Process token binding
+      def process_token_binding(token, hostname = nil)
+        # Parse token
+        token = token.strip
+        # -1 keeps trailing empties so "IL5:key:20:" yields 4 parts and the empty serverPublicKey
+        # segment is rejected, instead of default split() dropping it and looking like 3 parts.
+        token_parts = token.split(':', -1)
+
+        il5_key_id = nil
+        il5_server_public_key = nil
+
+        # Modern format: REGION:BASE64_TOKEN
+        if token_parts.length >= 2
+          region = token_parts[0].upcase
+          @hostname = KeeperGlobals::KEEPER_SERVERS[region] || KeeperGlobals::DEFAULT_SERVER
+
+          if region == 'IL5' && token_parts.length > 2
+            # IL5 dynamic-key OTT: IL5:clientKey:serverPublicKeyId:serverPublicKey
+            unless token_parts.length == 4
+              raise Error, 'Malformed IL5 one-time token: expected exactly 4 colon-separated ' \
+                           "segments 'IL5:clientKey:keyId:serverPublicKey', got #{token_parts.length}"
+            end
+            il5_key_id = token_parts[2]
+            il5_server_public_key = token_parts[3]
+            if il5_key_id.to_s.empty? || il5_server_public_key.to_s.empty?
+              raise Error, 'Malformed IL5 one-time token: keyId and serverPublicKey segments must be non-empty'
+            end
+            begin
+              Utils.url_safe_str_to_bytes(il5_server_public_key)
+            rescue StandardError
+              raise Error, 'Malformed IL5 one-time token: serverPublicKey segment is not valid url-safe base64'
+            end
+            @token = token_parts[1]
+          else
+            @token = token_parts[1..].join(':')
+          end
+        else
+          # Legacy format
+          @token = token
+          @hostname = hostname || KeeperGlobals::DEFAULT_SERVER
+        end
+
+        # Precedence: programmatic override > OTT segment.
+        effective_key_id     = @server_public_key_id_override || il5_key_id
+        effective_public_key = @server_public_key_override    || il5_server_public_key
+
+        # Bind the one-time token
+        bound_config = bind_one_time_token(@token, @hostname, effective_key_id, effective_public_key)
+
+        # Merge bound config into existing config if present
+        if @config
+          # Copy all values from bound config to existing config
+          bound_data = bound_config.instance_variable_get(:@data)
+          bound_data&.each do |key, value|
+            if value.is_a?(String)
+              @config.save_string(key, value)
+            else
+              @config.save_bytes(key, value)
+            end
+          end
+        else
+          @config = bound_config
+        end
+      end
+
+      # Bind one-time token
+      def bind_one_time_token(token, hostname, key_id = nil, server_public_key = nil)
+        storage = Storage::InMemoryStorage.new
+
+        # Generate EC key pair
+        keys = Crypto.generate_ecc_keys
+
+        # Convert token to bytes and create client ID hash
+        token_bytes = Utils.url_safe_str_to_bytes(token)
+        client_id_hash = OpenSSL::HMAC.digest(
+          'SHA512',
+          token_bytes,
+          'KEEPER_SECRETS_MANAGER_CLIENT_ID'
+        )
+        client_id = Utils.bytes_to_base64(client_id_hash)
+
+        # Store configuration
+        storage.save_string(ConfigKeys::KEY_HOSTNAME, hostname)
+        storage.save_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY_ID, (key_id || DEFAULT_KEY_ID).to_s)
+        # persist the dynamic (IL5) server public key so it is used for binding AND all
+        # later requests, and survives restart via the saved config.
+        storage.save_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY, server_public_key) if server_public_key && !server_public_key.empty?
+        storage.save_string(ConfigKeys::KEY_CLIENT_KEY, token)
+        storage.save_bytes(ConfigKeys::KEY_PRIVATE_KEY, keys[:private_key_bytes])
+        storage.save_string(ConfigKeys::KEY_CLIENT_ID, client_id)
+
+        # Prepare binding payload
+        payload = Dto::GetPayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = client_id
+        payload.public_key = keys[:public_key_str]
+
+        # Send binding request
+        response = post_query('get_secret', payload, storage)
+        response_dict = JSON.parse(response)
+
+        # Process binding response
+        if response_dict['encryptedAppKey']
+          # Decrypt app key
+          encrypted_app_key = Utils.url_safe_str_to_bytes(response_dict['encryptedAppKey'])
+          client_key_bytes = Utils.url_safe_str_to_bytes(token)
+
+          app_key = Crypto.decrypt_aes_gcm(encrypted_app_key, client_key_bytes)
+          storage.save_bytes(ConfigKeys::KEY_APP_KEY, app_key)
+
+          # Store app owner public key if present
+          if response_dict['appOwnerPublicKey']
+            owner_key = Utils.url_safe_str_to_bytes(response_dict['appOwnerPublicKey'])
+            storage.save_bytes(ConfigKeys::KEY_OWNER_PUBLIC_KEY, owner_key)
+          end
+
+          # Clean up client key after successful binding
+          storage.delete(ConfigKeys::KEY_CLIENT_KEY)
+        else
+          raise Error, 'Failed to bind one-time token - no encrypted app key in response'
+        end
+
+        storage
+      end
+
+      # Fetch and decrypt secrets
+      def fetch_and_decrypt_secrets(query_options = nil)
+        payload = prepare_get_payload(query_options)
+
+        response = post_query('get_secret', payload)
+        response_dict = JSON.parse(response)
+
+        # Decrypt app key if present (during token binding)
+        if response_dict['encryptedAppKey']
+          encrypted_app_key = Utils.url_safe_str_to_bytes(response_dict['encryptedAppKey'])
+          client_key = Utils.url_safe_str_to_bytes(@config.get_string(ConfigKeys::KEY_CLIENT_KEY))
+
+          # Decrypt app key using AES with client key (the original token)
+          app_key = Crypto.decrypt_aes_gcm(encrypted_app_key, client_key)
+          @config.save_bytes(ConfigKeys::KEY_APP_KEY, app_key)
+
+          # Clean up client key after successful binding
+          @config.delete(ConfigKeys::KEY_CLIENT_KEY)
+
+          # Store app owner public key if present
+          if response_dict['appOwnerPublicKey']
+            owner_key = Utils.url_safe_str_to_bytes(response_dict['appOwnerPublicKey'])
+            @config.save_bytes(ConfigKeys::KEY_OWNER_PUBLIC_KEY, owner_key)
+          end
+
+          # Set just bound flag
+          just_bound = true
+        else
+          just_bound = false
+        end
+
+        # Get app key
+        app_key = @config.get_bytes(ConfigKeys::KEY_APP_KEY)
+        raise Error, 'No app key available' unless app_key
+
+        # Decrypt records
+        records = []
+        if response_dict['records']
+          response_dict['records'].each do |encrypted_record|
+            record = decrypt_record(encrypted_record, app_key)
+            records << record
+          rescue StandardError => e
+            @logger.error("Failed to decrypt record: #{e.message}")
+          end
+        end
+
+        # Decrypt folders - need to handle them in order for shared folder keys
+        folders = []
+        response_folders = response_dict['folders'] || []
+
+        # First pass - decrypt folders in order
+        response_folders.each do |encrypted_folder|
+          folder = decrypt_folder(encrypted_folder, app_key, folders, response_folders)
+          if folder
+            folders << folder
+            # Add folder's records to the main records list
+            records.concat(folder.records) if folder.records && !folder.records.empty?
+          end
+        rescue StandardError => e
+          @logger.error("Failed to decrypt folder: #{e.message}")
+        end
+
+        # Build response
+        response = Dto::SecretsManagerResponse.new(
+          records: records,
+          folders: folders,
+          warnings: response_dict['warnings']
+        )
+
+        response.just_bound = just_bound if response.respond_to?(:just_bound=)
+        response
+      end
+
+      # Decrypt record
+      def decrypt_record(encrypted_record, app_key)
+        record_uid = encrypted_record['recordUid']
+        record_key_encrypted = Utils.base64_to_bytes(encrypted_record['recordKey'])
+        data_encrypted = Utils.base64_to_bytes(encrypted_record['data'])
+
+        # Decrypt record key
+        record_key = Crypto.decrypt_aes_gcm(record_key_encrypted, app_key)
+
+        # Decrypt data
+        data_json = Crypto.decrypt_aes_gcm(data_encrypted, record_key)
+        data = JSON.parse(data_json)
+
+        # Decrypt files if present
+        decrypted_files = []
+        if encrypted_record['files']
+          encrypted_record['files'].each do |file|
+            # Decrypt file key with record key
+            file_key_encrypted = Utils.base64_to_bytes(file['fileKey'])
+            file_key = Crypto.decrypt_aes_gcm(file_key_encrypted, record_key)
+
+            # Decrypt file metadata with file key
+            if file['data']
+              file_data_encrypted = Utils.base64_to_bytes(file['data'])
+              file_metadata_json = Crypto.decrypt_aes_gcm(file_data_encrypted, file_key)
+              file_metadata = JSON.parse(file_metadata_json)
+            else
+              file_metadata = {}
+            end
+
+            # Create decrypted file object
+            decrypted_file = {
+              'fileUid' => file['fileUid'],
+              'fileKey' => Utils.bytes_to_base64(file_key), # Store decrypted key
+              'url' => file['url'],
+              'thumbnailUrl' => file['thumbnailUrl'],
+              'name' => file_metadata['name'],
+              'title' => file_metadata['title'] || file_metadata['name'],
+              'type' => file_metadata['type'],
+              'size' => file_metadata['size'],
+              'lastModified' => file_metadata['lastModified']
+            }
+
+            decrypted_files << decrypted_file
+          rescue StandardError => e
+            @logger&.error("Failed to decrypt file #{file['fileUid']}: #{e.message}")
+          end
+        end
+
+        # Create record object
+        record = Dto::KeeperRecord.new(
+          'recordUid' => record_uid,
+          'folderUid' => encrypted_record['folderUid'],
+          'innerFolderUid' => encrypted_record['innerFolderUid'],
+          'isEditable' => encrypted_record['isEditable'],
+          'data' => data,
+          'revision' => encrypted_record['revision'],
+          'files' => decrypted_files,
+          'links' => encrypted_record['links'] || []
+        )
+
+        # Store record key for later use (e.g., file downloads)
+        record.instance_variable_set(:@record_key, record_key)
+        record.define_singleton_method(:record_key) { @record_key }
+
+        record
+      end
+
+      # Get shared folder key by traversing up the folder hierarchy
+      def get_shared_folder_key(folders, response_folders, parent_uid)
+        while parent_uid
+          # Find parent folder in response
+          parent_folder = response_folders.find { |f| f['folderUid'] == parent_uid }
+          return nil unless parent_folder
+
+          # If parent has no parent, it's the shared folder root
+          if !parent_folder['parent'] || parent_folder['parent'].empty?
+            # Find the decrypted folder object
+            shared_folder = folders.find { |f| f.uid == parent_uid }
+            return shared_folder&.folder_key
+          end
+
+          # Continue up the hierarchy
+          parent_uid = parent_folder['parent']
+        end
+
+        nil
+      end
+
+      # Decrypt folder
+      def decrypt_folder(encrypted_folder, app_key, existing_folders = [], response_folders = [])
+        folder_uid = encrypted_folder['folderUid']
+        folder_parent = encrypted_folder['parent']
+
+        @logger.debug("Decrypting folder #{folder_uid}, parent: #{folder_parent || 'none'}")
+
+        # Determine the decryption key to use
+        decryption_key = if !folder_parent || folder_parent.empty?
+                           # Root folder - use app key
+                           @logger.debug("Using app key for root folder #{folder_uid}")
+                           app_key
+                         else
+                           # Child folder - use shared folder key
+                           shared_folder_key = get_shared_folder_key(existing_folders, response_folders, folder_parent)
+                           unless shared_folder_key
+                             @logger.error("Cannot find shared folder key for parent #{folder_parent}")
+                             return nil
+                           end
+                           @logger.debug("Using shared folder key from parent for folder #{folder_uid}")
+                           shared_folder_key
+                         end
+
+        # Some folders might not have encryption data
+        unless encrypted_folder['folderKey']
+          # Create a basic folder object without decrypted data
+          return Dto::KeeperFolder.new(
+            'folderUid' => folder_uid,
+            'folderKey' => nil,
+            'data' => {},
+            'name' => encrypted_folder['name'] || folder_uid,
+            'parent' => folder_parent,
+            'records' => []
+          )
+        end
+
+        # Decrypt folder key
+        folder_key_encrypted = Utils.base64_to_bytes(encrypted_folder['folderKey'])
+        folder_key = if !folder_parent || folder_parent.empty?
+                       # Root folder key uses AES-GCM
+                       Crypto.decrypt_aes_gcm(folder_key_encrypted, decryption_key)
+                     else
+                       # Child folder key uses AES-CBC
+                       Crypto.decrypt_aes_cbc(folder_key_encrypted, decryption_key)
+                     end
+
+        # Get folder name - either from encrypted data or direct field
+        folder_name = ''
+        folder_type = nil
+
+        # Check if there's a direct name field (unencrypted)
+        if encrypted_folder['name']
+          folder_name = encrypted_folder['name']
+          @logger.debug("Using direct name field for folder #{folder_uid}: #{folder_name}")
+        elsif encrypted_folder['data'] && !encrypted_folder['data'].empty?
+          # Decrypt folder data if present
+          begin
+            data_encrypted = Utils.base64_to_bytes(encrypted_folder['data'])
+            # Folder data always uses CBC
+            data_json = Crypto.decrypt_aes_cbc(data_encrypted, folder_key)
+            data = JSON.parse(data_json)
+            folder_name = data['name'] || ''
+            folder_type = data['folderType']
+            @logger.debug("Successfully decrypted folder #{folder_uid}: #{folder_name}")
+          rescue StandardError => e
+            @logger.error("Failed to decrypt folder data for #{folder_uid}: #{e.class} - #{e.message}")
+            @logger.debug("Backtrace: #{e.backtrace.first(3).join("\n")}")
+          end
+        else
+          @logger.debug("Folder #{folder_uid} has no name or data field - using UID as name")
+          folder_name = folder_uid
+        end
+
+        # Decrypt records in this folder
+        folder_records = []
+        if encrypted_folder['records']
+          encrypted_folder['records'].each do |encrypted_record|
+            # Decrypt the record using folder key
+            record = decrypt_record(encrypted_record, folder_key)
+
+            # Set folder_uid on the record
+            record.folder_uid = folder_uid if record
+            folder_records << record if record
+          rescue StandardError => e
+            @logger.error("Failed to decrypt record in folder #{folder_uid}: #{e.message}")
+          end
+        end
+
+        # Create folder object
+        Dto::KeeperFolder.new(
+          'folderUid' => folder_uid,
+          'name' => folder_name,
+          'folderType' => folder_type,
+          'folderKey' => folder_key,
+          'parent' => folder_parent,
+          'records' => folder_records
+        )
+      end
+
+      # Prepare get payload
+      def prepare_get_payload(query_options = nil)
+        payload = Dto::GetPayload.new
+        payload.client_version = KeeperGlobals.client_version
+
+        # Client ID should be URL-safe base64
+        client_id_str = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.client_id = client_id_str
+
+        @logger.debug("Client ID for payload: #{client_id_str}")
+
+        # Public key is sent during initial binding only
+
+        if query_options
+          payload.requested_records = query_options.records_filter
+          payload.requested_folders = query_options.folders_filter
+          payload.request_links = query_options.request_links if query_options.request_links
+        end
+
+        payload
+      end
+
+      # Prepare create payload
+      def prepare_create_payload(record_uid:, record_key:, folder_uid:, folder_key:, data:, subfolder_uid: nil)
+        owner_public_key = @config.get_string(ConfigKeys::KEY_OWNER_PUBLIC_KEY)
+        unless owner_public_key && !owner_public_key.empty?
+          raise Error, 'Unable to create record - owner key is missing. Looks like application was created ' \
+                       'using out of date client (Web Vault or Commander)'
+        end
+
+        owner_public_key_bytes = Utils.url_safe_str_to_bytes(owner_public_key)
+        record_key_encrypted = Crypto.encrypt_ec(record_key, owner_public_key_bytes)
+
+        payload = Dto::CreatePayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.record_uid = record_uid
+        payload.record_key = Utils.bytes_to_base64(record_key_encrypted)
+        payload.folder_uid = folder_uid
+        payload.sub_folder_uid = subfolder_uid
+        payload.data = Utils.bytes_to_base64(data)
+
+        # Encrypt the record key with the folder key
+        if folder_key
+          folder_key_encrypted = Crypto.encrypt_aes_gcm(record_key, folder_key)
+          payload.folder_key = Utils.bytes_to_base64(folder_key_encrypted)
+        end
+
+        payload
+      end
+
+      # Other payload preparation methods...
+
+      # Post query to API
+      def post_query(path, payload, config = nil)
+        config ||= @config
+        server = get_server(@hostname)
+        url = "https://#{server}/api/rest/sm/v1/#{path}"
+
+        throttle_attempt = 0
+        loop do
+          # Generate transmission key
+          key_id = config.get_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY_ID) || DEFAULT_KEY_ID
+          # a custom (IL5) server public key in config overrides the built-in key table.
+          custom_public_key = config.get_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY)
+          transmission_key = generate_transmission_key(key_id, custom_public_key)
+
+          # Encrypt and sign payload
+          encrypted_payload = encrypt_and_sign_payload(config, transmission_key, payload)
+
+          # Make request
+          # Use custom post function for read-only operations (get_secret, get_folders)
+          # This enables caching for disaster recovery
+          response = if @custom_post_function && (path == 'get_secret' || path == 'get_folders')
+                       @custom_post_function.call(url, transmission_key, encrypted_payload, @verify_ssl_certs)
+                     else
+                       post_function(url, transmission_key, encrypted_payload)
+                     end
+
+          # Handle response
+          if response.success?
+            # Decrypt response if present
+            if response.data && !response.data.empty?
+              return Crypto.decrypt_aes_gcm(response.data, transmission_key.key)
+            else
+              return response.data
+            end
+          else
+            # Throttle retry with exponential backoff + jitter (KSM-876 / KSM-883). Checked before
+            # handle_http_error (which still drives key-rotation retry), and gated on the 403 status
+            # so a non-403 response carrying a {"error":"throttled"} body is not retried.
+            retry_after = parse_throttle(response)
+            if response.status_code == 403 && !retry_after.nil?
+              if throttle_attempt >= MAX_THROTTLE_RETRIES
+                raise ThrottledError.new('throttled',
+                                         "Request throttled by Keeper backend; exhausted #{MAX_THROTTLE_RETRIES} retries")
+              end
+              delay = throttle_delay(throttle_attempt, retry_after)
+              @logger.warn("Request throttled (attempt #{throttle_attempt + 1}/#{MAX_THROTTLE_RETRIES}); " \
+                           "retrying in #{delay.round(1)}s")
+              sleep(delay)
+              throttle_attempt += 1
+              next
+            end
+
+            handle_http_error(response, config)
+          end
+        end
+      end
+
+      # Generate transmission key
+      def generate_transmission_key(key_id, custom_public_key = nil)
+        # Get server public key. A custom (e.g. IL5) key overrides the built-in table,
+        # which only covers ids 1..18 - IL5 uses id 20 supplied out-of-band via the OTT/config.
+        if custom_public_key && !custom_public_key.empty?
+          server_public_key_str = custom_public_key
+        else
+          server_public_key_str = KeeperGlobals::KEEPER_PUBLIC_KEYS[key_id.to_s]
+          raise Error, "Unknown public key ID: #{key_id}" unless server_public_key_str
+        end
+
+        @logger.debug("Using server public key ID: #{key_id}")
+        @logger.debug("Server public key string: #{server_public_key_str[0..20]}...")
+
+        # Generate random key
+        key = Crypto.generate_encryption_key_bytes
+        @logger.debug("Generated transmission key: #{Utils.bytes_to_base64(key)[0..20]}...")
+
+        # Encrypt key with server public key
+        server_public_key = Utils.url_safe_str_to_bytes(server_public_key_str)
+        @logger.debug("Server public key bytes length: #{server_public_key.bytesize}")
+
+        encrypted_key = Crypto.encrypt_ec(key, server_public_key)
+        @logger.debug("Encrypted key length: #{encrypted_key.bytesize}")
+
+        Dto::TransmissionKey.new(
+          public_key_id: key_id,
+          key: key,
+          encrypted_key: encrypted_key
+        )
+      end
+
+      # Encrypt and sign payload
+      def encrypt_and_sign_payload(config, transmission_key, payload)
+        # Convert payload to JSON
+        payload_json = payload.to_json
+
+        @logger.debug("Payload: #{payload_json}")
+
+        # Encrypt payload
+        encrypted_payload = Crypto.encrypt_aes_gcm(payload_json, transmission_key.key)
+
+        # Generate signature
+        signature_base = transmission_key.encrypted_key + encrypted_payload
+
+        # After binding, use ECDSA signature with private key (not HMAC)
+        private_key_bytes = config.get_bytes(ConfigKeys::KEY_PRIVATE_KEY)
+        if private_key_bytes
+          # Load private key
+          private_key = load_ec_private_key(private_key_bytes)
+
+          # Generate ECDSA signature
+          signature = Crypto.sign_ec(signature_base, private_key)
+          @logger.debug("Using ECDSA signature, length: #{signature.bytesize}")
+        else
+          # Fallback to HMAC with client key (for one-time token binding)
+          client_key = config.get_string(ConfigKeys::KEY_CLIENT_KEY)
+          if client_key
+            signature_key = Utils.base64_to_bytes(client_key)
+            signature = Crypto.generate_hmac(signature_key, signature_base)
+            @logger.debug("Using HMAC signature, length: #{signature.bytesize}")
+          else
+            raise Error, 'No key available for signature'
+          end
+        end
+
+        Dto::EncryptedPayload.new(
+          encrypted_payload: encrypted_payload,
+          signature: signature
+        )
+      end
+
+      # Create Net::HTTP instance with proxy support
+      # Configures proxy if @proxy_url is set
+      def create_http_client(uri)
+        if @proxy_url
+          # Parse proxy URL
+          proxy_uri = URI(@proxy_url)
+
+          # Create HTTP client with proxy
+          http = Net::HTTP.new(
+            uri.host,
+            uri.port,
+            proxy_uri.host,
+            proxy_uri.port,
+            proxy_uri.user,
+            proxy_uri.password
+          )
+
+          @logger.debug("Using HTTP proxy: #{proxy_uri.host}:#{proxy_uri.port}")
+          @logger.debug("Proxy authentication: #{proxy_uri.user ? 'yes' : 'no'}")
+        else
+          # Create HTTP client without proxy
+          http = Net::HTTP.new(uri.host, uri.port)
+        end
+
+        http
+      end
+
+      # Configure SSL for HTTP connection
+      # Sets up certificate store and verification mode
+      def configure_http_ssl(http)
+        http.use_ssl = true
+
+        if @verify_ssl_certs
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+          # Set up certificate store with system defaults
+          # This ensures we use the system's trusted CA certificates
+          store = OpenSSL::X509::Store.new
+          store.set_default_paths
+
+          # Note: We don't enable CRL checking by default because:
+          # 1. CRL endpoints may be temporarily unreachable
+          # 2. Many enterprise networks block CRL endpoints
+          # 3. Certificate chain validation still provides strong security
+          # If strict CRL checking is needed, it can be added via configuration option
+          http.cert_store = store
+
+          @logger.debug('SSL verification enabled with system certificate store')
+          @logger.debug("OpenSSL version: #{OpenSSL::OPENSSL_VERSION}")
+        else
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          @logger.warn('SSL verification disabled - not recommended for production')
+        end
+      end
+
+      # HTTP post function
+      def post_function(url, transmission_key, encrypted_payload)
+        uri = URI(url)
+
+        @logger.debug("POST URL: #{url}")
+        @logger.debug("PublicKeyId header: #{transmission_key.public_key_id}")
+        @logger.debug("TransmissionKey header: #{Utils.bytes_to_base64(transmission_key.encrypted_key)[0..50]}...")
+        @logger.debug("TransmissionKey full base64 length: #{Utils.bytes_to_base64(transmission_key.encrypted_key).length}")
+        @logger.debug("Signature header: #{Utils.bytes_to_base64(encrypted_payload.signature)[0..50]}...")
+        @logger.debug("Request body length: #{encrypted_payload.encrypted_payload.bytesize} bytes")
+
+        request = Net::HTTP::Post.new(uri)
+        request['Content-Type'] = 'application/octet-stream'
+        request['PublicKeyId'] = transmission_key.public_key_id.to_s
+        request['TransmissionKey'] = Utils.bytes_to_base64(transmission_key.encrypted_key)
+        request['Authorization'] = "Signature #{Utils.bytes_to_base64(encrypted_payload.signature)}"
+        request['Content-Length'] = encrypted_payload.encrypted_payload.bytesize.to_s
+        request.body = encrypted_payload.encrypted_payload
+
+        http = create_http_client(uri)
+        configure_http_ssl(http)
+
+        response = http.request(request)
+
+        @logger.debug("Response status: #{response.code}")
+        @logger.debug("Response body length: #{response.body&.bytesize || 0} bytes")
+        @logger.debug("Response body (first 100 bytes): #{response.body&.[](0..100).inspect}") if response.body && !response.body.empty?
+
+        Dto::KSMHttpResponse.new(
+          status_code: response.code.to_i,
+          data: response.body,
+          http_response: response
+        )
+      rescue StandardError => e
+        raise NetworkError, "HTTP request failed: #{e.message}"
+      end
+
+      # Handle HTTP errors
+      def handle_http_error(response, config = nil)
+        error_data = JSON.parse(response.data)
+        result_code = error_data['result_code'] || error_data['error']
+        message = error_data['message']
+
+        @logger.debug("Server error response: #{error_data.inspect}")
+
+        # Handle specific errors
+        case result_code
+        when 'key'
+          # Server wants different key
+          key_id = error_data['key_id']
+          # Use passed config or fall back to instance config
+          config_to_use = config || @config
+          custom_key = config_to_use&.get_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY)
+
+          # when a custom (IL5) server public key is configured, the server rejecting it
+          # cannot be fixed by switching to a built-in key, and retrying would loop on the same key.
+          # Fail fast with actionable remediation instead.
+          if custom_key && !custom_key.empty?
+            raise Error, "Server rejected the custom server public key (requested key ID #{key_id.inspect}). " \
+                         'The IL5 server public key may have rotated; update your IL5 KSM configuration ' \
+                         '(serverPublicKey / serverPublicKeyId), e.g. by redeeming a freshly issued one-time token.'
+          end
+
+          raise Error, 'The public key is blank from the server' if key_id.nil?
+          unless KeeperGlobals::KEEPER_PUBLIC_KEYS.key?(key_id.to_s)
+            raise Error, "The public key at #{key_id} does not exist in the SDK"
+          end
+
+          @logger.info("Server requested key ID: #{key_id}")
+          config_to_use.save_string(ConfigKeys::KEY_SERVER_PUBLIC_KEY_ID, key_id.to_s) if config_to_use
+          nil # Retry
+        else
+          raise ErrorFactory.from_server_response(result_code, message)
+        end
+      rescue JSON::ParserError
+        raise NetworkError.new("Server error: HTTP #{response.status_code}",
+                               status_code: response.status_code,
+                               response_body: response.data)
+      end
+
+      # Returns the throttle retry_after (>= 0) when +response+ is a backend throttle error
+      # (result_code/error == "throttled"), otherwise nil so the caller falls through to
+      # handle_http_error. Non-JSON / non-object bodies return nil. (KSM-876 / KSM-883)
+      def parse_throttle(response)
+        data = JSON.parse(response.data)
+        return nil unless data.is_a?(Hash)
+
+        result_code = data['result_code'] || data['error']
+        return nil unless result_code == 'throttled'
+
+        retry_after = begin
+          Float(data['retry_after'])
+        rescue ArgumentError, TypeError
+          0.0
+        end
+        retry_after.negative? ? 0.0 : retry_after
+      rescue JSON::ParserError
+        nil
+      end
+
+      # Backoff delay (seconds) for a 0-based attempt: retry_after when > 0, otherwise exponential
+      # backoff (BASE_THROTTLE_DELAY_SEC * 2**attempt -> 11, 22, 44, 88, 176s). A 0 to +25% jitter
+      # is applied (one-sided so delay is always >= floor, preventing a retry before the backend's
+      # 10s memcached window expires).
+      def throttle_delay(attempt, retry_after = 0)
+        base = retry_after.to_f.positive? ? retry_after.to_f : BASE_THROTTLE_DELAY_SEC * (2**attempt)
+        base = MAX_THROTTLE_DELAY_SEC if base > MAX_THROTTLE_DELAY_SEC
+        delay = base + base * rand(0.0..0.25)
+        delay.negative? ? 0.0 : delay
+      end
+
+      # Get server hostname
+      def get_server(hostname)
+        return hostname if hostname.include?('.')
+
+        # Look up in server list
+        KeeperGlobals::KEEPER_SERVERS[hostname.upcase] || hostname
+      end
+
+      # Load EC private key from bytes
+      def load_ec_private_key(private_key_bytes)
+        # If it's already a key object, return it
+        return private_key_bytes if private_key_bytes.is_a?(OpenSSL::PKey::EC)
+
+        @logger.debug("Loading private key, bytes length: #{private_key_bytes.bytesize}")
+
+        # Try to load as DER format first
+        begin
+          key = OpenSSL::PKey.read(private_key_bytes, nil)
+          # Ensure it's an EC key
+          if key.is_a?(OpenSSL::PKey::EC)
+            @logger.debug('Successfully loaded EC private key from DER format')
+            key
+          else
+            raise "Not an EC key, got #{key.class}"
+          end
+        rescue StandardError => e
+          @logger.debug("DER format failed: #{e.message}, trying raw bytes")
+
+          # If DER fails, it might be raw key bytes (32 bytes)
+          if private_key_bytes.bytesize == 32
+            begin
+              # Create EC key from raw bytes (OpenSSL 3.0 compatible)
+              group = OpenSSL::PKey::EC::Group.new('prime256v1')
+
+              # Generate key components
+              private_key_bn = OpenSSL::BN.new(private_key_bytes, 2)
+              public_key_point = group.generator.mul(private_key_bn)
+
+              # Create ASN1 sequence for the key
+              asn1 = OpenSSL::ASN1::Sequence([
+                                               OpenSSL::ASN1::Integer(1),
+                                               OpenSSL::ASN1::OctetString(private_key_bytes),
+                                               OpenSSL::ASN1::ObjectId('prime256v1', 0, :EXPLICIT),
+                                               OpenSSL::ASN1::BitString(
+                                                 public_key_point.to_octet_string(:uncompressed), 1, :EXPLICIT
+                                               )
+                                             ])
+
+              # Create key from DER
+              key = OpenSSL::PKey::EC.new(asn1.to_der)
+
+              @logger.debug('Successfully created EC key from raw bytes')
+              key
+            rescue StandardError => raw_error
+              @logger.debug("Raw bytes failed: #{raw_error.message}")
+              raise CryptoError, "Failed to load private key: DER: #{e.message}, Raw: #{raw_error.message}"
+            end
+          else
+            raise CryptoError,
+                  "Failed to load private key: #{e.message} (got #{private_key_bytes.bytesize} bytes)"
+          end
+        end
+      end
+
+      # Other helper methods...
+      def prepare_update_payload(record_uid:, record_data:, record_key:, revision:, update_options: nil)
+        payload = Dto::UpdatePayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.record_uid = record_uid
+        payload.revision = revision
+
+        if update_options
+          # nil clears UpdatePayload's 'general' default (BasePayload#to_h skips nil fields).
+          payload.transaction_type = update_options.transaction_type
+
+          # Handle links_to_remove
+          if update_options.links_to_remove && !update_options.links_to_remove.empty?
+            payload.links2_remove = update_options.links_to_remove
+
+            # Filter fileRef field values - remove specified link UIDs from record data
+            # This modifies the data hash before encryption (matches Python SDK behavior)
+            fileref_field = record_data['fields']&.find { |f| f['type'] == 'fileRef' }
+            if fileref_field && fileref_field['value'].is_a?(Array)
+              original_values = fileref_field['value']
+              filtered_values = original_values.reject { |uid| update_options.links_to_remove.include?(uid) }
+              fileref_field['value'] = filtered_values if filtered_values.length != original_values.length
+            end
+          end
+        end
+        # When update_options is nil: UpdatePayload keeps its default 'general' from #initialize.
+
+        # Encrypt record data
+        json_data = Utils.dict_to_json(record_data)
+        encrypted_data = Crypto.encrypt_aes_gcm(json_data, record_key)
+        payload.data = Utils.bytes_to_base64(encrypted_data)
+
+        payload
+      end
+
+      def prepare_delete_payload(record_uids)
+        payload = Dto::DeletePayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.record_uids = record_uids
+        payload
+      end
+
+      def prepare_complete_transaction_payload(record_uid)
+        payload = Dto::CompleteTransactionPayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.record_uid = record_uid
+        payload
+      end
+
+      def prepare_create_folder_payload(folder_uid:, shared_folder_uid:, encrypted_folder_key:, data:, parent_uid:)
+        payload = Dto::CreateFolderPayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.folder_uid = folder_uid
+        payload.shared_folder_uid = shared_folder_uid
+        payload.data = Utils.bytes_to_url_safe_str(data)
+        payload.parent_uid = parent_uid
+
+        # Use encrypted folder key (already encrypted with shared folder's key using AES-CBC)
+        payload.shared_folder_key = Utils.bytes_to_url_safe_str(encrypted_folder_key)
+
+        payload
+      end
+
+      def prepare_update_folder_payload(folder_uid:, data:)
+        payload = Dto::UpdateFolderPayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.folder_uid = folder_uid
+        payload.data = Utils.bytes_to_url_safe_str(data)
+        payload
+      end
+
+      def prepare_delete_folder_payload(folder_uids, force)
+        payload = Dto::DeleteFolderPayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.folder_uids = folder_uids
+        payload.force_deletion = force
+        payload
+      end
+
+      def prepare_file_upload_payload(file_record_uid:, file_record_key:, file_record_data:, owner_record_uid:, owner_record_data:, owner_record_revision:, link_key:, file_size:)
+        payload = Dto::FileUploadPayload.new
+        payload.client_version = KeeperGlobals.client_version
+        payload.client_id = @config.get_string(ConfigKeys::KEY_CLIENT_ID)
+        payload.file_record_uid = file_record_uid
+        payload.file_record_key = Utils.bytes_to_base64(file_record_key)
+        payload.file_record_data = Utils.bytes_to_base64(file_record_data)
+        payload.owner_record_uid = owner_record_uid
+        payload.owner_record_data = Utils.bytes_to_base64(owner_record_data)
+        payload.owner_record_revision = owner_record_revision
+        payload.link_key = Utils.bytes_to_base64(link_key)
+        payload.file_size = file_size
+        payload
+      end
+
+      def upload_file_function(url, parameters, encrypted_file_data)
+        uri = URI(url)
+
+        # Parse parameters if it's a JSON string
+        params_hash = parameters.is_a?(String) ? JSON.parse(parameters) : parameters
+
+        # Build multipart form data
+        # Form data should include both the file and the parameters
+        form_data = params_hash.merge({ 'file' => encrypted_file_data })
+
+        request = Net::HTTP::Post.new(uri)
+
+        # Set form with file upload
+        # The file field needs to be handled as binary data
+        boundary = "----RubyMultipartPost#{rand(1000000)}"
+        body = []
+
+        # Add regular form fields first
+        params_hash.each do |key, value|
+          body << "--#{boundary}\r\n"
+          body << "Content-Disposition: form-data; name=\"#{key}\"\r\n\r\n"
+          body << "#{value}\r\n"
+        end
+
+        # Add file field
+        body << "--#{boundary}\r\n"
+        body << "Content-Disposition: form-data; name=\"file\"; filename=\"file\"\r\n"
+        body << "Content-Type: application/octet-stream\r\n\r\n"
+        body << encrypted_file_data
+        body << "\r\n--#{boundary}--\r\n"
+
+        request.body = body.join
+        request['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
+
+        http = create_http_client(uri)
+        configure_http_ssl(http)
+
+        response = http.request(request)
+
+        raise NetworkError, "File upload failed: HTTP #{response.code} - #{response.body}" unless response.code.to_i.between?(200, 299)
+
+        true
+      end
+    end
+  end
+end