vagrant-s3-multidownloader 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c4319db5427dee9a2ddaeb79f3494e2847bf9cdb261985a82a74fe86c462abc8
+  data.tar.gz: f33293ee80907e491d1d8c3c61c60984e0e7df3d9cea370d01a916e42d78da14
+SHA512:
+  metadata.gz: ef026874b3e549060199c63ff83f88e3dbe8af4d348b6084f95c626500145ebecf71eb7070a610b24b9f9965a2cd0f17879edfa24fa7b268fe09797d0f8ffaba
+  data.tar.gz: c400a6303e0d084b984c0ac4f14617bf22aa639a0282fedd9246e4af4e665e2e27aafa4f08a874a7cb7ca0303f57e3509a6df01e7cfeab35b2679eedda1dad94

data/lib/vagrant-s3-multidownloader/downloader.rb

@@ -0,0 +1,488 @@
+require 'vagrant'
+require 'aws-sdk-s3'
+require 'uri'
+require 'yaml'
+require 'thread'
+require 'fileutils'
+require 'json'
+require 'tmpdir'
+require_relative 'metadata_handler'
+
+module VagrantPlugins
+  module S3MultiDownloader
+    class Downloader
+
+      def self.patch_vagrant_downloader
+        ::Vagrant::Util::Downloader.class_eval do
+          alias_method :original_download!, :download! unless method_defined?(:original_download!)
+
+          def download!
+            if @source.to_s.start_with?("s3://")
+              logger = Log4r::Logger.new("vagrant::downloaders::s3")
+              logger.info("Detected S3 URL: #{@source}")
+
+              # Use our custom S3 downloader
+              downloader = VagrantPlugins::S3MultiD ownloader::Downloader.new(
+                @source, @destination, @ui, continue: @continue, headers: @headers
+              )
+              return downloader.download!
+            end
+            original_download!
+          end
+        end
+      end
+
+      # Automatically patch the downloader when this module is loaded
+      self.patch_vagrant_downloader
+
+      def initialize(source, destination, ui, options = {})
+        @source = source
+        @destination = destination
+        @ui = ui
+        @options = options
+        @env = options[:env]
+        @logger = Log4r::Logger.new("vagrant::s3-multidownloader")
+        @metadata_handler = MetadataHandler.new(ui)
+
+        @logger.info("S3MultiDownloader initialized")
+        @logger.info("Source: #{@source}")
+        @logger.info("Destination: #{@destination}")
+      end
+
+      def download!
+        # Parse the S3 URL
+        uri, bucket, key = parse_s3_url(@source)
+
+        # Log download information
+        log_download_info(bucket, key)
+
+        # Create S3 client
+        client = create_s3_client
+
+        begin
+          # Check for requested version from Vagrantfile
+          requested_version = @metadata_handler.get_requested_version(@env)
+          log_message(:info, "Box version requested: #{requested_version || 'latest'}") if requested_version
+
+          # Check if this is a metadata.json file
+          is_metadata = key.end_with?("metadata.json")
+          log_message(:detail, "File type: #{is_metadata ? 'metadata.json' : 'box file'}")
+
+          if is_metadata
+            # Handle metadata.json specially
+            handle_metadata_file(client, bucket, key, requested_version)
+          else
+            # Direct box download
+            download_s3_object(client, bucket, key)
+            log_message(:success, "Successfully downloaded #{@source} to #{@destination}")
+          end
+        rescue Aws::S3::Errors::Forbidden => e
+          handle_forbidden_error(e)
+        rescue Aws::S3::Errors::NotFound => e
+          handle_not_found_error(e, bucket, key)
+        rescue Aws::S3::Errors::ServiceError => e
+          handle_service_error(e)
+        rescue StandardError => e
+          handle_standard_error(e)
+        end
+      end
+
+      private
+
+      def parse_s3_url(url)
+        uri = URI.parse(url)
+        bucket = uri.host
+        key = uri.path[1..-1]
+        return uri, bucket, key
+      end
+
+      def s3_url_is_metadata?(url)
+        return false unless url
+        uri = URI.parse(url)
+        path = uri.path
+        path.end_with?('metadata.json')
+      end
+
+      def log_download_info(bucket, key)
+        log_message(:info, "Starting download from S3: #{@source}")
+        log_message(:detail, "Bucket: #{bucket}")
+        log_message(:detail, "Key: #{key}")
+
+        # Log headers if available for debugging
+        if @options[:headers]
+          @logger.debug("Request headers: #{@options[:headers].inspect}")
+          if @options[:headers].is_a?(Hash) && @options[:headers]["User-Agent"]
+            log_message(:detail, "User agent: #{@options[:headers]["User-Agent"]}")
+          end
+        end
+      end
+
+      def create_s3_client
+        # Load AWS config (credentials, region)
+        aws_config = load_aws_config
+        profile = aws_config[:profile]
+        log_message(:detail, "AWS config loaded with region: #{aws_config[:region]}")
+        log_message(:detail, "Credentials available: #{!aws_config[:access_key_id].nil?}")
+
+        # Make sure the destination directory exists
+        FileUtils.mkdir_p(File.dirname(@destination))
+
+        # Initialize client options
+        client_options = {
+          force_path_style: true,  # Required for most S3-compatible storage systems
+          ssl_verify_peer: false   # Don't verify SSL certificates
+        }
+
+        # Get endpoint URL from AWS config file
+        endpoint_url = find_endpoint_url(profile)
+
+        # If not found in config file, check environment as fallback
+        if endpoint_url.nil? && ENV['AWS_ENDPOINT_URL']
+          endpoint_url = ENV['AWS_ENDPOINT_URL']
+          log_message(:detail, "Using S3 endpoint from environment: #{endpoint_url}")
+        end
+
+        if endpoint_url
+          log_message(:info, "Using S3 endpoint URL: #{endpoint_url}")
+          # Add endpoint URL to client options
+          client_options[:endpoint] = endpoint_url
+        else
+          log_message(:detail, "No endpoint URL configured, will use AWS default endpoints")
+        end
+
+        log_message(:info, "Attempting to download actual box from S3")
+
+        # Setup credentials from config files first, not from environment
+        if aws_config[:access_key_id] && aws_config[:secret_access_key]
+          # Use explicit credentials from config files
+          client_options[:credentials] = Aws::Credentials.new(
+            aws_config[:access_key_id],
+            aws_config[:secret_access_key]
+          )
+          log_message(:detail, "Using credentials from config files")
+        elsif profile && !profile.empty?
+          # Try to use profile directly through SharedCredentials
+          begin
+            credentials = Aws::SharedCredentials.new(profile_name: profile)
+            client_options[:credentials] = credentials
+            log_message(:detail, "Using AWS shared credentials from profile: #{profile}")
+          rescue Aws::Errors::NoSuchProfileError => e
+            log_message(:warn, "AWS profile '#{profile}' not found in credentials file: #{e.message}")
+            log_message(:warn, "Falling back to default credential provider chain")
+          end
+        else
+          log_message(:detail, "No specific credentials provided, falling back to default credential provider chain")
+          # Let the AWS SDK use its default credential provider chain
+        end
+
+        # Ensure we have a region set (required by AWS SDK)
+        client_options[:region] = aws_config[:region] || 'us-east-1'
+
+        log_message(:detail, "Creating S3 client with profile credentials and options: #{client_options.inspect}")
+        Aws::S3::Client.new(client_options)
+      end
+
+      def download_s3_object(client, bucket, key, destination = nil)
+        dest = destination || @destination
+        log_message(:info, "Downloading #{bucket}/#{key} using single thread")
+
+        begin
+          # Make sure the destination directory exists
+          FileUtils.mkdir_p(File.dirname(dest))
+
+          # Get the entire object
+          response = client.get_object(bucket: bucket, key: key)
+
+          # Write the file to destination
+          File.open(dest, 'wb') do |file|
+            file.write(response.body.read)
+          end
+
+          log_message(:detail, "Download completed")
+        rescue StandardError => e
+          log_message(:error, "Download error: #{e.message}")
+          raise e
+        end
+      end
+
+      def handle_metadata_file(client, bucket, key, requested_version = nil)
+        log_message(:info, "Processing metadata.json file for versioning")
+
+        # First download the metadata file
+        download_s3_object(client, bucket, key)
+
+        # Process the metadata file
+        if File.exist?(@destination)
+          metadata = @metadata_handler.process_metadata_file(@destination)
+
+          if metadata
+            # Find the requested or latest version
+            version_data = @metadata_handler.find_version_in_metadata(metadata, requested_version)
+
+            if version_data
+              # Find the provider URL
+              provider_name = @env&.dig(:machine)&.provider&.name&.to_s
+              provider_url = @metadata_handler.find_provider_url(version_data, provider_name)
+
+              if provider_url && provider_url.start_with?('s3://')
+                log_message(:info, "Found box URL for version #{version_data['version']}: #{provider_url}")
+
+                # Create a temp path for the box file
+                box_temp_path = File.join(Dir.tmpdir, "vagrant_box_#{Time.now.to_i}")
+
+                # Download the actual box
+                box_uri, box_bucket, box_key = parse_s3_url(provider_url)
+
+                log_message(:info, "Downloading box from: #{provider_url}")
+                download_s3_object(client, box_bucket, box_key, box_temp_path)
+
+                # Copy the downloaded box to the destination
+                FileUtils.cp(box_temp_path, @destination)
+                FileUtils.rm(box_temp_path) if File.exist?(box_temp_path)
+
+                log_message(:success, "Successfully downloaded box version #{version_data['version']} to #{@destination}")
+              else
+                log_message(:warn, "No S3 URL found for provider #{provider_name || 'any'} in version #{version_data['version']}")
+              end
+            else
+              log_message(:warn, "No suitable version found in metadata.json")
+            end
+          end
+        else
+          log_message(:error, "Failed to download metadata.json file")
+        end
+      end
+
+      def process_metadata_file(metadata_path)
+        @metadata_handler.process_metadata_file(metadata_path)
+      end
+
+      def handle_forbidden_error(error)
+        log_message(:error, "S3 access forbidden. Check your AWS credentials.")
+        log_message(:error, "Error details: #{error.message}")
+        raise Vagrant::Errors::DownloaderError.new(message: "S3 access forbidden: #{error.message}. Please check your AWS credentials.")
+      end
+
+      def handle_not_found_error(error, bucket, key)
+        log_message(:error, "S3 file not found: #{error.message}")
+        raise Vagrant::Errors::DownloaderError.new(message: "S3 file not found: #{bucket}/#{key}. Please check the S3 URL.")
+      end
+
+      def handle_service_error(error)
+        log_message(:error, "S3 error: #{error.message}")
+        raise Vagrant::Errors::DownloaderError.new(message: "S3 error: #{error.message}")
+      end
+
+      def handle_standard_error(error)
+        log_message(:error, "Error: #{error.class.name} - #{error.message}")
+        raise Vagrant::Errors::DownloaderError.new(message: "Download error: #{error.message}")
+      end
+
+      # Helper method to log messages to UI or logger
+      def log_message(level, message)
+        if @ui.respond_to?(level)
+          @ui.send(level, message)
+        else
+          # Default to using the logger with an appropriate log level
+          case level
+          when :detail
+            @logger.info(message)
+          when :success
+            @logger.info("SUCCESS: #{message}")
+          when :warn
+            @logger.warn(message)
+          when :error
+            @logger.error(message)
+          else
+            @logger.send(level, message)
+          end
+        end
+      end
+
+      def load_aws_config
+        aws_config_file = File.expand_path("~/.aws/config")
+        aws_credentials_file = File.expand_path("~/.aws/credentials")
+
+        # Initialize with empty config
+        config = { region: "default" }
+
+        # Get profile from environment variable or use default
+        profile = ENV['AWS_PROFILE'] || ENV['AWS_DEFAULT_PROFILE'] || 'default'
+        config[:profile] = profile
+
+        log_message(:info, "Loading AWS credentials from profile: #{profile}")
+
+        # Try to load credentials from AWS credentials file first
+        if File.exist?(aws_credentials_file)
+          begin
+            # Parse AWS credentials file (INI format)
+            credentials_content = File.read(aws_credentials_file)
+            section = 'default'
+            credentials = {}
+
+            credentials_content.each_line do |line|
+              line = line.strip
+              next if line.empty? || line.start_with?('#')
+
+              if line =~ /^\[([^\]]+)\]/
+                section = $1
+                credentials[section] ||= {}
+              elsif line =~ /^([^=]+)=(.*)$/
+                key = $1.strip
+                value = $2.strip
+                credentials[section] ||= {}
+                credentials[section][key] = value
+              end
+            end
+
+            # First try to load the specified profile, then fall back to default
+            if credentials[profile]
+              log_message(:detail, "Found credentials for profile: #{profile}")
+              config[:access_key_id] = credentials[profile]['aws_access_key_id']
+              config[:secret_access_key] = credentials[profile]['aws_secret_access_key']
+            elsif credentials['default'] && profile != 'default'
+              log_message(:detail, "Profile #{profile} not found, using default profile")
+              config[:access_key_id] = credentials['default']['aws_access_key_id']
+              config[:secret_access_key] = credentials['default']['aws_secret_access_key']
+            end
+          rescue => e
+            # Log the error but continue
+            log_message(:warn, "Error reading AWS credentials file: #{e.message}")
+          end
+        else
+          log_message(:detail, "AWS credentials file not found at #{aws_credentials_file}")
+        end
+
+        # Fall back to environment variables if no credentials found in file
+        if config[:access_key_id].nil? && ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY']
+          log_message(:info, "Using AWS credentials from environment variables")
+          config[:access_key_id] = ENV['AWS_ACCESS_KEY_ID']
+          config[:secret_access_key] = ENV['AWS_SECRET_ACCESS_KEY']
+        end
+
+        # Load region from config file first
+        if File.exist?(aws_config_file)
+          begin
+            # Parse AWS config file (INI format)
+            config_content = File.read(aws_config_file)
+            section = 'default'
+            aws_configs = {}
+
+            config_content.each_line do |line|
+              line = line.strip
+              next if line.empty? || line.start_with?('#')
+
+              if line =~ /^\[([^\]]+)\]/
+                section = $1
+                # Handle "profile" prefix in config file
+                if section.start_with?('profile ')
+                  section = section.sub('profile ', '')
+                end
+                aws_configs[section] ||= {}
+              elsif line =~ /^([^=]+)=(.*)$/
+                key = $1.strip
+                value = $2.strip
+                aws_configs[section] ||= {}
+                aws_configs[section][key] = value
+              end
+            end
+
+            # Try to get region from the specified profile first
+            if aws_configs[profile] && aws_configs[profile]['region']
+              config[:region] = aws_configs[profile]['region']
+              log_message(:detail, "Using region '#{config[:region]}' from profile '#{profile}'")
+            # Then try the default profile
+            elsif aws_configs['default'] && aws_configs['default']['region']
+              config[:region] = aws_configs['default']['region']
+              log_message(:detail, "Using region '#{config[:region]}' from default profile")
+            end
+          rescue => e
+            # Log the error but continue
+            log_message(:warn, "Error reading AWS config file: #{e.message}")
+          end
+        else
+          log_message(:detail, "AWS config file not found at #{aws_config_file}")
+        end
+
+        # Fall back to environment variable if no region in config file
+        if config[:region].nil? && ENV['AWS_REGION']
+          config[:region] = ENV['AWS_REGION']
+          log_message(:detail, "Using region from environment: #{config[:region]}")
+        end
+
+        # Default to us-east-1 if no region specified
+        config[:region] ||= "us-east-1"
+        log_message(:info, "Using AWS region: #{config[:region]}")
+
+        config
+      end
+
+      def find_endpoint_url(profile)
+        aws_config_file = File.expand_path("~/.aws/config")
+
+        if File.exist?(aws_config_file)
+          begin
+            # Parse AWS config file (INI format)
+            config_content = File.read(aws_config_file)
+            section = 'default'
+            aws_configs = {}
+
+            config_content.each_line do |line|
+              line = line.strip
+              next if line.empty? || line.start_with?('#')
+
+              if line =~ /^\[([^\]]+)\]/
+                section = $1
+                # Handle "profile" prefix in config file
+                if section.start_with?('profile ')
+                  section = section.sub('profile ', '')
+                end
+                aws_configs[section] ||= {}
+              elsif line =~ /^([^=]+)=(.*)$/
+                key = $1.strip
+                value = $2.strip
+                aws_configs[section] ||= {}
+                aws_configs[section][key] = value
+              end
+            end
+
+            # Check for endpoint_url or endpoint in the specified profile
+            if aws_configs[profile]
+              endpoint = aws_configs[profile]['endpoint_url'] ||
+                         aws_configs[profile]['endpoint'] ||
+                         aws_configs[profile]['s3_endpoint_url'] ||
+                         aws_configs[profile]['s3_endpoint']
+
+              if endpoint
+                log_message(:detail, "Found endpoint URL in profile '#{profile}': #{endpoint}")
+                return endpoint
+              end
+            end
+
+            # Check default profile if no endpoint found in specified profile
+            if aws_configs['default'] && profile != 'default'
+              endpoint = aws_configs['default']['endpoint_url'] ||
+                         aws_configs['default']['endpoint'] ||
+                         aws_configs['default']['s3_endpoint_url'] ||
+                         aws_configs['default']['s3_endpoint']
+
+              if endpoint
+                log_message(:detail, "Found endpoint URL in default profile: #{endpoint}")
+                return endpoint
+              end
+            end
+
+            log_message(:detail, "No endpoint URL found in AWS config profiles")
+            return nil
+          rescue => e
+            log_message(:warn, "Error reading AWS config file for endpoint: #{e.message}")
+            return nil
+          end
+        else
+          log_message(:detail, "AWS config file not found at #{aws_config_file}")
+          return nil
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-s3-multidownloader/metadata_handler.rb

@@ -0,0 +1,187 @@
+require 'json'
+require 'vagrant/util/downloader'
+require 'uri'
+
+module VagrantPlugins
+  module S3MultiDownloader
+    class MetadataHandler
+      attr_reader :ui, :logger
+
+      def initialize(ui)
+        @ui = ui
+        @logger = Log4r::Logger.new("vagrant::s3-multidownloader::metadata")
+      end
+
+      # Extract version from the environment if available
+      def get_requested_version(env)
+        return nil if !env || !env[:machine] || !env[:machine].config || !env[:machine].config.vm
+
+        if env[:machine].config.vm.box_version
+          version = env[:machine].config.vm.box_version
+          @ui.detail("Found box_version in Vagrantfile: #{version}")
+          return version
+        end
+
+        nil
+      end
+
+      # Check if the URL points to a metadata.json file
+      def metadata_url?(url)
+        return false unless url
+        url.to_s.end_with?('metadata.json')
+      end
+
+      # Download and parse a metadata.json file
+      def download_metadata(url, destination, client, bucket, key)
+        @ui.info("Downloading metadata from S3: #{url}")
+
+        # Use the client directly to download the metadata file
+        client.get_object(
+          response_target: destination,
+          bucket: bucket,
+          key: key
+        )
+
+        # Parse the metadata
+        begin
+          metadata = JSON.parse(File.read(destination))
+          return metadata
+        rescue JSON::ParserError => e
+          @ui.error("Failed to parse metadata.json: #{e.message}")
+          return nil
+        rescue StandardError => e
+          @ui.error("Error processing metadata.json: #{e.message}")
+          return nil
+        end
+      end
+
+      # Find a specific version in the metadata
+      def find_version_in_metadata(metadata, version = nil)
+        return nil unless metadata && metadata.key?('versions') && metadata['versions'].is_a?(Array)
+
+        available_versions = metadata['versions']
+        @ui.detail("Available versions: #{available_versions.map { |v| v['version'] }.join(', ')}")
+
+        if version
+          # This handles both exact version number and version constraints
+          if version.include?('>') || version.include?('<') || version.include?('~>')
+            @ui.detail("Detected version constraint: #{version}")
+            # Parse the constraint and find the highest satisfying version
+            begin
+              require 'vagrant/util/template_renderer'
+              version_constraint = Gem::Requirement.new(version)
+
+              # Find all versions that satisfy the constraint
+              satisfying_versions = available_versions.select do |v|
+                begin
+                  version_number = Gem::Version.new(v['version'])
+                  version_constraint.satisfied_by?(version_number)
+                rescue
+                  false
+                end
+              end
+
+              if satisfying_versions.empty?
+                @ui.warn("No versions matching constraint #{version} found in metadata")
+                return nil
+              end
+
+              # Select the highest version that satisfies the constraint
+              selected_version = satisfying_versions.sort_by { |v| Gem::Version.new(v['version']) rescue Gem::Version.new('0.0.0') }.last
+              @ui.success("Selected version #{selected_version['version']} for constraint #{version}")
+              return selected_version
+            rescue => e
+              @ui.error("Error parsing version constraint: #{e.message}")
+              # Fall back to exact match
+              @ui.warn("Falling back to exact version match")
+            end
+          end
+
+          # Find the requested version (exact match)
+          requested_version = available_versions.find { |v| v['version'] == version }
+          if requested_version
+            @ui.success("Found requested version: #{version}")
+            return requested_version
+          else
+            @ui.warn("Requested version #{version} not found in metadata")
+            return nil
+          end
+        else
+          # No specific version requested, use the latest version
+          latest_version = available_versions.sort_by { |v| Gem::Version.new(v['version']) rescue Gem::Version.new('0.0.0') }.last
+          @ui.detail("Using latest version: #{latest_version['version']}")
+          return latest_version
+        end
+      end
+
+      # Find the provider URL in a version
+      def find_provider_url(version_data, provider_name = nil)
+        return nil unless version_data && version_data.key?('providers') && version_data['providers'].is_a?(Array)
+
+        providers = version_data['providers']
+
+        # If provider_name is specified, find that specific provider
+        if provider_name && !provider_name.empty?
+          provider = providers.find { |p| p['name'] == provider_name }
+          if provider && provider.key?('url')
+            @ui.detail("Found URL for provider #{provider_name}: #{provider['url']}")
+            return provider['url']
+          end
+        end
+
+        # Otherwise, just return the first provider with a URL
+        providers.each do |provider|
+          if provider.key?('url')
+            @ui.detail("Using provider #{provider['name']}: #{provider['url']}")
+            return provider['url']
+          end
+        end
+
+        nil
+      end
+
+      # Process a metadata file to ensure all S3 URLs are properly formatted
+      def process_metadata_file(metadata_path)
+        @ui.info("Processing metadata.json file for S3 URLs")
+        begin
+          # Read and parse the metadata.json file
+          metadata = JSON.parse(File.read(metadata_path))
+
+          # Check if the metadata has versions
+          if metadata.key?('versions') && metadata['versions'].is_a?(Array)
+            @ui.detail("Found #{metadata['versions'].size} versions in metadata")
+
+            # Update all S3 URLs in the metadata to ensure they'll be handled correctly
+            metadata['versions'].each_with_index do |version, index|
+              if version.key?('providers') && version['providers'].is_a?(Array)
+                version['providers'].each do |provider|
+                  if provider.key?('url') && provider['url'].to_s.start_with?('s3://')
+                    @ui.detail("Found S3 URL in metadata: #{provider['url']}")
+                    # No need to modify the URL as the plugin already handles s3:// URLs
+                  end
+                end
+              end
+            end
+
+            # Write the updated metadata back to the file
+            File.open(metadata_path, 'w') do |file|
+              file.write(JSON.pretty_generate(metadata))
+            end
+
+            @ui.detail("Metadata processing complete")
+            return metadata
+          else
+            @ui.detail("No versions found in metadata.json")
+            return nil
+          end
+        rescue JSON::ParserError => e
+          @ui.error("Failed to parse metadata.json: #{e.message}")
+          return nil
+        rescue StandardError => e
+          @ui.error("Error processing metadata.json: #{e.message}")
+          return nil
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-s3-multidownloader/middleware/handle_metadata_versions.rb

@@ -0,0 +1,87 @@
+require 'vagrant'
+require 'vagrant/action/builtin/box_add'
+
+module VagrantPlugins
+  module S3MultiDownloader
+    class HandleMetadataVersions
+      def initialize(app, env)
+        @app = app
+        @env = env
+        @logger = Log4r::Logger.new("vagrant::s3-multidownloader::metadata_versions")
+      end
+
+      def call(env)
+        @env = env
+
+        # Patch Vagrant's box add action to recognize S3 URLs as metadata URLs
+        if !defined? @@patched_box_add
+          patch_box_add_action
+          @@patched_box_add = true
+        end
+
+        @app.call(env)
+      end
+
+      private
+
+      def patch_box_add_action
+        @logger.info("Patching Vagrant::Action::Builtin::BoxAdd to support S3 metadata URLs")
+
+        # Override the method that determines if a URL is a metadata URL
+        if Vagrant::Action::Builtin::BoxAdd.instance_methods.include?(:metadata_url?)
+          Vagrant::Action::Builtin::BoxAdd.class_eval do
+            alias_method :original_metadata_url?, :metadata_url?
+
+            def metadata_url?(url, env=nil)
+              if url.to_s.start_with?("s3://")
+                # For S3 URLs, consider them as metadata URLs if they end with metadata.json
+                # or if they don't have a file extension (implying they might be a base URL)
+                path = URI.parse(url.to_s).path
+                return true if path.end_with?("metadata.json")
+                return true if File.extname(path).empty?
+              end
+
+              # Otherwise use the original logic
+              original_metadata_url?(url, env)
+            end
+          end
+        end
+
+        # Override the check that prevents version constraints with direct file URLs
+        if Vagrant::Action::Builtin::BoxAdd.instance_methods.include?(:validate_box_url)
+          Vagrant::Action::Builtin::BoxAdd.class_eval do
+            alias_method :original_validate_box_url, :validate_box_url
+
+            def validate_box_url
+              if @url && @url.to_s.start_with?("s3://")
+                # Allow version constraints with S3 URLs
+                return true
+              end
+
+              # Otherwise use the original validation
+              original_validate_box_url
+            end
+          end
+        end
+
+        # Also patch the method that validates version constraints
+        if Vagrant::Action::Builtin::BoxAdd.instance_methods.include?(:validate_version)
+          Vagrant::Action::Builtin::BoxAdd.class_eval do
+            alias_method :original_validate_version, :validate_version
+
+            def validate_version
+              # If we have an S3 URL, skip the version validation completely
+              if @url && @url.to_s.start_with?("s3://")
+                @logger.debug("Skipping version validation for S3 URL")
+                return true
+              end
+
+              # Fall back to the original validation for non-S3 URLs
+              original_validate_version
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vagrant-s3-multidownloader/middleware/handle_s3_urls.rb

@@ -0,0 +1,106 @@
+require 'uri'
+require 'vagrant/util/downloader'
+
+module VagrantPlugins
+  module S3MultiDownloader
+    class HandleS3Urls
+      def initialize(app, env)
+        @app = app
+        @env = env
+        @logger = Log4r::Logger.new("vagrant::s3-multidownloader")
+      end
+
+      def call(env)
+        @env = env
+
+        # Save references to the original Downloader methods
+        unless defined? @@original_curl_execute
+          @@original_curl_execute = Vagrant::Util::Downloader.instance_method(:execute_curl)
+        end
+
+        unless defined? @@original_download
+          @@original_download = Vagrant::Util::Downloader.instance_method(:download!)
+        end
+
+        # Override both download! and execute_curl to handle s3:// URLs
+        # We need to intercept at both levels to ensure we catch all S3 URL usage
+        unless Vagrant::Util::Downloader.instance_methods.include?(:original_download)
+          Vagrant::Util::Downloader.class_eval do
+            alias_method :original_download, :download!
+
+            def download!
+              if @source.to_s.start_with?("s3://")
+                ui = @ui || @logger
+                ui.info("Intercepted S3 download at download! level: #{@source}")
+                downloader = VagrantPlugins::S3MultiDownloader::Downloader.new(
+                  @source, @destination, ui,
+                  continue: @continue,
+                  headers: @headers,
+                  env: @env
+                )
+                return downloader.download!
+              end
+              original_download
+            end
+          end
+        end
+
+        # Also override execute_curl to catch any direct curl executions with S3 URLs
+        unless Vagrant::Util::Downloader.instance_methods.include?(:original_execute_curl)
+          Vagrant::Util::Downloader.class_eval do
+            alias_method :original_execute_curl, :execute_curl
+
+            def execute_curl(options, subprocess_options, &data_proc)
+              # More aggressively check for S3 URLs in the options array
+              # This needs to catch any occurrence of s3:// in the curl command
+              s3_url = nil
+              options.each do |opt|
+                if opt.to_s =~ /^s3:\/\//
+                  s3_url = opt.to_s
+                  break
+                end
+              end
+
+              if s3_url
+                ui = @ui || @logger
+                ui.info("Intercepted S3 URL in curl command: #{s3_url}")
+
+                # Find the output path from the curl options
+                output_index = options.find_index("--output")
+                output_path = output_index ? options[output_index + 1] : nil
+
+                if output_path
+                  ui.detail("Output path: #{output_path}")
+                  # Use our S3 downloader instead
+                  downloader = VagrantPlugins::S3MultiDownloader::Downloader.new(
+                    s3_url, output_path, ui, env: @env
+                  )
+
+                  begin
+                    ui.detail("Downloading from S3 via custom downloader: #{s3_url}")
+                    downloader.download!
+                    ui.detail("S3 download completed successfully")
+                    return 0 # Return success status code
+                  rescue => e
+                    ui.error("S3 download error: #{e.message}")
+                    ui.error(e.backtrace.join("\n"))
+                    return 1 # Return error status code
+                  end
+                else
+                  ui.error("Could not determine output path for S3 download")
+                  return 1
+                end
+              end
+
+              # For non-S3 URLs, use the original method
+              original_execute_curl(options, subprocess_options, &data_proc)
+            end
+          end
+        end
+
+        # Continue middleware chain
+        @app.call(env)
+      end
+    end
+  end
+end

data/lib/vagrant-s3-multidownloader/plugin.rb

@@ -0,0 +1,40 @@
+require 'vagrant'
+require_relative 'downloader'
+require_relative 'middleware/handle_s3_urls'
+require_relative 'middleware/handle_metadata_versions'
+
+module VagrantPlugins
+  module S3MultiDownloader
+    class Plugin < Vagrant.plugin('2')
+      name 'S3MultiDownloader'
+      description 'Enables downloading Vagrant boxes from S3-compatible storage with multithreading'
+
+      # Hook into all possible places where box downloads might be triggered
+      action_hook(:s3_multi_downloader_box_url, :authenticate_box_url) do |hook|
+        hook.prepend(HandleS3Urls)
+        hook.prepend(HandleMetadataVersions)
+      end
+
+      action_hook(:s3_multi_downloader_box_add, :box_add) do |hook|
+        hook.prepend(HandleS3Urls)
+        hook.prepend(HandleMetadataVersions)
+      end
+
+      action_hook(:s3_multi_downloader_box_update, :box_update) do |hook|
+        hook.prepend(HandleS3Urls)
+        hook.prepend(HandleMetadataVersions)
+      end
+
+      # Add more hooks to ensure coverage of all possible download paths
+      action_hook(:s3_multi_downloader_initialize, :environment_load) do |hook|
+        hook.append(HandleS3Urls)
+        hook.append(HandleMetadataVersions)
+      end
+
+      action_hook(:s3_multi_downloader_download, :download_box) do |hook|
+        hook.prepend(HandleS3Urls)
+        hook.prepend(HandleMetadataVersions)
+      end
+    end
+  end
+end

data/lib/vagrant-s3-multidownloader.rb

@@ -0,0 +1,31 @@
+begin
+  require 'vagrant'
+rescue LoadError
+  raise 'The Vagrant S3 Multi Downloader plugin must be run within Vagrant.'
+end
+
+# Load downloader first so it can be used by the plugin
+require_relative 'vagrant-s3-multidownloader/downloader'
+require_relative 'vagrant-s3-multidownloader/middleware/handle_s3_urls'
+require_relative 'vagrant-s3-multidownloader/middleware/handle_metadata_versions'
+require_relative 'vagrant-s3-multidownloader/plugin'
+
+module VagrantPlugins
+  module S3MultiDownloader
+    # Register S3 protocol handler early
+    def self.register_handlers
+      # Apply patches to Vagrant's downloader system immediately
+      VagrantPlugins::S3MultiDownloader::Downloader.patch_vagrant_downloader
+
+      # Log that we've loaded
+      logger = Log4r::Logger.new("vagrant::s3-multidownloader")
+      logger.info("S3 Multi Downloader plugin loaded")
+      logger.info("S3 URL protocol handler registered")
+    end
+
+    # Automatically register our handlers when this module is loaded
+    register_handlers
+  end
+end
+
+

data/vagrant-s3-multidownloader.gemspec

@@ -0,0 +1,12 @@
+Gem::Specification.new do |spec|
+  spec.name          = 'vagrant-s3-multidownloader'
+  spec.version       = '0.3.0'
+  spec.summary       = 'Download Vagrant boxes from S3-compatible storage using AWS SDK.'
+  spec.description   = 'Registers a custom downloader for the s3:// protocol to fetch boxes from any S3 API endpoint.'
+  spec.authors       = ['Maksim Razumov']
+  spec.email         = ['maksim.razumov@xiag.ch']
+  spec.files         = Dir.glob('lib/**/*') + ['vagrant-s3-multidownloader.gemspec']
+  spec.require_paths = ['lib']
+  spec.add_runtime_dependency 'vagrant', '>= 2.2.0'
+  spec.add_runtime_dependency 'aws-sdk-s3', '~> 1.0'
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: vagrant-s3-multidownloader
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Maksim Razumov
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-05-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: vagrant
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Registers a custom downloader for the s3:// protocol to fetch boxes from
+  any S3 API endpoint.
+email:
+- maksim.razumov@xiag.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/vagrant-s3-multidownloader.rb
+- lib/vagrant-s3-multidownloader/downloader.rb
+- lib/vagrant-s3-multidownloader/metadata_handler.rb
+- lib/vagrant-s3-multidownloader/middleware/handle_metadata_versions.rb
+- lib/vagrant-s3-multidownloader/middleware/handle_s3_urls.rb
+- lib/vagrant-s3-multidownloader/plugin.rb
+- vagrant-s3-multidownloader.gemspec
+homepage:
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.27
+signing_key:
+specification_version: 4
+summary: Download Vagrant boxes from S3-compatible storage using AWS SDK.
+test_files: []