iostreams 0.20.3 → 1.0.0.beta

data/lib/io_streams/file/writer.rb

@@ -1,22 +0,0 @@
-module IOStreams
-  module File
-    class Writer
-      # Write to a named file
-      #
-      # Note:
-      #   If an exception is raised whilst the file is being written to the file is removed to
-      #   prevent incomplete / partial files from being created.
-      def self.open(file_name, **args, &block)
-        raise(ArgumentError, 'File name must be a string') unless file_name.is_a?(String)
-
-        IOStreams::File::Path.mkpath(file_name)
-        begin
-          ::File.open(file_name, 'wb', &block)
-        rescue StandardError => e
-          ::File.unlink(file_name) if ::File.exist?(file_name)
-          raise(e)
-        end
-      end
-    end
-  end
-end

data/lib/io_streams/http/reader.rb

@@ -1,71 +0,0 @@
-require 'net/http'
-require 'uri'
-module IOStreams
-  module HTTP
-    # Read a file using an http get.
-    #
-    # For example:
-    #   IOStreams.reader('https://www5.fdic.gov/idasp/Offices2.zip') {|file| puts file.read}
-    #
-    # Direct example without unzipping the above file:
-    #   IOStreams::HTTP::Reader.new('https://www5.fdic.gov/idasp/Offices2.zip') {|file| puts file.read}
-    #
-    # Parameters:
-    #   uri: [String|URI]
-    #      URI of the file to download.
-    #     Example:
-    #       https://www5.fdic.gov/idasp/Offices2.zip
-    #
-    #   :username
-    #     When supplied, basic authentication is used with the username and password.
-    #     Default: nil
-    #
-    #   :password
-    #     Password to use use with basic authentication when the username is supplied.
-    #
-    # Notes:
-    # * Since Net::HTTP download only supports a push stream, the data is streamed into a tempfile first.
-    class Reader
-      def self.open(uri, username: nil, password: nil, **args, &block)
-        raise(ArgumentError, 'file_name must be a URI string') unless uri.is_a?(String) || uri.is_a?(URI)
-        handle_redirects(uri, username: username, password: password, **args, &block)
-      end
-
-      def self.handle_redirects(uri, username: nil, password: nil, http_redirect_count: 10, **args, &block)
-        uri    = URI.parse(uri) unless uri.is_a?(URI)
-        result = nil
-        raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
-
-        Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
-          request = Net::HTTP::Get.new(uri)
-          request.basic_auth(username, password) if username
-
-          http.request(request) do |response|
-            if response.is_a?(Net::HTTPNotFound)
-              raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}")
-            end
-            if response.is_a?(Net::HTTPUnauthorized)
-              raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
-            end
-            if response.is_a?(Net::HTTPRedirection)
-              new_uri = response['location']
-              return handle_redirects(new_uri, username: username, password: password, http_redirect_count: http_redirect_count - 1, **args, &block)
-            end
-
-            raise(IOStreams::Errors::CommunicationsFailure, "Invalid response code: #{response.code}") unless response.is_a?(Net::HTTPSuccess)
-
-            # Since Net::HTTP download only supports a push stream, write it to a tempfile first.
-            IOStreams::File::Path.temp_file_name('iostreams_http') do |file_name|
-              IOStreams::File::Writer.open(file_name) do |io|
-                response.read_body { |chunk| io.write(chunk) }
-              end
-              # Return a read stream
-              result = IOStreams::File::Reader.open(file_name, &block)
-            end
-          end
-        end
-        result
-      end
-    end
-  end
-end

data/lib/io_streams/s3.rb

@@ -1,26 +0,0 @@
-require 'uri'
-module IOStreams
-  module S3
-    autoload :Path, 'io_streams/s3/path'
-    autoload :Reader, 'io_streams/s3/reader'
-    autoload :Writer, 'io_streams/s3/writer'
-
-    # Sample URI: s3://mybucket/user/abc.zip
-    def self.parse_uri(uri)
-      uri = URI.parse(uri)
-      raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
-      {
-        bucket: uri.host,
-        key:    uri.path.sub(/\A\//, '')
-      }
-    end
-
-    def self.load_dependencies
-      return if defined?(::Aws::S3::Resource)
-
-      require 'aws-sdk-s3'
-    rescue LoadError => exc
-      raise(LoadError, "Install gem 'aws-sdk-s3' to read and write AWS S3 files: #{exc.message}")
-    end
-  end
-end

data/lib/io_streams/s3/path.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-module IOStreams
-  module S3
-    class Path < IOStreams::BasePath
-      def initialize(path)
-        IOStreams::S3.load_dependencies
-        @s3      = Aws::S3::Resource.new
-        @options = IOStreams::S3.parse_uri(path)
-        @object  = s3.bucket(options[:bucket]).object(options[:key])
-        super(path)
-      end
-
-      # S3 logically creates paths when a key is set.
-      def mkpath
-        self
-      end
-
-      def mkdir
-        self
-      end
-
-      def exist?
-        object.exists?
-      end
-
-      def size
-        object.size
-      end
-
-      def delete
-        object.delete
-        self
-      end
-
-      private
-
-      attr_reader :s3, :options, :object
-    end
-  end
-end

data/lib/io_streams/s3/reader.rb

@@ -1,28 +0,0 @@
-module IOStreams
-  module S3
-    class Reader
-      # Read from a AWS S3 file
-      def self.open(uri, region: nil, **args, &block)
-        raise(ArgumentError, 'file_name must be a URI string') unless uri.is_a?(String)
-
-        IOStreams::S3.load_dependencies
-
-        # https://aws.amazon.com/blogs/developer/using-resources/
-        s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
-        options = IOStreams::S3.parse_uri(uri)
-        object  = s3.bucket(options[:bucket]).object(options[:key])
-
-        begin
-          # Since S3 download only supports a push stream, write it to a tempfile first.
-          IOStreams::File::Path.temp_file_name('iostreams_s3') do |file_name|
-            args[:response_target] = file_name
-            object.get(args)
-
-            # Return a read stream
-            IOStreams::File::Reader.open(file_name, &block)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/io_streams/s3/writer.rb

@@ -1,85 +0,0 @@
-module IOStreams
-  module S3
-    class Writer
-      # Write to AWS S3
-      #
-      # Arguments:
-      #
-      # uri: [String]
-      #   Prefix must be: `s3://`
-      #   followed by bucket name,
-      #   followed by path and file_name (key).
-      #   Examples:
-      #     s3://my-bucket-name/file_name.txt
-      #     s3://my-bucket-name/some_path/file_name.csv
-      #
-      # region: [String]
-      #   AWS Region.
-      #   Default: ENV['AWS_REGION'], or supplied by ruby driver
-      #
-      # thread_count: [Integer]
-      #   The number of parallel multipart uploads
-      #   Default: 10
-      #
-      # tempfile: [Boolean]
-      #   Normally read data is stored in memory when building the parts in order to complete
-      #   the underlying multipart upload. By passing `:tempfile => true` data read will be
-      #   temporarily stored on disk reducing the memory footprint vastly.
-      #   Default: false
-      #
-      # part_size: [Integer]
-      #   Define how big each part size but the last should be.
-      #   Default: 5 * 1024 * 1024
-      #
-      # Other possible options extracted from AWS source code:
-      #   # See: https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#create_multipart_upload-instance_method
-      #
-      #   acl: "private", # accepts private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control
-      #   cache_control: "CacheControl",
-      #   content_disposition: "ContentDisposition",
-      #   content_encoding: "ContentEncoding",
-      #   content_language: "ContentLanguage",
-      #   content_type: "ContentType",
-      #   expires: Time.now,
-      #   grant_full_control: "GrantFullControl",
-      #   grant_read: "GrantRead",
-      #   grant_read_acp: "GrantReadACP",
-      #   grant_write_acp: "GrantWriteACP",
-      #   metadata: {
-      #     "MetadataKey" => "MetadataValue",
-      #   },
-      #   server_side_encryption: "AES256", # accepts AES256, aws:kms
-      #   storage_class: "STANDARD", # accepts STANDARD, REDUCED_REDUNDANCY, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING, GLACIER, DEEP_ARCHIVE
-      #   website_redirect_location: "WebsiteRedirectLocation",
-      #   sse_customer_algorithm: "SSECustomerAlgorithm",
-      #   sse_customer_key: "SSECustomerKey",
-      #   sse_customer_key_md5: "SSECustomerKeyMD5",
-      #   ssekms_key_id: "SSEKMSKeyId",
-      #   ssekms_encryption_context: "SSEKMSEncryptionContext",
-      #   request_payer: "requester", # accepts requester
-      #   tagging: "TaggingHeader",
-      #   object_lock_mode: "GOVERNANCE", # accepts GOVERNANCE, COMPLIANCE
-      #   object_lock_retain_until_date: Time.now,
-      #   object_lock_legal_hold_status: "ON", # accepts ON, OFF
-      #
-      # Raises [MultipartUploadError] If an object is being uploaded in
-      #   parts, and the upload can not be completed, then the upload is
-      #   aborted and this error is raised.  The raised error has a `#errors`
-      #   method that returns the failures that caused the upload to be
-      #   aborted.
-      def self.open(uri, region: nil, **args)
-        raise(ArgumentError, 'file_name must be a URI string') unless uri.is_a?(String)
-
-        IOStreams::S3.load_dependencies
-
-        options = IOStreams::S3.parse_uri(uri)
-        s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
-        object  = s3.bucket(options[:bucket]).object(options[:key])
-        object.upload_stream(args) do |s3|
-          s3.binmode
-          yield(s3)
-        end
-      end
-    end
-  end
-end

data/lib/io_streams/sftp/reader.rb

@@ -1,67 +0,0 @@
-module IOStreams
-  # Example:
-  #   IOStreams::SFTP::Reader.open(
-  #     'file.txt',
-  #     user:     'jbloggs',
-  #     password: 'secret',
-  #     host:     'example.org'
-  #   ) do |input|
-  #     puts input.read
-  #   end
-  module SFTP
-    class Reader
-      include SemanticLogger::Loggable if defined?(SemanticLogger)
-
-      # Stream to a remote file over sftp.
-      #
-      # file_name: [String]
-      #   Name of file to read from.
-      #
-      # user: [String]
-      #   Name of user to login with.
-      #
-      # password: [String]
-      #   Password for the user.
-      #
-      # host: [String]
-      #   Name of the host to connect to.
-      #
-      # port: [Integer]
-      #   Port to connect to at the above host.
-      #
-      # binary [true|false]
-      #   Whether to write in binary mode
-      #   Default: true
-      #
-      # options: [Hash]
-      #   Any options supported by Net::SSH.start
-      #
-      # Note:
-      # - Net::SFTP::StatusException means the file could not be read
-      def self.open(file_name, user:, password:, host:, port: 22, binary: true, options: {}, &block)
-        raise(NotImplementedError, 'Can only SFTP directly to a file name, not another stream.') if IOStreams.writer_stream?(file_name)
-
-        begin
-          require 'net/sftp' unless defined?(Net::SFTP)
-        rescue LoadError => e
-          raise(LoadError, "Please install the 'net-sftp' gem for SFTP streaming support. #{e.message}")
-        end
-
-        options                = options.dup
-        options[:logger]       ||= self.logger if defined?(SemanticLogger)
-        options[:port]         ||= 22
-        options[:max_pkt_size] ||= 65536
-        options[:password]     = password
-        options[:port]         = port
-        mode                   = binary ? 'rb' : 'r'
-
-        result = nil
-        Net::SFTP.start(host, user, options) do |sftp|
-          result = sftp.file.open(file_name, mode, &block)
-        end
-        result
-      end
-
-    end
-  end
-end

data/lib/io_streams/sftp/writer.rb

@@ -1,68 +0,0 @@
-module IOStreams
-  # Example:
-  #   IOStreams::SFTP::Writer.open('file.txt',
-  #     user:     'jbloggs',
-  #     password: 'secret',
-  #     host:     'example.org',
-  #     options: {compression: false}
-  #   ) do |output|
-  #       output.write('Hello World')
-  #   end
-  module SFTP
-    class Writer
-      include SemanticLogger::Loggable if defined?(SemanticLogger)
-
-      # Stream to a remote file over sftp.
-      #
-      # file_name: [String]
-      #   Name of file to write to.
-      #
-      # user: [String]
-      #   Name of user to login with.
-      #
-      # password: [String]
-      #   Password for the user.
-      #
-      # host: [String]
-      #   Name of the host to connect to.
-      #
-      # port: [Integer]
-      #   Port to connect to at the above host.
-      #
-      # mkdir [true|false]
-      #   Whether to create the output directory on the target system before writing the file.
-      #   The path is created recursively if any portions of the path that are missing.
-      #   Default: false
-      #
-      # binary [true|false]
-      #   Whether to write in binary mode
-      #   Default: true
-      #
-      # options: [Hash]
-      #   Any options supported by Net::SSH.start
-      def self.open(file_name, user:, password:, host:, port: 22, mkdir: false, binary: true, options: {}, &block)
-        raise(NotImplementedError, 'Can only SFTP directly to a file name, not another stream.') if IOStreams.writer_stream?(file_name)
-
-        begin
-          require 'net/sftp' unless defined?(Net::SFTP)
-        rescue LoadError => e
-          raise(LoadError, "Please install the 'net-sftp' gem for SFTP streaming support. #{e.message}")
-        end
-
-        options                = options.dup
-        options[:logger]       ||= logger if defined?(SemanticLogger)
-        options[:port]         ||= 22
-        options[:max_pkt_size] ||= 65536
-        options[:password]     = password
-        options[:port]         = port
-        mode                   = binary ? 'wb' : 'w'
-
-        Net::SFTP.start(host, user, options) do |sftp|
-          sftp.session.exec!("mkdir -p '#{::File.dirname(file_name)}'") if mkdir
-          sftp.file.open(file_name, mode, &block)
-        end
-      end
-
-    end
-  end
-end

data/test/base_path_test.rb

@@ -1,35 +0,0 @@
-require_relative 'test_helper'
-
-module IOStreams
-  class BasePathTest < Minitest::Test
-    describe IOStreams::BasePath do
-      describe '.join' do
-        let(:path) { IOStreams::BasePath.new('some_path') }
-
-        it 'returns self when no elements' do
-          assert_equal path.object_id, path.join.object_id
-        end
-
-        it 'adds element to path' do
-          assert_equal ::File.join('some_path', 'test'), path.join('test').to_s
-        end
-
-        it 'adds paths to root' do
-          assert_equal ::File.join('some_path', 'test', 'second', 'third'), path.join('test', 'second', 'third').to_s
-        end
-
-        it 'returns path and filename' do
-          assert_equal ::File.join('some_path', 'file.xls'), path.join('file.xls').to_s
-        end
-
-        it 'adds elements to path' do
-          assert_equal ::File.join('some_path', 'test', 'second', 'third', 'file.xls'), path.join('test', 'second', 'third', 'file.xls').to_s
-        end
-
-        it 'return path as sent in when full path' do
-          assert_equal ::File.join('some_path', 'test', 'second', 'third', 'file.xls'), path.join('some_path', 'test', 'second', 'third', 'file.xls').to_s
-        end
-      end
-    end
-  end
-end