iostreams 0.14.0 → 0.15.0

data/lib/io_streams/s3.rb

@@ -0,0 +1,25 @@
+begin
+  require 'aws-sdk-s3'
+rescue LoadError => exc
+  raise(LoadError, "Install gem 'aws-sdk-s3' to read and write AWS S3 files: #{exc.message}")
+end
+
+require 'uri'
+module IOStreams
+  module S3
+    # Sample URI: s3://mybucket/user/abc.zip
+    def self.parse_uri(uri)
+      # 's3://mybucket/user/abc.zip'
+      uri = URI.parse(uri)
+      # Filename and bucket only
+      if uri.scheme.nil?
+        segments = uri.path.split('/')
+        raise "S3 URI must at the very least contain '<bucket_name>/<key>'" if (segments.size == 1) || (segments[0] == '')
+        {
+          bucket: segments.shift,
+          key:    segments.join('/')
+        }
+      end
+    end
+  end
+end

data/lib/io_streams/s3/reader.rb

@@ -0,0 +1,64 @@
+module IOStreams
+  module S3
+    class Reader
+      # Read from a AWS S3 file
+      def self.open(uri = nil, bucket: nil, region: nil, key: nil, &block)
+        options = uri.nil? ? args : parse_uri(uri).merge(args)
+        s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
+        object  = s3.bucket(options[:bucket]).object(options[:key])
+
+        IO.pipe do |read_io, write_io|
+          object.get(response_target: write_io)
+          write_io.close
+          block.call(read_io)
+        end
+      end
+
+      def self.open2(uri = nil, **args, &block)
+        if !uri.nil? && IOStreams.reader_stream?(uri)
+          raise(ArgumentError, 'S3 can only accept a URI, not an IO stream when reading.')
+        end
+
+        unless defined?(Aws::S3::Resource)
+          begin
+            require 'aws-sdk-s3'
+          rescue LoadError => exc
+            raise(LoadError, "Install gem 'aws-sdk-s3' to read and write AWS S3 files: #{exc.message}")
+          end
+        end
+
+        options = uri.nil? ? args : parse_uri(uri).merge(args)
+
+        begin
+          io = new(**options)
+          block.call(io)
+        ensure
+          io.close if io && (io.respond_to?(:closed?) && !io.closed?)
+        end
+      end
+
+      def initialize(region: nil, bucket:, key:)
+        s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
+        @object = s3.bucket(bucket).object(key)
+        @buffer = []
+      end
+
+      def read(length = nil, outbuf = nil)
+        # Sufficient data already in the buffer
+        return @buffer.slice!(0, length) if length && (length <= @buffer.length)
+
+        # Fetch in chunks
+        @object.get do |chunk|
+          @buffer << chunk
+          return @buffer.slice!(0, length) if length && (length <= @buffer.length)
+        end
+        @buffer if @buffer.size > 0
+      end
+
+      private
+
+      attr_reader :object
+
+    end
+  end
+end

data/lib/io_streams/s3/writer.rb

@@ -0,0 +1,13 @@
+module IOStreams
+  module S3
+    class Writer
+      # Write to AWS S3
+      def self.open(uri = nil, bucket: nil, region: nil, key: nil, &block)
+        options = uri.nil? ? args : parse_uri(uri).merge(args)
+        s3      = region.nil? ? Aws::S3::Resource.new : Aws::S3::Resource.new(region: region)
+        object  = s3.bucket(options[:bucket]).object(options[:key])
+        object.upload_stream(file_name_or_io, &block)
+      end
+    end
+  end
+end

data/lib/io_streams/streams.rb

@@ -92,7 +92,7 @@ module IOStreams
     #   RocketJob::Formatter::Formats.streams_for_file_name('myfile.csv')
     #   => [ :file ]
     def streams_for_file_name(file_name)
-      raise ArgumentError.new("RocketJob Cannot detect file format when uploading to stream: #{file_name.inspect}") if reader_stream?(file_name)
+      raise ArgumentError.new("Cannot auto-detect streams when already a stream: #{file_name.inspect}") if reader_stream?(file_name)
 
       parts      = file_name.split('.')
       extensions = []

data/lib/io_streams/tabular.rb

@@ -0,0 +1,163 @@
+module IOStreams
+  # Common handling for efficiently processing tabular data such as CSV, spreadsheet or other tabular files
+  # on a line by line basis.
+  #
+  # Tabular consists of a table of data where the first row is usually the header, and subsequent
+  # rows are the data elements.
+  #
+  # Tabular applies the header information to every row of data when #as_hash is called.
+  #
+  # Example using the default CSV parser:
+  #
+  #   tabular = Tabular.new
+  #   tabular.parse_header("first field,Second,thirD")
+  #   # => ["first field", "Second", "thirD"]
+  #
+  #   tabular.cleanse_header!
+  #   # => ["first_field", "second", "third"]
+  #
+  #   tabular.record_parse("1,2,3")
+  #   # => {"first_field"=>"1", "second"=>"2", "third"=>"3"}
+  #
+  #   tabular.record_parse([1,2,3])
+  #   # => {"first_field"=>1, "second"=>2, "third"=>3}
+  #
+  #   tabular.render([5,6,9])
+  #   # => "5,6,9"
+  #
+  #   tabular.render({"third"=>"3", "first_field"=>"1" })
+  #   # => "1,,3"
+  class Tabular
+    autoload :Errors, 'io_streams/tabular/errors'
+    autoload :Header, 'io_streams/tabular/header'
+
+    module Parser
+      autoload :Array, 'io_streams/tabular/parser/array'
+      autoload :Base, 'io_streams/tabular/parser/base'
+      autoload :Csv, 'io_streams/tabular/parser/csv'
+      autoload :Fixed, 'io_streams/tabular/parser/fixed'
+      autoload :Hash, 'io_streams/tabular/parser/hash'
+      autoload :Json, 'io_streams/tabular/parser/json'
+      autoload :Psv, 'io_streams/tabular/parser/psv'
+    end
+
+    module Utility
+      autoload :CSVRow, 'io_streams/tabular/utility/csv_row'
+    end
+
+    attr_reader :format, :header, :parser
+
+    # Parse a delimited data source.
+    #
+    # Parameters
+    #   format: [Symbol]
+    #     :csv, :hash, :array, :json, :psv, :fixed
+    #
+    #   For all other parameters, see Tabular::Header.new
+    def initialize(format: nil, file_name: nil, **args)
+      @header = Header.new(**args)
+      klass   =
+        if file_name && format.nil?
+          self.class.parser_class_for_file_name(file_name)
+        else
+          self.class.parser_class(format)
+        end
+      @parser = klass.new
+    end
+
+    # Returns [true|false] whether a header row needs to be read first.
+    def requires_header?
+      parser.requires_header? && IOStreams.blank?(header.columns)
+    end
+
+    # Returns [Array] the header row/line after parsing and cleansing.
+    # Returns `nil` if the row/line is blank, or a header is not required for the supplied format (:json, :hash).
+    #
+    # Notes:
+    # * Call `parse_header?` first to determine if the header should be parsed first.
+    # * The header columns are set after parsing the row, but the header is not cleansed.
+    def parse_header(line)
+      return if IOStreams.blank?(line) || !parser.requires_header?
+
+      header.columns = parser.parse(line)
+    end
+
+    # Returns [Hash<String,Object>] the line as a hash.
+    # Returns nil if the line is blank.
+    def record_parse(line)
+      line = row_parse(line)
+      header.to_hash(line) if line
+    end
+
+    # Returns [Array] the row/line as a parsed Array of values.
+    # Returns nil if the row/line is blank.
+    def row_parse(line)
+      return if IOStreams.blank?(line)
+
+      parser.parse(line)
+    end
+
+    # Renders the output row
+    def render(row)
+      return if IOStreams.blank?(row)
+
+      parser.render(row, header)
+    end
+
+    # Returns [Array<String>] the cleansed columns
+    def cleanse_header!
+      header.cleanse!
+      header.columns
+    end
+
+    # Register a file extension and the reader and writer classes to use to format it
+    #
+    # Example:
+    #   # MyXls::Reader and MyXls::Writer must implement .open
+    #   register_extension(:xls, MyXls::Reader, MyXls::Writer)
+    def self.register_extension(extension, parser)
+      raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
+      @extensions[extension.nil? ? nil : extension.to_sym] = parser
+    end
+
+    # De-Register a file extension
+    #
+    # Returns [Symbol] the extension removed, or nil if the extension was not registered
+    #
+    # Example:
+    #   register_extension(:xls)
+    def self.deregister_extension(extension)
+      raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
+      @extensions.delete(extension.to_sym)
+    end
+
+    private
+
+    # A registry to hold formats for processing files during upload or download
+    @extensions = {}
+
+    def self.parser_class(format)
+      @extensions[format.nil? ? nil : format.to_sym] || raise(ArgumentError, "Unknown Tabular Format: #{format.inspect}")
+    end
+
+    # Returns the parser to use with tabular for the supplied file_name
+    def self.parser_class_for_file_name(file_name)
+      extension = nil
+      file_name.to_s.split('.').reverse_each do |ext|
+        if @extensions.include?(ext.to_sym)
+          extension = ext.to_sym
+          break
+        end
+      end
+      parser_class(extension)
+    end
+
+    register_extension(nil, IOStreams::Tabular::Parser::Csv)
+    register_extension(:array, IOStreams::Tabular::Parser::Array)
+    register_extension(:csv, IOStreams::Tabular::Parser::Csv)
+    register_extension(:fixed, IOStreams::Tabular::Parser::Fixed)
+    register_extension(:hash, IOStreams::Tabular::Parser::Hash)
+    register_extension(:json, IOStreams::Tabular::Parser::Json)
+    register_extension(:psv, IOStreams::Tabular::Parser::Psv)
+  end
+end

data/lib/io_streams/tabular/errors.rb

@@ -0,0 +1,14 @@
+module IOStreams
+  class Tabular
+    module Errors
+      class Error < StandardError;
+      end
+
+      class InvalidHeader < Error;
+      end
+
+      class TypeMismatch < Error;
+      end
+    end
+  end
+end

data/lib/io_streams/tabular/header.rb

@@ -0,0 +1,146 @@
+module IOStreams
+  class Tabular
+    # Process files / streams that start with a header.
+    class Header
+      attr_accessor :columns, :allowed_columns, :required_columns, :skip_unknown
+
+      # Header
+      #
+      # Parameters
+      #   columns [Array<String>]
+      #     Columns in this header.
+      #     Note:
+      #       It is recommended to keep all columns as strings to avoid any issues when persistence
+      #       with MongoDB when it converts symbol keys to strings.
+      #
+      #   allowed_columns [Array<String>]
+      #     List of columns to allow.
+      #     Default: nil ( Allow all columns )
+      #     Note:
+      #       When supplied any columns that are rejected will be returned in the cleansed columns
+      #       as nil so that they can be ignored during processing.
+      #
+      #   required_columns [Array<String>]
+      #     List of columns that must be present, otherwise an Exception is raised.
+      #
+      #   skip_unknown [true|false]
+      #     true:
+      #       Skip columns not present in the whitelist by cleansing them to nil.
+      #       #as_hash will skip these additional columns entirely as if they were not in the file at all.
+      #     false:
+      #       Raises Tabular::InvalidHeader when a column is supplied that is not in the whitelist.
+      def initialize(columns: nil, allowed_columns: nil, required_columns: nil, skip_unknown: true)
+        @columns          = columns
+        @required_columns = required_columns
+        @allowed_columns  = allowed_columns
+        @skip_unknown     = skip_unknown
+      end
+
+      # Returns [Array<String>] list columns that were ignored during cleansing.
+      #
+      # Each column is cleansed as follows:
+      # - Leading and trailing whitespace is stripped.
+      # - All characters converted to lower case.
+      # - Spaces and '-' are converted to '_'.
+      # - All characters except for letters, digits, and '_' are stripped.
+      #
+      # Notes
+      # * Raises Tabular::InvalidHeader when there are no non-nil columns left after cleansing.
+      def cleanse!
+        return [] if columns.nil? || columns.empty?
+
+        ignored_columns = []
+        self.columns    = columns.collect do |column|
+          cleansed = cleanse_column(column)
+          if allowed_columns.nil? || allowed_columns.include?(cleansed)
+            cleansed
+          else
+            ignored_columns << column
+            nil
+          end
+        end
+
+        if !skip_unknown && !ignored_columns.empty?
+          raise(IOStreams::Tabular::Errors::InvalidHeader, "Unknown columns after cleansing: #{ignored_columns.join(',')}")
+        end
+
+        if ignored_columns.size == columns.size
+          raise(IOStreams::Tabular::Errors::InvalidHeader, "All columns are unknown after cleansing: #{ignored_columns.join(',')}")
+        end
+
+        if required_columns
+          missing_columns = required_columns - columns
+          unless missing_columns.empty?
+            raise(IOStreams::Tabular::Errors::InvalidHeader, "Missing columns after cleansing: #{missing_columns.join(',')}")
+          end
+        end
+
+        ignored_columns
+      end
+
+      # Marshal to Hash from Array or Hash by applying this header
+      #
+      # Parameters:
+      #   cleanse [true|false]
+      #     Whether to cleanse and narrow the supplied hash to just those columns in this header.
+      #     Only Applies to when the hash is already a Hash.
+      #     Useful to turn off narrowing when the input data is already trusted.
+      def to_hash(row, cleanse = true)
+        return if IOStreams.blank?(row)
+
+        case row
+        when Array
+          raise(Tabular::Errors::InvalidHeader, "Missing mandatory header when trying to convert a row into a hash") unless columns
+          array_to_hash(row)
+        when Hash
+          cleanse && columns ? cleanse_hash(row) : row
+        else
+          raise(Tabular::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to a Hash")
+        end
+      end
+
+      def to_array(row, cleanse = true)
+        if row.is_a?(Hash) && columns
+          row = cleanse_hash(row) if cleanse
+          row = columns.collect { |column| row[column] }
+        end
+        raise(Tabular::Errors::TypeMismatch, "Don't know how to convert #{row.class.name} to an Array without the header columns being set.") unless row.is_a?(Array)
+        row
+      end
+
+      private
+
+      def array_to_hash(row)
+        h = {}
+        columns.each_with_index { |col, i| h[col] = row[i] unless IOStreams.blank?(col) }
+        h
+      end
+
+      # Perform cleansing on returned Hash keys during the narrowing process.
+      # For example, avoids issues with case etc.
+      def cleanse_hash(hash)
+        h = {}
+        hash.each_pair do |key, value|
+          cleansed_key    =
+            if columns.include?(key)
+              key
+            else
+              key = cleanse_column(key)
+              key if columns.include?(key)
+            end
+          h[cleansed_key] = value if cleansed_key
+        end
+        h
+      end
+
+      def cleanse_column(name)
+        cleansed = name.to_s.strip.downcase
+        cleansed.gsub!(/\s+/, '_')
+        cleansed.gsub!(/-+/, '_')
+        cleansed.gsub!(/\W+/, '')
+        cleansed
+      end
+
+    end
+  end
+end

data/lib/io_streams/tabular/parser/array.rb

@@ -0,0 +1,26 @@
+require 'json'
+module IOStreams
+  class Tabular
+    module Parser
+      class Array < Base
+        # Returns [Array<String>] the header row.
+        # Returns nil if the row is blank.
+        def parse_header(row)
+          raise(Tabular::Errors::InvalidHeader, "Format is :array. Invalid input header: #{row.class.name}") unless row.is_a?(::Array)
+          row
+        end
+
+        # Returns Array
+        def parse(row)
+          raise(Tabular::Errors::TypeMismatch, "Format is :array. Invalid input: #{row.class.name}") unless row.is_a?(::Array)
+          row
+        end
+
+        def render(row, header)
+          header.to_array(row)
+        end
+
+      end
+    end
+  end
+end