daru-io 0.1.0

data/lib/daru/io/exporters/r_data.rb

@@ -0,0 +1,66 @@
+require 'daru/io/exporters/rds'
+
+module Daru
+  module IO
+    module Exporters
+      # RData Exporter Class, that can be used to export multiple `Daru::DataFrame`s
+      # to a RData file
+      class RData < RDS
+        # Initializes a RData Exporter instance.
+        #
+        # @param options [Hash] A set of key-value pairs wherein the key depicts the name of
+        #   the R `data.frame` variable name to be saved in the RData file, and the corresponding
+        #   value depicts the `Daru::DataFrame` (or any Ruby variable in scope)
+        #
+        # @example Initializing RData Exporter instance
+        #   df1 = Daru::DataFrame.new([[1,2],[3,4]], order: [:a, :b])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      a   b
+        #   #  0   1   3
+        #   #  1   2   4
+        #
+        #   df2 = Daru::DataFrame.new([[5,6],[7,8]], order: [:x, :y])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      x   y
+        #   #  0   5   7
+        #   #  1   6   8
+        #
+        #   instance = Daru::IO::Exporters::RData.new("first.df": df1, "second.df": df2)
+        def initialize(**options)
+          optional_gem 'rsruby'
+
+          @options = options
+        end
+
+        # Exports a RData Exporter instance to a file-writable String.
+        #
+        # @return [String] A file-writable string
+        #
+        # @example Writing to a RData file
+        #   instance.to_s
+        #
+        #   #=> "\u001F\x8B\b\u0000\u0000\u0000\u0000\u0000\u0000\u0003\vr\x890\xE2\x8A\xE0b```b..."
+        def to_s
+          super
+        end
+
+        # Exports an RData Exporter instance to a rdata file.
+        #
+        # @param path [String] Path of RData file where the dataframe(s) is/are to be saved
+        #
+        # @example Writing to a RData file
+        #   instance.write("daru_dataframes.RData")
+        def write(path)
+          @instance    = RSRuby.instance
+          @statements  = @options.map do |r_variable, dataframe|
+            process_statements(r_variable, dataframe)
+          end.flatten
+          @statements << "save(#{@options.keys.map(&:to_s).join(', ')}, file='#{path}')"
+          @statements.each { |statement| @instance.eval_R(statement) }
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/exporters/rds.rb

@@ -0,0 +1,79 @@
+require 'daru/io/exporters/base'
+
+module Daru
+  module IO
+    module Exporters
+      # RDS Exporter Class, that extends `to_rds_string` and `write_rds` methods to
+      # `Daru::DataFrame` instance variables
+      class RDS < Base
+        Daru::DataFrame.register_io_module :to_rds_string, self
+        Daru::DataFrame.register_io_module :write_rds, self
+
+        # Initializes a RDS Exporter instance.
+        #
+        # @param dataframe [Daru::DataFrame] A dataframe to export
+        # @param r_variable [String] Name of the R `data.frame` variable name to be saved in the RDS file
+        #
+        # @example Initializing an RData Exporter
+        #   df = Daru::DataFrame.new([[1,2],[3,4]], order: [:a, :b])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      a   b
+        #   #  0   1   3
+        #   #  1   2   4
+        #
+        #   instance = Daru::IO::Exporters::RDS.new(df, "sample.dataframe")
+        def initialize(dataframe, r_variable)
+          optional_gem 'rsruby'
+
+          super(dataframe)
+          @r_variable = r_variable
+        end
+
+        # Exports a RDS Exporter instance to a file-writable String.
+        #
+        # @return [String] A file-writable string
+        #
+        # @example Getting a file-writable string from RDS Exporter instance
+        #   instance.to_s #! same as df.to_rds_string("sample.dataframe")
+        #
+        #   #=> "\u001F\x8B\b\u0000\u0000\u0000\u0000\u0000\u0000\u0003\x8B\xE0b```b..."
+        def to_s
+          super
+        end
+
+        # Exports a RDS Exporter instance to a rds file.
+        #
+        # @param path [String] Path of RDS file where the dataframe is to be saved
+        #
+        # @example Writing an RDS Exporter instance to a rds file
+        #   instance.write("daru_dataframe.rds")
+        def write(path)
+          @instance    = RSRuby.instance
+          @statements  = process_statements(@r_variable, @dataframe)
+          @statements << "saveRDS(#{@r_variable}, file='#{path}')"
+          @statements.each { |statement| @instance.eval_R(statement) }
+        end
+
+        private
+
+        def process_statements(r_variable, dataframe)
+          [
+            *dataframe.map_vectors_with_index do |vector, i|
+              "#{i} = c(#{vector.to_a.map { |val| convert_datatype(val) }.join(', ')})"
+            end,
+            "#{r_variable} = data.frame(#{dataframe.vectors.to_a.map(&:to_s).join(', ')})"
+          ]
+        end
+
+        def convert_datatype(value)
+          case value
+          when nil    then 'NA'
+          when String then "'#{value}'"
+          else value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/exporters/sql.rb

@@ -0,0 +1,55 @@
+require 'daru/io/exporters/base'
+
+module Daru
+  module IO
+    module Exporters
+      # SQL Exporter Class, that extends `to_sql` method to `Daru::DataFrame`
+      # instance variables
+      class SQL < Base
+        Daru::DataFrame.register_io_module :to_sql, self
+
+        # Initializes a SQL Exporter instance.
+        #
+        # @param dataframe [Daru::DataFrame] A dataframe to export.
+        # @param dbh [DBI] A DBI database connection object.
+        # @param table [String] The SQL table to export to.
+        #
+        # @example Initializing with database credentials
+        #   df = Daru::DataFrame.new([[1,2],[3,4]], order: [:a, :b])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      a   b
+        #   #  0   1   3
+        #   #  1   2   4
+        #
+        #   table = 'test'
+        #
+        #   dbh = DBI.connect("DBI:Mysql:database:localhost", "user", "password")
+        #   # Enter the actual SQL database credentials in the above line
+        #
+        #   instance = Daru::IO::Exporters::SQL.new(df, dbh, table)
+        def initialize(dataframe, dbh, table)
+          optional_gem 'dbd-sqlite3', requires: 'dbd/SQLite3'
+          optional_gem 'dbi'
+          optional_gem 'sqlite3'
+
+          super(dataframe)
+          @dbh       = dbh
+          @table     = table
+        end
+
+        # Exports a SQL Exporter instance to an SQL table.
+        #
+        # @example Exports SQL Exporter instance into given SQL table
+        #   instance.to
+        def to
+          query = "INSERT INTO #{@table} (#{@dataframe.vectors.to_a.join(',')}"\
+                  ") VALUES (#{(['?']*@dataframe.vectors.size).join(',')})"
+          sth   = @dbh.prepare(query)
+          @dataframe.each_row { |c| sth.execute(*c.to_a) }
+          true
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers.rb

@@ -0,0 +1 @@
+Dir["#{__dir__}/importers/*.rb"].each { |file| require "daru/io#{file.gsub(__dir__, '')}" }

data/lib/daru/io/importers/active_record.rb

@@ -0,0 +1,75 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # ActiveRecord Importer Class, that extends `from_activerecord` method to
+      # `Daru::DataFrame`
+      class ActiveRecord < Base
+        Daru::DataFrame.register_io_module :from_activerecord, self
+
+        # Checks for required gem dependencies of ActiveRecord Importer
+        def initialize
+          optional_gem 'activerecord', '~> 4.0', requires: 'active_record'
+        end
+
+        # Loads data from a given relation
+        #
+        # @!method self.from(relation)
+        #
+        # @param relation [ActiveRecord::Relation] A relation to be used to load
+        #   the contents of DataFrame
+        #
+        # @return [Daru::IO::Importers::ActiveRecord]
+        #
+        # @example Loading from a ActiveRecord instance
+        #   instance = Daru::IO::Importers::ActiveRecord.from(Account.all)
+        def from(relation)
+          @relation = relation
+          self
+        end
+
+        # Imports a `Daru::DataFrame` from an ActiveRecord Importer instance
+        #
+        # @param fields [String or Array of Strings] A set of fields to load from.
+        #
+        # @return [Daru::DataFrame]
+        #
+        # @example Importing from an instance without specifying fields
+        #   instance.call
+        #
+        #   #=> #<Daru::DataFrame(2x3)>
+        #   #=>        id  name   age
+        #   #=>   0     1 Homer    20
+        #   #=>   1     2 Marge    30
+        #
+        # @example Importing from an instance with specific fields
+        #   instance.call(:id, :name)
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #=>        id  name
+        #   #=>   0     1 Homer
+        #   #=>   1     2 Marge
+        def call(*fields)
+          @fields = fields
+
+          if @fields.empty?
+            records = @relation.map { |record| record.attributes.symbolize_keys }
+            return Daru::DataFrame.new(records)
+          else
+            @fields.map!(&:to_sym)
+          end
+
+          vectors = @fields.map { |name| [name, Daru::Vector.new([], name: name)] }.to_h
+
+          Daru::DataFrame.new(vectors, order: @fields).tap do |df|
+            @relation.pluck(*@fields).each do |record|
+              df.add_row(Array(record))
+            end
+            df.update
+          end
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers/avro.rb

@@ -0,0 +1,54 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # Avro Importer Class, that extends `read_avro` method to `Daru::DataFrame`
+      class Avro < Base
+        Daru::DataFrame.register_io_module :read_avro, self
+
+        # Checks for required gem dependencies of Avro Importer
+        #
+        # @note The 'snappy' gem handles compressions and is used within Avro gem. Yet, it isn't
+        #   specified as a dependency in Avro gem. Hence, it has been added separately.
+        def initialize
+          optional_gem 'avro'
+          optional_gem 'snappy'
+        end
+
+        # Reads data from an avro file
+        #
+        # @!method self.read(path)
+        #
+        # @param path [String] Path to Avro file, where the dataframe is to be imported from.
+        #
+        # @return [Daru::IO::Importers::Avro]
+        #
+        # @example Reading from avro file
+        #   instance = Daru::IO::Importers::Avro.read("azorahai.avro")
+        def read(path)
+          @path   = path
+          @buffer = StringIO.new(File.read(@path))
+          @data = ::Avro::DataFile::Reader.new(@buffer, ::Avro::IO::DatumReader.new).to_a
+          self
+        end
+
+        # Imports a `Daru::DataFrame` from an Avro Importer instance
+        #
+        # @return [Daru::DataFrame]
+        #
+        # @example Importing from an Avro file
+        #   df = instance.call
+        #
+        #   #=> #<Daru::DataFrame(3x3)>
+        #   #        name points winner
+        #   #    0   Dany    100   true
+        #   #    1    Jon    100   true
+        #   #    2 Tyrion    100   true
+        def call
+          Daru::DataFrame.new(@data)
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers/base.rb

@@ -0,0 +1,62 @@
+require 'daru/io/base'
+
+module Daru
+  module IO
+    module Importers
+      # Base Importer Class that contains generic helper methods, to be
+      # used by other Importers via inheritence
+      class Base < Daru::IO::Base
+        # Guesses the `Daru::DataFrame` from the parsed set of key-value pairs.
+        #
+        # @param keys [Array] A set of keys from given key-value pairs
+        # @param vals [Array] A set of values from given key-value pairs
+        #
+        # @example When key-value pairs contains values that is Array of Hashes
+        #   Daru::IO::Importers::Base.guess_parse([:a], [[{ x: 1, y: 2 },{ x: 3, y: 4 }]])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      x   y
+        #   #  0   1   2
+        #   #  1   3   4
+        #
+        # @example When key-value pairs contains values that is Arrays
+        #   Daru::IO::Importers::Base.guess_parse([:x, :y], [[1,3], [2,4]])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      x   y
+        #   #  0   1   2
+        #   #  1   3   4
+        #
+        # @example When key-value pairs contains Array of keys contain value Hashes
+        #   Daru::IO::Importers::Base.guess_parse([:a, :b], [{ x: 1, y: 2 }, { x: 3, y: 4 }])
+        #
+        #   #=> #<Daru::DataFrame(2x2)>
+        #   #      x   y
+        #   #  a   1   2
+        #   #  b   3   4
+        def self.guess_parse(keys, vals)
+          case vals.first
+          when Array
+            case vals.first.first
+            when Hash then Daru::DataFrame.new(vals.flatten)
+            else Daru::DataFrame.rows(vals.transpose, order: keys)
+            end
+          when Hash then Daru::DataFrame.new(vals.flatten, index: keys)
+          end
+        end
+
+        # Adds the `from` class method to all inheriting children Importer classes, which
+        # calls corresponding Importer's `initialize` and instance method `from`.
+        def self.from(relation)
+          new.from(relation)
+        end
+
+        # Adds the `read` class method to all inheriting children Importer classes, which
+        # calls corresponding Importer's `initialize` and instance method `read`.
+        def self.read(path)
+          new.read(path)
+        end
+      end
+    end
+  end
+end

data/lib/daru/io/importers/csv.rb

@@ -0,0 +1,190 @@
+require 'daru/io/importers/base'
+
+module Daru
+  module IO
+    module Importers
+      # CSV Importer Class, that extends `read_csv` method to `Daru::DataFrame`
+      class CSV < Base
+        Daru::DataFrame.register_io_module :read_csv, self
+
+        CONVERTERS = {
+          boolean: lambda { |f, _|
+            case f.downcase.strip
+            when 'true'  then true
+            when 'false' then false
+            else f
+            end
+          }
+        }.freeze
+
+        # Checks for required gem dependencies of CSV Importer
+        def initialize
+          require 'csv'
+          require 'open-uri'
+          require 'zlib'
+        end
+
+        # Reads data from a csv / csv.gz file
+        #
+        # @!method self.read(path)
+        #
+        # @param path [String] Path to csv / csv.gz file, where the dataframe is to be imported
+        #   from.
+        #
+        # @return [Daru::IO::Importers::CSV]
+        #
+        # @example Reading from csv file
+        #   instance = Daru::IO::Importers::CSV.read("matrix_test.csv")
+        #
+        # @example Reading from csv.gz file
+        #   instance = Daru::IO::Importers::CSV.read("matrix_test.csv.gz")
+        def read(path)
+          @path      = path
+          @file_data = open(@path)
+          self
+        end
+
+        # Imports a `Daru::DataFrame` from a CSV Importer instance
+        #
+        # @param headers [Boolean] If this option is `true`, only those columns
+        #   will be used to import the `Daru::DataFrame` whose header is given.
+        # @param skiprows [Integer] Skips the first `:skiprows` number of rows from
+        #   the CSV file. Defaults to 0.
+        # @param compression [Symbol] Defaults to `:infer`, to parse depending on file format
+        #   like `.csv.gz`. For explicitly parsing data from a `.csv.gz` file, set
+        #   `:compression` as `:gzip`.
+        # @param clone [Boolean] Have a look at `:clone` option
+        #   [here](http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize)
+        # @param index [Array or Daru::Index or Daru::MultiIndex] Have a look at
+        #   `:index` option
+        #   [here](http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize)
+        # @param order [Array or Daru::Index or Daru::MultiIndex] Have a look at
+        #   `:order` option
+        #   [here](http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize)
+        # @param name [String] Have a look at `:name` option
+        #   [here](http://www.rubydoc.info/gems/daru/0.1.5/Daru%2FDataFrame:initialize)
+        # @param options [Hash] CSV standard library options such as `:col_sep`
+        #   (defaults to `','`), `:converters` (defaults to `:numeric`),
+        #   `:header_converters` (defaults to `:symbol`).
+        #
+        # @return [Daru::DataFrame]
+        #
+        # @example Calling with csv options
+        #   df = instance.call(col_sep: ' ', headers: true)
+        #
+        #   #=> #<Daru::DataFrame(99x3)>
+        #   #        image_reso        mls true_trans
+        #   #      0    6.55779          0 -0.2362347
+        #   #      1    2.14746          0 -0.1539447
+        #   #      2    8.31104          0 0.3832846,
+        #   #      3    3.47872          0 0.3832846,
+        #   #      4    4.16725          0 -0.2362347
+        #   #      5    5.79983          0 -0.2362347
+        #   #      6     1.9058          0 -0.895577,
+        #   #      7     1.9058          0 -0.2362347
+        #   #      8    4.11806          0 -0.895577,
+        #   #      9    6.26622          0 -0.2362347
+        #   #     10    2.57805          0 -0.1539447
+        #   #     11    4.76151          0 -0.2362347
+        #   #     12    7.11002          0 -0.895577,
+        #   #     13    5.40811          0 -0.2362347
+        #   #     14    8.19567          0 -0.1539447
+        #   #    ...        ...        ...        ...
+        #
+        # @example Calling with csv.gz options
+        #   df = instance.call(compression: :gzip, col_sep: ' ', headers: true)
+        #
+        #   #=> #<Daru::DataFrame(99x3)>
+        #   #        image_reso        mls true_trans
+        #   #      0    6.55779          0 -0.2362347
+        #   #      1    2.14746          0 -0.1539447
+        #   #      2    8.31104          0 0.3832846,
+        #   #      3    3.47872          0 0.3832846,
+        #   #      4    4.16725          0 -0.2362347
+        #   #      5    5.79983          0 -0.2362347
+        #   #      6     1.9058          0 -0.895577,
+        #   #      7     1.9058          0 -0.2362347
+        #   #      8    4.11806          0 -0.895577,
+        #   #      9    6.26622          0 -0.2362347
+        #   #     10    2.57805          0 -0.1539447
+        #   #     11    4.76151          0 -0.2362347
+        #   #     12    7.11002          0 -0.895577,
+        #   #     13    5.40811          0 -0.2362347
+        #   #     14    8.19567          0 -0.1539447
+        #   #    ...        ...        ...        ...
+        def call(headers: nil, skiprows: 0, compression: :infer,
+          clone: nil, index: nil, order: nil, name: nil, **options)
+          init_opts(headers: headers, skiprows: skiprows, compression: compression,
+                    clone: clone, index: index, order: order, name: name, **options)
+          process_compression
+
+          # Preprocess headers for detecting and correcting repetition in
+          # case the :headers option is not specified.
+          hsh =
+            if @headers
+              hash_with_headers
+            else
+              hash_without_headers.tap { |hash| @daru_options[:order] = hash.keys }
+            end
+
+          Daru::DataFrame.new(hsh, @daru_options)
+        end
+
+        private
+
+        def compression?(algorithm, *formats)
+          @compression == algorithm || formats.any? { |f| @path.end_with?(f) }
+        end
+
+        def hash_with_headers
+          ::CSV
+            .parse(@file_data, @options)
+            .tap { |c| yield c if block_given? }
+            .by_col
+            .map do |col_name, values|
+              [col_name, values.nil? ? [] : values[@skiprows..-1]]
+            end
+            .to_h
+        end
+
+        def hash_without_headers
+          csv_as_arrays =
+            ::CSV
+            .parse(@file_data, @options)
+            .tap { |c| yield c if block_given? }
+            .to_a
+          headers       = ArrayHelper.recode_repeated(csv_as_arrays.shift)
+          csv_as_arrays = csv_as_arrays[@skiprows..-1].transpose
+          headers
+            .each_with_index
+            .map do |h, i|
+              [h, csv_as_arrays[i] || []]
+            end
+            .to_h
+        end
+
+        def init_opts(headers: nil, skiprows: 0, compression: :infer,
+          clone: nil, index: nil, order: nil, name: nil, **options)
+          @headers      = headers
+          @skiprows     = skiprows
+          @compression  = compression
+          @daru_options = {clone: clone, index: index, order: order, name: name}
+          @options      = {
+            col_sep: ',', converters: [:numeric], header_converters: :symbol,
+            headers: @headers, skip_blanks: true
+          }.merge(options)
+
+          @options[:converters] = @options[:converters].flat_map do |c|
+            next ::CSV::Converters[c] if ::CSV::Converters[c]
+            next CONVERTERS[c] if CONVERTERS[c]
+            c
+          end
+        end
+
+        def process_compression
+          @file_data = ::Zlib::GzipReader.new(@file_data).read if compression?(:gzip, '.csv.gz')
+        end
+      end
+    end
+  end
+end