data_kit 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6cb35e25f3fbf1a5444fbd581b8ae9225c038653
+  data.tar.gz: 8933a43e911a6e8c36c92d7fcb9ea004875c0cb6
+SHA512:
+  metadata.gz: abf4c32f7bc1c7d001d2acd90dfd23c652382f652bb1066205df76642855c77bb94abbeea6cd1c27cd39a7784f846f58f3cdd7be08e87c2edc9541cdacf3edbf
+  data.tar.gz: f248370d7f60840a9a82229409bf4d8316e3d5b40340bd6cf7e6cbe0dc40f226ec05813107f9a6bdee8607c81a4253fe7381525a11719904d43858c9cd54cf0c

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+
+script:
+  - bundle exec rake

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mode-ruby.gemspec
+gemspec
+
+gem "codeclimate-test-reporter", group: :test, require: nil

data/Gemfile.lock

@@ -0,0 +1,44 @@
+PATH
+  remote: .
+  specs:
+    data_kit (0.0.1)
+      rcsv
+      timeliness
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    codeclimate-test-reporter (0.2.0)
+      simplecov (>= 0.7.1, < 1.0.0)
+    diff-lcs (1.2.5)
+    docile (1.1.1)
+    multi_json (1.8.2)
+    profile (0.3.3)
+    rake (10.1.0)
+    rcsv (0.1.1)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+    simplecov (0.8.2)
+      docile (~> 1.1.0)
+      multi_json
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    timeliness (0.3.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  codeclimate-test-reporter
+  data_kit!
+  profile
+  rake
+  rspec
+  simplecov

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Mode
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,22 @@
+Data Kit
+==========
+
+Library for ingesting, analyzing and cleaning normalizing datasets
+
+## Installation
+
+### Standalone
+
+From your terminal run:
+
+    $ gem install data_kit
+
+### Bundler
+
+Add this line to your application's Gemfile:
+
+    gem 'data_kit'
+
+And then execute:
+
+    $ bundle

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/data_kit.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'data_kit/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "data_kit"
+  spec.version       = DataKit::VERSION
+  spec.authors       = ["Mode Analytics"]
+  spec.email         = ["support@modeanalytics.com"]
+  spec.description   = %q{Library for ingesting, analyzing and normalizing datasets in various formats}
+  spec.summary       = %q{Provides parsers, analyzers and converters for datasets stored in various formats}
+  spec.homepage      = "http://www.modeanalytics.com/"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  # Runtime Dependencies
+  spec.add_runtime_dependency "rcsv"
+  spec.add_runtime_dependency "timeliness"
+
+  # Development Dependencies
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "profile"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "simplecov"
+end

data/lib/data_kit/converters/boolean.rb

@@ -0,0 +1,16 @@
+module DataKit
+  module Converters
+    class Boolean
+      class << self
+        def convert(value)
+          downcased = value.downcase
+          downcased == 'true' || downcased == 't'
+        end
+
+        def match?(value)
+          (value =~ /\A(true|t|false|f)\z/i) == 0
+        end
+      end
+    end
+  end
+end

data/lib/data_kit/converters/date_time.rb

@@ -0,0 +1,21 @@
+require 'timeliness'
+
+module DataKit
+  module Converters
+    class DateTime
+      class << self
+        def convert(value)
+          Timeliness.parse(value, :datetime, :zone => :utc)
+        end
+
+        def match?(value)
+          Timeliness::Definitions.format_sets(:datetime, value).any?{|set| value =~ set.regexp}
+        end
+
+        # Additional Date/Time Formats
+        Timeliness.add_formats(:datetime, "yyyy-m-dTh:nn:ss")
+        Timeliness.add_formats(:datetime, "m/d/yy h:nn:ss.u ampm")
+      end
+    end
+  end
+end

data/lib/data_kit/converters/integer.rb

@@ -0,0 +1,24 @@
+module DataKit
+  module Converters
+    class Integer
+      class << self
+        def convert(value)
+          Integer(value)
+        end
+
+        def match?(value)
+          begin
+            Integer(value)
+            true
+          rescue
+            false
+          end
+        end
+
+        def reformat(value)
+          value.tr(',', '').tr('$', '')
+        end
+      end
+    end
+  end
+end

data/lib/data_kit/converters/number.rb

@@ -0,0 +1,28 @@
+module DataKit
+  module Converters
+    class Number
+      class << self
+        def convert(value)
+          Float(value)
+        end
+
+        def match?(value)
+          begin
+            Float(value)
+            true
+          rescue
+            false
+          end
+        end
+
+        def reformat(value)
+          if value.is_a?(String)
+            value.gsub(/(\p{Sc}|\,)/, '')
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/data_kit/csv/analysis.rb

@@ -0,0 +1,69 @@
+module DataKit
+  module CSV
+    class Analysis
+      attr_reader :fields
+      attr_reader :types
+      attr_reader :row_count
+      attr_reader :sample_count
+
+      def initialize(fields)
+        @fields, @types = fields, {}
+        @row_count, @sample_count = 0, 0
+
+        fields.each do |field_name|
+          @types[field_name] = {}
+          Dataset::Field::Types.each do |type|
+            @types[field_name][type] = 0
+          end
+        end
+      end
+
+      def increment_total
+        @row_count += 1
+      end
+
+      def increment_sample
+        @sample_count += 1
+      end
+
+      def insert(field_name, value)
+        @types[field_name][Dataset::Field.type?(value)] += 1
+      end
+
+      def field_types
+        fields.inject({}) do |result, field_name|
+          result[field_name] = type?(field_name)
+          result
+        end
+      end
+
+      def type?(field)
+        if has_single_type?(field)
+          type_list(field).first
+        elsif has_only_numeric_types?(field)
+          :number
+        else
+          :string
+        end
+      end
+
+      def type_count(field, type)
+        types[field][type] || 0
+      end
+
+      def type_list(field)
+        types[field].keys.select do |type|
+          type_count(field, type) > 0
+        end
+      end
+
+      def has_single_type?(field)
+        type_list(field).length == 1
+      end
+
+      def has_only_numeric_types?(field)
+        (type_list(field) - [:integer, :number, :null]).length == 0
+      end
+    end
+  end
+end

data/lib/data_kit/csv/analyzer.rb

@@ -0,0 +1,52 @@
+module DataKit
+  module CSV
+    class Analyzer
+      attr_accessor :csv
+      attr_accessor :keys
+      attr_accessor :sample_rate
+      
+      def initialize(csv, options = {})
+        @csv = csv
+        @keys = options[:keys] || []
+        @sample_rate = options[:sample_rate] || 0.1
+      end
+
+      def execute
+        random = Random.new
+        analysis = Analysis.new(csv.headers)
+
+        csv.each_row do |row|
+          analysis.increment_total
+          if random.rand <= sample_rate
+            analysis.increment_sample
+            row.keys.each do |field_name|
+              analysis.insert(field_name.to_s, row[field_name])
+            end
+          end
+        end
+
+        analysis
+      end
+      
+      class << self
+        def analyze(csv, options = {})
+          analyzer = new(csv,
+            :keys => options[:keys],
+            :sample_rate => options[:sample_rate]
+          )
+
+          analyzer.execute
+        end
+
+        def sample_rate(file_size)
+          if file_size < (1024 * 1024)
+            sample_rate = 1.0
+          else
+            scale_factor = 500
+            sample_rate = (scale_factor / Math.sqrt(file_size)).round(4)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/data_kit/csv/converter.rb

@@ -0,0 +1,62 @@
+require 'csv'
+
+module DataKit
+  module CSV
+    class Converter
+
+      attr_accessor :csv
+      attr_accessor :analysis
+      attr_accessor :output_path
+
+      def initialize(csv, analysis, output_path)
+        @csv = csv
+        @analysis = analysis
+        @output_path = File.expand_path(output_path)
+      end
+
+      def execute
+        ::CSV.open(output_path, 'wb') do |writer|
+          writer << csv.headers
+          csv.each_row do |row|
+            writer << csv.headers.collect do |field_name|
+              convert(row[field_name], field_types[field_name])
+            end
+          end
+        end
+      end
+
+      def field_types
+        @field_types ||= analysis.field_types
+      end
+
+      class << self
+        def convert(csv, analysis, output_path)
+          converter = new(csv, analysis, output_path)
+          converter.execute
+          converter
+        end
+      end
+
+      private
+
+      def convert(value, type)
+        if value.nil? || type == :string || type == :empty
+          return value.to_s
+        else
+          formatted = Converters::Number.reformat(value)
+
+          case type
+          when :integer
+            return Converters::Integer.convert(formatted)
+          when :number
+            return Converters::Number.convert(formatted)
+          when :boolean
+            return Converters::Boolean.convert(value)
+          when :datetime
+            return Converters::DateTime.convert(value).strftime("%Y-%m-%dT%H:%M:%SZ")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/data_kit/csv/parser.rb

@@ -0,0 +1,55 @@
+require 'rcsv'
+
+module DataKit
+  module CSV
+    class Parser
+      # Encode streams from BINARY into UTF-8
+      InternalEnc = Encoding.find("UTF-8")
+      ExternalEnc = Encoding.find("BINARY")
+
+      attr_reader :path
+      attr_reader :handle
+      attr_reader :headers
+
+      def initialize(path)
+        @path = path
+        
+        set_handle
+        set_headers
+      end
+
+      def each_row(&block)
+        handle.rewind
+        Rcsv.parse(handle, :header => :skip, :columns => columns, :row_as_hash => true) do |row|
+          yield row
+        end
+      end
+
+      private
+
+      def columns
+        index = -1
+        @columns ||= headers.inject({}) do |result, field_name|
+          index += 1
+          result[index] = { :alias => field_name }
+          result
+        end
+      end
+
+      def set_handle
+        if path.is_a?(IO)
+          @handle = path
+        else
+          @handle = File.open(path)
+        end
+
+        @handle.set_encoding(ExternalEnc, InternalEnc)
+      end
+
+      def set_headers
+        handle.rewind
+        Rcsv.parse(handle, :header => :none) { |row| @headers = row; break }
+      end
+    end
+  end
+end

data/lib/data_kit/dataset/field.rb

@@ -0,0 +1,58 @@
+module DataKit
+  module Dataset
+    class Field
+
+      Types = [:string, :integer, :number, :datetime, :boolean, :null].freeze
+
+      attr_accessor :name
+      attr_accessor :key
+      attr_accessor :type
+
+      def initialize(name, options = {})
+        @name = name
+        @key = options[:key] || false
+        @type = (options[:type] || :string).to_sym
+      end
+
+      def key?
+        key == true
+      end
+
+      def to_hash
+        { 'name' => name, 'type' => type.to_s, 'key' => key?}
+      end
+
+      class << self
+        def type?(value)
+          return :null if value.nil?
+          reformatted = Converters::Number.reformat(value)
+
+          if Converters::Integer.match?(reformatted)
+            :integer
+          elsif Converters::Number.match?(reformatted)
+            :number
+          elsif Converters::Boolean.match?(value)
+            :boolean
+          elsif Converters::DateTime.match?(value)
+            :datetime
+          else
+            :string
+          end
+        end
+
+        def convert(value, type)
+          return nil if type == :null || value.nil?
+          reformatted = Converters::Number.reformat(value)
+
+          case type
+          when :integer  then Converters::Integer.convert(reformatted)
+          when :number   then Converters::Number.convert(reformatted)
+          when :boolean  then Converters::Boolean.convert(value)
+          when :datetime then Converters::DateTime.convert(value)
+          else value.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/data_kit/dataset/schema.rb

@@ -0,0 +1,21 @@
+require 'yaml'
+
+module DataKit
+  module Dataset
+    class Schema
+      attr_accessor :fields
+
+      def initialize
+        @fields = []
+      end
+
+      def keys
+        fields.select{ |f| f.key? }
+      end
+
+      def to_yaml
+        fields.collect(&:to_hash).to_yaml
+      end
+    end
+  end
+end