schema-inference 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e9966b0286d35c69194293d9895fe6e7198c893d
+  data.tar.gz: d15a656c7a2c654a618b29f976f27926de913180
+SHA512:
+  metadata.gz: 7d33da3b607dba3848fc05d8a19d5c24554e39809302146b7581749923afa34fe980caa4cf18cf32d2ca2d40f885fedc0c4f273f7a25661f1baa878bde4d2bf4
+  data.tar.gz: 2fa3693d384696779211b7f5706a80453e473d2d986579293d0ee7ceb8a4de3b3a1774aff80b1e041437539317cdf749fe56956e809d98c2cee00043c3cc487d

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.byebug*

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.12.5

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at eurico@phybbit.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in schema-inference.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 株式会社Phybbit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# Schema::Inference
+
+Supports inferring tabular schemas from deep nested data structures.
+There 2 main uses for this gem:
+- gives schema information on a nested data structure (useful when converting to a tabular format)
+- recover types from data that has been serialized to string (e.g. JSON or CSV)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'schema-inference'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install schema-inference
+
+## Usage
+
+1. Report information on nested data structure
+
+```
+schema = Schema::Inference.schema(dataset: [
+  {
+    'person' => {
+      'name' => 'Bob',
+      'age' => 30,
+      'weight' => 60
+    },
+    'updated_at' => '2016-01-01T00:00:00Z'
+  },
+  {
+    'person' => {
+      'name' => 'Alice',
+      # Alice does not want to show her age
+      'weight' => 50.5
+    },
+    'updated_at' => '2016-01-01T00:00:00Z'
+  },
+])
+
+schema['person.name'][:type]  # String
+schema['person.name'][:usage] # 1.0 (100% of the entries have a name)
+
+schema['person.age'][:type]  # Integer
+schema['person.age'][:usage] # 0.5 (50% of the entries have an age)
+
+schema['person.weight'][:type] # Numeric (inferred to be numeric, even though an integer was present)
+schema['updated_at'][:type]    # Time
+```
+
+2. Recover types from string serialization
+
+```
+schema = Schema::Inference.schema(dataset: {
+  'serialized_time' => '2016-01-01T00:00:00Z',
+  'serialized_integer' => '100',
+  'serialized_numeric' => '0.5',
+  'serialized_boolean' => 'true',
+})
+schema['serialized_time'][:type]    # Time
+schema['serialized_integer'][:type] # Integer
+schema['serialized_numeric'][:type] # Numeric
+schema['serialized_boolean'][:type] # Boolean
+```
+
+3. If you need to load a lot of data consider using the following pattern:
+```
+schema = Schema::Inference.schema(batch_count: 10) do |idx\
+ # Pull and return some large amount of data.
+ # Fetching/accessing the data here would avoid the IPC cost of
+ # sending the data to the child process for parallel processing.
+ # e.g.:
+ MongoClient.find.limit(1000).offset(1000 * idx)
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. In certain cases, you maybe want to use the debug flag e.g. `DEBUG=true rake spec` to disable parallel schema processing. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/phybbit/schema-inference. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'schema/inference'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/extensions/boolean.rb

@@ -0,0 +1,3 @@
+# Just declare it, do not add any behavior
+class Boolean
+end unless defined?(Boolean)

data/lib/schema/inference.rb

@@ -0,0 +1,21 @@
+require 'active_support'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/hash/reverse_merge'
+
+require 'parallel'
+require 'timeliness'
+
+require 'extensions/boolean'
+require 'schema/inference/version'
+require 'schema/inference/schema_inferrer'
+
+
+module Schema
+  module Inference
+
+    def Inference.schema(*args)
+      SchemaInferrer.new.infer_schema(*args)
+    end
+
+  end
+end

data/lib/schema/inference/schema_inferrer.rb

@@ -0,0 +1,251 @@
+module Schema
+  module Inference
+    class SchemaInferrer
+      attr_accessor :separator
+
+      def initialize(separator: '.')
+        @separator = separator
+      end
+
+      # Generate a schema based on this collection's records.
+      # We evaluate the schema of each record and then merge all
+      # the information together.
+      # @param dataset [Array] of samples on which we will
+      #        perform the schema analysis.
+      # @param extended [Boolean] Set to true to keep each field as a basic type.
+      #        Set to false to reduce the terminal arrays to a single key (under the type array).
+      # @return [Hash] with one entry per 'column'/'field'. The values
+      #         contains information about the type and usage.
+      def infer_schema(dataset: [], batch_count: 0, extended: false)
+        # support detecting schemas of single objects
+        dataset = [dataset] if dataset.is_a?(Hash)
+        validate_dataset(dataset)
+
+        has_dataset = dataset.count > 0 || (block_given? && batch_count > 0)
+        raise ArgumentError, 'a dataset or a block with a batch count must be passed' unless has_dataset
+
+        if dataset.is_a?(Array) && dataset.count > 0
+          # divide in batches to process in parallel
+          per_process = (dataset.count / Parallel.processor_count.to_f).ceil
+          batch_count = (dataset.count / per_process.to_f).ceil
+        end
+
+        results = parallel_map(batch_count.times) { |i|
+          batch = block_given? ? yield(i) : dataset[i*per_process...(i+1)*per_process]
+          { partial_schema: data_schema(batch), count: batch.count }
+        }
+
+        partial_schemas = results.map { |r| r[:partial_schema] }
+        total_count = results.map { |r| r[:count] }.reduce(:+)
+
+        table_schema = process_schema_results(partial_schemas, total_count, extended)
+        table_schema.sort_by { |k, v| -v[:usage] }.to_h
+      end
+
+      private
+
+      def validate_dataset(dataset)
+        return if dataset.is_a?(Array)
+        raise ArgumentError, 'dataset must be an array or a hash'
+      end
+
+      def data_schema(data)
+        table_schema = {}
+        data.each do |record|
+          # fetch the record schema & update the general schema
+          rec_schema = record_schema(record)
+          rec_schema.each do |field_schema|
+            table_schema[field_schema[:field]] ||= {type: field_schema[:type], usage_count: 0}
+            if table_schema[field_schema[:field]][:type] != field_schema[:type]
+              if table_schema[field_schema[:field]][:type] == NilClass
+                table_schema[field_schema[:field]][:type] = field_schema[:type]
+              elsif field_schema[:type] != nil
+                table_schema[field_schema[:field]][:type] = lowest_common_type(field_schema[:type], table_schema[field_schema[:field]][:type])
+              end
+            end
+            table_schema[field_schema[:field]][:usage_count] += 1
+            table_schema[field_schema[:field]][:types] ||= {}
+            table_schema[field_schema[:field]][:types][field_schema[:type]] ||= 0
+            table_schema[field_schema[:field]][:types][field_schema[:type]] += 1
+          end
+        end
+
+        table_schema
+      end
+
+      def process_schema_results(results, total_count, extended)
+        # aggregate the results
+        table_schema = results[0]
+        results[1..-1].each { |res|
+          table_schema.each { |k, v|
+            next if res[k].blank?
+
+            # aggregate types count
+            res[k][:types].each { |type, count|
+              table_schema[k][:types][type] ||= 0
+              table_schema[k][:types][type] += count
+            }
+
+            # aggregate other informations
+            table_schema[k][:usage_count] += res[k][:usage_count].to_i
+            if (table_schema[k][:type] != res[k][:type])
+              if table_schema[k][:type] == NilClass
+                table_schema[k][:type] = res[k][:type]
+              elsif res[k][:type] != NilClass
+                table_schema[k][:type] = lowest_common_type(res[k][:type], table_schema[k][:type])
+              end
+            end
+          }
+
+          # make sure keys that were not in table_schema are now added.
+          table_schema.reverse_merge!(res)
+        }
+
+        # detect and remove nulls that are part of other schemas
+        # e.g. { 'some_data': null } and { 'some_data': { 'hash': 1 } }
+        # shouldn't be reported as different keys
+        table_schema.each { |k, v|
+          next unless v[:type] == NilClass
+          # check if there is any key that match this one plus an hash/array extension
+          full_key_exists = table_schema.find {|full_key, _| full_key =~ /^#{k}#{Regexp.quote(separator)}.*/}.present?
+          table_schema.delete(k) if full_key_exists
+        }
+
+        # detect and process array information
+        unless extended
+          terminal_array_keys = {}
+          table_schema.keys.each { |key|
+            is_terminal_array = /.*#{Regexp.quote(separator)}[0-9]+$/ =~ key
+            next unless is_terminal_array
+            key_prefix = key.split(separator)[0...-1].join(separator)
+            terminal_array_keys[key_prefix] ||= []
+            terminal_array_keys[key_prefix] << key
+          }
+
+          terminal_array_keys.each do |key_prefix, keys|
+            keys_usage_count = keys.map{ |x| table_schema[x][:usage_count] }
+            usage_count = keys_usage_count.max
+            # min size = how many keys have "always" been used
+            # As the keys may not have been used at the same time,
+            # this may not be valid depending on the array usage.
+            min_size = keys_usage_count.count { |x| x == usage_count }
+            max_size = keys.map { |x| x.split(separator)[-1].to_i }.max + 1
+
+            # delete keys that are part of they array
+            keys.each { |key, _| table_schema.delete(key) }
+
+            table_schema[key_prefix] = {
+              type: Array,
+              usage_count: usage_count,
+              min_size: min_size,
+              max_size: max_size
+            }
+          end
+        end
+
+        # add a percentage in terms of usage
+        table_schema.each { |k, v|
+          table_schema[k][:usage] = table_schema[k][:usage_count] / total_count.to_f
+        }
+
+        table_schema
+      end
+
+      NumericTypes = [Numeric, Integer].freeze
+      def lowest_common_type(type1, type2)
+        return type1 if type1 == type2
+        return Numeric if NumericTypes.include?(type1) && NumericTypes.include?(type2)
+        Object
+      end
+
+      # Recursively explore a record and return its schema
+      def record_schema(record, name = "")
+        if record.is_a? Hash
+          record.flat_map { |k, v|
+            field_name = "#{name}#{separator}#{k}" if name.present?
+            field_name ||= k
+            record_schema(v, field_name)
+          }
+        elsif record.is_a? Array
+          record.each_with_index.flat_map { |x, index|
+            field_name = "#{name}#{separator}#{index}" if name.present?
+            field_name ||= k
+            record_schema(x, field_name)
+          }
+        else
+          { field: name, type: detect_type_of(record) }
+        end
+      end
+
+      def detect_type_of(value)
+        return Boolean  if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+        return Integer  if value.is_a? Integer
+        return Numeric  if value.is_a? Numeric
+        return Time     if value.is_a? Time
+        return NilClass if value.is_a? NilClass
+
+        if value.is_a? String
+          return Integer if value =~ /^[-+]?[0-9]+$/
+          return Numeric if value =~ /^[-+]?[0-9]*\.?[0-9]+$/
+          return Boolean if %w(false true).include?(value.downcase)
+          return Time if Timeliness.parse(value) != nil
+          return String
+        end
+
+        Object
+      end
+
+      def key_access_tokens(key:)
+        key.split(separator).map { |token|
+          # only parse integers for array indexing
+          next token unless is_integer?(token)
+          token.to_i
+        }
+      end
+
+      def record_value(record:, key:)
+        tokens = key_access_tokens(key: key)
+        record.dig(*tokens)
+      end
+
+      def add_value_to_record(record:, key:, value:)
+        tokens = key.split(separator)
+        current_ref = record
+        previous_token = tokens[0]
+
+        tokens[1..-1].each_with_index { |token|
+          if is_integer?(token)
+            current_ref[previous_token] ||= []
+            current_ref = current_ref[previous_token]
+            previous_token = token.to_i
+          else
+            current_ref[previous_token] ||= {}
+            current_ref = current_ref[previous_token]
+            previous_token = token
+          end
+        }
+
+        current_ref[previous_token] = value
+      end
+
+      def is_integer?(value)
+        (/^[+-]?[0-9]+$/ =~ value).present?
+      end
+
+      def parallel_map(itr, &block)
+        # set to true to debug code in the iteration
+        is_debugging_impl = ENV['DEBUG']
+        if is_debugging_impl
+          itr.map do |arg|
+            block.call(arg)
+          end
+        else
+          Parallel.map(itr) do |arg|
+            block.call(arg)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/schema/inference/version.rb

@@ -0,0 +1,5 @@
+module Schema
+  module Inference
+    VERSION = '1.0.0'
+  end
+end

data/schema-inference.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'schema/inference/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'schema-inference'
+  spec.version       = Schema::Inference::VERSION
+  spec.authors       = ['Eurico Doirado']
+  spec.email         = ['eurico@phybbit.com']
+
+  spec.summary       = %q{Supports inferring tabular schemas from deep nested structures.}
+  spec.homepage      = 'https://github.com/Phybbit/schema-inference'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.12'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry-byebug'
+
+  spec.add_dependency 'activesupport', '>= 4.0.0'
+  spec.add_dependency 'parallel', '~>1.8'
+  spec.add_dependency 'timeliness', '~>0.3'
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: schema-inference
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Eurico Doirado
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: timeliness
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+description: 
+email:
+- eurico@phybbit.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/extensions/boolean.rb
+- lib/schema/inference.rb
+- lib/schema/inference/schema_inferrer.rb
+- lib/schema/inference/version.rb
+- schema-inference.gemspec
+homepage: https://github.com/Phybbit/schema-inference
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Supports inferring tabular schemas from deep nested structures.
+test_files: []