json-inference 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in json-inference.gemspec
+gemspec
+
+group :test do
+  gem 'shoulda-context', '~> 1.1.6'
+end

data/Gemfile.lock

@@ -0,0 +1,19 @@
+PATH
+  remote: .
+  specs:
+    json-inference (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.1.1)
+    shoulda-context (1.1.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  json-inference!
+  rake
+  shoulda-context (~> 1.1.6)

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Francis Hwang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,13 @@
+# JsonInference
+
+Given a bunch of JSON documents that are assumed to be similar, collects
+info about common structure.  This can be useful for getting a top-level
+overview of a document datastore.
+
+## Example
+
+    report = JsonInference.new_report
+    huge_json['docs'].each do |doc|
+      report << doc
+    end
+    puts report.to_s

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/json-inference.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'json-inference/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "json-inference"
+  spec.version       = Json::Inference::VERSION
+  spec.authors       = ["Francis Hwang"]
+  spec.email         = ["sera@fhwang.net"]
+  spec.description   = %q{Given a bunch of JSON documents that are assumed to be similar, collects info about common structure.}
+  spec.summary       = %q{Given a bunch of JSON documents that are assumed to be similar, collects info about common structure.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/json-inference.rb

@@ -0,0 +1,149 @@
+module JsonInference
+  def self.new_report
+    Report.new
+  end
+
+  class BaseNode
+    def initialize
+      @value_classes = Hash.new 0
+      @sub_nodes = Hash.new { |h,k| 
+        if k == :nth_child
+          sub_node = NthChildNode.new(self)
+        else
+          sub_node = Node.new(k, self)
+        end
+        h[k] = sub_node
+      }
+    end
+
+    def <<(value)
+      if value.is_a?(Hash)
+        value.each do |key, sub_value|
+          @sub_nodes[key] << sub_value
+        end
+      elsif value.is_a?(Array)
+        @sub_nodes[:nth_child]
+        value.each do |sub_value|
+          @sub_nodes[:nth_child] << sub_value
+        end
+      end
+      if value.class == String && value =~ /^(\d){4}-(\d){2}-(\d){2}T(\d){2}:(\d){2}:(\d){2}\.(\d){3}Z$/
+        @value_classes[Date] += 1
+      elsif [true, false].include?(value)
+        @value_classes['Boolean'] += 1
+      else
+        @value_classes[value.class] += 1
+      end
+    end
+
+    def each_sub_node
+      @sub_nodes.keys.sort.each do |key|
+        sub_node = @sub_nodes[key]
+        yield sub_node
+      end
+    end
+
+    def indent_level
+      @parent.indent_level + 1
+    end
+
+    def indent
+      '  ' * indent_level
+    end
+
+    def total_count
+      @value_classes.values.inject { |sum, i| sum + i } || 0
+    end
+  end
+
+  class Node < BaseNode
+    def initialize(name = nil, parent = nil)
+      super()
+      @name, @parent = name, parent
+    end
+
+    def selector
+      "#{@parent.selector} > .#{@name}"
+    end
+
+    def selector_line(documents_count)
+      "#{indent}#{selector}: #{total_count}/#{documents_count} (#{(total_count.to_f / documents_count * 100).round}%)\n"
+    end
+
+    def to_s(documents_count)
+      str = ""
+      str << selector_line(documents_count)
+      @value_classes.each do |klass, count|
+        str << "  #{indent}#{klass}: #{(count / total_count.to_f * 100).round}%\n"
+      end
+      each_sub_node do |sub_node|
+        str << sub_node.to_s(documents_count)
+      end
+      str
+    end
+  end
+
+  class NthChildNode < BaseNode
+    def initialize(parent)
+      super()
+      @parent = parent
+    end
+
+    def selector
+      "#{@parent.selector}:nth-child()"
+    end
+
+    def selector_line(documents_count)
+      "#{indent}#{selector}: #{total_count} child#{'ren' unless total_count == 1}\n"
+    end
+
+    def to_s(documents_count)
+      str = ""
+      str << selector_line(documents_count)
+      @value_classes.each do |klass, count|
+        str << "  #{indent}#{klass}: #{(count / total_count.to_f * 100).round}%\n"
+      end
+      each_sub_node do |sub_node|
+        str << sub_node.to_s(total_count)
+      end
+      str
+    end
+  end
+
+  class RootNode < BaseNode
+    def indent_level
+      -1
+    end
+
+    def selector
+      ':root'
+    end
+
+    def to_s(documents_count)
+      str = ""
+      each_sub_node do |sub_node|
+        str << sub_node.to_s(documents_count)
+        str << "\n"
+      end
+      str
+    end
+  end
+
+  class Report
+    def initialize
+      @documents = []
+      @root = RootNode.new
+    end
+
+    def <<(document)
+      @documents << document
+      @root << document
+    end
+
+    def to_s
+      str = "JsonInference report: #{@documents.size} documents\n"
+      str << @root.to_s(@documents.size)
+      str
+    end
+  end
+end

data/lib/json-inference/version.rb

@@ -0,0 +1,5 @@
+module Json
+  module Inference
+    VERSION = "0.0.1"
+  end
+end

data/test/json_inference_test.rb

@@ -0,0 +1,161 @@
+require 'test/unit'
+$: << '.'
+require 'lib/json-inference'
+require 'shoulda-context'
+
+class JsonInferenceTestCase < Test::Unit::TestCase
+  context "no depth, only strings" do
+    setup do
+      report = JsonInference.new_report
+      report << {foo: 'one'}
+      report << {foo: 'two', bar: 'ONE'}
+      report << {foo: 'three', baz: 'won'}
+      @string = report.to_s
+    end
+
+    should "count selectors as part of the total" do
+      assert_match(/:root > .foo/, @string)
+      assert_match(/3\/3 \(100%\)/, @string)
+    end
+
+    should "count classes per selector" do
+      assert_match(/String: 100%/, @string)
+    end
+
+    should "sort report by selector" do
+      assert_match(/bar.*baz.*foo/m, @string)
+    end
+  end
+
+  context "no depth, date fields" do
+    setup do
+      report = JsonInference.new_report
+      report << {created_at: '2013-08-21T20:50:16.921Z'}
+      report << {created_at: '2013-08-21T20:50:16.555Z'}
+      @string = report.to_s
+    end
+
+    should "recognize date fields based on format" do
+      assert_match(/Date: 100%/, @string)
+    end
+  end
+
+  context "no depth, boolean fields" do
+    setup do
+      report = JsonInference.new_report
+      report << {featured: true}
+      report << {featured: false}
+      @string = report.to_s
+    end
+
+    should "group boolean fields" do
+      assert_match(/Boolean: 100%/, @string)
+    end
+  end
+
+  context "hash with uniform keys" do
+    setup do
+      report = JsonInference.new_report
+      report << {embedded: {title: 'title', position: 1}}
+      report << {embedded: {title: 'title two', position: 2}}
+      @string = report.to_s
+    end
+
+    should "show full selectors" do
+      assert_match(/:root > .embedded > .title/, @string)
+      assert_match(/2\/2 \(100%\)/, @string)
+    end
+
+    should "count classes per selector" do
+      assert_match(/String: 100%/, @string)
+    end
+
+    should "sort report by selector" do
+      assert_match(/embedded.*position/m, @string)
+    end
+    
+    should "display count for the overall hash too" do
+      assert_match(/:root > .embedded: 2\/2 \(100%\)/, @string)
+    end
+  end
+
+  context "hash with inconsistent keys" do
+    setup do
+      report = JsonInference.new_report
+      report << {embedded: {title: 'title'}}
+      report << {embedded: {}}
+      @string = report.to_s
+    end
+
+    should "calculate percentages related to occurrences of the field" do
+      assert_match(/String: 100%/, @string)
+    end
+  end
+
+  context "field that is sometimes a hash and sometimes not" do
+    setup do
+      report = JsonInference.new_report
+      report << {embedded: {title: 'title'}}
+      report << {embedded: "what's this doing here"}
+      @string = report.to_s
+    end
+
+    should "display all top-level classes" do
+      assert_match(/Hash: 50%/, @string)
+      assert_match(/String: 50%/, @string)
+    end
+
+    should "display sub nodes" do
+      assert_match(/:root > .embedded > .title: 1\/2/, @string)
+    end
+  end
+
+  context "array" do
+    setup do
+      report = JsonInference.new_report
+      report << {items: [1, 2, 3]}
+      report << {items: [4, 5, 6]}
+      @string = report.to_s
+    end
+
+    should "display a different sort of selector" do
+      assert_match(/:root > .items:nth-child\(\): 6 children$/, @string)
+    end
+
+    should "count types of children" do
+      assert_match(/Fixnum: 100%/, @string)
+    end
+  end
+
+  context "array of hashes" do
+    setup do
+      report = JsonInference.new_report
+      report << {items: [{one: 'one', two: 'two'}, {one: 'ONE', two: 'TWO'}]}
+      report << {items: [{one: 'won', two: 'too'}, {one: 1, two: 'two'}]}
+      @string = report.to_s
+    end
+    
+    should "count elements in each hash" do
+      assert_match(/:root > .items:nth-child\(\) > .one: 4\/4 \(100%\)$/, @string)
+      assert_match(/:root > .items:nth-child\(\) > .two: 4\/4 \(100%\)$/, @string)
+    end
+
+    should "count value classes in hashes too" do
+      assert_match(/String: 75%/, @string)
+      assert_match(/Fixnum: 25%/, @string)
+    end
+  end
+  
+  context "empty array" do
+    setup do
+      report = JsonInference.new_report
+      report << {items: []}
+      report << {items: []}
+      @string = report.to_s
+    end
+    
+    should "display that there are zero children" do
+      assert_match(/:root > .items:nth-child\(\): 0 children$/, @string)
+    end
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: json-inference
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Francis Hwang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Given a bunch of JSON documents that are assumed to be similar, collects
+  info about common structure.
+email:
+- sera@fhwang.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- json-inference.gemspec
+- lib/json-inference.rb
+- lib/json-inference/version.rb
+- test/json_inference_test.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2986096997133027698
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2986096997133027698
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Given a bunch of JSON documents that are assumed to be similar, collects
+  info about common structure.
+test_files:
+- test/json_inference_test.rb