goldmine 2.1.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 214cfc93706ed23076acfc401061d060e1753b59
-  data.tar.gz: 754cf23c6dd3c8437d18e83daf7dd9f86e063781
+  metadata.gz: d630803c577229ae72b1ca7bd4e7125840e3f9b6
+  data.tar.gz: 8684e82fa2691af08618d5cb845ec500c99176b5
 SHA512:
-  metadata.gz: 43fdf070ee47b3d767027bcea6c1518c6f948bff9885a0f283dbbfb419d2c560fbd62cb1670027281fdce5c17989414270786b968db0a17b14ac19932b797a1b
-  data.tar.gz: e82d4461294d81e9afd982b429dac780ebd9fd840d4c55176896df95d05f8a38f2a6479446b31ea8dc219e397144c3a976cc48caa27df94ac1069dffaafb3d28
+  metadata.gz: bfc80b08483357bd6cb61260782b7424cff279c8bbdc8d59247edc54fb2e0ed070d9d6797a063bc5af6ab767547854beb9ca1929f20dee9a94b2406081bb550c
+  data.tar.gz: c7dab7e7f518893b98f5213238d10ad0975b9556209084267d7de1bdec1612bde78e3483e27490f201dd8e5c2ed1cd6924723a0bab6b828995123cd9f011faa7

data/Gemfile

@@ -1,3 +1,2 @@
 source "https://rubygems.org"
 gemspec
-

data/Gemfile.lock

@@ -1,41 +1,33 @@
 PATH
   remote: .
   specs:
-    goldmine (2.0.0)
+    goldmine (3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     binding_of_caller (0.7.2)
       debug_inspector (>= 0.0.1)
-    byebug (5.0.0)
-      columnize (= 0.9.0)
-    coderay (1.1.0)
-    columnize (0.9.0)
-    coveralls (0.8.3)
+    byebug (8.2.4)
+    coderay (1.1.1)
+    coveralls (0.8.13)
       json (~> 1.8)
-      rest-client (>= 1.6.8, < 2)
-      simplecov (~> 0.10.0)
+      simplecov (~> 0.11.0)
       term-ansicolor (~> 1.3)
       thor (~> 0.19.1)
+      tins (~> 1.6.0)
     debug_inspector (0.0.2)
     docile (1.1.5)
-    domain_name (0.5.25)
-      unf (>= 0.0.5, < 1.0.0)
-    http-cookie (1.0.2)
-      domain_name (~> 0.5)
     interception (0.5)
     json (1.8.3)
     method_source (0.8.2)
-    mime-types (2.6.2)
-    netrc (0.10.3)
     os (0.9.6)
     pry (0.10.3)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    pry-byebug (3.2.0)
-      byebug (~> 5.0)
+    pry-byebug (3.3.0)
+      byebug (~> 8.0)
       pry (~> 0.10)
     pry-rescue (1.4.2)
       interception (>= 0.5)
@@ -52,29 +44,22 @@ GEM
     rack (1.6.4)
     rack-protection (1.5.3)
       rack
-    rake (10.4.2)
-    rest-client (1.8.0)
-      http-cookie (>= 1.0.2, < 2.0)
-      mime-types (>= 1.16, < 3.0)
-      netrc (~> 0.7)
-    simplecov (0.10.0)
+    rake (11.1.2)
+    simplecov (0.11.2)
       docile (~> 1.1.0)
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
-    sinatra (1.4.6)
-      rack (~> 1.4)
+    sinatra (1.4.7)
+      rack (~> 1.5)
       rack-protection (~> 1.4)
       tilt (>= 1.3, < 3)
     slop (3.6.0)
     term-ansicolor (1.3.2)
       tins (~> 1.0)
     thor (0.19.1)
-    tilt (2.0.1)
+    tilt (2.0.2)
     tins (1.6.0)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.7.1)
 
 PLATFORMS
   ruby
@@ -87,4 +72,4 @@ DEPENDENCIES
   sinatra
 
 BUNDLED WITH
-   1.10.6
+   1.11.2

data/README.md

@@ -1,4 +1,4 @@
-[![Lines of Code](http://img.shields.io/badge/lines_of_code-193-brightgreen.svg?style=flat)](http://blog.codinghorror.com/the-best-code-is-no-code-at-all/)
+[![Lines of Code](http://img.shields.io/badge/lines_of_code-181-brightgreen.svg?style=flat)](http://blog.codinghorror.com/the-best-code-is-no-code-at-all/)
 [![Code Status](http://img.shields.io/codeclimate/github/hopsoft/goldmine.svg?style=flat)](https://codeclimate.com/github/hopsoft/goldmine)
 [![Dependency Status](http://img.shields.io/gemnasium/hopsoft/goldmine.svg?style=flat)](https://gemnasium.com/hopsoft/goldmine)
 [![Build Status](http://img.shields.io/travis/hopsoft/goldmine.svg?style=flat)](https://travis-ci.org/hopsoft/goldmine)
@@ -7,7 +7,7 @@
 
 # Goldmine
 
-Extract a wealth of information from Arrays & Hashes.
+Extract a wealth of information from Arrays.
 
 Goldmine is especially helpful when working with source data that is difficult to query.
 e.g. CSV files, API results, etc...
@@ -28,158 +28,129 @@ gem install goldmine
 
 ```ruby
 require "goldmine"
-
-list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
-  .pivot { |i| i < 5 }
-# result:
-{
-  true  => [1, 2, 3, 4],
-  false => [5, 6, 7, 8, 9]
-}
 ```
 
-## Chained Pivots
-
 ```ruby
 list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
-  .pivot { |i| i < 5 }
-  .pivot { |i| i % 2 == 0 }
-# result:
-{
-  [true, false]  => [1, 3],
-  [true, true]   => [2, 4],
-  [false, false] => [5, 7, 9],
-  [false, true]  => [6, 8]
-}
-```
 
-## Named Pivots
+Goldmine(list)
+  .pivot("< 5") { |i| i < 5 }
+  .result
+  .to_h
+```
 
 ```ruby
-list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
-  .pivot(:less_than_5) { |i| i < 5 }
-# result:
 {
-  { :less_than_5 => true }  => [1, 2, 3, 4],
-  { :less_than_5 => false } => [5, 6, 7, 8, 9]
+  [["< 5", true]]  => [1, 2, 3, 4],
+  [["< 5", false]] => [5, 6, 7, 8, 9]
 }
 ```
 
-## Value Pivots
+## Array Value Pivots
 
 ```ruby
-list = [
+users = [
   { :name => "Sally",   :favorite_colors => [:blue] },
   { :name => "John",    :favorite_colors => [:blue, :green] },
   { :name => "Stephen", :favorite_colors => [:red, :pink, :purple] },
   { :name => "Emily",   :favorite_colors => [:orange, :green] },
   { :name => "Joe",     :favorite_colors => [:red] }
 ]
-list = Goldmine::ArrayMiner.new(list)
-list.pivot { |record| record[:favorite_colors] }
-# result:
+
+Goldmine(users)
+  .pivot(:favorite_color) { |record| record[:favorite_colors] }
+  .result
+  .to_h
+```
+
+```ruby
 {
-  :blue => [
-    { :name => "Sally", :favorite_colors => [:blue] },
-    { :name => "John",  :favorite_colors => [:blue, :green] }
-  ],
-  :green => [
-    { :name => "John",  :favorite_colors => [:blue, :green] },
-    { :name => "Emily", :favorite_colors => [:orange, :green] }
-  ],
-  :red => [
-    { :name => "Stephen", :favorite_colors => [:red, :pink, :purple] },
-    { :name => "Joe",     :favorite_colors => [:red] }
-  ],
-  :pink => [
-    { :name => "Stephen", :favorite_colors => [:red, :pink, :purple] }
-  ],
-  :purple => [
-    { :name => "Stephen", :favorite_colors => [:red, :pink, :purple] }
-  ],
-  :orange => [
-    { :name => "Emily", :favorite_colors => [:orange, :green] }
-  ]
+  [:favorite_color, :blue]   => [{:name=>"Sally", :favorite_colors=>[:blue]}, {:name=>"John", :favorite_colors=>[:blue, :green]}],
+  [:favorite_color, :green]  => [{:name=>"John", :favorite_colors=>[:blue, :green]}, {:name=>"Emily", :favorite_colors=>[:orange, :green]}],
+  [:favorite_color, :red]    => [{:name=>"Stephen", :favorite_colors=>[:red, :pink, :purple]}, {:name=>"Joe", :favorite_colors=>[:red]}],
+  [:favorite_color, :pink]   => [{:name=>"Stephen", :favorite_colors=>[:red, :pink, :purple]}],
+  [:favorite_color, :purple] => [{:name=>"Stephen", :favorite_colors=>[:red, :pink, :purple]}],
+  [:favorite_color, :orange] => [{:name=>"Emily", :favorite_colors=>[:orange, :green]}]
 }
 ```
 
-## Stacked pivots
+## Chained pivots
 
 ```ruby
-list = [
+users = [
   { :name => "Sally",   :age => 21 },
   { :name => "John",    :age => 28 },
   { :name => "Stephen", :age => 37 },
   { :name => "Emily",   :age => 32 },
   { :name => "Joe",     :age => 18 }
 ]
-list = Goldmine::ArrayMiner.new(list)
-mined = list.pivot("Name has an 'e'") do |record|
-  !!record[:name].match(/e/i)
-end
-mined = mined.pivot(">= 21 years old") do |record|
-  record[:age] >= 21
-end
-# result:
+
+Goldmine(users).
+  pivot("'e' in name") { |user| !!user[:name].match(/e/i) }.
+  pivot("21 or over") { |user| user[:age] >= 21 }.
+  result.
+  to_h
+```
+
+```ruby
 {
-  { "Name has an 'e'" => false, ">= 21 years old" => true } => [
-    { :name => "Sally", :age => 21 },
-    { :name => "John",  :age => 28 }
-  ],
-  { "Name has an 'e'" => true, ">= 21 years old" => true } => [
-    { :name => "Stephen", :age => 37 },
-    { :name => "Emily",   :age => 32 }
-  ],
-  { "Name has an 'e'" => true, ">= 21 years old" => false } => [
-    { :name => "Joe", :age => 18 }
-  ]
+  [["'e' in name", false], ["21 or over", true]]  => [{:name=>"Sally", :age=>21}, {:name=>"John", :age=>28}],
+  [["'e' in name", true],  ["21 or over", true]]  => [{:name=>"Stephen", :age=>37}, {:name=>"Emily", :age=>32}],
+  [["'e' in name", true],  ["21 or over", false]] => [{:name=>"Joe", :age=>18}]
 }
 ```
 
 ## Rollups
 
-Rollups provide a clean way to aggregate pivoted data...
-think computed columns.
+An intuitive way to aggregate pivoted data...
+i.e. computed columns.
 
-Rollup `blocks` are executed once for each pivot.
-_Like pivots, rollups can be chained._
+Rollups are `blocks` that get executed once for each pivot entry.
+_They can be also be chained._
 
 ```ruby
 list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
-  .pivot(:less_than_5) { |i| i < 5 }
-  .pivot(:even) { |i| i % 2 == 0 }
-  .rollup(:count) { |matched| matched.size }
-# result:
+
+Goldmine(list)
+  .pivot("< 5") { |i| i < 5 }
+  .pivot("even") { |i| i % 2 == 0 }
+  .result
+  .rollup("count", &:count)
+  .result
+  .to_h
+```
+
+```ruby
 {
-  { :less_than_5 => true, :even => false }  => { :count => 2 },
-  { :less_than_5 => true, :even => true }   => { :count => 2 },
-  { :less_than_5 => false, :even => false } => { :count => 3 },
-  { :less_than_5 => false, :even => true }  => { :count => 2 }
+  [["< 5", true],  ["even", false]] => [["count", 2]],
+  [["< 5", true],  ["even", true]]  => [["count", 2]],
+  [["< 5", false], ["even", false]] => [["count", 3]],
+  [["< 5", false], ["even", true]]  => [["count", 2]]
 }
 ```
 
-### Pre-Computed Results
+### Rollup Caching
 
-Rollups can be computationally expensive _(depending upon how much logic you stuff into the `block`)_.
-Goldmine caches rollup results & makes them available to subsequent rollups.
+Rollups can be computationally expensive.
+Optional caching can be used to reduce this computational overhead.
 
 ```ruby
 list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
+
+Goldmine(list)
   .pivot(:less_than_5) { |i| i < 5 }
-  .rollup(:count, &:size)
-  .rollup(:evens) { |list| list.select { |i| i % 2 == 0 }.size }
-  .rollup(:even_percentage) { |list|
-    computed(:evens).for(list) / computed(:count).for(list).to_f
-  }
-# result:
+  .result
+  .rollup(:count, &:count)
+  .rollup(:evens) { |list| list.select { |i| i % 2 == 0 }.count }
+  .rollup(:even_percentage) { |list| cache[:evens] / cache[:count].to_f }
+  .result(cache: true)
+  .to_h
+```
+
+```ruby
 {
-  { :less_than_5 => true } => { :count => 4, :evens => 2, :even_percentage => 0.5 },
-  { :less_than_5 => false } => { :count => 5, :evens => 2, :even_percentage => 0.4 }
+  [[:less_than_5, true]]  => [[:count, 4], [:evens, 2], [:even_percentage, 0.5]],
+  [[:less_than_5, false]] => [[:count, 5], [:evens, 2], [:even_percentage, 0.4]]
 }
 ```
 
@@ -189,18 +160,35 @@ It's often helpful to flatten rollups into rows.
 
 ```ruby
 list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
+
+rollup = Goldmine(list)
   .pivot(:less_than_5) { |i| i < 5 }
-  .rollup(:count, &:size)
-  .rollup(:evens) { |list| list.select { |i| i % 2 == 0 }.size }
-  .rollup(:even_percentage) { |list|
-    computed(:evens).for(list) / computed(:count).for(list).to_f
-  }
-  .to_rows
-# result:
+  .result
+  .rollup(:count, &:count)
+  .rollup(:evens) { |list| list.select { |i| i % 2 == 0 }.count }
+  .rollup(:even_percentage) { |list| cache[:evens] / cache[:count].to_f }
+  .result(cache: true)
+```
+
+```ruby
+rollup.to_rows
+```
+
+```ruby
 [
-  { "less_than_5" => true, "count" => 4, "evens" => 2, "even_percentage" => 0.5 },
-  { "less_than_5" => false, "count" => 5, "evens" => 2, "even_percentage" => 0.4 }
+  [[:less_than_5, true], [:count, 4], [:evens, 2], [:even_percentage, 0.5]],
+  [[:less_than_5, false], [:count, 5], [:evens, 2], [:even_percentage, 0.4]]
+]
+```
+
+```ruby
+rollup.to_hash_rows
+```
+
+```ruby
+[
+  {:less_than_5=>true, :count=>4, :evens=>2, :even_percentage=>0.5},
+  {:less_than_5=>false, :count=>5, :evens=>2, :even_percentage=>0.4}
 ]
 ```
 
@@ -210,14 +198,19 @@ Rollups can also be converted into tabular format.
 
 ```ruby
 list = [1,2,3,4,5,6,7,8,9]
-Goldmine::ArrayMiner.new(list)
+
+Goldmine(list)
   .pivot(:less_than_5) { |i| i < 5 }
   .pivot(:even) { |i| i % 2 == 0 }
-  .rollup(:count) { |matched| matched.size }
+  .result
+  .rollup(:count, &:size)
+  .result
   .to_tabular
-# result:
+```
+
+```ruby
 [
-  ["less_than_5", "even", "count"],
+  [:less_than_5, :even, :count],
   [true, false, 2],
   [true, true, 2],
   [false, false, 3],
@@ -230,22 +223,25 @@ Goldmine::ArrayMiner.new(list)
 Goldmine makes producing CSV output simple.
 
 ```ruby
-csv_table = Goldmine::ArrayMiner.new(list)
+list = [1,2,3,4,5,6,7,8,9]
+
+Goldmine(list)
   .pivot(:less_than_5) { |i| i < 5 }
   .pivot(:even) { |i| i % 2 == 0 }
+  .result
   .rollup(:count) { |matched| matched.size }
+  .result
   .to_csv_table
-# result:
-#<CSV::Table mode:col_or_row row_count:5>
+  .to_csv
+```
 
-csv_table.to_csv
-# result:
+```ruby
 "less_than_5,even,count\ntrue,false,2\ntrue,true,2\nfalse,false,3\nfalse,true,2\n"
 ```
 
-## Examples
+## Example Apps
 
-All examples are simple Sinatra apps.
+All examples are small Sinatra apps.
 They are designed to help communicate Goldmine use-cases.
 
 ### Setup
@@ -258,7 +254,9 @@ bundle
 
 ### [New York Wifi Hotspots](https://github.com/hopsoft/goldmine/tree/master/examples/new_york_wifi_hotspots)
 
-In this example, we mine the following data.
+Uses data from https://github.com/hopsoft/goldmine/blob/master/examples/new_york_wifi_hotspots/DOITT_WIFI_HOTSPOT_01_13SEPT2010.csv
+
+In this example, we mine out the following information.
 
 * Total hotspots by city, zip, & area code
 * Free hotspots by city, zip, & area code
@@ -284,7 +282,7 @@ curl http://localhost:3000/csv
 
 Uses data from http://dev.socrata.com/foundry/#/data.medicare.gov/aeay-dfax
 
-In this example, we mine the following data.
+In this example, we mine out the following information.
 
 * Total doctors by state & specialty
 * Preferred doctors by state & specialty
@@ -319,22 +317,22 @@ My Macbook Pro yields the following benchmarks.
 
 ```
                       user     system      total        real
-pivoted           1.000000   0.020000   1.020000 (  1.027810)
-rolled_up         1.090000   0.020000   1.110000 (  1.101082)
-rows              0.020000   0.000000   0.020000 (  0.022978)
-tabular           0.010000   0.000000   0.010000 (  0.005423)
-csv               0.030000   0.000000   0.030000 (  0.037245)
+pivoted           0.630000   0.030000   0.660000 (  0.670409)
+rolled_up         0.570000   0.030000   0.600000 (  0.626413)
+rows              0.010000   0.000000   0.010000 (  0.003258)
+tabular           0.010000   0.000000   0.010000 (  0.010110)
+csv               0.050000   0.000000   0.050000 (  0.057677)
 ```
 
 ##### 1,000,000 Records
 
 ```
                       user     system      total        real
-pivoted          15.700000   0.490000  16.190000 ( 16.886677)
-rolled_up         7.070000   0.350000   7.420000 (  7.544060)
-rows              0.020000   0.000000   0.020000 (  0.028432)
-tabular           0.010000   0.010000   0.020000 (  0.007663)
-csv               0.050000   0.000000   0.050000 (  0.058925)
+pivoted           7.270000   0.300000   7.570000 (  8.053166)
+rolled_up         6.800000   0.830000   7.630000 (  8.051707)
+rows              0.000000   0.000000   0.000000 (  0.003934)
+tabular           0.010000   0.000000   0.010000 (  0.011825)
+csv               0.210000   0.010000   0.220000 (  0.222752)
 ```
 
 ## Summary