goldmine 0.0.3 → 0.0.4

data/README.md

@@ -1,180 +1,387 @@
 # Goldmine
 
-## Data mining made easy... the Ruby way.
-### Turn any list into a treasure trove.
+## Pivot tables for the Rubyist
 
-Goldmine allows you to apply pivot table logic to any list for powerful data mining capabilities.
+### Pivot any list into a wealth of information.
 
-In the nomenclature of Goldmine, we call this digging for data. So we've added a `dig` method to `Array`.
+Goldmine allows you to apply pivot table logic to any list for powerful data mining capabilities.
 
-#### More reasons to love it
+### Reasons to love it
 
 * Provides ETL like functionality... but simple and elegant
+* Easily build OLAP cubes using Ruby
 * Supports method chaining for deep data mining
 * Handles values that are lists themselves
-* Allows you to name your pivots
 
-What does this all mean for you? Lets have a look.
+[Why use it?](#putting-it-all-together)
+
+## Quick start
+
+Install
+
+```
+$gem install goldmine
+```
+
+Use
+
+```ruby
+[1,2,3,4,5,6,7,8,9].pivot { |i| i < 5 }
+```
+
+### Usage examples
+
+* [Pivot a list](#pivot-a-list-of-numbers-based-on-whether-or-not-they-are-less-than-5)
+* [Create a named pivot](#explicitly-name-a-pivot)
+* [Pivot values that are lists themselves](#pivot-values-that-are-lists-themselves)
+* [Chain pivots](#chain-pivots-together)
+* [Chain pivots conditionally](#conditionally-chain-pivots-together)
+* [Dig deep and extract meaningful data](#deep-cuts)
 
 ## The Basics
 
-#### Pivot a simple list of numbers based on whether or not they are less than 5
+### Pivot a list of numbers based on whether or not they are less than 5
 
 ```ruby
+# operation
 list = [1,2,3,4,5,6,7,8,9]
-data = list.dig { |i| i < 5 }
+data = list.pivot { |i| i < 5 }
 
-# {
-#   true  => [1, 2, 3, 4],
-#   false => [5, 6, 7, 8, 9]
-# }
+# resulting data
+{
+  true  => [1, 2, 3, 4],
+  false => [5, 6, 7, 8, 9]
+}
 ```
 
-#### The same pivot as above but explicitly named
+### Explicitly name a pivot
 
 ```ruby
+# operation
 list = [1,2,3,4,5,6,7,8,9]
-data = list.dig("less than 5") { |i| i < 5 }
+data = list.pivot("less than 5") { |i| i < 5 }
 
-# {
-#   "less than 5: true"  => [1, 2, 3, 4],
-#   "less than 5: false" => [5, 6, 7, 8, 9]
-# }
+# resulting data
+{
+  { "less than 5" => true }  => [1, 2, 3, 4],
+  { "less than 5" => false } => [5, 6, 7, 8, 9]
+}
 ```
 
 ## Next Steps
 
-#### Chain pivots together
+### Pivot values that are lists themselves
 
 ```ruby
+# operation
+list = [
+  { :name => "one",   :list => [1] },
+  { :name => "two",   :list => [1, 2] },
+  { :name => "three", :list => [1, 2, 3] },
+  { :name => "four",  :list => [1, 2, 3, 4] },
+]
+data = list.pivot { |record| record[:list] }
+
+# resulting data
+{
+  1 => [ { :name => "one",   :list => [1] },
+         { :name => "two",   :list => [1, 2] },
+         { :name => "three", :list => [1, 2, 3] },
+         { :name => "four",  :list => [1, 2, 3, 4] } ],
+  2 => [ { :name => "two",   :list => [1, 2] },
+         { :name => "three", :list => [1, 2, 3] },
+         { :name => "four",  :list => [1, 2, 3, 4] } ],
+  3 => [ { :name => "three", :list => [1, 2, 3] },
+         { :name => "four",  :list => [1, 2, 3, 4] } ],
+  4 => [ { :name => "four",  :list => [1, 2, 3, 4] } ]
+}
+```
+
+### Chain pivots together
+
+```ruby
+# operation
 list = [1,2,3,4,5,6,7,8,9]
-data = list.dig { |i| i < 5 }.dig { |i| i % 2 == 0 }
-
-# {
-#   [true, false]  => [1, 3],
-#   [true, true]   => [2, 4],
-#   [false, false] => [5, 7, 9],
-#   [false, true]  => [6, 8]
-# }
+data = list.pivot { |i| i < 5 }.pivot { |i| i % 2 == 0 }
+
+# resulting data
+{
+  [true, false]  => [1, 3],
+  [true, true]   => [2, 4],
+  [false, false] => [5, 7, 9],
+  [false, true]  => [6, 8]
+}
 ```
 
-#### The same pivot as above but explicitly named
+### Conditionally chain pivots together
 
 ```ruby
+# operation
+params = { :divisible_by_two => false, :next_greater_than_five => true }
 list = [1,2,3,4,5,6,7,8,9]
-data = list.dig("less than 5") { |i| i < 5 }.dig("divisible by 2") { |i| i % 2 == 0 }
-
-# {
-#   ["less than 5: true", "divisible by 2: false"]  => [1, 3],
-#   ["less than 5: true", "divisible by 2: true"]   => [2, 4],
-#   ["less than 5: false", "divisible by 2: false"] => [5, 7, 9],
-#   ["less than 5: false", "divisible by 2: true"]  => [6, 8]
-# }
+data = list.pivot("less than 5") { |i| i < 5 }
+data = data.pivot("divisible by 2") { |i| i % 2 == 0 } if params[:divisible_by_two]
+data = data.pivot("next greater than 5") { |i| i.next > 5 } if params[:next_greater_than_five]
+
+# resulting data
+{
+  { "less than 5" => true,  "next greater than 5" => false } => [1, 2, 3, 4],
+  { "less than 5" => false, "next greater than 5" => true } => [5, 6, 7, 8, 9]
+}
 ```
 
 ## Deep Cuts
 
-#### Pivot a list of users based on a value that is itself a list
+### Build a moderately complex dataset of Cities
 
 ```ruby
-list = [
-  { :name => "Nathan",  :projects => [:a, :b] },
-  { :name => "Eric",    :projects => [:a, :d, :g] },
-  { :name => "Brian",   :projects => [:b, :c, :e, :f] },
-  { :name => "Mark",    :projects => [:g] },
-  { :name => "Josh",    :projects => [:a, :c] },
-  { :name => "Matthew", :projects => [:b, :c, :d] }
+cities = [
+  { :name => "San Francisco",
+    :state => "CA",
+    :population => 805235,
+    :airlines => [ "Delta", "United", "SouthWest" ]
+  },
+  {
+    :name => "Mountain View",
+    :state => "CA",
+    :population => 74066,
+    :airlines => [ "SkyWest", "United", "SouthWest" ]
+  },
+  {
+    :name => "Manhattan",
+    :state => "NY",
+    :population => 1586698,
+    :airlines => [ "Delta", "JetBlue", "United" ]
+  },
+  {
+    :name => "Brooklyn",
+    :state => "NY",
+    :population => 2504700,
+    :airlines => [ "Delta", "American", "US Airways" ]
+  },
+  {
+    :name => "Boston",
+    :state => "MA",
+    :population => 617594,
+    :airlines => [ "Delta", "JetBlue", "American" ]
+  },
+  {
+    :name => "Atlanta",
+    :state => "GA",
+    :population => 420003,
+    :airlines => [ "Delta", "United", "SouthWest" ]
+  },
+  {
+    :name => "Dallas",
+    :state => "TX",
+    :population => 1197816,
+    :airlines => [ "Delta", "SouthWest", "Frontier" ]
+  }
 ]
-data = list.dig { |record| record[:projects] }
-
-# {
-#   :a => [ { :name => "Nathan",  :projects => [:a, :b] },
-#           { :name => "Eric",    :projects => [:a, :d, :g] },
-#           { :name => "Josh",    :projects => [:a, :c] } ],
-#   :b => [ { :name => "Nathan",  :projects => [:a, :b] },
-#           { :name => "Brian",   :projects => [:b, :c, :e, :f] },
-#           { :name => "Matthew", :projects => [:b, :c, :d] } ],
-#   :d => [ { :name => "Eric",    :projects => [:a, :d, :g] },
-#           { :name => "Matthew", :projects => [:b, :c, :d] } ],
-#   :g => [ { :name => "Eric",    :projects => [:a, :d, :g] },
-#           { :name => "Mark",    :projects => [:g] } ],
-#   :c => [ { :name => "Brian",   :projects => [:b, :c, :e, :f] },
-#           { :name => "Josh",    :projects => [:a, :c] },
-#           { :name => "Matthew", :projects => [:b, :c, :d] } ],
-#   :e => [ { :name => "Brian",   :projects => [:b, :c, :e, :f] } ],
-#   :f => [ { :name => "Brian",   :projects => [:b, :c, :e, :f] } ]
-# }
-
 ```
 
-#### Pivot a list of users based on lang and number of projects owned
+### Pivot cities by state for population over 750k
 
 ```ruby
-list = [
-  { :name => "Nathan",  :langs => [:ruby, :javascript],          :projects => [:a, :b] },
-  { :name => "Eric",    :langs => [:ruby, :javascript, :groovy], :projects => [:a, :d, :g] },
-  { :name => "Brian",   :langs => [:ruby, :javascript, :c, :go], :projects => [:b, :c, :e, :f] },
-  { :name => "Mark",    :langs => [:ruby, :java, :scala],        :projects => [:g] },
-  { :name => "Josh",    :langs => [:ruby, :lisp, :clojure],      :projects => [:a, :c] },
-  { :name => "Matthew", :langs => [:ruby, :c, :clojure],         :projects => [:b, :c, :d] }
-]
-data = list
-  .dig("lang") { |rec| rec[:langs] }
-  .dig("project count") { |rec| rec[:projects].length }
-
-# {
-#   ["lang: ruby", "project count: 2"]       => [ { :name => "Nathan", ... }, { :name => "Josh", ... } ],
-#   ["lang: ruby", "project count: 3"]       => [ { :name => "Eric", ... }, { :name => "Matthew", ... } ],
-#   ["lang: ruby", "project count: 4"]       => [ { :name => "Brian", ... } ],
-#   ["lang: ruby", "project count: 1"]       => [ { :name => "Mark", ... } ],
-#   ["lang: javascript", "project count: 2"] => [ { :name => "Nathan", ... } ],
-#   ["lang: javascript", "project count: 3"] => [ { :name => "Eric", ... } ],
-#   ["lang: javascript", "project count: 4"] => [ { :name => "Brian", ... } ],
-#   ["lang: groovy", "project count: 3"]     => [ { :name => "Eric", ... } ],
-#   ["lang: c", "project count: 4"]          => [ { :name => "Brian", ... } ],
-#   ["lang: c", "project count: 3"]          => [ { :name => "Matthew", ... } ],
-#   ["lang: go", "project count: 4"]         => [ { :name => "Brian", ... } ],
-#   ["lang: java", "project count: 1"]       => [ { :name => "Mark", ... } ],
-#   ["lang: scala", "project count: 1"]      => [ { :name => "Mark", ... } ],
-#   ["lang: lisp", "project count: 2"]       => [ { :name => "Josh", ... } ],
-#   ["lang: clojure", "project count: 2"]    => [ { :name => "Josh", ... } ],
-#   ["lang: clojure", "project count: 3"]    => [ { :name => "Matthew", ... } ]
-# }
+# operation
+data = cities
+  .pivot("state") { |city| city[:state] }
+  .pivot("population >= 750k") { |city| city[:population] >= 750000 }
+
+# resulting data
+{
+  { "state" => "CA", "population >= 750k" => true }  => [ { :name => "San Francisco", ... } ],
+  { "state" => "CA", "population >= 750k" => false } => [ { :name => "Mountain View", ... } ],
+  { "state" => "NY", "population >= 750k" => true }  => [ { :name => "Manhattan", ... }, { :name => "Brooklyn", ... } ],
+  { "state" => "MA", "population >= 750k" => false } => [ { :name => "Boston", ... } ],
+  { "state" => "GA", "population >= 750k" => false } => [ { :name => "Atlanta", ... } ],
+  { "state" => "TX", "population >= 750k" => true }  => [ { :name => "Dallas", ... } ]
+}
 ```
 
-#### Pivot a list of users based on whether or not they know javascript, what other languages they know, and whether or not their name contains the letter 'a'
-
-*Pretty contrived example here, but hopefully illustrates the type of power thats available.*
+### Putting it all together
+
+**The end goal of all this is to support the creation of aggregate reports.**
+
+*You can think of these reports as individual data cubes.*
+
+Here is a table view of the pivoted city data from above.
+
+<table>
+  <thead>
+    <tr>
+      <th>state</th>
+      <th>population >= 750k</th>
+      <th>cities</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>CA</td>
+      <td>true</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>CA</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>NY</td>
+      <td>true</td>
+      <td>2</td>
+    </tr>
+    <tr>
+      <td>MA</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>GA</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>TX</td>
+      <td>true</td>
+      <td>1</td>
+    </tr>
+  </tbody>
+</table>
+
+Lets try another one.
+
+### Determine which airlines service cities with fewer than 750k people
 
 ```ruby
-list = [
-  { :name => "Nathan",  :langs => [:ruby, :javascript],          :projects => [:a, :b] },
-  { :name => "Eric",    :langs => [:ruby, :javascript, :groovy], :projects => [:a, :d, :g] },
-  { :name => "Brian",   :langs => [:ruby, :javascript, :c, :go], :projects => [:b, :c, :e, :f] },
-  { :name => "Mark",    :langs => [:ruby, :java, :scala],        :projects => [:g] },
-  { :name => "Josh",    :langs => [:ruby, :lisp, :clojure],      :projects => [:a, :c] },
-  { :name => "Matthew", :langs => [:ruby, :c, :clojure],         :projects => [:b, :c, :d] }
-]
-data = list
-  .dig("knows javascript") { |rec| rec[:langs].include?(:javascript) }
-  .dig("lang") { |rec| rec[:langs] }
-  .dig("name includes 'a'") { |rec| rec[:name].include?("a") }
-
-# {
-#   ["knows javascript: true", "lang: ruby", "name includes 'a': true"]        => [ { :name => "Nathan", ... }, { :name => "Brian", ... } ],
-#   ["knows javascript: true", "lang: ruby", "name includes 'a': false"]       => [ { :name => "Eric", ... } ],
-#   ["knows javascript: true", "lang: javascript", "name includes 'a': true"]  => [ { :name => "Nathan", ... }, { :name => "Brian", ... } ],
-#   ["knows javascript: true", "lang: javascript", "name includes 'a': false"] => [ { :name => "Eric", ... } ],
-#   ["knows javascript: true", "lang: groovy", "name includes 'a': false"]     => [ { :name => "Eric", ... } ],
-#   ["knows javascript: true", "lang: c", "name includes 'a': true"]           => [ { :name => "Brian", ... } ],
-#   ["knows javascript: true", "lang: go", "name includes 'a': true"]          => [ { :name => "Brian", ... } ],
-#   ["knows javascript: false", "lang: ruby", "name includes 'a': true"]       => [ { :name => "Mark", ... }, { :name => "Matthew", ... } ],
-#   ["knows javascript: false", "lang: ruby", "name includes 'a': false"]      => [ { :name => "Josh", ... } ],
-#   ["knows javascript: false", "lang: java", "name includes 'a': true"]       => [ { :name => "Mark", ... } ],
-#   ["knows javascript: false", "lang: scala", "name includes 'a': true"]      => [ { :name => "Mark", ... } ],
-#   ["knows javascript: false", "lang: lisp", "name includes 'a': false"]      => [ { :name => "Josh", ... } ],
-#   ["knows javascript: false", "lang: clojure", "name includes 'a': false"]   => [ { :name => "Josh", ... } ],
-#   ["knows javascript: false", "lang: clojure", "name includes 'a': true"]    => [ { :name => "Matthew", ... } ],
-#   ["knows javascript: false", "lang: c", "name includes 'a': true"]          => [ { :name => "Matthew", ... } ]
-# }
+# operation
+data = cities
+  .pivot("airline") { |city| city[:airlines] }
+  .pivot("population < 750k") { |city| city[:population] < 750000 }
+
+# resulting data
+{
+  { "airline" => "Delta", "population < 750k" => false } => [
+    { :name => "San Francisco", ... },
+    { :name => "Manhattan", ... },
+    { :name => "Brooklyn", ... },
+    { :name => "Dallas", ... }],
+  { "airline" => "Delta", "population < 750k" => true } => [
+    { :name => "Boston", ... },
+    { :name => "Atlanta", ... }],
+  { "airline" => "United", "population < 750k" => false } => [
+    { :name => "San Francisco", ... },
+    { :name => "Manhattan", ... }],
+  { "airline" => "United", "population < 750k" => true } => [
+    { :name => "Mountain View", ... },
+    { :name => "Atlanta", ... }],
+  { "airline" => "SouthWest", "population < 750k" => false } => [
+    { :name => "San Francisco", ... },
+    { :name => "Dallas", ... }],
+  { "airline" => "SouthWest", "population < 750k" => true } => [
+    { :name => "Mountain View", ... },
+    { :name => "Atlanta", ... }],
+  { "airline" => "SkyWest", "population < 750k" => true } => [
+    { :name => "Mountain View", ... }],
+  { "airline" => "JetBlue", "population < 750k" => false } => [
+    { :name => "Manhattan", ... }],
+  { "airline" => "JetBlue", "population < 750k" => true } => [
+    { :name => "Boston", ... }],
+  { "airline" => "American", "population < 750k" => false } => [
+    { :name => "Brooklyn", ... }],
+  { "airline" => "American", "population < 750k" => true } => [
+    { :name => "Boston", ... }],
+  { "airline" => "US Airways", "population < 750k" => false } => [
+    { :name => "Brooklyn", ... }],
+  { "airline" => "Frontier", "population < 750k" => false } => [
+    { :name => "Dallas", ... }]
+}
 ```
+
+Here is the corresponding table view for the above dataset.
+
+<table>
+  <thead>
+    <tr>
+      <th>airline</th>
+      <th>population &lt; 750k</th>
+      <th>cities</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>Delta</td>
+      <td>false</td>
+      <td>4</td>
+    </tr>
+    <tr>
+      <td>Delta</td>
+      <td>true</td>
+      <td>2</td>
+    </tr>
+    <tr>
+      <td>United</td>
+      <td>false</td>
+      <td>2</td>
+    </tr>
+    <tr>
+      <td>United</td>
+      <td>true</td>
+      <td>2</td>
+    </tr>
+    <tr>
+      <td>SouthWest</td>
+      <td>false</td>
+      <td>2</td>
+    </tr>
+    <tr>
+      <td>SouthWest</td>
+      <td>true</td>
+      <td>2</td>
+    </tr>
+    <tr>
+      <td>SkyWest</td>
+      <td>true</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>JetBlue</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>JetBlue</td>
+      <td>true</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>American</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>American</td>
+      <td>true</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>US Airways</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+    <tr>
+      <td>Frontier</td>
+      <td>false</td>
+      <td>1</td>
+    </tr>
+  </tbody>
+</table>
+
+Hopefully you can see the potential even though the above examples are somewhat contrived.
+
+## Special thanks
+
+* One on One Marketing - for sponsoring the development of Goldmine
+* Eric Berry - for constructive feedback
+* Josh Bowles - for early adoption and feedback
+* Brett Beers - for early adoption and feedback

data/lib/goldmine.rb

@@ -2,7 +2,7 @@ require "rubygems"
 
 module Goldmine
   module ArrayMiner
-    def dig(name=nil, &block)
+    def pivot(name=nil, &block)
       reduce({}) do |memo, item|
         value = yield(item)
 
@@ -24,19 +24,18 @@ module Goldmine
 
   module HashMiner
     attr_accessor :goldmine
-    def dig(name=nil, &block)
+    def pivot(name=nil, &block)
       return self unless goldmine
       reduce({}) do |memo, item|
         key = item.first
         value = item.last
-        value.dig(name, &block).each do |k, v|
-          new_key = []
-          if key.is_a? Array
-            new_key = new_key + key
+        value.pivot(name, &block).each do |k, v|
+          if key.is_a? Hash
+            k = { block.to_s => k } unless k.is_a?(Hash)
+            new_key = key.merge(k)
           else
-            new_key << key
+            new_key = [key, k]
           end
-          new_key << k
           memo[new_key] = v
         end
         memo.goldmine = true
@@ -51,7 +50,7 @@ module Goldmine
     end
 
     def goldmine_key(name, key)
-      mine_key = "#{name}: #{key}" if name
+      mine_key = { name => key } if name
       mine_key ||= key
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: goldmine
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -50,5 +50,5 @@ rubyforge_project:
 rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
-summary: Data mining made easy... the Ruby way.
+summary: Pivot tables for the Rubyist
 test_files: []