data_hut 0.0.5 → 0.0.6

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.0.6 
+
+* externalized the Sequel database logger so that it can be set by DataHut clients.  See DataHut::DataWarehouse#logger=
+
+* added type checking on extract and transform to ensure safe operation with underlying Sequel sqlite3 database.
+
 ## 0.0.5
 
 * added rdoc

data/README.md

@@ -229,7 +229,6 @@ Have fun!
 
 ## TODOS
 
-* fill out tests
 * further optimizations
 * time-based series and binning helpers (by week/day/hour/5-min/etc).
 

data/Rakefile

@@ -11,5 +11,5 @@ task :default => :test
 
 desc "clean up"
 task :clean do
-  FileUtils.rm(FileList["*.db"], force: true, verbose: true)
+  FileUtils.rm(FileList["samples/**/*.db"], force: true, verbose: true)
 end

data/lib/data_hut/data_warehouse.rb

@@ -78,7 +78,6 @@ module DataHut
     # @yield [record, element] lets you control the mapping of data elements to record fields
     # @yieldparam record an OpenStruct that allows you to create fields dynamically on the record as needed. 
     #   These fields will automatically be added to the schema behind the DataHut using the ruby data type you assigned to the record.
-    #   *NOTE* that you must use DateTime or Time objects as Date objects are not supported.
     #   See {http://sequel.rubyforge.org/rdoc/files/doc/schema_modification_rdoc.html Sequel Schema Modification Methods} for 
     #   more information about supported ruby data types you can use.
     # @yieldparam element an element from your data.
@@ -108,7 +107,6 @@ module DataHut
     # @yieldparam record an OpenStruct that fronts the DataHut record.  You may access existing fields on this record or create new 
     #   fields to store synthetic data from a transform pass. 
     #   These fields will automatically be added to the schema behind the DataHut using the ruby data type you assigned to the record.
-    #   *NOTE* that you must use DateTime or Time objects as Date objects are not supported.
     #   See {http://sequel.rubyforge.org/rdoc/files/doc/schema_modification_rdoc.html Sequel Schema Modification Methods} for 
     #   more information about supported ruby data types you can use.
     # @raise [ArgumentError] if you don't provide a block
@@ -150,12 +148,25 @@ module DataHut
       @db[:data_warehouse].update(:dw_processed => true)
     end
 
+    # attach a Logger to the underlying Sequel database so that you can debug or monitor database actions.
+    # See {http://sequel.rubyforge.org/rdoc/classes/Sequel/Database.html#method-i-logger-3D Sequel::Database#logger=}.
+    #
+    # @example 
+    #   dh.logger = Logger.new(STDOUT)
+    #
+    # @param logger [Logger] a logger for the underlying Sequel actions.
+    # @raise [ArgumentError] if passed a logger that is not a kind of {http://www.ruby-doc.org/stdlib-1.9.3//libdoc/logger/rdoc/Logger.html Logger}.
+    def logger=(logger)
+      raise(ArgumentError, "logger must be a type of Logger.") unless logger.kind_of?(Logger)
+      @db.logger = logger
+    end
+
     private 
 
     def initialize(name)
       @db_file = "#{name}.db"
       @db = Sequel.sqlite(@db_file)
-      #@db.logger = ::Logger.new(STDOUT)
+      
       unless @db.table_exists?(:data_warehouse)
         @db.create_table(:data_warehouse) do
           primary_key :dw_id
@@ -177,6 +188,9 @@ module DataHut
       h = r.marshal_dump
       h.keys.each do |key|
         type = h[key].class
+        unless Sequel::Schema::CreateTableGenerator::GENERIC_TYPES.include?(type)
+          raise(ArgumentError, "DataHut: Ruby type '#{type}' not supported by Sequel. Must be one of the supported types: #{Sequel::Schema::CreateTableGenerator::GENERIC_TYPES.inspect}", caller)
+        end
         unless @db[:data_warehouse].columns.include?(key)
           @db.alter_table(:data_warehouse) do 
             add_column key, type

data/lib/data_hut/version.rb

@@ -1,3 +1,3 @@
 module DataHut
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/samples/basic.rb

@@ -0,0 +1,55 @@
+
+# run from the samples dir with:
+# $ ruby basic.rb 
+
+require_relative 'sample_helper.rb'
+
+require 'data_hut'
+require 'pry'
+
+dh = DataHut.connect("sample")
+
+data = [{name: "barney", age: 27, login: DateTime.parse('2008-05-03') },
+        {name: "phil", age: 31},
+        {name: "fred", age: 44, login: DateTime.parse('2013-02-07')},
+        {name: "sarah", age: 24, login: DateTime.parse('2011-04-01')},
+        {name: "robin", age: 45},
+        {name: "jane", age: 19, login: DateTime.parse('2012-10-14')}]
+
+# extract your data by iterating over your data format (from whatever source) and map it to a record model...
+puts "extracting data"
+dh.extract(data) do |r, d|
+  r.name = d[:name]
+  r.age = d[:age]
+  # data quality step:
+  d[:login] = DateTime.new unless d.has_key?(:login)
+  r.last_active = d[:login]
+  print '.'
+end
+
+# and only transform the new records automatically
+puts "\ntransforming data"
+dh.transform do |r|
+  r.eligible = r.age < 30
+  print '*'
+end
+
+dh.transform_complete
+puts "\ndone."
+
+# operate on your dataset by using chained queries
+ds = dh.dataset
+
+ds.each{|d| puts d.inspect}
+
+puts "Average age: #{ds.avg(:age)}"
+
+puts "Eligible:"
+eligible = ds.where(eligible:true)
+eligible.each{|d| puts d.inspect}
+
+binding.pry
+
+# clean up scratch demo
+FileUtils.rm("sample.db")
+puts "done."

data/samples/league_of_legends.rb

@@ -0,0 +1,97 @@
+
+# run from the samples dir with: 
+# $ ruby league_of_legends.rb 
+
+require_relative 'sample_helper.rb'
+
+require 'data_hut'
+require 'nokogiri'
+require 'open-uri'
+require 'pry'
+
+root = 'http://na.leagueoflegends.com'
+
+# load the data once... (manually delete it to refresh)
+unless File.exists?("lolstats.db")
+  dh = DataHut.connect("lolstats")
+
+  champions_page = Nokogiri::HTML(open("#{root}/champions"))
+
+  urls = champions_page.css('table.champion_item td.description span a').collect{|e| e.attribute('href').value}
+
+  # keep the powers for later since they are on different pages.
+  powers = {}
+  champions_page.css('table.champion_item').each do |c|
+    name        = c.css('td.description span.highlight a').text
+    attack      = c.css('td.graphing td.filled_attack').count
+    health      = c.css('td.graphing td.filled_health').count
+    spells      = c.css('td.graphing td.filled_spells').count
+    difficulty  = c.css('td.graphing td.filled_difficulty').count
+    powers.store(name, {attack_power: attack, defense_power: health, ability_power: spells, difficulty: difficulty})
+  end
+
+  puts "loading champion data"
+  dh.extract(urls) do |r, url|
+    champion_page = Nokogiri::HTML(open("#{root}#{url}"))
+    r.name = champion_page.css('div.page_header_text').text
+
+    st = champion_page.css('table.stats_table')
+    names = st.css('td.stats_name').collect{|e| e.text.strip}
+    values = st.css('td.stats_value').collect{|e| e.text.strip}
+    modifiers = st.css('td.stats_modifier').collect{|e| e.text.strip}
+
+    (0..names.count-1).collect do |i| 
+      stat = (names[i].downcase.gsub(/ /,'_') << "=").to_sym
+      r.send(stat, values[i].to_f)
+      stat_per_level = (names[i].downcase.gsub(/ /,'_') << "_per_level=").to_sym
+      per_level_value = modifiers[i].match(/\+([\d\.]+)/)[1].to_f rescue 0
+      r.send(stat_per_level, per_level_value)
+    end
+
+    # add the powers for this champion...
+    power = powers[r.name]
+    r.attack_power = power[:attack_power]
+    r.defense_power = power[:defense_power]
+    r.ability_power = power[:ability_power]
+    r.difficulty = power[:difficulty]
+
+    print "."
+  end
+  puts "done."
+end
+
+dh = DataHut.connect("lolstats")
+
+puts "first transform"
+dh.transform do |r|
+  r.total_damage = r.damage + (r.damage_per_level * 18.0)
+  r.total_health = r.health + (r.health_per_level * 18.0)
+  r.total_mana = r.mana + (r.mana_per_level * 18.0)
+  r.total_move_speed = r.move_speed + (r.move_speed_per_level * 18.0)
+  r.total_armor = r.armor + (r.armor_per_level * 18.0)
+  r.total_spell_block = r.spell_block + (r.spell_block_per_level * 18.0)
+  r.total_health_regen = r.health_regen + (r.health_regen_per_level * 18.0)
+  r.total_mana_regen = r.mana_regen + (r.mana_regen_per_level * 18.0)
+  print '.'
+end
+
+puts "second transform"
+# there's no need to do transforms all in one batch either... you can layer them...
+dh.transform(true) do |r|
+  # this index combines the tank dimensions above for best combination (simple Euclidean metric)
+  r.nuke_index = r.total_damage * r.total_move_speed * r.total_mana * (r.ability_power)
+  r.easy_nuke_index = r.total_damage * r.total_move_speed * r.total_mana * (r.ability_power) * (1.0/r.difficulty)
+  r.tenacious_index = r.total_armor * r.total_health * r.total_spell_block * r.total_health_regen * (r.defense_power)
+  r.support_index = r.total_mana * r.total_armor * r.total_spell_block * r.total_health * r.total_health_regen * r.total_mana_regen * (r.ability_power * r.defense_power)
+  print '.'
+end
+
+# use once at the end to mark records processed.
+dh.transform_complete
+puts "transforms complete"
+
+ds = dh.dataset
+
+binding.pry
+
+puts "done."

data/samples/sample_helper.rb

@@ -0,0 +1,7 @@
+# sample helper takes care of loading the gem from source without requiring it to be rebuilt and installed.
+# this is useful in allowing the samples in this directory to evolve the behavior of the actual gem.
+
+lp = File.expand_path(File.join(*%w[.. lib]), File.dirname(__FILE__))
+unless $LOAD_PATH.include?(lp)
+  $LOAD_PATH.unshift(lp)
+end

data/test/spec/basic_test.rb

@@ -136,5 +136,43 @@ describe DataHut do
 
   end
 
+
+  describe "nice usage" do  
+
+    class Foo
+    end
+
+    it "should provide logging services to see or debug underlying Sequel" do
+      dh = DataHut.connect("foo")
+
+      dh.logger = ::Logger.new(STDOUT)
+
+      assert_raises(ArgumentError) do
+        dh.logger = Foo.new
+      end
+
+    end
+
+    it "should handle type errors" do
+      dh = DataHut.connect("foo")
+
+      data = [{name: "fred", birthday: '1978-02-11'}]
+
+      # how about dates?
+      dh.extract(data) do |r, d|
+        r.name = d[:name]
+        r.birthday = Date.parse(d[:birthday])
+      end
+
+      # ok, but what about a custom type... that's guaranteed to fail!
+      assert_raises(ArgumentError) do
+        dh.transform do |r|
+          r.my_foo = Foo.new
+        end
+      end
+    end
+
+  end
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_hut
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-09 00:00:00.000000000 Z
+date: 2013-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -141,7 +141,9 @@ files:
 - lib/data_hut.rb
 - lib/data_hut/data_warehouse.rb
 - lib/data_hut/version.rb
-- scratch
+- samples/basic.rb
+- samples/league_of_legends.rb
+- samples/sample_helper.rb
 - test/spec/basic_test.rb
 - test/test_helper.rb
 - test/unit/data_warehouse_test.rb

data/scratch

@@ -1,164 +0,0 @@
-#!/usr/bin/env ruby
-# allows this script to behave as if the gem were installed.
-lp = File.expand_path(File.join(*%w[lib]), File.dirname(__FILE__))
-unless $LOAD_PATH.include?(lp)
-  $LOAD_PATH.unshift(lp)
-end
-
-# ------------------ manual example of client usage for docs, etc. -----------
-
-    require 'data_hut'
-    require 'pry'
-
-    dh = DataHut.connect("scratch")
-
-    data = [{name: "barney", age: 27, login: DateTime.parse('2008-05-03') },
-            {name: "phil", age: 31},
-            {name: "fred", age: 44, login: DateTime.parse('2013-02-07')}]
-
-    # extract your data by iterating over your data format (from whatever source) and map it to a record model...
-    dh.extract(data) do |r, d|
-      r.name = d[:name]
-      r.age = d[:age]
-      # data quality step:
-      d[:login] = DateTime.new unless d.has_key?(:login)
-      r.last_active = d[:login]
-      print 'v'
-    end
-
-    # transform your data by adding fields to it
-    dh.transform do |r|
-      r.eligible = r.age < 30
-      print '*'
-    end
-
-    dh.transform_complete
-
-    # later... you have more data...
-    data = [{name: "sarah", age: 24, login: DateTime.parse('2011-04-01') },
-            {name: "robin", age: 45},
-            {name: "jane", age: 19, login: DateTime.parse('2012-10-14')}]
-
-    # add it too...
-    dh.extract(data) do |r, d|
-      r.name = d[:name]
-      r.age = d[:age]
-      # data quality step:
-      d[:login] = DateTime.new unless d.has_key?(:login)
-      r.last_active = d[:login]
-      print 'v'
-    end
-
-    # and only transform the new records automatically
-    dh.transform do |r|
-      r.eligible = r.age < 30
-      print '*'
-    end
-
-    dh.transform_complete
-
-
-    # operate on your dataset by using chained queries
-    ds = dh.dataset
-
-    #binding.pry
-
-
-    # clean up scratch demo
-    FileUtils.rm("scratch.db")
-    puts "done."
-
-#exit
-
-# ------------------------------ a more ambitious example -------------------------
-
-    require 'nokogiri'
-    require 'open-uri'
-    require 'pry'
-
-    root = 'http://na.leagueoflegends.com'
-
-    # load the data once... (manually delete it to refresh)
-    unless File.exists?("lolstats.db")
-      dh = DataHut.connect("lolstats")
-
-      champions_page = Nokogiri::HTML(open("#{root}/champions"))
-
-      urls = champions_page.css('table.champion_item td.description span a').collect{|e| e.attribute('href').value}
-
-      # keep the powers for later since they are on different pages.
-      powers = {}
-      champions_page.css('table.champion_item').each do |c|
-        name        = c.css('td.description span.highlight a').text
-        attack      = c.css('td.graphing td.filled_attack').count
-        health      = c.css('td.graphing td.filled_health').count
-        spells      = c.css('td.graphing td.filled_spells').count
-        difficulty  = c.css('td.graphing td.filled_difficulty').count
-        powers.store(name, {attack_power: attack, defense_power: health, ability_power: spells, difficulty: difficulty})
-      end
-
-      puts "loading champion data"
-      dh.extract(urls) do |r, url|
-        champion_page = Nokogiri::HTML(open("#{root}#{url}"))
-        r.name = champion_page.css('div.page_header_text').text
-
-        st = champion_page.css('table.stats_table')
-        names = st.css('td.stats_name').collect{|e| e.text.strip}
-        values = st.css('td.stats_value').collect{|e| e.text.strip}
-        modifiers = st.css('td.stats_modifier').collect{|e| e.text.strip}
-
-        (0..names.count-1).collect do |i| 
-          stat = (names[i].downcase.gsub(/ /,'_') << "=").to_sym
-          r.send(stat, values[i].to_f)
-          stat_per_level = (names[i].downcase.gsub(/ /,'_') << "_per_level=").to_sym
-          per_level_value = modifiers[i].match(/\+([\d\.]+)/)[1].to_f rescue 0
-          r.send(stat_per_level, per_level_value)
-        end
-
-        # add the powers for this champion...
-        power = powers[r.name]
-        r.attack_power = power[:attack_power]
-        r.defense_power = power[:defense_power]
-        r.ability_power = power[:ability_power]
-        r.difficulty = power[:difficulty]
-
-        print "."
-      end
-      puts "done."
-    end
-
-    dh = DataHut.connect("lolstats")
-
-    puts "first transform"
-    dh.transform do |r|
-      r.total_damage = r.damage + (r.damage_per_level * 18.0)
-      r.total_health = r.health + (r.health_per_level * 18.0)
-      r.total_mana = r.mana + (r.mana_per_level * 18.0)
-      r.total_move_speed = r.move_speed + (r.move_speed_per_level * 18.0)
-      r.total_armor = r.armor + (r.armor_per_level * 18.0)
-      r.total_spell_block = r.spell_block + (r.spell_block_per_level * 18.0)
-      r.total_health_regen = r.health_regen + (r.health_regen_per_level * 18.0)
-      r.total_mana_regen = r.mana_regen + (r.mana_regen_per_level * 18.0)
-      print '.'
-    end
-
-    puts "second transform"
-    # there's no need to do transforms all in one batch either... you can layer them...
-    dh.transform(true) do |r|
-      # this index combines the tank dimensions above for best combination (simple Euclidean metric)
-      r.nuke_index = r.total_damage * r.total_move_speed * r.total_mana * (r.ability_power)
-      r.easy_nuke_index = r.total_damage * r.total_move_speed * r.total_mana * (r.ability_power) * (1.0/r.difficulty)
-      r.tenacious_index = r.total_armor * r.total_health * r.total_spell_block * r.total_health_regen * (r.defense_power)
-      r.support_index = r.total_mana * r.total_armor * r.total_spell_block * r.total_health * r.total_health_regen * r.total_mana_regen * (r.ability_power * r.defense_power)
-      print '.'
-    end
-
-    # use once at the end to mark records processed.
-    dh.transform_complete
-    puts "transforms complete"
-
-    ds = dh.dataset
-
-    binding.pry
-
-puts "done."