fuzzy_infer 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,10 @@
+source :rubygems
+
+gemspec
+
+# development dependencies
+gem 'minitest'
+gem 'minitest-reporters'
+gem 'mysql2'
+gem 'earth'
+gem 'pg'

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Seamus Abshere
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,48 @@
+# FuzzyInfer
+
+## Where it's used
+
+* [Brighter Planet CM1 Impact Estimate web service](http://impact.brighterplanet.com) 
+
+## setup
+
+1) gem install a bleeding edge earth
+
+    cd earth
+    git pull
+    gem build earth.gemspec
+    gem install earth-0.11.10.gem --ignore-dependencies --no-rdoc --no-ri
+
+2) create your test database
+
+    mysql -u root -ppassword -e "create database test_fuzzy_infer charset utf8"
+
+3) load cbecs (just the first time - note that it is hardcoded to ONLY run cbecs data_miner)
+
+    RUN_DATA_MINER=true rake
+
+## further testing
+
+    rake
+
+## future plans
+
+**in the future the fuzzy inference machine will make TEMPORARY tables, rather than gum up your db**
+
+for now, it makes permanent tables so that you can examine them
+
+wishlist:
+
+* re-use FIM for multiple targets
+* cache #fuzzy_infer
+* randomize names of all added columns
+* use arel to generate sql(?)
+
+## Database compatibility
+
+* mysql
+* postgresql
+
+## Copyright
+
+Copyright 2012 Brighter Planet, Inc.

data/Rakefile

@@ -0,0 +1,12 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake'
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fuzzy_infer.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/fuzzy_infer/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Seamus Abshere", "Ian Hough", "Matt Kling"]
+  gem.email         = ["seamus@abshere.net", 'ijhough@gmail.com', 'mattkling@gmail.com']
+  desc = %q{Use fuzzy set analysis to infer missing values. You provide a sigma function, a membership function, and a kernel.}
+  gem.description   = desc
+  gem.summary       = desc
+  gem.homepage      = "https://github.com/seamusabshere/fuzzy_infer"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "fuzzy_infer"
+  gem.require_paths = ["lib"]
+  gem.version       = FuzzyInfer::VERSION
+  
+  gem.add_runtime_dependency 'activesupport', '>=3'
+  gem.add_runtime_dependency 'activerecord', '>=3'
+  gem.add_runtime_dependency 'hashie'
+end

data/lib/fuzzy_infer.rb

@@ -0,0 +1,14 @@
+require 'active_support/core_ext'
+
+require "fuzzy_infer/version"
+require 'fuzzy_infer/registry'
+require 'fuzzy_infer/active_record_class_methods'
+require 'fuzzy_infer/active_record_instance_methods'
+require 'fuzzy_infer/fuzzy_inference_machine'
+
+module FuzzyInfer
+  # Your code goes here...
+end
+
+ActiveRecord::Base.send :include, FuzzyInfer::ActiveRecordInstanceMethods
+ActiveRecord::Base.extend FuzzyInfer::ActiveRecordClassMethods

data/lib/fuzzy_infer/active_record_class_methods.rb

@@ -0,0 +1,15 @@
+require 'hashie/mash'
+
+module FuzzyInfer
+  module ActiveRecordClassMethods
+    # Configure fuzzy inferences
+    # see test/helper.rb for an example
+    def fuzzy_infer(options = {})
+      options = ::Hashie::Mash.new options
+      options.target.each do |target|
+        Registry.instance[name] ||= {}
+        Registry.instance[name][target] = options
+      end
+    end
+  end
+end

data/lib/fuzzy_infer/active_record_instance_methods.rb

@@ -0,0 +1,14 @@
+module FuzzyInfer
+  module ActiveRecordInstanceMethods
+    # Returns a new FuzzyInferenceMachine instance that can infer this target (field)
+    def fuzzy_inference_machine(target)
+      target = target.to_sym
+      FuzzyInferenceMachine.new self, target, Registry.config_for(self.class.name, target)
+    end
+    
+    # Shortcut to creating a FIM and immediately calling it
+    def fuzzy_infer(target)
+      fuzzy_inference_machine(target).infer
+    end
+  end
+end

data/lib/fuzzy_infer/fuzzy_inference_machine.rb

@@ -0,0 +1,126 @@
+module FuzzyInfer
+  class FuzzyInferenceMachine
+    
+    attr_reader :kernel
+    attr_reader :target
+    attr_reader :config
+
+    def initialize(kernel, target, config)
+      @kernel = kernel
+      @target = target
+      @config = config
+    end
+    
+    def infer
+      calculate_table!
+      retval = select_value(%{SELECT SUM(fuzzy_weighted_value)/SUM(fuzzy_membership) FROM #{table_name}}).to_f
+      execute %{DROP TABLE #{table_name}}
+      retval
+    end
+    
+    # TODO technically I could use this to generate the SQL
+    def arel_table
+      calculate_table!
+      Arel::Table.new table_name
+    end
+    
+    def basis
+      @basis ||= kernel.attributes.symbolize_keys.slice(*config.basis).reject { |k, v| v.nil? }
+    end
+    
+    def sigma
+      @sigma ||= basis.inject({}) do |memo, (k, v)|
+        memo[k] = select_value(%{SELECT #{sigma_sql(k, v)} FROM #{kernel.class.quoted_table_name} WHERE #{target_not_null_sql} AND #{basis_not_null_sql}}).to_f
+        memo
+      end
+    end
+    
+    def membership
+      return @membership if @membership
+      sql = kernel.send(config.membership, basis).dup
+      basis.keys.each do |k|
+        sql.gsub! ":#{k}_n_w", quote_column_name("#{k}_n_w")
+      end
+      @membership = sql
+    end
+    
+    # In case you want to `cache_method :infer` with https://github.com/seamusabshere/cache_method
+    def method_cache_hash
+      [kernel.class.name, basis, target, config].hash
+    end
+    
+    private
+    
+    def calculate_table!
+      return if table_exists?(table_name)
+      execute %{CREATE TEMPORARY TABLE #{table_name} AS SELECT * FROM #{kernel.class.quoted_table_name} WHERE #{target_not_null_sql} AND #{basis_not_null_sql}}
+      execute %{ALTER TABLE #{table_name} #{weight_create_columns_sql}}
+      execute %{ALTER TABLE #{table_name} ADD COLUMN fuzzy_membership FLOAT default null}
+      execute %{ALTER TABLE #{table_name} ADD COLUMN fuzzy_weighted_value FLOAT default null}
+      execute %{UPDATE #{table_name} SET #{weight_calculate_sql}}
+      weight_normalize_frags.each do |sql|
+        execute sql
+      end
+      execute %{UPDATE #{table_name} SET fuzzy_membership = #{membership_sql}}
+      execute %{UPDATE #{table_name} SET fuzzy_weighted_value = fuzzy_membership * #{quote_column_name(target)}}
+      nil
+    end
+    
+    def membership_sql
+      if config.weight
+        "(#{membership}) * #{quote_column_name(config.weight.to_s)}"
+      else
+        membership
+      end
+    end
+    
+    def weight_normalize_frags
+      basis.keys.map do |k|
+        max = select_value("SELECT MAX(#{quote_column_name("#{k}_w")}) FROM #{table_name}").to_f
+        "UPDATE #{table_name} SET #{quote_column_name("#{k}_n_w")} = #{quote_column_name("#{k}_w")} / #{max}"
+      end
+    end
+    
+    def weight_calculate_sql
+      basis.keys.map do |k|
+        "#{quote_column_name("#{k}_w")} = 1.0 / (#{sigma[k]}*SQRT(2*PI())) * EXP(-(POW(#{quote_column_name(k)} - #{basis[k]},2))/(2*POW(#{sigma[k]},2)))"
+      end.join(', ')
+    end
+    
+    def sigma_sql(column_name, value)
+      sql = config.sigma.dup
+      sql.gsub! ':column', quote_column_name(column_name)
+      sql.gsub! ':value', value.to_f.to_s
+      sql
+    end
+    
+    def table_name
+      @table_name ||= "fuzzy_infer_#{Time.now.strftime('%Y_%m_%d_%H_%M_%S')}_#{Kernel.rand(1e11)}"
+    end
+    
+    def weight_create_columns_sql
+      basis.keys.inject([]) do |memo, k|
+        memo << "ADD COLUMN #{quote_column_name("#{k}_w")} FLOAT default null"
+        memo << "ADD COLUMN #{quote_column_name("#{k}_n_w")} FLOAT default null"
+        memo
+      end.flatten.join ', '
+    end
+    
+    def basis_not_null_sql
+      basis.keys.map do |basis|
+        "#{quote_column_name(basis)} IS NOT NULL"
+      end.join ' AND '
+    end
+    
+    def target_not_null_sql
+      "#{quote_column_name(target)} IS NOT NULL"
+    end
+    
+    def connection
+      kernel.connection
+    end
+    
+    delegate :execute, :quote_column_name, :select_value, :table_exists?, :to => :connection
+  end
+end
+

data/lib/fuzzy_infer/registry.rb

@@ -0,0 +1,15 @@
+require 'singleton'
+
+module FuzzyInfer
+  class Registry < ::Hash
+    class << self
+      def config_for(class_name, target)
+        raise %{[fuzzy_infer] Zero machines are defined on #{class_name}.} unless instance.has_key?(class_name)
+        raise %{[fuzzy_infer] Target #{target.inspect} is not available on #{class_name}.} unless instance[class_name].has_key?(target)
+        instance[class_name][target]
+      end
+    end
+    
+    include ::Singleton
+  end
+end

data/lib/fuzzy_infer/version.rb

@@ -0,0 +1,3 @@
+module FuzzyInfer
+  VERSION = "0.0.1"
+end

data/test/helper.rb

@@ -0,0 +1,83 @@
+require 'bundler/setup'
+
+require 'active_record'
+case ENV['DB_ADAPTER']
+when 'postgresql'
+  adapter = 'postgresql'
+  username = ENV['POSTGRES_USERNAME'] || `whoami`.chomp
+  password = ENV['POSTGRES_PASSWORD']
+  database = ENV['POSTGRES_DATABASE'] || 'test_fuzzy_infer'
+else
+  adapter = 'mysql2'
+  database = 'test_fuzzy_infer'
+  username = 'root'
+  password = 'password'
+end
+config = {
+  'encoding' => 'utf8',
+  'adapter' => adapter,
+  'database' => database,
+}
+config['username'] = username if username
+config['password'] = password if password
+ActiveRecord::Base.establish_connection config
+require 'logger'
+ActiveRecord::Base.logger = Logger.new $stderr
+ActiveRecord::Base.logger.level = Logger::DEBUG
+
+require 'earth'
+if ENV['RUN_DATA_MINER'] == 'true'
+  Earth.init :hospitality, :load_data_miner => true
+  ActiveRecord::Base.logger.level = Logger::INFO
+  CommercialBuildingEnergyConsumptionSurveyResponse.run_data_miner!
+  $stderr.puts "Done!"
+  exit
+end
+
+Earth.init :hospitality
+
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'minitest/reporters'
+MiniTest::Unit.runner = MiniTest::SuiteRunner.new
+MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'fuzzy_infer'
+# class MiniTest::Spec
+# end
+
+CBECS = CommercialBuildingEnergyConsumptionSurveyResponse
+class CBECS < ActiveRecord::Base
+  fuzzy_infer :target     => [:electricity_per_room_night, :natural_gas_per_room_night,
+                              :fuel_oil_per_room_night, :district_heat_per_room_night], # list of columns that this model is designed to infer
+              :basis      => [:lodging_rooms, :construction_year, :heating_degree_days,
+                              :cooling_degree_days, :floors, :ac_coverage],             # list of columns that are believed to affect energy use (aka MU)
+              :sigma      => "(STDDEV_SAMP(:column)/5)+(ABS(AVG(:column)-:value)/3)",   # empirically determined formula (SQL!) that captures the desired sample size once all the weights are compiled, across the full range of possible mu values
+              :membership => :energy_use_membership,                                    # name of instance method to be called on kernel
+              :weight     => :weighting                                                 # (optional) a pre-existing row weighting, if any, provided by the dataset authors
+  
+  # empirically determined formula that minimizes variance between real and predicted energy use
+  # SQL! - :heating_degree_days_n_w will be replaced with, for example, `tmp_table_9301293.hdd_normalized_weight`
+  def energy_use_membership(basis)
+    keys = basis.keys
+    
+    formula = "(POW(:heating_degree_days_n_w, 0.8) + POW(:cooling_degree_days_n_w, 0.8))"
+    
+    formula += if keys.include? :lodging_rooms and keys.include? :floors
+      " * (POW(:lodging_rooms_n_w, 0.8) + POW(:floors_n_w, 0.8))"
+    elsif keys.include? :lodging_rooms
+      " * POW(:lodging_rooms_n_w, 0.8)"
+    elsif keys.include? :floors
+      " * POW(:floors_n_w, 0.8)"
+    else
+      ""
+    end
+    
+    formula += " * POW(:percent_cooled_n_w, 0.8)" if keys.include? :percent_cooled
+    formula += " * POW(:construction_year_n_w, 0.8)" if keys.include? :construction_year
+    
+    formula
+  end
+end

data/test/test_fuzzy_infer.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+require 'helper'
+
+describe FuzzyInfer do
+  describe FuzzyInfer::ActiveRecordClassMethods do
+    it 'adds a way to configure a FuzzyInferenceMachine' do
+      CBECS.respond_to?(:fuzzy_infer).must_equal true
+    end
+  end
+  
+  describe FuzzyInfer::ActiveRecordInstanceMethods do
+    it 'adds a way to infer a particular target (field)' do
+      CBECS.new.respond_to?(:fuzzy_infer).must_equal true
+    end
+    it 'adds a way to get a FIM object' do
+      CBECS.new.respond_to?(:fuzzy_inference_machine).must_equal true
+    end
+    it "creates a new FIM object" do
+      e = CBECS.new.fuzzy_inference_machine(:electricity_per_room_night)
+      e.must_be_instance_of FuzzyInfer::FuzzyInferenceMachine
+    end
+  end
+  # CBECS.new(:heating_degree_days => 2778, :lodging_rooms => 20).fuzzy_infer(:electricity_per_room_night)
+  
+  describe FuzzyInfer::FuzzyInferenceMachine do
+    before do
+      @kernel = CBECS.new(:heating_degree_days => 2778, :cooling_degree_days => 400, :lodging_rooms => 20, :principal_activity => 'Partying')
+      @e = @kernel.fuzzy_inference_machine(:electricity_per_room_night)
+    end
+    describe '#basis' do
+      it "is the union of the kernel's attributes with the basis" do
+        @e.basis.must_equal :lodging_rooms => 20, :heating_degree_days => 2778.0, :cooling_degree_days => 400.0
+      end
+    end
+    describe "the temp table" do
+      it "excludes rows from the original table where basis or target is nil, but includes rows where they are 0" do
+        ActiveRecord::Base.connection.select_value(@e.arel_table.project('COUNT(*)').to_sql).to_f.must_equal 192
+      end
+    end
+    describe '#sigma' do
+      it "is calculated from the original table, but only those rows that are also in the temp table" do
+        @e.sigma[:heating_degree_days].must_be_close_to 411.9, 0.1
+        @e.sigma[:cooling_degree_days].must_be_close_to 267.6, 0.1
+        @e.sigma[:lodging_rooms].must_be_close_to 55.0, 0.1
+      end
+    end
+    describe '#membership' do
+      it 'depends on the kernel' do
+        @e.membership.must_match %r{\(POW\(.heating_degree_days_n_w.,\ 0\.8\)\ \+\ POW\(.cooling_degree_days_n_w.,\ 0\.8\)\)\ \*\ POW\(.lodging_rooms_n_w.,\ 0\.8\)}
+      end
+    end
+    describe '#infer' do
+      it 'guesses!' do
+        @e.infer.must_be_close_to 17.75, 0.01
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: fuzzy_infer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Seamus Abshere
+- Ian Hough
+- Matt Kling
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: &2153031100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153031100
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: &2153029920 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153029920
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: &2153029260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153029260
+description: Use fuzzy set analysis to infer missing values. You provide a sigma function,
+  a membership function, and a kernel.
+email:
+- seamus@abshere.net
+- ijhough@gmail.com
+- mattkling@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.markdown
+- Rakefile
+- fuzzy_infer.gemspec
+- lib/fuzzy_infer.rb
+- lib/fuzzy_infer/active_record_class_methods.rb
+- lib/fuzzy_infer/active_record_instance_methods.rb
+- lib/fuzzy_infer/fuzzy_inference_machine.rb
+- lib/fuzzy_infer/registry.rb
+- lib/fuzzy_infer/version.rb
+- test/helper.rb
+- test/test_fuzzy_infer.rb
+homepage: https://github.com/seamusabshere/fuzzy_infer
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Use fuzzy set analysis to infer missing values. You provide a sigma function,
+  a membership function, and a kernel.
+test_files:
+- test/helper.rb
+- test/test_fuzzy_infer.rb
+has_rdoc: