linkage 0.0.1 → 0.0.2

data/lib/linkage/function.rb

@@ -0,0 +1,108 @@
+module Linkage
+  # Abstract class to represent SQL functions. No attempts are made to
+  # ensure that the function actually exists in the database you're using.
+  #
+  # @abstract
+  class Function < Data
+    # Register a new function.
+    #
+    # @param [Class] klass Function class (probably a subclass of {Function})
+    def self.register(klass)
+      if klass.instance_methods(false).any? { |m| m.to_s == "ruby_type" }
+        @functions ||= {}
+        @functions[klass.function_name] = klass
+      else
+        raise ArgumentError, "ruby_type instance method must be defined"
+      end
+    end
+
+    def self.[](name)
+      @functions ? @functions[name] : nil
+    end
+
+    # Subclasses must define this.
+    def self.function_name
+      raise NotImplementedError
+    end
+
+    # Subclasses can define this to require a specific number of arguments
+    # of a certain class. To require two parameters of either String or
+    # Integer, do something like this:
+    #
+    #   @@parameters = [[String, Integer], [String, Integer]]
+    #   def self.parameters
+    #     @@parameters
+    #   end
+    #
+    def self.parameters
+      nil
+    end
+
+    attr_reader :dataset
+
+    # @param [Linkage::Field, Object] args Function arguments
+    def initialize(*args)
+      @names = [self.class.function_name]
+      @args = args
+      @dataset = nil
+      process_args
+    end
+
+    def name
+      @name ||= @names.join("_").to_sym
+    end
+
+    def static?
+      @static
+    end
+
+    # Subclasses must define this. The return value should be a Hash with
+    # the following elements:
+    #   :type - column type (Ruby class) of the result
+    #   :opts - Optional hash with additional options (like :size)
+    def ruby_type
+      raise NotImplementedError
+    end
+
+    # @return [Sequel::SQL::Function]
+    def to_expr
+      self.class.function_name.to_sym.sql_function(*@values)
+    end
+
+    private
+
+    def process_args
+      parameters = self.class.parameters
+      if parameters && parameters.length != @args.length
+        raise ArgumentError, "wrong number of arguments (#{@args.length} for #{parameters.length})"
+      end
+
+      @static = true
+      @values = []
+      @args.each_with_index do |arg, i|
+        if arg.kind_of?(Data)
+          @names << arg.name
+          @static &&= arg.static?
+          if @dataset && !arg.static? && @dataset != arg.dataset
+            raise ArgumentError, "You cannot supply fields from different datasets as arguments to the same function)"
+          end
+          @dataset ||= arg.dataset
+          type = arg.ruby_type[:type]
+          value = arg.is_a?(Field) ? arg.name : arg.to_expr
+        else
+          @names << arg.to_s.gsub(/\W/, "")
+          type = arg.class
+          value = arg
+        end
+        if parameters && !parameters[i].include?(type)
+          raise TypeError, "expected type #{parameters[i].join(" or ")}, got #{type}"
+        end
+        @values << value
+      end
+    end
+  end
+end
+
+Dir.glob(File.expand_path(File.join(File.dirname(__FILE__), "functions", "*.rb"))).each do |filename|
+  require filename
+end

data/lib/linkage/functions/trim.rb

@@ -0,0 +1,22 @@
+module Linkage
+  module Functions
+    class Trim < Function
+      def self.function_name
+        "trim"
+      end
+
+      def self.parameters
+        [[String]]
+      end
+
+      def ruby_type
+        if @args[0].kind_of?(Data)
+          @args[0].ruby_type
+        else
+          {:type => String}
+        end
+      end
+    end
+    Function.register(Trim)
+  end
+end

data/lib/linkage/warnings.rb

@@ -0,0 +1,5 @@
+module Linkage
+  module Warnings
+    # TODO :)
+  end
+end

data/lib/linkage.rb

@@ -6,10 +6,13 @@ end
 
 path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'linkage'
 require path + 'utils'
+require path + 'warnings'
 require path + 'dataset'
 require path + 'runner'
 require path + 'expectation'
+require path + 'data'
 require path + 'field'
+require path + 'function'
 require path + 'group'
 require path + 'import_buffer'
 require path + 'configuration'

data/linkage.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "linkage"
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jeremy Stephens"]
-  s.date = "2011-10-25"
+  s.date = "2011-12-06"
   s.description = "Wraps Sequel to perform record linkage between one or two datasets"
   s.email = "jeremy.f.stephens@vanderbilt.edu"
   s.extra_rdoc_files = [
@@ -28,34 +28,42 @@ Gem::Specification.new do |s|
     "VERSION",
     "lib/linkage.rb",
     "lib/linkage/configuration.rb",
+    "lib/linkage/data.rb",
     "lib/linkage/dataset.rb",
     "lib/linkage/expectation.rb",
     "lib/linkage/field.rb",
+    "lib/linkage/function.rb",
+    "lib/linkage/functions/trim.rb",
     "lib/linkage/group.rb",
     "lib/linkage/import_buffer.rb",
     "lib/linkage/runner.rb",
     "lib/linkage/runner/single_threaded.rb",
     "lib/linkage/utils.rb",
+    "lib/linkage/warnings.rb",
     "linkage.gemspec",
+    "test/config.yml",
     "test/helper.rb",
     "test/integration/test_cross_linkage.rb",
     "test/integration/test_dual_linkage.rb",
     "test/integration/test_self_linkage.rb",
+    "test/unit/functions/test_trim.rb",
+    "test/unit/runner/test_single_threaded.rb",
     "test/unit/test_configuration.rb",
+    "test/unit/test_data.rb",
     "test/unit/test_dataset.rb",
     "test/unit/test_expectation.rb",
     "test/unit/test_field.rb",
+    "test/unit/test_function.rb",
     "test/unit/test_group.rb",
     "test/unit/test_import_buffer.rb",
     "test/unit/test_linkage.rb",
     "test/unit/test_runner.rb",
-    "test/unit/test_single_threaded_runner.rb",
     "test/unit/test_utils.rb"
   ]
   s.homepage = "http://github.com/coupler/linkage"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.10"
+  s.rubygems_version = "1.8.11"
   s.summary = "Sequel-based record linkage"
 
   if s.respond_to? :specification_version then
@@ -74,6 +82,10 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<rake>, [">= 0"])
       s.add_development_dependency(%q<versionomy>, [">= 0"])
       s.add_development_dependency(%q<guard-yard>, [">= 0"])
+      s.add_development_dependency(%q<rb-inotify>, [">= 0"])
+      s.add_development_dependency(%q<mysql2>, [">= 0"])
+      s.add_development_dependency(%q<pry>, [">= 0"])
+      s.add_development_dependency(%q<rdiscount>, [">= 0"])
     else
       s.add_dependency(%q<sequel>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
@@ -87,6 +99,10 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rake>, [">= 0"])
       s.add_dependency(%q<versionomy>, [">= 0"])
       s.add_dependency(%q<guard-yard>, [">= 0"])
+      s.add_dependency(%q<rb-inotify>, [">= 0"])
+      s.add_dependency(%q<mysql2>, [">= 0"])
+      s.add_dependency(%q<pry>, [">= 0"])
+      s.add_dependency(%q<rdiscount>, [">= 0"])
     end
   else
     s.add_dependency(%q<sequel>, [">= 0"])
@@ -101,6 +117,10 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rake>, [">= 0"])
     s.add_dependency(%q<versionomy>, [">= 0"])
     s.add_dependency(%q<guard-yard>, [">= 0"])
+    s.add_dependency(%q<rb-inotify>, [">= 0"])
+    s.add_dependency(%q<mysql2>, [">= 0"])
+    s.add_dependency(%q<pry>, [">= 0"])
+    s.add_dependency(%q<rdiscount>, [">= 0"])
   end
 end
 

data/test/config.yml

@@ -0,0 +1,5 @@
+mysql:
+  host: localhost
+  port: 3306
+  database: test
+  user: test

data/test/helper.rb

@@ -13,6 +13,7 @@ require 'tmpdir'
 require 'logger'
 require 'pp'
 require 'versionomy'
+require 'pry'
 
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
@@ -21,15 +22,35 @@ require 'linkage'
 class Test::Unit::TestCase
   def stub_field(name, options = {}, &block)
     f = Linkage::Field.allocate
+    f.stubs({:static? => false}.merge(options))
+    if block
+      f.send(:instance_eval, &block)
+    end
+    f
+  end
+
+  def stub_function(name, options = {}, &block)
+    f = Linkage::Function.allocate
     f.stubs(options)
     if block
       f.send(:instance_eval, &block)
     end
-    f.stubs(:is_a?).returns(false)
-    f.stubs(:is_a?).with(Linkage::Field).returns(true)
     f
   end
 
+  def new_function(name, ruby_type = nil, params = nil, &block)
+    klass = Class.new(Linkage::Function)
+    klass.send(:define_singleton_method, :function_name) { name }
+    if ruby_type
+      klass.send(:define_method, :ruby_type) { ruby_type }
+    end
+    if params
+      klass.send(:define_singleton_method, :parameters) { params }
+    end
+    klass
+  end
+
+
   def self.current_ruby_version
     @current_ruby_version ||= Versionomy.parse(RUBY_VERSION)
   end
@@ -37,6 +58,10 @@ class Test::Unit::TestCase
   def self.ruby19
     @ruby19 ||= Versionomy.parse("1.9")
   end
+
+  def test_config
+    @test_config ||= YAML.load_file(File.join(File.dirname(__FILE__), "config.yml"))
+  end
 end
 
 module UnitTests; end

data/test/integration/test_dual_linkage.rb

@@ -81,5 +81,38 @@ module IntegrationTests
         end
       end
     end
+
+    test "handles MySQL's ignorance of trailing spaces when comparing strings" do
+      pend
+      if !test_config['mysql']
+        omission("No MySQL test configuration found")
+      end
+      uri = "mysql2://%s:%s/%s?user=%s" % test_config['mysql'].values_at('host', 'port', 'database', 'user')
+      Sequel.connect(uri) do |db|
+        db.create_table!(:foo) { primary_key(:id); String(:one); String(:two) }
+        db[:foo].import([:id, :one, :two], [[1, "", "test"], [2, "", "test"], [3, " ", "test "], [4, "", "test"], [5, "", "junk"]])
+
+        db.create_table!(:bar) { primary_key(:id); String(:one); String(:two) }
+        db[:bar].import([:id, :one, :two], [[1, "", "junk"]])
+
+        db.run("DROP TABLE IF EXISTS groups")
+        db.run("DROP TABLE IF EXISTS groups_records")
+      end
+
+      ds_1 = Linkage::Dataset.new(uri, "foo", :single_threaded => true)
+      ds_2 = Linkage::Dataset.new(uri, "bar", :single_threaded => true)
+      conf = ds_1.link_with(ds_2) do
+        lhs[:one].must == rhs[:one]
+        lhs[:two].must == rhs[:two]
+      end
+
+      logger = Logger.new(STDERR)
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri, :logger => logger)
+      runner.execute
+
+      Sequel.connect(@tmpuri) do |db|
+        assert_equal 1, db[:groups].count
+      end
+    end
   end
 end

data/test/integration/test_self_linkage.rb

@@ -205,5 +205,26 @@ module IntegrationTests
         end
       end
     end
+
+    test "match functions" do
+      # insert the test data
+      database do |db|
+        db.create_table(:foo) { primary_key(:id); String(:bar) }
+        db[:foo].import([:id, :bar],
+          Array.new(100) { |i| [i, "bar%s" % (" " * (i % 10))] })
+      end
+
+      ds = Linkage::Dataset.new(@tmpuri, "foo", :single_threaded => true)
+      conf = ds.link_with(ds) do
+        trim(lhs[:bar]).must == trim(rhs[:bar])
+      end
+      assert_equal :self, conf.linkage_type
+      runner = Linkage::SingleThreadedRunner.new(conf, @tmpuri)
+      runner.execute
+
+      database do |db|
+        assert_equal 1, db[:groups].count
+      end
+    end
   end
 end

data/test/unit/functions/test_trim.rb

@@ -0,0 +1,36 @@
+require 'helper'
+
+class UnitTests::TestTrim < Test::Unit::TestCase
+  test "subclass of Function" do
+    assert_equal Linkage::Function, Linkage::Functions::Trim.superclass
+  end
+
+  test "ruby_type for string literal" do
+    assert_equal({:type => String}, Linkage::Functions::Trim.new("foo").ruby_type)
+  end
+
+  test "ruby_type for string field" do
+    field_1 = stub_field('field 1', :name => :bar, :ruby_type => {:type => String})
+    assert_equal({:type => String}, Linkage::Functions::Trim.new(field_1).ruby_type)
+
+    field_2 = stub_field('field 2', :name => :bar, :ruby_type => {:type => String, :opts => {:size => 123}})
+    assert_equal({:type => String, :opts => {:size => 123}}, Linkage::Functions::Trim.new(field_2).ruby_type)
+  end
+
+  test "ruby_type for string function" do
+    func = new_function('foo', {:type => String, :opts => {:junk => '123'}})
+    assert_equal({:type => String, :opts => {:junk => '123'}}, Linkage::Functions::Trim.new(func.new).ruby_type)
+  end
+
+  test "parameters" do
+    assert_equal [[String]], Linkage::Functions::Trim.parameters
+  end
+
+  test "name" do
+    assert_equal "trim", Linkage::Functions::Trim.function_name
+  end
+
+  test "registers itself" do
+    assert_equal Linkage::Function["trim"], Linkage::Functions::Trim
+  end
+end

data/test/unit/test_configuration.rb

@@ -23,7 +23,7 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
   end
 
   test "linkage_type is cross when there's different filters on both sides" do
-    field = stub('field')
+    field = stub_field('field')
     dataset = stub('dataset', :set_new_id => nil)
     dataset.stubs(:fields).returns({:foo => field})
     c = Linkage::Configuration.new(dataset, dataset)
@@ -39,7 +39,7 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
   end
 
   test "linkage_type is self when there's identical static filters on each side" do
-    field = stub('field')
+    field = stub_field('field')
     dataset = stub('dataset', :set_new_id => nil)
     dataset.stubs(:fields).returns({:foo => field})
     c = Linkage::Configuration.new(dataset, dataset)
@@ -53,8 +53,8 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
   end
 
   test "linkage_type is self when there's a two-field filter on one side" do
-    field_1 = stub('field 1')
-    field_2 = stub('field 2')
+    field_1 = stub_field('field 1')
+    field_2 = stub_field('field 2')
     dataset = stub('dataset', :set_new_id => nil)
     dataset.stubs(:fields).returns({:foo => field_1, :bar => field_2})
     c = Linkage::Configuration.new(dataset, dataset)
@@ -71,7 +71,7 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
 
   test "static expectation" do
     dataset_1 = stub('dataset')
-    field = stub('field')
+    field = stub_field('field')
     dataset_1.stubs(:fields).returns({:foo => field})
     dataset_2 = stub('dataset')
     c = Linkage::Configuration.new(dataset_1, dataset_2)
@@ -84,7 +84,7 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
   ## Maybe in the future
   #test "static expectation, flopped" do
     #dataset_1 = stub('dataset')
-    #field = stub('field')
+    #field = stub_field('field')
     #dataset_1.stubs(:fields).returns({:foo => field})
     #dataset_2 = stub('dataset')
     #c = Linkage::Configuration.new(dataset_1, dataset_2)
@@ -142,4 +142,40 @@ class UnitTests::TestConfiguration < Test::Unit::TestCase
       lhs[:foo].must_not == 123
     end
   end
+
+  test "dynamic database function" do
+    dataset_1 = stub('dataset')
+    field_1 = stub_field('field 1')
+    dataset_1.stubs(:fields).returns({:foo => field_1})
+    dataset_2 = stub('dataset')
+    field_2 = stub_field('field 2')
+    dataset_2.stubs(:fields).returns({:foo => field_2})
+
+    func = stub_function('function', :static? => false)
+    Linkage::Functions::Trim.expects(:new).with(field_1).returns(func)
+
+    c = Linkage::Configuration.new(dataset_1, dataset_2)
+    Linkage::MustExpectation.expects(:new).with(:==, func, field_2, nil)
+    c.send(:instance_eval) do
+      trim(lhs[:foo]).must == rhs[:foo]
+    end
+  end
+
+  test "static database function" do
+    dataset_1 = stub('dataset')
+    field_1 = stub_field('field 1')
+    dataset_1.stubs(:fields).returns({:foo => field_1})
+    dataset_2 = stub('dataset')
+    field_2 = stub_field('field 2')
+    dataset_2.stubs(:fields).returns({:foo => field_2})
+
+    func = stub_function('function', :static? => true)
+    Linkage::Functions::Trim.expects(:new).with("foo").returns(func)
+
+    c = Linkage::Configuration.new(dataset_1, dataset_2)
+    Linkage::MustExpectation.expects(:new).with(:==, field_1, func, :filter)
+    c.send(:instance_eval) do
+      lhs[:foo].must == trim("foo")
+    end
+  end
 end