linkage 0.1.0.pre → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad6a9ee6a6add94a342e3d02d49d8a4bfeb9122b
-  data.tar.gz: a989e8e810602dfcd4da596fcc1154a922dedccd
+  metadata.gz: 57f4bad92110063c64ed24a43b2a805f4fe6d051
+  data.tar.gz: 9d9ff5fda254dae02bde47dac69c94af56300d51
 SHA512:
-  metadata.gz: 0d78da4904100679826cf76ae07f7daf984c12e2a762aceb0ad2d1387c5a5778403a782ff104ce48c1fd3ec5588c728fe2b8ea7a79a896da153a8d91c7e4cb14
-  data.tar.gz: 6797e5598f47413022d18c6732cbf0613efaccba886cda56eb390e3344d131ccfd977b5992c5a2c4557cfbc4f93bf747ddffe849770be5860e793f27fe3e2286
+  metadata.gz: 63552888a854815988985d54a628e7594d072765027767ea95b159ef80408c64cc2d5ec608892be15c356aecd028529a02df6ca2792827c26aaffa605ab28b65
+  data.tar.gz: 7531bca7bec718605f940557a1572fd3f74528883d08d7137b4c775f95e8b7b4036fe00a6c270f24b30ebf093bb345f97f425f4813aae620f4e69c28b99abde3

data/.yardopts

@@ -1 +1,3 @@
 -m markdown
+--plugin redcarpet-ext
+--private

data/Guardfile

@@ -2,7 +2,6 @@ guard 'test' do
   watch(%r{^lib/linkage/([^/]+/)*([^/]+)\.rb$}) { |m| "test/unit/#{m[1]}test_#{m[2]}.rb" }
   watch(%r{^test/unit/([^/]+/)*test_.+\.rb$})
   watch(%r{^test/integration/test_.+\.rb$})
-  watch('lib/linkage/configuration.rb') { "test/unit/test_dataset.rb" }
   watch('test/helper.rb')  { "test" }
 end
 

data/TODO

@@ -2,3 +2,5 @@ Features
 - add matcher algorithms
 - change comparator_id to more than just an index; need to get comparator ids
   from result set instead of configuration
+- serialize configuration into different formats
+- disallow bad options in result sets

data/lib/linkage.rb

@@ -9,6 +9,7 @@ module Linkage
 end
 
 path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'linkage'
+require path + 'helpers'
 require path + 'comparator'
 require path + 'configuration'
 require path + 'dataset'

data/lib/linkage/comparator.rb

@@ -1,7 +1,7 @@
 module Linkage
   # {Comparator} is the superclass for comparators in Linkage. Comparators are
-  # used to compare two records and compute scores based on how closely the two
-  # records relate.
+  # used to compare records and compute scores based on how closely the records
+  # relate.
   #
   # Each comparator should inherit from {Comparator} and declare itself as
   # simple or advanced by overriding {#type} (the default is simple). Simple
@@ -22,6 +22,16 @@ module Linkage
   class Comparator
     include Observable
 
+    attr_reader :weight
+
+    def weigh(weight)
+      return if weight.nil?
+      if not weight.is_a?(Numeric)
+        raise "weight must be numeric type"
+      end
+      @weight = weight
+    end
+
     class << self
       # Register a new comparator. Subclasses must define at least {#score} for
       # simple comparators, or {#score_dataset} and {#score_datasets} for

data/lib/linkage/comparators/strcompare.rb

@@ -7,6 +7,7 @@ module Linkage
     # the comparison, along with an operator. Valid operators are:
     #
     # * `:jarowinkler` ([Jaro-Winkler distance](http://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance))
+    # * `:damerau_levenshtein` ([Damerau-Levenshtein distance](http://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance))
     #
     # Consider the following example, using a {Configuration} as part of
     # {Dataset#link_with}:
@@ -17,8 +18,11 @@ module Linkage
     #
     # For each record, the values of the `foo` and `bar` fields are compared
     # using the Jaro-Winkler distance algorithm.
+    #
+    # Damerau-Levenshtein is a modified Levenshtein that allows for transpositions
+    # It has additionally been modified to make costs of additions or deletions only 0.5
     class Strcompare < Comparator
-      VALID_OPERATIONS = [:jarowinkler]
+      VALID_OPERATIONS = [:jarowinkler, :reverse_jarowinkler, :damerau_levenshtein]
 
       def initialize(field_1, field_2, operation)
         if field_1.ruby_type[:type] != String || field_2.ruby_type[:type] != String
@@ -38,6 +42,10 @@ module Linkage
           case @operation
           when :jarowinkler
             jarowinkler(record_1[@name_1], record_2[@name_2])
+          when :reverse_jarowinkler
+            reverse_jarowinkler(record_1[@name_1], record_2[@name_2])
+          when :damerau_levenshtein
+            damerau_levenshtein(record_1[@name_1], record_2[@name_2])
           end
 
         result
@@ -50,33 +58,77 @@ module Linkage
         ba = b.split('')
         al = a.length
         bl = b.length
+        return 0 if al == 0 || bl == 0
         l = 0
         for i in Range.new(0, [[al, bl].min, 4].min-1)
           break if aa[i] != ba[i]
           l += 1
         end
-        aj = aa - (aa - ba)
-        bj = ba - (ba - aa)
-        nm = 0
-        nt = 0
-        md = [[al, bl].max/2 - 1, 0].max
+        md = [[al, bl].max/2 - 1, 1].max
+        usea = []
+        useb = []
+        # simplify to matching characters
         for i in Range.new(0, al-1)
-          bi = ba.index(aa[i])
-          aji = aj.index(aa[i])
-          bji = bj.index(aa[i])
-          if !bi.nil? && (bi + nm - i).abs <= md
-            nm += 1
-            nt += 1 if !bji.nil? && aji != bji
+          fi = [[i - md, 0].max, bl-1].min
+          li = [i + md, bl-1].min
+          for j in Range.new(fi, li)
+            if aa[i] == ba[j] and not useb.include?(j)
+              usea << i
+              useb << j
+              break
+            end
           end
-          ba.delete_at(bi) if !bi.nil?
-          aj.delete_at(aji) if !aji.nil?
-          bj.delete_at(bji) if !bji.nil?
         end
+        bada = Range.new(0, al-1).to_a - usea
+        badb = Range.new(0, bl-1).to_a - useb
+        bada.reverse.each { |x| aa.delete_at(x) }
+        badb.reverse.each { |x| ba.delete_at(x) }
+        nm = aa.length
         return 0 if nm == 0
-        d = (nm/al.to_f + nm/bl.to_f + (nm-nt)/nm.to_f)/3.0
+        # count transpositions
+        nt = 0
+        for i in Range.new(0, nm-1)
+          nt +=1 if aa[i] != ba[i]
+        end
+        d = (nm/al.to_f + nm/bl.to_f + (nm-nt/2.0)/nm.to_f)/3.0
         w = (d + l * 0.1 * (1 - d)).round(3)
         w
       end
+
+      def reverse_jarowinkler(w1, w2)
+        jarowinkler(w1.reverse, w2.reverse)
+      end
+
+      def damerau_levenshtein(w1, w2)
+        a = w1.downcase
+        b = w2.downcase
+        aa = a.split('')
+        ba = b.split('')
+        al = a.length
+        bl = b.length
+        denom = [al, bl].max
+        return 0 if denom == 0
+        oneago = nil
+        thisrow = (1..bl).to_a + [0]
+        al.times do |x|
+          twoago, oneago, thisrow = oneago, thisrow, [0] * bl + [x + 1]
+          bl.times do |y|
+            if aa[x] == ba[y]
+              thisrow[y] = oneago[y - 1]
+            else
+              delcost = oneago[y] + 0.5
+              addcost = thisrow[y - 1] + 0.5
+              subcost = oneago[y - 1] + 1
+              thisrow[y] = [delcost, addcost, subcost].min
+              # remove this statement for original levenshtein
+              if x > 0 and y > 0 and aa[x] == ba[y-1] and aa[x-1] == ba[y]
+                thisrow[y] = [thisrow[y], twoago[y-2] + 1].min
+              end
+            end
+          end
+        end
+        return (1 - thisrow[bl - 1] / denom.to_f).round(3)
+      end
     end
 
     Comparator.register('strcompare', Strcompare)

data/lib/linkage/configuration.rb

@@ -1,21 +1,124 @@
 module Linkage
+  # {Configuration} keeps track of everything needed to run a record linkage,
+  # including which datasets you want to link, how you want to link them, and
+  # where you want to store the results. Once created, you can supply the
+  # {Configuration} to {Runner#initialize} and run it with {Runner#execute}.
+  #
+  # To create a configuration, usually you will want to use {Dataset#link_with},
+  # but you can create it directly if you like (see {#initialize}), like so:
+  #
+  # ```ruby
+  # dataset_1 = Linkage::Dataset.new('mysql://example.com/database_name', 'foo')
+  # dataset_2 = Linkage::Dataset.new('postgres://example.com/other_name', 'bar')
+  # result_set = Linkage::ResultSet['csv'].new('/home/foo/linkage')
+  # config = Linkage::Configuration.new(dataset_1, dataset_2, result_set)
+  # ```
+  #
+  # To add comparators to {Configuration}, you can call methods with the same
+  # name as registered comparators. Here's the list of builtin comparators:
+  #
+  # | Name       | Class                     |
+  # |------------|---------------------------|
+  # | compare    | {Comparators::Compare}    |
+  # | strcompare | {Comparators::Strcompare} |
+  # | within     | {Comparators::Within}     |
+  #
+  # For example, if you want to add a {Comparators::Compare} comparator to
+  # your configuration, run this:
+  #
+  # ```ruby
+  # config.compare([:foo], [:bar], :equal_to)
+  # ```
+  #
+  # This works via {Configuration#method_missing}. First, the comparator class
+  # is fetched via {Comparator.[]}. Then fields are looked up in the {FieldSet}
+  # of the {Dataset}. Those {Field}s along with any other arguments you specify
+  # are passed to the constructor of the comparator you chose.
+  #
+  # {Configuration} also contains information about how records are matched.
+  # Once scores are computed, the scores for each pair of records are averaged
+  # and compared against a threshold value. Record pairs that have an average
+  # score greater than or equal to the threshold value are considered matches.
+  #
+  # The threshold value is `0.5` by default, but you can change it by setting
+  # {#threshold} like so:
+  #
+  # ```ruby
+  # config.threshold = 0.75
+  # ```
+  #
+  # Since scores range between 0 and 1 (inclusive), be sure to set a threshold
+  # value within the same range. The actual matching work is done by the
+  # {Matcher} class.
+  #
+  # @see Dataset
+  # @see ResultSet
+  # @see Comparator
+  # @see Matcher
+  # @see Runner
   class Configuration
-    attr_reader :dataset_1, :dataset_2, :result_set, :comparators
-    attr_accessor :record_cache_size, :algorithm, :threshold
+    attr_reader :dataset_1, :dataset_2, :result_set, :comparators, :threshold
+    attr_accessor :algorithm
 
+    def threshold=(threshold)
+      if not threshold.is_a?(Numeric)
+        raise "threshold must be numeric type"
+      end
+      @threshold = threshold
+    end
+    # Create a new instance of {Configuration}.
+    #
+    # @overload initialize(dataset_1, dataset_2, result_set)
+    #   Create a linkage configuration for two datasets and a result set.
+    #   @param [Linkage::Dataset] dataset_1
+    #   @param [Linkage::Dataset] dataset_2
+    #   @param [Linkage::ResultSet] result_set
+    # @overload initialize(dataset, result_set)
+    #   Create a linkage configuration for one dataset and a result set.
+    #   @param [Linkage::Dataset] dataset
+    #   @param [Linkage::ResultSet] result_set
+    # @overload initialize(dataset_1, dataset_2, score_set, match_set)
+    #   Create a linkage configuration for two datasets, a score set, and a
+    #   match set.
+    #   @param [Linkage::Dataset] dataset_1
+    #   @param [Linkage::Dataset] dataset_2
+    #   @param [Linkage::ScoreSet] score_set
+    #   @param [Linkage::MatchSet] match_set
+    # @overload initialize(dataset, score_set, match_set)
+    #   Create a linkage configuration for one dataset, a score set, and a
+    #   match set.
+    #   @param [Linkage::Dataset] dataset
+    #   @param [Linkage::ScoreSet] score_set
+    #   @param [Linkage::MatchSet] match_set
     def initialize(*args)
-      if args.length < 2 || args.length > 3
-        raise ArgumentError, "wrong number of arguments (#{args.length} for 3..4)"
+      if args.length < 2 || args.length > 4
+        raise ArgumentError, "wrong number of arguments (#{args.length} for 2..4)"
       end
 
       @dataset_1 = args[0]
-      if args.length > 2 && args[1]
+      case args.length
+      when 2
+        # dataset and result set
+        @result_set = args[1]
+      when 3
+        # dataset 1, dataset 2, and result set
+        # dataset, score set, and match set
+        case args[1]
+        when Dataset, nil
+          @dataset_2 = args[1]
+          @result_set = args[2]
+        when ScoreSet
+          @result_set = ResultSet.new(args[1], args[2])
+        end
+      when 4
+        # dataset 1, dataset 2, score set, and match set
         @dataset_2 = args[1]
+        @result_set = ResultSet.new(args[2], args[3])
       end
-      @result_set = args[-1]
 
       @comparators = []
-      @record_cache_size = 10_000
+      @algorithm = :mean
+      @threshold = 0.5
     end
 
     def score_recorder
@@ -29,7 +132,7 @@ module Linkage
     end
 
     def matcher
-      Matcher.new(@comparators, @result_set.score_set, @algorithm || :mean, @threshold || 0.5)
+      Matcher.new(@comparators, @result_set.score_set, @algorithm, @threshold)
     end
 
     def match_recorder(matcher)
@@ -60,6 +163,7 @@ module Linkage
 
       comparator = klass.new(*args, &block)
       @comparators << comparator
+      return comparator
     end
 
     protected

data/lib/linkage/dataset.rb

@@ -1,8 +1,111 @@
 module Linkage
-  # Delegator around Sequel::Dataset with some extra functionality.
+  # {Dataset} is a representation of a database table. It is a thin wrapper
+  # around a
+  # {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}.
+  #
+  # There are three ways to create a {Dataset}.
+  #
+  # Pass in a {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}:
+  #
+  # ```ruby
+  # Linkage::Dataset.new(db[:foo])
+  # ```
+  #
+  # Pass in a {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Database.html `Sequel::Database`}
+  # and a table name:
+  #
+  # ```ruby
+  # Linkage::Dataset.new(db, :foo)
+  # ```
+  #
+  # Pass in a
+  # {http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html Sequel-style}
+  # connection URI, a table name, and any options you want to pass to
+  # {http://sequel.jeremyevans.net/rdoc/classes/Sequel.html#method-c-connect `Sequel.connect`}.
+  #
+  # ```ruby
+  # Linkage::Dataset.new("mysql2://example.com/foo", :bar, :user => 'viking', :password => 'secret')
+  # ```
+  #
+  # Once you've made a {Dataset}, you can use any
+  # {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}
+  # method on it you wish. For example, if you want to limit the dataset to
+  # records that refer to people born after 1985 (assuming date of birth is
+  # stored as a date type):
+  #
+  # ```ruby
+  # filtered_dataset = dataset.where('dob > :date', :date => Date.new(1985, 1, 1))
+  # ```
+  #
+  # Note that
+  # {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}
+  # methods return a __clone__ of a dataset, so you must assign the return value
+  # to a variable.
+  #
+  # Once you have your {Dataset} how you want it, you can use the {#link_with}
+  # method to create a {Configuration} for record linkage. The {#link_with}
+  # method takes another {Dataset} object and a {ResultSet} and returns a
+  # {Configuration}.
+  #
+  # ```ruby
+  # config = dataset.link_with(other_dataset, result_set)
+  # config.compare([:foo], [:bar], :equal_to)
+  # ```
+  #
+  # You can pass in a {ScoreSet} and {MatchSet} instead of a {ResultSet} if you
+  # wish:
+  #
+  # ```ruby
+  # config = dataset.link_with(other_dataset, score_set, match_set)
+  # ```
+  #
+  # Note that a dataset can be linked with itself the same way, like so:
+  #
+  # ```ruby
+  # config = dataset.link_with(dataset, result_set)
+  # config.compare([:foo], [:bar], :equal_to)
+  # ```
+  #
+  # If you give {#link_with} a block, it will yield the same {Configuration}
+  # object to the block that it returns.
+  #
+  # ```ruby
+  # config = dataset.link_with(other_dataset, result_set) do |c|
+  #   c.compare([:foo], [:bar], :equal_to)
+  # end
+  # ```
+  #
+  # Once that's done, use a {Runner} to run the record linkage:
+  #
+  # ```ruby
+  # runner = Linkage::Runner.new(config)
+  # runner.execute
+  # ```
+  #
+  # @see http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html Connecting to a database
   class Dataset
-    attr_reader :field_set, :table_name
+    # @return [Symbol] Returns this dataset's table name.
+    attr_reader :table_name
 
+    # @return [FieldSet] Returns this dataset's {FieldSet}.
+    attr_reader :field_set
+
+    # Returns a new instance of {Dataset}.
+    #
+    # @overload initialize(dataset)
+    #   Use a specific {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}.
+    #   @param dataset [Sequel::Dataset]
+    # @overload initialize(database, table_name)
+    #   Use a specific {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Database.html `Sequel::Database`}.
+    #   @param database [Sequel::Database]
+    #   @param table_name [Symbol, String]
+    # @overload initialize(uri, table_name, options = {})
+    #   Use {http://sequel.jeremyevans.net/rdoc/classes/Sequel.html#method-c-connect `Sequel.connect`}
+    #   to connect to a database.
+    #   @param uri [String, Hash]
+    #   @param table_name [Symbol, String]
+    #   @param options [Hash]
+    #
     def initialize(*args)
       if args.length == 0 || args.length > 3
         raise ArgumentError, "wrong number of arguments (#{args.length} for 1..3)"
@@ -31,17 +134,23 @@ module Linkage
       @field_set = FieldSet.new(self)
     end
 
+    # Returns the underlying {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}.
+    # @return [Sequel::Dataset]
     def obj
       @dataset
     end
 
+    # Set the underlying {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}.
     def obj=(value)
       @dataset = value
     end
+    private :obj=
 
-    # Setup a linkage with another dataset
+    # Create a {Configuration} for record linkage.
     #
-    # @return [Linkage::Configuration]
+    # @param dataset [Dataset]
+    # @param result_set [ResultSet]
+    # @return [Configuration]
     def link_with(dataset, result_set)
       other = dataset.eql?(self) ? nil : dataset
       conf = Configuration.new(self, other, result_set)
@@ -51,25 +160,31 @@ module Linkage
       conf
     end
 
-    def database_type
-      @db.database_type
-    end
-
+    # Return the dataset's schema.
+    #
+    # @return [Array]
+    # @see http://sequel.jeremyevans.net/rdoc/classes/Sequel/Database.html#method-i-schema Sequel::Database#schema
     def schema
       @db.schema(@table_name)
     end
 
+    # Returns {FieldSet#primary_key}.
+    #
+    # @return [Field]
+    # @see FieldSet#primary_key
     def primary_key
       @field_set.primary_key
     end
 
     protected
 
+    # Delegate methods to the underlying
+    # {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Dataset.html `Sequel::Dataset`}.
     def method_missing(name, *args, &block)
       result = @dataset.send(name, *args, &block)
       if result.kind_of?(Sequel::Dataset)
         new_object = clone
-        new_object.obj = result
+        new_object.send(:obj=, result)
         new_object
       else
         result