linkage 0.0.8 → 0.1.0.pre

data/lib/linkage/comparators/compare.rb

@@ -1,46 +1,253 @@
 module Linkage
   module Comparators
-    class Compare < Binary
-      @@parameters = [
-        [:any, :static => false, :side => :first],
-        [String, :values => %w{> >= <= < !=}],
-        [:any, :same_type_as => 0, :static => false, :side => :second]
+    # Compare is the most basic comparator in Linkage, conceptually. It scores
+    # two records based on whether or not field values satisfy the specified
+    # operator. Score is either 0 or 1.
+    #
+    # To use Compare, you must specify two sets of fields to use in the
+    # comparison, along with an operator. Valid operators are:
+    #
+    # * `:equal`
+    # * `:not_equal`
+    # * `:greater_than`
+    # * `:greater_than_or_equal`
+    # * `:less_than`
+    # * `:less_than_or_equal`
+    #
+    # Sets of fields must be of equal length. If you specify more than one
+    # field, each field will be compared to its counterpart in the other set.
+    # All of the field values must meet the conditions in order for the score to
+    # be 1. Otherwise, the score is 0.
+    #
+    # Consider the following example, using a {Configuration} as part of
+    # {Dataset#link_with}:
+    #
+    # ```ruby
+    # config.compare([:foo, :bar], [:baz, :qux], :equal)
+    # ```
+    #
+    # For each record, the values of `foo` and `baz` are compared, and the
+    # values of `bar` and `qux` are compared. If both of these two comparisons
+    # are `true`, then the score of 1 is given. If `foo` and `baz` are equal but
+    # `bar` and `qux` are not equal, or if both comparisons are false, then a
+    # score of 0 is given.
+    #
+    # Algorithms
+    # ----------
+    #
+    # The way records are chosen for comparison depends on which operator you
+    # use. The `:equal` operator is treated differently than the other
+    # operators. When using operators other than `:equal`, each record is
+    # compared to every other record (and {#type} returns `:simple`). When using
+    # `:equal`, {#type} is `:advanced` and a different algorithm is used.
+    #
+    # "Equal" mode uses an algorithm similar to the sorted neighborhood method.
+    # Values are sorted (via database query) and then compared. This way, only
+    # adjacent records are compared. Using the transitive property of equality,
+    # records are grouped together. All pairs of records in the group are scored
+    # as 1. Scores of 0 are not given at all (absence of score means 0).
+    class Compare < Comparator
+      VALID_OPERATIONS = [
+        :not_equal, :greater_than, :greater_than_or_equal,
+        :less_than_or_equal, :less_than, :equal
       ]
-      def self.parameters
-        @@parameters
-      end
 
-      @@comparator_name = 'compare'
-      def self.comparator_name
-        @@comparator_name
-      end
+      def initialize(set_1, set_2, operation)
+        if set_1.length != set_2.length
+          raise "sets must be of equal length"
+        end
+
+        # Check value data types
+        set_1.each_with_index do |value_1, index|
+          value_2 = set_2[index]
+          if value_1.ruby_type != value_2.ruby_type
+            raise "values at index #{index} had different types"
+          end
+        end
 
-      def initialize(*args)
-        super
-        @name_1 = @args[0].name
-        @operator = @args[1].object
-        @name_2 = @args[2].name
+        # Check compare operator
+        if !VALID_OPERATIONS.include?(operation)
+          raise "operation is not valid"
+        end
+        @type = operation == :equal ? :advanced : :simple
+        @names_1 = set_1.collect(&:name)
+        @names_2 = set_2.collect(&:name)
+        @operation = operation
       end
 
       def score(record_1, record_2)
+        values_1 = record_1.values_at(*@names_1)
+        values_2 = record_2.values_at(*@names_2)
         result =
-          case @operator
-          when '!='
-            record_1[@name_1] != record_2[@name_2]
-          when '>'
-            record_1[@name_1] > record_2[@name_2]
-          when '>='
-            record_1[@name_1] >= record_2[@name_2]
-          when '<='
-            record_1[@name_1] <= record_2[@name_2]
-          when '<'
-            record_1[@name_1] < record_2[@name_2]
+          case @operation
+          when :not_equal
+            values_1.each_with_index.all? do |value_1, i|
+              value_1 != values_2[i]
+            end
+          when :greater_than
+            values_1.each_with_index.all? do |value_1, i|
+              value_1 > values_2[i]
+            end
+          when :greater_than_or_equal
+            values_1.each_with_index.all? do |value_1, i|
+              value_1 >= values_2[i]
+            end
+          when :less_than_or_equal
+            values_1.each_with_index.all? do |value_1, i|
+              value_1 <= values_2[i]
+            end
+          when :less_than
+            values_1.each_with_index.all? do |value_1, i|
+              value_1 < values_2[i]
+            end
           end
 
         result ? 1 : 0
       end
+
+      def score_datasets(dataset_1, dataset_2)
+        # FIXME: nil value equality
+
+        _score_datasets(dataset_1, dataset_2)
+      end
+
+      def score_dataset(dataset)
+        # FIXME: nil value equality
+
+        if @names_1 != @names_2
+          return _score_datasets(dataset, dataset)
+        end
+
+        enum = dataset.order(*@names_1).to_enum
+        begin
+          record = enum.next
+        rescue StopIteration
+          return
+        end
+        group = [record]
+        last_value = record.values_at(*@names_1)
+        loop do
+          begin
+            record = enum.next
+          rescue StopIteration
+            break
+          end
+          value = record.values_at(*@names_1)
+          if value == last_value
+            group << record
+          else
+            score_group(group)
+            group.clear
+            group << record
+            last_value = value
+          end
+        end
+        score_group(group)
+      end
+
+      private
+
+      def _score_datasets(dataset_1, dataset_2)
+        enum_1 = dataset_1.order(*@names_1).to_enum
+        enum_2 = dataset_2.order(*@names_2).to_enum
+
+        begin
+          record_1 = enum_1.next
+          record_2 = enum_2.next
+        rescue StopIteration
+          # no pairs to score
+          return
+        end
+        group_1 = []
+        group_2 = []
+        loop do
+          value_1 = record_1.values_at(*@names_1)
+          value_2 = record_2.values_at(*@names_2)
+          result = value_1 <=> value_2
+          if result == 0
+            last_value = value_1
+            group_1 << record_1
+            group_2 << record_2
+
+            state = :right
+            loop do
+              begin
+                case state
+                when :left
+                  record_1 = enum_1.next
+                  value_1 = record_1.values_at(*@names_1)
+                  result = last_value == value_1
+                when :right
+                  record_2 = enum_2.next
+                  value_2 = record_2.values_at(*@names_2)
+                  result = last_value == value_2
+                end
+              rescue StopIteration
+                result = false
+                case state
+                when :left
+                  record_1 = :eof
+                when :right
+                  record_2 = :eof
+                end
+              end
+
+              if result
+                case state
+                when :left
+                  group_1 << record_1
+                when :right
+                  group_2 << record_2
+                end
+              else
+                case state
+                when :left
+                  # done with this group
+                  score_groups(group_1, group_2)
+                  group_1.clear
+                  group_2.clear
+                  break
+                when :right
+                  state = :left
+                end
+              end
+            end
+            if record_1 == :eof || record_2 == :eof
+              break
+            end
+          else
+            begin
+              if result < 0
+                record_1 = enum_1.next
+              else
+                record_2 = enum_2.next
+              end
+            rescue StopIteration
+              break
+            end
+          end
+        end
+      end
+
+      def score_groups(group_1, group_2)
+        group_1.each do |record_1|
+          group_2.each do |record_2|
+            changed
+            notify_observers(self, record_1, record_2, 1)
+          end
+        end
+      end
+
+      def score_group(group)
+        (group.length - 1).times do |i|
+          ((i+1)...group.length).each do |j|
+            changed
+            notify_observers(self, group[i], group[j], 1)
+          end
+        end
+      end
     end
 
-    Comparator.register(Compare)
+    Comparator.register('compare', Compare)
   end
 end

data/lib/linkage/comparators/strcompare.rb

@@ -0,0 +1,85 @@
+module Linkage
+  module Comparators
+    # Strcompare is a string comparison comparator. It uses the specified
+    # operation to compare string-type fields. Score ranges from 0 to 1.
+    #
+    # To use Strcompare, you must specify one field for each record to use in
+    # the comparison, along with an operator. Valid operators are:
+    #
+    # * `:jarowinkler` ([Jaro-Winkler distance](http://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance))
+    #
+    # Consider the following example, using a {Configuration} as part of
+    # {Dataset#link_with}:
+    #
+    # ```ruby
+    # config.strcompare(:foo, :bar, :jarowinkler)
+    # ```
+    #
+    # For each record, the values of the `foo` and `bar` fields are compared
+    # using the Jaro-Winkler distance algorithm.
+    class Strcompare < Comparator
+      VALID_OPERATIONS = [:jarowinkler]
+
+      def initialize(field_1, field_2, operation)
+        if field_1.ruby_type[:type] != String || field_2.ruby_type[:type] != String
+          raise "fields must be string types"
+        end
+        if !VALID_OPERATIONS.include?(operation)
+          raise "#{operation.inspect} is not a valid operation"
+        end
+
+        @name_1 = field_1.name
+        @name_2 = field_2.name
+        @operation = operation
+      end
+
+      def score(record_1, record_2)
+        result =
+          case @operation
+          when :jarowinkler
+            jarowinkler(record_1[@name_1], record_2[@name_2])
+          end
+
+        result
+      end
+
+      def jarowinkler(w1, w2)
+        a = w1.downcase
+        b = w2.downcase
+        aa = a.split('')
+        ba = b.split('')
+        al = a.length
+        bl = b.length
+        l = 0
+        for i in Range.new(0, [[al, bl].min, 4].min-1)
+          break if aa[i] != ba[i]
+          l += 1
+        end
+        aj = aa - (aa - ba)
+        bj = ba - (ba - aa)
+        nm = 0
+        nt = 0
+        md = [[al, bl].max/2 - 1, 0].max
+        for i in Range.new(0, al-1)
+          bi = ba.index(aa[i])
+          aji = aj.index(aa[i])
+          bji = bj.index(aa[i])
+          if !bi.nil? && (bi + nm - i).abs <= md
+            nm += 1
+            nt += 1 if !bji.nil? && aji != bji
+          end
+          ba.delete_at(bi) if !bi.nil?
+          aj.delete_at(aji) if !aji.nil?
+          bj.delete_at(bji) if !bji.nil?
+        end
+        return 0 if nm == 0
+        d = (nm/al.to_f + nm/bl.to_f + (nm-nt)/nm.to_f)/3.0
+        w = (d + l * 0.1 * (1 - d)).round(3)
+        w
+      end
+    end
+
+    Comparator.register('strcompare', Strcompare)
+  end
+end
+

data/lib/linkage/comparators/within.rb

@@ -1,25 +1,29 @@
 module Linkage
   module Comparators
-    class Within < Binary
-      @@parameters = [
-        [:any, :static => false, :side => :first],
-        [Fixnum],
-        [:any, :same_type_as => 0, :static => false, :side => :second]
-      ]
-      def self.parameters
-        @@parameters
-      end
-
-      @@comparator_name = 'within'
-      def self.comparator_name
-        @@comparator_name
-      end
+    # Within is a integer comparator. It checks if two values are within a
+    # specified range. Score is either 0 to 1.
+    #
+    # To use Within, you must specify one field for each record to use in
+    # the comparison, along with a range value.
+    #
+    # Consider the following example, using a {Configuration} as part of
+    # {Dataset#link_with}:
+    #
+    # ```ruby
+    # config.within(:foo, :bar, 5)
+    # ```
+    #
+    # For each pair of records, if value of `foo` is within 5 (inclusive) of
+    # the value of `bar`, the score is 1. Otherwise, the score is 0.
+    class Within < Comparator
+      def initialize(field_1, field_2, value)
+        if field_1.ruby_type != field_2.ruby_type
+          raise "fields must have the same type"
+        end
 
-      def initialize(*args)
-        super
-        @name_1 = @args[0].name
-        @value = @args[1].object
-        @name_2 = @args[2].name
+        @name_1 = field_1.name
+        @name_2 = field_2.name
+        @value = value
       end
 
       def score(record_1, record_2)
@@ -27,6 +31,6 @@ module Linkage
       end
     end
 
-    Comparator.register(Within)
+    Comparator.register('within', Within)
   end
 end