linkage 0.1.0.pre → 0.1.0

data/lib/linkage/matcher.rb

@@ -1,9 +1,29 @@
 module Linkage
+  # {Matcher} is responsible for combining scores from a {ScoreSet} and deciding
+  # which pairs of records match. There are two parameters you can use to
+  # determine how {Matcher} does this: `algorithm` and `threshold`.
+  #
+  # There are currently two algorithm options: `:mean` and `:sum`. The mean
+  # algorithm will create a mean score for each pair of records. The sum
+  # algorithm will create a total score for each pair of records.
+  #
+  # The `threshold` parameter determines what is considered a match. If the
+  # result score for a pair of records (depending on the algorithm used) is
+  # greater than or equal to the threshold, then the pair is considered to be a
+  # match.
+  #
+  # Whenever {Matcher} finds a match, it uses the observer pattern to notify
+  # other objects that a match has been found. Usually the only observer is a
+  # {MatchRecorder}.
   class Matcher
     include Observable
 
     attr_reader :comparators, :score_set, :algorithm, :threshold
 
+    # @param comparators [Array<Comparator>]
+    # @param score_set [ScoreSet]
+    # @param algorithm [Symbol] `:mean` or `:sum`
+    # @param threshold [Numeric]
     def initialize(comparators, score_set, algorithm, threshold)
       @comparators = comparators
       @score_set = score_set
@@ -11,20 +31,46 @@ module Linkage
       @threshold = threshold
     end
 
+    # Find matches.
     def run
       send(@algorithm)
     end
 
-    private
-
+    # Combine scores for each pair of records via mean, then compare the
+    # combined score to the threshold. Notify observers if there's a match.
     def mean
+      w = @comparators.collect { |comparator| comparator.weight || 1 }
+      @score_set.open_for_reading
       @score_set.each_pair do |id_1, id_2, scores|
-        mean = scores.values.inject(:+) / @comparators.length.to_f
+        sum = 0
+        scores.each do |key, value|
+          sum += value * w[key-1]
+        end
+        mean = sum / @comparators.length.to_f
         if mean >= @threshold
           changed
           notify_observers(id_1, id_2, mean)
         end
       end
+      @score_set.close
+    end
+
+    # Combine scores for each pair of records via sum, then compare the
+    # combined score to the threshold. Notify observers if there's a match.
+    def sum
+      w = @comparators.collect { |comparator| comparator.weight || 1 }
+      @score_set.open_for_reading
+      @score_set.each_pair do |id_1, id_2, scores|
+        sum = 0
+        scores.each do |key, value|
+          sum += value * w[key-1]
+        end
+        if sum >= @threshold
+          changed
+          notify_observers(id_1, id_2, sum)
+        end
+      end
+      @score_set.close
     end
   end
 end

data/lib/linkage/result_set.rb

@@ -1,37 +1,75 @@
 module Linkage
+  # A {ResultSet} is a convenience class for wrapping a {ScoreSet} and a
+  # {MatchSet}. Most of the time, you'll want to use the same storage format for
+  # both scores and matches. {ResultSet} provides a way to group both sets
+  # together.
+  #
+  # The default implementation of {ResultSet} merely returns whatever {ScoreSet}
+  # and {MatchSet} you pass to it during creation (see {#initialize}). However,
+  # {ResultSet} can be subclassed to provide easy initialization of sets of the
+  # same format. Currently there are two subclasses:
+  #
+  # * CSV ({ResultSets::CSV})
+  # * Database ({ResultSets::Database})
+  #
+  # If you want to implement a custom {ResultSet}, create a class that inherits
+  # {ResultSet} and defines both {#score_set} and {#match_set} to return a
+  # {ScoreSet} and {MatchSet} respectively. You can then register that class via
+  # {.register} to make it easier to use.
   class ResultSet
-    # Register a result set.
-    #
-    # @param [Class] klass
-    def self.register(name, klass)
-      methods = klass.instance_methods(false)
-      missing = []
-      unless methods.include?(:score_set)
-        missing.push("#score_set")
-      end
-      unless methods.include?(:match_set)
-        missing.push("#match_set")
-      end
-      unless missing.empty?
-        raise ArgumentError, "class must define #{missing.join(" and ")}"
+    class << self
+      # Register a new result set. Subclasses must define {#score_set} and
+      # {#match_set}.  Otherwise, an `ArgumentError` will be raised when you try
+      # to call {.register}.
+      #
+      # @param [String] name Result set name used in {.klass_for}
+      # @param [Class] klass ResultSet subclass
+      def register(name, klass)
+        methods = klass.instance_methods
+        missing = []
+        unless methods.include?(:score_set)
+          missing.push("#score_set")
+        end
+        unless methods.include?(:match_set)
+          missing.push("#match_set")
+        end
+        unless missing.empty?
+          raise ArgumentError, "class must define #{missing.join(" and ")}"
+        end
+
+        @result_set ||= {}
+        @result_set[name] = klass
       end
 
-      @result_set ||= {}
-      @result_set[name] = klass
+      # Return a registered ResultSet subclass or `nil` if it doesn't exist.
+      #
+      # @param [String] name of registered result set
+      # @return [Class, nil]
+      def klass_for(name)
+        @result_set ? @result_set[name] : nil
+      end
+      alias :[] :klass_for
     end
 
-    def self.[](name)
-      @result_set ? @result_set[name] : nil
+    # @param [ScoreSet] score_set
+    # @param [MatchSet] match_set
+    def initialize(score_set, match_set)
+      @score_set = score_set
+      @match_set = match_set
     end
 
-    # @abstract
+    # Returns a {ScoreSet}.
+    #
+    # @return [ScoreSet]
     def score_set
-      raise NotImplementedError
+      @score_set
     end
 
-    # @abstract
+    # Returns a {MatchSet}.
+    #
+    # @return [MatchSet]
     def match_set
-      raise NotImplementedError
+      @match_set
     end
   end
 end

data/lib/linkage/result_sets/csv.rb

@@ -1,8 +1,51 @@
 module Linkage
   module ResultSets
+    # {CSV ResultSets::CSV} is a subclass of {ResultSet ResultSet} that makes it
+    # convenient to set up a {ScoreSets::CSV} and {MatchSets::CSV} at the same
+    # time. For example:
+    #
+    # ```ruby
+    # result_set = Linkage::ResultSets::CSV.new('/some/path')
+    # ```
+    #
+    # Or by using {ResultSet.[] ResultSet.[]}:
+    #
+    # ```ruby
+    # result_set = Linkage::ResultSet['csv'].new('/some/path')
+    # ```
+    #
+    # {#initialize ResultSets::CSV.new} takes either a directory name as its
+    # argument or a Hash of options. Passing in a directory name is equivalent
+    # to passing in a Hash with the `:dir` key. For example:
+    #
+    # ```ruby
+    # result_set = Linkage::ResultSet['csv'].new('/some/path')
+    # ```
+    #
+    # is the same as:
+    #
+    # ```ruby
+    # result_set = Linkage::ResultSet['csv'].new({:dir => '/some/path'})
+    # ```
+    #
+    # The `:dir` option lets you specify the parent directory for the score set
+    # and result set files (which are `scores.csv` and `results.csv` by default).
+    #
+    # The only other relevant option is `:overwrite`, which controls whether or
+    # not overwriting existing files is permitted.
+    #
+    # @see ScoreSets::CSV
+    # @see MatchSets::CSV
     class CSV < ResultSet
+      # @overload initialize(dir)
+      #   @param [String] dir parent directory of CSV files
+      # @overload initialize(options)
+      #   @param [Hash] options
+      #   @option options [String] :dir parent directory of CSV files
+      #   @option options [Boolean] :overwrite (false) whether or not to allow
+      #     overwriting existing files
       def initialize(dir_or_options = nil)
-        opts =
+        @options =
           case dir_or_options
           when nil
             {}
@@ -13,39 +56,14 @@ module Linkage
           else
             raise ArgumentError, "expected nil, a String, or a Hash, got #{dir_or_options.class}"
           end
-
-        if opts[:dir]
-          opts[:dir] = File.expand_path(opts[:dir])
-          FileUtils.mkdir_p(opts[:dir])
-        end
-
-        @score_set_args = extract_args_for(:scores, opts)
-        @match_set_args = extract_args_for(:matches, opts)
       end
 
       def score_set
-        @score_set ||= ScoreSet['csv'].new(*@score_set_args)
+        @score_set ||= ScoreSet['csv'].new(@options)
       end
 
       def match_set
-        @match_set ||= MatchSet['csv'].new(*@match_set_args)
-      end
-
-      private
-
-      def extract_args_for(name, opts)
-        dir = opts[:dir] || '.'
-        opts = opts[name]
-
-        filename =
-          case opts
-          when Hash, nil
-            opts = opts ? opts.dup : {}
-            opts.delete(:filename) || "#{name}.csv"
-          when String
-            opts
-          end
-        [File.join(dir, filename), opts]
+        @match_set ||= MatchSet['csv'].new(@options)
       end
     end
 

data/lib/linkage/result_sets/database.rb

@@ -1,39 +1,57 @@
 module Linkage
   module ResultSets
+    # {Database ResultSets::Database} is the {ResultSet ResultSet} for writing
+    # to database tables. You can use it by either referencing it directly like
+    # so:
+    #
+    # ```ruby
+    # result_set = Linkage::ResultSets::Database.new(connection_options, options)
+    # ```
+    #
+    # Or by using {ResultSet.[] ResultSet.[]}:
+    #
+    # ```ruby
+    # result_set = Linkage::ResultSet['database'].new(connection_options, options)
+    # ```
+    #
+    # You can setup a database connection in a few different ways. By default, a
+    # SQLite database with the filename of `results.db` will be created in the
+    # current working directory. If you want something different, you can either
+    # specify a Sequel-style URI, provide connection options for
+    # `Sequel.connect`, or you can just specify a
+    # {http://sequel.jeremyevans.net/rdoc/classes/Sequel/Database.html Sequel::Database}
+    # object to use.
+    #
+    # There are a couple of non-Sequel connection options:
+    #  * `:filename` - specify filename to use for a SQLite database
+    #  * `:dir` - specify the parent directory for a SQLite database
+    #
+    # This result set creates a {ScoreSets::Database database-backed score set}
+    # and a {Matchsets::Database database-backed match set} with their default
+    # table names (`scores` and `matches` respectively.  If either table already
+    # exists, an {ExistsError} will be raised unless you set the `:overwrite`
+    # option to a truthy value in the second options hash.
+    #
+    # @see ScoreSets::Database
+    # @see MatchSets::Database
     class Database < ResultSet
-      def initialize(database_or_options = nil)
-        @database = nil
-        @options = {}
+      include Helpers::Database
 
-        if database_or_options.kind_of?(Sequel::Database)
-          @database = database_or_options
-        else
-          database_opts = nil
-          case database_or_options
-          when String
-            database_opts = database_or_options
-          when Hash
-            database_opts = {}
-            database_or_options.each_pair do |key, value|
-              if key == :scores || key == :matches
-                @options[key] = value
-              else
-                database_opts[key] = value
-              end
-            end
-          else
-            raise ArgumentError, "expected Sequel::Database, a String, or a Hash, got #{database_or_options.class}"
-          end
-          @database = Sequel.connect(database_opts)
-        end
+      DEFAULT_OPTIONS = {
+        :filename => 'results.db'
+      }
+
+      def initialize(connection_options = {}, options = {})
+        @database = database_connection(connection_options, DEFAULT_OPTIONS)
+        @options = options
       end
 
       def score_set
-        @score_set ||= ScoreSet['database'].new(@database, @options[:scores] || {})
+        @score_set ||= ScoreSet['database'].new(@database, @options)
       end
 
       def match_set
-        @match_set ||= MatchSet['database'].new(@database, @options[:matches] || {})
+        @match_set ||= MatchSet['database'].new(@database, @options)
       end
     end
 

data/lib/linkage/runner.rb

@@ -1,5 +1,15 @@
 module Linkage
   # Use this class to run a configuration created by {Dataset#link_with}.
+  #
+  # During a record linkage, one or more {Comparator}s generate scores. Each
+  # score is recorded by a {ScoreRecorder}, which uses a {ScoreSet} to actually
+  # save the score. After the scoring is complete, a {Matcher} combines the
+  # scores to create matches. Each match is recorded by a {MatchRecorder}, which
+  # uses a {MatchSet} to actually save the match information.
+  #
+  # So to save scores and matches, we need both a {ScoreSet} and a {MatchSet}.
+  # To make this easier, a {ResultSet} can be used to configure both {ScoreSet}s
+  # and {MatchSet}s.
   class Runner
     attr_reader :config
 

data/lib/linkage/score_recorder.rb

@@ -1,5 +1,10 @@
 module Linkage
+  # {ScoreRecorder} is responsible for observing a set of {Comparator} for
+  # changes and saving matches to a {ScoreSet} via {ScoreSet#add_score}.
   class ScoreRecorder
+    # @param comparators [Array<Comparator>]
+    # @param score_set [ScoreSet]
+    # @param primary_keys [Array<Symbol>]
     def initialize(comparators, score_set, primary_keys)
       @comparators = comparators
       @score_set = score_set

data/lib/linkage/score_set.rb

@@ -1,45 +1,103 @@
 module Linkage
+  # A {ScoreSet} is responsible for keeping track of scores. During the record
+  # linkage process, one or more {Comparator}s generate scores. These scores are
+  # handled by a {ScoreRecorder}, which uses a {ScoreSet} to actually save the
+  # scores. {ScoreSet} is also used to fetch the linkage scores so that a
+  # {Matcher} can create matches.
+  #
+  # {ScoreSet} is the superclass of implementations for different formats.
+  # Currently there are two formats for storing scores:
+  #
+  # * CSV ({ScoreSets::CSV})
+  # * Database ({ScoreSets::Database})
+  #
+  # See the documentation for score set you're interested in for more
+  # information.
+  #
+  # If you want to implement a custom {ScoreSet}, create a class that inherits
+  # {ScoreSet} and defines at least {#add_score} and {#each_pair}. You can then
+  # register that class via {.register}.
+  #
+  # @abstract
   class ScoreSet
-    # Register a score set.
-    #
-    # @param [Class] klass
-    def self.register(name, klass)
-      methods = klass.instance_methods(false)
-      missing = []
-      unless methods.include?(:add_score)
-        missing.push("#add_score")
-      end
-      unless methods.include?(:each_pair)
-        missing.push("#each_pair")
-      end
-      unless missing.empty?
-        raise ArgumentError, "class must define #{missing.join(" and ")}"
-      end
+    class << self
+      # Register a new score set. Subclasses must define at least {#add_score}
+      # and {#each_pair}. Otherwise, an `ArgumentError` will be raised when you
+      # try to call {.register}.
+      #
+      # @param [String] name Score set name used in {.klass_for}
+      # @param [Class] klass ScoreSet subclass
+      def register(name, klass)
+        methods = klass.instance_methods(false)
+        missing = []
+        unless methods.include?(:add_score)
+          missing.push("#add_score")
+        end
+        unless methods.include?(:each_pair)
+          missing.push("#each_pair")
+        end
+        unless missing.empty?
+          raise ArgumentError, "class must define #{missing.join(" and ")}"
+        end
 
-      @score_sets ||= {}
-      @score_sets[name] = klass
-    end
+        @score_sets ||= {}
+        @score_sets[name] = klass
+      end
 
-    def self.[](name)
-      @score_sets ? @score_sets[name] : nil
+      # Return a registered ScoreSet subclass or `nil` if it doesn't exist.
+      #
+      # @param [String] name of registered score set
+      # @return [Class, nil]
+      def klass_for(name)
+        @score_sets ? @score_sets[name] : nil
+      end
+      alias :[] :klass_for
     end
 
+    # This is called by {Matcher#run}, before any scores are read via
+    # {#each_pair}. Subclasses can redefine this to perform any setup needed
+    # for reading scores.
     def open_for_reading
     end
 
+    # This is called by {ScoreRecorder#start}, before any scores are added via
+    # {#add_score}. Subclasses can redefine this to perform any setup needed
+    # for saving scores.
     def open_for_writing
     end
 
+    # Add a score to the ScoreSet. Subclasses must redefine this.
+    #
+    # @param comparator_id [Fixnum] 1-indexed comparator index
+    # @param id_1 [Object] record id from first dataset
+    # @param id_2 [Object] record id from second dataset
+    # @param value [Fixnum, Float] score value
     # @abstract
     def add_score(comparator_id, id_1, id_2, value)
       raise NotImplementedError
     end
 
+    # Yield scores for each pair of records. Subclasses must redefine this.
+    # This method is called by {Matcher#run} with a block with three
+    # parameters:
+    #
+    # ```ruby
+    # score_set.each_pair do |id_1, id_2, scores|
+    # end
+    # ```
+    #
+    # `scores` should be a Hash where comparator ids are keys and scores are
+    # values. For example: `{ 1 => 0.5, 2 => 0.75, 3 => 1 }`. Note that not all
+    # comparators (including {Comparators::Compare}) create scores for each
+    # pair. A missing score means that pair was given a score of 0.
+    #
     # @abstract
     def each_pair(&block)
       raise NotImplementedError
     end
 
+    # This is called by {ScoreRecorder#stop}, after all scores have been added.
+    # Subclasses can redefine this to perform any teardown needed.
     def close
     end
   end