linkage 0.1.0.pre → 0.1.0

data/lib/linkage/exceptions.rb

@@ -1,5 +1,10 @@
 module Linkage
+  # Generic error.
   class Error < Exception; end
+
+  # Error raised when a file would be overwritten.
   class ExistsError < Error; end
+
+  # Error raised when trying to read a file that doesn't exist.
   class MissingError < Error; end
 end

data/lib/linkage/field.rb

@@ -1,15 +1,14 @@
 module Linkage
-  # This class is for holding information about a particular field in a
-  # dataset.
+  # {Field} describes a field in a dataset, otherwise known as database table
+  # column.
   class Field
-    # @!attribute [r] name
-    #   @return [Symbol] This object's name
+    # @return [Symbol] This field's name
     attr_reader :name
 
-    # @return [Symbol] This field's schema information
+    # @return [Array] This field's schema information
     attr_reader :schema
 
-    # Create a new instance of Field.
+    # Returns a new instance of Field.
     #
     # @param [Symbol] name The field's name
     # @param [Hash] schema The field's schema information
@@ -19,23 +18,58 @@ module Linkage
     end
 
     # Convert the column schema information to a hash of column options, one of
-    # which must be :type. The other options added should modify that type
-    # (e.g. :size). If a database type is not recognized, return it as a String
-    # type.
+    # which is `:type`. The other options modify that type (e.g. `:size`).
     #
-    # @note This method comes more or less straight from Sequel
-    #   (lib/sequel/extensions/schema_dumper.rb).
+    # Here are some examples:
+    #
+    # | Database type    | Ruby type          | Other modifiers       |
+    # |------------------|--------------------|-----------------------|
+    # | mediumint        | Fixnum             |                       |
+    # | smallint         | Fixnum             |                       |
+    # | int              | Fixnum             |                       |
+    # | int(10) unsigned | Bignum             |                       |
+    # | tinyint          | TrueClass, Integer |                       |
+    # | bigint           | Bignum             |                       |
+    # | real             | Float              |                       |
+    # | float            | Float              |                       |
+    # | double           | Float              |                       |
+    # | boolean          | TrueClass          |                       |
+    # | text             | String             | text: true            |
+    # | date             | Date               |                       |
+    # | datetime         | DateTime           |                       |
+    # | timestamp        | DateTime           |                       |
+    # | time             | Time               | only_time: true       |
+    # | varchar(255)     | String             | size: 255             |
+    # | char(10)         | String             | size: 10, fixed: true |
+    # | money            | BigDecimal         | size: [19, 2]         |
+    # | decimal          | BigDecimal         |                       |
+    # | numeric          | BigDecimal         |                       |
+    # | number           | BigDecimal         |                       |
+    # | blob             | File               |                       |
+    # | year             | Integer            |                       |
+    # | identity         | Integer            |                       |
+    # | **other types**  | String             |                       |
+    #
+    # @note This method is copied from
+    #   {http://sequel.jeremyevans.net/rdoc-plugins/classes/Sequel/SchemaDumper.html `Sequel::SchemaDumper`}.
+    # @return [Hash]
     def ruby_type
       unless @ruby_type
         hsh =
-          case t = @schema[:db_type].downcase
-          when /\A(?:medium|small)?int(?:eger)?(?:\((?:\d+)\))?(?: unsigned)?\z/o
-            {:type=>Integer}
-          when /\Atinyint(?:\((\d+)\))?\z/o
-            {:type =>@schema[:type] == :boolean ? TrueClass : Integer}
+          case @schema[:db_type].downcase
+          when /\A(medium|small)?int(?:eger)?(?:\((\d+)\))?( unsigned)?\z/o
+            if !$1 && $2 && $2.to_i >= 10 && $3
+              # Unsigned integer type with 10 digits can potentially contain values which
+              # don't fit signed integer type, so use bigint type in target database.
+              {:type=>Bignum}
+            else
+              {:type=>Integer}
+            end
+          when /\Atinyint(?:\((\d+)\))?(?: unsigned)?\z/o
+            {:type =>schema[:type] == :boolean ? TrueClass : Integer}
           when /\Abigint(?:\((?:\d+)\))?(?: unsigned)?\z/o
             {:type=>Bignum}
-          when /\A(?:real|float|double(?: precision)?)\z/o
+          when /\A(?:real|float|double(?: precision)?|double\(\d+,\d+\)(?: unsigned)?)\z/o
             {:type=>Float}
           when 'boolean'
             {:type=>TrueClass}
@@ -60,7 +94,7 @@ module Linkage
             {:type=>BigDecimal, :size=>(s.empty? ? nil : s)}
           when /\A(?:bytea|(?:tiny|medium|long)?blob|(?:var)?binary)(?:\((\d+)\))?\z/o
             {:type=>File, :size=>($1.to_i if $1)}
-          when 'year'
+          when /\A(?:year|(?:int )?identity)\z/o
             {:type=>Integer}
           else
             {:type=>String}
@@ -73,6 +107,9 @@ module Linkage
       @ruby_type
     end
 
+    # Returns whether or not this field is a primary key.
+    #
+    # @return [Boolean]
     def primary_key?
       schema && schema[:primary_key]
     end

data/lib/linkage/field_set.rb

@@ -1,5 +1,11 @@
 module Linkage
+  # {FieldSet} is a `Hash` of {Field} values. It is usually associated with a
+  # {Dataset}. It looks up keys in a case-insensitive manner and doesn't care if
+  # you use strings or symbols.
+  #
+  # @see Dataset#field_set
   class FieldSet < Hash
+    # @return [Field] primary key of this field set.
     attr_reader :primary_key
 
     # Create a new FieldSet.
@@ -16,15 +22,29 @@ module Linkage
       end
     end
 
+    # Returns whether or not `key` is contained in the field set
+    # (case-insensitive).
+    #
+    # @param key [String, Symbol]
+    # @return [Boolean]
     def has_key?(key)
       !fetch_key(key).nil?
     end
 
+    # Returns a key that matches the parameter in a case-insensitive manner.
+    #
+    # @param key [String, Symbol]
+    # @return [Symbol]
     def fetch_key(key)
       string_key = key.to_s
       keys.detect { |k| k.to_s.casecmp(string_key) == 0 }
     end
 
+    # Returns the value for `key`, where `key` is matched in a case-insensitive
+    # manner.
+    #
+    # @param key [String, Symbol]
+    # @return [Field]
     def [](key)
       k = fetch_key(key)
       k ? super(k) : nil

data/lib/linkage/helpers.rb

@@ -0,0 +1,7 @@
+module Linkage
+  module Helpers
+  end
+end
+
+require 'linkage/helpers/csv'
+require 'linkage/helpers/database'

data/lib/linkage/helpers/csv.rb

@@ -0,0 +1,28 @@
+module Linkage
+  module Helpers
+    module CSV
+      def csv_filename(options)
+        File.expand_path(options[:filename], options[:dir] || '.')
+      end
+
+      def open_csv_for_reading(options)
+        filename = csv_filename(options)
+        if !File.exist?(filename)
+          raise MissingError, "#{filename} does not exist"
+        end
+        ::CSV.open(filename, 'rb', :headers => true)
+      end
+
+      def open_csv_for_writing(options)
+        filename = csv_filename(options)
+        if !options[:overwrite] && File.exist?(filename)
+          raise ExistsError, "#{filename} exists and not in overwrite mode"
+        end
+        if options[:dir]
+          FileUtils.mkdir_p(File.dirname(filename))
+        end
+        ::CSV.open(filename, 'wb')
+      end
+    end
+  end
+end

data/lib/linkage/helpers/database.rb

@@ -0,0 +1,47 @@
+module Linkage
+  module Helpers
+    module Database
+      # Returns a `Sequel::Database`.
+      #
+      # @overload database_connection(connection_options = {}, default_options = {})
+      #   @param connection_options [Hash] Options to establish a connection. Any
+      #     options not explicitly listed below are passed directly to `Sequel.connect`.
+      #   @option connection_options [String] :dir Parent directory to use for SQLite database
+      #   @option connection_options [String] :filename SQLite database filename
+      # @overload database_connection(url)
+      #   @param url [String] Sequel-style connection url
+      # @overload database_connection(database)
+      #   @param database [Sequel::Database]
+      def database_connection(connection_options = {}, default_options = {})
+        sequel_options = nil
+        connection_options ||= default_options
+
+        case connection_options
+        when Hash
+          connection_options = default_options.merge(connection_options)
+          sequel_options = connection_options.reject do |key, value|
+            key == :dir || key == :filename
+          end
+
+          if sequel_options.empty?
+            filename = connection_options[:filename] || 'linkage.db'
+            if connection_options[:dir]
+              dir = File.expand_path(connection_options[:dir])
+              FileUtils.mkdir_p(dir)
+              filename = File.join(dir, filename)
+            end
+            sequel_options[:adapter] = :sqlite
+            sequel_options[:database] = filename
+          end
+        when String
+          sequel_options = connection_options
+        when Sequel::Database
+          return connection_options
+        else
+          raise ArgumentError, "Expected Hash or String, got #{connection_options.class}"
+        end
+        Sequel.connect(sequel_options)
+      end
+    end
+  end
+end

data/lib/linkage/import_buffer.rb

@@ -1,8 +1,8 @@
 module Linkage
   class ImportBuffer
-    # @param [Sequel::Dataset] dataset
-    # @param [Array<Symbol>] headers List of fields you want to insert
-    # @param [Fixnum] limit Number of records to insert at a time
+    # @param dataset [Sequel::Dataset]
+    # @param headers [Array<Symbol>] List of fields you want to insert
+    # @param limit [Fixnum] Number of records to insert at a time
     def initialize(dataset, headers, limit = 1000)
       @dataset = dataset
       @headers = headers

data/lib/linkage/match_recorder.rb

@@ -1,5 +1,9 @@
 module Linkage
+  # {MatchRecorder} is responsible for observing {Matcher} for changes and
+  # saving matches to a {MatchSet} via {MatchSet#add_match}.
   class MatchRecorder
+    # @param matcher [Matcher]
+    # @param match_set [MatchSet]
     def initialize(matcher, match_set)
       @matcher = matcher
       @match_set = match_set

data/lib/linkage/match_set.rb

@@ -1,30 +1,68 @@
 module Linkage
+  # A {MatchSet} is responsible for keeping track of matches. After the scoring
+  # process, a {Matcher} uses scores from a {ScoreSet} to calculate which record
+  # pairs match. Those pairs are then recorded by a {MatchRecorder} to a
+  # {MatchSet}.
+  #
+  # {MatchSet} is the superclass of implementations for different formats.
+  # Currently there are two formats for storing matches:
+  #
+  # * CSV ({MatchSets::CSV})
+  # * Database ({MatchSets::Database})
+  #
+  # See the documentation for match set you're interested in for more
+  # information.
+  #
+  # If you want to implement a custom {MatchSet}, create a class that inherits
+  # {MatchSet} and defines at least {#add_match}. You can then register that
+  # class via {.register}.
+  #
+  # @abstract
   class MatchSet
-    # Register a match set.
-    #
-    # @param [Class] klass
-    def self.register(name, klass)
-      methods = klass.instance_methods(false)
-      unless methods.include?(:add_match)
-        raise ArgumentError, "class must define #add_match"
-      end
+    class << self
+      # Register a new match set. Subclasses must define at least {#add_match},
+      # otherwise an `ArgumentError` will be raised.
+      #
+      # @param [String] name Match set name used in {.klass_for}
+      # @param [Class] klass MatchSet subclass
+      def register(name, klass)
+        methods = klass.instance_methods(false)
+        unless methods.include?(:add_match)
+          raise ArgumentError, "class must define #add_match"
+        end
 
-      @match_sets ||= {}
-      @match_sets[name] = klass
-    end
+        @match_sets ||= {}
+        @match_sets[name] = klass
+      end
 
-    def self.[](name)
-      @match_sets ? @match_sets[name] : nil
+      # Return a registered MatchSet subclass or `nil` if it doesn't exist.
+      #
+      # @param [String] name of registered match set
+      # @return [Class, nil]
+      def klass_for(name)
+        @match_sets ? @match_sets[name] : nil
+      end
+      alias :[] :klass_for
     end
 
+    # This is called by {MatchRecorder#start}, before any matches are added via
+    # {#add_match}. Subclasses can redefine this to perform any setup needed for
+    # saving matches.
     def open_for_writing
     end
 
+    # Add a match to the MatchSet. Subclasses must redefine this.
+    #
+    # @param id_1 [Object] record id from first dataset
+    # @param id_2 [Object] record id from second dataset
+    # @param value [Fixnum, Float] match value
     # @abstract
     def add_match(id_1, id_2, score)
       raise NotImplementedError
     end
 
+    # This is called by {MatchRecorder#stop}, after all matches have been added.
+    # Subclasses can redefine this to perform any teardown needed.
     def close
     end
   end

data/lib/linkage/match_sets/csv.rb

@@ -2,27 +2,54 @@ require 'csv'
 
 module Linkage
   module MatchSets
+    # {CSV MatchSets::CSV} is an implementation of {MatchSet} for saving
+    # matches in a CSV file.
+    #
+    # There are three options available:
+    #
+    #  * `:filename` - which file to store matches in; can be an absolute path
+    #     or relative path
+    #  * `:dir` - which directory to put the file in; used if `:filename` is a
+    #     relative path
+    #  * `:overwrite` - indicate whether or not to overwrite an existing file
+    #
+    # By default, `:filename` is `'matches.csv'`, and the other options are
+    # blank. This means that it will write matches to the `'matches.csv'` file
+    # in the current working directory and will raise an error if the file
+    # already exists.
+    #
+    # If you specify `:dir`, that path will be created if it doesn't exist yet.
+    #
+    # The resulting file looks like this:
+    #
+    #     id_1,id_2,score
+    #     123,456,0.75
+    #     124,457,1
+    #
+    # @see Helpers::CSV
     class CSV < MatchSet
-      def initialize(filename, options = {})
-        @filename = filename
-        @overwrite = options[:overwrite]
+      include Helpers::CSV
+
+      DEFAULT_OPTIONS = {
+        :filename => 'matches.csv'
+      }
+
+      def initialize(options = {})
+        @options = DEFAULT_OPTIONS.merge(options.reject { |k, v| v.nil? })
       end
 
       def open_for_writing
         return if @mode == :write
 
-        if !@overwrite && File.exist?(@filename)
-          raise ExistsError, "#{@filename} exists and not in overwrite mode"
-        end
-
-        @csv = ::CSV.open(@filename, 'wb')
+        @csv = open_csv_for_writing(@options)
         @csv << %w{id_1 id_2 score}
         @mode = :write
       end
 
       def add_match(id_1, id_2, score)
         raise "not in write mode" if @mode != :write
-        if score.equal?(1.0) || score.equal?(0.0)
+
+        if score.floor.equal?(score.ceil)
           score = score.floor
         end
         @csv << [id_1, id_2, score]

data/lib/linkage/match_sets/database.rb

@@ -1,8 +1,49 @@
 module Linkage
   module MatchSets
+    # {Database MatchSets::Database} is an implementation of {MatchSet} for saving
+    # matches in a relational database.
+    #
+    # Matches are saved in a database table with the following columns:
+    # - id_1 (string)
+    # - id_2 (string)
+    # - score (float)
+    #
+    # You can setup a database connection in a few different ways. By default, a
+    # SQLite database with the filename of `matches.db` will be created in the
+    # current working directory. If you want something different, you can either
+    # specify a Sequel-style URI, provide connection options for
+    # `Sequel.connect`, or you can just specify a `Sequel::Database` object to
+    # use.
+    #
+    # There are a couple of non-Sequel connection options:
+    #  * `:filename` - specify filename to use for a SQLite database
+    #  * `:dir` - specify the parent directory for a SQLite database
+    #
+    # In addition to connection options, there are behavioral options you can
+    # set. By default, the table name used is called `matches`, but you change
+    # that by setting the `:table_name` option in the second options hash. If
+    # the table already exists, an {ExistsError} will be raised unless you set
+    # the `:overwrite` option to a truthy value in the second options hash.
+    #
+    # @see Helpers::Database
     class Database < MatchSet
-      def initialize(database, options = {})
-        @database = database
+      include Helpers::Database
+
+      DEFAULT_OPTIONS = {
+        :filename => 'matches.db'
+      }
+
+      # @override initialize(connection_options = {}, options = {})
+      #   @param connection_options [Hash]
+      #   @param options [Hash]
+      # @override initialize(uri, options = {})
+      #   @param uri [String]
+      #   @param options [Hash]
+      # @override initialize(database, options = {})
+      #   @param database [Sequel::Database]
+      #   @param options [Hash]
+      def initialize(connection_options = {}, options = {})
+        @database = database_connection(connection_options, DEFAULT_OPTIONS)
         @table_name = options[:table_name] || :matches
         @overwrite = options[:overwrite]
       end