linkage 0.0.6 → 0.0.8

data/.gitignore

@@ -0,0 +1,10 @@
+coverage
+rdoc
+doc
+.yardoc
+.bundle
+pkg
+test.rb
+results.db
+.rbenv-version
+bin

data/Gemfile

@@ -1,19 +1,21 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
-gem "sequel"
+gemspec
 
 group :development do
-  gem "bundler"
-  gem "jeweler"
-  gem "test-unit"
-  gem "mocha"
-  gem "sqlite3"
-  gem "yard"
-  gem "rake"
-  gem "versionomy"
-  gem "mysql2"
-  gem 'pry'
+  gem 'bundler'
+  gem 'test-unit'
+  gem 'mocha'
+  gem 'yard'
+  gem 'rake'
+  gem 'versionomy'
+  gem 'sqlite3', :platforms => :ruby
+  gem 'mysql2', :platforms => :ruby
+  gem 'jdbc-sqlite3', :platforms => :jruby
+  gem 'jdbc-mysql', :platforms => :jruby
   gem 'rdiscount'
   gem 'guard-test'
-  gem 'guard-yard'
+  gem 'guard-yard', :platforms => :ruby_19
+  gem 'rb-inotify', '~> 0.8.8'
+  gem 'debugger'
 end

data/Gemfile.lock

@@ -1,61 +1,91 @@
+PATH
+  remote: .
+  specs:
+    linkage (0.0.8)
+      hashery
+      sequel
+      sequel-collation
+
 GEM
   remote: http://rubygems.org/
   specs:
-    blockenspiel (0.4.3)
-    coderay (1.0.6)
-    ffi (1.0.11)
-    git (1.2.5)
-    guard (1.0.1)
-      ffi (>= 0.5.0)
-      thor (~> 0.14.6)
-    guard-test (0.4.3)
-      guard (>= 0.4)
+    blockenspiel (0.4.5)
+    coderay (1.0.8)
+    columnize (0.3.6)
+    debugger (1.3.0)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.1.1)
+      debugger-ruby_core_source (~> 1.1.7)
+    debugger-linecache (1.1.2)
+      debugger-ruby_core_source (>= 1.1.1)
+    debugger-ruby_core_source (1.1.7)
+    ffi (1.3.1)
+    ffi (1.3.1-java)
+    guard (1.6.2)
+      listen (>= 0.6.0)
+      lumberjack (>= 1.0.2)
+      pry (>= 0.9.10)
+      terminal-table (>= 1.4.3)
+      thor (>= 0.14.6)
+    guard-test (0.7.0)
+      guard (>= 1.1)
       test-unit (~> 2.2)
-    guard-yard (1.0.2)
-      guard (>= 0.2.2)
+    guard-yard (2.0.1)
+      guard (>= 1.1.0)
       yard (>= 0.7.0)
-    jeweler (1.8.3)
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      rake
-      rdoc
-    json (1.6.6)
+    hashery (2.1.0)
+    jdbc-mysql (5.1.22.1)
+    jdbc-sqlite3 (3.7.2.1)
+    listen (0.7.2)
+    lumberjack (1.0.2)
     metaclass (0.0.1)
-    method_source (0.7.1)
-    mocha (0.10.5)
+    method_source (0.8.1)
+    mocha (0.13.2)
       metaclass (~> 0.0.1)
     mysql2 (0.3.11)
-    pry (0.9.9)
+    pry (0.9.11.4)
       coderay (~> 1.0.5)
-      method_source (~> 0.7.1)
-      slop (>= 2.4.4, < 3)
-    rake (0.9.2.2)
-    rdiscount (1.6.8)
-    rdoc (3.12)
-      json (~> 1.4)
-    sequel (3.34.1)
-    slop (2.4.4)
-    sqlite3 (1.3.6)
-    test-unit (2.4.8)
-    thor (0.14.6)
-    versionomy (0.4.3)
-      blockenspiel (>= 0.4.3)
-    yard (0.7.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    pry (0.9.11.4-java)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    rake (10.0.3)
+    rb-inotify (0.8.8)
+      ffi (>= 0.5.0)
+    rdiscount (2.0.7)
+    sequel (3.44.0)
+    sequel-collation (0.1.0)
+      sequel
+    slop (3.4.3)
+    spoon (0.0.1)
+    sqlite3 (1.3.7)
+    terminal-table (1.4.5)
+    test-unit (2.5.4)
+    thor (0.17.0)
+    versionomy (0.4.4)
+      blockenspiel (>= 0.4.5)
+    yard (0.8.4.1)
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES
   bundler
+  debugger
   guard-test
   guard-yard
-  jeweler
+  jdbc-mysql
+  jdbc-sqlite3
+  linkage!
   mocha
   mysql2
-  pry
   rake
+  rb-inotify (~> 0.8.8)
   rdiscount
-  sequel
   sqlite3
   test-unit
   versionomy

data/Guardfile

@@ -9,5 +9,3 @@ end
 guard 'yard' do
   watch(%r{lib/[^.].*\.rb$})
 end
-
-notification :notifysend, :u => :normal

data/Rakefile

@@ -10,20 +10,7 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'rake'
-
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "linkage"
-  gem.homepage = "http://github.com/coupler/linkage"
-  gem.license = "MIT"
-  gem.summary = %Q{Record linkage library}
-  gem.description = %Q{Performs record linkage between one or two datasets, using Sequel on the backend}
-  gem.email = "jeremy.f.stephens@vanderbilt.edu"
-  gem.authors = ["Jeremy Stephens"]
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
+require "bundler/gem_tasks"
 
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
@@ -34,19 +21,129 @@ end
 
 task :default => :test
 
-require 'rdoc/task'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "linkage #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
-
 require 'yard'
 YARD::Rake::YardocTask.new do |t|
   t.files = ['lib/**/*.rb']
 end
 
-task :build => :gemspec
+# Yoinked from https://github.com/rails/rails/blob/master/railties/lib/rails/tasks/annotations.rake
+namespace :notes do
+  ["OPTIMIZE", "FIXME", "TODO"].each do |annotation|
+    desc "Enumerate all #{annotation} annotations"
+    task annotation.downcase.intern do
+      SourceAnnotationExtractor.enumerate annotation
+    end
+  end
+
+  desc "Enumerate a custom annotation, specify with ANNOTATION=CUSTOM"
+  task :custom do
+    SourceAnnotationExtractor.enumerate ENV['ANNOTATION']
+  end
+end
+
+# Yoinked from https://github.com/rails/rails/blob/master/railties/lib/rails/source_annotation_extractor.rb
+#
+# Implements the logic behind the rake tasks for annotations like
+#
+#   rake notes
+#   rake notes:optimize
+#
+# and friends. See <tt>rake -T notes</tt> and <tt>railties/lib/tasks/annotations.rake</tt>.
+#
+# Annotation objects are triplets <tt>:line</tt>, <tt>:tag</tt>, <tt>:text</tt> that
+# represent the line where the annotation lives, its tag, and its text. Note
+# the filename is not stored.
+#
+# Annotations are looked for in comments and modulus whitespace they have to
+# start with the tag optionally followed by a colon. Everything up to the end
+# of the line (or closing ERB comment tag) is considered to be their text.
+class SourceAnnotationExtractor
+  class Annotation < Struct.new(:line, :tag, :text)
+
+    # Returns a representation of the annotation that looks like this:
+    #
+    #   [126] [TODO] This algorithm is simple and clearly correct, make it faster.
+    #
+    # If +options+ has a flag <tt>:tag</tt> the tag is shown as in the example above.
+    # Otherwise the string contains just line and text.
+    def to_s(options={})
+      s = "[%3d] " % line
+      s << "[#{tag}] " if options[:tag]
+      s << text
+    end
+  end
+
+  # Prints all annotations with tag +tag+ under the root directories +app+, +config+, +lib+,
+  # +script+, and +test+ (recursively). Only filenames with extension 
+  # +.builder+, +.rb+, and +.erb+ are taken into account. The +options+
+  # hash is passed to each annotation's +to_s+.
+  #
+  # This class method is the single entry point for the rake tasks.
+  def self.enumerate(tag, options={})
+    extractor = new(tag)
+    extractor.display(extractor.find, options)
+  end
+
+  attr_reader :tag
+
+  def initialize(tag)
+    @tag = tag
+  end
+
+  # Returns a hash that maps filenames under +dirs+ (recursively) to arrays
+  # with their annotations.
+  def find(dirs=%w(app config lib script test))
+    dirs.inject({}) { |h, dir| h.update(find_in(dir)) }
+  end
+
+  # Returns a hash that maps filenames under +dir+ (recursively) to arrays
+  # with their annotations. Only files with annotations are included, and only
+  # those with extension +.builder+, +.rb+, +.erb+, +.haml+, +.slim+ and +.coffee+
+  # are taken into account.
+  def find_in(dir)
+    results = {}
+
+    Dir.glob("#{dir}/*") do |item|
+      next if File.basename(item)[0] == ?.
+
+      if File.directory?(item)
+        results.update(find_in(item))
+      elsif item =~ /\.(builder|rb|coffee)$/
+        results.update(extract_annotations_from(item, /#\s*(#{tag}):?\s*(.*)$/))
+      elsif item =~ /\.erb$/
+        results.update(extract_annotations_from(item, /<%\s*#\s*(#{tag}):?\s*(.*?)\s*%>/))
+      elsif item =~ /\.haml$/
+        results.update(extract_annotations_from(item, /-\s*#\s*(#{tag}):?\s*(.*)$/))
+      elsif item =~ /\.slim$/
+        results.update(extract_annotations_from(item, /\/\s*\s*(#{tag}):?\s*(.*)$/))
+      end
+    end
+
+    results
+  end
+
+  # If +file+ is the filename of a file that contains annotations this method returns
+  # a hash with a single entry that maps +file+ to an array of its annotations.
+  # Otherwise it returns an empty hash.
+  def extract_annotations_from(file, pattern)
+    lineno = 0
+    result = File.readlines(file).inject([]) do |list, line|
+      lineno += 1
+      next list unless line =~ pattern
+      list << Annotation.new(lineno, $1, $2)
+    end
+    result.empty? ? {} : { file => result }
+  end
+
+  # Prints the mapping from filenames to annotations in +results+ ordered by filename.
+  # The +options+ hash is passed to each annotation's +to_s+.
+  def display(results, options={})
+    results.keys.sort.each do |file|
+      puts "#{file}:"
+      results[file].each do |note|
+        puts "  * #{note.to_s(options)}"
+      end
+      puts
+    end
+  end
+end

data/lib/linkage/comparator.rb

@@ -0,0 +1,172 @@
+module Linkage
+  # @abstract Abstract class to represent record comparators.
+  class Comparator
+    # Register a new comparator.
+    #
+    # @param [Class] klass Comparator subclass
+    def self.register(klass)
+      name = nil
+      begin
+        name = klass.comparator_name
+      rescue NotImplementedError
+        raise ArgumentError, "comparator_name class method must be defined"
+      end
+
+      if !klass.instance_methods(false).include?(:score)
+        raise ArgumentError, "class must define the score method"
+      end
+
+      begin
+        if klass.parameters.length > 0
+          @comparators ||= {}
+          @comparators[name] = klass
+        else
+          raise ArgumentError, "class must have at least one parameter"
+        end
+      rescue NotImplementedError
+        raise ArgumentError, "parameters class method must be defined"
+      end
+
+      begin
+        range = klass.score_range
+        if !range.is_a?(Range) || !range.first.is_a?(Numeric) ||
+              !range.last.is_a?(Numeric)
+          raise ArgumentError, "score_range must be a Range of two numbers"
+        end
+      rescue NotImplementedError
+        raise ArgumentError, "score_range class method must be defined"
+      end
+    end
+
+    def self.[](name)
+      @comparators ? @comparators[name] : nil
+    end
+
+    # @abstract Override this to return the name of the comparator.
+    # @return [String]
+    def self.comparator_name
+      raise NotImplementedError
+    end
+
+    # @abstract Override this to require a specific number of arguments of a
+    #   certain class. To require two parameters of either String or Integer,
+    #   do something like this:
+    #
+    #     @@parameters = [[String, Integer], [String, Integer]]
+    #     def self.parameters
+    #       @@parameters
+    #     end
+    #
+    #   At least one argument must be defined.
+    # @return [Array]
+    def self.parameters
+      raise NotImplementedError
+    end
+
+    # @abstract Override this to return a Range of the possible scores for the
+    #   comparator.
+    # @return [Range]
+    def self.score_range
+      raise NotImplementedError
+    end
+
+    attr_reader :args, :lhs_args, :rhs_args
+
+    # Create a new Comparator object.
+    # @param [Linkage::MetaObject, Hash] args Comparator arguments
+    def initialize(*args)
+      @args = args
+      @lhs_args = []
+      @rhs_args = []
+      @options = args.last.is_a?(Hash) ? args.pop : {}
+      process_args
+    end
+
+    # @abstract Override this to return the score of the linkage strength of
+    #   two records.
+    # @return [Numeric]
+    def score(record_1, record_2)
+      raise NotImplementedError
+    end
+
+    private
+
+    def process_args
+      parameters = self.class.parameters
+      if parameters.length != @args.length
+        raise ArgumentError, "wrong number of arguments (#{@args.length} for #{parameters.length})"
+      end
+
+      first_side = nil
+      second_side = nil
+      @args.each_with_index do |arg, i|
+        type = arg.ruby_type[:type]
+
+        parameter_types = parameters[i]
+        if parameter_types.last.is_a?(Hash)
+          parameter_options = parameter_types[-1]
+          parameter_types = parameter_types[0..-2]
+        else
+          parameter_options = {}
+        end
+
+        if parameter_types[0] != :any && !parameter_types.include?(type)
+          raise TypeError, "expected type #{parameters[i].join(" or ")}, got #{type}"
+        end
+
+        if parameter_options.has_key?(:values) && arg.raw? && !parameter_options[:values].include?(arg.object)
+          raise ArgumentError, "argument #{i + 1} (#{arg.object.inspect}) was not one of the expected values: #{parameter_options[:values].inspect}"
+        end
+
+        if parameter_options.has_key?(:same_type_as)
+          arg_index = parameter_options[:same_type_as]
+          other_type = @args[arg_index].ruby_type[:type]
+          if type != other_type
+            raise TypeError, "argument #{i + 1} (#{type}) was expected to have the same type as argument #{arg_index + 1} (#{other_type})"
+          end
+        end
+
+        if parameter_options.has_key?(:static) &&
+              parameter_options[:static] != arg.static?
+          raise TypeError, "argument #{i + 1} was expected to #{arg.static? ? "not be" : "be"} static"
+        end
+
+        if !arg.static?
+          if first_side.nil?
+            first_side = arg.side
+          elsif arg.side != first_side && second_side.nil?
+            second_side = arg.side
+          end
+
+          valid_side = true
+          case parameter_options[:side]
+          when :first
+            if arg.side != first_side
+              valid_side = false
+            end
+          when :second
+            if second_side.nil? || arg.side != second_side
+              valid_side = false
+            end
+          end
+
+          if !valid_side
+            raise TypeError, "argument #{i + 1} was expected to have a different side value"
+          end
+
+          case arg.side
+          when :lhs
+            @lhs_args << arg
+          when :rhs
+            @rhs_args << arg
+          end
+        end
+      end
+    end
+  end
+end
+
+path = File.expand_path(File.join(File.dirname(__FILE__), "comparators"))
+require File.join(path, "binary")
+require File.join(path, "compare")
+require File.join(path, "within")

data/lib/linkage/comparators/binary.rb

@@ -0,0 +1,12 @@
+module Linkage
+  module Comparators
+    # @abstract Convenient abstract class for comparators that only return
+    #   true/false values (0 or 1).
+    class Binary < Comparator
+      @@score_range = 0..1
+      def self.score_range
+        @@score_range
+      end
+    end
+  end
+end

data/lib/linkage/comparators/compare.rb

@@ -0,0 +1,46 @@
+module Linkage
+  module Comparators
+    class Compare < Binary
+      @@parameters = [
+        [:any, :static => false, :side => :first],
+        [String, :values => %w{> >= <= < !=}],
+        [:any, :same_type_as => 0, :static => false, :side => :second]
+      ]
+      def self.parameters
+        @@parameters
+      end
+
+      @@comparator_name = 'compare'
+      def self.comparator_name
+        @@comparator_name
+      end
+
+      def initialize(*args)
+        super
+        @name_1 = @args[0].name
+        @operator = @args[1].object
+        @name_2 = @args[2].name
+      end
+
+      def score(record_1, record_2)
+        result =
+          case @operator
+          when '!='
+            record_1[@name_1] != record_2[@name_2]
+          when '>'
+            record_1[@name_1] > record_2[@name_2]
+          when '>='
+            record_1[@name_1] >= record_2[@name_2]
+          when '<='
+            record_1[@name_1] <= record_2[@name_2]
+          when '<'
+            record_1[@name_1] < record_2[@name_2]
+          end
+
+        result ? 1 : 0
+      end
+    end
+
+    Comparator.register(Compare)
+  end
+end

data/lib/linkage/comparators/within.rb

@@ -0,0 +1,32 @@
+module Linkage
+  module Comparators
+    class Within < Binary
+      @@parameters = [
+        [:any, :static => false, :side => :first],
+        [Fixnum],
+        [:any, :same_type_as => 0, :static => false, :side => :second]
+      ]
+      def self.parameters
+        @@parameters
+      end
+
+      @@comparator_name = 'within'
+      def self.comparator_name
+        @@comparator_name
+      end
+
+      def initialize(*args)
+        super
+        @name_1 = @args[0].name
+        @value = @args[1].object
+        @name_2 = @args[2].name
+      end
+
+      def score(record_1, record_2)
+        (record_1[@name_1] - record_2[@name_2]).abs <= @value ? 1 : 0
+      end
+    end
+
+    Comparator.register(Within)
+  end
+end