linkage 0.0.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.vimrc

@@ -0,0 +1,34 @@
+function! s:AlternateFile()
+  let fn = substitute(expand('%'), "^".getcwd()."/", "", "")
+  let head = fnamemodify(fn, ':h')
+  let tail = fnamemodify(fn, ':t')
+
+  if match(head, '^lib') >= 0
+    return substitute(head, '^lib/linkage', 'test/unit', '').'/test_'.tail
+  elseif match(head, '^test') >= 0
+    return substitute(head, '^test/unit', 'lib/linkage', '').'/'.substitute(tail, '^test_', '', '')
+  endif
+  return ''
+endfunction
+
+function! s:Alternate(cmd)
+  let file = s:AlternateFile()
+  "if file != '' && filereadable(file)
+    if a:cmd == 'T'
+      let cmd = 'tabe'
+    elseif a:cmd == 'S'
+      let cmd = 'sp'
+    else
+      let cmd = 'e'
+    endif
+    exe ':'.cmd.' '.file
+  "else
+    "echomsg 'No alternate file is defined: '.file
+  "endif
+endfunction
+
+command! A  :call s:Alternate('')
+command! AE :call s:Alternate('E')
+command! AS :call s:Alternate('S')
+command! AV :call s:Alternate('V')
+command! AT :call s:Alternate('T')

data/Gemfile

@@ -0,0 +1,17 @@
+source "http://rubygems.org"
+
+gem "sequel"
+
+group :development do
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+  gem "guard-test"
+  gem "test-unit", "2.3.2"
+  gem "mocha"
+  gem "sqlite3"
+  gem "yard"
+  gem "rake"
+  gem "versionomy"
+  gem "guard-yard", :platforms => :ruby_19
+end

data/Gemfile.lock

@@ -0,0 +1,44 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    blockenspiel (0.4.3)
+    git (1.2.5)
+    guard (0.6.2)
+      thor (~> 0.14.6)
+    guard-test (0.3.0)
+      guard (>= 0.2.2)
+      test-unit (~> 2.2)
+    guard-yard (1.0.1)
+      guard (>= 0.2.2)
+      yard (>= 0.7.0)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    mocha (0.9.12)
+    rake (0.9.2)
+    rcov (0.9.10)
+    sequel (3.26.0)
+    sqlite3 (1.3.3)
+    test-unit (2.3.2)
+    thor (0.14.6)
+    versionomy (0.4.1)
+      blockenspiel (>= 0.4.1)
+    yard (0.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  guard-test
+  guard-yard
+  jeweler (~> 1.6.4)
+  mocha
+  rake
+  rcov
+  sequel
+  sqlite3
+  test-unit (= 2.3.2)
+  versionomy
+  yard

data/Guardfile

@@ -0,0 +1,12 @@
+guard 'test' do
+  watch(%r{^lib/linkage/runner/([^/]+)\.rb$}) { |m| "test/unit/test_#{m[1]}_runner.rb" }
+  watch(%r{^lib/linkage/([^/]+)\.rb$}) { |m| "test/unit/test_#{m[1]}.rb" }
+  watch(%r{^test/unit/test_.+\.rb$})
+  watch(%r{^test/integration/test_.+\.rb$})
+  watch('lib/linkage/configuration.rb') { "test/unit/test_dataset.rb" }
+  watch('test/helper.rb')  { "test" }
+end
+
+guard 'yard' do
+  watch(%r{lib/[^.].*\.rb$})
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Vanderbilt University
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,64 @@
+# linkage
+
+Linkage is a library for record linkage between one or two database tables.
+
+## Usage
+
+Linkage uses Sequel to talk to databases, so any database that Sequel can
+talk to, Linkage can talk to. You just give Linkage the Sequel-style URI
+and the database table name:
+
+    ds = Linkage::Dataset.new('mysql://example.com/database_name', 'table_name')
+
+To describe a linkage, you use the `Dataset#link_with` method.
+
+    parents = Linkage::Dataset.new('postgres://example.com/foo', 'parents')
+    children = Linkage::Dataset.new('mysql://some-other-host.net/bar', 'children')
+    config = parents.link_with(children) do
+      lhs[:first_name].must == rhs[:parent_first_name]
+      lhs[:last_name].must == rhs[:parent_last_name]
+      lhs[:last_name].must_not == "Smith"  # exclude parents with the last
+                                           # name "Smith"
+    end
+
+Note that the datasets don't have to be in the same database, or even on
+the same machine.
+
+To run a linkage, use a Runner with the resulting configuration from
+`Dataset#link_with`:
+
+    runner = Linkage::SingleThreadedRunner.new(config, 'sqlite://results.db')
+    runner.execute
+
+The runner needs a database URI, since it stores its results in two
+database tables: `groups` and `groups_records`. The `groups` table contains
+all of the unique combinations of values in your datasets, and
+`groups_records` maps records to groups.
+
+You can also link a dataset to itself:
+
+    births = Linkage::Dataset.new('postgres://example.com/hospital_data', 'births')
+    config = births.link_with(births) do
+      lhs[:mother_first_name].must == rhs[:mother_first_name]
+      lhs[:mother_last_name].must == rhs[:mother_last_name]
+    end
+    runner = Linkage::SingleThreadedRunner.new(config, 'sqlite://results.db')
+    runner.execute
+
+The above example would find birth records that have mothers with the same
+name.
+
+## Contributing to linkage
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+
+## Copyright
+
+Copyright (c) 2011 Vanderbilt University. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,58 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "linkage"
+  gem.homepage = "http://github.com/coupler/linkage"
+  gem.license = "MIT"
+  gem.summary = %Q{Sequel-based record linkage}
+  gem.description = %Q{Wraps Sequel to perform record linkage between one or two datasets}
+  gem.email = "jeremy.f.stephens@vanderbilt.edu"
+  gem.authors = ["Jeremy Stephens"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "linkage #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'yard'
+YARD::Rake::YardocTask.new do |t|
+  t.files = ['lib/**/*.rb']
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/lib/linkage.rb

@@ -0,0 +1,15 @@
+require 'pathname'
+require 'sequel'
+
+module Linkage
+end
+
+path = Pathname.new(File.expand_path(File.dirname(__FILE__))) + 'linkage'
+require path + 'utils'
+require path + 'dataset'
+require path + 'runner'
+require path + 'expectation'
+require path + 'field'
+require path + 'group'
+require path + 'import_buffer'
+require path + 'configuration'

data/lib/linkage/configuration.rb

@@ -0,0 +1,178 @@
+module Linkage
+  # {Configuration} is used to configure linkages. When you call
+  # {Dataset#link_with}, the block you supply gets called in the context of
+  # an instance of {Configuration}.
+  #
+  # @example
+  #   dataset_1 = Linkage::Dataset.new("mysql://example.com/database_name", "table_1")
+  #   dataset_2 = Linkage::Dataset.new("mysql://example.com/database_name", "table_2")
+  #   dataset_1.link_with(dataset_2) do
+  #     # this gets run inside of a Configuration instance
+  #   end
+  #
+  # @see Dataset#link_with
+  class Configuration
+    # @private
+    class ExpectationWrapper
+      def initialize(type, field, config)
+        @type = type
+        @field = field
+        @config = config
+        @side = nil
+        @forced_kind = nil
+      end
+
+      Linkage::Expectation::VALID_OPERATORS.each do |op|
+        define_method(op) do |other|
+          case other
+          when FieldWrapper
+            @other = other.field
+            if other.side == @field.side
+              @forced_kind = :filter
+              @side = @field.side
+            end
+          else
+            @other = other
+            @side = @field.side
+          end
+          add_expectation(op)
+        end
+      end
+
+      private
+
+      def add_expectation(operator)
+        klass = Expectation.get(@type)
+        exp = klass.new(operator, @field.field, @other, @forced_kind)
+        @config.add_expectation(exp, @side)
+      end
+    end
+
+    # @private
+    class FieldWrapper
+      attr_reader :field, :side
+      def initialize(field, side, config)
+        @field = field
+        @side = side
+        @config = config
+      end
+
+      def must
+        ExpectationWrapper.new(:must, self, @config)
+      end
+
+      def must_not
+        ExpectationWrapper.new(:must_not, self, @config)
+      end
+    end
+
+    # @private
+    class DatasetWrapper
+      def initialize(dataset, side, config)
+        @dataset = dataset
+        @side = side
+        @config = config
+      end
+
+      def [](field_name)
+        field = @dataset.fields[field_name]
+        if field.nil?
+          raise ArgumentError, "The '#{field_name}' field doesn't exist for that dataset!"
+        end
+        FieldWrapper.new(field, @side, @config)
+      end
+    end
+
+    include Utils
+
+    # @return [Symbol] :self, :dual, or :cross
+    attr_reader :linkage_type
+
+    # @return [Array<Linkage::Expectation>]
+    attr_reader :expectations
+
+    # @return [Linkage::Dataset]
+    attr_reader :dataset_1
+
+    # @return [Linkage::Dataset]
+    attr_reader :dataset_2
+
+    def initialize(dataset_1, dataset_2)
+      @dataset_1 = dataset_1.clone
+      @dataset_2 = dataset_2.clone
+      @expectations = []
+      @linkage_type = dataset_1 == dataset_2 ? :self : :dual
+      @lhs_filters = []
+      @rhs_filters = []
+    end
+
+    def lhs
+      @lhs ||= DatasetWrapper.new(@dataset_1, :lhs, self)
+    end
+
+    def rhs
+      @rhs ||= DatasetWrapper.new(@dataset_2, :rhs, self)
+    end
+
+    # @private
+    def add_expectation(expectation, side = nil)
+      # If the expectation created turns the linkage type from a self to a
+      # cross, then the dataset gets a new id. This is so that
+      # Expectation#apply does the right thing.
+
+      @expectations << expectation
+      if @linkage_type == :self
+        cross = false
+
+        case expectation.kind
+        when :cross
+          cross = true
+        when :filter
+          # If there different filters on both 'sides' of a self-linkage,
+          # it turns into a cross linkage.
+          these_filters, other_filters =
+            case side
+            when :lhs
+              [@lhs_filters, @rhs_filters]
+            when :rhs
+              [@rhs_filters, @lhs_filters]
+            end
+
+          if !other_filters.empty? && !other_filters.include?(expectation)
+            cross = true
+          else
+            these_filters << expectation
+          end
+        end
+
+        if cross
+          @linkage_type = :cross
+          @dataset_2.send(:set_new_id)
+        end
+      end
+    end
+
+    # @private
+    def groups_table_schema
+      schema = []
+
+      # add id
+      schema << [:id, Integer, {:primary_key => true}]
+
+      # add values
+      @expectations.each do |exp|
+        next  if exp.kind == :filter
+
+        merged_type = exp.merged_field.ruby_type
+        schema << [exp.name, merged_type[:type], merged_type[:opts] || {}]
+      end
+
+      schema
+    end
+
+    # @private
+    def inspect
+      to_s
+    end
+  end
+end