coupler 0.0.1-java

data/lib/coupler/models/scenario/runner.rb

@@ -0,0 +1,357 @@
+module Coupler
+  module Models
+    class Scenario
+      class Runner
+        LIMIT = 10000
+
+        def initialize(parent, &progress)
+          @parent = parent
+          @progress = progress
+          @matcher = parent.matcher
+          @type = parent.linkage_type
+          if @type == 'cross-linkage'
+            @resources = [parent.resource_1, parent.resource_1]
+          else
+            @resources = parent.resources
+          end
+          @run_number = @parent.run_count + 1
+          @mutex = Mutex.new
+          @group_number = 0
+          setup_pairs
+          create_tables
+        end
+
+        def run!
+          @parent.local_database do |scenario_db|
+            @groups_dataset = scenario_db[@groups_table_name]
+            @groups_buffer = ImportBuffer.new(@groups_column_names, @groups_dataset)
+            @join_dataset = scenario_db[@join_table_name]
+            @join_buffer = ImportBuffer.new([:record_id, :which, :group_id], @join_dataset)
+
+            # Group records for each dataset.  This step is the same for both
+            # self-linkage and dual-linkage.  However, this is the only step
+            # for self-linkage.  Dual-linkage requires another pass after
+            # this.
+            #
+            # Cross-matching on a single dataset is treated as a dual-linkage.
+            #
+            @pairs = @phase_one_pairs
+            tw = ThreadsWait.new
+            databases_to_close = []
+            @resources.each_with_index do |resource, i|
+              dataset = resource.final_dataset
+              databases_to_close << dataset.db
+              primary_key = resource.primary_key_sym
+              which = @type == 'self-linkage' ? nil : i
+              resource_thread = phase_one_thread(dataset, primary_key, which)
+              tw.join_nowait(resource_thread)
+            end
+            tw.all_waits
+            databases_to_close.each do |db|   # tidy up
+              db.disconnect
+              ::Sequel::DATABASES.delete(db)
+            end
+
+            if @type != 'self-linkage'
+              # Phase 2!
+              secondary_groups_ds = scenario_db[@secondary_groups_table_name].order(:group_1_id, :group_2_id)
+              @join_buffer = ImportBuffer.new([:group_1_id, :group_2_id], secondary_groups_ds)
+              @pairs = @phase_two_pairs
+              phase_two(@groups_dataset, :id)
+              @join_buffer.flush
+
+              # Update groups and groups_records
+              tw = ThreadsWait.new
+              count = secondary_groups_ds.count
+              offset = 0
+              last_group_id = @group_number
+              while offset < count
+                dataset = secondary_groups_ds.limit(10, offset)
+                offset += 10
+                dataset.each do |row|
+                  thread = Thread.new(row[:group_1_id], row[:group_2_id]) do |group_1_id, group_2_id|
+                    new_group_id = get_next_group_id
+                    @join_dataset.filter(:group_id => [group_1_id, group_2_id]).update(:group_id => new_group_id)
+                    @groups_dataset.filter(:id => group_1_id).update(:id => new_group_id)
+                    @groups_dataset.filter(:id => group_2_id).delete
+                  end
+                  thread.abort_on_exception = true
+                  tw.join_nowait(thread)
+                end
+                tw.all_waits
+              end
+
+              # Clean up groups and records that don't match
+              @groups_dataset.filter(:id <= last_group_id).delete
+              @join_dataset.filter(:group_id <= last_group_id).delete
+
+              # Don't need the secondary table anymore
+              scenario_db.drop_table(@secondary_groups_table_name)
+            end
+
+            # Calculate some summary stats
+            @join_dataset.group_and_count(:group_id, :which).each do |row|
+              col = row[:which] ? :"resource_#{row[:which]+1}_count" : :"resource_1_count"
+              @groups_dataset.filter(:id => row[:group_id]).update(col => row[:count])
+            end
+          end
+        end
+
+        private
+          def setup_pairs
+            @field_pairs = []
+            @phase_one_pairs = []
+            @phase_two_pairs = @type == 'self-linkage' ? nil : []
+            @matcher.comparisons.each do |comparison|
+              if !comparison.blocking?
+                fields = comparison.fields
+                @field_pairs << fields
+                @phase_one_pairs.push(fields.collect(&:name_sym))
+                if @phase_two_pairs
+                  pair_sym = :"pair_#{@phase_two_pairs.length}"
+                  @phase_two_pairs.push([pair_sym, pair_sym])
+                end
+              end
+            end
+          end
+
+          def create_tables
+            # Yes, this could be done during setup_pairs, but I think
+            # this is more clean.  Also, there will only be very few pairs.
+            groups_columns = [
+              {:name => :id, :type => Integer, :primary_key => true}
+            ]
+            @group_value_fields = []
+            @field_pairs.each_with_index do |(field_1, field_2), i|
+              type_1 = field_1.local_column_options[:type]
+              type_2 = field_2.local_column_options[:type]
+              if type_1 != type_2
+                # FIXME!
+                raise "BOOM!!"
+              end
+              @group_value_fields.push([field_1.name_sym, field_2.name_sym])
+              groups_columns.push({:name => :"pair_#{i}", :type => type_1})
+            end
+            @groups_column_names = groups_columns.collect { |c| c[:name] }
+            @groups_table_name = :"groups_#{@run_number}"
+
+            # Add extra columns to the groups table for summary stats
+            @resources.length.times do |i|
+              groups_columns << {:name => :"resource_#{i+1}_count", :type => Integer}
+            end
+
+            key_types = @resources.collect { |r| r.primary_key_type }.uniq
+            record_id_type = key_types.length == 1 ? key_types[0] : String
+            @join_table_name = :"groups_records_#{@run_number}"
+
+            @parent.local_database do |scenario_db|
+              scenario_db.create_table!(@groups_table_name) do
+                columns.push(*groups_columns)
+              end
+              scenario_db.create_table!(@join_table_name) do
+                column :record_id, record_id_type
+                Integer :which
+                Integer :group_id, :index => true
+                index [:group_id, :which, :record_id]   # speedy filtering when showing results
+              end
+              if @type != 'self-linkage'
+                # Need another groups table
+                @secondary_groups_table_name = :"groups_groups_#{@run_number}"
+                scenario_db.create_table!(@secondary_groups_table_name) do
+                  Integer :group_1_id
+                  Integer :group_2_id
+                end
+              end
+            end
+          end
+
+          def phase_one_thread(dataset, primary_key, which)
+            thread = Thread.new do
+              # Apply filters and what not from comparisons
+              dataset = dataset.select(primary_key)
+              @matcher.comparisons.each do |comparison|
+                dataset = comparison.apply(dataset, which)
+              end
+              dataset = dataset.order_more(primary_key)
+
+              # Do the work
+              local_tw = ThreadsWait.new
+              threads = []
+              count = dataset.count
+              segments = count / LIMIT
+              segments += 1  if count % LIMIT > 0
+
+              segments.times do |segment_num|
+                ds = dataset.limit(LIMIT, LIMIT * segment_num)
+                thread = Thread.new do
+                  prev_row = nil
+                  group_id = nil
+
+                  ds.each_with_index do |row, row_num|
+                    if row_num > 0
+                      result = compare_rows(prev_row, row, which)
+                      if (result || which) && group_id.nil?
+                        # If `which` is not nil, that means we're in the first
+                        # stage of a dual or cross linkage.  So, we should save
+                        # groups that only have 1 record in them.
+                        group_id = create_group(prev_row, which)
+                        @join_buffer.add([prev_row[primary_key], which, group_id])
+                      end
+                      if result
+                        @join_buffer.add([row[primary_key], which, group_id])
+                      else
+                        group_id = nil
+                      end
+
+                      # This stores the first record of this segment in order
+                      # to check it against the last record of the previous
+                      # segment.  I don't like putting this inside the loop
+                      # really, but it's better than making another database
+                      # query.
+                      if segment_num > 0 && row_num == 1
+                        Thread.current[:head] = { :row => prev_row, :group_id => group_id }
+                      end
+                    end
+                    prev_row = row
+                  end
+                  if which && group_id.nil?
+                    # See above comment about `which`
+                    group_id = create_group(prev_row, which)
+                    @join_buffer.add([prev_row[primary_key], which, group_id])
+                  end
+
+                  # This stores the last record of this segment in order to
+                  # check it against the first record of the next segment.
+                  if segment_num < (segments - 1)
+                    Thread.current[:tail] = { :row => prev_row, :group_id => group_id }
+                  end
+                end
+                thread.abort_on_exception = true
+                threads << thread
+                local_tw.join_nowait(thread)
+                local_tw.next_wait    if local_tw.threads.length == 10
+              end
+              local_tw.all_waits
+              @groups_buffer.flush
+              @join_buffer.flush
+
+              # Compare heads and tails from segments
+              if segments > 1
+                head = tail = nil
+                threads.each do |thread|
+                  if tail
+                    head = thread[:head]
+                    if compare_rows(tail, head, which)
+                      @join_dataset.filter({
+                        :group_id => head[:group_id]
+                      }).update(:group_id => tail[:group_id])
+                      @groups_dataset.filter({
+                        :id => head[:group_id]
+                      }).delete
+                    end
+                  end
+                  tail = thread[:tail]
+                end
+              end
+            end
+            thread.abort_on_exception = true
+            thread
+          end
+
+          def phase_two(dataset, primary_key)
+            # Add sorting only
+            @pairs.each do |fields|
+              dataset = dataset.order_more(*fields.uniq)
+            end
+            dataset = dataset.order_more(primary_key)
+
+            local_tw = ThreadsWait.new
+            threads = []
+            count = dataset.count
+            segments = count / LIMIT
+            segments += 1  if count % LIMIT > 0
+
+            segments.times do |segment_num|
+              ds = dataset.limit(LIMIT, LIMIT * segment_num)
+              thread = Thread.new do
+                prev_result = nil
+                prev_row    = nil
+                skip        = false
+
+                ds.each_with_index do |row, row_num|
+                  if row_num > 0 && !skip
+                    result = compare_rows(prev_row, row)
+                    # There should only be one-to-one matches here.
+                    if result
+                      @join_buffer.add([prev_row[primary_key], row[primary_key]])
+                      skip = true  # Skip comparing this row to the next
+                    end
+
+                    if segment_num > 0 && row_num == 1 && !result
+                      Thread.current[:head] = { :row => prev_row }
+                    end
+                  elsif skip
+                    skip = false
+                  end
+                  prev_row = row
+                  prev_result = result
+                end
+                if segment_num < (segments - 1) && !prev_result
+                  Thread.current[:tail] = { :row => prev_row }
+                end
+              end
+              thread.abort_on_exception = true
+              threads << thread
+              local_tw.join_nowait(thread)
+              local_tw.next_wait    if local_tw.threads.length == 10
+            end
+            local_tw.all_waits
+            @join_buffer.flush
+
+            # Compare heads and tails from segments
+            if threads.length > 1
+              head = tail = nil
+              threads.each do |thread|
+                if tail
+                  head = thread[:head]
+                  if head && compare_rows(tail[:row], head[:row])
+                    @join_buffer.add([tail[:row][primary_key], head[:row][primary_key]])
+                  end
+                end
+                tail = thread[:tail]
+              end
+            end
+          end
+
+          def compare_rows(row_1, row_2, which = nil)
+            values = []
+            @pairs.each do |fields|
+              value_1 = row_1[fields[which || 0]]
+              value_2 = row_2[fields[which || 1]]
+              if value_1 == value_2
+                values << value_1
+              else
+                values = nil
+                break
+              end
+            end
+            values
+          end
+
+          def create_group(row, which)
+            group_id = get_next_group_id
+            group_row = [group_id]
+            @group_value_fields.each do |fields|
+              group_row.push(row[fields[which || 0]])
+            end
+            @groups_buffer.add(group_row)
+            group_id
+          end
+
+          def get_next_group_id
+            @mutex.synchronize { @group_number += 1 }
+          end
+      end
+    end
+  end
+end

data/lib/coupler/models/scenario.rb

@@ -0,0 +1,115 @@
+module Coupler
+  module Models
+    class Scenario < Sequel::Model
+      class NoMatcherError < Exception; end
+      class ResourcesOutOfDateError < Exception; end
+
+      include CommonModel
+      include Jobify
+
+      attr_writer :resource_ids
+      many_to_one :project
+      many_to_one :resource_1, :class => "Coupler::Models::Resource"
+      many_to_one :resource_2, :class => "Coupler::Models::Resource"
+      one_to_one :matcher
+      one_to_many :results
+
+      def self.count_by_project
+        dataset.naked.group_and_count(:project_id).to_hash(:project_id, :count)
+      end
+
+      def status
+        if matcher.nil?
+          "no_matcher"
+        elsif resources.any? { |r| r.status == "out_of_date" }
+          "resources_out_of_date"
+        else
+          "ok"
+        end
+      end
+
+      def resources
+        if resource_1
+          resource_2 ? [resource_1, resource_2] : [resource_1]
+        else
+          []
+        end
+      end
+
+      def local_database(&block)
+        Sequel.connect(local_connection_string, {
+          :loggers => [Coupler::Logger.instance],
+          :max_connections => 50,
+          :pool_timeout => 60
+        }, &block)
+      end
+
+      def run!(&progress)
+        case status
+        when 'no_matcher'             then raise NoMatcherError
+        when 'resources_out_of_date'  then raise ResourcesOutOfDateError
+        end
+
+        runner = Runner.new(self, &progress)
+        runner.run!
+
+        update(:run_count => run_count + 1, :last_run_at => Time.now)
+        result = Result.new(:scenario => self, :run_number => run_count)
+        result.save
+      end
+
+      def set_linkage_type
+        if resource_1
+          if resource_2
+            self.linkage_type = "dual-linkage"
+          elsif matcher && matcher.cross_match?
+            self.linkage_type = "cross-linkage"
+          else
+            self.linkage_type = "self-linkage"
+          end
+        else
+          self.linkage_type = "N/A"
+        end
+      end
+
+      private
+
+        def local_connection_string
+          Base.connection_string("scenario_#{id}")
+        end
+
+        def before_validation
+          super
+          if @resource_ids.is_a?(Array)
+            objects = project.resources_dataset.filter(:id => @resource_ids[0..1].compact).all
+            self.resource_1_id = objects[0].nil? ? nil : objects[0].id
+            self.resource_2_id = objects[1].nil? ? nil : objects[1].id
+          end
+        end
+
+        def validate
+          super
+          validates_presence :name
+          validates_unique [:name, :project_id]
+          if resource_1_id.nil?
+            errors.add(:base, "At least one resource is required")
+          end
+        end
+
+        def before_create
+          super
+          self.slug ||= name.downcase.gsub(/\s+/, "_")
+          set_linkage_type
+        end
+
+        # There is currently no plan to let a user change resources
+        # after a scenario has been created.
+        #def before_update
+          #super
+          #set_linkage_type
+        #end
+    end
+  end
+end
+
+require File.join(File.dirname(__FILE__), 'scenario', 'runner')

data/lib/coupler/models/transformation.rb

@@ -0,0 +1,117 @@
+module Coupler
+  module Models
+    class Transformation < Sequel::Model
+      include CommonModel
+      many_to_one :resource
+      many_to_one :source_field, :class => Field
+      many_to_one :result_field, :class => Field
+      many_to_one :transformer
+
+      plugin :nested_attributes
+      nested_attributes :result_field
+      nested_attributes :transformer, :destroy => false
+
+      def transform(data)
+        transformer.transform(data, {
+          :in => source_field.name.to_sym,
+          :out => result_field.name.to_sym
+        })
+      end
+
+      def field_changes
+        transformer.field_changes(source_field)
+      end
+
+      # NOTE: The fact that the aliased name doesn't have an = at the end is
+      # important.  Ruby methods with names that have = at the end always
+      # return the RHS value, regardless of what the method actually returns.
+      # The only way to grab the associated object that gets created from the
+      # nested attributes methods is by fetching the return value.
+      #
+      alias :original_result_field_attributes :result_field_attributes=
+      def result_field_attributes=(h)
+        @staged_result_field = self.original_result_field_attributes(h.merge({
+          :is_generated => true
+        }))
+      end
+
+      def deletable?
+        position == self.class.max(:position) &&
+          (result_field.nil? || !result_field.is_generated || result_field.scenarios_dataset.count == 0)
+      end
+
+      private
+        def before_validation
+          super
+          if source_field_id && !result_field_id && !@staged_result_field
+            self.result_field_id = source_field_id
+          end
+
+          if @staged_result_field && transformer && source_field && resource_id
+            hash = transformer.field_changes(source_field).values[0]
+            if hash.empty?
+              hash.update({
+                :type => source_field[:type],
+                :db_type => source_field[:db_type]
+              })
+            end
+            hash[:resource_id] = resource_id
+            @staged_result_field.set(hash)
+          end
+        end
+
+        def validate
+          super
+          validates_presence [:resource_id, :source_field_id]
+          if transformer.nil?
+            errors.add(:transformer_id, "is not present")
+          end
+          if errors.empty?
+            source_field = resource.fields_dataset[:id => source_field_id]
+            if source_field.nil?
+              errors.add(:source_field_id, "is invalid")
+            else
+              if transformer.allowed_types.is_a?(Array) && !transformer.allowed_types.include?(source_field.final_type)
+                errors.add(:base, "#{transformer.name} cannot transform type '#{source_field.final_type}'")
+              end
+            end
+
+            if !@staged_result_field
+              result_field = result_field_id ? resource.fields_dataset[:id => result_field_id] : nil
+              if result_field.nil?
+                errors.add(:result_field_id, "is invalid")
+              end
+            end
+          end
+        end
+
+        def before_create
+          super
+          self.position ||= self.class.filter(:resource_id => resource_id).count + 1
+        end
+
+        def after_save
+          super
+          resource.refresh_fields!
+        end
+
+        def before_destroy
+          # Prevent all but the last transformation from being destroyed
+          #
+          # TODO: I'm putting off programming the complex logic required to
+          # enable deletion from the middle of a transformation stack.
+          #
+          super
+          deletable?
+        end
+
+        def after_destroy
+          super
+          if result_field && result_field.is_generated && self.class.filter(:result_field_id => result_field.id).count == 0
+            result_field.destroy
+          end
+          resource.refresh_fields!  if resource
+        end
+    end
+  end
+end

data/lib/coupler/models/transformer/runner.rb

@@ -0,0 +1,28 @@
+module Coupler
+  module Models
+    class Transformer
+      class Runner
+        instance_methods.each do |m|
+          undef_method m unless m =~ /^__|^instance_eval$/
+        end
+
+        def initialize(code, input)
+          @input = input
+          @code = code
+        end
+
+        def run
+          instance_eval(@code, __FILE__, __LINE__)
+        end
+
+        def value
+          @input
+        end
+
+        def method_missing(name)
+          raise NoMethodError
+        end
+      end
+    end
+  end
+end