coupler 0.0.1-java

data/lib/coupler/import_buffer.rb

@@ -0,0 +1,48 @@
+module Coupler
+  # This class is used during resource transformation.  Its purpose
+  # is for mass inserts into the local database for speed.
+  class ImportBuffer
+    attr_writer :dataset
+    def initialize(columns, dataset, &progress)
+      @columns = columns
+      @dataset = dataset
+      @mutex = Mutex.new
+      @progress = progress
+      @pending = 0
+      @max_query_size = 1_048_576
+    end
+
+    def add(row)
+      fragment = " " + @dataset.literal(row.is_a?(Hash) ? row.values_at(*@columns) : row) + ","
+      @mutex.synchronize do
+        init_query  if @query.nil?
+        if (@query.length + fragment.length) > @max_query_size
+          flush(false)
+          init_query
+        end
+        @query << fragment
+        @pending += 1
+      end
+    end
+
+    def flush(lock = true)
+      begin
+        @mutex.lock   if lock
+        if @query
+          @dataset.db.run(@query.chomp(","))
+          @progress.call(@pending) if @progress
+          @pending = 0
+          @query = nil
+        end
+      ensure
+        @mutex.unlock if lock
+      end
+    end
+
+    private
+      def init_query
+        @query = String.alloc(@max_query_size)
+        @query << @dataset.insert_sql(@columns, Sequel::LiteralString.new('VALUES'))
+      end
+  end
+end

data/lib/coupler/logger.rb

@@ -0,0 +1,16 @@
+module Coupler
+  class Logger < Delegator
+    include Singleton
+
+    def initialize
+      log_path = Base.settings.log_path
+      Dir.mkdir(log_path)    if !File.exist?(log_path)
+      @logger = ::Logger.new(File.join(log_path, "#{Base.settings.environment}.log"))
+      super(@logger)
+    end
+
+    def __getobj__
+      @logger
+    end
+  end
+end

data/lib/coupler/models/common_model.rb

@@ -0,0 +1,104 @@
+module Coupler
+  module Models
+    module CommonModel
+      module ClassMethods
+        def create!(*args)
+          new(*args).save!
+        end
+
+        def recently_accessed
+          col = columns.include?(:last_accessed_at) ? :last_accessed_at : :updated_at
+          order(col.desc).limit(3).all
+        end
+
+        def as_of_version(id, version)
+          versions_dataset[:current_id => id, :version => version]
+        end
+
+        def as_of_time(id, time)
+          versions_dataset.filter(["current_id = ? AND updated_at <= ?", id, time]).first
+        end
+
+        def versions_table_name
+          "#{table_name}_versions".to_sym
+        end
+
+        def versions_dataset
+          db[versions_table_name]
+        end
+
+        def const_missing(name)
+          Models.const_missing(name)
+        end
+      end
+
+      @@versioned = {}
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.raise_on_save_failure = false
+        base.plugin :validation_helpers
+
+        # decide whether or not to version this model
+        versions_table_name = base.versions_table_name
+        if base.db.tables.include?(versions_table_name)
+          @@versioned[base] = versions_table_name
+          base.send(:attr_accessor, :delete_versions_on_destroy)
+        end
+      end
+
+      def before_create
+        super
+        now = Time.now
+        self[:created_at] = now
+        self[:updated_at] = now
+      end
+
+      def before_update
+        super
+        now = Time.now
+        self[:updated_at] = now
+      end
+
+      def before_save
+        super
+        if @@versioned[self.class] && !@skip_new_version
+          self[:version] = self[:version].nil? ? 1 : self[:version] + 1
+        end
+      end
+
+      def after_save
+        super
+        if @skip_new_version
+          @skip_new_version = nil
+        else
+          if versions_table_name = @@versioned[self.class]
+            dataset = self.db[versions_table_name]
+            hash = self.values.clone
+            hash[:current_id] = hash.delete(:id)
+            dataset.insert(hash)
+          end
+        end
+      end
+
+      def after_destroy
+        super
+        if @delete_versions_on_destroy && (versions_table_name = @@versioned[self.class])
+          dataset = self.db[versions_table_name]
+          dataset.filter(:current_id => id).delete
+        end
+      end
+
+      def save!(*args)
+        if !save(*args)
+          raise "couldn't save: " + errors.full_messages.join("; ")
+        end
+        self
+      end
+
+      def touch!
+        @skip_new_version = true
+        update(:last_accessed_at => Time.now)
+      end
+    end
+  end
+end

data/lib/coupler/models/comparison.rb

@@ -0,0 +1,166 @@
+module Coupler
+  module Models
+    class Comparison < Sequel::Model
+      include CommonModel
+
+      OPERATORS = {
+        "equals" => "=",
+        "does_not_equal" => "!=",
+        "greater_than" => ">",
+        "less_than" => "<",
+      }
+      TYPES = %w{field integer string}
+
+      many_to_one :matcher
+      plugin :serialization, :marshal, :raw_lhs_value, :raw_rhs_value
+
+      def lhs_rhs_value(name)
+        case self[:"#{name}_type"]
+        when "field"
+          Field[:id => send("raw_#{name}_value")]
+        else
+          send("raw_#{name}_value")
+        end
+      end
+      def lhs_value; lhs_rhs_value("lhs"); end
+      def rhs_value; lhs_rhs_value("rhs"); end
+
+      def lhs_rhs_label(name)
+        case self[:"#{name}_type"]
+        when "field"
+          field = lhs_rhs_value(name)
+          result = field.name
+          resource_name = field.resource.name
+          if self[:"#{name}_which"]
+            resource_name += %{<span class="sup">#{self[:"#{name}_which"]}</span>}
+          end
+          result += " (#{resource_name})"
+        else
+          lhs_rhs_value(name).inspect
+        end
+      end
+      def lhs_label; lhs_rhs_label("lhs"); end
+      def rhs_label; lhs_rhs_label("rhs"); end
+
+      def fields
+        result = []
+        result << lhs_value if lhs_type == 'field'
+        result << rhs_value if rhs_type == 'field'
+        result
+      end
+
+      def operator_symbol
+        OPERATORS[operator]
+      end
+
+      def apply(dataset, which = nil)
+        lhs = lhs_type == 'field' ? lhs_value.name.to_sym : lhs_value
+        rhs = rhs_type == 'field' ? rhs_value.name.to_sym : rhs_value
+        if !blocking?
+          filters = []
+          tmp = dataset.opts
+          opts = {
+            :select => tmp[:select] ? tmp[:select].dup : [],
+            :order  => tmp[:order]  ? tmp[:order].dup  : []
+          }
+
+          fields =
+            case which
+            when nil then lhs == rhs ? [lhs] : [lhs, rhs]
+            when 0   then [lhs]
+            when 1   then [rhs]
+            end
+          fields.each_with_index do |field, i|
+            index = i == 0 ? 0 : -1
+
+            # NOTE: This assumes that the presence of a field name in the
+            # select array implies that the filters for it are already in
+            # place.  I don't want to go searching through Sequel's filter
+            # expressions to find out what's in there.
+            if !opts[:select].include?(field)
+              opts[:select].push(field)
+              opts[:order].push(field)
+              opts[:modified] = true
+              filters.push(~{field => nil})
+            end
+          end
+          if opts.delete(:modified)
+            dataset = dataset.clone(opts).filter(*filters)
+          end
+        else
+          # Figure out which side to apply this comparison to.
+          tmp_which = nil
+          if !which.nil?
+            if lhs_type == 'field' && rhs_type == 'field'
+              if lhs_which == rhs_which
+                tmp_which = lhs_which == 1 ? 0 : 1
+              else
+                raise "unsupported" # FIXME
+              end
+            elsif lhs_type == 'field'
+              tmp_which = lhs_which == 1 ? 0 : 1
+            elsif rhs_type == 'field'
+              tmp_which = rhs_which == 1 ? 0 : 1
+            else
+              # Doesn't matter.  Apply to either side.
+            end
+          end
+
+          if which.nil? || tmp_which.nil? || which == tmp_which
+            expr = Sequel::SQL::BooleanExpression.new(operator_symbol.to_sym, lhs, rhs)
+            dataset = dataset.filter(expr)
+          end
+        end
+        dataset
+      end
+
+      def blocking?
+        lhs_type != 'field' || rhs_type != 'field' || lhs_which == rhs_which || operator != 'equals'
+      end
+
+      def cross_match?
+        lhs_type == 'field' && rhs_type == 'field' && lhs_which != rhs_which && lhs_value.id != rhs_value.id && lhs_value.resource_id == rhs_value.resource_id
+      end
+
+      private
+        def coerce_value(type, value)
+          case type
+          when "field", "integer"
+            value.to_i
+          else
+            value
+          end
+        end
+
+        def before_validation
+          super
+          self.lhs_which ||= 1  if lhs_type == 'field'
+          self.rhs_which ||= 2  if rhs_type == 'field'
+        end
+
+        def validate
+          super
+          validates_presence [:raw_lhs_value, :raw_rhs_value]
+          validates_includes TYPES, [:lhs_type, :rhs_type]
+          validates_includes OPERATORS.keys, :operator
+          validates_includes [1, 2], :lhs_which   if lhs_type == 'field'
+          validates_includes [1, 2], :rhs_which   if rhs_type == 'field'
+
+          if lhs_type == 'field' && rhs_type == 'field' && (lhs_field = lhs_value) && (rhs_field = rhs_value)
+            if lhs_field[:type] != rhs_field[:type]
+              errors.add(:base, "Comparing fields of different types is currently disallowed.")
+            end
+            if lhs_which != rhs_which && operator != 'equals'
+              errors.add(:operator, "is invalid; can't compare fields with anything but equals at the moment.")
+            end
+          end
+        end
+
+        def before_save
+          self.raw_lhs_value = coerce_value(lhs_type, raw_lhs_value)
+          self.raw_rhs_value = coerce_value(rhs_type, raw_rhs_value)
+          super
+        end
+    end
+  end
+end

data/lib/coupler/models/connection.rb

@@ -0,0 +1,59 @@
+module Coupler
+  module Models
+    class Connection < Sequel::Model
+      include CommonModel
+
+      ADAPTERS = [%w{mysql MySQL h2 H2}]
+
+      one_to_many :resources
+
+      def database(&block)
+        Sequel.connect(connection_string, {
+          :loggers => [Coupler::Logger.instance],
+          :max_connections => 20
+        }, &block)
+      end
+
+      def deletable?
+        resources_dataset.count == 0
+      end
+
+      private
+        def connection_string
+          case adapter
+          when 'mysql'
+            misc = '&zeroDateTimeBehavior=convertToNull'
+            "jdbc:mysql://%s:%d/%s?user=%s&password=%s%s" % [
+              host, port, database_name, username, password, misc
+            ]
+          when 'h2'
+            "jdbc:h2:#{path}"
+          end
+        end
+
+        def before_validation
+          super
+          self.slug ||= name.downcase.gsub(/\s+/, "_") if name
+        end
+
+        def validate
+          super
+          validates_presence :name
+          validates_unique :name, :slug
+
+          begin
+            database { |db| db.test_connection }
+          rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError => e
+            errors.add(:base, "Couldn't connect to the database")
+          end
+        end
+
+        def before_destroy
+          super
+
+          # Prevent destruction of connections in use by resources.
+          deletable?
+        end
+    end
+  end
+end

data/lib/coupler/models/field.rb

@@ -0,0 +1,55 @@
+module Coupler
+  module Models
+    class Field < Sequel::Model
+      include CommonModel
+      many_to_one :resource
+      one_to_many :transformations, :key => :source_field_id
+
+      def original_column_options
+        { :name => name, :type => db_type, :primary_key => is_primary_key }
+      end
+
+      def local_column_options
+        { :name => name, :type => final_db_type,
+          :primary_key => is_primary_key }
+      end
+
+      def final_type
+        local_type || self[:type]
+      end
+
+      def final_db_type
+        local_db_type || db_type
+      end
+
+      def scenarios_dataset
+        marshalled_id = [Marshal.dump(id)].pack('m')
+        Scenario.
+          select(:scenarios.*).
+          filter({:project_id => resource.project_id} & ({:resource_1_id => resource_id} | {:resource_2_id => resource_id})).
+          join(Matcher, :scenario_id => :id).
+          join(Comparison, :matcher_id => :id).
+          filter({:lhs_type => 'field', :raw_lhs_value => marshalled_id} | {:rhs_type => 'field', :raw_rhs_value => marshalled_id})
+      end
+
+      def name_sym
+        @name_sym ||= name.to_sym
+      end
+
+      private
+        def validate
+          super
+          validates_presence [:name, :resource_id]
+          validates_unique [:name, :resource_id]
+        end
+
+        def before_save
+          super
+          case is_primary_key
+          when TrueClass, 1
+            self.is_selected = 1
+          end
+        end
+    end
+  end
+end

data/lib/coupler/models/import.rb

@@ -0,0 +1,238 @@
+module Coupler
+  module Models
+    class Import < Sequel::Model
+      include CommonModel
+
+      # NOTE: yoinked from FasterCSV
+      # A Regexp used to find and convert some common Date formats.
+      DateMatcher     = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4} |
+                                \d{4}-\d{2}-\d{2} )\z /x
+      # A Regexp used to find and convert some common DateTime formats.
+      DateTimeMatcher =
+        / \A(?: (\w+,?\s+)?\w+\s+\d{1,2}\s+\d{1,2}:\d{1,2}:\d{1,2},?\s+\d{2,4} |
+                \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2} )\z /x
+
+      many_to_one :project
+      plugin :serialization
+      serialize_attributes :marshal, :field_types, :field_names
+      mount_uploader :data, DataUploader
+
+      def data=(value)
+        result = super
+        self.name ||= File.basename(data.file.original_filename).sub(/\.\w+?$/, "").gsub(/[_-]+/, " ").capitalize
+        discover_fields
+        result
+      end
+
+      def primary_key_sym
+        primary_key_name.to_sym
+      end
+
+      def table_name
+        :"import_#{id}"
+      end
+
+      def preview
+        if @preview.nil?
+          @preview = []
+          FasterCSV.open(data.file.file) do |csv|
+            csv.rewind
+            csv.shift   if self.has_headers
+            50.times do |i|
+              row = csv.shift
+              break if row.nil?
+              @preview << row
+            end
+          end
+        end
+        @preview
+      end
+
+      def import!
+        project.local_database do |db|
+          column_info = []
+          column_names = []
+          column_types = []
+          field_names.each_with_index do |name, i|
+            name_sym = name.to_sym
+            column_names << name_sym
+            column_types << {
+              :name => name_sym,
+              :type =>
+                case field_types[i]
+                when 'integer' then Integer
+                when 'string' then String
+                end,
+              :null => !(name == primary_key_name)
+            }
+          end
+          column_names << :dup_key_count
+          column_types << {:name => :dup_key_count, :type => Integer}
+          db.create_table!(table_name) do
+            columns.push(*column_types)
+          end
+
+          ds = db[table_name]
+          key_frequencies = Hash.new { |h, k| h[k] = 0 }
+          buffer = ImportBuffer.new(column_names, ds)
+          skip = has_headers
+          primary_key_index = field_names.index(primary_key_name)
+          FasterCSV.foreach(data.file.file) do |row|
+            if skip
+              # skip header if necessary
+              skip = false
+              next
+            end
+
+            key = row[primary_key_index]
+            num = key_frequencies[key] += 1
+            row.push(num > 1 ? num : nil)
+            self.has_duplicate_keys = true if num > 1
+
+            buffer.add(row)
+          end
+          buffer.flush
+
+          primary_key = self.primary_key_sym
+          if has_duplicate_keys
+            # flag duplicate primary keys
+            key_frequencies.each_pair do |key, count|
+              next  if count == 1
+              ds.filter(primary_key => key, :dup_key_count => nil).update(:dup_key_count => 1)
+            end
+          else
+            # alter table to set primary key
+            db.alter_table(table_name) do
+              drop_column(:dup_key_count)
+              add_primary_key([primary_key])
+            end
+          end
+        end
+        update(:occurred_at => Time.now)
+        !has_duplicate_keys
+      end
+
+      def dataset
+        project.local_database do |db|
+          yield(db[table_name])
+        end
+      end
+
+      def repair_duplicate_keys!(rows_to_remove = nil)
+        pkey = primary_key_sym
+        project.local_database do |db|
+          ds = db[table_name]
+          if rows_to_remove
+            filtered_ds = nil
+            rows_to_remove.each_pair do |key, dups|
+              hsh = {pkey => key, :dup_key_count => dups}
+              filtered_ds = filtered_ds ? filtered_ds.or(hsh) : ds.filter(hsh)
+            end
+            filtered_ds.delete if filtered_ds
+          end
+
+          # only reassign keys if there is more than 1 duplicate per key
+          keys = ds.group(pkey).having { count(pkey) > 1 }.select_map(pkey)
+
+          current_key = nil
+          next_key = ds.order(pkey).last[pkey].next
+          ds.filter(pkey => keys).order(:dup_key_count).each do |row|
+            # skip the first one, since it'll retain the key
+            if current_key != row[pkey]
+              current_key = row[pkey]
+            else
+              ds.filter(pkey => row[pkey], :dup_key_count => row[:dup_key_count]).
+                update(pkey => next_key)
+              next_key = next_key.next
+            end
+          end
+
+          db.alter_table(table_name) do
+            drop_column(:dup_key_count)
+            add_primary_key([pkey])
+          end
+        end
+      end
+
+      private
+        def discover_fields
+          FasterCSV.open(data.file.file) do |csv|
+            csv.rewind
+
+            count = 0
+            types = []
+            type_counts = []
+            headers = csv.shift
+            if headers.any? { |h| h !~ /[A-Za-z_$]/ }
+              row = headers
+              headers = nil
+              self.has_headers = false
+            else
+              self.has_headers = true
+              headers.each_with_index do |name, i|
+                if name =~ /^id$/i
+                  self.primary_key_name = name
+                end
+              end
+              row = csv.shift
+            end
+
+            while row && count < 50
+              row.each_with_index do |value, i|
+                hash = type_counts[i] ||= {}
+                type =
+                  case value
+                  when /^\d+$/ then 'integer'
+                  else 'string'
+                  end
+                hash[type] = (hash[type] || 0) + 1
+              end
+              row = csv.shift
+              count += 1
+            end
+
+            type_counts.each_with_index do |type_count, i|
+              types[i] = type_count.max { |a, b| a[1] <=> b[1] }[0]
+            end
+
+            self.field_types = types
+            self.field_names = headers
+          end
+        end
+
+        def validate
+          super
+
+          validates_presence :project_id
+          if project_id
+            # don't allow import to have the same name as an already existing resource
+            if project.resources_dataset.filter(:name => name).count > 0
+              errors.add(:name, "is already taken")
+            end
+          end
+          validates_presence [:field_names, :primary_key_name]
+          if field_names.is_a?(Array)
+            validates_includes field_names, [:primary_key_name]
+
+            expected = field_types.length
+            if field_names.length != expected
+              errors.add(:field_names, "must be of length #{expected}")
+            end
+
+            # check for duplicate field names
+            duplicates = {}
+            field_names.inject(Hash.new(0)) do |hash, field_name|
+              num = hash[field_name] += 1
+              duplicates[field_name] = num  if num > 1
+              hash
+            end
+            if !duplicates.empty?
+              message = "have duplicates (%s)" %
+                duplicates.inject("") { |s, (k, v)| s + "#{k} x #{v}, " }.chomp(", ")
+              errors.add(:field_names, message)
+            end
+          end
+        end
+    end
+  end
+end