coupler 0.0.7-java → 0.0.8-java

data/VERSION

@@ -1 +1 @@
-0.0.7
+0.0.8

data/coupler.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = %q{coupler}
-  s.version = "0.0.7"
+  s.version = "0.0.8"
   s.platform = %q{java}
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jeremy Stephens"]
-  s.date = %q{2011-07-19}
+  s.date = %q{2011-08-16}
   s.default_executable = %q{coupler}
   s.description = %q{Coupler is a (JRuby) desktop application designed to link datasets together}
   s.email = %q{jeremy.f.stephens@vanderbilt.edu}
@@ -59,6 +59,10 @@ Gem::Specification.new do |s|
     "db/migrate/021_add_fields_to_connections.rb",
     "db/migrate/022_remove_database_name_from_resources.rb",
     "db/migrate/023_add_import_jobs.rb",
+    "db/migrate/024_add_error_msg_to_jobs.rb",
+    "db/migrate/025_add_notifications.rb",
+    "db/migrate/026_add_status_to_resources.rb",
+    "db/migrate/027_add_notification_id_to_jobs.rb",
     "features/connections.feature",
     "features/matchers.feature",
     "features/projects.feature",
@@ -86,6 +90,7 @@ Gem::Specification.new do |s|
     "lib/coupler/extensions/imports.rb",
     "lib/coupler/extensions/jobs.rb",
     "lib/coupler/extensions/matchers.rb",
+    "lib/coupler/extensions/notifications.rb",
     "lib/coupler/extensions/projects.rb",
     "lib/coupler/extensions/resources.rb",
     "lib/coupler/extensions/results.rb",
@@ -104,6 +109,7 @@ Gem::Specification.new do |s|
     "lib/coupler/models/job.rb",
     "lib/coupler/models/jobify.rb",
     "lib/coupler/models/matcher.rb",
+    "lib/coupler/models/notification.rb",
     "lib/coupler/models/project.rb",
     "lib/coupler/models/resource.rb",
     "lib/coupler/models/result.rb",
@@ -138,6 +144,7 @@ Gem::Specification.new do |s|
     "test/functional/test_imports.rb",
     "test/functional/test_jobs.rb",
     "test/functional/test_matchers.rb",
+    "test/functional/test_notifications.rb",
     "test/functional/test_projects.rb",
     "test/functional/test_resources.rb",
     "test/functional/test_results.rb",
@@ -158,6 +165,7 @@ Gem::Specification.new do |s|
     "test/unit/models/test_import.rb",
     "test/unit/models/test_job.rb",
     "test/unit/models/test_matcher.rb",
+    "test/unit/models/test_notification.rb",
     "test/unit/models/test_project.rb",
     "test/unit/models/test_resource.rb",
     "test/unit/models/test_result.rb",
@@ -242,6 +250,7 @@ Gem::Specification.new do |s|
     "webroot/views/layout.erb",
     "webroot/views/matchers/form.erb",
     "webroot/views/matchers/list.erb",
+    "webroot/views/notifications/index.erb",
     "webroot/views/projects/form.erb",
     "webroot/views/projects/index.erb",
     "webroot/views/projects/show.erb",

data/db/migrate/024_add_error_msg_to_jobs.rb

@@ -0,0 +1,5 @@
+Sequel.migration do
+  up do
+    add_column :jobs, :error_msg, String, :text => true
+  end
+end

data/db/migrate/025_add_notifications.rb

@@ -0,0 +1,16 @@
+Sequel.migration do
+  up do
+    create_table(:notifications) do
+      primary_key :id
+      String :message
+      String :url
+      TrueClass :seen
+      Integer :import_id
+      Time :created_at
+      Time :updated_at
+    end
+  end
+  down do
+    drop_table(:notifications)
+  end
+end

data/db/migrate/026_add_status_to_resources.rb

@@ -0,0 +1,7 @@
+Sequel.migration do
+  up do
+    [:resources, :resources_versions].each do |name|
+      add_column name, :status, String
+    end
+  end
+end

data/db/migrate/027_add_notification_id_to_jobs.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  up do
+    add_column :jobs, :notification_id, Integer
+  end
+  down do
+    drop_column :jobs, :notification_id
+  end
+end

data/lib/coupler.rb

@@ -21,8 +21,12 @@ require 'json'
 require 'fastercsv'
 require 'carrierwave/sequel'
 require 'mongrel'
-#require 'jdbc/mysql'  # Sequel should load this when it needs to.
-#require 'jdbc/h2'     # Sequel should load this when it needs to.
+
+=begin
+# Sequel will automatically include these as needed
+require 'jdbc/mysql'
+require 'jdbc/h2'
+=end
 
 module Coupler
   def self.environment

data/lib/coupler/base.rb

@@ -25,6 +25,7 @@ module Coupler
     register Extensions::Jobs
     register Extensions::Transformers
     register Extensions::Imports
+    register Extensions::Notifications
     register Extensions::Exceptions
 
     helpers do

data/lib/coupler/extensions.rb

@@ -3,6 +3,8 @@ module Coupler
   end
 end
 
+require File.dirname(__FILE__) + "/extensions/exceptions"
+
 require File.dirname(__FILE__) + "/extensions/connections"
 require File.dirname(__FILE__) + "/extensions/projects"
 require File.dirname(__FILE__) + "/extensions/resources"
@@ -13,4 +15,4 @@ require File.dirname(__FILE__) + "/extensions/results"
 require File.dirname(__FILE__) + "/extensions/jobs"
 require File.dirname(__FILE__) + "/extensions/transformers"
 require File.dirname(__FILE__) + "/extensions/imports"
-require File.dirname(__FILE__) + "/extensions/exceptions"
+require File.dirname(__FILE__) + "/extensions/notifications"

data/lib/coupler/extensions/imports.rb

@@ -9,19 +9,16 @@ module Coupler
 
         app.post "/projects/:project_id/imports" do
           @import = Models::Import.new(params[:import].merge(:project_id => @project.id))
-
-          if @import.save
-            @resource = Models::Resource.new(:import => @import)
-            if @resource.valid?
-              if @import.import!
-                @resource.save
-                redirect("/projects/#{@project.id}/resources/#{@resource.id}")
-              else
-                redirect("/projects/#{@project.id}/imports/#{@import.id}/edit")
-              end
-            end
+          @resource = Models::Resource.new(:name => @import.name, :project_id => @project.id, :status => 'pending')
+          if @import.valid? && @resource.valid?
+            @import.save
+            @resource.import = @import
+            @resource.save
+            Scheduler.instance.schedule_import_job(@import)
+            redirect("/projects/#{@project.id}/resources")
+          else
+            erb(:'imports/new')
           end
-          erb(:'imports/new')
         end
 
         app.get "/projects/:project_id/imports/:id/edit" do
@@ -34,8 +31,8 @@ module Coupler
           @import = Models::Import[:id => params[:id], :project_id => @project.id]
           raise ImportNotFound    unless @import
           @import.repair_duplicate_keys!(params[:delete])
-          @resource = Models::Resource.create(:import => @import)
-          redirect("/projects/#{@project.id}/resources/#{@resource.id}")
+          @import.resource.activate!
+          redirect("/projects/#{@project.id}/resources/#{@import.resource.id}")
         end
       end
     end

data/lib/coupler/extensions/notifications.rb

@@ -0,0 +1,26 @@
+module Coupler
+  module Extensions
+    module Notifications
+      include Models
+
+      def self.registered(app)
+        app.before do
+          Notification.filter(~{:seen => true}, {:url => request.path_info}).update(:seen => true)
+        end
+
+        app.get "/notifications" do
+          @notifications = Notification.order(:created_at).all
+          erb :"notifications/index"
+        end
+
+        app.get "/notifications/unseen.json" do
+          content_type 'application/json'
+          notifications = Notification.filter(~{:seen => true}).order(:created_at).all
+          notifications.collect do |n|
+            { 'id' => n.id, 'message' => n.message, 'url' => n.url, 'created_at' => n.created_at.iso8601 }
+          end.to_json
+        end
+      end
+    end
+  end
+end

data/lib/coupler/models.rb

@@ -16,6 +16,7 @@ module Coupler
     autoload :Scenario,       File.dirname(__FILE__) + "/models/scenario"
     autoload :Transformation, File.dirname(__FILE__) + "/models/transformation"
     autoload :Transformer,    File.dirname(__FILE__) + "/models/transformer"
+    autoload :Notification,   File.dirname(__FILE__) + "/models/notification"
   end
 end
 

data/lib/coupler/models/comparison.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Comparison < Sequel::Model

data/lib/coupler/models/connection.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Connection < Sequel::Model

data/lib/coupler/models/field.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Field < Sequel::Model
@@ -6,13 +5,14 @@ module Coupler
       many_to_one :resource
       one_to_many :transformations, :key => :source_field_id
 
-      def original_column_options
-        { :name => name, :type => db_type, :primary_key => is_primary_key }
-      end
+      TYPES = {
+        'integer'  => {:type => Integer},
+        'string'   => {:type => String, :size => 255},
+        'datetime' => {:type => DateTime}
+      }
 
       def local_column_options
-        { :name => name, :type => final_db_type,
-          :primary_key => is_primary_key }
+        { :name => name, :primary_key => is_primary_key }.merge!(TYPES[final_type])
       end
 
       def final_type

data/lib/coupler/models/import.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Import < Sequel::Model
@@ -15,6 +14,7 @@ module Coupler
                 \d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2} )\z /x
 
       many_to_one :project
+      one_to_one :resource
       plugin :serialization
       serialize_attributes :marshal, :field_types, :field_names
       mount_uploader :data, DataUploader
@@ -50,7 +50,7 @@ module Coupler
         @preview
       end
 
-      def import!
+      def import!(&progress)
         project.local_database do |db|
           column_info = []
           column_names = []
@@ -79,7 +79,9 @@ module Coupler
           buffer = ImportBuffer.new(column_names, ds)
           skip = has_headers
           primary_key_index = field_names.index(primary_key_name)
-          FasterCSV.foreach(data.file.file) do |row|
+          io = File.open(data.file.file, 'rb')
+          csv = FasterCSV.new(io)
+          csv.each do |row|
             if skip
               # skip header if necessary
               skip = false
@@ -97,6 +99,10 @@ module Coupler
             self.has_duplicate_keys = true if num > 1
 
             buffer.add(row)
+
+            if block_given?
+              yield io.pos
+            end
           end
           buffer.flush
 

data/lib/coupler/models/job.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Job < Sequel::Model
@@ -6,6 +5,8 @@ module Coupler
 
       many_to_one :resource
       many_to_one :scenario
+      many_to_one :import
+      many_to_one :notification
 
       def percent_completed
         total > 0 ? completed * 100 / total : 0
@@ -13,31 +14,74 @@ module Coupler
 
       def execute
         Logger.instance.info("Starting job #{id} (#{name})")
+
+        opts = {}
         case name
         when 'transform'
-          update(:status => 'running', :started_at => Time.now, :total => resource.source_dataset_count)
-
-          new_status = 'failed'
-          begin
-            resource.transform! { |n| update(:completed => completed + n) }
-            new_status = 'done'
-          ensure
-            update(:status => new_status, :completed_at => Time.now)
-          end
+          opts[:total] = resource.source_dataset_count
+        when 'import'
+          opts[:total] = import.data.file.size
+        end
 
-        when 'run_scenario'
-          update(:status => 'running', :started_at => Time.now)
+        begin
+          opts[:status] = 'running'
+          opts[:started_at] = Time.now
+          update(opts)
+          send("execute_#{name}")
+        rescue Exception => e
+          message = "%s: %s\n  %s" % [e.class.to_s, e.to_s, e.backtrace.join("\n  ")]
+          update({
+            :status => 'failed',
+            :completed_at => Time.now,
+            :error_msg => message
+          })
+          Logger.instance.error("Job #{id} (#{name}) crashed: #{message}")
+          raise e
+        end
+        self.status = 'done'
+        self.completed_at = Time.now
+        save
+        Logger.instance.info("Job #{id} (#{name}) finished successfully")
+      end
+
+      private
 
-          new_status = 'failed'
-          begin
-            scenario.run!
-            new_status = 'done'
-          ensure
-            update(:status => new_status, :completed_at => Time.now)
+        def execute_transform
+          resource.transform! { |n| update(:completed => completed + n) }
+        end
+
+        def execute_run_scenario
+          scenario.run!
+        end
+
+        def execute_import
+          last = Time.now # don't slam the database
+          result = import.import! do |pos|
+            now = Time.now
+            if now - last >= 1
+              last = now
+              update(:completed => pos)
+            end
+          end
+          if result
+            # NOTE: This is a bug waiting to happen. Import doesn't verify
+            # that it has a resource, but supposedly, it will always have
+            # one. The resource gets created at the same time in the
+            # controller.
+
+            resource = import.resource
+            resource.activate!
+            self.notification = Notification.create({
+              :message => "Import finished successfully",
+              :url => "/projects/#{import.project_id}/resources/#{resource.id}"
+            })
+          else
+            self.notification = Notification.create({
+              :message => "Import finished, but with errors",
+              :url => "/projects/#{import.project_id}/imports/#{import.id}/edit"
+            })
           end
         end
-        Logger.instance.info("Job #{id} (#{name}) finished")
-      end
     end
   end
 end

data/lib/coupler/models/matcher.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Matcher < Sequel::Model

data/lib/coupler/models/notification.rb

@@ -0,0 +1,7 @@
+module Coupler
+  module Models
+    class Notification < Sequel::Model
+      include CommonModel
+    end
+  end
+end

data/lib/coupler/models/project.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Project < Sequel::Model

data/lib/coupler/models/resource.rb

@@ -1,4 +1,3 @@
-pp caller
 module Coupler
   module Models
     class Resource < Sequel::Model
@@ -30,8 +29,6 @@ module Coupler
       def import=(*args)
         result = super
         if new?
-          self.project = import.project
-          self.name = import.name
           self.table_name = "import_#{import.id}"
         end
         result
@@ -106,21 +103,20 @@ module Coupler
         end
       end
 
-      def status
-        if transformed_with.to_s != transformation_ids.join(",") || transformations_dataset.filter("updated_at > ?", transformed_at).count > 0
-          "out_of_date"
-        else
-          "ok"
-        end
-      end
-
       def scenarios
         Scenario.filter(["resource_1_id = ? OR resource_2_id = ?", id, id]).all
       end
 
-      def refresh_fields!
+      def transformations_updated!
+        last_updated_at = transformed_at
+        transformation_ids = []
+
         fields_dataset.update(:local_db_type => nil, :local_type => nil)
         transformations_dataset.order(:position).each do |transformation|
+          transformation_ids << transformation.id
+          if last_updated_at.nil? || transformation.updated_at > last_updated_at
+            last_updated_at = transformation.updated_at
+          end
           if transformation.source_field_id == transformation.result_field_id
             source_field = transformation.source_field
             changes = transformation.field_changes[source_field.id]
@@ -130,6 +126,12 @@ module Coupler
             })
           end
         end
+
+        if transformed_with.to_s != transformation_ids.join(",") || (last_updated_at && last_updated_at > transformed_at)
+          update(:status => "out_of_date")
+        else
+          update(:status => "ok")
+        end
       end
 
       def transform!(&progress)
@@ -137,6 +139,7 @@ module Coupler
         create_local_table!
         _transform(&progress)
         self.update({
+          :status => 'ok',
           :transformed_at => Time.now,
           :transformed_with => t_ids
         })
@@ -161,6 +164,13 @@ module Coupler
         primary_key_name.to_sym
       end
 
+      # Activate resource that was pending until import was completed
+      def activate!
+        set_primary_key
+        create_fields
+        update(:status => "ok")
+      end
+
       private
         def transformation_ids
           transformations_dataset.select(:id).order(:id).all.collect(&:id)
@@ -170,6 +180,15 @@ module Coupler
           Coupler.connection_string("project_#{project.id}")
         end
 
+        def set_primary_key
+          source_database do |db|
+            schema = db.schema(table_name.to_sym)
+            info = schema.detect { |x| x[1][:primary_key] }
+            self.primary_key_name = info[0].to_s
+            self.primary_key_type = info[1][:type].to_s
+          end
+        end
+
         def create_fields
           source_schema.each do |(name, info)|
             add_field({
@@ -233,19 +252,21 @@ module Coupler
           validates_presence [:project_id, :name]
           validates_presence :slug
           validates_unique [:name, :project_id], [:slug, :project_id]
-          validates_presence [:table_name]
-
-          if import.nil? && errors.on(:table_name).nil?
-            source_database do |db|
-              sym = self.table_name.to_sym
-              if !db.tables.include?(sym)
-                errors.add(:table_name, "is invalid")
-              else
-                keys = db.schema(sym).select { |info| info[1][:primary_key] }
-                if keys.empty?
-                  errors.add(:table_name, "doesn't have a primary key")
-                elsif keys.length > 1
-                  errors.add(:table_name, "has too many primary keys")
+
+          if status != 'pending'
+            validates_presence [:table_name]
+            if errors.on(:table_name).nil?
+              source_database do |db|
+                sym = self.table_name.to_sym
+                if !db.tables.include?(sym)
+                  errors.add(:table_name, "is invalid")
+                else
+                  keys = db.schema(sym).select { |info| info[1][:primary_key] }
+                  if keys.empty?
+                    errors.add(:table_name, "doesn't have a primary key")
+                  elsif keys.length > 1
+                    errors.add(:table_name, "has too many primary keys")
+                  end
                 end
               end
             end
@@ -257,11 +278,9 @@ module Coupler
             # NOTE: I'm doing this instead of using before_create because
             # serialization happens in before_save, which gets called before
             # the before_create hook
-            source_database do |db|
-              schema = db.schema(table_name.to_sym)
-              info = schema.detect { |x| x[1][:primary_key] }
-              self.primary_key_name = info[0].to_s
-              self.primary_key_type = info[1][:type].to_s
+            if status != "pending"
+              set_primary_key
+              self.status = "ok"
             end
           end
           super
@@ -269,7 +288,9 @@ module Coupler
 
         def after_create
           super
-          create_fields
+          if status != "pending"
+            create_fields
+          end
         end
 
         def after_destroy