coupler 0.0.7-java → 0.0.8-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/coupler.gemspec +11 -2
- data/db/migrate/024_add_error_msg_to_jobs.rb +5 -0
- data/db/migrate/025_add_notifications.rb +16 -0
- data/db/migrate/026_add_status_to_resources.rb +7 -0
- data/db/migrate/027_add_notification_id_to_jobs.rb +8 -0
- data/lib/coupler.rb +6 -2
- data/lib/coupler/base.rb +1 -0
- data/lib/coupler/extensions.rb +3 -1
- data/lib/coupler/extensions/imports.rb +11 -14
- data/lib/coupler/extensions/notifications.rb +26 -0
- data/lib/coupler/models.rb +1 -0
- data/lib/coupler/models/comparison.rb +0 -1
- data/lib/coupler/models/connection.rb +0 -1
- data/lib/coupler/models/field.rb +6 -6
- data/lib/coupler/models/import.rb +9 -3
- data/lib/coupler/models/job.rb +64 -20
- data/lib/coupler/models/matcher.rb +0 -1
- data/lib/coupler/models/notification.rb +7 -0
- data/lib/coupler/models/project.rb +0 -1
- data/lib/coupler/models/resource.rb +52 -31
- data/lib/coupler/models/result.rb +0 -1
- data/lib/coupler/models/scenario.rb +0 -1
- data/lib/coupler/models/transformation.rb +2 -3
- data/lib/coupler/models/transformer.rb +0 -1
- data/lib/coupler/scheduler.rb +8 -0
- data/tasks/db.rake +2 -2
- data/test/functional/test_imports.rb +21 -13
- data/test/functional/test_notifications.rb +38 -0
- data/test/integration/test_import.rb +25 -1
- data/test/unit/models/test_field.rb +2 -13
- data/test/unit/models/test_import.rb +4 -0
- data/test/unit/models/test_job.rb +59 -6
- data/test/unit/models/test_notification.rb +17 -0
- data/test/unit/models/test_resource.rb +114 -29
- data/test/unit/models/test_transformation.rb +4 -4
- data/test/unit/test_base.rb +1 -1
- data/test/unit/test_scheduler.rb +11 -0
- data/webroot/public/css/style.css +23 -0
- data/webroot/public/js/application.js +31 -10
- data/webroot/views/jobs/list.erb +2 -0
- data/webroot/views/layout.erb +3 -1
- data/webroot/views/notifications/index.erb +16 -0
- data/webroot/views/resources/list.erb +12 -7
- data/webroot/views/sidebar.erb +2 -0
- metadata +11 -2
| @@ -1,4 +1,3 @@ | |
| 1 | 
            -
            pp caller
         | 
| 2 1 | 
             
            module Coupler
         | 
| 3 2 | 
             
              module Models
         | 
| 4 3 | 
             
                class Transformation < Sequel::Model
         | 
| @@ -93,7 +92,7 @@ module Coupler | |
| 93 92 |  | 
| 94 93 | 
             
                    def after_save
         | 
| 95 94 | 
             
                      super
         | 
| 96 | 
            -
                      resource. | 
| 95 | 
            +
                      resource.transformations_updated!
         | 
| 97 96 | 
             
                    end
         | 
| 98 97 |  | 
| 99 98 | 
             
                    def before_destroy
         | 
| @@ -111,7 +110,7 @@ module Coupler | |
| 111 110 | 
             
                      if result_field && result_field.is_generated && self.class.filter(:result_field_id => result_field.id).count == 0
         | 
| 112 111 | 
             
                        result_field.destroy
         | 
| 113 112 | 
             
                      end
         | 
| 114 | 
            -
                      resource. | 
| 113 | 
            +
                      resource.transformations_updated!  if resource
         | 
| 115 114 | 
             
                    end
         | 
| 116 115 | 
             
                end
         | 
| 117 116 | 
             
              end
         | 
    
        data/lib/coupler/scheduler.rb
    CHANGED
    
    | @@ -23,6 +23,14 @@ module Coupler | |
| 23 23 | 
             
                  })
         | 
| 24 24 | 
             
                end
         | 
| 25 25 |  | 
| 26 | 
            +
                def schedule_import_job(import)
         | 
| 27 | 
            +
                  Models::Job.create({
         | 
| 28 | 
            +
                    :name => "import",
         | 
| 29 | 
            +
                    :import => import,
         | 
| 30 | 
            +
                    :status => "scheduled"
         | 
| 31 | 
            +
                  })
         | 
| 32 | 
            +
                end
         | 
| 33 | 
            +
             | 
| 26 34 | 
             
                def run_jobs
         | 
| 27 35 | 
             
                  @mutex.synchronize do
         | 
| 28 36 | 
             
                    count = Models::Job.filter(:status => 'running').count
         | 
    
        data/tasks/db.rake
    CHANGED
    
    | @@ -16,7 +16,7 @@ namespace :db do | |
| 16 16 | 
             
              desc "Run migrations"
         | 
| 17 17 | 
             
              task :migrate => :environment do
         | 
| 18 18 | 
             
                version = ENV['VERSION']
         | 
| 19 | 
            -
                Coupler::Database. | 
| 19 | 
            +
                Coupler::Database.migrate!(version ? version.to_i : nil)
         | 
| 20 20 | 
             
              end
         | 
| 21 21 |  | 
| 22 22 | 
             
              namespace :migrate do
         | 
| @@ -26,7 +26,7 @@ namespace :db do | |
| 26 26 |  | 
| 27 27 | 
             
              desc "Roll the database back a version"
         | 
| 28 28 | 
             
              task :rollback => [:start, :environment] do
         | 
| 29 | 
            -
                Coupler::Database. | 
| 29 | 
            +
                Coupler::Database.rollback!
         | 
| 30 30 | 
             
              end
         | 
| 31 31 |  | 
| 32 32 | 
             
              desc "Reset and bootstrap the database"
         | 
| @@ -33,12 +33,16 @@ module CouplerFunctionalTests | |
| 33 33 | 
             
                  find('label[for="resource-type-csv"]').click
         | 
| 34 34 | 
             
                  attach_file('data', fixture_path('people.csv'))
         | 
| 35 35 |  | 
| 36 | 
            +
                  job_count = Job.count
         | 
| 37 | 
            +
                  resource_count = Resource.count
         | 
| 36 38 | 
             
                  click_button('Begin Importing')
         | 
| 37 | 
            -
                  assert_match %r{^/projects/#{@project[:id]}/resources | 
| 39 | 
            +
                  assert_match %r{^/projects/#{@project[:id]}/resources$}, page.current_path
         | 
| 40 | 
            +
                  assert_equal resource_count + 1, Resource.count
         | 
| 41 | 
            +
                  assert_equal job_count + 1, Job.count
         | 
| 38 42 | 
             
                end
         | 
| 39 43 |  | 
| 40 44 | 
             
                attribute(:javascript, true)
         | 
| 41 | 
            -
                test "create with invalid import" do
         | 
| 45 | 
            +
                test "create with invalid import, fix, and continue" do
         | 
| 42 46 | 
             
                  visit "/projects/#{@project.id}/resources/new"
         | 
| 43 47 | 
             
                  find('label[for="resource-type-csv"]').click
         | 
| 44 48 | 
             
                  attach_file('data', fixture_path('people.csv'))
         | 
| @@ -47,30 +51,34 @@ module CouplerFunctionalTests | |
| 47 51 | 
             
                  click_button('Begin Importing')
         | 
| 48 52 |  | 
| 49 53 | 
             
                  assert page.has_selector?("div.errors")
         | 
| 54 | 
            +
             | 
| 55 | 
            +
                  fill_in('name', :with => 'Foo Bar')
         | 
| 56 | 
            +
                  click_button('Begin Importing')
         | 
| 57 | 
            +
                  assert_match %r{^/projects/#{@project[:id]}/resources$}, page.current_path
         | 
| 50 58 | 
             
                end
         | 
| 51 59 |  | 
| 52 60 | 
             
                attribute(:javascript, true)
         | 
| 53 | 
            -
                test " | 
| 54 | 
            -
                   | 
| 55 | 
            -
                   | 
| 56 | 
            -
                  attach_file('data', fixture_path('duplicate-keys.csv'))
         | 
| 57 | 
            -
                  click_button('Begin Importing')
         | 
| 61 | 
            +
                test "editing import with duplicate keys" do
         | 
| 62 | 
            +
                  import = Import.create(:data => fixture_file_upload("duplicate-keys.csv"), :project => @project)
         | 
| 63 | 
            +
                  import.import!
         | 
| 58 64 |  | 
| 65 | 
            +
                  visit "/projects/#{@project.id}/imports/#{import.id}/edit"
         | 
| 59 66 | 
             
                  assert find("h2").has_content?("Duplicate Keys")
         | 
| 60 | 
            -
                  assert_match %r{^/projects/#{@project.id}/imports/\d+/edit$}, page.current_path
         | 
| 61 67 | 
             
                end
         | 
| 62 68 |  | 
| 63 69 | 
             
                attribute(:javascript, true)
         | 
| 64 70 | 
             
                test "update import with duplicate keys" do
         | 
| 65 | 
            -
                   | 
| 66 | 
            -
                   | 
| 67 | 
            -
                   | 
| 68 | 
            -
                  click_button('Begin Importing')
         | 
| 71 | 
            +
                  import = Import.create(:data => fixture_file_upload("duplicate-keys.csv"), :project => @project)
         | 
| 72 | 
            +
                  resource = Resource.create(:name => import.name, :project => @project, :status => 'pending', :import => import)
         | 
| 73 | 
            +
                  import.import!
         | 
| 69 74 |  | 
| 75 | 
            +
                  visit "/projects/#{@project.id}/imports/#{import.id}/edit"
         | 
| 70 76 | 
             
                  find('input[name="delete[2][]"][value="1"]').click
         | 
| 71 77 | 
             
                  click_button('Submit')
         | 
| 72 78 |  | 
| 73 | 
            -
                   | 
| 79 | 
            +
                  resource.refresh
         | 
| 80 | 
            +
                  assert_equal "ok", resource.status
         | 
| 81 | 
            +
                  assert_match %r{^/projects/#{@project[:id]}/resources/#{resource.id}$}, page.current_path
         | 
| 74 82 | 
             
                end
         | 
| 75 83 | 
             
              end
         | 
| 76 84 | 
             
            end
         | 
| @@ -0,0 +1,38 @@ | |
| 1 | 
            +
            require 'helper'
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            module CouplerFunctionalTests
         | 
| 4 | 
            +
              class TestNotifications < Coupler::Test::FunctionalTest
         | 
| 5 | 
            +
             | 
| 6 | 
            +
                test "empty index" do
         | 
| 7 | 
            +
                  visit "/notifications"
         | 
| 8 | 
            +
                  assert_equal 200, page.status_code
         | 
| 9 | 
            +
                end
         | 
| 10 | 
            +
             | 
| 11 | 
            +
                test "index" do
         | 
| 12 | 
            +
                  n1 = Notification.create(:message => "Test!")
         | 
| 13 | 
            +
                  n2 = Notification.create(:message => "Another Test!", :url => "/projects")
         | 
| 14 | 
            +
                  n3 = Notification.create(:message => "Foo", :seen => true)
         | 
| 15 | 
            +
                  n4 = Notification.create(:message => "Bar", :url => "/connections", :seen => true)
         | 
| 16 | 
            +
                  visit "/notifications"
         | 
| 17 | 
            +
                  assert_equal 200, page.status_code
         | 
| 18 | 
            +
                end
         | 
| 19 | 
            +
             | 
| 20 | 
            +
                test "flags notifications as seen when url is visited" do
         | 
| 21 | 
            +
                  n = Notification.create(:message => "Foo bar", :url => "/connections")
         | 
| 22 | 
            +
                  visit "/connections"
         | 
| 23 | 
            +
                  n.reload
         | 
| 24 | 
            +
                  assert n.seen
         | 
| 25 | 
            +
                end
         | 
| 26 | 
            +
             | 
| 27 | 
            +
                test "unseen json" do
         | 
| 28 | 
            +
                  now = DateTime.now
         | 
| 29 | 
            +
                  Timecop.freeze(now) do
         | 
| 30 | 
            +
                    n1 = Notification.create(:message => "Foo bar", :url => "/connections")
         | 
| 31 | 
            +
                    n2 = Notification.create(:message => "Seen foo bar", :url => "/projects", :seen => true)
         | 
| 32 | 
            +
                  end
         | 
| 33 | 
            +
                  page.driver.get "/notifications/unseen.json"
         | 
| 34 | 
            +
                  result = JSON.parse(page.driver.response.body)
         | 
| 35 | 
            +
                  assert_equal([{'id' => n1.id, 'message' => 'Foo bar', 'url' => '/connections', 'created_at' => now.to_s}], result)
         | 
| 36 | 
            +
                end
         | 
| 37 | 
            +
              end
         | 
| 38 | 
            +
            end
         | 
| @@ -5,7 +5,13 @@ class TestImport < Coupler::Test::IntegrationTest | |
| 5 5 | 
             
              test "import!" do
         | 
| 6 6 | 
             
                project = Project.create(:name => "foo")
         | 
| 7 7 | 
             
                import = Import.create(:data => fixture_file_upload("people.csv"), :project => project)
         | 
| 8 | 
            -
             | 
| 8 | 
            +
             | 
| 9 | 
            +
                total = import.data.file.size
         | 
| 10 | 
            +
                completed = 0
         | 
| 11 | 
            +
                import.import! do |pos|
         | 
| 12 | 
            +
                  assert pos > completed
         | 
| 13 | 
            +
                  completed = pos
         | 
| 14 | 
            +
                end
         | 
| 9 15 | 
             
                assert_not_nil import.occurred_at
         | 
| 10 16 |  | 
| 11 17 | 
             
                project.local_database do |db|
         | 
| @@ -22,6 +28,24 @@ class TestImport < Coupler::Test::IntegrationTest | |
| 22 28 | 
             
                end
         | 
| 23 29 | 
             
              end
         | 
| 24 30 |  | 
| 31 | 
            +
              test "import job" do
         | 
| 32 | 
            +
                project = Project.create(:name => "foo")
         | 
| 33 | 
            +
             | 
| 34 | 
            +
                tempfile = Tempfile.new('coupler-import')
         | 
| 35 | 
            +
                tempfile.write("id,foo,bar\n1,2,3\n4,5,6\n7,8,9\n2,3,4\n5,6,7\n8,9,0\n")
         | 
| 36 | 
            +
                tempfile.close
         | 
| 37 | 
            +
                import = Import.create(:data => file_upload(tempfile.path), :project => project)
         | 
| 38 | 
            +
                resource = Resource.create(:name => 'foo', :status => 'pending', :project => project, :import => import)
         | 
| 39 | 
            +
             | 
| 40 | 
            +
                job = Job.create(:name => 'import', :import => import, :status => "scheduled")
         | 
| 41 | 
            +
                job.execute
         | 
| 42 | 
            +
             | 
| 43 | 
            +
                project.local_database do |db|
         | 
| 44 | 
            +
                  name = :"import_#{import.id}"
         | 
| 45 | 
            +
                  assert db.tables.include?(name)
         | 
| 46 | 
            +
                end
         | 
| 47 | 
            +
              end
         | 
| 48 | 
            +
             | 
| 25 49 | 
             
              test "flags duplicate primary keys" do
         | 
| 26 50 | 
             
                tempfile = Tempfile.new('coupler-import')
         | 
| 27 51 | 
             
                tempfile.write("id,foo,bar\n1,abc,def\n2,ghi,jkl\n2,mno,pqr")
         | 
| @@ -46,17 +46,6 @@ module CouplerUnitTests | |
| 46 46 | 
             
                    assert field_2.is_selected
         | 
| 47 47 | 
             
                  end
         | 
| 48 48 |  | 
| 49 | 
            -
                  test "original_column_options" do
         | 
| 50 | 
            -
                    field = new_field({
         | 
| 51 | 
            -
                      :local_type => 'integer',
         | 
| 52 | 
            -
                      :local_db_type => 'int(11)',
         | 
| 53 | 
            -
                    })
         | 
| 54 | 
            -
                    assert_equal({
         | 
| 55 | 
            -
                      :name => 'foo', :type => 'varchar(255)',
         | 
| 56 | 
            -
                      :primary_key => false
         | 
| 57 | 
            -
                    }, field.original_column_options)
         | 
| 58 | 
            -
                  end
         | 
| 59 | 
            -
             | 
| 60 49 | 
             
                  test "local_column_options" do
         | 
| 61 50 | 
             
                    field_1 = new_field({
         | 
| 62 51 | 
             
                      :type => 'integer', :db_type => 'int(11)',
         | 
| @@ -69,11 +58,11 @@ module CouplerUnitTests | |
| 69 58 | 
             
                    })
         | 
| 70 59 |  | 
| 71 60 | 
             
                    assert_equal({
         | 
| 72 | 
            -
                      :name => 'foo', :type =>  | 
| 61 | 
            +
                      :name => 'foo', :type => String, :size => 255,
         | 
| 73 62 | 
             
                      :primary_key => false
         | 
| 74 63 | 
             
                    }, field_1.local_column_options)
         | 
| 75 64 | 
             
                    assert_equal({
         | 
| 76 | 
            -
                      :name => 'foo', :type =>  | 
| 65 | 
            +
                      :name => 'foo', :type => Integer,
         | 
| 77 66 | 
             
                      :primary_key => false
         | 
| 78 67 | 
             
                    }, field_2.local_column_options)
         | 
| 79 68 | 
             
                  end
         | 
| @@ -31,6 +31,10 @@ module CouplerUnitTests | |
| 31 31 | 
             
                    assert_respond_to Models::Import.new, :project
         | 
| 32 32 | 
             
                  end
         | 
| 33 33 |  | 
| 34 | 
            +
                  test "one to one resource" do
         | 
| 35 | 
            +
                    assert_respond_to Models::Import.new, :resource
         | 
| 36 | 
            +
                  end
         | 
| 37 | 
            +
             | 
| 34 38 | 
             
                  test "gets name from original filename" do
         | 
| 35 39 | 
             
                    import = new_import
         | 
| 36 40 | 
             
                    assert_equal "People", import.name
         | 
| @@ -11,13 +11,20 @@ module CouplerUnitTests | |
| 11 11 | 
             
                    if attribs[:scenario]
         | 
| 12 12 | 
             
                      j.stubs(:scenario_dataset).returns(stub({:all => [attribs[:scenario]]}))
         | 
| 13 13 | 
             
                    end
         | 
| 14 | 
            +
                    if attribs[:import]
         | 
| 15 | 
            +
                      j.stubs(:import_dataset).returns(stub({:all => [attribs[:import]]}))
         | 
| 16 | 
            +
                    end
         | 
| 14 17 | 
             
                    j
         | 
| 15 18 | 
             
                  end
         | 
| 16 19 |  | 
| 17 20 | 
             
                  def setup
         | 
| 18 21 | 
             
                    super
         | 
| 22 | 
            +
                    @project = stub('project', :pk => 1, :id => 1, :associations => {})
         | 
| 19 23 | 
             
                    @resource = stub('resource', :pk => 456, :id => 456, :associations => {})
         | 
| 20 24 | 
             
                    @scenario = stub('scenario', :pk => 456, :id => 456, :associations => {})
         | 
| 25 | 
            +
                    @import = stub('import', :pk => 123, :id => 123, :associations => {}, :project_id => 1, :project => @project, :name => 'foo')
         | 
| 26 | 
            +
                    @notification = stub('notification', :pk => 222, :id => 222, :associations => {})
         | 
| 27 | 
            +
                    Coupler::Models::Notification.stubs(:create).returns(@notification)
         | 
| 21 28 | 
             
                  end
         | 
| 22 29 |  | 
| 23 30 | 
             
                  test "sequel model" do
         | 
| @@ -33,6 +40,14 @@ module CouplerUnitTests | |
| 33 40 | 
             
                    assert_respond_to Job.new, :scenario
         | 
| 34 41 | 
             
                  end
         | 
| 35 42 |  | 
| 43 | 
            +
                  test "belongs to import" do
         | 
| 44 | 
            +
                    assert_respond_to Job.new, :import
         | 
| 45 | 
            +
                  end
         | 
| 46 | 
            +
             | 
| 47 | 
            +
                  test "belongs to notification" do
         | 
| 48 | 
            +
                    assert_respond_to Job.new, :notification
         | 
| 49 | 
            +
                  end
         | 
| 50 | 
            +
             | 
| 36 51 | 
             
                  test "percent completed" do
         | 
| 37 52 | 
             
                    job = new_job(:name => 'transform', :resource => @resource, :total => 200, :completed => 54)
         | 
| 38 53 | 
             
                    assert_equal 27, job.percent_completed
         | 
| @@ -48,7 +63,9 @@ module CouplerUnitTests | |
| 48 63 | 
             
                    seq = sequence("update")
         | 
| 49 64 | 
             
                    job.expects(:update).with(:status => 'running', :total => 12345, :started_at => now).in_sequence(seq)
         | 
| 50 65 | 
             
                    @resource.expects(:transform!).in_sequence(seq)
         | 
| 51 | 
            -
                    job.expects(: | 
| 66 | 
            +
                    job.expects(:status=).with('done').in_sequence(seq)
         | 
| 67 | 
            +
                    job.expects(:completed_at=).with(now).in_sequence(seq)
         | 
| 68 | 
            +
                    job.expects(:save).in_sequence(seq)
         | 
| 52 69 | 
             
                    Timecop.freeze(now) { job.execute }
         | 
| 53 70 | 
             
                  end
         | 
| 54 71 |  | 
| @@ -60,8 +77,9 @@ module CouplerUnitTests | |
| 60 77 | 
             
                    seq = sequence("update")
         | 
| 61 78 | 
             
                    job.expects(:update).with(:status => 'running', :total => 12345, :started_at => now).in_sequence(seq)
         | 
| 62 79 | 
             
                    fake_exception_klass = Class.new(Exception)
         | 
| 63 | 
            -
                     | 
| 64 | 
            -
                     | 
| 80 | 
            +
                    exception = fake_exception_klass.new("someone set us up the bomb")
         | 
| 81 | 
            +
                    @resource.expects(:transform!).raises(exception).in_sequence(seq)
         | 
| 82 | 
            +
                    job.expects(:update).with(has_entries(:status => 'failed', :completed_at => now, :error_msg => kind_of(String))).in_sequence(seq)
         | 
| 65 83 |  | 
| 66 84 | 
             
                    Timecop.freeze(now) do
         | 
| 67 85 | 
             
                      begin
         | 
| @@ -78,7 +96,9 @@ module CouplerUnitTests | |
| 78 96 | 
             
                    seq = sequence("update")
         | 
| 79 97 | 
             
                    job.expects(:update).with(:status => 'running', :started_at => now).in_sequence(seq)
         | 
| 80 98 | 
             
                    @scenario.expects(:run!).in_sequence(seq)
         | 
| 81 | 
            -
                    job.expects(: | 
| 99 | 
            +
                    job.expects(:status=).with('done').in_sequence(seq)
         | 
| 100 | 
            +
                    job.expects(:completed_at=).with(now).in_sequence(seq)
         | 
| 101 | 
            +
                    job.expects(:save).in_sequence(seq)
         | 
| 82 102 |  | 
| 83 103 | 
             
                    Timecop.freeze(now) { job.execute }
         | 
| 84 104 | 
             
                  end
         | 
| @@ -90,8 +110,9 @@ module CouplerUnitTests | |
| 90 110 | 
             
                    seq = sequence("update")
         | 
| 91 111 | 
             
                    job.expects(:update).with(:status => 'running', :started_at => now).in_sequence(seq)
         | 
| 92 112 | 
             
                    fake_exception_klass = Class.new(Exception)
         | 
| 93 | 
            -
                     | 
| 94 | 
            -
                     | 
| 113 | 
            +
                    exception = fake_exception_klass.new("someone set us up the bomb")
         | 
| 114 | 
            +
                    @scenario.expects(:run!).raises(exception).in_sequence(seq)
         | 
| 115 | 
            +
                    job.expects(:update).with(has_entries(:status => 'failed', :completed_at => now, :error_msg => kind_of(String))).in_sequence(seq)
         | 
| 95 116 |  | 
| 96 117 | 
             
                    Timecop.freeze(now) do
         | 
| 97 118 | 
             
                      begin
         | 
| @@ -110,6 +131,38 @@ module CouplerUnitTests | |
| 110 131 | 
             
                    Timecop.freeze(now    ) { job_4 = new_job(:name => "run_scenario", :scenario => @scenario).save! }
         | 
| 111 132 | 
             
                    assert_equal [job_4, job_3, job_2], Job.recently_accessed
         | 
| 112 133 | 
             
                  end
         | 
| 134 | 
            +
             | 
| 135 | 
            +
                  test "import job" do
         | 
| 136 | 
            +
                    job = new_job(:name => 'import', :import => @import).save!
         | 
| 137 | 
            +
             | 
| 138 | 
            +
                    now = Time.now
         | 
| 139 | 
            +
                    seq = sequence("update")
         | 
| 140 | 
            +
                    @import.expects(:data).returns(mock(:file => mock(:size => 12345)))
         | 
| 141 | 
            +
                    job.expects(:update).with(:status => 'running', :total => 12345, :started_at => now).in_sequence(seq)
         | 
| 142 | 
            +
                    @import.expects(:import!).returns(true).in_sequence(seq)
         | 
| 143 | 
            +
                    @import.expects(:resource).returns(mock({:id => 456, :activate! => nil})).in_sequence(seq)
         | 
| 144 | 
            +
                    Notification.expects(:create).with(:message => "Import finished successfully", :url => "/projects/1/resources/456").returns(@notification).in_sequence(seq)
         | 
| 145 | 
            +
                    job.expects(:notification=).with(@notification).in_sequence(seq)
         | 
| 146 | 
            +
                    job.expects(:status=).with('done').in_sequence(seq)
         | 
| 147 | 
            +
                    job.expects(:completed_at=).with(now).in_sequence(seq)
         | 
| 148 | 
            +
                    job.expects(:save).in_sequence(seq)
         | 
| 149 | 
            +
                    Timecop.freeze(now) { job.execute }
         | 
| 150 | 
            +
                  end
         | 
| 151 | 
            +
             | 
| 152 | 
            +
                  test "import job with more interaction needed" do
         | 
| 153 | 
            +
                    job = new_job(:name => 'import', :import => @import).save!
         | 
| 154 | 
            +
             | 
| 155 | 
            +
                    now = Time.now
         | 
| 156 | 
            +
                    seq = sequence("update")
         | 
| 157 | 
            +
                    @import.expects(:data).returns(mock(:file => mock(:size => 12345)))
         | 
| 158 | 
            +
                    job.expects(:update).with(:status => 'running', :total => 12345, :started_at => now).in_sequence(seq)
         | 
| 159 | 
            +
                    @import.expects(:import!).returns(false).in_sequence(seq)
         | 
| 160 | 
            +
                    Notification.expects(:create).with(:message => "Import finished, but with errors", :url => "/projects/1/imports/123/edit").returns(@notification)
         | 
| 161 | 
            +
                    job.expects(:status=).with('done').in_sequence(seq)
         | 
| 162 | 
            +
                    job.expects(:completed_at=).with(now).in_sequence(seq)
         | 
| 163 | 
            +
                    job.expects(:save).in_sequence(seq)
         | 
| 164 | 
            +
                    Timecop.freeze(now) { job.execute }
         | 
| 165 | 
            +
                  end
         | 
| 113 166 | 
             
                end
         | 
| 114 167 | 
             
              end
         | 
| 115 168 | 
             
            end
         | 
| @@ -0,0 +1,17 @@ | |
| 1 | 
            +
            require 'helper'
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            module TestModels
         | 
| 4 | 
            +
              class TestNotification < Coupler::Test::UnitTest
         | 
| 5 | 
            +
                include Coupler::Models
         | 
| 6 | 
            +
                Coupler::Models::Notification  # force load
         | 
| 7 | 
            +
             | 
| 8 | 
            +
                test "sequel model" do
         | 
| 9 | 
            +
                  assert_equal ::Sequel::Model, Notification.superclass
         | 
| 10 | 
            +
                  assert_equal :notifications, Notification.table_name
         | 
| 11 | 
            +
                end
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                test "common model" do
         | 
| 14 | 
            +
                  assert Notification.ancestors.include?(CommonModel)
         | 
| 15 | 
            +
                end
         | 
| 16 | 
            +
              end
         | 
| 17 | 
            +
            end
         | 
| @@ -29,8 +29,15 @@ module CouplerUnitTests | |
| 29 29 | 
             
                      :connection => @connection
         | 
| 30 30 | 
             
                    }.update(attribs)
         | 
| 31 31 | 
             
                    r = Resource.new(values)
         | 
| 32 | 
            -
                     | 
| 33 | 
            -
             | 
| 32 | 
            +
                    if values[:project]
         | 
| 33 | 
            +
                      r.stubs(:project_dataset).returns(stub({:all => [values[:project]]}))
         | 
| 34 | 
            +
                    end
         | 
| 35 | 
            +
                    if values[:connection]
         | 
| 36 | 
            +
                      r.stubs(:connection_dataset).returns(stub({:all => [values[:connection]]}))
         | 
| 37 | 
            +
                    end
         | 
| 38 | 
            +
                    if values[:import]
         | 
| 39 | 
            +
                      r.stubs(:import_dataset).returns(stub({:all => [values[:import]]}))
         | 
| 40 | 
            +
                    end
         | 
| 34 41 | 
             
                    r
         | 
| 35 42 | 
             
                  end
         | 
| 36 43 |  | 
| @@ -311,23 +318,27 @@ module CouplerUnitTests | |
| 311 318 | 
             
                    assert_equal dataset, resource.local_dataset
         | 
| 312 319 | 
             
                  end
         | 
| 313 320 |  | 
| 314 | 
            -
                  test " | 
| 321 | 
            +
                  test "transformations_updated!" do
         | 
| 315 322 | 
             
                    resource = new_resource.save!
         | 
| 316 323 | 
             
                    field_1 = stub("id field", :id => 1)
         | 
| 317 324 | 
             
                    field_2 = stub("first_name field", :id => 2)
         | 
| 318 325 | 
             
                    field_3 = stub("last_name field", :id => 3)
         | 
| 319 326 |  | 
| 327 | 
            +
                    now = Time.now
         | 
| 320 328 | 
             
                    transformation_1 = stub("transformation 1", {
         | 
| 329 | 
            +
                      :id => 1, :updated_at => now,
         | 
| 321 330 | 
             
                      :source_field_id => 2, :result_field_id => 2,
         | 
| 322 331 | 
             
                      :field_changes => {2 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 323 332 | 
             
                      :source_field => field_2
         | 
| 324 333 | 
             
                    })
         | 
| 325 334 | 
             
                    transformation_2 = stub("transformation 2", {
         | 
| 335 | 
            +
                      :id => 2, :updated_at => now,
         | 
| 326 336 | 
             
                      :source_field_id => 3, :result_field_id => 3,
         | 
| 327 337 | 
             
                      :field_changes => {3 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 328 338 | 
             
                      :source_field => field_3
         | 
| 329 339 | 
             
                    })
         | 
| 330 340 | 
             
                    transformation_3 = stub("transformation 3", {
         | 
| 341 | 
            +
                      :id => 3, :updated_at => now,
         | 
| 331 342 | 
             
                      :source_field_id => 3, :result_field_id => 3,
         | 
| 332 343 | 
             
                      :field_changes => {3 => {:type => :datetime, :db_type => 'datetime'}},
         | 
| 333 344 | 
             
                      :source_field => field_3
         | 
| @@ -349,10 +360,10 @@ module CouplerUnitTests | |
| 349 360 | 
             
                      :local_type => :datetime, :local_db_type => 'datetime'
         | 
| 350 361 | 
             
                    }).in_sequence(seq)
         | 
| 351 362 |  | 
| 352 | 
            -
                    resource. | 
| 363 | 
            +
                    resource.transformations_updated!
         | 
| 353 364 | 
             
                  end
         | 
| 354 365 |  | 
| 355 | 
            -
                  test " | 
| 366 | 
            +
                  test "transformations_updated does not change newly created result field" do
         | 
| 356 367 | 
             
                    resource = new_resource.save!
         | 
| 357 368 | 
             
                    field_1 = stub("id field", :id => 1)
         | 
| 358 369 | 
             
                    field_2 = stub("first_name field", :id => 2)
         | 
| @@ -360,6 +371,7 @@ module CouplerUnitTests | |
| 360 371 | 
             
                    new_field = stub("new field", :id => 4)
         | 
| 361 372 |  | 
| 362 373 | 
             
                    transformation = stub("transformation 1", {
         | 
| 374 | 
            +
                      :id => 1, :updated_at => Time.now,
         | 
| 363 375 | 
             
                      :source_field_id => 2, :result_field_id => 4,
         | 
| 364 376 | 
             
                      :field_changes => {2 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 365 377 | 
             
                      :source_field => field_2
         | 
| @@ -373,10 +385,10 @@ module CouplerUnitTests | |
| 373 385 | 
             
                    field_2.expects(:update).never
         | 
| 374 386 | 
             
                    new_field.expects(:update).never
         | 
| 375 387 |  | 
| 376 | 
            -
                    resource. | 
| 388 | 
            +
                    resource.transformations_updated!
         | 
| 377 389 | 
             
                  end
         | 
| 378 390 |  | 
| 379 | 
            -
                  test " | 
| 391 | 
            +
                  test "transformations_updated resets local_type and local_db_type" do
         | 
| 380 392 | 
             
                    resource = new_resource.save!
         | 
| 381 393 |  | 
| 382 394 | 
             
                    resource.expects(:fields_dataset).returns(mock {
         | 
| @@ -386,7 +398,7 @@ module CouplerUnitTests | |
| 386 398 | 
             
                    td.expects(:order).with(:position).returns(td)
         | 
| 387 399 | 
             
                    resource.expects(:transformations_dataset).returns(td)
         | 
| 388 400 |  | 
| 389 | 
            -
                    resource. | 
| 401 | 
            +
                    resource.transformations_updated!
         | 
| 390 402 | 
             
                  end
         | 
| 391 403 |  | 
| 392 404 | 
             
                  test "initial status" do
         | 
| @@ -396,39 +408,108 @@ module CouplerUnitTests | |
| 396 408 |  | 
| 397 409 | 
             
                  test "status after adding first transformation" do
         | 
| 398 410 | 
             
                    resource = new_resource.save!
         | 
| 399 | 
            -
             | 
| 411 | 
            +
             | 
| 412 | 
            +
                    field_1 = stub("id field", :id => 1, :update => nil)
         | 
| 413 | 
            +
                    transformation_1 = stub("transformation 1", {
         | 
| 414 | 
            +
                      :id => 1, :updated_at => Time.now,
         | 
| 415 | 
            +
                      :source_field_id => 1, :result_field_id => 1,
         | 
| 416 | 
            +
                      :field_changes => {1 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 417 | 
            +
                      :source_field => field_1
         | 
| 418 | 
            +
                    })
         | 
| 419 | 
            +
                    resource.stubs(:fields_dataset).returns(stub(:update => nil))
         | 
| 420 | 
            +
                    td = [transformation_1]
         | 
| 421 | 
            +
                    td.expects(:order).with(:position).returns(td)
         | 
| 422 | 
            +
                    resource.expects(:transformations_dataset).returns(td)
         | 
| 423 | 
            +
             | 
| 424 | 
            +
                    resource.transformations_updated!
         | 
| 400 425 | 
             
                    assert_equal "out_of_date", resource.status
         | 
| 401 426 | 
             
                  end
         | 
| 402 427 |  | 
| 403 428 | 
             
                  test "status when new transformation is created since transforming" do
         | 
| 404 | 
            -
                     | 
| 405 | 
            -
                    resource | 
| 429 | 
            +
                    now = Time.now
         | 
| 430 | 
            +
                    resource = new_resource(:transformed_with => "1", :transformed_at => now - 5).save!
         | 
| 431 | 
            +
             | 
| 432 | 
            +
                    field_1 = stub("id field", :id => 1, :update => nil)
         | 
| 433 | 
            +
                    transformation_1 = stub("transformation 1", {
         | 
| 434 | 
            +
                      :id => 1, :updated_at => now - 10,
         | 
| 435 | 
            +
                      :source_field_id => 1, :result_field_id => 1,
         | 
| 436 | 
            +
                      :field_changes => {1 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 437 | 
            +
                      :source_field => field_1
         | 
| 438 | 
            +
                    })
         | 
| 439 | 
            +
                    transformation_2 = stub("transformation 2", {
         | 
| 440 | 
            +
                      :id => 2, :updated_at => now,
         | 
| 441 | 
            +
                      :source_field_id => 1, :result_field_id => 1,
         | 
| 442 | 
            +
                      :field_changes => {1 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 443 | 
            +
                      :source_field => field_1
         | 
| 444 | 
            +
                    })
         | 
| 445 | 
            +
                    resource.stubs(:fields_dataset).returns(stub(:update => nil))
         | 
| 446 | 
            +
                    td = [transformation_1, transformation_2]
         | 
| 447 | 
            +
                    td.expects(:order).with(:position).returns(td)
         | 
| 448 | 
            +
                    resource.expects(:transformations_dataset).returns(td)
         | 
| 449 | 
            +
             | 
| 450 | 
            +
                    resource.transformations_updated!
         | 
| 406 451 | 
             
                    assert_equal "out_of_date", resource.status
         | 
| 407 452 | 
             
                  end
         | 
| 408 453 |  | 
| 409 454 | 
             
                  test "status when transformation is updated since transforming" do
         | 
| 410 455 | 
             
                    now = Time.now
         | 
| 411 | 
            -
                    resource = new_resource( | 
| 412 | 
            -
             | 
| 413 | 
            -
             | 
| 414 | 
            -
             | 
| 415 | 
            -
             | 
| 456 | 
            +
                    resource = new_resource({
         | 
| 457 | 
            +
                      :transformed_with => "1",
         | 
| 458 | 
            +
                      :transformed_at => now - 20
         | 
| 459 | 
            +
                    }).save!
         | 
| 460 | 
            +
                    field_1 = stub("id field", :id => 1, :update => nil)
         | 
| 461 | 
            +
             | 
| 462 | 
            +
                    transformation_1 = stub("transformation 1", {
         | 
| 463 | 
            +
                      :id => 1, :updated_at => now,
         | 
| 464 | 
            +
                      :source_field_id => 1, :result_field_id => 1,
         | 
| 465 | 
            +
                      :field_changes => {1 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 466 | 
            +
                      :source_field => field_1
         | 
| 416 467 | 
             
                    })
         | 
| 468 | 
            +
             | 
| 469 | 
            +
                    resource.stubs(:fields_dataset).returns(stub(:update => nil))
         | 
| 470 | 
            +
                    td = [transformation_1]
         | 
| 471 | 
            +
                    td.expects(:order).with(:position).returns(td)
         | 
| 472 | 
            +
                    resource.expects(:transformations_dataset).returns(td)
         | 
| 473 | 
            +
             | 
| 474 | 
            +
                    resource.transformations_updated!
         | 
| 417 475 | 
             
                    assert_equal "out_of_date", resource.status
         | 
| 418 476 | 
             
                  end
         | 
| 419 477 |  | 
| 420 478 | 
             
                  test "status when transformation is removed since transforming" do
         | 
| 421 479 | 
             
                    now = Time.now
         | 
| 422 | 
            -
                    resource = new_resource( | 
| 423 | 
            -
             | 
| 480 | 
            +
                    resource = new_resource({
         | 
| 481 | 
            +
                      :transformed_with => "1",
         | 
| 482 | 
            +
                      :transformed_at => now - 20
         | 
| 483 | 
            +
                    }).save!
         | 
| 484 | 
            +
                    td = []
         | 
| 485 | 
            +
                    td.stubs(:order).returns(td)
         | 
| 486 | 
            +
                    resource.stubs(:transformations_dataset).returns(td)
         | 
| 487 | 
            +
             | 
| 488 | 
            +
                    resource.transformations_updated!
         | 
| 424 489 | 
             
                    assert_equal "out_of_date", resource.status
         | 
| 425 490 | 
             
                  end
         | 
| 426 491 |  | 
| 427 492 | 
             
                  test "status when new transformation is removed before transforming" do
         | 
| 428 493 | 
             
                    resource = new_resource.save!
         | 
| 429 | 
            -
             | 
| 494 | 
            +
             | 
| 495 | 
            +
                    field_1 = stub("id field", :id => 1, :update => nil)
         | 
| 496 | 
            +
                    transformation_1 = stub("transformation 1", {
         | 
| 497 | 
            +
                      :id => 1, :updated_at => Time.now,
         | 
| 498 | 
            +
                      :source_field_id => 1, :result_field_id => 1,
         | 
| 499 | 
            +
                      :field_changes => {1 => {:type => :integer, :db_type => 'int(11)'}},
         | 
| 500 | 
            +
                      :source_field => field_1
         | 
| 501 | 
            +
                    })
         | 
| 502 | 
            +
                    resource.stubs(:fields_dataset).returns(stub(:update => nil))
         | 
| 503 | 
            +
                    td = [transformation_1]
         | 
| 504 | 
            +
                    td.expects(:order).with(:position).returns(td)
         | 
| 505 | 
            +
                    resource.expects(:transformations_dataset).returns(td)
         | 
| 506 | 
            +
                    resource.transformations_updated!
         | 
| 430 507 | 
             
                    assert_equal "out_of_date", resource.status
         | 
| 431 | 
            -
             | 
| 508 | 
            +
             | 
| 509 | 
            +
                    td = []
         | 
| 510 | 
            +
                    td.stubs(:order).returns(td)
         | 
| 511 | 
            +
                    resource.stubs(:transformations_dataset).returns(td)
         | 
| 512 | 
            +
                    resource.transformations_updated!
         | 
| 432 513 | 
             
                    assert_equal "ok", resource.status
         | 
| 433 514 | 
             
                  end
         | 
| 434 515 |  | 
| @@ -511,18 +592,21 @@ module CouplerUnitTests | |
| 511 592 | 
             
                    resource.destroy
         | 
| 512 593 | 
             
                  end
         | 
| 513 594 |  | 
| 514 | 
            -
                  test "creating resource | 
| 595 | 
            +
                  test "creating pending resource" do
         | 
| 596 | 
            +
                    resource = Resource.new(:name => "People", :project => @project, :status => 'pending')
         | 
| 597 | 
            +
                    assert resource.valid?
         | 
| 598 | 
            +
                    resource.save!
         | 
| 599 | 
            +
                  end
         | 
| 600 | 
            +
             | 
| 601 | 
            +
                  test "activate pending resource" do
         | 
| 515 602 | 
             
                    import = stub("import", {
         | 
| 516 603 | 
             
                      :id => 123, :pk => 123, :project => @project,
         | 
| 517 | 
            -
                      :name => "People"
         | 
| 604 | 
            +
                      :name => "People", :project_id => @project.id,
         | 
| 605 | 
            +
                      :associations => {}
         | 
| 518 606 | 
             
                    })
         | 
| 519 | 
            -
                    resource =  | 
| 520 | 
            -
                     | 
| 521 | 
            -
                    assert_equal @project, resource.project
         | 
| 522 | 
            -
                    assert_equal "import_123", resource.table_name
         | 
| 523 | 
            -
                    assert resource.valid?
         | 
| 607 | 
            +
                    resource = new_resource(:name => 'People', :status => 'pending', :import => import)
         | 
| 608 | 
            +
                    resource.save!
         | 
| 524 609 |  | 
| 525 | 
            -
                    # import.import! would happen here
         | 
| 526 610 | 
             
                    @local_database.stubs({
         | 
| 527 611 | 
             
                      :tables => [:import_123],
         | 
| 528 612 | 
             
                      :schema => [
         | 
| @@ -531,9 +615,10 @@ module CouplerUnitTests | |
| 531 615 | 
             
                        [:bar, {:primary_key => false, :type => :string, :db_type => "VARCHAR"}]
         | 
| 532 616 | 
             
                      ]
         | 
| 533 617 | 
             
                    })
         | 
| 534 | 
            -
                    resource. | 
| 618 | 
            +
                    resource.activate!
         | 
| 535 619 | 
             
                    assert_equal "id", resource.primary_key_name
         | 
| 536 620 | 
             
                    assert_equal "integer", resource.primary_key_type
         | 
| 621 | 
            +
                    assert_equal "ok", resource.status
         | 
| 537 622 | 
             
                  end
         | 
| 538 623 |  | 
| 539 624 | 
             
                  test "source_dataset count" do
         |