coupler 0.0.1-java

data/db/migrate/014_add_last_accessed_at_to_some_tables.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  up do
+    [:projects, :projects_versions, :connections, :transformers, :transformers_versions].each do |name|
+      alter_table(name) { add_column(:last_accessed_at, Time) }
+    end
+  end
+
+  down do
+    [:projects, :projects_versions, :connections, :transformers, :transformers_versions].each do |name|
+      alter_table(name) { drop_column(:last_accessed_at) }
+    end
+  end
+end

data/db/migrate/015_add_run_number_to_results.rb

@@ -0,0 +1,15 @@
+Sequel.migration do
+  up do
+    alter_table(:results) do
+      drop_column(:score_set_id)
+      add_column(:run_number, Integer)
+    end
+  end
+
+  down do
+    alter_table(:results) do
+      drop_column(:run_number)
+      add_column(:score_set_id, Integer)
+    end
+  end
+end

data/db/migrate/016_fix_scenario_run_count.rb

@@ -0,0 +1,27 @@
+Sequel.migration do
+  up do
+    # Fix scenario run_count
+    scenarios = self[:scenarios]
+    ds = scenarios.join(:results, :scenario_id => :id)
+    run_counts = ds.group_and_count(:scenario_id).all
+    run_counts.each do |row|
+      run_count = row[:count] > 1 ? 1 : 0
+      scenarios.filter(:id => row[:scenario_id]).update(:run_count => run_count)
+    end
+
+    # Delete all but the last result, since the other ones were overwritten
+    last_scenario_id = nil
+    results = self[:results]
+    results.order(:scenario_id, :id.desc).each do |result|
+      if result[:scenario_id] != last_scenario_id
+        last_scenario_id = result[:scenario_id]
+        results.filter(:id => result[:id]).update(:run_number => 1)
+      else
+        results.filter(:id => result[:id]).delete
+      end
+    end
+  end
+
+  down do
+  end
+end

data/db/migrate/017_rename_comparison_columns.rb

@@ -0,0 +1,14 @@
+Sequel.migration do
+  up do
+    alter_table(:comparisons) do
+      rename_column :lhs_value, :raw_lhs_value
+      rename_column :rhs_value, :raw_rhs_value
+    end
+  end
+  down do
+    alter_table(:comparisons) do
+      rename_column :raw_lhs_value, :lhs_value
+      rename_column :raw_rhs_value, :rhs_value
+    end
+  end
+end

data/db/migrate/018_fix_scenario_linkage_type.rb

@@ -0,0 +1,8 @@
+Sequel.migration do
+  up do
+    Coupler::Models::Scenario.each do |scenario|
+      scenario.set_linkage_type
+      scenario.save
+    end
+  end
+end

data/db/migrate/019_add_columns_to_imports.rb

@@ -0,0 +1,24 @@
+Sequel.migration do
+  up do
+    alter_table(:imports) do
+      add_column :field_names, String, :text => true
+      add_column :primary_key_name, String
+      add_column :has_headers, TrueClass
+      add_column :has_duplicate_keys, TrueClass
+      add_column :occurred_at, DateTime
+      rename_column :fields, :field_types
+      rename_column :data, :file_name
+    end
+  end
+  down do
+    alter_table(:imports) do
+      rename_column :file_name, :data
+      rename_column :field_types, :fields
+      drop_column :field_names
+      drop_column :primary_key_name
+      drop_column :has_headers
+      drop_column :has_duplicate_keys
+      drop_column :occurred_at
+    end
+  end
+end

data/db/migrate/020_rename_import_columns.rb

@@ -0,0 +1,12 @@
+Sequel.migration do
+  up do
+    alter_table(:imports) do
+      rename_column :file_name, :data
+    end
+  end
+  down do
+    alter_table(:imports) do
+      rename_column :data, :file_name
+    end
+  end
+end

data/db/migrate/021_add_fields_to_connections.rb

@@ -0,0 +1,15 @@
+Sequel.migration do
+  up do
+    alter_table(:connections) do
+      add_column :path, String
+      add_column :database_name, String
+    end
+  end
+
+  down do
+    alter_table(:connections) do
+      drop_column :path
+      drop_column :database_name
+    end
+  end
+end

data/db/migrate/022_remove_database_name_from_resources.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up do
+    alter_table(:resources) { drop_column(:database_name) }
+    alter_table(:resources_versions) { drop_column(:database_name) }
+  end
+
+  down do
+    alter_table(:resources) { add_column(:database_name, String) }
+    alter_table(:resources_versions) { add_column(:database_name, String) }
+  end
+end

data/features/connections.feature

@@ -0,0 +1,28 @@
+Feature: managing connections
+
+  Scenario: creating a connection
+    When I go to the connections page
+    And I click the "New Connection" link
+    And I fill in the form:
+      | Name     | Fake data |
+      | Type     | MySQL     |
+      | Host     | localhost |
+      | Port     | 12345     |
+      | Username | coupler   |
+      | Password | cupla     |
+    And I click the "Submit" button
+    Then it should show me a confirmation notice
+
+  Scenario: editing a connection
+    Given that I have created a connection called "My Connection"
+    When I go to the connections page
+    And I click the "Edit" link
+    And I change "Name" to "Server X"
+    And I click the "Submit" button
+    Then it should take me back to the connections page
+
+  Scenario: changing a connection that orphans resources
+
+  Scenario: deleting a connection
+
+  Scenario: deleting a connection that orphans resources

data/features/matchers.feature

@@ -0,0 +1,35 @@
+Feature: managing matchers
+
+  Background:
+    Given that I have created a connection called "My Connection"
+    And that I have created a project called "My Project"
+    And that I have added the "People" resource
+
+  Scenario: creating a matcher for a self-linkage scenario
+    Given that I have created a self-linkage scenario called "Link by Last name"
+    When I go to the scenario page
+    And I click the "Add matcher" link
+    And I click the "Add comparison" link
+    And I select "Field" as the first type
+    And I select "last_name" for "People [1]" as the first value
+    And I select "Field" as the second type
+    And I select "last_name" for "People [2]" as the second value
+    And I click the "Add" button
+    And I click the "Submit" button
+    Then it should take me back to the scenario page
+    And it should show me a confirmation notice
+
+  Scenario: creating a matcher for a dual-join scenario
+    Given that I have added the "Pets" resource
+    And that I have created a dual-linkage scenario called "Link by Last name"
+    When I go to the scenario page
+    And I click the "Add matcher" link
+    And I click the "Add comparison" link
+    And I select "Field" as the first type
+    And I select "last_name" for "People" as the first value
+    And I select "Field" as the second type
+    And I select "owner_last_name" for "Pets" as the second value
+    And I click the "Add" button
+    And I click the "Submit" button
+    Then it should take me back to the scenario page
+    And it should show me a confirmation notice

data/features/projects.feature

@@ -0,0 +1,11 @@
+Feature: managing projects
+
+  Scenario: creating a project
+    When I go to the projects page
+    And I click the "New Project" link
+    And I fill in the form:
+      | Name        | My Project             |
+      | Description | This is a test project |
+    And I click the "Submit" button
+    Then it should show me a confirmation notice
+    And ask me to add a resource

data/features/resources.feature

@@ -0,0 +1,62 @@
+Feature: managing resources
+  Background:
+    Given that I have created a project called "My Project"
+    And that I have created a transformer called "My Transformer"
+
+  Scenario: adding a resource
+    Given that I have created a connection called "My Connection"
+    When I go to the project page
+    And I click the "Add resource" link
+    And I choose a "Database" resource
+    And I fill in the form:
+      | Name         | People        |
+      | Connection   | My Connection |
+      | Database (2) | fake_data     |
+      | Table        | people        |
+    And I click the "Submit" button
+    Then it should show me a confirmation notice
+    And ask me to choose which fields I wish to select
+
+  Scenario: adding a resource and connection at the same time
+    When I go to the project page
+    And I click the "Add resource" link
+    And I choose a "Database" resource
+    And I fill in the form:
+      | Name (1)     | Fake data |
+      | Type         | MySQL     |
+      | Host         | localhost |
+      | Port         | 12345     |
+      | Username     | coupler   |
+      | Password     | cupla     |
+      | Name (2)     | People    |
+      | Database (2) | fake_data |
+      | Table        | people    |
+    And I click the "Submit" button
+    Then it should show me a confirmation notice
+
+  Scenario: adding a transformation
+    Given that I have created a connection called "My Connection"
+    And that I have added the "People" resource
+    When I go to the resource page
+    And I click on the hammer icon next to the "first_name" field
+    And I fill in the form:
+      | Transformer | My Transformer |
+    And I click the "Finish" button
+    Then it should show me a confirmation notice
+
+  Scenario: deleting a transformation
+    Given that I have created a connection called "My Connection"
+    And that I have added the "People" resource
+    And that I have added a transformation for "first_name"
+    When I go to the resource page
+    And I click on the cog icon next to the "first_name" field
+    And I click the "Delete" link
+    Then there should be no more transformations
+
+  Scenario: transforming a resource
+    Given that I have created a connection called "My Connection"
+    And that I have added the "People" resource
+    And that I have added a transformation for "first_name"
+    When I go to the resource page
+    And I click the "Transform now" button with confirmation
+    Then it should start transforming

data/features/scenarios.feature

@@ -0,0 +1,45 @@
+Feature: managing scenarios
+  Background:
+    Given that I have created a connection called "My Connection"
+    And that I have created a project called "My Project"
+    And that I have added the "People" resource
+
+  Scenario: creating a self-linkage scenario
+    When I go to the project page
+    And I click the "Create scenario" link
+    And I fill in the form:
+      | Name | Link by Last name |
+    And I click the "People" resource
+    And I click the "Submit" button
+    Then it should show me a confirmation notice
+    And ask me to add matchers
+
+  Scenario: creating a dual-linkage scenario
+    Given that I have added the "Pets" resource
+    When I go to the project page
+    And I click the "Create scenario" link
+    And I fill in the form:
+      | Name | Link by name |
+    And I click the "People" resource
+    And I click the "Pets" resource
+    And I click the "Submit" button
+    Then it should show me a confirmation notice
+    And ask me to add matchers
+
+  Scenario: running a self-linkage scenario
+    Given that I have created a self-linkage scenario called "Link by Last name"
+    And that I have added a matcher with these options:
+      | Type 1 | Value 1   | Operator | Type 2 | Value 2   |
+      | field  | last_name | equals   | field  | last_name |
+    When I go to the scenario page
+    And I click the "Run now" button with confirmation
+    Then it should start the linkage process
+
+  Scenario: running a scenario with a non-equality matcher
+    Given that I have created a self-linkage scenario called "Link by Last name"
+    And that I have added a matcher with these options:
+      | Type 1 | Value 1   | Operator       | Type 2 | Value 2   |
+      | field  | last_name | does_not_equal | field  | last_name |
+    When I go to the scenario page
+    And I click the "Run now" button with confirmation
+    Then it should start the linkage process

data/features/step_definitions/coupler_steps.rb

@@ -0,0 +1,145 @@
+Given /^that I have created a connection called "(.+?)"$/ do |connection_name|
+  @connection_name = connection_name
+  @connection = Factory(:connection, :name => connection_name)
+end
+
+Given /^that I have created a project called "(.+?)"$/ do |project_name|
+  @project_name = project_name
+  @project = Factory(:project, :name => project_name)
+end
+
+Given /^that I have created a transformer called "(.+?)"$/ do |transformer_name|
+  @transformer_name = transformer_name
+  @transformer = Factory(:transformer, :name => transformer_name)
+end
+
+Given /^that I have added the "(.+?)" resource$/ do |resource_name|
+  @resource_name = resource_name
+  options = case resource_name
+            when "People"
+              { :table_name => "people" }
+            when "Pets"
+              { :table_name => "pets" }
+            end
+  @resources ||= []
+  @resources << Factory(:resource, {:name => resource_name, :project => @project, :connection => @connection}.merge(options))
+end
+
+Given /^that I have added a transformation for "([^\"]*)"$/ do |field|
+  resource = @resources[0]
+  @transformation = Factory(:transformation, {
+    :resource => resource, :transformer => @transformer,
+    :source_field => resource.fields_dataset[:name => field]
+  })
+end
+
+Given /^that I have created a self-linkage scenario called "([^"]*)"$/ do |scenario_name|
+  @scenario = Factory(:scenario, {
+    :name => scenario_name, :resource_1_id => @resources[0].id,
+    :project => @project
+  })
+end
+
+Given /^that I have created a dual-linkage scenario called "([^"]*)"$/ do |scenario_name|
+  @scenario = Factory(:scenario, {
+    :name => scenario_name, :project => @project,
+    :resource_1_id => @resources[0].id, :resource_2_id => @resources[1].id
+  })
+end
+
+Given /^that I have added a matcher with these options:$/ do |table|
+  comparisons_attributes = []
+  resources = @scenario.resources
+  table.hashes.each do |hash|
+    lhs_value = hash["Type 1"] == "field" ? resources[0].fields_dataset[:name => hash["Value 1"]].id : hash["Value 1"]
+    rhs_value = hash["Type 2"] == "field" ? resources[-1].fields_dataset[:name => hash["Value 2"]].id : hash["Value 2"]
+    comparisons_attributes << {
+      "lhs_type" => hash["Type 1"], "lhs_value" => lhs_value,
+      "rhs_type" => hash["Type 2"], "rhs_value" => rhs_value,
+      "operator" => hash["Operator"]
+    }
+  end
+  @matcher = Factory(:matcher, {
+    :comparisons_attributes => comparisons_attributes,
+    :scenario => @scenario
+  })
+end
+
+When /^I go to the (.+?) page$/ do |page_name|
+  @page_name = page_name
+  path = case page_name
+         when "front"
+           "/"
+         when "connections"
+           "/connections"
+         when "projects"
+           "/projects"
+         when "project"
+           "/projects/#{@project.id}"
+         when "resource"
+           "/projects/#{@project.id}/resources/#{@resources[0].id}"
+         when "transformations"
+           "/projects/#{@project.id}/resources/#{@resources[0].id}/transformations"
+         when "scenario"
+           "/projects/#{@project.id}/scenarios/#{@scenario.id}"
+         end
+  visit("http://localhost:4567#{path}")
+end
+
+When /^I click the "(.+?)" link$/ do |link_name|
+  link = browser.link(:text, link_name)
+  assert link.exist?, "can't find the '#{link_name}' link"
+  link.click
+end
+
+When /^I fill in the form:$/ do |table|
+  form = browser.forms.first
+  table.raw.each do |(label_or_name, value)|
+    md = label_or_name.match(/\s*\((\d+)\)\s*$/)
+    args = md ? [md.pre_match, md[1].to_i - 1] : [label_or_name]
+    elt = find_visible_element_by_label_or_id(*args)
+    if elt.nil?
+      puts current_page_source
+    end
+    assert elt && elt.exist?, "can't find element with label or name of '#{label_or_name}'"
+
+    case elt
+    when Celerity::TextField
+      elt.value = value
+    when Celerity::SelectList
+      value.split("/").each do |v|
+        elt.object.get_options.find do |option|
+          option.text_content == v || option.value_attribute == v
+        end.click
+      end
+    else
+      raise "unknown field type: #{elt.to_xml}"
+    end
+  end
+end
+
+When /^I click the "(.+?)" button$/ do |button_name|
+  click_button(button_name)
+end
+
+When /^I click the "(.+?)" button with confirmation$/ do |button_name|
+  browser.confirm(true) do
+    click_button(button_name)
+  end
+end
+
+Then /^(?:it should )?show me a confirmation notice$/ do
+  assert_match /successfully created|created successfully/, current_page_source
+end
+
+Then /^ask me to (.+)$/ do |question|
+  question = question.sub(/\bI\b/, "you")
+  assert current_page_source.include?(question)
+end
+
+Then /^it should take me back to the (\w+) page$/ do |page_name|
+  case page_name
+  when 'resource'
+    assert_match %r{/projects/#{@project.id}/resources/#{@resources[0].id}$}, current_url
+  end
+end

data/features/step_definitions/matchers_steps.rb

@@ -0,0 +1,26 @@
+When /^I select "([^\"]*)" as the (first|second) type$/ do |type, which|
+  name = case which
+         when "first" then "lhs"
+         when "second" then "rhs"
+         else raise "bad which"
+         end
+  browser.select_list(:id => "#{name}_type").select(type)
+end
+
+When /^I select "([^\"]*)" for "([^\"]*)" as the (first|second) value$/ do |field, resource, which|
+  name = case which
+         when "first" then "lhs"
+         when "second" then "rhs"
+         else raise "bad which"
+         end
+  s = browser.select_list(:id => "#{name}_value_select")
+  optgroup = s.object.child_nodes.detect do |node|
+    Java::ComGargoylesoftwareHtmlunitHtml::HtmlOptionGroup === node &&
+      node.label_attribute == resource
+  end
+  option = optgroup.child_nodes.detect do |node|
+    Java::ComGargoylesoftwareHtmlunitHtml::HtmlOption === node &&
+      node.text == field
+  end
+  s.object.set_selected_attribute(option, true)
+end

data/features/step_definitions/resources_steps.rb

@@ -0,0 +1,12 @@
+When /I click on the (\w+) icon next to the "([^"]+)" field/ do |icon, field|
+  image = browser.cell(:text => field).parent.image(:src => /#{icon}/)
+  image.parent.click
+end
+
+When /I choose a "([^"]+)" resource/ do |name|
+  browser.execute_script(%{$('#resource-#{name.downcase}').show();})
+end
+
+Then /^it should start transforming$/ do
+  assert_match /running/, current_page_source
+end

data/features/step_definitions/scenarios_steps.rb

@@ -0,0 +1,7 @@
+When /^I click the "([^"]+)" resource/ do |resource_name|
+  browser.li(:text, resource_name).click
+end
+
+Then /^it should start the linkage process$/ do
+  assert_match /running/, current_page_source
+end

data/features/step_definitions/transformations_steps.rb

@@ -0,0 +1,3 @@
+Then /^there should be no more transformations$/ do
+  assert_equal 0, @resources.last.transformations_dataset.count
+end

data/features/support/env.rb

@@ -0,0 +1,128 @@
+require File.dirname(__FILE__) + "/../../test/helper"
+require 'test/unit/assertions'
+require 'celerity'
+
+module CouplerWorld
+  include Test::Unit::Assertions
+
+  def app
+    Coupler::Base.set :environment, :test
+    Coupler::Base
+  end
+
+  def browser
+    unless @browser
+      @browser = Celerity::Browser.new({
+        :javascript_exceptions => true, :viewer => "127.0.0.1:6429",
+        :resynchronize => true, :css => true
+      })
+    end
+    @browser
+  end
+
+  def visit(url)
+    browser.goto(url)
+  end
+
+  def current_url
+    browser.page.web_response.request_url.to_string
+  end
+
+  def current_page_source
+    return nil    unless browser.page
+
+    string_writer = java.io.StringWriter.new
+    print_writer  = java.io.PrintWriter.new(string_writer)
+
+    root = browser.page.document_element
+    node_to_xml(root, print_writer)
+    print_writer.close
+    string_writer.to_string
+  end
+
+  # Fill in a text field with a value
+  def fill_in(label_or_name, options = {})
+    elt = find_element_by_label_or_name(:text_field, label_or_name)
+    if elt.exist?
+      elt.value = options[:with]
+    end
+  end
+
+  def select(option_text, options = {})
+    elt = find_element_by_label_or_name(:select_list, options[:from])
+    if elt.exist?
+      begin
+        elt.select(option_text)
+      rescue Celerity::Exception::CelerityException
+        elt.select_value(option_text)
+      end
+    end
+  end
+
+  def click_button(button_value)
+    browser.button(button_value).click
+  end
+
+  def find_visible_element_by_label_or_id(label_or_id, which = 0)
+    labels = browser.labels.select do |label|
+      object = label.object
+      object.textContent == label_or_id && object.displayed?
+    end
+    id = labels.empty? ? label_or_id : labels[which].for
+    browser.elements_by_xpath("id('#{id}')")[0]
+  end
+
+  private
+    # NOTE: I have to do this because HtmlUnit's asXml likes to put spaces
+    #       and newlines in the middle of whitespace-sensitive tags (like
+    #       <a> and <textarea>).  Fail.
+    def node_to_xml(node, print_writer)
+      closing_tag = false
+
+      case node
+      when HtmlUnit::Html::DomText
+        # just print out the text
+        print_writer.write(HtmlUnit::Util::StringUtils.escape_xml_chars(node.data))
+      when HtmlUnit::Html::DomCDataSection, HtmlUnit::Html::DomProcessingInstruction
+        # use default printXml here
+        node.print_xml("", print_writer)
+      when HtmlUnit::Html::DomComment
+        print_writer.write("<!-- #{node.data} -->")
+      when HtmlUnit::Html::HtmlTextArea
+        node_print_opening_tag(node, print_writer)
+        print_writer.write(node.text)
+        closing_tag = true
+      else
+        node_print_opening_tag(node, print_writer)
+        closing_tag = true
+      end
+
+      child = node.first_child
+      while child
+        node_to_xml(child, print_writer)
+        child = child.next_sibling
+      end
+
+      node_print_closing_tag(node, print_writer) if closing_tag
+    end
+
+    def node_print_opening_tag(node, print_writer)
+      print_writer.write("<")
+      node.print_opening_tag_content_as_xml(print_writer)
+      print_writer.write(">")
+    end
+
+    def node_print_closing_tag(node, print_writer)
+      print_writer.write("</#{node.tag_name}>")
+    end
+end
+
+Before do
+  database = Coupler::Database.instance
+  database.tables.each do |name|
+    next  if name == :schema_info
+    database[name].delete
+  end
+end
+
+World(CouplerWorld)