surveyor 0.19.7 → 0.20.0

data/CHANGELOG

@@ -1,3 +1,25 @@
+0.20.0
+
+* allow answers to be images. closes #171
+* moving from answer.hide_label => true to answer.display_type => 'hidden_label'
+* don't parse redcap formulas. really closes #179
+* allow more of less whitespace in answer options. closes #179
+* fixing date, time, and datetime saving via ajax. closes #133
+* not using separate gemset, using passenger in development
+* fixed an issue where the last date was sent instead of the currently choosen date for a date field
+* Save grid responses and ids. Closes #161
+* dropdowns should save their response ids. closes #180
+* README should reflect that it is 2011
+* separate file for custom css and gem upgrade message. closes #151
+* benchmarking surveyor dump for #166
+* two-lettered rule keys. this one actually closes #159
+* two-lettered rule keys. closes #159
+* fix parsing of question and answer references with q_ and a_ in their names. closes #170
+* more redcap branching logic. closes #177
+* dynamically require some columns. closes #178
+* Fixes problem with updating checkboxes on postgers.
+* Fix problem when calling Answer.exists?('') under postgers.
+
 0.19.7
 
 * fix accidentally allowing through blank pick => any answers

data/README.md

@@ -111,4 +111,4 @@ To work on the code fork this github project. Run:
 
 which will generate a test app in testbed. Run rake spec and rake cucumber there, and start writing tests!
 
-Copyright (c) 2008-2010 Brian Chamberlain and Mark Yoon, released under the MIT license
+Copyright (c) 2008-2011 Brian Chamberlain and Mark Yoon, released under the MIT license

data/Rakefile

@@ -15,6 +15,7 @@ begin
     gem.add_dependency 'formtastic'
     gem.add_dependency 'uuid'
     gem.add_development_dependency "yard", ">= 0"
+    gem.post_install_message = "Thanks for installing surveyor! The time has come to run the surveyor generator and migrate your database, even if you are upgrading."
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.19.7
+0.20.0

data/app/helpers/surveyor_helper.rb

@@ -4,7 +4,7 @@ module SurveyorHelper
     surveyor_stylsheets + surveyor_javascripts    
   end
   def surveyor_stylsheets
-    stylesheet_link_tag 'surveyor/reset', 'surveyor/dateinput', 'surveyor'
+    stylesheet_link_tag 'surveyor/reset', 'surveyor/dateinput', 'surveyor', 'custom'
   end
   def surveyor_javascripts
     javascript_include_tag 'surveyor/jquery.tools.min', 'surveyor/jquery.surveyor'
@@ -50,6 +50,10 @@ module SurveyorHelper
   # end
   
   # Answers
+  def a_text(obj, pos=nil)
+    return image_tag(obj.text) if obj.is_a?(Answer) and obj.display_type == "image"
+    obj.split_or_hidden_text(pos)
+  end  
   def rc_to_attr(type_sym)
     case type_sym.to_s
     when /^date|time$/ then :datetime_value

data/app/views/partials/_answer.html.haml

@@ -8,12 +8,12 @@
   = ff.quiet_input :response_group, :value => rg if q.pick != "one" && g && g.display_type == "repeater"
   - case q.pick
   - when "one"
-    = ff.input :answer_id, :as => :surveyor_radio, :collection => [[a.text, a.id]], :label => false, :input_html => {:class => a.css_class}, :response_class => a.response_class
+    = ff.input :answer_id, :as => :surveyor_radio, :collection => [[a_text(a), a.id]], :label => false, :input_html => {:class => a.css_class}, :response_class => a.response_class
   - when "any"
-    = ff.input :answer_id, :as => :surveyor_check_boxes, :collection => [[a.text, a.id]], :label => false, :input_html => {:class => a.css_class}, :response_class => a.response_class
+    = ff.input :answer_id, :as => :surveyor_check_boxes, :collection => [[a_text(a), a.id]], :label => false, :input_html => {:class => a.css_class}, :response_class => a.response_class
   - when "none"
     - if %w(date datetime time float integer string text).include? a.response_class
       = ff.quiet_input :answer_id, :input_html => {:class => a.css_class, :value => a.id}
-      = ff.input rc_to_attr(a.response_class), :as => rc_to_as(a.response_class), :label => a.split_or_hidden_text(:pre).blank? ? false : a.split_or_hidden_text(:pre), :hint => a.split_or_hidden_text(:post), :input_html => generate_pick_none_input_html(r.as(a.response_class), a.default_value, a.css_class)
+      = ff.input rc_to_attr(a.response_class), :as => rc_to_as(a.response_class), :label => a_text(a, :pre).blank? ? false : a_text(a, :pre), :hint => a_text(a, :post), :input_html => generate_pick_none_input_html(r.as(a.response_class), a.default_value, a.css_class)
     - else
-      = a.text
+      = a_text(a)

data/app/views/partials/_question.html.haml

@@ -11,7 +11,8 @@
     - f.semantic_fields_for i, r do |ff|
       = ff.quiet_input :question_id
       = ff.quiet_input :response_group, :value => rg if g && g.display_type == "repeater"
-      = ff.input :answer_id, :as => :select, :collection => q.answers.map{|a| [a.text, a.id]}, :label => false
+      = ff.quiet_input :id unless r.new_record?
+      = ff.input :answer_id, :as => :select, :collection => q.answers.map{|a| [a.text, a.id]}, :label => q.text
   - else # :default, :inline, :inline_default
     - if q.pick == "one"
       - r = response_for(@response_set, q, nil, rg)

data/app/views/partials/_question_group.html.haml

@@ -14,12 +14,17 @@
               %tr
                 %th  
                 - ten_questions.first.answers.each do |a|
-                  %th= a.text
+                  %th= a_text(a)
                 %th  
               - ten_questions.each_with_index do |q, i|
                 %tr{:id => "q_#{q.id}", :class => "q_#{renderer} #{q.css_class(@response_set)}"}
+                  - if q.pick == "one"
+                    - r = response_for(@response_set, q, nil, g)
+                    - i = response_idx  # increment the response index since the answer partial skips for q.pick == one
+                    - f.semantic_fields_for i, r do |ff|
+                      = ff.quiet_input :question_id
+                      = ff.quiet_input :id unless r.new_record?
                   %th= q.split_text(:pre)
-                  - response_idx if q.pick == "one" # increment the response index since the answer partial skips for q.pick == one
                   - q.answers.each do |a|
                     %td= render a.custom_renderer || '/partials/answer', :g => g, :q => q, :a => a, :f => f
                   %th= q.split_text(:post)

data/features/redcap_parser.feature

@@ -31,3 +31,27 @@ Feature: Survey creation
     And there should be 1 dependencies with:
       | rule    |
       | A       |
+  Scenario: with different headers
+    Given I parse redcap file "redcap_new_headers.csv"
+    Then there should be 1 survey with:
+      ||
+    And there should be 1 questions with:
+      ||
+    And there should be 2 answers with:
+      ||
+  @focus
+  Scenario: with different whitespace
+    Given I parse redcap file "redcap_whitespace.csv"
+    Then there should be 1 survey with:
+      ||
+    And there should be 2 questions with:
+      ||
+    And there should be 7 answers with:
+      | reference_identifier | text    |
+      | 1                    | Lexapro |
+      | 2                    | Celexa  |
+      | 3                    | Prozac  |
+      | 4                    | Paxil   |
+      | 5                    | Zoloft  |
+      | 0                    | No      |
+      | 1                    | Yes     |

data/features/step_definitions/surveyor_steps.rb

@@ -22,7 +22,7 @@ Then /^there should be (\d+) response set with (\d+) responses? with:$/ do |rs_n
   end
 end
 
-Then /^there should be (\d+) dependencies$/ do |x|
+Then /^there should be (\d+) dependenc(?:y|ies)$/ do |x|
   Dependency.count.should == x.to_i
 end
 
@@ -60,3 +60,24 @@ Then /^there should be (\d+) text areas$/ do |count|
   response.should have_selector('textarea', :count => count.to_i)
 end
 
+Then /^the question "([^"]*)" should be triggered$/ do |text|
+  response.should have_selector %(fieldset[name="#{text}"][class!="q_hidden"])
+end
+
+Then /^there should be (\d+) response with answer "([^"]*)"$/ do |count, answer_text|
+  Response.count.should == count.to_i
+  Response.find_by_answer_id(Answer.find_by_text(answer_text)).should_not be_blank
+end
+
+Then /^there should be (\d+) datetime responses with$/ do |count, table|
+  Response.count.should == count.to_i
+  table.hashes.each do |hash|
+    if hash.keys == ["datetime_value"]
+      Response.find_by_datetime_value(DateTime.parse(hash["datetime_value"])).should_not be_blank
+    end
+  end
+end
+
+Then /^I should see the image "([^"]*)"$/ do |src|
+  response.should have_selector %(img[src^="#{src}"])
+end

data/features/support/redcap_new_headers.csv

@@ -0,0 +1 @@
+Variable / Field Name,Form Name,Field Units,Section Header,Field Type,Field Label,Choices Calculations OR Slider Labels,Field Note,Text Validation Type OR Show Slider Number,Text Validation Min,Text Validation Max,Identifier?,Branching Logic (Show field only if...),Required Field?

data/features/support/redcap_whitespace.csv

@@ -0,0 +1 @@
+Variable / Field Name,Form Name,Field Units,Section Header,Field Type,Field Label,Choices OR Calculations,Field Note,Text Validation Type,Text Validation Min,Text Validation Max,Identifier?,Branching Logic (Show field only if...),Required Field?

data/features/surveyor.feature

@@ -168,4 +168,164 @@ Feature: Survey creation
     """
     When I start the "Websites" survey
     Then there should be 3 checkboxes
-    And there should be 3 text areas
+    And there should be 3 text areas
+
+  Scenario: "Double letter rule keys"
+    Given the survey
+    """
+      survey "Doubles" do
+        section "Two" do
+          q_twin "Are you a twin?", :pick => :one
+          a_yes "Oh yes"
+          a_no "Oh no"
+
+          q_two_first_names "Do you have two first names?", :pick => :one
+          a_yes "Why yes"
+          a_no "Why no"
+
+          q "Do you want to be part of an SNL skit?", :pick => :one
+          a_yes "Um yes"
+          a_no "Um no"
+          dependency :rule => "A or AA"
+          condition_A :q_twin, "==", :a_yes
+          condition_AA :q_two_first_names, "==", :a_yes
+        end
+        section "Deux" do
+          label "Here for the ride"
+        end
+        section "Three" do
+          label "Here for the ride"
+        end
+      end
+    """
+    When I start the "Doubles" survey
+    Then I choose "Oh yes"
+    And I press "Deux"
+    And I press "Two"
+    Then the question "Do you want to be part of an SNL skit?" should be triggered
+
+  Scenario: "Changing dropdowns"
+    Given the survey
+    """
+      survey "Drop" do
+        section "Like it is hot" do
+          q "Name", :pick => :one, :display_type => :dropdown
+          a "Snoop"
+          a "Dogg"
+          a "D-O double G"
+          a "S-N double O-P, D-O double G"
+        end
+        section "Two" do
+          label "Here for the ride"
+        end
+        section "Three" do
+          label "Here for the ride"
+        end
+      end
+    """
+    When I start the "Drop" survey
+    Then I select "Snoop" from "Name"
+    And I press "Two"
+    And I press "Like it is hot"
+    And I select "Dogg" from "Name"
+    And I press "Two"
+    Then there should be 1 response with answer "Dogg"
+
+  Scenario: "Saving grids"
+    Given the survey
+    """
+      survey "Grid" do
+        section "One" do
+          grid "Tell us how often do you cover these each day" do
+            a "1"
+            a "2"
+            a "3"
+            q "Head", :pick => :one
+            q "Knees", :pick => :one
+            q "Toes", :pick => :one
+          end
+        end
+        section "Two" do
+          label "Here for the ride"
+        end
+        section "Three" do
+          label "Here for the ride"
+        end
+      end
+    """
+    When I start the "Grid" survey
+    Then I choose "1"
+    And I press "Two"
+    And I press "One"
+    Then there should be 1 response with answer "1"
+
+  Scenario: "Dates"
+    Given the survey
+    """
+      survey "When" do
+        section "One" do
+          q "Tell us when you want to meet"
+          a "Give me a date", :date
+        end
+        section "Two" do
+          q "Tell us when you'd like to eat"
+          a :time
+        end
+        section "Three" do
+          q "Tell us when you'd like a phone call"
+          a :datetime
+        end
+      end
+    """
+    When I start the "When" survey
+    # 2/14/11
+    And I fill in "Give me a date" with "2011-02-14"
+    # 1:30am
+    And I press "Two"
+    And I select "01" from "Hour"
+    And I select "30" from "Minute"
+    # 2/15/11 5:30pm
+    And I press "Three"
+    And I select "2011" from "Year"
+    And I select "February" from "Month"
+    And I select "15" from "Day"
+    And I select "17" from "Hour"
+    And I select "30" from "Minute"
+    And I press "One"
+
+    Then there should be 3 datetime responses with
+      | datetime_value      |
+      | 2011-02-14 00:00:00 |
+      | 2001-01-01 01:30:00 |
+      | 2011-02-15 17:30:00 |
+    
+    # 2/13/11
+    And I fill in "Give me a date" with "2011-02-13"
+    # 1:30pm
+    And I press "Two"
+    And I select "13" from "Hour"
+    # 2/15/11 5:00pm
+    And I press "Three"
+    And I select "00" from "Minute"
+    And I press "Click here to finish"
+    
+    Then there should be 3 datetime responses with
+      | datetime_value      |
+      | 2011-02-13 00:00:00 |
+      | 2001-01-01 13:30:00 |
+      | 2011-02-15 17:00:00 |
+
+  Scenario: "Images"
+    Given the survey
+    """
+      survey "Images" do
+        section "One" do
+          q "Which way?"
+          a "/images/surveyor/next.gif", :display_type => "image"
+          a "/images/surveyor/prev.gif", :display_type => "image"
+        end
+      end
+    """
+    When I start the "Images" survey
+    Then I should see the image "/images/surveyor/next.gif"
+    And I should see the image "/images/surveyor/prev.gif"

data/features/surveyor_parser.feature

@@ -238,4 +238,52 @@ Feature: Survey creation
     """
     Then there should be 4 dependencies
     And 2 dependencies should depend on questions
-    And 2 dependencies should depend on question groups
+    And 2 dependencies should depend on question groups
+
+  Scenario: Dependencies with "a"
+    Given the survey
+    """
+      survey "Dependencies with 'a'" do
+        section "First" do
+          q_data_collection "Disease data collection", :pick => :one
+          a_via_chart_review "Via chart review"
+          a_via_patient_interview "Via patient interview/questionnaire"
+
+          q_myocardial_infaction "Myocardinal Infarction", :pick => :one
+          dependency :rule => "A"
+          condition_A :q_data_collection, "==", :a_via_chart_review
+          a_yes "Yes"
+          a_no "No"
+        end
+      end
+    """
+    And there should be 1 dependency with:
+      | rule |
+      | A    |
+    And there should be 1 resolved dependency_condition with:
+      | rule_key |
+      | A        |
+  
+  Scenario: Dependencies with "q"
+    Given the survey
+    """
+      survey "Dependencies with 'q'" do
+        section "First" do
+          q_rawq_collection "Your rockin rawq collection", :pick => :one
+          a_rawqs "Rawqs"
+          a_doesnt_rawq "Doesn't rawq"
+
+          q_do_you_rawq "Do you rawq with your rockin rawq collection?", :pick => :one
+          dependency :rule => "A"
+          condition_A :q_rawq_collection, "==", :a_rawqs
+          a_yes "Yes"
+          a_no "No"
+        end
+      end
+    """
+    And there should be 1 dependency with:
+      | rule |
+      | A    |
+    And there should be 1 resolved dependency_condition with:
+      | rule_key |
+      | A        |

data/generators/surveyor/surveyor_generator.rb

@@ -28,7 +28,8 @@ class SurveyorGenerator < Rails::Generator::Base
         "add_display_order_to_surveys", "add_correct_answer_id_to_questions",
         "add_index_to_response_sets", "add_index_to_surveys", 
         "add_unique_indicies", "add_section_id_to_responses",
-        "add_default_value_to_answers", "add_api_ids"].each_with_index do |model, i|
+        "add_default_value_to_answers", "add_api_ids",
+        "add_display_type_to_answers"].each_with_index do |model, i|
         unless (prev_migrations = Dir.glob("db/migrate/[0-9]*_*.rb").grep(/[0-9]+_#{model}.rb$/)).empty?
           prev_migration_timestamp = prev_migrations[0].match(/([0-9]+)_#{model}.rb$/)[1]
         end
@@ -45,7 +46,7 @@ class SurveyorGenerator < Rails::Generator::Base
       end
       m.directory "public/stylesheets/sass"
       m.file "assets/stylesheets/sass/surveyor.sass", "public/stylesheets/sass/surveyor.sass"
-      
+      m.file "assets/stylesheets/sass/custom.sass", "public/stylesheets/sass/custom.sass"
 
       # Locales
       m.directory "config/locales"

data/generators/surveyor/templates/assets/javascripts/jquery.surveyor.js

@@ -2,25 +2,37 @@
 jQuery(document).ready(function(){
   // if(jQuery.browser.msie){
   //  // IE has trouble with the change event for form radio/checkbox elements - bind click instead
-  //  jQuery("form#survey_form input[type=radio], form#survey_form [type=checkbox]").bind("click", function(){ 
+  //  jQuery("form#survey_form input[type=radio], form#survey_form [type=checkbox]").bind("click", function(){
   //    jQuery(this).parents("form").ajaxSubmit({dataType: 'json', success: successfulSave});
   //  });
   //  // IE fires the change event for all other (not radio/checkbox) elements of the form
-  //  jQuery("form#survey_form *").not("input[type=radio], input[type=checkbox]").bind("change", function(){ 
-  //    jQuery(this).parents("form").ajaxSubmit({dataType: 'json', success: successfulSave}); 
+  //  jQuery("form#survey_form *").not("input[type=radio], input[type=checkbox]").bind("change", function(){
+  //    jQuery(this).parents("form").ajaxSubmit({dataType: 'json', success: successfulSave});
   //  });
   // }else{
   //  // Other browsers just use the change event on the form
+
+  // For a date input, i.e. using dateinput from jQuery tools, the value is not updated
+  // before the onChange or change event is fired, so we hang this in before the update is
+  // sent to the server and set the correct value from the dateinput object.
+  jQuery('li.date input').change(function(){
+      if ( $(this).data('dateinput') ) {
+          var date_obj = $(this).data('dateinput').getValue();
+          this.value = date_obj.getFullYear() + "-" + (date_obj.getMonth()+1) + "-" +
+              date_obj.getDate() + " 00:00:00 UTC";
+      }
+  });
+
   jQuery("form#survey_form input, form#survey_form select, form#survey_form textarea").change(function(){
-    question_data = $(this).parents('fieldset[id^="q_"]').find("input, select, textarea").add($("form#survey_form input[name='authenticity_token']")).serialize();
+    question_data = $(this).parents('fieldset[id^="q_"],tr[id^="q_"]').find("input, select, textarea").add($("form#survey_form input[name='authenticity_token']")).serialize();
     // console.log(unescape(question_data));
     $.ajax({ type: "PUT", url: $(this).parents('form#survey_form').attr("action"), data: question_data, dataType: 'json', success: successfulSave })
   });
   // }
-  
+
   // If javascript works, we don't need to show dependents from previous sections at the top of the page.
   jQuery("#dependents").remove();
-  
+
   function successfulSave(responseText){ // for(key in responseText) { console.log("key is "+[key]+", value is "+responseText[key]); }
     // surveyor_controller returns a json object to show/hide elements and insert/remove ids e.g. {"ids": {"2" => 234}, "remove": {"4" => 21}, "hide":["question_12","question_13"],"show":["question_14"]}
     jQuery.each(responseText.show, function(){ jQuery('#' + this).show("fast"); });
@@ -29,7 +41,7 @@ jQuery(document).ready(function(){
     jQuery.each(responseText.remove, function(k,v){ jQuery('#r_'+k+'_id[value="'+v+'"]').remove(); });
     return false;
   }
-  
+
   // is_exclusive checkboxes should disble sibling checkboxes
   $('input.exclusive:checked').parents('fieldset[id^="q_"]').find(':checkbox').not(".exclusive").attr('checked', false).attr('disabled', true);
   $('input.exclusive:checkbox').click(function(){
@@ -41,6 +53,8 @@ jQuery(document).ready(function(){
       others.attr('disabled', false);
     }
   });
-  
-  $('li.date input').dateinput();
+
+  $('li.date input').dateinput({
+      format: 'dd mmm yyyy'
+  });
 });

data/generators/surveyor/templates/assets/stylesheets/sass/custom.sass

@@ -0,0 +1,5 @@
+/* Add custom styles here. They will be
+ * loaded after the surveyor stylesheets */
+ 
+#surveyor
+ :font-family "Century Gothic"

data/generators/surveyor/templates/assets/stylesheets/sass/surveyor.sass

@@ -1,3 +1,9 @@
+/* Do not change surveyor.sass or the generated
+ * surveyor.css. They will be overwritten by
+ * the surveyor generator. Instead, add
+ * custom styles to custom.sass, which will be
+ * loaded after the surveyor stylesheets */
+
 $background_color:#EEEEEE
 $surveyor_flash_background_color:#FFF1A8
 $surveyor_color:#FFFFF1 
@@ -103,8 +109,9 @@ body
     legend
       :background-color $surveyor_dependent_color
       :padding 3px 3px 3px 0
-
-    
+  fieldset.q_dropdown, fieldset.q_inline_dropdown, fieldset.q_slider, fieldset.q_repeater_dropdown
+    label
+      :display none
     
   // buttons
   input[type="submit"]

data/generators/surveyor/templates/migrate/add_display_type_to_answers.rb

@@ -0,0 +1,13 @@
+class AddDisplayTypeToAnswers < ActiveRecord::Migration
+  def self.up
+    add_column :answers, :display_type, :string
+    Answer.all.each{|a| a.update_attributes(:display_type => "hidden_label") if a.hide_label == true}
+    remove_column :answers, :hide_label
+  end
+
+  def self.down
+    add_column :answers, :hide_label, :boolean
+    Answer.all.each{|a| a.update_attributes(:hide_label => true) if a.display_type == "hidden_label"}
+    remove_column :answers, :display_type
+  end
+end

data/lib/surveyor/models/answer_methods.rb

@@ -31,7 +31,7 @@ module Surveyor
       def default_args
         self.display_order ||= self.question ? self.question.answers.count : 0
         self.is_exclusive ||= false
-        self.hide_label ||= false
+        self.display_type ||= "default"
         self.response_class ||= "answer"
         self.short_text ||= text
         self.data_export_identifier ||= Surveyor::Common.normalize(text)
@@ -43,7 +43,7 @@ module Surveyor
       end
       
       def split_or_hidden_text(part = nil)
-        return "" if hide_label.to_s == "true"
+        return "" if display_type == "hidden_label"
         part == :pre ? text.split("|",2)[0] : (part == :post ? text.split("|",2)[1] : text)
       end
       

data/lib/surveyor/models/dependency_methods.rb

@@ -42,7 +42,7 @@ module Surveyor
         # logger.debug "rexp: #{rgx.inspect}"
         # logger.debug "keyp: #{ch.inspect}"
         # logger.debug "subd: #{self.rule.gsub(rgx){|m| ch[m.to_sym]}}"
-        rgx = Regexp.new(self.dependency_conditions.map{|dc| ["a","o"].include?(dc.rule_key) ? "#{dc.rule_key}(?!nd|r)" : dc.rule_key}.join("|")) # exclude and, or
+        rgx = Regexp.new(self.dependency_conditions.map{|dc| ["a","o"].include?(dc.rule_key) ? "\\b#{dc.rule_key}(?!nd|r)\\b" : "\\b#{dc.rule_key}\\b"}.join("|")) # exclude and, or
         eval(self.rule.gsub(rgx){|m| ch[m.to_sym]})
       end
 

data/lib/surveyor/models/response_methods.rb

@@ -20,7 +20,7 @@ module Surveyor
           def applicable_attributes(attrs)
             result = HashWithIndifferentAccess.new(attrs)
             answer_id = result[:answer_id].is_a?(Array) ? result[:answer_id].last : result[:answer_id] # checkboxes are arrays / radio buttons are not arrays
-            if result[:string_value] && Answer.exists?(answer_id)
+            if result[:string_value] && !answer_id.blank? && Answer.exists?(answer_id)
               answer = Answer.find(answer_id)
               result.delete(:string_value) unless answer.response_class && answer.response_class.to_sym == :string
             end

data/lib/surveyor/models/response_set_methods.rb

@@ -42,6 +42,13 @@ module Surveyor
             return false if (q = Question.find_by_id(hash["question_id"])) and q.pick == "one"
             hash.any?{|k,v| v.is_a?(Array) ? v.all?{|x| x.to_s.blank?} : v.to_s.blank?}
           end
+          def trim_for_lookups(hash_of_hashes)
+            result = {}
+            (reject_or_destroy_blanks(hash_of_hashes) || {}).each_pair do |k, hash|
+              result.merge!({k => {"question_id" => hash["question_id"], "answer_id" => hash["answer_id"]}.merge(hash.has_key?("response_group") ? {"response_group" => hash["response_group"]} : {} ).merge(hash.has_key?("id") ? {"id" => hash["id"]} : {} ).merge(hash.has_key?("_destroy") ? {"_destroy" => hash["_destroy"]} : {} )})
+            end
+            result
+          end
         end
       end
 

data/lib/surveyor/parser.rb

@@ -192,11 +192,11 @@ class DependencyCondition < ActiveRecord::Base
         { 
           :context_reference => context,
           :operator => a1 || "==",
-          :question_reference => a0.to_s.gsub("q_", ""),
+          :question_reference => a0.to_s.gsub(/^q_/, ""),
           :rule_key => reference_identifier 
         }.merge(
             a2.is_a?(Hash) ? a2 : { :answer_reference => 
-                                      a2.to_s.gsub("a_", "") }
+                                      a2.to_s.gsub(/^a_/, "") }
           )
       )
   end
@@ -258,7 +258,7 @@ class Answer < ActiveRecord::Base
     when :none, :omit # is_exclusive erases and disables other checkboxes and input elements
       self.text_args(arg.to_s.humanize).merge({:is_exclusive => true})
     when :integer, :date, :time, :datetime, :text, :datetime, :string
-      self.text_args(arg.to_s.humanize).merge({:response_class => arg.to_s, :hide_label => true})
+      self.text_args(arg.to_s.humanize).merge({:response_class => arg.to_s, :display_type => "hidden_label"})
     end
   end
 end

data/lib/surveyor/redcap_parser.rb

@@ -24,7 +24,7 @@ module Surveyor
       begin
         csvlib.parse(str, :headers => :first_row, :return_headers => true, :header_converters => :symbol) do |r|
           if r.header_row? # header row
-            return puts "Missing headers: #{missing_columns(r).inspect}\n\n" unless missing_columns(r).blank?
+            return puts "Missing headers: #{missing_columns(r.headers).inspect}\n\n" unless missing_columns(r.headers).blank?
             context[:survey] = Survey.new(:title => filename)
             print "survey_#{context[:survey].access_code} "
           else # non-header rows
@@ -44,10 +44,14 @@ module Surveyor
       return context[:survey]
     end
     def missing_columns(r)
-      required_columns - r.headers.map(&:to_s)
+      missing = []
+      missing << "choices_or_calculations" unless r.map(&:to_s).include?("choices_or_calculations") or r.map(&:to_s).include?("choices_calculations_or_slider_labels")
+      missing << "text_validation_type" unless r.map(&:to_s).include?("text_validation_type") or r.map(&:to_s).include?("text_validation_type_or_show_slider_number") 
+      missing += (static_required_columns - r.map(&:to_s))
     end
-    def required_columns
-      %w(variable__field_name form_name field_units section_header field_type field_label choices_or_calculations field_note text_validation_type text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
+    def static_required_columns
+      # no longer requiring field_units
+      %w(variable__field_name form_name section_header field_type field_label field_note text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
     end
   end
 end
@@ -110,19 +114,18 @@ class Dependency < ActiveRecord::Base
     end
   end
   def self.decompose_component(str)
-     # [initial_52] = "1"
-    if match = str.match(/^\[(\w+)\] ?([!=><]+) ?"(\w+)"$/)
-      {:question_reference => match[1], :operator => match[2].gsub(/^=$/, "=="), :answer_reference => match[3]}
-    # [initial_119(2)] = "1"
-    elsif match = str.match(/^\[(\w+)\((\w+)\)\] ?([!=><]+) ?"1"$/)
-      {:question_reference => match[1], :operator => match[3].gsub(/^=$/, "=="), :answer_reference => match[2]}
-    # [f1_q15] >= 21
-    elsif match = str.match(/^\[(\w+)\] ?([!=><]+) ?(\d+)$/)
-      {:question_reference => match[1], :operator => match[2].gsub(/^=$/, "=="), :integer_value => match[3]}
-    # uhoh
+    # [initial_52] = "1" or [f1_q15] = '' or [f1_q15] = '-2' or [hi_event1_type] <> ''
+    if match = str.match(/^\[(\w+)\] ?([!=><]+) ?['"](-?\w*)['"]$/)
+      {:question_reference => match[1], :operator => match[2].gsub(/^=$/, "==").gsub(/^<>$/, "!="), :answer_reference => match[3]}
+    # [initial_119(2)] = "1" or [hiprep_heat2(97)] = '1'
+    elsif match = str.match(/^\[(\w+)\((\w+)\)\] ?([!=><]+) ?['"]1['"]$/)
+      {:question_reference => match[1], :operator => match[3].gsub(/^=$/, "==").gsub(/^<>$/, "!="), :answer_reference => match[2]}
+    # [f1_q15] >= 21 or [f1_q15] >= -21
+    elsif match = str.match(/^\[(\w+)\] ?([!=><]+) ?(-?\d+)$/)
+      {:question_reference => match[1], :operator => match[2].gsub(/^=$/, "==").gsub(/^<>$/, "!="), :integer_value => match[3]}
     else
       puts "\n!!! skipping dependency_condition #{str}"
-    end
+    end    
   end
   def self.decompose_rule(str)
     # see spec/lib/redcap_parser_spec.rb for examples
@@ -181,9 +184,9 @@ class Answer < ActiveRecord::Base
     when "file"
       puts "\n!!! skipping answer: file"
     end
-    r[:choices_or_calculations].to_s.split("|").each do |pair|
-      aref, atext = pair.strip.split(", ")
-      if aref.blank? or atext.blank?
+    (r[:choices_or_calculations] || r[:choices_calculations_or_slider_labels]).to_s.split("|").each do |pair|
+      aref, atext = pair.split(",").map(&:strip)
+      if aref.blank? or atext.blank? or (aref.to_i.to_s != aref)
         puts "\n!!! skipping answer #{pair}"
       else
         context[:answer] = context[:question].answers.build(:reference_identifier => aref, :text => atext)

data/lib/surveyor/surveyor_controller_methods.rb

@@ -75,8 +75,9 @@ module Surveyor
         end
         format.js do
           ids, remove, question_ids = {}, {}, []
-          ResponseSet.reject_or_destroy_blanks(params[:r]).each do |k,v|
-            ids[k] = @response_set.responses.find(:first, :conditions => v).id if !v.has_key?("id")
+          ResponseSet.trim_for_lookups(params[:r]).each do |k,v|
+            v[:answer_id].reject!(&:blank?) if v[:answer_id].is_a?(Array)
+            ids[k] = @response_set.responses.find(:first, :conditions => v, :order => "created_at DESC").id if !v.has_key?("id")
             remove[k] = v["id"] if v.has_key?("id") && v.has_key?("_destroy")
             question_ids << v["question_id"]
           end

data/lib/surveyor/unparser.rb

@@ -111,7 +111,7 @@ class Answer < ActiveRecord::Base
     dsl << "  " if question.part_of_group?
     dsl << "    a"
     dsl << "_#{reference_identifier}" unless reference_identifier.blank?
-    if response_class.to_s.titlecase == text && attrs == {:hide_label => true}
+    if response_class.to_s.titlecase == text && attrs == {:display_type => "hidden_label"}
       dsl << " :#{response_class}"
     else    
       dsl << [ text.blank? ? nil : text == "Other" ? " :other" : text == "Omit" ? " :omit" : " \"#{text}\"",

data/spec/factories.rb

@@ -71,7 +71,7 @@ Factory.define :answer do |a|
   a.common_identifier         {}
   a.display_order             {Factory.next :answer_display_order}
   a.is_exclusive              {}
-  a.hide_label                {}
+  a.display_type              "default"
   a.display_length            {}
   a.custom_class              {}
   a.custom_renderer           {}

data/spec/lib/benchmark_spec.rb

@@ -0,0 +1,22 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Surveyor do
+  it "should write thousands of response sets" do
+    Surveyor::Parser.parse(File.read(File.join(Rails.root, 'surveys', 'kitchen_sink_survey.rb')))
+    survey = Survey.last
+    rs = ResponseSet.create(:survey => survey)
+    survey.sections.each{|s| s.questions.each{|q| rs.responses.create(:question => q, :answer => q.answers.first)}}
+    Benchmark.bm 20 do |x|
+      x.report "a test" do
+        full_path = File.join(Rails.root,"#{survey.access_code}_#{Time.now.to_i}.csv")
+        File.open(full_path, 'w') do |f|
+          100.times do # adjust this to test
+            survey.response_sets.each_with_index{|r,i| f.write(r.to_csv(true, i == 0)) } # print access code every time, print_header first time
+          end
+        end        
+      end
+    end
+  end
+end
+
+

data/spec/lib/redcap_parser_spec.rb

@@ -4,6 +4,20 @@ describe Surveyor::RedcapParser do
   before(:each) do
     # @parser = Surveyor::Parser.new
   end
+  it "should require specific columns" do
+    # with standard fields
+    x = %w(field_units choices_or_calculations text_validation_type variable__field_name form_name  section_header field_type field_label field_note text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
+    Surveyor::RedcapParser.new.missing_columns(x).should be_blank
+    # without field_units
+    y = %w(choices_or_calculations text_validation_type variable__field_name form_name  section_header field_type field_label field_note text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
+    Surveyor::RedcapParser.new.missing_columns(y).should be_blank
+    # choices_or_calculations => choices_calculations_or_slider_labels
+    z = %w(field_units choices_calculations_or_slider_labels text_validation_type variable__field_name form_name  section_header field_type field_label field_note text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
+    Surveyor::RedcapParser.new.missing_columns(z).should be_blank
+    # text_validation_type => text_validation_type_or_show_slider_number
+    a = %w(field_units choices_or_calculations text_validation_type_or_show_slider_number variable__field_name form_name  section_header field_type field_label field_note text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
+    Surveyor::RedcapParser.new.missing_columns(a).should be_blank
+  end
   it "should decompose dependency rules" do
     # basic
     Dependency.decompose_rule('[f1_q12]="1"').should == {:rule => "A", :components => ['[f1_q12]="1"']}
@@ -44,6 +58,14 @@ describe Surveyor::RedcapParser do
     Dependency.decompose_component('[initial_52] = "1"').should == {:question_reference => 'initial_52', :operator => '==', :answer_reference => '1'}
     Dependency.decompose_component('[initial_119(2)] = "1"').should == {:question_reference => 'initial_119', :operator => '==', :answer_reference => '2'}
     Dependency.decompose_component('[f1_q15] >= 21').should == {:question_reference => 'f1_q15', :operator => '>=', :integer_value => '21'}
+    # basic, blanks
+    Dependency.decompose_component("[f1_q15]=''").should == {:question_reference => 'f1_q15', :operator => '==', :answer_reference => ''}
+    # basic, negatives
+    Dependency.decompose_component("[f1_q15]='-2'").should == {:question_reference => 'f1_q15', :operator => '==', :answer_reference => '-2'}
+    # internal parenthesis
+    Dependency.decompose_component("[hiprep_heat2(97)] = '1'").should == {:question_reference => 'hiprep_heat2', :operator => '==', :answer_reference => '97'}
+    Dependency.decompose_component("[hi_event1_type] <> ''").should == {:question_reference => 'hi_event1_type', :operator => '!=', :answer_reference => ''}
+    
   end
   it "should return a survey object" do
     x = %("Variable / Field Name","Form Name","Field Units","Section Header","Field Type","Field Label","Choices OR Calculations","Field Note","Text Validation Type","Text Validation Min","Text Validation Max",Identifier?,"Branching Logic (Show field only if...)","Required Field?"\nstudy_id,demographics,,,text,"Study ID",,,,,,,,)

data/spec/models/answer_spec.rb

@@ -22,9 +22,9 @@ describe Answer, "when creating a new answer" do
     @answer.css_class.should == "exclusive foo bar"
   end
   
-  it "should hide the label when hide_label is set" do
+  it "should hide the label when display_type hidden_label is set" do
     @answer.split_or_hidden_text.should == "Red"
-    @answer.hide_label = true
+    @answer.display_type = "hidden_label"
     @answer.split_or_hidden_text.should == ""
   end
   it "should split up pre/post labels" do

data/spec/models/response_set_spec.rb

@@ -125,6 +125,26 @@ describe ResponseSet do
       "32" => {"question_id" => @qone.id, "answer_id" => "291", "string_value" => ""} # new radio with blank string value, selected
     }
   end
+  it "should clean up responses for lookups to get ids after saving via ajax" do
+    hash_of_hashes = {"1"=>{"question_id"=>"2", "answer_id"=>"1"},
+      "2"=>{"question_id"=>"3", "answer_id"=>["", "6"]},
+      "9"=>{"question_id"=>"6", "string_value"=>"jack", "answer_id"=>"13"},
+      "17"=>{"question_id"=>"13", "datetime_value(1i)"=>"2006", "datetime_value(2i)"=>"2", "datetime_value(3i)"=>"4", "datetime_value(4i)"=>"02", "datetime_value(5i)"=>"05", "answer_id"=>"21"},
+      "18"=>{"question_id"=>"14", "datetime_value(1i)"=>"1", "datetime_value(2i)"=>"1", "datetime_value(3i)"=>"1", "datetime_value(4i)"=>"01", "datetime_value(5i)"=>"02", "answer_id"=>"22"},
+      "19"=>{"question_id"=>"15", "datetime_value"=>"", "answer_id"=>"23", "id" => "1"},
+      "47"=>{"question_id"=>"38", "answer_id"=>"220", "integer_value"=>"2", "id" => "2"},
+      "61"=>{"question_id"=>"44", "response_group"=>"0", "answer_id"=>"241", "integer_value"=>"12"}}
+    ResponseSet.trim_for_lookups(hash_of_hashes).should == 
+    { "1"=>{"question_id"=>"2", "answer_id"=>"1"},
+      "2"=>{"question_id"=>"3", "answer_id"=>["", "6"]},
+      "9"=>{"question_id"=>"6", "answer_id"=>"13"},
+      "17"=>{"question_id"=>"13", "answer_id"=>"21"},
+      "18"=>{"question_id"=>"14", "answer_id"=>"22"},
+      "19"=>{"question_id"=>"15", "answer_id"=>"23", "id" => "1", "_destroy" => "true"},
+      "47"=>{"question_id"=>"38", "answer_id"=>"220", "id" => "2"},
+      "61"=>{"question_id"=>"44", "response_group"=>"0", "answer_id"=>"241"}
+    }
+  end
   it "should remove responses" do
     r = @response_set.responses.create(:question_id => 1, :answer_id => 2)
     r.id.should_not be nil

data/surveyor.gemspec

@@ -5,17 +5,16 @@
 
 Gem::Specification.new do |s|
   s.name = %q{surveyor}
-  s.version = "0.19.7"
+  s.version = "0.20.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brian Chamberlain", "Mark Yoon"]
-  s.date = %q{2011-06-23}
+  s.date = %q{2011-07-11}
   s.email = %q{yoon@northwestern.edu}
   s.extra_rdoc_files = [
     "README.md"
   ]
   s.files = [
-    ".rvmrc",
     "CHANGELOG",
     "MIT-LICENSE",
     "README.md",
@@ -61,7 +60,9 @@ Gem::Specification.new do |s|
     "features/support/REDCapDemoDatabase_DataDictionary.csv",
     "features/support/env.rb",
     "features/support/paths.rb",
+    "features/support/redcap_new_headers.csv",
     "features/support/redcap_siblings.csv",
+    "features/support/redcap_whitespace.csv",
     "features/surveyor.feature",
     "features/surveyor_parser.feature",
     "generators/extend_surveyor/extend_surveyor_generator.rb",
@@ -77,6 +78,7 @@ Gem::Specification.new do |s|
     "generators/surveyor/templates/assets/stylesheets/dateinput.css",
     "generators/surveyor/templates/assets/stylesheets/reset.css",
     "generators/surveyor/templates/assets/stylesheets/results.css",
+    "generators/surveyor/templates/assets/stylesheets/sass/custom.sass",
     "generators/surveyor/templates/assets/stylesheets/sass/surveyor.sass",
     "generators/surveyor/templates/locales/surveyor_en.yml",
     "generators/surveyor/templates/locales/surveyor_es.yml",
@@ -85,6 +87,7 @@ Gem::Specification.new do |s|
     "generators/surveyor/templates/migrate/add_correct_answer_id_to_questions.rb",
     "generators/surveyor/templates/migrate/add_default_value_to_answers.rb",
     "generators/surveyor/templates/migrate/add_display_order_to_surveys.rb",
+    "generators/surveyor/templates/migrate/add_display_type_to_answers.rb",
     "generators/surveyor/templates/migrate/add_index_to_response_sets.rb",
     "generators/surveyor/templates/migrate/add_index_to_surveys.rb",
     "generators/surveyor/templates/migrate/add_section_id_to_responses.rb",
@@ -129,6 +132,7 @@ Gem::Specification.new do |s|
     "spec/controllers/surveyor_controller_spec.rb",
     "spec/factories.rb",
     "spec/helpers/surveyor_helper_spec.rb",
+    "spec/lib/benchmark_spec.rb",
     "spec/lib/common_spec.rb",
     "spec/lib/parser_spec.rb",
     "spec/lib/redcap_parser_spec.rb",
@@ -151,6 +155,7 @@ Gem::Specification.new do |s|
     "testbed/Gemfile"
   ]
   s.homepage = %q{http://github.com/NUBIC/surveyor}
+  s.post_install_message = %q{Thanks for installing surveyor! The time has come to run the surveyor generator and migrate your database, even if you are upgrading.}
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.6.2}
   s.summary = %q{A rails (gem) plugin to enable surveys in your application}

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: surveyor
 version: !ruby/object:Gem::Version 
-  hash: 93
+  hash: 79
   prerelease: 
   segments: 
   - 0
-  - 19
-  - 7
-  version: 0.19.7
+  - 20
+  - 0
+  version: 0.20.0
 platform: ruby
 authors: 
 - Brian Chamberlain
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-23 00:00:00 -05:00
+date: 2011-07-11 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -112,7 +112,6 @@ extensions: []
 extra_rdoc_files: 
 - README.md
 files: 
-- .rvmrc
 - CHANGELOG
 - MIT-LICENSE
 - README.md
@@ -158,7 +157,9 @@ files:
 - features/support/REDCapDemoDatabase_DataDictionary.csv
 - features/support/env.rb
 - features/support/paths.rb
+- features/support/redcap_new_headers.csv
 - features/support/redcap_siblings.csv
+- features/support/redcap_whitespace.csv
 - features/surveyor.feature
 - features/surveyor_parser.feature
 - generators/extend_surveyor/extend_surveyor_generator.rb
@@ -174,6 +175,7 @@ files:
 - generators/surveyor/templates/assets/stylesheets/dateinput.css
 - generators/surveyor/templates/assets/stylesheets/reset.css
 - generators/surveyor/templates/assets/stylesheets/results.css
+- generators/surveyor/templates/assets/stylesheets/sass/custom.sass
 - generators/surveyor/templates/assets/stylesheets/sass/surveyor.sass
 - generators/surveyor/templates/locales/surveyor_en.yml
 - generators/surveyor/templates/locales/surveyor_es.yml
@@ -182,6 +184,7 @@ files:
 - generators/surveyor/templates/migrate/add_correct_answer_id_to_questions.rb
 - generators/surveyor/templates/migrate/add_default_value_to_answers.rb
 - generators/surveyor/templates/migrate/add_display_order_to_surveys.rb
+- generators/surveyor/templates/migrate/add_display_type_to_answers.rb
 - generators/surveyor/templates/migrate/add_index_to_response_sets.rb
 - generators/surveyor/templates/migrate/add_index_to_surveys.rb
 - generators/surveyor/templates/migrate/add_section_id_to_responses.rb
@@ -226,6 +229,7 @@ files:
 - spec/controllers/surveyor_controller_spec.rb
 - spec/factories.rb
 - spec/helpers/surveyor_helper_spec.rb
+- spec/lib/benchmark_spec.rb
 - spec/lib/common_spec.rb
 - spec/lib/parser_spec.rb
 - spec/lib/redcap_parser_spec.rb
@@ -250,7 +254,7 @@ has_rdoc: true
 homepage: http://github.com/NUBIC/surveyor
 licenses: []
 
-post_install_message: 
+post_install_message: Thanks for installing surveyor! The time has come to run the surveyor generator and migrate your database, even if you are upgrading.
 rdoc_options: []
 
 require_paths: 

data/.rvmrc

@@ -1 +0,0 @@
-rvm_gemset_create_on_use_flag=1; rvm gemset use surveyor-dev