affectiva-surveyor 1.5.0.pre.disco.2

data/lib/surveyor/redcap_parser.rb

@@ -0,0 +1,288 @@
+%w(survey survey_section question_group question dependency dependency_condition answer validation validation_condition).each {|model| require model }
+require 'active_support' # for humanize
+module Surveyor
+  class RedcapParserError < StandardError; end
+  class RedcapParser
+    class << self; attr_accessor :options end
+
+    # Attributes
+    attr_accessor :context
+
+    # Class methods
+    def self.parse(str, filename, options={})
+      self.options = options
+      Surveyor::RedcapParser.rake_trace "\n"
+      Surveyor::RedcapParser.new.parse(str, filename)
+      Surveyor::RedcapParser.rake_trace "\n"
+    end
+    def self.rake_trace(str)
+      self.options ||= {}
+      print str if self.options[:trace] == true
+    end
+
+    # Instance methods
+    def initialize
+      self.context = {}
+      self.context[:dependency_conditions] = []
+    end
+    def parse(str, filename)
+      csvlib = Surveyor::Common.csv_impl
+      begin
+        csvlib.parse(str, :headers => :first_row, :return_headers => true, :header_converters => :symbol) do |r|
+          if r.header_row? # header row
+            return Surveyor::RedcapParser.rake_trace "Missing headers: #{missing_columns(r.headers).inspect}\n\n" unless missing_columns(r.headers).blank?
+            context[:survey] = Survey.new(:title => filename)
+            Surveyor::RedcapParser.rake_trace "survey_#{context[:survey].access_code} "
+          else # non-header rows
+            SurveySection.new.extend(SurveyorRedcapParserSurveySectionMethods).build_or_set(context, r)
+            Question.new.extend(SurveyorRedcapParserQuestionMethods).build_and_set(context, r)
+            Answer.new.extend(SurveyorRedcapParserAnswerMethods).build_and_set(context, r)
+            Validation.new.extend(SurveyorRedcapParserValidationMethods).build_and_set(context, r)
+            Dependency.new.extend(SurveyorRedcapParserDependencyMethods).build_and_set(context, r)
+          end
+        end
+        resolve_references
+        Surveyor::RedcapParser.rake_trace context[:survey].save ? "saved. " : " not saved! #{context[:survey].errors.full_messages.join(", ")} "
+        # Surveyor::RedcapParser.rake_trace context[:survey].sections.map(&:questions).flatten.map(&:answers).flatten.map{|x| x.errors.each_full{|y| y}.join}.join
+      rescue csvlib::MalformedCSVError
+        raise Surveyor::RedcapParserError, "Oops. Not a valid CSV file."
+      # ensure
+      end
+      return context[:survey]
+    end
+    def missing_columns(r)
+      missing = []
+      missing << "choices_or_calculations" unless r.map(&:to_s).include?("choices_or_calculations") or r.map(&:to_s).include?("choices_calculations_or_slider_labels")
+      missing << "text_validation_type" unless r.map(&:to_s).include?("text_validation_type") or r.map(&:to_s).include?("text_validation_type_or_show_slider_number")
+      missing += (static_required_columns - r.map(&:to_s))
+    end
+    def static_required_columns
+      # no longer requiring field_units
+      %w(variable__field_name form_name section_header field_type field_label field_note text_validation_min text_validation_max identifier branching_logic_show_field_only_if required_field)
+    end
+    def resolve_references
+      context[:dependency_conditions].each do |dc|
+        Surveyor::RedcapParser.rake_trace "resolve(#{dc.question_reference},#{dc.answer_reference})"
+        if dc.answer_reference.blank? and (context[:question_references][dc.question_reference].answers.size == 1)
+          Surveyor::RedcapParser.rake_trace "...found "
+          dc.question = context[:question_references][dc.question_reference]
+          dc.answer = dc.question.answers.first
+        elsif answer = context[:answer_references][dc.question_reference][dc.answer_reference]
+          Surveyor::RedcapParser.rake_trace "...found "
+          dc.answer = answer
+          dc.question = context[:question_references][dc.question_reference]
+        else
+          Surveyor::RedcapParser.rake_trace "\n!!! failed lookup for dependency_condition q: #{question_reference} a: #{question_reference}"
+        end
+      end
+    end
+  end
+end
+
+# Surveyor models with extra parsing methods
+
+# SurveySection model
+module SurveyorRedcapParserSurveySectionMethods
+  def build_or_set(context, r)
+    unless context[:survey_section] && context[:survey_section].reference_identifier == r[:form_name]
+      if match = context[:survey].sections.detect{|ss| ss.reference_identifier == r[:form_name]}
+        context[:current_survey_section] = match
+      else
+        self.attributes = (
+          {:title => r[:form_name].to_s.humanize,
+          :reference_identifier => r[:form_name],
+          :display_order => context[:survey].sections.size })
+        context[:survey].sections << context[:survey_section] = self
+        Surveyor::RedcapParser.rake_trace "survey_section_#{context[:survey_section].reference_identifier} "
+      end
+    end
+  end
+end
+
+# Question model
+module SurveyorRedcapParserQuestionMethods
+  def build_and_set(context, r)
+    if !r[:section_header].blank?
+      context[:survey_section].questions.build({:display_type => "label", :text => r[:section_header], :display_order => context[:survey_section].questions.size})
+      Surveyor::RedcapParser.rake_trace "label_ "
+    end
+    self.attributes = ({
+      :reference_identifier => r[:variable__field_name],
+      :text => r[:field_label],
+      :help_text => r[:field_note],
+      :is_mandatory => (/^y/i.match r[:required_field]) ? true : false,
+      :pick => pick_from_field_type(r[:field_type]),
+      :display_type => display_type_from_field_type(r[:field_type]),
+      :display_order => context[:survey_section].questions.size
+    })
+    context[:survey_section].questions << context[:question] = self
+    unless context[:question].reference_identifier.blank?
+      context[:question_references] ||= {}
+      context[:question_references][context[:question].reference_identifier] = context[:question]
+    end
+    Surveyor::RedcapParser.rake_trace "question_#{context[:question].reference_identifier} "
+  end
+  def pick_from_field_type(ft)
+    {"checkbox" => :any, "radio" => :one}[ft] || :none
+  end
+  def display_type_from_field_type(ft)
+    {"text" => :string, "dropdown" => :dropdown, "notes" => :text}[ft]
+  end
+end
+
+# Dependency model
+module SurveyorRedcapParserDependencyMethods
+  def build_and_set(context, r)
+    unless (bl = r[:branching_logic_show_field_only_if]).blank?
+      # TODO: forgot to tie rule key to component, counting on the sequence of components
+      letters = ('A'..'Z').to_a
+      hash = decompose_rule(bl)
+      self.attributes = {:rule => hash[:rule]}
+      context[:question].dependency = context[:dependency] = self
+      hash[:components].each do |component|
+        dc = context[:dependency].dependency_conditions.build(decompose_component(component).merge({ :rule_key => letters.shift } ))
+        context[:dependency_conditions] << dc
+      end
+      Surveyor::RedcapParser.rake_trace "dependency(#{hash[:rule]}) "
+    end
+  end
+  def decompose_component(str)
+    # [initial_52] = "1" or [f1_q15] = '' or [f1_q15] = '-2' or [hi_event1_type] <> ''
+    if match = str.match(/^\[(\w+)\] ?([!=><]+) ?['"](-?\w*)['"]$/)
+      {:question_reference => match[1], :operator => match[2].gsub(/^=$/, "==").gsub(/^<>$/, "!="), :answer_reference => match[3]}
+    # [initial_119(2)] = "1" or [hiprep_heat2(97)] = '1'
+    elsif match = str.match(/^\[(\w+)\((\w+)\)\] ?([!=><]+) ?['"]1['"]$/)
+      {:question_reference => match[1], :operator => match[3].gsub(/^=$/, "==").gsub(/^<>$/, "!="), :answer_reference => match[2]}
+    # [f1_q15] >= 21 or [f1_q15] >= -21
+    elsif match = str.match(/^\[(\w+)\] ?([!=><]+) ?(-?\d+)$/)
+      {:question_reference => match[1], :operator => match[2].gsub(/^=$/, "==").gsub(/^<>$/, "!="), :integer_value => match[3]}
+    else
+      Surveyor::RedcapParser.rake_trace "\n!!! skipping dependency_condition #{str}"
+    end
+  end
+  def decompose_rule(str)
+    # see spec/lib/redcap_parser_spec.rb for examples
+    letters = ('A'..'Z').to_a
+    rule = str
+    components = str.split(/\band\b|\bor\b|\((?!\d)|\)(?!\(|\])/).reject(&:blank?).map(&:strip)
+    components.each_with_index do |part, i|
+      # internal commas on the right side of the operator e.g. '[initial_189] = "1, 2, 3"'
+      if match = part.match(/^(\[[^\]]+\][^\"]+)"([0-9 ]+,[0-9 ,]+)"$/)
+        nums = match[2].split(",").map(&:strip)
+        components[i] = nums.map{|x| "#{match[1]}\"#{x}\""}
+        # sub in rule key
+        rule = rule.gsub(part, "(#{nums.map{letters.shift}.join(' and ')})")
+      # multiple internal parenthesis on the left  e.g. '[initial_119(1)(2)(3)(4)(6)] = "1"'
+      elsif match = part.match(/^\[(\w+)(\(\d+\)\([\d\(\)]+)\]([^\"]+"\d+")$/)
+        nums = match[2].split(/\(|\)/).reject(&:blank?).map(&:strip)
+        components[i] = nums.map{|x| "[#{match[1]}(#{x})]#{match[3]}"}
+        # sub in rule key
+        rule = rule.gsub(part, "(#{nums.map{letters.shift}.join(' and ')})")
+      else
+        # 'or' on the right of the operator
+        components[i] = components[i-1].gsub(/"(\d+)"/, part) if part.match(/^"(\d+)"$/) && i != 0
+        # sub in rule key
+        rule = rule.gsub(part){letters.shift}
+      end
+    end
+    {:rule => rule, :components => components.flatten}
+  end
+end
+
+# DependencyCondition model
+module SurveyorRedcapParserDependencyConditionMethods
+  DependencyCondition.instance_eval do
+    attr_accessor :question_reference, :answer_reference
+  end
+end
+
+# Answer model
+module SurveyorRedcapParserAnswerMethods
+  def build_and_set(context, r)
+    case r[:field_type]
+    when "text"
+      self.attributes = {
+        :response_class => "string",
+        :text => "Text",
+        :display_order => context[:question].answers.size }
+      context[:question].answers << context[:answer] = self
+    when "notes"
+      self.attributes = {
+        :response_class => "text",
+        :text => "Notes",
+        :display_order => context[:question].answers.size }
+      context[:question].answers << context[:answer] = self
+    when "file"
+      Surveyor::RedcapParser.rake_trace "\n!!! skipping answer: file"
+    end
+    (r[:choices_or_calculations] || r[:choices_calculations_or_slider_labels]).to_s.split("|").each do |pair|
+      aref, atext = pair.split(",").map(&:strip)
+      if aref.blank? or atext.blank? or (aref.to_i.to_s != aref)
+        Surveyor::RedcapParser.rake_trace "\n!!! skipping answer #{pair}"
+      else
+        a = Answer.new({
+          :reference_identifier => aref,
+          :text => atext,
+          :display_order => context[:question].answers.size })
+        context[:question].answers << context[:answer] = a
+        unless context[:question].reference_identifier.blank? or aref.blank? or !context[:answer].valid?
+          context[:answer_references] ||= {}
+          context[:answer_references][context[:question].reference_identifier] ||= {}
+          context[:answer_references][context[:question].reference_identifier][aref] = context[:answer]
+        end
+        Surveyor::RedcapParser.rake_trace "#{context[:answer].errors.full_messages}, #{context[:answer].inspect}" unless context[:answer].valid?
+        Surveyor::RedcapParser.rake_trace "answer_#{context[:answer].reference_identifier} "
+      end
+    end
+  end
+end
+
+# Validation model
+module SurveyorRedcapParserValidationMethods
+  def build_and_set(context, r)
+    # text_validation_type text_validation_min text_validation_max
+    min = r[:text_validation_min].to_s.blank? ? nil : r[:text_validation_min].to_s
+    max = r[:text_validation_max].to_s.blank? ? nil : r[:text_validation_max].to_s
+    type = r[:text_validation_type].to_s.blank? ? nil : r[:text_validation_type].to_s
+    if min or max
+      context[:question].answers.each do |a|
+        self.rule = (min ? max ? "A and B" : "A" : "B")
+        a.validations << context[:validation] = self
+        context[:validation].validation_conditions.build(:rule_key => "A", :operator => ">=", :integer_value => min) if min
+        context[:validation].validation_conditions.build(:rule_key => "B", :operator => "<=", :integer_value => max) if max
+      end
+    elsif type
+      # date email integer number phone
+      case r[:text_validation_type]
+      when "date"
+        context[:question].display_type = :date if context[:question].display_type == :string
+      when "email"
+        context[:question].answers.each do |a|
+          self.rule = "A"
+          a.validations << context[:validation] = self
+          context[:validation].validation_conditions.build(:rule_key => "A", :operator => "=~", :regexp => "^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}$")
+        end
+      when "integer"
+        context[:question].display_type = :integer if context[:question].display_type == :string
+        context[:question].answers.each do |a|
+          self.rule = "A"
+          a.validations << context[:validation] = self
+          context[:validation].validation_conditions.build(:rule_key => "A", :operator => "=~", :regexp => "\d+")
+        end
+      when "number"
+        context[:question].display_type = :float if context[:question].display_type == :string
+        context[:question].answers.each do |a|
+          self.rule = "A"
+          a.validations << context[:validation] = self
+          context[:validation].validation_conditions.build(:rule_key => "A", :operator => "=~", :regexp => "^\d*(,\d{3})*(\.\d*)?$")
+        end
+      when "phone"
+        context[:question].answers.each do |a|
+                    self.rule = "A"
+          a.validations << context[:validation] = self
+          context[:validation].validation_conditions.build(:rule_key => "A", :operator => "=~", :regexp => "\d{3}.*\d{4}")
+        end
+      end
+    end
+  end
+end

data/lib/surveyor/surveyor_controller_methods.rb

@@ -0,0 +1,236 @@
+require 'rabl'
+Rabl.register!
+Rabl.configure {|config| config.include_child_root = false }
+Rabl.configure {|config| config.include_json_root = false }
+module Surveyor
+  module SurveyorControllerMethods
+    def self.included(base)
+      base.send :before_filter, :get_current_user, :only => [:new, :create]
+      base.send :before_filter, :determine_if_javascript_is_enabled, :only => [:create, :update]
+      base.send :before_filter, :set_response_set_and_render_context, :only => [:edit, :show]
+      base.send :layout, 'surveyor_default'
+      base.send :before_filter, :set_locale
+    end
+
+    # Actions
+    def new
+      @surveys_by_access_code = Survey.order("created_at DESC, survey_version DESC").group_by(&:access_code)
+      redirect_to surveyor_index unless surveyor_index == surveyor.available_surveys_path
+    end
+
+    def create
+      surveys = Survey.where(:access_code => params[:survey_code]).order("survey_version DESC")
+      if params[:survey_version].blank?
+        @survey = surveys.first
+      else
+        @survey = surveys.where(:survey_version => params[:survey_version]).first
+      end
+      @response_set = ResponseSet.
+        create(:survey => @survey, :user_id => (@current_user.nil? ? @current_user : @current_user.id))
+      if (@survey && @response_set)
+        flash[:notice] = t('surveyor.survey_started_success')
+        redirect_to(surveyor.edit_my_survey_path(
+          :survey_code => @survey.access_code, :response_set_code  => @response_set.access_code))
+      else
+        flash[:notice] = t('surveyor.Unable_to_find_that_survey')
+        redirect_to surveyor_index
+      end
+    end
+
+    def show
+      # @response_set is set in before_filter - set_response_set_and_render_context
+      if @response_set
+        @survey = @response_set.survey
+        respond_to do |format|
+          format.html #{render :action => :show}
+          format.csv {
+            send_data(@response_set.to_csv, :type => 'text/csv; charset=utf-8; header=present',
+              :filename => "#{@response_set.updated_at.strftime('%Y-%m-%d')}_#{@response_set.access_code}.csv")
+          }
+          format.json
+        end
+      else
+        flash[:notice] = t('surveyor.unable_to_find_your_responses')
+        redirect_to surveyor_index
+      end
+    end
+
+    def edit
+      # @response_set is set in before_filter - set_response_set_and_render_context
+      if @response_set
+        @survey = Survey.with_sections.find_by_id(@response_set.survey_id)
+        @sections = @survey.sections
+        @section = section_id_from(params) ? @sections.with_includes.find(section_id_from(params)) : @sections.with_includes.first
+        set_dependents
+      else
+        flash[:notice] = t('surveyor.unable_to_find_your_responses')
+        redirect_to surveyor_index
+      end
+    end
+
+    def update
+      question_ids_for_dependencies = (params[:r] || []).map{|k,v| v["question_id"] }.compact.uniq
+      saved = load_and_update_response_set_with_retries
+
+      return redirect_with_message(surveyor_finish, :notice, t('surveyor.completed_survey')) if saved && params[:finish]
+
+      respond_to do |format|
+        format.html do
+          if @response_set.nil?
+            return redirect_with_message(surveyor.available_surveys_path, :notice, t('surveyor.unable_to_find_your_responses'))
+          else
+            flash[:notice] = t('surveyor.unable_to_update_survey') unless saved
+            redirect_to surveyor.edit_my_survey_path(:anchor => anchor_from(params[:section]), :section => section_id_from(params))
+          end
+        end
+        format.js do
+          if @response_set
+            render :json => @response_set.reload.all_dependencies(question_ids_for_dependencies)
+          else
+            render :text => "No response set #{params[:response_set_code]}",
+              :status => 404
+          end
+        end
+      end
+    end
+
+    def load_and_update_response_set_with_retries(remaining=2)
+      begin
+        load_and_update_response_set
+      rescue ActiveRecord::StatementInvalid => e
+        if remaining > 0
+          load_and_update_response_set_with_retries(remaining - 1)
+        else
+          raise e
+        end
+      end
+    end
+
+    def load_and_update_response_set
+      ResponseSet.transaction do
+        @response_set = ResponseSet.includes(:responses => :answer).find_by(:access_code => params[:response_set_code])
+        if @response_set
+          saved = true
+          if params[:r]
+            @response_set.update_from_ui_hash(params[:r])
+          end
+          if params[:finish]
+            @response_set.complete!
+            saved &= @response_set.save
+          end
+          saved
+        else
+          false
+        end
+      end
+    end
+    private :load_and_update_response_set
+
+    def export
+      surveys = Survey.where(:access_code => params[:survey_code]).order("survey_version DESC")
+      s = params[:survey_version].blank? ? surveys.first : surveys.where(:survey_version => params[:survey_version]).first
+      render_404 and return if s.blank?
+      @survey = s.filtered_for_json
+    end
+
+    def render_404
+      head :status => 404
+      true
+    end
+
+    def url_options
+      ((I18n.locale == I18n.default_locale) ? {} : {:locale => I18n.locale}).merge(super)
+    end
+
+    private
+
+    # This is a hook method for surveyor-using applications to override and provide the context object
+    def render_context
+      nil
+    end
+
+    # Filters
+    def get_current_user
+      @current_user = self.respond_to?(:current_user) ? self.current_user : nil
+    end
+
+    def set_response_set_and_render_context
+      @response_set = ResponseSet.includes(:responses => [:question, :answer])
+        .find_by(:access_code => params[:response_set_code])
+      @render_context = render_context
+    end
+
+     def set_locale
+      if params[:new_locale]
+        I18n.locale = params[:new_locale]
+      elsif params[:locale]
+        I18n.locale = params[:locale]
+      else
+        I18n.locale = I18n.default_locale
+      end
+    end
+
+    # Params: the name of some submit buttons store the section we'd like to go
+    # to. for repeater questions, an anchor to the repeater group is also stored
+    # e.g. params[:section] = {"1"=>{"question_group_1"=>"<= add row"}}
+    def section_id_from(p = {})
+      if p[:section] && p[:section].respond_to?(:keys)
+        p[:section].keys.first
+      elsif p[:section]
+        p[:section]
+      elsif p[:current_section]
+        p[:current_section]
+      end
+    end
+
+    def anchor_from(p)
+      p.respond_to?(:keys) && p[p.keys.first].respond_to?(:keys) ? p[p.keys.first].keys.first : nil
+    end
+
+    def surveyor_index
+      surveyor.available_surveys_path
+    end
+    def surveyor_finish
+      surveyor.available_surveys_path
+    end
+
+    def redirect_with_message(path, message_type, message)
+      respond_to do |format|
+        format.html do
+          flash[message_type] = message if !message.blank? and !message_type.blank?
+          redirect_to path
+        end
+        format.js do
+          render :text => message, :status => 403
+        end
+      end
+    end
+
+    ##
+    # @dependents are necessary in case the client does not have javascript enabled
+    # Whether or not javascript is enabled is determined by a hidden field set in the surveyor/edit.html form
+    def set_dependents
+      if session[:surveyor_javascript] && session[:surveyor_javascript] == "enabled"
+        @dependents = []
+      else
+        @dependents = get_unanswered_dependencies_minus_section_questions
+      end
+    end
+
+    def get_unanswered_dependencies_minus_section_questions
+      @response_set.unanswered_dependencies - @section.questions || []
+    end
+
+    ##
+    # If the hidden field surveyor_javascript_enabled is set to true
+    # cf. surveyor/edit.html.haml
+    # the set the session variable [:surveyor_javascript] to "enabled"
+    def determine_if_javascript_is_enabled
+      if params[:surveyor_javascript_enabled] && params[:surveyor_javascript_enabled].to_s == "true"
+        session[:surveyor_javascript] = "enabled"
+      else
+        session[:surveyor_javascript] = "not_enabled"
+      end
+    end
+  end
+end