surveyor_warehouse 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YjkwYWQxMGNhMTJiYzJlNzY4MWM5NjM3N2ZkMDM5MDE5OGY2NDAyMA==
+  data.tar.gz: !binary |-
+    NDQxMGFlY2EzNjYyY2UxOTVlZGVlMjI2NTY5NWE4ZDgwZTM3NDY0OQ==
+SHA512:
+  metadata.gz: !binary |-
+    ZjE2MmYyNDQ0MGQ3NzNjNmEyN2NkZDRjMzRkOTM0ZGFmMDUxZDY4Y2VjMGM0
+    MzRjNDhlYzkwNThhZTc4ZWQ1ZWJiMGNiZWZiODEyMWFmMDI1ZTJmNmE0Njdi
+    Y2QzZWJkYWZmMjg4YWFmMzkwZjEyMmU3N2U3YWUwOWQyZWEwY2I=
+  data.tar.gz: !binary |-
+    MGM2ODIyZmE2MmJlZjc0ZjkxMDc3ZTk0ZjJiZmNiMGIxNmI5ODczMDA1NjA2
+    OTU4MWIzYTgxNjMyYTNlYzIyODU1Njg5OGNhOGQ2NzliNTgxMTc5ZDI0Zjhl
+    ZTYwNTNhZWQzN2QzMTRhNzc3MDI5NGEyMzYzODFhN2Y4ZmU4YjM=

data/README.md

@@ -0,0 +1,70 @@
+# Surveyor Warehouse
+
+Surveyor Warehouse allows you to extract and transform the responses from 
+surveyor response sets into an alternate database table structure. This 
+can be generally helpful for querying surveyor responses within a table
+structure of one's choice.
+
+The alternate database table structure is defined by adding a
+`:data_export_identifier` attribute to each question with the 
+format "table.column" (e.g. 'subjects.name').
+
+## Example
+
+Given the survey:
+```
+  q 'What is your name?', :data_export_identifier => 'subjects.name'
+  a :string
+
+  q 'What is your age?', :data_export_identifier => 'subjects.age'
+  a :integer
+
+  q 'What is your gender?', :pick => :one, :data_export_identifier => 'subjects.gender'
+  a 'Male', :reference_identifier => "male"
+  a 'Female', :reference_identifier => "female"
+
+  q 'Which brand of shoes do you wear?', :pick => :any, :data_export_identifier => 'subjects.shoes'
+  a 'Nike', :reference_identifier => 'nike'
+  a 'Adidas', :reference_identifier => 'adidas'
+  a 'None', :omit, :reference_identifier => 'none'
+
+  repeater 'List all your siblings' do
+    q 'Name', :data_export_identifier => 'siblings.name' 
+    a :string
+  end
+```
+
+Responses would be transformed into:
+
+```
+  db=# select * from subjects;
+    name   | age | gender |     shoes     | access_code |      id
+  ---------+-----+--------+---------------+-------------+--------------
+   raphael |  18 | male   | {nike,adidas} | QUhwHtMPyQ  | QUhwHtMPyQ.0
+   stephen |  50 | male   | {none}        | zZu1OV1hmw  | zZu1OV1hmw.0
+  (2 rows)
+
+  db=# select * from siblings;
+       name     | access_code |      id
+  --------------+-------------+--------------
+   leonardo     | QUhwHtMPyQ  | QUhwHtMPyQ.0
+   michelangelo | QUhwHtMPyQ  | QUhwHtMPyQ.1
+   elizabeth    | zZu1OV1hmw  | zZu1OV1hmw.0
+   james        | zZu1OV1hmw  | zZu1OV1hmw.1
+  (4 rows)
+```
+
+
+## Getting Started
+
+1. Survey questions must have data_export_identifiers with the 
+   format "table.column" (e.g. 'subjects.name')
+2. Take the survey
+3. `bundle exec rake surveyor:warehouse`
+4. Query the results
+
+
+
+## Limitations
+
+* Currently only works with PostgresSQL

data/lib/surveyor_warehouse.rb

@@ -0,0 +1,47 @@
+require 'surveyor'
+require 'active_support/concern'
+
+require 'surveyor_warehouse/normalized_survey_structure'
+require 'surveyor_warehouse/response_bin'
+require 'surveyor_warehouse/response_row'
+require 'surveyor_warehouse/railtie' if defined?(Rails)
+require 'surveyor_warehouse/extensions/question'
+require 'surveyor_warehouse/extensions/survey'
+
+module SurveyorWarehouse
+  def self.logger
+    @logger ||= Logger.new(STDOUT)
+  end
+
+  def self.transform
+    Survey.send(:include, SurveyorWarehouse::Extensions::Survey)
+
+    surveys = Survey.current_versions
+
+    surveys.each do |s|
+      ns = NormalizedSurveyStructure.new(s)
+      ns.create!
+      s.response_sets.each do |rs|
+        # logger.debug("Transforming [ResponseSet id:#{rs.id}] for [Survey id:#{s.id} title:'#{s.title}")
+        bins = ResponseBin.bins(rs.responses)
+        bins.map(&:rows).flatten.each(&:insert!)
+        logger.info("Transformed [ResponseSet id:#{rs.id}] for [Survey id:#{s.id} title:'#{s.title}']")
+      end
+    end
+  end
+
+  def self.clobber
+    Survey.send(:include, SurveyorWarehouse::Extensions::Survey)
+
+    surveys = Survey.current_versions
+
+    surveys.map(&:response_sets).flatten.each do |rs|
+      ns = NormalizedSurveyStructure.new(rs.survey)
+      ns.destroy!
+    end
+
+  end
+end
+
+# Survey.send(:include, SurveyorWarehouse::Extensions::Survey)
+# Question.send(:include, SurveyorWarehouse::Extensions::Question)

data/lib/surveyor_warehouse/db.rb

@@ -0,0 +1,58 @@
+module SurveyorWarehouse
+  module DB
+    def self.connection
+      @connection ||= Sequel.connect(
+          :adapter=> adapter, 
+          :host=>'localhost', 
+          :database=> database, 
+          :user=> username, 
+          :password=> password).extension(:pg_array)
+      @connection.extend(SequelExtension::Connection)
+    end
+
+    def self.configurations
+      @configurations ||= ::ActiveRecord::Base.configurations[Rails.env]
+    end
+
+    def self.username
+      @username ||= configurations['username']
+    end
+    
+    def self.password
+      @password ||= configurations['password']
+    end
+    
+    def self.database
+      @database ||= configurations['database']
+    end
+
+    def self.adapter
+      @adapter ||=
+        if 'postgresql' == configurations['adapter']
+          'postgres'
+        else
+          raise "Unsupported database adapter: #{db['adapter']}"
+        end
+    end
+
+    ##
+    # Columns are returned a hash like below:
+    #
+    # { :column1 => [:primary_key], :column2 => [:not_null] }
+    def self.columns(tablename)
+      connection.schema(tablename.to_sym).inject({}) do |attrs, (cname, cattrs)| 
+        attrs.merge(cname => cattrs)
+      end
+    end
+  end
+
+  module SequelExtension
+    module Connection
+      def columns(tablename)
+        self.schema(tablename.to_sym).inject({}) do |attrs, (cname, cattrs)| 
+          attrs.merge(cname => cattrs)
+        end
+      end
+    end
+  end
+end

data/lib/surveyor_warehouse/extensions/question.rb

@@ -0,0 +1,18 @@
+require 'active_support/concern'
+
+module SurveyorWarehouse
+  module Extensions
+    module Question
+      extend ActiveSupport::Concern
+      included do
+        ##
+        # A question's data_export_identifier should follow the
+        # format 'table.column' (e.g. 'patients.name')
+        def valid_data_export_identifier?
+          dei = self.try(:data_export_identifier)
+          dei.present? && dei.split('.').size == 2
+        end
+      end
+    end
+  end
+end

data/lib/surveyor_warehouse/extensions/survey.rb

@@ -0,0 +1,15 @@
+require 'active_support/concern'
+
+module SurveyorWarehouse
+  module Extensions
+    module Survey
+      extend ActiveSupport::Concern
+      module ClassMethods
+        def current_versions
+          group = ::Survey.order("created_at DESC, survey_version DESC").all.group_by(&:access_code)
+          group.map { |access_code, surveys|  surveys.first }
+        end
+      end
+    end
+  end
+end

data/lib/surveyor_warehouse/normalized_survey_structure.rb

@@ -0,0 +1,96 @@
+require 'surveyor_warehouse/db'
+
+module SurveyorWarehouse
+  class NormalizedSurveyStructure
+    def initialize(survey)
+      @survey = survey
+    end
+
+    ##
+    # response_classes
+    def tables
+      questions = @survey.sections_with_questions.map(&:questions).flatten
+      Question.send(:include, SurveyorWarehouse::Extensions::Question)
+      predefs = questions.select {|q| q.valid_data_export_identifier? }.map do |question| 
+        dei_tokens = question.data_export_identifier.split('.')
+
+        table = dei_tokens[0]
+        column = dei_tokens[1]
+        type = database_type(question)
+          
+        [table, column, type]
+      end
+
+      tables = {}
+      predefs.each do |d|
+        table_name, column_name, column_type = d
+        table = tables[table_name] || TableDefinition.new(table_name)
+        table.columns << ColumnDefinition.new(column_name, column_type)
+        tables.merge!(table_name => table)
+      end
+
+      # Add primary key and response set access code columns
+      tables.each do |name, tdef|
+        tdef.columns << ColumnDefinition.new('access_code', 'text')
+        tdef.columns << ColumnDefinition.new('id', 'text')
+      end
+
+      @tables ||= tables.values
+    end
+
+    ##
+    # Force creates the structure by destroying all the tables first
+    #
+    def create!
+      destroy!
+      tables.each(&:create)
+    end
+
+    ##
+    # Destroy all the tables defined in the survey data_export_identifiers
+    #
+    def destroy!
+      drop_ddl = tables.map(&:name).map do |t|
+        "drop table if exists #{t};\n"
+      end.join
+      ActiveRecord::Base.connection.execute(drop_ddl)
+    end
+
+    private
+    SUPPORTED_TYPES = %w(date datetime decimal float integer string text)
+    def database_type(question)
+      case question.pick
+      when 'none'
+        response_classes = question.answers.map(&:response_class).uniq
+        raise "Muliple answer types are unsupported: #{response_classes.join(', ')}" if response_classes.size > 1
+        type = response_classes[0].tap do |t|
+          raise "Unsupported column type '#{t}' for question: #{question.inspect}}" unless SUPPORTED_TYPES.include?(t)
+        end
+        type == 'string' ? 'text' : type
+      when 'one'
+        'text'
+      when 'any'
+        "text[]"
+      else
+        raise "Unable to find type for question: #{question.inspect}"
+      end
+    end
+
+    class TableDefinition < Struct.new(:name)
+      def columns
+        @columns ||= []
+      end
+
+      def create
+        ActiveRecord::Base.connection.create_table(name.to_sym, :id => false) do |t|
+          columns.each do |c|
+            t.column c.name, c.type
+          end
+        end
+      end
+    end
+
+    class ColumnDefinition < Struct.new(:name, :type)
+    end
+  end
+end

data/lib/surveyor_warehouse/railtie.rb

@@ -0,0 +1,9 @@
+require 'surveyor_warehouse'
+require 'rails'
+module SurveyorWarehouse
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      Dir[File.join(File.dirname(__FILE__),'../tasks/*.rake')].each { |f| load f }
+    end
+  end
+end

data/lib/surveyor_warehouse/response_bin.rb

@@ -0,0 +1,67 @@
+require 'surveyor_warehouse/response_row'
+
+module SurveyorWarehouse
+  class ResponseBin < Struct.new(:key, :access_code, :response_group)
+    def <<(response)
+      q = response.question
+      Question.send(:include, SurveyorWarehouse::Extensions::Question)
+      if q.valid_data_export_identifier?
+        (@responses ||= []) << response
+      else
+        puts "Ignoring response with data_export_identifier: #{q.data_export_identifier}"
+      end
+      self
+    end
+
+    def responses
+      (@responses || []).dup.freeze
+    end
+
+    ##
+    # Groups responses into "bins" using the 'access_code.response_group' as
+    # the bin key.
+    #
+    # See below for an example how the binning behavior works:
+    # 
+    #   responses = []
+    #   responses.push( 
+    #     [Response access_code:'abc' response_group:0 string_value:'Jeff'],
+    #     [Response access_code:'abc' response_group:0 string_value:'Bezos'],
+    #     [Response access_code:'xyz' response_group:0 string_value:'Kindle'],
+    #     [Response access_code:'xyz' response_group:1 string_value:'EC2'])
+    #
+    #   ResponseBinner.new(responses).bins # => Results in the three bins below
+    #
+    #   [ResponseBin key:'abc:0' responses: [
+    #       [Response access_code:'abc' response_group:0 string_value:'Jeff'],
+    #       [Response access_code:'abc' response_group:0 string_value:'Bezos']]
+    #   [ResponseBin key:'xyz:0' responses: [
+    #       [Response access_code:'xyz' response_group:0 string_value:'Kindle']]
+    #   [ResponseBin key:'xyz:1' responses: [
+    #       [Response access_code:'xyz' response_group:1 string_value:'EC2']]
+    DEFAULT_RESPONSE_GROUP = 0
+    def self.bins(responses)
+      bins = {}
+      responses.each do |r|
+        ac = r.response_set.access_code
+        rg = r.response_group || DEFAULT_RESPONSE_GROUP
+        key = "#{ac}.#{rg}"
+        bin = bins[key] || ResponseBin.new(key, ac, rg)
+        bin << r
+        bins.merge!(key => bin)
+      end
+      bins.values
+    end
+
+    def rows
+      rows = {}
+      @responses.each do |r|
+        table_name = r.question.data_export_identifier.split('.')[0]
+        table = rows[table_name] || ResponseRow.new(table_name, key, access_code)
+        table.responses << r
+        rows.merge!(table_name => table)
+      end
+      rows.values
+    end
+  end
+end

data/lib/surveyor_warehouse/response_row.rb

@@ -0,0 +1,53 @@
+require 'surveyor_warehouse/db'
+require 'sequel'
+
+module SurveyorWarehouse
+  class ResponseRow < Struct.new(:name, :id, :access_code)
+    def responses
+      @responses ||= []
+    end
+
+    def name
+      super.to_sym
+    end
+
+    def insert!
+      column_and_values = {}
+      col_and_val = @responses.map do |r|
+        column = r.question.data_export_identifier.split('.')[1]
+
+        value_field = case r.answer.response_class
+          when 'datetime'; 'datetime_value'
+          when 'date'; 'date_value'
+          when 'time'; 'time_value'
+          when 'float'; 'float_value'
+          when 'integer'; 'integer_value'
+          when 'string'; 'string_value'
+          when 'text'; 'text_value'
+          end
+
+        value =  value_field.present? ? r.send(value_field) : r.answer.try(:reference_identifier)
+        [column.to_sym, value]
+      end
+
+      mDB = SurveyorWarehouse::DB.connection
+      schema = mDB.schema(name.to_sym).inject({}) { |attrs, (cname, cattrs)| attrs.merge(cname => cattrs) }
+
+      column_and_values = col_and_val.inject({}) do |accum, (col, val)|
+        if schema[col][:db_type] =~ /\[\]/
+          accum[col] = Sequel.pg_array((accum[col] || []) << val)
+        else
+          accum[col] = val
+        end
+        accum
+      end
+
+      column_and_values.merge!('id' => id, 'access_code' => access_code)
+
+      ds = mDB[name]
+      
+      ds.insert(column_and_values)
+      
+    end
+  end
+end

data/lib/tasks/transform.rake

@@ -0,0 +1,13 @@
+require 'surveyor_warehouse'
+
+namespace :surveyor do
+  desc 'Transforms the survey response sets into the defined warehouse form'
+  task :warehouse => [:environment, :clobber] do
+    SurveyorWarehouse.transform
+  end
+
+  desc 'Clobber the warehouse'
+  task :clobber do
+    SurveyorWarehouse.clobber
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,47 @@
+# This file is copied to spec/ when you run 'rails generate rspec:install'
+ENV["RAILS_ENV"] ||= 'test'
+begin
+  require File.expand_path("../../testbed/config/environment", __FILE__)
+rescue LoadError => e
+  fail "Could not load the testbed app. Have you generated it?\n#{e.class}: #{e}"
+end
+
+require 'rspec/autorun'
+require 'database_cleaner'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+
+  # == Mock Framework
+  #
+  # If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+  config.mock_with :rspec
+
+  ## Database Cleaner
+
+  config.before(:suite) do
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.before(:each, :clean_with_truncation) do
+    DatabaseCleaner.strategy = :truncation
+  end
+
+  config.after(:each, :clean_with_truncation) do
+    DatabaseCleaner.strategy = :transaction
+  end
+
+  config.before(:each) do
+    DatabaseCleaner.start
+  end
+
+  config.after(:each) do
+    DatabaseCleaner.clean
+  end
+end
+

data/spec/surveyor_warehouse/normalized_survey_structure_spec.rb

@@ -0,0 +1,144 @@
+require 'spec_helper'
+require 'surveyor_warehouse'
+
+describe SurveyorWarehouse::NormalizedSurveyStructure do
+  let(:survey) do
+    Surveyor::Parser.new.parse( 
+      <<-SURVEY
+        survey "Favorites" do
+          section "One" do
+            q "What is your favorite name?", :data_export_identifier => 'favorites.name'
+            a :string
+
+            q "What is your favorite color?", :pick => :one, :data_export_identifier => 'favorites.color'
+            a "red"
+            a "blue"
+            
+            q "Choose the colors you don't like", :pick => :any, :data_export_identifier => 'hated.colors'
+            a "red"
+            a "blue"
+            a "green"
+            
+          end
+        end
+      SURVEY
+    )
+  end
+
+  let(:normalized) { SurveyorWarehouse::NormalizedSurveyStructure.new(survey) }
+
+  let(:connection) do
+    ActiveRecord::Base.connection
+  end
+
+  def drop_tables(*tables)
+    tables.each do |t|
+      connection.drop_table(t) if connection.table_exists?(t)
+    end
+  end
+
+  describe '#create!' do
+    after(:each) do
+      drop_tables(:favorites, :hated)
+    end
+
+    it 'creates tables with scalar types' do
+      normalized.create!
+      connection.table_exists?(:favorites).should be(true)
+      columns = connection.columns(:favorites).map{ |c| [c.name, c.sql_type] }.sort
+      columns.should == [
+        %w(access_code text),
+        %w(color text),
+        %w(id text),
+        %w(name text)
+      ]
+    end
+
+    it 'creates tables with array types' do
+      normalized.create!
+      connection.table_exists?(:hated).should be(true)
+      columns = connection.columns(:hated).map(&:name).sort
+      columns.should == %w(access_code colors id)
+    end
+  end
+
+  describe '#destroy!' do
+    before(:each) do
+      connection.create_table(:foo) do |t|
+        t.integer :id
+      end
+    end
+
+    after(:each) do
+      drop_tables(:favorites, :hated, :foo)
+    end
+
+    it "drops all normalized tables" do
+      normalized.create!
+      normalized.destroy!
+      connection.table_exists?(:favorites).should be(false)
+      connection.table_exists?(:hated).should be(false)
+      connection.table_exists?(:foo).should be(true)
+    end
+  end
+
+  describe '#tables' do
+    let (:tables) { normalized.tables }
+    let (:columns) { tables.inject({}){ |accum, t| accum.merge(t.name => t.columns) } }
+
+    it 'has tables from data export identifiers' do
+      tables.map(&:name).sort.should == %w(favorites hated)
+    end
+
+    it 'has columns with scalar types' do
+      columns['favorites'].map { |col| [col.name, col.type] }.sort.should == [
+        ['access_code', 'text'],
+        ['color', 'text'],
+        ['id', 'text'],
+        ['name', 'text']        
+      ]
+    end
+
+    it 'has columns with array types' do
+      columns['hated'].map { |col| [col.name, col.type] }.sort.should == [
+        ['access_code', 'text'],
+        ['colors', 'text[]'],
+        ['id', 'text']
+      ]
+    end
+
+    it 'ignores questions without a data export identifier' do
+      blank_survey = Surveyor::Parser.new.parse(
+        <<-SURVEY
+          survey "Pet" do
+            section "Name" do
+              q "What is your pet's name?"
+              a :string
+            end
+          end
+        SURVEY
+      )
+
+      norm = SurveyorWarehouse::NormalizedSurveyStructure.new(blank_survey)
+
+      expect(norm.tables).to eq([])
+    end
+
+    it 'ignores questions with an incomplete data export identifier' do
+      odd_survey = Surveyor::Parser.new.parse(
+        <<-SURVEY
+          survey "Pet" do
+            section "Name" do
+              q "What is your pet's name?", :data_export_identifier => 'pet'
+              a :string
+            end
+          end
+        SURVEY
+      )
+
+      norm = SurveyorWarehouse::NormalizedSurveyStructure.new(odd_survey)
+
+      expect(norm.tables).to eq([])
+    end
+  end
+end

data/spec/surveyor_warehouse/response_bin_spec.rb

@@ -0,0 +1,45 @@
+require 'spec_helper'
+
+describe 'SurveyorWarehouse::ResponseBin' do
+  let(:color)   { Question.new(:data_export_identifier => 'favorite.color') }
+  let(:food)    { Question.new(:data_export_identifier => 'favorite.food') }
+  let(:utencil) { Question.new(:data_export_identifier => 'hated.utencil') }
+  let(:tree)    { Question.new(:data_export_identifier => 'meh.tree') }
+
+  def r(q)
+    Response.new(:question => q)
+  end
+
+  describe '#tables' do
+   let(:bin) { SurveyorWarehouse::ResponseBin.new.tap{ |bin| bin << r(color) << r(food) << r(utencil) << r(tree) } }
+
+    it 'groups into tables' do
+      bin.rows.map(&:name).sort.should == [:favorite, :hated, :meh]
+    end    
+  end
+
+  describe '::bins' do
+    let(:rs_abc) { ResponseSet.new.tap { |r| r.access_code = 'abc' } }
+    let(:rs_def) { ResponseSet.new.tap { |r| r.access_code = 'def' } }
+
+    let(:r0) { Response.new(:response_set => rs_abc, :question => food) }
+    let(:r1) { Response.new(:response_set => rs_abc, :question => utencil) }
+    let(:r2) { Response.new(:response_set => rs_def, :question => food) }
+    let(:r3) { Response.new(:response_set => rs_def, :question => utencil, :response_group => 2) }
+
+    let(:responses) { [r0, r1, r2, r3] }
+
+    it 'should bin each access_code and response_group combination' do
+      bins = SurveyorWarehouse::ResponseBin.bins(responses)
+      bins.map(&:key).sort.should == %w(abc.1 def.1 def.2)
+      bins.map(&:access_code).sort.should == %w(abc def def)
+    end
+
+    it 'should ignore responses with invalid data export identifiers' do
+      bad_1 = Question.new(:data_export_identifier => '')
+      bad_2 = Question.new(:data_export_identifier => 'tree')
+
+      Response.new(:response_set => rs_abc)
+    end
+  end
+end

data/spec/surveyor_warehouse/response_row_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+
+describe 'SurveyorWarehouse::ResponseRow' do
+  let(:connection) do
+    SurveyorWarehouse::DB.connection
+  end
+
+  let(:favorites) { connection[:favorites] }
+  let(:hated) { connection[:hated] }
+
+  before(:each) do
+    connection.create_table!(:favorites) do
+      primary_key :id, String
+      String :access_code
+      String :name
+      String :color
+    end
+    connection.extension :pg_array
+    Sequel::Model.db.extension :pg_array
+    connection.create_table!(:hated) do
+      primary_key :id, String
+      String :access_code
+      column :colors, 'text[]'
+    end
+  end
+
+  describe '#insert!' do
+    let(:survey) do 
+      Surveyor::Parser.new.parse(<<-SURVEY
+        survey "Favorites" do
+          section "One" do
+            q "What is your favorite name?", :data_export_identifier => 'favorites.name'
+            a :string
+
+            q "What is your favorite color?", :pick => :one, :data_export_identifier => 'favorites.color'
+            a_3 "red"
+            a_5 "green"
+            a_7 "blue"
+            
+            q "Choose the colors you don't like", :pick => :any, :data_export_identifier => 'hated.colors'
+            a_2 "red"
+            a_4 "blue"
+            a_6 "green"
+            
+          end
+        end
+        SURVEY
+      )
+    end
+
+    def find_question_by_dei(dei)
+        questions = survey.sections.map(&:questions).flatten.select { |q| q.data_export_identifier == dei}
+
+      if questions.size == 1
+        questions.first
+      elsif questions.size > 1
+        raise "Too many questions found with dei: #{dei}"
+      else
+        raise "No questions found with dei: #{dei}"
+      end        
+    end
+
+    def find_answer_by_refid(q, refid)
+      answers = q.answers.select { |a| a.reference_identifier == refid}
+
+      if answers.size == 1
+        answers.first
+      elsif answers.size > 1
+        raise "Too many answers found with refid: #{refid}"
+      else
+        raise "No answers found with refid: #{refid}"
+      end 
+    end
+
+    let (:fav_color) { find_question_by_dei('favorites.color') }
+    let (:fav_name) { find_question_by_dei('favorites.name') }
+    let (:hated_colors) { find_question_by_dei('hated.colors') }
+
+    it 'inserts string data' do
+      r = Response.new(:question => fav_name, :answer => fav_name.answers[0], :string_value => 'homer')
+      row = SurveyorWarehouse::ResponseRow.new('favorites', 'abc.1', 'abc')
+      row.responses << r
+      row.insert!
+
+      favorites.map(:name).should == ['homer']
+    end
+
+    it 'inserts integer data' do
+      r = Response.new(:question => fav_color, :answer => find_answer_by_refid(fav_color, '5'))
+      row = SurveyorWarehouse::ResponseRow.new('favorites', 'abc.1', 'abc')
+      row.responses << r
+      row.insert!
+      
+      favorites.map(:color).should == ['5']
+    end
+
+    it 'inserts array data' do
+      r0 = Response.new(:question => hated_colors, :answer => find_answer_by_refid(hated_colors, '4'))
+      r1 = Response.new(:question => hated_colors, :answer => find_answer_by_refid(hated_colors, '6'))
+      row = SurveyorWarehouse::ResponseRow.new('hated', 'abc.1', 'abc')
+      row.responses << r0 << r1
+      row.insert!
+
+      hated.map(:colors).sort.should == [%w(4 6)]
+    end
+  end
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: surveyor_warehouse
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- John Dzak
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: surveyor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: haml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: actionpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Transform surveyor responses into an alternate structure
+email: j-dzak@northwestern.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/surveyor_warehouse.rb
+- lib/surveyor_warehouse/db.rb
+- lib/surveyor_warehouse/extensions/question.rb
+- lib/surveyor_warehouse/extensions/survey.rb
+- lib/surveyor_warehouse/normalized_survey_structure.rb
+- lib/surveyor_warehouse/railtie.rb
+- lib/surveyor_warehouse/response_bin.rb
+- lib/surveyor_warehouse/response_row.rb
+- lib/tasks/transform.rake
+- spec/spec_helper.rb
+- spec/surveyor_warehouse/normalized_survey_structure_spec.rb
+- spec/surveyor_warehouse/response_bin_spec.rb
+- spec/surveyor_warehouse/response_row_spec.rb
+homepage: https://github.com/NUBIC/surveyor_warehouse
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Transform surveyor responses into an alternate structure
+test_files: []