surveyor_warehouse 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/README.md +70 -0
- data/lib/surveyor_warehouse.rb +47 -0
- data/lib/surveyor_warehouse/db.rb +58 -0
- data/lib/surveyor_warehouse/extensions/question.rb +18 -0
- data/lib/surveyor_warehouse/extensions/survey.rb +15 -0
- data/lib/surveyor_warehouse/normalized_survey_structure.rb +96 -0
- data/lib/surveyor_warehouse/railtie.rb +9 -0
- data/lib/surveyor_warehouse/response_bin.rb +67 -0
- data/lib/surveyor_warehouse/response_row.rb +53 -0
- data/lib/tasks/transform.rake +13 -0
- data/spec/spec_helper.rb +47 -0
- data/spec/surveyor_warehouse/normalized_survey_structure_spec.rb +144 -0
- data/spec/surveyor_warehouse/response_bin_spec.rb +45 -0
- data/spec/surveyor_warehouse/response_row_spec.rb +107 -0
- metadata +169 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
YjkwYWQxMGNhMTJiYzJlNzY4MWM5NjM3N2ZkMDM5MDE5OGY2NDAyMA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
NDQxMGFlY2EzNjYyY2UxOTVlZGVlMjI2NTY5NWE4ZDgwZTM3NDY0OQ==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
ZjE2MmYyNDQ0MGQ3NzNjNmEyN2NkZDRjMzRkOTM0ZGFmMDUxZDY4Y2VjMGM0
|
10
|
+
MzRjNDhlYzkwNThhZTc4ZWQ1ZWJiMGNiZWZiODEyMWFmMDI1ZTJmNmE0Njdi
|
11
|
+
Y2QzZWJkYWZmMjg4YWFmMzkwZjEyMmU3N2U3YWUwOWQyZWEwY2I=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
MGM2ODIyZmE2MmJlZjc0ZjkxMDc3ZTk0ZjJiZmNiMGIxNmI5ODczMDA1NjA2
|
14
|
+
OTU4MWIzYTgxNjMyYTNlYzIyODU1Njg5OGNhOGQ2NzliNTgxMTc5ZDI0Zjhl
|
15
|
+
ZTYwNTNhZWQzN2QzMTRhNzc3MDI5NGEyMzYzODFhN2Y4ZmU4YjM=
|
data/README.md
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# Surveyor Warehouse
|
2
|
+
|
3
|
+
Surveyor Warehouse allows you to extract and transform the responses from
|
4
|
+
surveyor response sets into an alternate database table structure. This
|
5
|
+
can be generally helpful for querying surveyor responses within a table
|
6
|
+
structure of one's choice.
|
7
|
+
|
8
|
+
The alternate database table structure is defined by adding a
|
9
|
+
`:data_export_identifier` attribute to each question with the
|
10
|
+
format "table.column" (e.g. 'subjects.name').
|
11
|
+
|
12
|
+
## Example
|
13
|
+
|
14
|
+
Given the survey:
|
15
|
+
```
|
16
|
+
q 'What is your name?', :data_export_identifier => 'subjects.name'
|
17
|
+
a :string
|
18
|
+
|
19
|
+
q 'What is your age?', :data_export_identifier => 'subjects.age'
|
20
|
+
a :integer
|
21
|
+
|
22
|
+
q 'What is your gender?', :pick => :one, :data_export_identifier => 'subjects.gender'
|
23
|
+
a 'Male', :reference_identifier => "male"
|
24
|
+
a 'Female', :reference_identifier => "female"
|
25
|
+
|
26
|
+
q 'Which brand of shoes do you wear?', :pick => :any, :data_export_identifier => 'subjects.shoes'
|
27
|
+
a 'Nike', :reference_identifier => 'nike'
|
28
|
+
a 'Adidas', :reference_identifier => 'adidas'
|
29
|
+
a 'None', :omit, :reference_identifier => 'none'
|
30
|
+
|
31
|
+
repeater 'List all your siblings' do
|
32
|
+
q 'Name', :data_export_identifier => 'siblings.name'
|
33
|
+
a :string
|
34
|
+
end
|
35
|
+
```
|
36
|
+
|
37
|
+
Responses would be transformed into:
|
38
|
+
|
39
|
+
```
|
40
|
+
db=# select * from subjects;
|
41
|
+
name | age | gender | shoes | access_code | id
|
42
|
+
---------+-----+--------+---------------+-------------+--------------
|
43
|
+
raphael | 18 | male | {nike,adidas} | QUhwHtMPyQ | QUhwHtMPyQ.0
|
44
|
+
stephen | 50 | male | {none} | zZu1OV1hmw | zZu1OV1hmw.0
|
45
|
+
(2 rows)
|
46
|
+
|
47
|
+
db=# select * from siblings;
|
48
|
+
name | access_code | id
|
49
|
+
--------------+-------------+--------------
|
50
|
+
leonardo | QUhwHtMPyQ | QUhwHtMPyQ.0
|
51
|
+
michelangelo | QUhwHtMPyQ | QUhwHtMPyQ.1
|
52
|
+
elizabeth | zZu1OV1hmw | zZu1OV1hmw.0
|
53
|
+
james | zZu1OV1hmw | zZu1OV1hmw.1
|
54
|
+
(4 rows)
|
55
|
+
```
|
56
|
+
|
57
|
+
|
58
|
+
## Getting Started
|
59
|
+
|
60
|
+
1. Survey questions must have data_export_identifiers with the
|
61
|
+
format "table.column" (e.g. 'subjects.name')
|
62
|
+
2. Take the survey
|
63
|
+
3. `bundle exec rake surveyor:warehouse`
|
64
|
+
4. Query the results
|
65
|
+
|
66
|
+
|
67
|
+
|
68
|
+
## Limitations
|
69
|
+
|
70
|
+
* Currently only works with PostgresSQL
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'surveyor'
|
2
|
+
require 'active_support/concern'
|
3
|
+
|
4
|
+
require 'surveyor_warehouse/normalized_survey_structure'
|
5
|
+
require 'surveyor_warehouse/response_bin'
|
6
|
+
require 'surveyor_warehouse/response_row'
|
7
|
+
require 'surveyor_warehouse/railtie' if defined?(Rails)
|
8
|
+
require 'surveyor_warehouse/extensions/question'
|
9
|
+
require 'surveyor_warehouse/extensions/survey'
|
10
|
+
|
11
|
+
module SurveyorWarehouse
|
12
|
+
def self.logger
|
13
|
+
@logger ||= Logger.new(STDOUT)
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.transform
|
17
|
+
Survey.send(:include, SurveyorWarehouse::Extensions::Survey)
|
18
|
+
|
19
|
+
surveys = Survey.current_versions
|
20
|
+
|
21
|
+
surveys.each do |s|
|
22
|
+
ns = NormalizedSurveyStructure.new(s)
|
23
|
+
ns.create!
|
24
|
+
s.response_sets.each do |rs|
|
25
|
+
# logger.debug("Transforming [ResponseSet id:#{rs.id}] for [Survey id:#{s.id} title:'#{s.title}")
|
26
|
+
bins = ResponseBin.bins(rs.responses)
|
27
|
+
bins.map(&:rows).flatten.each(&:insert!)
|
28
|
+
logger.info("Transformed [ResponseSet id:#{rs.id}] for [Survey id:#{s.id} title:'#{s.title}']")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.clobber
|
34
|
+
Survey.send(:include, SurveyorWarehouse::Extensions::Survey)
|
35
|
+
|
36
|
+
surveys = Survey.current_versions
|
37
|
+
|
38
|
+
surveys.map(&:response_sets).flatten.each do |rs|
|
39
|
+
ns = NormalizedSurveyStructure.new(rs.survey)
|
40
|
+
ns.destroy!
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Survey.send(:include, SurveyorWarehouse::Extensions::Survey)
|
47
|
+
# Question.send(:include, SurveyorWarehouse::Extensions::Question)
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module SurveyorWarehouse
|
2
|
+
module DB
|
3
|
+
def self.connection
|
4
|
+
@connection ||= Sequel.connect(
|
5
|
+
:adapter=> adapter,
|
6
|
+
:host=>'localhost',
|
7
|
+
:database=> database,
|
8
|
+
:user=> username,
|
9
|
+
:password=> password).extension(:pg_array)
|
10
|
+
@connection.extend(SequelExtension::Connection)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.configurations
|
14
|
+
@configurations ||= ::ActiveRecord::Base.configurations[Rails.env]
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.username
|
18
|
+
@username ||= configurations['username']
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.password
|
22
|
+
@password ||= configurations['password']
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.database
|
26
|
+
@database ||= configurations['database']
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.adapter
|
30
|
+
@adapter ||=
|
31
|
+
if 'postgresql' == configurations['adapter']
|
32
|
+
'postgres'
|
33
|
+
else
|
34
|
+
raise "Unsupported database adapter: #{db['adapter']}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
##
|
39
|
+
# Columns are returned a hash like below:
|
40
|
+
#
|
41
|
+
# { :column1 => [:primary_key], :column2 => [:not_null] }
|
42
|
+
def self.columns(tablename)
|
43
|
+
connection.schema(tablename.to_sym).inject({}) do |attrs, (cname, cattrs)|
|
44
|
+
attrs.merge(cname => cattrs)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
module SequelExtension
|
50
|
+
module Connection
|
51
|
+
def columns(tablename)
|
52
|
+
self.schema(tablename.to_sym).inject({}) do |attrs, (cname, cattrs)|
|
53
|
+
attrs.merge(cname => cattrs)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
|
3
|
+
module SurveyorWarehouse
|
4
|
+
module Extensions
|
5
|
+
module Question
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
included do
|
8
|
+
##
|
9
|
+
# A question's data_export_identifier should follow the
|
10
|
+
# format 'table.column' (e.g. 'patients.name')
|
11
|
+
def valid_data_export_identifier?
|
12
|
+
dei = self.try(:data_export_identifier)
|
13
|
+
dei.present? && dei.split('.').size == 2
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
|
3
|
+
module SurveyorWarehouse
|
4
|
+
module Extensions
|
5
|
+
module Survey
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
module ClassMethods
|
8
|
+
def current_versions
|
9
|
+
group = ::Survey.order("created_at DESC, survey_version DESC").all.group_by(&:access_code)
|
10
|
+
group.map { |access_code, surveys| surveys.first }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'surveyor_warehouse/db'
|
2
|
+
|
3
|
+
module SurveyorWarehouse
|
4
|
+
class NormalizedSurveyStructure
|
5
|
+
def initialize(survey)
|
6
|
+
@survey = survey
|
7
|
+
end
|
8
|
+
|
9
|
+
##
|
10
|
+
# response_classes
|
11
|
+
def tables
|
12
|
+
questions = @survey.sections_with_questions.map(&:questions).flatten
|
13
|
+
Question.send(:include, SurveyorWarehouse::Extensions::Question)
|
14
|
+
predefs = questions.select {|q| q.valid_data_export_identifier? }.map do |question|
|
15
|
+
dei_tokens = question.data_export_identifier.split('.')
|
16
|
+
|
17
|
+
table = dei_tokens[0]
|
18
|
+
column = dei_tokens[1]
|
19
|
+
type = database_type(question)
|
20
|
+
|
21
|
+
[table, column, type]
|
22
|
+
end
|
23
|
+
|
24
|
+
tables = {}
|
25
|
+
predefs.each do |d|
|
26
|
+
table_name, column_name, column_type = d
|
27
|
+
table = tables[table_name] || TableDefinition.new(table_name)
|
28
|
+
table.columns << ColumnDefinition.new(column_name, column_type)
|
29
|
+
tables.merge!(table_name => table)
|
30
|
+
end
|
31
|
+
|
32
|
+
# Add primary key and response set access code columns
|
33
|
+
tables.each do |name, tdef|
|
34
|
+
tdef.columns << ColumnDefinition.new('access_code', 'text')
|
35
|
+
tdef.columns << ColumnDefinition.new('id', 'text')
|
36
|
+
end
|
37
|
+
|
38
|
+
@tables ||= tables.values
|
39
|
+
end
|
40
|
+
|
41
|
+
##
|
42
|
+
# Force creates the structure by destroying all the tables first
|
43
|
+
#
|
44
|
+
def create!
|
45
|
+
destroy!
|
46
|
+
tables.each(&:create)
|
47
|
+
end
|
48
|
+
|
49
|
+
##
|
50
|
+
# Destroy all the tables defined in the survey data_export_identifiers
|
51
|
+
#
|
52
|
+
def destroy!
|
53
|
+
drop_ddl = tables.map(&:name).map do |t|
|
54
|
+
"drop table if exists #{t};\n"
|
55
|
+
end.join
|
56
|
+
ActiveRecord::Base.connection.execute(drop_ddl)
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
SUPPORTED_TYPES = %w(date datetime decimal float integer string text)
|
61
|
+
def database_type(question)
|
62
|
+
case question.pick
|
63
|
+
when 'none'
|
64
|
+
response_classes = question.answers.map(&:response_class).uniq
|
65
|
+
raise "Muliple answer types are unsupported: #{response_classes.join(', ')}" if response_classes.size > 1
|
66
|
+
type = response_classes[0].tap do |t|
|
67
|
+
raise "Unsupported column type '#{t}' for question: #{question.inspect}}" unless SUPPORTED_TYPES.include?(t)
|
68
|
+
end
|
69
|
+
type == 'string' ? 'text' : type
|
70
|
+
when 'one'
|
71
|
+
'text'
|
72
|
+
when 'any'
|
73
|
+
"text[]"
|
74
|
+
else
|
75
|
+
raise "Unable to find type for question: #{question.inspect}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
class TableDefinition < Struct.new(:name)
|
80
|
+
def columns
|
81
|
+
@columns ||= []
|
82
|
+
end
|
83
|
+
|
84
|
+
def create
|
85
|
+
ActiveRecord::Base.connection.create_table(name.to_sym, :id => false) do |t|
|
86
|
+
columns.each do |c|
|
87
|
+
t.column c.name, c.type
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
class ColumnDefinition < Struct.new(:name, :type)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'surveyor_warehouse/response_row'
|
2
|
+
|
3
|
+
module SurveyorWarehouse
|
4
|
+
class ResponseBin < Struct.new(:key, :access_code, :response_group)
|
5
|
+
def <<(response)
|
6
|
+
q = response.question
|
7
|
+
Question.send(:include, SurveyorWarehouse::Extensions::Question)
|
8
|
+
if q.valid_data_export_identifier?
|
9
|
+
(@responses ||= []) << response
|
10
|
+
else
|
11
|
+
puts "Ignoring response with data_export_identifier: #{q.data_export_identifier}"
|
12
|
+
end
|
13
|
+
self
|
14
|
+
end
|
15
|
+
|
16
|
+
def responses
|
17
|
+
(@responses || []).dup.freeze
|
18
|
+
end
|
19
|
+
|
20
|
+
##
|
21
|
+
# Groups responses into "bins" using the 'access_code.response_group' as
|
22
|
+
# the bin key.
|
23
|
+
#
|
24
|
+
# See below for an example how the binning behavior works:
|
25
|
+
#
|
26
|
+
# responses = []
|
27
|
+
# responses.push(
|
28
|
+
# [Response access_code:'abc' response_group:0 string_value:'Jeff'],
|
29
|
+
# [Response access_code:'abc' response_group:0 string_value:'Bezos'],
|
30
|
+
# [Response access_code:'xyz' response_group:0 string_value:'Kindle'],
|
31
|
+
# [Response access_code:'xyz' response_group:1 string_value:'EC2'])
|
32
|
+
#
|
33
|
+
# ResponseBinner.new(responses).bins # => Results in the three bins below
|
34
|
+
#
|
35
|
+
# [ResponseBin key:'abc:0' responses: [
|
36
|
+
# [Response access_code:'abc' response_group:0 string_value:'Jeff'],
|
37
|
+
# [Response access_code:'abc' response_group:0 string_value:'Bezos']]
|
38
|
+
# [ResponseBin key:'xyz:0' responses: [
|
39
|
+
# [Response access_code:'xyz' response_group:0 string_value:'Kindle']]
|
40
|
+
# [ResponseBin key:'xyz:1' responses: [
|
41
|
+
# [Response access_code:'xyz' response_group:1 string_value:'EC2']]
|
42
|
+
DEFAULT_RESPONSE_GROUP = 0
|
43
|
+
def self.bins(responses)
|
44
|
+
bins = {}
|
45
|
+
responses.each do |r|
|
46
|
+
ac = r.response_set.access_code
|
47
|
+
rg = r.response_group || DEFAULT_RESPONSE_GROUP
|
48
|
+
key = "#{ac}.#{rg}"
|
49
|
+
bin = bins[key] || ResponseBin.new(key, ac, rg)
|
50
|
+
bin << r
|
51
|
+
bins.merge!(key => bin)
|
52
|
+
end
|
53
|
+
bins.values
|
54
|
+
end
|
55
|
+
|
56
|
+
def rows
|
57
|
+
rows = {}
|
58
|
+
@responses.each do |r|
|
59
|
+
table_name = r.question.data_export_identifier.split('.')[0]
|
60
|
+
table = rows[table_name] || ResponseRow.new(table_name, key, access_code)
|
61
|
+
table.responses << r
|
62
|
+
rows.merge!(table_name => table)
|
63
|
+
end
|
64
|
+
rows.values
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'surveyor_warehouse/db'
|
2
|
+
require 'sequel'
|
3
|
+
|
4
|
+
module SurveyorWarehouse
|
5
|
+
class ResponseRow < Struct.new(:name, :id, :access_code)
|
6
|
+
def responses
|
7
|
+
@responses ||= []
|
8
|
+
end
|
9
|
+
|
10
|
+
def name
|
11
|
+
super.to_sym
|
12
|
+
end
|
13
|
+
|
14
|
+
def insert!
|
15
|
+
column_and_values = {}
|
16
|
+
col_and_val = @responses.map do |r|
|
17
|
+
column = r.question.data_export_identifier.split('.')[1]
|
18
|
+
|
19
|
+
value_field = case r.answer.response_class
|
20
|
+
when 'datetime'; 'datetime_value'
|
21
|
+
when 'date'; 'date_value'
|
22
|
+
when 'time'; 'time_value'
|
23
|
+
when 'float'; 'float_value'
|
24
|
+
when 'integer'; 'integer_value'
|
25
|
+
when 'string'; 'string_value'
|
26
|
+
when 'text'; 'text_value'
|
27
|
+
end
|
28
|
+
|
29
|
+
value = value_field.present? ? r.send(value_field) : r.answer.try(:reference_identifier)
|
30
|
+
[column.to_sym, value]
|
31
|
+
end
|
32
|
+
|
33
|
+
mDB = SurveyorWarehouse::DB.connection
|
34
|
+
schema = mDB.schema(name.to_sym).inject({}) { |attrs, (cname, cattrs)| attrs.merge(cname => cattrs) }
|
35
|
+
|
36
|
+
column_and_values = col_and_val.inject({}) do |accum, (col, val)|
|
37
|
+
if schema[col][:db_type] =~ /\[\]/
|
38
|
+
accum[col] = Sequel.pg_array((accum[col] || []) << val)
|
39
|
+
else
|
40
|
+
accum[col] = val
|
41
|
+
end
|
42
|
+
accum
|
43
|
+
end
|
44
|
+
|
45
|
+
column_and_values.merge!('id' => id, 'access_code' => access_code)
|
46
|
+
|
47
|
+
ds = mDB[name]
|
48
|
+
|
49
|
+
ds.insert(column_and_values)
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'surveyor_warehouse'
|
2
|
+
|
3
|
+
namespace :surveyor do
|
4
|
+
desc 'Transforms the survey response sets into the defined warehouse form'
|
5
|
+
task :warehouse => [:environment, :clobber] do
|
6
|
+
SurveyorWarehouse.transform
|
7
|
+
end
|
8
|
+
|
9
|
+
desc 'Clobber the warehouse'
|
10
|
+
task :clobber do
|
11
|
+
SurveyorWarehouse.clobber
|
12
|
+
end
|
13
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
# This file is copied to spec/ when you run 'rails generate rspec:install'
|
2
|
+
ENV["RAILS_ENV"] ||= 'test'
|
3
|
+
begin
|
4
|
+
require File.expand_path("../../testbed/config/environment", __FILE__)
|
5
|
+
rescue LoadError => e
|
6
|
+
fail "Could not load the testbed app. Have you generated it?\n#{e.class}: #{e}"
|
7
|
+
end
|
8
|
+
|
9
|
+
require 'rspec/autorun'
|
10
|
+
require 'database_cleaner'
|
11
|
+
|
12
|
+
RSpec.configure do |config|
|
13
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
14
|
+
|
15
|
+
# == Mock Framework
|
16
|
+
#
|
17
|
+
# If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
|
18
|
+
#
|
19
|
+
# config.mock_with :mocha
|
20
|
+
# config.mock_with :flexmock
|
21
|
+
# config.mock_with :rr
|
22
|
+
config.mock_with :rspec
|
23
|
+
|
24
|
+
## Database Cleaner
|
25
|
+
|
26
|
+
config.before(:suite) do
|
27
|
+
DatabaseCleaner.strategy = :transaction
|
28
|
+
DatabaseCleaner.clean_with(:truncation)
|
29
|
+
end
|
30
|
+
|
31
|
+
config.before(:each, :clean_with_truncation) do
|
32
|
+
DatabaseCleaner.strategy = :truncation
|
33
|
+
end
|
34
|
+
|
35
|
+
config.after(:each, :clean_with_truncation) do
|
36
|
+
DatabaseCleaner.strategy = :transaction
|
37
|
+
end
|
38
|
+
|
39
|
+
config.before(:each) do
|
40
|
+
DatabaseCleaner.start
|
41
|
+
end
|
42
|
+
|
43
|
+
config.after(:each) do
|
44
|
+
DatabaseCleaner.clean
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'surveyor_warehouse'
|
3
|
+
|
4
|
+
describe SurveyorWarehouse::NormalizedSurveyStructure do
|
5
|
+
let(:survey) do
|
6
|
+
Surveyor::Parser.new.parse(
|
7
|
+
<<-SURVEY
|
8
|
+
survey "Favorites" do
|
9
|
+
section "One" do
|
10
|
+
q "What is your favorite name?", :data_export_identifier => 'favorites.name'
|
11
|
+
a :string
|
12
|
+
|
13
|
+
q "What is your favorite color?", :pick => :one, :data_export_identifier => 'favorites.color'
|
14
|
+
a "red"
|
15
|
+
a "blue"
|
16
|
+
|
17
|
+
q "Choose the colors you don't like", :pick => :any, :data_export_identifier => 'hated.colors'
|
18
|
+
a "red"
|
19
|
+
a "blue"
|
20
|
+
a "green"
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
24
|
+
SURVEY
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
let(:normalized) { SurveyorWarehouse::NormalizedSurveyStructure.new(survey) }
|
29
|
+
|
30
|
+
let(:connection) do
|
31
|
+
ActiveRecord::Base.connection
|
32
|
+
end
|
33
|
+
|
34
|
+
def drop_tables(*tables)
|
35
|
+
tables.each do |t|
|
36
|
+
connection.drop_table(t) if connection.table_exists?(t)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe '#create!' do
|
41
|
+
after(:each) do
|
42
|
+
drop_tables(:favorites, :hated)
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'creates tables with scalar types' do
|
46
|
+
normalized.create!
|
47
|
+
connection.table_exists?(:favorites).should be(true)
|
48
|
+
columns = connection.columns(:favorites).map{ |c| [c.name, c.sql_type] }.sort
|
49
|
+
columns.should == [
|
50
|
+
%w(access_code text),
|
51
|
+
%w(color text),
|
52
|
+
%w(id text),
|
53
|
+
%w(name text)
|
54
|
+
]
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'creates tables with array types' do
|
58
|
+
normalized.create!
|
59
|
+
connection.table_exists?(:hated).should be(true)
|
60
|
+
columns = connection.columns(:hated).map(&:name).sort
|
61
|
+
columns.should == %w(access_code colors id)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
describe '#destroy!' do
|
66
|
+
before(:each) do
|
67
|
+
connection.create_table(:foo) do |t|
|
68
|
+
t.integer :id
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
after(:each) do
|
73
|
+
drop_tables(:favorites, :hated, :foo)
|
74
|
+
end
|
75
|
+
|
76
|
+
it "drops all normalized tables" do
|
77
|
+
normalized.create!
|
78
|
+
normalized.destroy!
|
79
|
+
connection.table_exists?(:favorites).should be(false)
|
80
|
+
connection.table_exists?(:hated).should be(false)
|
81
|
+
connection.table_exists?(:foo).should be(true)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
describe '#tables' do
|
86
|
+
let (:tables) { normalized.tables }
|
87
|
+
let (:columns) { tables.inject({}){ |accum, t| accum.merge(t.name => t.columns) } }
|
88
|
+
|
89
|
+
it 'has tables from data export identifiers' do
|
90
|
+
tables.map(&:name).sort.should == %w(favorites hated)
|
91
|
+
end
|
92
|
+
|
93
|
+
it 'has columns with scalar types' do
|
94
|
+
columns['favorites'].map { |col| [col.name, col.type] }.sort.should == [
|
95
|
+
['access_code', 'text'],
|
96
|
+
['color', 'text'],
|
97
|
+
['id', 'text'],
|
98
|
+
['name', 'text']
|
99
|
+
]
|
100
|
+
end
|
101
|
+
|
102
|
+
it 'has columns with array types' do
|
103
|
+
columns['hated'].map { |col| [col.name, col.type] }.sort.should == [
|
104
|
+
['access_code', 'text'],
|
105
|
+
['colors', 'text[]'],
|
106
|
+
['id', 'text']
|
107
|
+
]
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'ignores questions without a data export identifier' do
|
111
|
+
blank_survey = Surveyor::Parser.new.parse(
|
112
|
+
<<-SURVEY
|
113
|
+
survey "Pet" do
|
114
|
+
section "Name" do
|
115
|
+
q "What is your pet's name?"
|
116
|
+
a :string
|
117
|
+
end
|
118
|
+
end
|
119
|
+
SURVEY
|
120
|
+
)
|
121
|
+
|
122
|
+
norm = SurveyorWarehouse::NormalizedSurveyStructure.new(blank_survey)
|
123
|
+
|
124
|
+
expect(norm.tables).to eq([])
|
125
|
+
end
|
126
|
+
|
127
|
+
it 'ignores questions with an incomplete data export identifier' do
|
128
|
+
odd_survey = Surveyor::Parser.new.parse(
|
129
|
+
<<-SURVEY
|
130
|
+
survey "Pet" do
|
131
|
+
section "Name" do
|
132
|
+
q "What is your pet's name?", :data_export_identifier => 'pet'
|
133
|
+
a :string
|
134
|
+
end
|
135
|
+
end
|
136
|
+
SURVEY
|
137
|
+
)
|
138
|
+
|
139
|
+
norm = SurveyorWarehouse::NormalizedSurveyStructure.new(odd_survey)
|
140
|
+
|
141
|
+
expect(norm.tables).to eq([])
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'SurveyorWarehouse::ResponseBin' do
|
4
|
+
let(:color) { Question.new(:data_export_identifier => 'favorite.color') }
|
5
|
+
let(:food) { Question.new(:data_export_identifier => 'favorite.food') }
|
6
|
+
let(:utencil) { Question.new(:data_export_identifier => 'hated.utencil') }
|
7
|
+
let(:tree) { Question.new(:data_export_identifier => 'meh.tree') }
|
8
|
+
|
9
|
+
def r(q)
|
10
|
+
Response.new(:question => q)
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#tables' do
|
14
|
+
let(:bin) { SurveyorWarehouse::ResponseBin.new.tap{ |bin| bin << r(color) << r(food) << r(utencil) << r(tree) } }
|
15
|
+
|
16
|
+
it 'groups into tables' do
|
17
|
+
bin.rows.map(&:name).sort.should == [:favorite, :hated, :meh]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '::bins' do
|
22
|
+
let(:rs_abc) { ResponseSet.new.tap { |r| r.access_code = 'abc' } }
|
23
|
+
let(:rs_def) { ResponseSet.new.tap { |r| r.access_code = 'def' } }
|
24
|
+
|
25
|
+
let(:r0) { Response.new(:response_set => rs_abc, :question => food) }
|
26
|
+
let(:r1) { Response.new(:response_set => rs_abc, :question => utencil) }
|
27
|
+
let(:r2) { Response.new(:response_set => rs_def, :question => food) }
|
28
|
+
let(:r3) { Response.new(:response_set => rs_def, :question => utencil, :response_group => 2) }
|
29
|
+
|
30
|
+
let(:responses) { [r0, r1, r2, r3] }
|
31
|
+
|
32
|
+
it 'should bin each access_code and response_group combination' do
|
33
|
+
bins = SurveyorWarehouse::ResponseBin.bins(responses)
|
34
|
+
bins.map(&:key).sort.should == %w(abc.1 def.1 def.2)
|
35
|
+
bins.map(&:access_code).sort.should == %w(abc def def)
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should ignore responses with invalid data export identifiers' do
|
39
|
+
bad_1 = Question.new(:data_export_identifier => '')
|
40
|
+
bad_2 = Question.new(:data_export_identifier => 'tree')
|
41
|
+
|
42
|
+
Response.new(:response_set => rs_abc)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'SurveyorWarehouse::ResponseRow' do
|
4
|
+
let(:connection) do
|
5
|
+
SurveyorWarehouse::DB.connection
|
6
|
+
end
|
7
|
+
|
8
|
+
let(:favorites) { connection[:favorites] }
|
9
|
+
let(:hated) { connection[:hated] }
|
10
|
+
|
11
|
+
before(:each) do
|
12
|
+
connection.create_table!(:favorites) do
|
13
|
+
primary_key :id, String
|
14
|
+
String :access_code
|
15
|
+
String :name
|
16
|
+
String :color
|
17
|
+
end
|
18
|
+
connection.extension :pg_array
|
19
|
+
Sequel::Model.db.extension :pg_array
|
20
|
+
connection.create_table!(:hated) do
|
21
|
+
primary_key :id, String
|
22
|
+
String :access_code
|
23
|
+
column :colors, 'text[]'
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
describe '#insert!' do
|
28
|
+
let(:survey) do
|
29
|
+
Surveyor::Parser.new.parse(<<-SURVEY
|
30
|
+
survey "Favorites" do
|
31
|
+
section "One" do
|
32
|
+
q "What is your favorite name?", :data_export_identifier => 'favorites.name'
|
33
|
+
a :string
|
34
|
+
|
35
|
+
q "What is your favorite color?", :pick => :one, :data_export_identifier => 'favorites.color'
|
36
|
+
a_3 "red"
|
37
|
+
a_5 "green"
|
38
|
+
a_7 "blue"
|
39
|
+
|
40
|
+
q "Choose the colors you don't like", :pick => :any, :data_export_identifier => 'hated.colors'
|
41
|
+
a_2 "red"
|
42
|
+
a_4 "blue"
|
43
|
+
a_6 "green"
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
SURVEY
|
48
|
+
)
|
49
|
+
end
|
50
|
+
|
51
|
+
def find_question_by_dei(dei)
|
52
|
+
questions = survey.sections.map(&:questions).flatten.select { |q| q.data_export_identifier == dei}
|
53
|
+
|
54
|
+
if questions.size == 1
|
55
|
+
questions.first
|
56
|
+
elsif questions.size > 1
|
57
|
+
raise "Too many questions found with dei: #{dei}"
|
58
|
+
else
|
59
|
+
raise "No questions found with dei: #{dei}"
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def find_answer_by_refid(q, refid)
|
64
|
+
answers = q.answers.select { |a| a.reference_identifier == refid}
|
65
|
+
|
66
|
+
if answers.size == 1
|
67
|
+
answers.first
|
68
|
+
elsif answers.size > 1
|
69
|
+
raise "Too many answers found with refid: #{refid}"
|
70
|
+
else
|
71
|
+
raise "No answers found with refid: #{refid}"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
let (:fav_color) { find_question_by_dei('favorites.color') }
|
76
|
+
let (:fav_name) { find_question_by_dei('favorites.name') }
|
77
|
+
let (:hated_colors) { find_question_by_dei('hated.colors') }
|
78
|
+
|
79
|
+
it 'inserts string data' do
|
80
|
+
r = Response.new(:question => fav_name, :answer => fav_name.answers[0], :string_value => 'homer')
|
81
|
+
row = SurveyorWarehouse::ResponseRow.new('favorites', 'abc.1', 'abc')
|
82
|
+
row.responses << r
|
83
|
+
row.insert!
|
84
|
+
|
85
|
+
favorites.map(:name).should == ['homer']
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'inserts integer data' do
|
89
|
+
r = Response.new(:question => fav_color, :answer => find_answer_by_refid(fav_color, '5'))
|
90
|
+
row = SurveyorWarehouse::ResponseRow.new('favorites', 'abc.1', 'abc')
|
91
|
+
row.responses << r
|
92
|
+
row.insert!
|
93
|
+
|
94
|
+
favorites.map(:color).should == ['5']
|
95
|
+
end
|
96
|
+
|
97
|
+
it 'inserts array data' do
|
98
|
+
r0 = Response.new(:question => hated_colors, :answer => find_answer_by_refid(hated_colors, '4'))
|
99
|
+
r1 = Response.new(:question => hated_colors, :answer => find_answer_by_refid(hated_colors, '6'))
|
100
|
+
row = SurveyorWarehouse::ResponseRow.new('hated', 'abc.1', 'abc')
|
101
|
+
row.responses << r0 << r1
|
102
|
+
row.insert!
|
103
|
+
|
104
|
+
hated.map(:colors).sort.should == [%w(4 6)]
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
metadata
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: surveyor_warehouse
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- John Dzak
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-05-27 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: surveyor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: haml
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rails
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activesupport
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ! '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: actionpack
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ! '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ! '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: activerecord
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ! '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ! '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: sequel
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ! '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ! '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: pg
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ! '>='
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
description: Transform surveyor responses into an alternate structure
|
126
|
+
email: j-dzak@northwestern.edu
|
127
|
+
executables: []
|
128
|
+
extensions: []
|
129
|
+
extra_rdoc_files: []
|
130
|
+
files:
|
131
|
+
- README.md
|
132
|
+
- lib/surveyor_warehouse.rb
|
133
|
+
- lib/surveyor_warehouse/db.rb
|
134
|
+
- lib/surveyor_warehouse/extensions/question.rb
|
135
|
+
- lib/surveyor_warehouse/extensions/survey.rb
|
136
|
+
- lib/surveyor_warehouse/normalized_survey_structure.rb
|
137
|
+
- lib/surveyor_warehouse/railtie.rb
|
138
|
+
- lib/surveyor_warehouse/response_bin.rb
|
139
|
+
- lib/surveyor_warehouse/response_row.rb
|
140
|
+
- lib/tasks/transform.rake
|
141
|
+
- spec/spec_helper.rb
|
142
|
+
- spec/surveyor_warehouse/normalized_survey_structure_spec.rb
|
143
|
+
- spec/surveyor_warehouse/response_bin_spec.rb
|
144
|
+
- spec/surveyor_warehouse/response_row_spec.rb
|
145
|
+
homepage: https://github.com/NUBIC/surveyor_warehouse
|
146
|
+
licenses:
|
147
|
+
- MIT
|
148
|
+
metadata: {}
|
149
|
+
post_install_message:
|
150
|
+
rdoc_options: []
|
151
|
+
require_paths:
|
152
|
+
- lib
|
153
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
|
+
requirements:
|
160
|
+
- - ! '>='
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: '0'
|
163
|
+
requirements: []
|
164
|
+
rubyforge_project:
|
165
|
+
rubygems_version: 2.2.2
|
166
|
+
signing_key:
|
167
|
+
specification_version: 4
|
168
|
+
summary: Transform surveyor responses into an alternate structure
|
169
|
+
test_files: []
|