topographer 0.0.7 → 0.0.8

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/lib/topographer/exceptions.rb +5 -3
  3. data/lib/topographer/importer.rb +25 -9
  4. data/lib/topographer/importer/helpers.rb +19 -13
  5. data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
  6. data/lib/topographer/importer/importable.rb +7 -3
  7. data/lib/topographer/importer/input.rb +11 -4
  8. data/lib/topographer/importer/input/base.rb +21 -15
  9. data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
  10. data/lib/topographer/importer/input/roo.rb +31 -24
  11. data/lib/topographer/importer/input/source_data.rb +14 -8
  12. data/lib/topographer/importer/logger.rb +10 -6
  13. data/lib/topographer/importer/logger/base.rb +75 -66
  14. data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
  15. data/lib/topographer/importer/logger/log_entry.rb +31 -25
  16. data/lib/topographer/importer/logger/simple.rb +25 -19
  17. data/lib/topographer/importer/mapper.rb +58 -53
  18. data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
  19. data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
  20. data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
  21. data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
  22. data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
  23. data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
  24. data/lib/topographer/importer/mapper/result.rb +21 -15
  25. data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
  26. data/lib/topographer/importer/strategy.rb +12 -6
  27. data/lib/topographer/importer/strategy/base.rb +43 -38
  28. data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
  29. data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
  30. data/lib/topographer/importer/strategy/import_status.rb +27 -20
  31. data/lib/topographer/importer/strategy/update_record.rb +28 -24
  32. data/lib/topographer/version.rb +1 -1
  33. data/spec/assets/test_files/a_csv.csv +3 -0
  34. data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
  35. data/spec/topographer/importer/importer_spec.rb +21 -5
  36. data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
  37. data/spec/topographer/importer/input/source_data_spec.rb +2 -2
  38. data/spec/topographer/importer/logger/base_spec.rb +19 -0
  39. data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
  40. data/spec/topographer/importer/logger/simple_spec.rb +3 -3
  41. data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
  42. data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
  43. data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
  44. data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
  45. data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
  46. data/spec/topographer/importer/mapper_spec.rb +25 -25
  47. data/spec/topographer/importer/strategy/base_spec.rb +16 -5
  48. data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
  49. data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
  50. data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
  51. data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
  52. data/topographer.gemspec +3 -2
  53. metadata +101 -102
@@ -1,22 +1,28 @@
1
- class Topographer::Importer::Mapper::DefaultFieldMapping < Topographer::Importer::Mapper::FieldMapping
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ class DefaultFieldMapping < Topographer::Importer::Mapper::FieldMapping
2
5
 
3
- def initialize(output_column, &output_block)
4
- unless block_given?
5
- raise Topographer::InvalidMappingError, 'Static fields must have an output block'
6
- end
7
- @output_field = output_column
8
- @output_block = output_block
9
- end
6
+ def initialize(output_column, &output_block)
7
+ unless block_given?
8
+ raise Topographer::InvalidMappingError, 'Static fields must have an output block'
9
+ end
10
+ @output_field = output_column
11
+ @output_block = output_block
12
+ end
10
13
 
11
- def process_input(_, result)
12
- @output_data = @output_block.()
13
- result.add_data(@output_field, @output_data)
14
- rescue => exception
15
- result.add_error(@output_field, exception.message)
16
- end
14
+ def process_input(_, result)
15
+ @output_data = @output_block.()
16
+ result.add_data(@output_field, @output_data)
17
+ rescue => exception
18
+ result.add_error(@output_field, exception.message)
19
+ end
17
20
 
18
- def required?
19
- true
20
- end
21
+ def required?
22
+ true
23
+ end
21
24
 
25
+ end
26
+ end
27
+ end
22
28
  end
@@ -1,55 +1,62 @@
1
1
  require 'active_support/core_ext/hash'
2
2
  require 'active_support/core_ext/object/blank'
3
- class Topographer::Importer::Mapper::FieldMapping
4
- attr_reader :input_columns, :output_field
5
-
6
- def initialize(required, input_columns, output_field, &mapping_behavior)
7
- @required = required
8
- @input_columns = Array(input_columns)
9
- @output_field = output_field
10
- @mapping_behavior = mapping_behavior
11
- @invalid_keys = []
12
- end
13
-
14
- def process_input(input, result)
15
- mapping_input = input.slice(*input_columns)
16
- @invalid_keys = get_invalid_keys(mapping_input)
17
- data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
18
- if !data.nil?
19
- result.add_data(output_field, data)
20
- elsif required?
21
- result.add_error(output_field, invalid_input_error)
22
- end
23
-
24
- rescue => exception
25
- result.add_error(output_field, exception.message)
26
-
27
- end
28
-
29
- def required?
30
- @required
31
- end
32
-
33
- private
34
3
 
35
- def apply_mapping(mapping_input)
36
- if @mapping_behavior
37
- @mapping_behavior.(mapping_input)
38
- else
39
- (mapping_input.size > 1) ? mapping_input.values.join(', ') : mapping_input.values.first
4
+ module Topographer
5
+ class Importer
6
+ class Mapper
7
+ class FieldMapping
8
+ attr_reader :input_columns, :output_field
9
+
10
+ def initialize(required, input_columns, output_field, &mapping_behavior)
11
+ @required = required
12
+ @input_columns = Array(input_columns)
13
+ @output_field = output_field
14
+ @mapping_behavior = mapping_behavior
15
+ @invalid_keys = []
16
+ end
17
+
18
+ def process_input(input, result)
19
+ mapping_input = input.slice(*input_columns)
20
+ @invalid_keys = get_invalid_keys(mapping_input)
21
+ data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
22
+ if !data.nil?
23
+ result.add_data(output_field, data)
24
+ elsif required?
25
+ result.add_error(output_field, invalid_input_error)
26
+ end
27
+
28
+ rescue => exception
29
+ result.add_error(output_field, exception.message)
30
+
31
+ end
32
+
33
+ def required?
34
+ @required
35
+ end
36
+
37
+ private
38
+
39
+ def apply_mapping(mapping_input)
40
+ if @mapping_behavior
41
+ @mapping_behavior.(mapping_input)
42
+ else
43
+ (mapping_input.size > 1) ? mapping_input.values.join(', ') : mapping_input.values.first
44
+ end
45
+ end
46
+
47
+ def invalid_input_error
48
+ "Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
49
+ end
50
+
51
+ def get_invalid_keys(input)
52
+ missing_columns = @input_columns - input.keys
53
+ #reject input that is not blank or the value `false`
54
+ #this allows boolean inputs for required fields
55
+ missing_data = @required ? input.reject { |k, v| !v.blank? || v == false }.keys : []
56
+ missing_columns + missing_data
57
+ end
58
+
59
+ end
40
60
  end
41
61
  end
42
-
43
- def invalid_input_error
44
- "Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
45
- end
46
-
47
- def get_invalid_keys(input)
48
- missing_columns = @input_columns - input.keys
49
- #reject input that is not blank or the value `false`
50
- #this allows boolean inputs for required fields
51
- missing_data = @required ? input.reject{|k,v| !v.blank? || v == false }.keys : []
52
- missing_columns + missing_data
53
- end
54
-
55
62
  end
@@ -1,10 +1,17 @@
1
- class Topographer::Importer::Mapper::IgnoredFieldMapping < Topographer::Importer::Mapper::FieldMapping
2
- def initialize(input_columns)
3
- @input_columns = input_columns
4
- @output_field = nil
5
- end
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ class IgnoredFieldMapping < Topographer::Importer::Mapper::FieldMapping
5
+ def initialize(input_columns)
6
+ @input_columns = input_columns
7
+ @output_field = nil
8
+ end
6
9
 
7
- def required?
8
- false
10
+ def required?
11
+ false
12
+ end
13
+ end
14
+ end
9
15
  end
10
16
  end
17
+
@@ -1,46 +1,59 @@
1
- class Topographer::Importer::Mapper::MapperBuilder
2
- include Topographer::Importer::Mapper::MappingColumns
3
- include Topographer::Importer::Mapper::MappingValidator
4
-
5
- attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
6
- :validation_mappings, :default_values, :key_fields
7
-
8
- def initialize
9
- @required_mappings = {}
10
- @optional_mappings = {}
11
- @ignored_mappings = {}
12
- @validation_mappings = {}
13
- @default_values = {}
14
- @key_fields = []
15
- end
16
-
17
- def required_mapping(input_columns, output_field, &mapping_behavior)
18
- validate_unique_mapping(input_columns, output_field)
19
- @required_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
20
- end
21
-
22
- def optional_mapping(input_columns, output_field, &mapping_behavior)
23
- validate_unique_mapping(input_columns, output_field)
24
- @optional_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
25
- end
26
-
27
- def validation_field(name, input_columns, &mapping_behavior)
28
- validate_unique_validation_name(name)
29
- @validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
30
- end
31
-
32
- def default_value(output_field, &mapping_behavior)
33
- validate_unique_mapping([], output_field)
34
- @default_values[output_field] = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
35
- end
36
-
37
- def key_field(output_field)
38
- validate_key_field(output_field)
39
- @key_fields << output_field
40
- end
41
-
42
- def ignored_column(input_column)
43
- validate_unique_column_mapping_type(input_column, ignored: true)
44
- @ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ class MapperBuilder
5
+ include Topographer::Importer::Mapper::MappingColumns
6
+ include Topographer::Importer::Mapper::MappingValidator
7
+
8
+ attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
9
+ :validation_mappings, :default_values, :key_fields, :field_mappings
10
+
11
+ def initialize
12
+ @required_mappings = {}
13
+ @optional_mappings = {}
14
+ @ignored_mappings = {}
15
+ @validation_mappings = {}
16
+ @default_values = {}
17
+ @key_fields = []
18
+ @field_mappings = {}
19
+ end
20
+
21
+ def required_mapping(input_columns, output_field, &mapping_behavior)
22
+ validate_unique_mapping(input_columns, output_field)
23
+ mapping = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
24
+ @required_mappings[output_field] = mapping
25
+ @field_mappings[output_field] = mapping
26
+ end
27
+
28
+ def optional_mapping(input_columns, output_field, &mapping_behavior)
29
+ validate_unique_mapping(input_columns, output_field)
30
+ mapping = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
31
+ @optional_mappings[output_field] = mapping
32
+ @field_mappings[output_field] = mapping
33
+ end
34
+
35
+ def validation_field(name, input_columns, &mapping_behavior)
36
+ validate_unique_validation_name(name)
37
+ @validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
38
+ end
39
+
40
+ def default_value(output_field, &mapping_behavior)
41
+ validate_unique_mapping([], output_field)
42
+ mapping = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
43
+ @default_values[output_field] = mapping
44
+ @field_mappings[output_field] = mapping
45
+ end
46
+
47
+ def key_field(output_field)
48
+ validate_key_field(output_field)
49
+ @key_fields << output_field
50
+ end
51
+
52
+ def ignored_column(input_column)
53
+ validate_unique_column_mapping_type(input_column, ignored: true)
54
+ @ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
55
+ end
56
+ end
57
+ end
45
58
  end
46
59
  end
@@ -1,46 +1,53 @@
1
- module Topographer::Importer::Mapper::MappingColumns
2
- def output_fields
3
- (required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
4
- end
5
-
6
- def required_input_columns
7
- (required_mapping_columns + validation_mapping_columns).uniq
8
- end
9
-
10
- def input_columns
11
- (required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
12
- end
13
-
14
- def default_fields
15
- default_values.keys
16
- end
17
-
18
- def validation_mapping_columns
19
- validation_mappings.values.flat_map(&:input_columns)
20
- end
21
-
22
- def ignored_mapping_columns
23
- ignored_mappings.values.flat_map(&:input_columns)
24
- end
25
-
26
- def optional_mapping_columns
27
- optional_mappings.values.flat_map(&:input_columns)
28
- end
29
-
30
- def required_mapping_columns
31
- required_mappings.values.flat_map(&:input_columns)
32
- end
33
-
34
- private
35
- def mapped_input_columns
36
- required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
37
- end
38
-
39
- def mappings
40
- required_mappings.merge(optional_mappings).merge(ignored_mappings).merge(default_values)
41
- end
42
-
43
- def non_ignored_columns
44
- required_mappings.merge(optional_mappings)
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ module MappingColumns
5
+ def output_fields
6
+ (required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
7
+ end
8
+
9
+ def required_input_columns
10
+ (required_mapping_columns + validation_mapping_columns).uniq
11
+ end
12
+
13
+ def input_columns
14
+ (required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
15
+ end
16
+
17
+ def default_fields
18
+ default_values.keys
19
+ end
20
+
21
+ def validation_mapping_columns
22
+ validation_mappings.values.flat_map(&:input_columns)
23
+ end
24
+
25
+ def ignored_mapping_columns
26
+ ignored_mappings.values.flat_map(&:input_columns)
27
+ end
28
+
29
+ def optional_mapping_columns
30
+ optional_mappings.values.flat_map(&:input_columns)
31
+ end
32
+
33
+ def required_mapping_columns
34
+ required_mappings.values.flat_map(&:input_columns)
35
+ end
36
+
37
+ private
38
+ def mapped_input_columns
39
+ required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
40
+ end
41
+
42
+ def mappings
43
+ @field_mappings
44
+ end
45
+
46
+ def non_ignored_columns
47
+ required_mappings.merge(optional_mappings)
48
+ end
49
+ end
45
50
  end
51
+ end
46
52
  end
53
+
@@ -1,40 +1,47 @@
1
- module Topographer::Importer::Mapper::MappingValidator
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ module MappingValidator
2
5
 
3
- def validate_unique_validation_name(name)
4
- raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
5
- end
6
+ def validate_unique_validation_name(name)
7
+ raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
8
+ end
6
9
 
7
- def validate_unique_output_mapping(output_field)
8
- if output_fields.include?(output_field)
9
- raise Topographer::InvalidMappingError, 'Output column already mapped.'
10
- end
11
- end
10
+ def validate_unique_output_mapping(output_field)
11
+ if output_fields.include?(output_field)
12
+ raise Topographer::InvalidMappingError, 'Output column already mapped.'
13
+ end
14
+ end
12
15
 
13
- def validate_unique_column_mapping_type(mapping_input_columns, options = {})
14
- ignored = options.fetch(:ignored, false)
15
- mapping_input_columns = Array(mapping_input_columns)
16
- mapping_input_columns.each do |col|
17
- if ignored && ((input_columns + ignored_mapping_columns).include?(col))
18
- raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
19
- elsif (ignored_mapping_columns.include?(col))
20
- raise Topographer::InvalidMappingError, 'Input column already ignored.'
21
- end
22
- end
23
- end
16
+ def validate_unique_column_mapping_type(mapping_input_columns, options = {})
17
+ ignored = options.fetch(:ignored, false)
18
+ mapping_input_columns = Array(mapping_input_columns)
19
+ mapping_input_columns.each do |col|
20
+ if ignored && ((input_columns + ignored_mapping_columns).include?(col))
21
+ raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
22
+ elsif (ignored_mapping_columns.include?(col))
23
+ raise Topographer::InvalidMappingError, 'Input column already ignored.'
24
+ end
25
+ end
26
+ end
24
27
 
25
- def validate_unique_mapping(mapping_input_columns, output_field)
26
- if (output_field.is_a?(Array))
27
- raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
28
- end
29
- validate_unique_column_mapping_type(mapping_input_columns)
30
- validate_unique_output_mapping(output_field)
31
- end
28
+ def validate_unique_mapping(mapping_input_columns, output_field)
29
+ if (output_field.is_a?(Array))
30
+ raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
31
+ end
32
+ validate_unique_column_mapping_type(mapping_input_columns)
33
+ validate_unique_output_mapping(output_field)
34
+ end
32
35
 
33
- def validate_key_field(field)
34
- if field.is_a?(Array)
35
- raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
36
- elsif key_fields.include?(field)
37
- raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
36
+ def validate_key_field(field)
37
+ if field.is_a?(Array)
38
+ raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
39
+ elsif key_fields.include?(field)
40
+ raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
41
+ end
42
+ end
43
+ end
38
44
  end
39
45
  end
40
46
  end
47
+