topographer 0.0.7 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/lib/topographer/exceptions.rb +5 -3
  3. data/lib/topographer/importer.rb +25 -9
  4. data/lib/topographer/importer/helpers.rb +19 -13
  5. data/lib/topographer/importer/helpers/write_log_to_csv.rb +75 -67
  6. data/lib/topographer/importer/importable.rb +7 -3
  7. data/lib/topographer/importer/input.rb +11 -4
  8. data/lib/topographer/importer/input/base.rb +21 -15
  9. data/lib/topographer/importer/input/delimited_spreadsheet.rb +55 -0
  10. data/lib/topographer/importer/input/roo.rb +31 -24
  11. data/lib/topographer/importer/input/source_data.rb +14 -8
  12. data/lib/topographer/importer/logger.rb +10 -6
  13. data/lib/topographer/importer/logger/base.rb +75 -66
  14. data/lib/topographer/importer/logger/fatal_error_entry.rb +22 -16
  15. data/lib/topographer/importer/logger/log_entry.rb +31 -25
  16. data/lib/topographer/importer/logger/simple.rb +25 -19
  17. data/lib/topographer/importer/mapper.rb +58 -53
  18. data/lib/topographer/importer/mapper/default_field_mapping.rb +23 -17
  19. data/lib/topographer/importer/mapper/field_mapping.rb +56 -49
  20. data/lib/topographer/importer/mapper/ignored_field_mapping.rb +14 -7
  21. data/lib/topographer/importer/mapper/mapper_builder.rb +57 -44
  22. data/lib/topographer/importer/mapper/mapping_columns.rb +51 -44
  23. data/lib/topographer/importer/mapper/mapping_validator.rb +39 -32
  24. data/lib/topographer/importer/mapper/result.rb +21 -15
  25. data/lib/topographer/importer/mapper/validation_field_mapping.rb +35 -28
  26. data/lib/topographer/importer/strategy.rb +12 -6
  27. data/lib/topographer/importer/strategy/base.rb +43 -38
  28. data/lib/topographer/importer/strategy/create_or_update_record.rb +39 -34
  29. data/lib/topographer/importer/strategy/import_new_record.rb +16 -10
  30. data/lib/topographer/importer/strategy/import_status.rb +27 -20
  31. data/lib/topographer/importer/strategy/update_record.rb +28 -24
  32. data/lib/topographer/version.rb +1 -1
  33. data/spec/assets/test_files/a_csv.csv +3 -0
  34. data/spec/topographer/importer/helpers/write_log_to_csv_spec.rb +4 -4
  35. data/spec/topographer/importer/importer_spec.rb +21 -5
  36. data/spec/topographer/importer/input/delimited_spreadsheet_spec.rb +90 -0
  37. data/spec/topographer/importer/input/source_data_spec.rb +2 -2
  38. data/spec/topographer/importer/logger/base_spec.rb +19 -0
  39. data/spec/topographer/importer/logger/fatal_error_entry_spec.rb +2 -2
  40. data/spec/topographer/importer/logger/simple_spec.rb +3 -3
  41. data/spec/topographer/importer/mapper/default_field_mapping_spec.rb +2 -2
  42. data/spec/topographer/importer/mapper/field_mapping_spec.rb +21 -21
  43. data/spec/topographer/importer/mapper/mapper_builder_spec.rb +9 -9
  44. data/spec/topographer/importer/mapper/mapping_validator_spec.rb +10 -10
  45. data/spec/topographer/importer/mapper/validation_field_mapping_spec.rb +3 -3
  46. data/spec/topographer/importer/mapper_spec.rb +25 -25
  47. data/spec/topographer/importer/strategy/base_spec.rb +16 -5
  48. data/spec/topographer/importer/strategy/create_or_update_record_spec.rb +3 -3
  49. data/spec/topographer/importer/strategy/import_new_records_spec.rb +5 -5
  50. data/spec/topographer/importer/strategy/mapped_model.rb +9 -0
  51. data/spec/topographer/importer/strategy/update_record_spec.rb +3 -3
  52. data/topographer.gemspec +3 -2
  53. metadata +101 -102
@@ -1,22 +1,28 @@
1
- class Topographer::Importer::Mapper::DefaultFieldMapping < Topographer::Importer::Mapper::FieldMapping
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ class DefaultFieldMapping < Topographer::Importer::Mapper::FieldMapping
2
5
 
3
- def initialize(output_column, &output_block)
4
- unless block_given?
5
- raise Topographer::InvalidMappingError, 'Static fields must have an output block'
6
- end
7
- @output_field = output_column
8
- @output_block = output_block
9
- end
6
+ def initialize(output_column, &output_block)
7
+ unless block_given?
8
+ raise Topographer::InvalidMappingError, 'Static fields must have an output block'
9
+ end
10
+ @output_field = output_column
11
+ @output_block = output_block
12
+ end
10
13
 
11
- def process_input(_, result)
12
- @output_data = @output_block.()
13
- result.add_data(@output_field, @output_data)
14
- rescue => exception
15
- result.add_error(@output_field, exception.message)
16
- end
14
+ def process_input(_, result)
15
+ @output_data = @output_block.()
16
+ result.add_data(@output_field, @output_data)
17
+ rescue => exception
18
+ result.add_error(@output_field, exception.message)
19
+ end
17
20
 
18
- def required?
19
- true
20
- end
21
+ def required?
22
+ true
23
+ end
21
24
 
25
+ end
26
+ end
27
+ end
22
28
  end
@@ -1,55 +1,62 @@
1
1
  require 'active_support/core_ext/hash'
2
2
  require 'active_support/core_ext/object/blank'
3
- class Topographer::Importer::Mapper::FieldMapping
4
- attr_reader :input_columns, :output_field
5
-
6
- def initialize(required, input_columns, output_field, &mapping_behavior)
7
- @required = required
8
- @input_columns = Array(input_columns)
9
- @output_field = output_field
10
- @mapping_behavior = mapping_behavior
11
- @invalid_keys = []
12
- end
13
-
14
- def process_input(input, result)
15
- mapping_input = input.slice(*input_columns)
16
- @invalid_keys = get_invalid_keys(mapping_input)
17
- data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
18
- if !data.nil?
19
- result.add_data(output_field, data)
20
- elsif required?
21
- result.add_error(output_field, invalid_input_error)
22
- end
23
-
24
- rescue => exception
25
- result.add_error(output_field, exception.message)
26
-
27
- end
28
-
29
- def required?
30
- @required
31
- end
32
-
33
- private
34
3
 
35
- def apply_mapping(mapping_input)
36
- if @mapping_behavior
37
- @mapping_behavior.(mapping_input)
38
- else
39
- (mapping_input.size > 1) ? mapping_input.values.join(', ') : mapping_input.values.first
4
+ module Topographer
5
+ class Importer
6
+ class Mapper
7
+ class FieldMapping
8
+ attr_reader :input_columns, :output_field
9
+
10
+ def initialize(required, input_columns, output_field, &mapping_behavior)
11
+ @required = required
12
+ @input_columns = Array(input_columns)
13
+ @output_field = output_field
14
+ @mapping_behavior = mapping_behavior
15
+ @invalid_keys = []
16
+ end
17
+
18
+ def process_input(input, result)
19
+ mapping_input = input.slice(*input_columns)
20
+ @invalid_keys = get_invalid_keys(mapping_input)
21
+ data = (@invalid_keys.any?) ? nil : apply_mapping(mapping_input)
22
+ if !data.nil?
23
+ result.add_data(output_field, data)
24
+ elsif required?
25
+ result.add_error(output_field, invalid_input_error)
26
+ end
27
+
28
+ rescue => exception
29
+ result.add_error(output_field, exception.message)
30
+
31
+ end
32
+
33
+ def required?
34
+ @required
35
+ end
36
+
37
+ private
38
+
39
+ def apply_mapping(mapping_input)
40
+ if @mapping_behavior
41
+ @mapping_behavior.(mapping_input)
42
+ else
43
+ (mapping_input.size > 1) ? mapping_input.values.join(', ') : mapping_input.values.first
44
+ end
45
+ end
46
+
47
+ def invalid_input_error
48
+ "Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
49
+ end
50
+
51
+ def get_invalid_keys(input)
52
+ missing_columns = @input_columns - input.keys
53
+ #reject input that is not blank or the value `false`
54
+ #this allows boolean inputs for required fields
55
+ missing_data = @required ? input.reject { |k, v| !v.blank? || v == false }.keys : []
56
+ missing_columns + missing_data
57
+ end
58
+
59
+ end
40
60
  end
41
61
  end
42
-
43
- def invalid_input_error
44
- "Missing required input(s): `#{@invalid_keys.join(", ")}` for `#{@output_field}`"
45
- end
46
-
47
- def get_invalid_keys(input)
48
- missing_columns = @input_columns - input.keys
49
- #reject input that is not blank or the value `false`
50
- #this allows boolean inputs for required fields
51
- missing_data = @required ? input.reject{|k,v| !v.blank? || v == false }.keys : []
52
- missing_columns + missing_data
53
- end
54
-
55
62
  end
@@ -1,10 +1,17 @@
1
- class Topographer::Importer::Mapper::IgnoredFieldMapping < Topographer::Importer::Mapper::FieldMapping
2
- def initialize(input_columns)
3
- @input_columns = input_columns
4
- @output_field = nil
5
- end
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ class IgnoredFieldMapping < Topographer::Importer::Mapper::FieldMapping
5
+ def initialize(input_columns)
6
+ @input_columns = input_columns
7
+ @output_field = nil
8
+ end
6
9
 
7
- def required?
8
- false
10
+ def required?
11
+ false
12
+ end
13
+ end
14
+ end
9
15
  end
10
16
  end
17
+
@@ -1,46 +1,59 @@
1
- class Topographer::Importer::Mapper::MapperBuilder
2
- include Topographer::Importer::Mapper::MappingColumns
3
- include Topographer::Importer::Mapper::MappingValidator
4
-
5
- attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
6
- :validation_mappings, :default_values, :key_fields
7
-
8
- def initialize
9
- @required_mappings = {}
10
- @optional_mappings = {}
11
- @ignored_mappings = {}
12
- @validation_mappings = {}
13
- @default_values = {}
14
- @key_fields = []
15
- end
16
-
17
- def required_mapping(input_columns, output_field, &mapping_behavior)
18
- validate_unique_mapping(input_columns, output_field)
19
- @required_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
20
- end
21
-
22
- def optional_mapping(input_columns, output_field, &mapping_behavior)
23
- validate_unique_mapping(input_columns, output_field)
24
- @optional_mappings[output_field] = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
25
- end
26
-
27
- def validation_field(name, input_columns, &mapping_behavior)
28
- validate_unique_validation_name(name)
29
- @validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
30
- end
31
-
32
- def default_value(output_field, &mapping_behavior)
33
- validate_unique_mapping([], output_field)
34
- @default_values[output_field] = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
35
- end
36
-
37
- def key_field(output_field)
38
- validate_key_field(output_field)
39
- @key_fields << output_field
40
- end
41
-
42
- def ignored_column(input_column)
43
- validate_unique_column_mapping_type(input_column, ignored: true)
44
- @ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ class MapperBuilder
5
+ include Topographer::Importer::Mapper::MappingColumns
6
+ include Topographer::Importer::Mapper::MappingValidator
7
+
8
+ attr_reader :required_mappings, :optional_mappings, :ignored_mappings,
9
+ :validation_mappings, :default_values, :key_fields, :field_mappings
10
+
11
+ def initialize
12
+ @required_mappings = {}
13
+ @optional_mappings = {}
14
+ @ignored_mappings = {}
15
+ @validation_mappings = {}
16
+ @default_values = {}
17
+ @key_fields = []
18
+ @field_mappings = {}
19
+ end
20
+
21
+ def required_mapping(input_columns, output_field, &mapping_behavior)
22
+ validate_unique_mapping(input_columns, output_field)
23
+ mapping = Topographer::Importer::Mapper::FieldMapping.new(true, input_columns, output_field, &mapping_behavior)
24
+ @required_mappings[output_field] = mapping
25
+ @field_mappings[output_field] = mapping
26
+ end
27
+
28
+ def optional_mapping(input_columns, output_field, &mapping_behavior)
29
+ validate_unique_mapping(input_columns, output_field)
30
+ mapping = Topographer::Importer::Mapper::FieldMapping.new(false, input_columns, output_field, &mapping_behavior)
31
+ @optional_mappings[output_field] = mapping
32
+ @field_mappings[output_field] = mapping
33
+ end
34
+
35
+ def validation_field(name, input_columns, &mapping_behavior)
36
+ validate_unique_validation_name(name)
37
+ @validation_mappings[name] = Topographer::Importer::Mapper::ValidationFieldMapping.new(name, input_columns, &mapping_behavior)
38
+ end
39
+
40
+ def default_value(output_field, &mapping_behavior)
41
+ validate_unique_mapping([], output_field)
42
+ mapping = Topographer::Importer::Mapper::DefaultFieldMapping.new(output_field, &mapping_behavior)
43
+ @default_values[output_field] = mapping
44
+ @field_mappings[output_field] = mapping
45
+ end
46
+
47
+ def key_field(output_field)
48
+ validate_key_field(output_field)
49
+ @key_fields << output_field
50
+ end
51
+
52
+ def ignored_column(input_column)
53
+ validate_unique_column_mapping_type(input_column, ignored: true)
54
+ @ignored_mappings[input_column] = Topographer::Importer::Mapper::IgnoredFieldMapping.new(input_column)
55
+ end
56
+ end
57
+ end
45
58
  end
46
59
  end
@@ -1,46 +1,53 @@
1
- module Topographer::Importer::Mapper::MappingColumns
2
- def output_fields
3
- (required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
4
- end
5
-
6
- def required_input_columns
7
- (required_mapping_columns + validation_mapping_columns).uniq
8
- end
9
-
10
- def input_columns
11
- (required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
12
- end
13
-
14
- def default_fields
15
- default_values.keys
16
- end
17
-
18
- def validation_mapping_columns
19
- validation_mappings.values.flat_map(&:input_columns)
20
- end
21
-
22
- def ignored_mapping_columns
23
- ignored_mappings.values.flat_map(&:input_columns)
24
- end
25
-
26
- def optional_mapping_columns
27
- optional_mappings.values.flat_map(&:input_columns)
28
- end
29
-
30
- def required_mapping_columns
31
- required_mappings.values.flat_map(&:input_columns)
32
- end
33
-
34
- private
35
- def mapped_input_columns
36
- required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
37
- end
38
-
39
- def mappings
40
- required_mappings.merge(optional_mappings).merge(ignored_mappings).merge(default_values)
41
- end
42
-
43
- def non_ignored_columns
44
- required_mappings.merge(optional_mappings)
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ module MappingColumns
5
+ def output_fields
6
+ (required_mappings.merge(optional_mappings).merge(default_values)).values.map(&:output_field)
7
+ end
8
+
9
+ def required_input_columns
10
+ (required_mapping_columns + validation_mapping_columns).uniq
11
+ end
12
+
13
+ def input_columns
14
+ (required_mapping_columns + optional_mapping_columns + validation_mapping_columns).uniq
15
+ end
16
+
17
+ def default_fields
18
+ default_values.keys
19
+ end
20
+
21
+ def validation_mapping_columns
22
+ validation_mappings.values.flat_map(&:input_columns)
23
+ end
24
+
25
+ def ignored_mapping_columns
26
+ ignored_mappings.values.flat_map(&:input_columns)
27
+ end
28
+
29
+ def optional_mapping_columns
30
+ optional_mappings.values.flat_map(&:input_columns)
31
+ end
32
+
33
+ def required_mapping_columns
34
+ required_mappings.values.flat_map(&:input_columns)
35
+ end
36
+
37
+ private
38
+ def mapped_input_columns
39
+ required_mapping_columns + optional_mapping_columns + ignored_mapping_columns + validation_mapping_columns
40
+ end
41
+
42
+ def mappings
43
+ @field_mappings
44
+ end
45
+
46
+ def non_ignored_columns
47
+ required_mappings.merge(optional_mappings)
48
+ end
49
+ end
45
50
  end
51
+ end
46
52
  end
53
+
@@ -1,40 +1,47 @@
1
- module Topographer::Importer::Mapper::MappingValidator
1
+ module Topographer
2
+ class Importer
3
+ class Mapper
4
+ module MappingValidator
2
5
 
3
- def validate_unique_validation_name(name)
4
- raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
5
- end
6
+ def validate_unique_validation_name(name)
7
+ raise Topographer::InvalidMappingError, "A validation already exists with the name `#{name}`" if validation_mappings.has_key?(name)
8
+ end
6
9
 
7
- def validate_unique_output_mapping(output_field)
8
- if output_fields.include?(output_field)
9
- raise Topographer::InvalidMappingError, 'Output column already mapped.'
10
- end
11
- end
10
+ def validate_unique_output_mapping(output_field)
11
+ if output_fields.include?(output_field)
12
+ raise Topographer::InvalidMappingError, 'Output column already mapped.'
13
+ end
14
+ end
12
15
 
13
- def validate_unique_column_mapping_type(mapping_input_columns, options = {})
14
- ignored = options.fetch(:ignored, false)
15
- mapping_input_columns = Array(mapping_input_columns)
16
- mapping_input_columns.each do |col|
17
- if ignored && ((input_columns + ignored_mapping_columns).include?(col))
18
- raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
19
- elsif (ignored_mapping_columns.include?(col))
20
- raise Topographer::InvalidMappingError, 'Input column already ignored.'
21
- end
22
- end
23
- end
16
+ def validate_unique_column_mapping_type(mapping_input_columns, options = {})
17
+ ignored = options.fetch(:ignored, false)
18
+ mapping_input_columns = Array(mapping_input_columns)
19
+ mapping_input_columns.each do |col|
20
+ if ignored && ((input_columns + ignored_mapping_columns).include?(col))
21
+ raise Topographer::InvalidMappingError, 'Input column already mapped to an output column.'
22
+ elsif (ignored_mapping_columns.include?(col))
23
+ raise Topographer::InvalidMappingError, 'Input column already ignored.'
24
+ end
25
+ end
26
+ end
24
27
 
25
- def validate_unique_mapping(mapping_input_columns, output_field)
26
- if (output_field.is_a?(Array))
27
- raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
28
- end
29
- validate_unique_column_mapping_type(mapping_input_columns)
30
- validate_unique_output_mapping(output_field)
31
- end
28
+ def validate_unique_mapping(mapping_input_columns, output_field)
29
+ if (output_field.is_a?(Array))
30
+ raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
31
+ end
32
+ validate_unique_column_mapping_type(mapping_input_columns)
33
+ validate_unique_output_mapping(output_field)
34
+ end
32
35
 
33
- def validate_key_field(field)
34
- if field.is_a?(Array)
35
- raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
36
- elsif key_fields.include?(field)
37
- raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
36
+ def validate_key_field(field)
37
+ if field.is_a?(Array)
38
+ raise Topographer::InvalidMappingError, 'One to many mapping is not supported'
39
+ elsif key_fields.include?(field)
40
+ raise Topographer::InvalidMappingError, "Field `#{field}` has already been included as a key"
41
+ end
42
+ end
43
+ end
38
44
  end
39
45
  end
40
46
  end
47
+