csv_row_model 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.rspec +2 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +10 -0
- data/.yardopts +10 -0
- data/CONTRIBUTING.md +7 -0
- data/Gemfile +20 -0
- data/LICENSE.txt +21 -0
- data/README.md +369 -0
- data/Rakefile +1 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/csv_row_model.gemspec +23 -0
- data/lib/csv_row_model.rb +46 -0
- data/lib/csv_row_model/concerns/deep_class_var.rb +86 -0
- data/lib/csv_row_model/concerns/inspect.rb +10 -0
- data/lib/csv_row_model/engine.rb +5 -0
- data/lib/csv_row_model/export.rb +70 -0
- data/lib/csv_row_model/export/csv.rb +43 -0
- data/lib/csv_row_model/export/single_model.rb +47 -0
- data/lib/csv_row_model/import.rb +125 -0
- data/lib/csv_row_model/import/attributes.rb +105 -0
- data/lib/csv_row_model/import/csv.rb +136 -0
- data/lib/csv_row_model/import/csv/row.rb +17 -0
- data/lib/csv_row_model/import/file.rb +133 -0
- data/lib/csv_row_model/import/file/callbacks.rb +16 -0
- data/lib/csv_row_model/import/file/validations.rb +39 -0
- data/lib/csv_row_model/import/presenter.rb +160 -0
- data/lib/csv_row_model/import/single_model.rb +41 -0
- data/lib/csv_row_model/model.rb +68 -0
- data/lib/csv_row_model/model/children.rb +79 -0
- data/lib/csv_row_model/model/columns.rb +73 -0
- data/lib/csv_row_model/model/csv_string_model.rb +16 -0
- data/lib/csv_row_model/model/single_model.rb +16 -0
- data/lib/csv_row_model/validators/boolean_format.rb +11 -0
- data/lib/csv_row_model/validators/date_format.rb +9 -0
- data/lib/csv_row_model/validators/default_change.rb +6 -0
- data/lib/csv_row_model/validators/float_format.rb +9 -0
- data/lib/csv_row_model/validators/integer_format.rb +9 -0
- data/lib/csv_row_model/validators/number_validator.rb +16 -0
- data/lib/csv_row_model/validators/validate_attributes.rb +27 -0
- data/lib/csv_row_model/version.rb +3 -0
- metadata +116 -0
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'csv_row_model/validators/boolean_format'
|
2
|
+
|
3
|
+
module CsvRowModel
|
4
|
+
module Import
|
5
|
+
module Attributes
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
# Classes with a validations associated with them in csv_row_model/validators
|
8
|
+
PARSE_VALIDATION_CLASSES = [Boolean, Integer, Float, Date].freeze
|
9
|
+
|
10
|
+
# Mapping of column type classes to a parsing lambda. These are applied after {Import.format_cell}.
|
11
|
+
# Can pass custom Proc with :parse option.
|
12
|
+
CLASS_TO_PARSE_LAMBDA = {
|
13
|
+
nil => ->(s) { s },
|
14
|
+
Boolean => ->(s) { s =~ BooleanFormatValidator::FALSE_BOOLEAN_REGEX ? false : true },
|
15
|
+
String => ->(s) { s },
|
16
|
+
Integer => ->(s) { s.to_i },
|
17
|
+
Float => ->(s) { s.to_f },
|
18
|
+
Date => ->(s) { s.present? ? Date.parse(s) : s }
|
19
|
+
}.freeze
|
20
|
+
|
21
|
+
# @return [Hash] a map of `column_name => original_attribute(column_name)`
|
22
|
+
def original_attributes
|
23
|
+
@original_attributes ||= begin
|
24
|
+
values = self.class.column_names.map { |column_name| original_attribute(column_name) }
|
25
|
+
self.class.column_names.zip(values).to_h
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# @return [Object] the column's attribute before override
|
30
|
+
def original_attribute(column_name)
|
31
|
+
@default_changes ||= {}
|
32
|
+
|
33
|
+
csv_string_model.valid?
|
34
|
+
return nil unless csv_string_model.errors[column_name].blank?
|
35
|
+
|
36
|
+
value = self.class.format_cell(mapped_row[column_name], column_name, self.class.index(column_name))
|
37
|
+
if value.present?
|
38
|
+
instance_exec(value, &self.class.parse_lambda(column_name))
|
39
|
+
elsif self.class.options(column_name)[:default]
|
40
|
+
original_value = value
|
41
|
+
value = instance_exec(value, &self.class.default_lambda(column_name))
|
42
|
+
@default_changes[column_name] = [original_value, value]
|
43
|
+
value
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# return [Hash] a map changes from {.column}'s default option': `column_name -> [value_before_default, default_set]`
|
48
|
+
def default_changes
|
49
|
+
original_attributes
|
50
|
+
@default_changes
|
51
|
+
end
|
52
|
+
|
53
|
+
class_methods do
|
54
|
+
# Safe to override. Method applied to each cell by default
|
55
|
+
#
|
56
|
+
# @param cell [String] the cell's string
|
57
|
+
# @param column_name [Symbol] the cell's column_name
|
58
|
+
# @param column_index [Integer] the column_name's index
|
59
|
+
def format_cell(cell, column_name, column_index)
|
60
|
+
cell
|
61
|
+
end
|
62
|
+
|
63
|
+
# @return [Lambda] returns a Lambda: ->(original_value) { default_exists? ? default : original_value }
|
64
|
+
def default_lambda(column_name)
|
65
|
+
default = options(column_name)[:default]
|
66
|
+
default.is_a?(Proc) ? ->(s) { instance_exec(&default) } : ->(s) { default.nil? ? s : default }
|
67
|
+
end
|
68
|
+
|
69
|
+
# @return [Lambda, Proc] returns the Lambda/Proc given in the parse option or:
|
70
|
+
# ->(original_value) { parse_proc_exists? ? parsed_value : original_value }
|
71
|
+
def parse_lambda(column_name)
|
72
|
+
options = options(column_name)
|
73
|
+
|
74
|
+
raise ArgumentError.new("You need either :parse OR :type but not both of them") if options[:parse] && options[:type]
|
75
|
+
|
76
|
+
parse_lambda = options[:parse] || CLASS_TO_PARSE_LAMBDA[options[:type]]
|
77
|
+
return parse_lambda if parse_lambda
|
78
|
+
raise ArgumentError.new("type must be #{CLASS_TO_PARSE_LAMBDA.keys.reject(:nil?).join(", ")}")
|
79
|
+
end
|
80
|
+
|
81
|
+
protected
|
82
|
+
# Define default attribute method for a column
|
83
|
+
# @param column_name [Symbol] the cell's column_name
|
84
|
+
def define_attribute_method(column_name)
|
85
|
+
add_type_validation(column_name)
|
86
|
+
define_method(column_name) { original_attribute(column_name) }
|
87
|
+
end
|
88
|
+
|
89
|
+
# Adds the type validation based on :validate_type option
|
90
|
+
def add_type_validation(column_name)
|
91
|
+
options = options(column_name)
|
92
|
+
validate_type = options[:validate_type]
|
93
|
+
|
94
|
+
return unless validate_type
|
95
|
+
|
96
|
+
type = options[:type]
|
97
|
+
raise ArgumentError.new("invalid :type given for :validate_type for column") unless PARSE_VALIDATION_CLASSES.include? type
|
98
|
+
validate_type = Proc.new { validates column_name, "#{type.name.underscore}_format".to_sym => true, allow_blank: true }
|
99
|
+
|
100
|
+
csv_string_model(&validate_type)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
module CsvRowModel
|
2
|
+
module Import
|
3
|
+
# Abstraction of Ruby's CSV library. Keeps current row and index, skips empty rows, handles errors.
|
4
|
+
class Csv
|
5
|
+
# @return [String] the file path of the CSV
|
6
|
+
attr_reader :file_path
|
7
|
+
# @return [Integer, nil] return `-1` at start of file, `0 to infinity` is index of row_model, `nil` is end of file (row is also `nil`)
|
8
|
+
attr_reader :index
|
9
|
+
# @return [Array, nil] the current row, or nil at the beginning or end of file
|
10
|
+
attr_reader :current_row
|
11
|
+
# @return [Hash{Integer => Symbol}] hash of skipped rows from last change in position, `index => :reason`
|
12
|
+
attr_reader :skipped_rows
|
13
|
+
|
14
|
+
include ActiveModel::Validations
|
15
|
+
|
16
|
+
validate do
|
17
|
+
begin; _ruby_csv
|
18
|
+
rescue => e; errors.add(:ruby_csv, e.message) end
|
19
|
+
end
|
20
|
+
|
21
|
+
def initialize(file_path)
|
22
|
+
@file_path = file_path
|
23
|
+
reset
|
24
|
+
end
|
25
|
+
|
26
|
+
# http://stackoverflow.com/questions/2650517/count-the-number-of-lines-in-a-file-without-reading-entire-file-into-memory
|
27
|
+
# @return [Integer] the number of rows in the file, including empty new lines
|
28
|
+
def size
|
29
|
+
@size ||= `wc -l #{file_path}`.split[0].to_i + 1
|
30
|
+
end
|
31
|
+
|
32
|
+
# If the current position is at the header, skip it and return it. Otherwise, only return false.
|
33
|
+
# @return [Boolean, Array] returns false, if header is already skipped, otherwise returns the header
|
34
|
+
def skip_header
|
35
|
+
start_of_file? ? (@header = read_row) : false
|
36
|
+
end
|
37
|
+
|
38
|
+
# Returns the header __without__ changing the position of the CSV
|
39
|
+
# @return [Array, nil] the header
|
40
|
+
def header
|
41
|
+
return unless valid?
|
42
|
+
return @header if @header
|
43
|
+
|
44
|
+
ruby_csv = _ruby_csv
|
45
|
+
@header = _read_row({}, 0, ruby_csv)
|
46
|
+
ruby_csv.close
|
47
|
+
@header
|
48
|
+
end
|
49
|
+
|
50
|
+
# Resets the file to the start of file
|
51
|
+
def reset
|
52
|
+
return unless valid?
|
53
|
+
|
54
|
+
@index = -1
|
55
|
+
@current_row = nil
|
56
|
+
@ruby_csv = _ruby_csv
|
57
|
+
true
|
58
|
+
end
|
59
|
+
|
60
|
+
# @return [Boolean] true, if the current position is at the start of the file
|
61
|
+
def start_of_file?
|
62
|
+
index == -1
|
63
|
+
end
|
64
|
+
|
65
|
+
# @return [Boolean] true, if the current position is at the end of the file
|
66
|
+
def end_of_file?
|
67
|
+
index.nil?
|
68
|
+
end
|
69
|
+
|
70
|
+
# Returns the next row __without__ changing the position of the CSV
|
71
|
+
# @return [Array, nil] the next row, or `nil` at the end of file
|
72
|
+
def next_row
|
73
|
+
@next_skipped_rows = {}
|
74
|
+
@next_row ||= _read_row(@next_skipped_rows)
|
75
|
+
end
|
76
|
+
|
77
|
+
# Returns the next row, while changing the position of the CSV
|
78
|
+
# @return [Array, nil] the changed current row, or `nil` at the end of file
|
79
|
+
def read_row
|
80
|
+
if @next_row
|
81
|
+
@current_row, @skipped_rows = @next_row, @next_skipped_rows
|
82
|
+
@next_row = nil
|
83
|
+
increment_index(@current_row)
|
84
|
+
else
|
85
|
+
@skipped_rows = {}
|
86
|
+
@current_row = _read_row(@skipped_rows) do |row|
|
87
|
+
increment_index(row)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
current_row
|
91
|
+
end
|
92
|
+
|
93
|
+
protected
|
94
|
+
def set_end_of_file
|
95
|
+
@current_row = @index = nil
|
96
|
+
end
|
97
|
+
|
98
|
+
def _ruby_csv
|
99
|
+
CSV.open(file_path)
|
100
|
+
end
|
101
|
+
|
102
|
+
def _read_row(skipped_rows={}, index=@index, ruby_csv=@ruby_csv)
|
103
|
+
return unless valid?
|
104
|
+
|
105
|
+
loop do
|
106
|
+
row = ruby_csv.readline
|
107
|
+
|
108
|
+
raise "empty?" if row.try(:empty?)
|
109
|
+
|
110
|
+
index += 1 if index
|
111
|
+
|
112
|
+
yield row if block_given?
|
113
|
+
return row
|
114
|
+
end
|
115
|
+
rescue Exception => e
|
116
|
+
index += 1 if index
|
117
|
+
yield [] if block_given?
|
118
|
+
|
119
|
+
# thanks to the ruby CSV library, quotes can still escape to the next line
|
120
|
+
reason = case e.message
|
121
|
+
when "empty?"; :empty
|
122
|
+
when /Illegal quoting/i; :illegal_quote
|
123
|
+
when /Unclosed quoted/i; :unclosed_quote
|
124
|
+
end
|
125
|
+
|
126
|
+
skipped_rows.merge!(index => reason)
|
127
|
+
|
128
|
+
retry
|
129
|
+
end
|
130
|
+
|
131
|
+
def increment_index(current_row)
|
132
|
+
current_row.nil? ? set_end_of_file : @index += 1
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module CsvRowModel
|
2
|
+
module Import
|
3
|
+
class Csv
|
4
|
+
class Row
|
5
|
+
attr_reader :row, :skipped_rows
|
6
|
+
|
7
|
+
def initialize(row, index, skipped_rows)
|
8
|
+
@row, @index, @skipped_rows = row, index, skipped_rows
|
9
|
+
end
|
10
|
+
|
11
|
+
def empty?
|
12
|
+
!!row.try(:empty?)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
require 'csv_row_model/import/file/callbacks'
|
2
|
+
require 'csv_row_model/import/file/validations'
|
3
|
+
|
4
|
+
module CsvRowModel
|
5
|
+
module Import
|
6
|
+
# Represents a csv file and handles parsing to return `Import` or `Mapper`
|
7
|
+
class File
|
8
|
+
include Callbacks
|
9
|
+
include Validations
|
10
|
+
|
11
|
+
# @return [Csv]
|
12
|
+
attr_reader :csv
|
13
|
+
# @return [Input,Mapper] model class returned for importing
|
14
|
+
attr_reader :row_model_class
|
15
|
+
|
16
|
+
# Current index of the row model
|
17
|
+
# @return [Integer] returns -1 = start of file, 0 to infinity = index of row_model, nil = end of file, no row_model
|
18
|
+
attr_reader :index
|
19
|
+
# @return [Input, Mapper] the current row model set by {#next}
|
20
|
+
attr_reader :current_row_model
|
21
|
+
# @return [Input, Mapper] the previous row model set by {#next}
|
22
|
+
attr_reader :previous_row_model
|
23
|
+
|
24
|
+
delegate :header, :size, :skipped_rows, to: :csv
|
25
|
+
|
26
|
+
# @param [String] file_path path of csv file
|
27
|
+
# @param [Import, Mapper] row_model_class model class returned for importing
|
28
|
+
def initialize(file_path, row_model_class)
|
29
|
+
@csv, @row_model_class = Csv.new(file_path), row_model_class
|
30
|
+
reset
|
31
|
+
end
|
32
|
+
|
33
|
+
# Resets the file back to the top
|
34
|
+
def reset
|
35
|
+
csv.reset
|
36
|
+
@index = -1
|
37
|
+
@current_row_model = nil
|
38
|
+
end
|
39
|
+
|
40
|
+
# Gets the next row model based on the context
|
41
|
+
#
|
42
|
+
# @param context [Hash] context passed to the {Import}
|
43
|
+
def next(context={})
|
44
|
+
if is_single_model?
|
45
|
+
return set_end_of_file if end_of_file?
|
46
|
+
set_single_model(context)
|
47
|
+
else
|
48
|
+
next_collection_model(context)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
# @return [Boolean] returns true, if the object is at the end of file
|
53
|
+
def end_of_file?
|
54
|
+
csv.end_of_file?
|
55
|
+
end
|
56
|
+
|
57
|
+
# Iterates through the entire csv file and provides the `current_row_model` in a block, while handing aborts and skips
|
58
|
+
# via. calling {Model#abort?} and {Model#skip?}
|
59
|
+
#
|
60
|
+
# @param context [Hash] context passed to the {Import}
|
61
|
+
def each(context={})
|
62
|
+
return to_enum(__callee__, context) unless block_given?
|
63
|
+
return false if _abort?
|
64
|
+
|
65
|
+
while self.next(context)
|
66
|
+
return false if _abort?
|
67
|
+
next if _skip?
|
68
|
+
|
69
|
+
run_callbacks :yield do
|
70
|
+
yield current_row_model
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
protected
|
76
|
+
|
77
|
+
# @return [boolean] if type of model is collection_model
|
78
|
+
def is_single_model?
|
79
|
+
@is_single_model ||= begin
|
80
|
+
row_model_class.respond_to?(:type) ? (row_model_class.type == :single_model) : false
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def set_current_collection_model(context)
|
85
|
+
@current_row_model = row_model_class.new(csv.current_row, index: csv.index, context: context, source_header: header, previous: previous_row_model)
|
86
|
+
@index += 1
|
87
|
+
end
|
88
|
+
|
89
|
+
def set_single_model(context={})
|
90
|
+
source_row = Array.new(row_model_class.header_matchers.size)
|
91
|
+
while !end_of_file?
|
92
|
+
csv.read_row
|
93
|
+
update_source_row(source_row)
|
94
|
+
end
|
95
|
+
@current_row_model = row_model_class.new(source_row, context: context)
|
96
|
+
end
|
97
|
+
|
98
|
+
def update_source_row(source_row)
|
99
|
+
current_row = csv.current_row
|
100
|
+
return unless current_row
|
101
|
+
current_row.each_with_index do |cell, position|
|
102
|
+
next if cell.blank?
|
103
|
+
index = row_model_class.index_header_match(cell)
|
104
|
+
next unless index
|
105
|
+
source_row[index] = current_row[position + 1]
|
106
|
+
break
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def next_collection_model(context)
|
111
|
+
csv.skip_header
|
112
|
+
|
113
|
+
next_row_is_parent = true
|
114
|
+
loop do
|
115
|
+
@previous_row_model = current_row_model if next_row_is_parent
|
116
|
+
|
117
|
+
csv.read_row
|
118
|
+
return set_end_of_file if csv.end_of_file?
|
119
|
+
|
120
|
+
set_current_collection_model(context) if next_row_is_parent
|
121
|
+
|
122
|
+
next_row_is_parent = !current_row_model.append_child(csv.next_row)
|
123
|
+
return current_row_model if next_row_is_parent
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def set_end_of_file
|
128
|
+
# please return nil
|
129
|
+
@current_row_model = @index = nil
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module CsvRowModel
|
2
|
+
module Import
|
3
|
+
class File
|
4
|
+
module Callbacks
|
5
|
+
extend ActiveSupport::Concern
|
6
|
+
|
7
|
+
included do
|
8
|
+
extend ActiveModel::Callbacks
|
9
|
+
|
10
|
+
define_model_callbacks :yield
|
11
|
+
define_model_callbacks :abort, :skip, only: :before
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module CsvRowModel
|
2
|
+
module Import
|
3
|
+
class File
|
4
|
+
module Validations
|
5
|
+
extend ActiveSupport::Concern
|
6
|
+
|
7
|
+
included do
|
8
|
+
include ActiveModel::Validations
|
9
|
+
include Validators::ValidateAttributes
|
10
|
+
|
11
|
+
validate_attributes :csv
|
12
|
+
end
|
13
|
+
|
14
|
+
# @return [Boolean] returns true, if the file should abort reading
|
15
|
+
def abort?
|
16
|
+
!valid? || !!current_row_model.try(:abort?)
|
17
|
+
end
|
18
|
+
|
19
|
+
# @return [Boolean] returns true, if the file should skip `current_row_model`
|
20
|
+
def skip?
|
21
|
+
!!current_row_model.try(:skip?)
|
22
|
+
end
|
23
|
+
|
24
|
+
protected
|
25
|
+
def _abort?
|
26
|
+
abort = abort?
|
27
|
+
run_callbacks(:abort) if abort
|
28
|
+
abort
|
29
|
+
end
|
30
|
+
|
31
|
+
def _skip?
|
32
|
+
skip = skip?
|
33
|
+
run_callbacks(:skip) if skip
|
34
|
+
skip
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|