rseed 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +18 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.rdoc +100 -0
- data/Rakefile +1 -0
- data/lib/generators/rseed/converter.rb +27 -0
- data/lib/generators/rseed/templates/converter.rb.erb +39 -0
- data/lib/generators/rseed/templates/data.csv.erb +1 -0
- data/lib/rseed/adapter.rb +37 -0
- data/lib/rseed/attribute.rb +44 -0
- data/lib/rseed/attribute_converters.rb +33 -0
- data/lib/rseed/converter.rb +74 -0
- data/lib/rseed/csv_adapter.rb +80 -0
- data/lib/rseed/hash_adapter.rb +23 -0
- data/lib/rseed/processor.rb +69 -0
- data/lib/rseed/utilities.rb +72 -0
- data/lib/rseed/version.rb +3 -0
- data/lib/rseed.rb +30 -0
- data/lib/tasks/rseed.rake +8 -0
- data/rseed.gemspec +26 -0
- metadata +120 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ODgyMmI1OTUxYmZlMTBkOTY4M2I5ZDRmOWQ1ODA0MDNjM2M1MTFkZg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
YzRlMzE4YzM3ZjI4MzFjMmFmZjcwYjYxODAwMmQyMTUwNGNiY2FhMA==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NDdhYWUyMTU2MzAyN2M0MjliYzliZWI1NjY5YmI1N2ZjNzdiMjcwMzU3Njc5
|
10
|
+
MjA1OTI4ZDFkZjg4MzZhN2UzZDNmYjIzYThkYzU2ZDQwNjhlZGEzNmVlMDk4
|
11
|
+
Yzg2ZGJhMmQxNDg4Mzg0Y2NhZTRjYTQ0NzI1ODQxNzllNjkwNDY=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NmZmNzFhYzRiMmEyYzYwNmEyMTVhYmI4OTk1MGRhNjNkMjBiYjU2ZWFlMGQz
|
14
|
+
NTA1NDNiMzU1YWM1MzIyMjUyYjgwYzBjNjQyYzBlOGEzMTI2NjZlZjVmMzEy
|
15
|
+
MzdkZTI5YjFhYWI4YmY4N2NhZjRmMTQxZmE2YzU5MDlmZTkxMTE=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 David Monagle
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
= Rseed
|
2
|
+
|
3
|
+
Rseed is a replacement for rseed. There are lots of improvements in order to make it easy to create and
|
4
|
+
maintain converters.
|
5
|
+
|
6
|
+
== Installation
|
7
|
+
|
8
|
+
Simple add the following to your Gemfile
|
9
|
+
|
10
|
+
gem 'rseed'
|
11
|
+
|
12
|
+
Then run:
|
13
|
+
|
14
|
+
bundle install
|
15
|
+
|
16
|
+
== Quick Example
|
17
|
+
|
18
|
+
rails g rseed:converter User
|
19
|
+
|
20
|
+
This will create an model converter in the directory app/rseed. You can read through this import file to see how the import works.
|
21
|
+
|
22
|
+
This also creates a default data file in db/rseed. This will be the CSV used for this converter.
|
23
|
+
|
24
|
+
== The Converter File
|
25
|
+
|
26
|
+
=== Options
|
27
|
+
|
28
|
+
:header
|
29
|
+
Defines the name of the attribute to be used for serialization. If there is no :match defined, it will also be used
|
30
|
+
to match the attribute name of the input to the attribute being defined.
|
31
|
+
|
32
|
+
:match
|
33
|
+
A regex string that is used to match the attribute name of the input to the attribute being defined. If this is not
|
34
|
+
defined, a match will be checked against :header and then the attribute name.
|
35
|
+
|
36
|
+
:type
|
37
|
+
Defines a type for the string.
|
38
|
+
|
39
|
+
:model
|
40
|
+
This can be set to the name of a model that this attribute should resolve to. The model is classified so using a symbol
|
41
|
+
works here. Alternately, if this is set to *true*, then the name of the attribute will be used as the model name. In
|
42
|
+
order for this to work, :model_attribute must also be set.
|
43
|
+
|
44
|
+
:model_attribute
|
45
|
+
Specify which attribute on the model is used for lookup.
|
46
|
+
|
47
|
+
:model_match
|
48
|
+
Specifies how the model should be resolved. The value here is called against the *where* that is used to look up the model.
|
49
|
+
For example, this defaults to *:first*. If your model is *Person* and the :model_attribute is *:name* then this is what
|
50
|
+
is called to set the attribute value:
|
51
|
+
|
52
|
+
Person.where(name: <value>).first
|
53
|
+
|
54
|
+
You may use any active record method in this case, such as :first_or_create, or :last.
|
55
|
+
|
56
|
+
:optional
|
57
|
+
Defines the attribute as optionsal. This has no effect in the *HashAdapter*.
|
58
|
+
|
59
|
+
== Rake Tasks
|
60
|
+
|
61
|
+
These rake tasks allow you to run seeds manually:
|
62
|
+
|
63
|
+
rake rseed:csv Load csv file into a model using a model converter
|
64
|
+
rake rseed:seed Seed a list of import files
|
65
|
+
|
66
|
+
=== Examples
|
67
|
+
|
68
|
+
rake rseed:csv FILE=user.csv CONVERTER=User CONVERTER_OPTIONS="give_admin_access=true"
|
69
|
+
|
70
|
+
In this case the file in db/rseed/user.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case @options["give_admin_access"] will evaluate to true.
|
71
|
+
|
72
|
+
The FILE parameter is not strictly necessary in this case either as the default file name will be an underscored value of the name of the converter.
|
73
|
+
|
74
|
+
== Seeding
|
75
|
+
|
76
|
+
Seeding allows you to import several files through different model converters in a single command. It involves the creation of a .seed file. Each file goes on a single line and the options are separated by pipe symbols.
|
77
|
+
|
78
|
+
=== Example
|
79
|
+
|
80
|
+
user_info/user.xls | User | give_admin_access=false,send_email=true
|
81
|
+
user_info/roles.xls | Role
|
82
|
+
user_info/permissions.xls | UserPermission
|
83
|
+
|
84
|
+
You could save this file as db/rseed/user_info.seed and run the command like this:
|
85
|
+
|
86
|
+
rake rseed:seed SET=user_info
|
87
|
+
|
88
|
+
Note this will get the data files from a subdirectory: db/rseed/user_info. Also the top conversion uses options but as they are optional, the following two do not.
|
89
|
+
|
90
|
+
If you do not specify a set, the rake task will look for a set based on the current development environment: ie development.seed.
|
91
|
+
|
92
|
+
|
93
|
+
== The Converter Class
|
94
|
+
|
95
|
+
=== Column Setup
|
96
|
+
|
97
|
+
==== Mandatory Columns
|
98
|
+
==== Column Types
|
99
|
+
|
100
|
+
=== Custom Type Conversions
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'rails/generators'
|
2
|
+
|
3
|
+
module Rseed
|
4
|
+
module Generators
|
5
|
+
class ConverterGenerator < Rails::Generators::NamedBase
|
6
|
+
class_option :converter_name, :type => :string, :default => nil, :desc => "Names the converter file, defaults to the model name"
|
7
|
+
|
8
|
+
def self.source_root
|
9
|
+
@source_root ||= File.join(File.dirname(__FILE__), 'templates')
|
10
|
+
end
|
11
|
+
|
12
|
+
def create_files
|
13
|
+
converter_dir = File.join("app", "rseed")
|
14
|
+
seed_dir = File.join("db", "rseed")
|
15
|
+
Dir.mkdir(converter_dir) unless File.directory?(converter_dir)
|
16
|
+
@model_name = file_name
|
17
|
+
@class_name = class_name
|
18
|
+
@model = eval(@class_name)
|
19
|
+
@columns = @model.columns_hash.except "id", "created_at", "updated_at"
|
20
|
+
@converter_name = options.converter_name || @class_name
|
21
|
+
template 'converter.rb.erb', File.join(converter_dir, "#{@converter_name.underscore}_converter.rb")
|
22
|
+
Dir.mkdir(seed_dir) unless File.directory?(seed_dir)
|
23
|
+
template 'data.csv.erb', File.join(seed_dir, "#{@converter_name.underscore}.csv")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# Converter created by Rseed
|
2
|
+
|
3
|
+
class <%= @converter_name.camelize %>Converter < Rseed::Converter
|
4
|
+
<% @columns.each_pair do |attribute, meta| -%>
|
5
|
+
attribute :<%= attribute %>, type: :<%= meta.type %>
|
6
|
+
<% end -%>
|
7
|
+
|
8
|
+
def <%= @model_name %>_attributes values
|
9
|
+
attributes = {}
|
10
|
+
|
11
|
+
[<%= [].tap { |list| @columns.each_pair{|attribute, meta| list << ":#{attribute}"}}.join(", ") %>].each do |attribute|
|
12
|
+
attributes[attribute] = values[attribute]
|
13
|
+
end
|
14
|
+
|
15
|
+
return attributes
|
16
|
+
end
|
17
|
+
|
18
|
+
def before_deserialize
|
19
|
+
end
|
20
|
+
|
21
|
+
def after_deserialize
|
22
|
+
end
|
23
|
+
|
24
|
+
def deserialize values
|
25
|
+
# Prevents nil values coming from the import overwriting the model attributes
|
26
|
+
remove_nil_from values
|
27
|
+
# Prevents blank values coming from the import overwriting the model attributes
|
28
|
+
# remove_blank_from values
|
29
|
+
|
30
|
+
# For create only
|
31
|
+
<%= @model_name %> = <%= @class_name %>.new
|
32
|
+
# For create or update, use the following instead and change the match attribute name as required
|
33
|
+
# match_attribute = :id
|
34
|
+
# <%= @model_name %> = <%= @class_name %>.where(match_attribute => values[match_attribute]).first_or_initialize
|
35
|
+
|
36
|
+
# This will return false if the record fails to update, signalling a failed import
|
37
|
+
<%= @model_name %>.update_attributes <%= @model_name %>_attributes(values)
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
<%= CSV.generate { |csv| csv << @columns.keys } %>
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Adapter
|
3
|
+
attr_writer :logger
|
4
|
+
attr_writer :options
|
5
|
+
attr_reader :error
|
6
|
+
attr_accessor :converter
|
7
|
+
|
8
|
+
def logger
|
9
|
+
@logger.nil? ? Rseed.logger : @logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def options
|
13
|
+
@options.nil? ? {} : @options
|
14
|
+
end
|
15
|
+
|
16
|
+
def converter_attributes
|
17
|
+
return [] unless converter
|
18
|
+
converter.class.converter_attributes
|
19
|
+
end
|
20
|
+
|
21
|
+
def mandatory_attributes
|
22
|
+
return [] unless converter
|
23
|
+
converter.class.mandatory_attributes
|
24
|
+
end
|
25
|
+
|
26
|
+
# Dummy process that should be overwritten by other adapters
|
27
|
+
def preprocess
|
28
|
+
return true
|
29
|
+
end
|
30
|
+
|
31
|
+
def process &block
|
32
|
+
values = {}
|
33
|
+
meta = {}
|
34
|
+
yield values, meta
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Attribute
|
3
|
+
attr_accessor :name
|
4
|
+
attr_accessor :options
|
5
|
+
|
6
|
+
def initialize(name, options = {})
|
7
|
+
@name = name
|
8
|
+
@options = options
|
9
|
+
end
|
10
|
+
|
11
|
+
def header
|
12
|
+
return options[:header] || name
|
13
|
+
end
|
14
|
+
|
15
|
+
def matches? match_name
|
16
|
+
unless options[:match]
|
17
|
+
return true if options[:header] and match_name == options[:header]
|
18
|
+
return match_name.to_s == self.name.to_s
|
19
|
+
end
|
20
|
+
re = Regexp.new(options[:match])
|
21
|
+
!re.match(match_name).nil?
|
22
|
+
end
|
23
|
+
|
24
|
+
def deserialize values
|
25
|
+
return nil if values[self.name].nil?
|
26
|
+
value = values[self.name]
|
27
|
+
|
28
|
+
if options[:model] && options[:model_attribute]
|
29
|
+
# The attribute is a model, we look up the model via the specified attribute
|
30
|
+
model_name = options[:model] == true ? self.name : options[:model]
|
31
|
+
klass = model_name.to_s.classify.constantize
|
32
|
+
model_match = options[:model_match] || :first
|
33
|
+
value = klass.where(options[:model_attribute] => value).send(model_match.to_s)
|
34
|
+
elsif options[:type]
|
35
|
+
# Check for a deserialize function for the type
|
36
|
+
dsf = "deserialize_#{options[:type].to_s}"
|
37
|
+
if self.respond_to? dsf
|
38
|
+
value = self.send(dsf, value)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
value
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Rseed
|
2
|
+
module AttributeConverters
|
3
|
+
def deserialize_string(value)
|
4
|
+
value = value.to_i if (value.to_i == value.to_f) if /^\s*[\d]+(\.0+){0,1}\s*$/.match(value.to_s)
|
5
|
+
return nil if value.to_s.blank? || value.to_s.nil?
|
6
|
+
value.to_s
|
7
|
+
end
|
8
|
+
|
9
|
+
def deserialize_clean_string(value)
|
10
|
+
value = value.to_i if (value.to_i == value.to_f) if /^\s*[\d]+(\.0+){0,1}\s*$/.match(value.to_s)
|
11
|
+
value = value.gsub(/[^A-Za-z0-9 \.,\?'""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]/, '').strip if value.is_a?(String)
|
12
|
+
return nil if value.to_s.blank? || value.to_s.nil?
|
13
|
+
value.to_s
|
14
|
+
end
|
15
|
+
|
16
|
+
def deserialize_boolean value
|
17
|
+
/^y|t/.match(value.strip.downcase) ? true : false
|
18
|
+
end
|
19
|
+
|
20
|
+
def deserialize_date s
|
21
|
+
return nil if (s.nil? || s.blank?)
|
22
|
+
return Date.strptime(s, "%d/%m/%y") if /^[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2}$/.match(s)
|
23
|
+
return DateTime.new(1899,12,30) + s.to_f if s.to_f unless s !~ /^\s*[+-]?((\d+_?)*\d+(\.(\d+_?)*\d+)?|\.(\d+_?)*\d+)(\s*|([eE][+-]?(\d+_?)*\d+)\s*)$/
|
24
|
+
begin
|
25
|
+
result = Date.parse(s)
|
26
|
+
rescue
|
27
|
+
Rseed.logger.error "Could not parse date ".red + "'#{s}'"
|
28
|
+
end
|
29
|
+
|
30
|
+
return result
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Converter
|
3
|
+
include AttributeConverters
|
4
|
+
|
5
|
+
class << self
|
6
|
+
attr_accessor :converter_attributes
|
7
|
+
end
|
8
|
+
|
9
|
+
attr_writer :logger
|
10
|
+
attr_reader :error
|
11
|
+
attr_writer :options
|
12
|
+
|
13
|
+
def name
|
14
|
+
class_name = self.class.to_s
|
15
|
+
m = /^(?<name>.*)Converter$/.match(class_name)
|
16
|
+
m ? m[:name] : class_name
|
17
|
+
end
|
18
|
+
|
19
|
+
def logger
|
20
|
+
@logger.nil? ? Rseed.logger : @logger
|
21
|
+
end
|
22
|
+
|
23
|
+
def options
|
24
|
+
@options ||= {}
|
25
|
+
@options
|
26
|
+
end
|
27
|
+
|
28
|
+
# Used to define an attribute when creating a converter.
|
29
|
+
def self.attribute name, options
|
30
|
+
@converter_attributes ||= []
|
31
|
+
converter_attributes << Attribute.new(name, options)
|
32
|
+
end
|
33
|
+
|
34
|
+
def before_deserialize
|
35
|
+
end
|
36
|
+
|
37
|
+
def after_deserialize
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.mandatory_attributes
|
41
|
+
converter_attributes.reject { |a| a.options[:optional] }
|
42
|
+
end
|
43
|
+
|
44
|
+
# Takes the raw values coming out of an adapter and converts them based on the attribute definitions in the
|
45
|
+
# converter.
|
46
|
+
def deserialize_raw values
|
47
|
+
converted_values = {}
|
48
|
+
self.class.converter_attributes.each do |attribute|
|
49
|
+
converted_values[attribute.name] = attribute.deserialize(values)
|
50
|
+
end
|
51
|
+
|
52
|
+
deserialize converted_values
|
53
|
+
end
|
54
|
+
|
55
|
+
# Dummy convert function
|
56
|
+
def deserialize values
|
57
|
+
logger.debug values
|
58
|
+
end
|
59
|
+
|
60
|
+
# Helpers for converters
|
61
|
+
def remove_nil_from values
|
62
|
+
values.delete_if { |k, v| v.nil? }
|
63
|
+
end
|
64
|
+
|
65
|
+
def remove_blank_from values
|
66
|
+
values.delete_if { |k, v| v.to_s.blank? }
|
67
|
+
end
|
68
|
+
|
69
|
+
def fail_with_error e
|
70
|
+
@error = e
|
71
|
+
false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
|
3
|
+
module Rseed
|
4
|
+
class CsvAdapter < Rseed::Adapter
|
5
|
+
attr_accessor :file
|
6
|
+
|
7
|
+
def preprocess
|
8
|
+
return false unless file
|
9
|
+
logger.info "Preprocessing CSV file: #{file.to_s.yellow}"
|
10
|
+
@estimated_rows = CSV.read(file).length - 1
|
11
|
+
logger.info "Estimated Rows: #{@estimated_rows}".magenta
|
12
|
+
true
|
13
|
+
end
|
14
|
+
|
15
|
+
def process &block
|
16
|
+
headers = {}
|
17
|
+
header = true
|
18
|
+
data_count = 0
|
19
|
+
row_number = 0
|
20
|
+
|
21
|
+
# Get an estimate of the number of rows in the file
|
22
|
+
CSV.foreach(file, {:encoding => 'windows-1251:utf-8'}) do |row|
|
23
|
+
row_number += 1
|
24
|
+
if (header)
|
25
|
+
column = 0
|
26
|
+
row.each do |column_value|
|
27
|
+
column += 1
|
28
|
+
converter_attributes.each do |attribute|
|
29
|
+
if attribute.matches? column_value
|
30
|
+
logger.debug "Found header for #{attribute.name} at column #{column}".green
|
31
|
+
if (headers[attribute.name].nil?)
|
32
|
+
headers[attribute.name] = column
|
33
|
+
else
|
34
|
+
logger.error "Found duplicate header '#{attribute.name}' on columns #{column} and #{headers[attribute.name]}.".red
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
unless all_headers_found(headers)
|
40
|
+
logger.error "Missing headers".red
|
41
|
+
break
|
42
|
+
end
|
43
|
+
header = false
|
44
|
+
else
|
45
|
+
import_row = {}
|
46
|
+
headers.each_pair do |name, column|
|
47
|
+
value = row[column - 1].to_s
|
48
|
+
import_row[name] = value
|
49
|
+
end
|
50
|
+
data_count += 1
|
51
|
+
yield import_row, record_count: data_count, total_records: @estimated_rows
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def all_headers_found(headers)
|
57
|
+
@missing_headers_mandatory = []
|
58
|
+
@missing_headers_optional = []
|
59
|
+
found_at_least_one = false
|
60
|
+
|
61
|
+
converter_attributes.each do |attribute|
|
62
|
+
if headers[attribute.name].nil?
|
63
|
+
unless attribute.options[:optional]
|
64
|
+
@missing_headers_mandatory << attribute.name
|
65
|
+
else
|
66
|
+
@missing_headers_optional << attribute.name
|
67
|
+
end
|
68
|
+
else
|
69
|
+
found_at_least_one = true
|
70
|
+
end
|
71
|
+
end
|
72
|
+
if found_at_least_one
|
73
|
+
logger.warning "Missing optional headers: #{@missing_headers_optional.join(',')}".yellow unless @missing_headers_optional.empty?
|
74
|
+
logger.warning "Missing mandatory headers: #{@missing_headers_mandatory.join(',')}".red unless @missing_headers_mandatory.empty?
|
75
|
+
end
|
76
|
+
return false unless @missing_headers_mandatory.empty?
|
77
|
+
true
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Rseed
|
2
|
+
class HashAdapter < Rseed::Adapter
|
3
|
+
attr_accessor :data
|
4
|
+
def initialize data = nil
|
5
|
+
@data = data
|
6
|
+
end
|
7
|
+
|
8
|
+
def preprocess
|
9
|
+
return false unless @data.is_a? Array or @data.is_a?(Hash)
|
10
|
+
@data = [@data] if @data.is_a?(Hash)
|
11
|
+
true
|
12
|
+
end
|
13
|
+
|
14
|
+
def process &block
|
15
|
+
meta = {}
|
16
|
+
meta[:total_records] = @data.length
|
17
|
+
@data.each_with_index do |d, i|
|
18
|
+
meta[:record_count] = i + 1
|
19
|
+
yield d, meta
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Processor
|
3
|
+
attr_writer :logger
|
4
|
+
attr_reader :adapter
|
5
|
+
attr_reader :converter
|
6
|
+
|
7
|
+
def initialize(options = {})
|
8
|
+
return nil unless options[:adapter]
|
9
|
+
return nil unless options[:converter]
|
10
|
+
|
11
|
+
adapter = options[:adapter].is_a?(Adapter) ? options[:adapter] : Rseed.const_get("#{options[:adapter].to_s.classify}Adapter").new
|
12
|
+
converter = options[:converter].is_a?(Converter) ? options[:converter] : Rseed.const_get("#{options[:converter].to_s.classify}Converter").new
|
13
|
+
|
14
|
+
@adapter = adapter
|
15
|
+
@converter = converter
|
16
|
+
end
|
17
|
+
|
18
|
+
def logger
|
19
|
+
@logger.nil? ? Rseed.logger : @logger
|
20
|
+
end
|
21
|
+
|
22
|
+
def deserialize options = {}, &block
|
23
|
+
converter.logger = logger
|
24
|
+
adapter.logger = logger
|
25
|
+
adapter.converter = converter
|
26
|
+
|
27
|
+
yield :preprocessing
|
28
|
+
begin
|
29
|
+
if @adapter.preprocess
|
30
|
+
@converter.before_deserialize
|
31
|
+
yield :processing
|
32
|
+
start_time = Time.now
|
33
|
+
adapter.process do |values, meta|
|
34
|
+
result = {}
|
35
|
+
meta ||= {}
|
36
|
+
begin
|
37
|
+
if @converter.deserialize_raw(values)
|
38
|
+
result[:success] = true
|
39
|
+
else
|
40
|
+
result[:success] = false
|
41
|
+
result[:message] = "Failed to convert"
|
42
|
+
result[:error] = @converter.error
|
43
|
+
end
|
44
|
+
rescue Exception => e
|
45
|
+
result[:success] = false
|
46
|
+
result[:message] = "Exception during deserialize"
|
47
|
+
result[:error] = e.message
|
48
|
+
result[:backtrace] = e.backtrace
|
49
|
+
end
|
50
|
+
|
51
|
+
# Calculate the ETA
|
52
|
+
if meta[:record_count] and meta[:total_records]
|
53
|
+
remaining = meta[:total_records] - meta[:record_count]
|
54
|
+
tpr = (Time.now - start_time)/meta[:record_count]
|
55
|
+
meta[:eta] = remaining * tpr
|
56
|
+
end
|
57
|
+
yield :processing, result, meta
|
58
|
+
end
|
59
|
+
@converter.after_deserialize
|
60
|
+
else
|
61
|
+
yield :error, {success: false, message: 'Preprocessing failed', error: @adapter.error}
|
62
|
+
end
|
63
|
+
rescue Exception => e
|
64
|
+
yield :error, {success: false, message: 'Exception during preprocessing', error: e.message, backtrace: e.backtrace}
|
65
|
+
end
|
66
|
+
yield :complete
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'ruby-progressbar'
|
2
|
+
require 'colorize'
|
3
|
+
|
4
|
+
module Rseed
|
5
|
+
class ProgressBarLogger < IO
|
6
|
+
def initialize(progress_bar)
|
7
|
+
@progress_bar = progress_bar
|
8
|
+
end
|
9
|
+
|
10
|
+
def write(*args)
|
11
|
+
@progress_bar.log args.join(',')
|
12
|
+
end
|
13
|
+
|
14
|
+
def close
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def process_with_status_bar processor, options = {}
|
19
|
+
title = options[:title] ? options[:title] : "Seed"
|
20
|
+
title = "#{processor.converter.name.cyan} #{title.blue}"
|
21
|
+
record_count = 0
|
22
|
+
progress_bar = ProgressBar.create(starting_at: nil, total: nil, format: "#{"Preprocessing".magenta} %t <%B>", title: title)
|
23
|
+
processor.logger = Logger.new(ProgressBarLogger.new(progress_bar))
|
24
|
+
processor.deserialize do |status, result, meta|
|
25
|
+
eta = meta ? meta[:eta] : nil
|
26
|
+
eta = eta ? Time.at(eta).utc.strftime("%H:%M:%S") : "??:??"
|
27
|
+
case status
|
28
|
+
when :processing
|
29
|
+
progress_bar.format "#{"Processing".yellow} %t <%B> %c/%C (#{eta.to_s.yellow})"
|
30
|
+
if meta
|
31
|
+
if record_count != meta[:record_count]
|
32
|
+
record_count = meta[:record_count]
|
33
|
+
progress_bar.total ||= meta[:total_records]
|
34
|
+
# Set the progress unless it is the finishing record.
|
35
|
+
progress_bar.progress = record_count unless record_count == progress_bar.total
|
36
|
+
end
|
37
|
+
end
|
38
|
+
when :complete
|
39
|
+
progress_bar.format "#{"Complete".green} %t <%B> %C (%a)"
|
40
|
+
progress_bar.finish
|
41
|
+
when :error
|
42
|
+
processor.logger.error result[:message].to_s.red
|
43
|
+
processor.logger.error result[:error]
|
44
|
+
processor.logger.error result[:backtrace].join('\n')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def from_file file, options = {}
|
50
|
+
p = Processor.new(options)
|
51
|
+
return nil unless p
|
52
|
+
p.adapter.file = file
|
53
|
+
process_with_status_bar p, title: file
|
54
|
+
end
|
55
|
+
|
56
|
+
def from_csv(file, options = {})
|
57
|
+
options[:adapter] = :csv
|
58
|
+
from_file(file, options)
|
59
|
+
end
|
60
|
+
|
61
|
+
def from_hash hash_or_array, options = {}
|
62
|
+
p = Processor.new(adapter: :hash, converter: options[:converter])
|
63
|
+
return nil unless p
|
64
|
+
p.adapter.data = hash_or_array
|
65
|
+
process_with_status_bar p, title: "Hash"
|
66
|
+
end
|
67
|
+
|
68
|
+
module_function :from_file
|
69
|
+
module_function :from_hash
|
70
|
+
module_function :from_csv
|
71
|
+
module_function :process_with_status_bar
|
72
|
+
end
|
data/lib/rseed.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require "csv"
|
2
|
+
require "rseed/attribute"
|
3
|
+
require "rseed/attribute_converters"
|
4
|
+
require "rseed/version"
|
5
|
+
require "rseed/adapter"
|
6
|
+
require "rseed/hash_adapter"
|
7
|
+
require "rseed/converter"
|
8
|
+
require "rseed/processor"
|
9
|
+
require "rseed/csv_adapter"
|
10
|
+
require "rseed/utilities"
|
11
|
+
|
12
|
+
module Rseed
|
13
|
+
class << self
|
14
|
+
attr_accessor :logger
|
15
|
+
end
|
16
|
+
|
17
|
+
@logger = Logger.new(STDOUT)
|
18
|
+
|
19
|
+
class Railtie < ::Rails::Railtie
|
20
|
+
railtie_name :rseed
|
21
|
+
|
22
|
+
rake_tasks do
|
23
|
+
load "tasks/rseed.rake"
|
24
|
+
end
|
25
|
+
|
26
|
+
generators do
|
27
|
+
require "generators/rseed/converter"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/rseed.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'rseed/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "rseed"
|
8
|
+
spec.version = Rseed::VERSION
|
9
|
+
spec.authors = ["David Monagle"]
|
10
|
+
spec.email = ["david.monagle@intrica.com.au"]
|
11
|
+
spec.description = ""
|
12
|
+
spec.summary = %q{Assist with seeding/import of external data into models.}
|
13
|
+
spec.homepage = "http://www.intrica.com.au"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "colorize"
|
22
|
+
spec.add_dependency "ruby-progressbar"
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
25
|
+
spec.add_development_dependency "rake"
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rseed
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Monagle
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-11-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: colorize
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: ruby-progressbar
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ! '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: ''
|
70
|
+
email:
|
71
|
+
- david.monagle@intrica.com.au
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- .gitignore
|
77
|
+
- Gemfile
|
78
|
+
- LICENSE.txt
|
79
|
+
- README.rdoc
|
80
|
+
- Rakefile
|
81
|
+
- lib/generators/rseed/converter.rb
|
82
|
+
- lib/generators/rseed/templates/converter.rb.erb
|
83
|
+
- lib/generators/rseed/templates/data.csv.erb
|
84
|
+
- lib/rseed.rb
|
85
|
+
- lib/rseed/adapter.rb
|
86
|
+
- lib/rseed/attribute.rb
|
87
|
+
- lib/rseed/attribute_converters.rb
|
88
|
+
- lib/rseed/converter.rb
|
89
|
+
- lib/rseed/csv_adapter.rb
|
90
|
+
- lib/rseed/hash_adapter.rb
|
91
|
+
- lib/rseed/processor.rb
|
92
|
+
- lib/rseed/utilities.rb
|
93
|
+
- lib/rseed/version.rb
|
94
|
+
- lib/tasks/rseed.rake
|
95
|
+
- rseed.gemspec
|
96
|
+
homepage: http://www.intrica.com.au
|
97
|
+
licenses:
|
98
|
+
- MIT
|
99
|
+
metadata: {}
|
100
|
+
post_install_message:
|
101
|
+
rdoc_options: []
|
102
|
+
require_paths:
|
103
|
+
- lib
|
104
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ! '>='
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - ! '>='
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '0'
|
114
|
+
requirements: []
|
115
|
+
rubyforge_project:
|
116
|
+
rubygems_version: 2.1.9
|
117
|
+
signing_key:
|
118
|
+
specification_version: 4
|
119
|
+
summary: Assist with seeding/import of external data into models.
|
120
|
+
test_files: []
|