rseed 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +18 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.rdoc +100 -0
- data/Rakefile +1 -0
- data/lib/generators/rseed/converter.rb +27 -0
- data/lib/generators/rseed/templates/converter.rb.erb +39 -0
- data/lib/generators/rseed/templates/data.csv.erb +1 -0
- data/lib/rseed/adapter.rb +37 -0
- data/lib/rseed/attribute.rb +44 -0
- data/lib/rseed/attribute_converters.rb +33 -0
- data/lib/rseed/converter.rb +74 -0
- data/lib/rseed/csv_adapter.rb +80 -0
- data/lib/rseed/hash_adapter.rb +23 -0
- data/lib/rseed/processor.rb +69 -0
- data/lib/rseed/utilities.rb +72 -0
- data/lib/rseed/version.rb +3 -0
- data/lib/rseed.rb +30 -0
- data/lib/tasks/rseed.rake +8 -0
- data/rseed.gemspec +26 -0
- metadata +120 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ODgyMmI1OTUxYmZlMTBkOTY4M2I5ZDRmOWQ1ODA0MDNjM2M1MTFkZg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
YzRlMzE4YzM3ZjI4MzFjMmFmZjcwYjYxODAwMmQyMTUwNGNiY2FhMA==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NDdhYWUyMTU2MzAyN2M0MjliYzliZWI1NjY5YmI1N2ZjNzdiMjcwMzU3Njc5
|
10
|
+
MjA1OTI4ZDFkZjg4MzZhN2UzZDNmYjIzYThkYzU2ZDQwNjhlZGEzNmVlMDk4
|
11
|
+
Yzg2ZGJhMmQxNDg4Mzg0Y2NhZTRjYTQ0NzI1ODQxNzllNjkwNDY=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NmZmNzFhYzRiMmEyYzYwNmEyMTVhYmI4OTk1MGRhNjNkMjBiYjU2ZWFlMGQz
|
14
|
+
NTA1NDNiMzU1YWM1MzIyMjUyYjgwYzBjNjQyYzBlOGEzMTI2NjZlZjVmMzEy
|
15
|
+
MzdkZTI5YjFhYWI4YmY4N2NhZjRmMTQxZmE2YzU5MDlmZTkxMTE=
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 David Monagle
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
= Rseed
|
2
|
+
|
3
|
+
Rseed is a replacement for rseed. There are lots of improvements in order to make it easy to create and
|
4
|
+
maintain converters.
|
5
|
+
|
6
|
+
== Installation
|
7
|
+
|
8
|
+
Simple add the following to your Gemfile
|
9
|
+
|
10
|
+
gem 'rseed'
|
11
|
+
|
12
|
+
Then run:
|
13
|
+
|
14
|
+
bundle install
|
15
|
+
|
16
|
+
== Quick Example
|
17
|
+
|
18
|
+
rails g rseed:converter User
|
19
|
+
|
20
|
+
This will create an model converter in the directory app/rseed. You can read through this import file to see how the import works.
|
21
|
+
|
22
|
+
This also creates a default data file in db/rseed. This will be the CSV used for this converter.
|
23
|
+
|
24
|
+
== The Converter File
|
25
|
+
|
26
|
+
=== Options
|
27
|
+
|
28
|
+
:header
|
29
|
+
Defines the name of the attribute to be used for serialization. If there is no :match defined, it will also be used
|
30
|
+
to match the attribute name of the input to the attribute being defined.
|
31
|
+
|
32
|
+
:match
|
33
|
+
A regex string that is used to match the attribute name of the input to the attribute being defined. If this is not
|
34
|
+
defined, a match will be checked against :header and then the attribute name.
|
35
|
+
|
36
|
+
:type
|
37
|
+
Defines a type for the string.
|
38
|
+
|
39
|
+
:model
|
40
|
+
This can be set to the name of a model that this attribute should resolve to. The model is classified so using a symbol
|
41
|
+
works here. Alternately, if this is set to *true*, then the name of the attribute will be used as the model name. In
|
42
|
+
order for this to work, :model_attribute must also be set.
|
43
|
+
|
44
|
+
:model_attribute
|
45
|
+
Specify which attribute on the model is used for lookup.
|
46
|
+
|
47
|
+
:model_match
|
48
|
+
Specifies how the model should be resolved. The value here is called against the *where* that is used to look up the model.
|
49
|
+
For example, this defaults to *:first*. If your model is *Person* and the :model_attribute is *:name* then this is what
|
50
|
+
is called to set the attribute value:
|
51
|
+
|
52
|
+
Person.where(name: <value>).first
|
53
|
+
|
54
|
+
You may use any active record method in this case, such as :first_or_create, or :last.
|
55
|
+
|
56
|
+
:optional
|
57
|
+
Defines the attribute as optionsal. This has no effect in the *HashAdapter*.
|
58
|
+
|
59
|
+
== Rake Tasks
|
60
|
+
|
61
|
+
These rake tasks allow you to run seeds manually:
|
62
|
+
|
63
|
+
rake rseed:csv Load csv file into a model using a model converter
|
64
|
+
rake rseed:seed Seed a list of import files
|
65
|
+
|
66
|
+
=== Examples
|
67
|
+
|
68
|
+
rake rseed:csv FILE=user.csv CONVERTER=User CONVERTER_OPTIONS="give_admin_access=true"
|
69
|
+
|
70
|
+
In this case the file in db/rseed/user.csv would be run through the converter UserConverter. The options specified are available within the converter. In this case @options["give_admin_access"] will evaluate to true.
|
71
|
+
|
72
|
+
The FILE parameter is not strictly necessary in this case either as the default file name will be an underscored value of the name of the converter.
|
73
|
+
|
74
|
+
== Seeding
|
75
|
+
|
76
|
+
Seeding allows you to import several files through different model converters in a single command. It involves the creation of a .seed file. Each file goes on a single line and the options are separated by pipe symbols.
|
77
|
+
|
78
|
+
=== Example
|
79
|
+
|
80
|
+
user_info/user.xls | User | give_admin_access=false,send_email=true
|
81
|
+
user_info/roles.xls | Role
|
82
|
+
user_info/permissions.xls | UserPermission
|
83
|
+
|
84
|
+
You could save this file as db/rseed/user_info.seed and run the command like this:
|
85
|
+
|
86
|
+
rake rseed:seed SET=user_info
|
87
|
+
|
88
|
+
Note this will get the data files from a subdirectory: db/rseed/user_info. Also the top conversion uses options but as they are optional, the following two do not.
|
89
|
+
|
90
|
+
If you do not specify a set, the rake task will look for a set based on the current development environment: ie development.seed.
|
91
|
+
|
92
|
+
|
93
|
+
== The Converter Class
|
94
|
+
|
95
|
+
=== Column Setup
|
96
|
+
|
97
|
+
==== Mandatory Columns
|
98
|
+
==== Column Types
|
99
|
+
|
100
|
+
=== Custom Type Conversions
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'rails/generators'
|
2
|
+
|
3
|
+
module Rseed
|
4
|
+
module Generators
|
5
|
+
class ConverterGenerator < Rails::Generators::NamedBase
|
6
|
+
class_option :converter_name, :type => :string, :default => nil, :desc => "Names the converter file, defaults to the model name"
|
7
|
+
|
8
|
+
def self.source_root
|
9
|
+
@source_root ||= File.join(File.dirname(__FILE__), 'templates')
|
10
|
+
end
|
11
|
+
|
12
|
+
def create_files
|
13
|
+
converter_dir = File.join("app", "rseed")
|
14
|
+
seed_dir = File.join("db", "rseed")
|
15
|
+
Dir.mkdir(converter_dir) unless File.directory?(converter_dir)
|
16
|
+
@model_name = file_name
|
17
|
+
@class_name = class_name
|
18
|
+
@model = eval(@class_name)
|
19
|
+
@columns = @model.columns_hash.except "id", "created_at", "updated_at"
|
20
|
+
@converter_name = options.converter_name || @class_name
|
21
|
+
template 'converter.rb.erb', File.join(converter_dir, "#{@converter_name.underscore}_converter.rb")
|
22
|
+
Dir.mkdir(seed_dir) unless File.directory?(seed_dir)
|
23
|
+
template 'data.csv.erb', File.join(seed_dir, "#{@converter_name.underscore}.csv")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# Converter created by Rseed
|
2
|
+
|
3
|
+
class <%= @converter_name.camelize %>Converter < Rseed::Converter
|
4
|
+
<% @columns.each_pair do |attribute, meta| -%>
|
5
|
+
attribute :<%= attribute %>, type: :<%= meta.type %>
|
6
|
+
<% end -%>
|
7
|
+
|
8
|
+
def <%= @model_name %>_attributes values
|
9
|
+
attributes = {}
|
10
|
+
|
11
|
+
[<%= [].tap { |list| @columns.each_pair{|attribute, meta| list << ":#{attribute}"}}.join(", ") %>].each do |attribute|
|
12
|
+
attributes[attribute] = values[attribute]
|
13
|
+
end
|
14
|
+
|
15
|
+
return attributes
|
16
|
+
end
|
17
|
+
|
18
|
+
def before_deserialize
|
19
|
+
end
|
20
|
+
|
21
|
+
def after_deserialize
|
22
|
+
end
|
23
|
+
|
24
|
+
def deserialize values
|
25
|
+
# Prevents nil values coming from the import overwriting the model attributes
|
26
|
+
remove_nil_from values
|
27
|
+
# Prevents blank values coming from the import overwriting the model attributes
|
28
|
+
# remove_blank_from values
|
29
|
+
|
30
|
+
# For create only
|
31
|
+
<%= @model_name %> = <%= @class_name %>.new
|
32
|
+
# For create or update, use the following instead and change the match attribute name as required
|
33
|
+
# match_attribute = :id
|
34
|
+
# <%= @model_name %> = <%= @class_name %>.where(match_attribute => values[match_attribute]).first_or_initialize
|
35
|
+
|
36
|
+
# This will return false if the record fails to update, signalling a failed import
|
37
|
+
<%= @model_name %>.update_attributes <%= @model_name %>_attributes(values)
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
<%= CSV.generate { |csv| csv << @columns.keys } %>
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Adapter
|
3
|
+
attr_writer :logger
|
4
|
+
attr_writer :options
|
5
|
+
attr_reader :error
|
6
|
+
attr_accessor :converter
|
7
|
+
|
8
|
+
def logger
|
9
|
+
@logger.nil? ? Rseed.logger : @logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def options
|
13
|
+
@options.nil? ? {} : @options
|
14
|
+
end
|
15
|
+
|
16
|
+
def converter_attributes
|
17
|
+
return [] unless converter
|
18
|
+
converter.class.converter_attributes
|
19
|
+
end
|
20
|
+
|
21
|
+
def mandatory_attributes
|
22
|
+
return [] unless converter
|
23
|
+
converter.class.mandatory_attributes
|
24
|
+
end
|
25
|
+
|
26
|
+
# Dummy process that should be overwritten by other adapters
|
27
|
+
def preprocess
|
28
|
+
return true
|
29
|
+
end
|
30
|
+
|
31
|
+
def process &block
|
32
|
+
values = {}
|
33
|
+
meta = {}
|
34
|
+
yield values, meta
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Attribute
|
3
|
+
attr_accessor :name
|
4
|
+
attr_accessor :options
|
5
|
+
|
6
|
+
def initialize(name, options = {})
|
7
|
+
@name = name
|
8
|
+
@options = options
|
9
|
+
end
|
10
|
+
|
11
|
+
def header
|
12
|
+
return options[:header] || name
|
13
|
+
end
|
14
|
+
|
15
|
+
def matches? match_name
|
16
|
+
unless options[:match]
|
17
|
+
return true if options[:header] and match_name == options[:header]
|
18
|
+
return match_name.to_s == self.name.to_s
|
19
|
+
end
|
20
|
+
re = Regexp.new(options[:match])
|
21
|
+
!re.match(match_name).nil?
|
22
|
+
end
|
23
|
+
|
24
|
+
def deserialize values
|
25
|
+
return nil if values[self.name].nil?
|
26
|
+
value = values[self.name]
|
27
|
+
|
28
|
+
if options[:model] && options[:model_attribute]
|
29
|
+
# The attribute is a model, we look up the model via the specified attribute
|
30
|
+
model_name = options[:model] == true ? self.name : options[:model]
|
31
|
+
klass = model_name.to_s.classify.constantize
|
32
|
+
model_match = options[:model_match] || :first
|
33
|
+
value = klass.where(options[:model_attribute] => value).send(model_match.to_s)
|
34
|
+
elsif options[:type]
|
35
|
+
# Check for a deserialize function for the type
|
36
|
+
dsf = "deserialize_#{options[:type].to_s}"
|
37
|
+
if self.respond_to? dsf
|
38
|
+
value = self.send(dsf, value)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
value
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module Rseed
|
2
|
+
module AttributeConverters
|
3
|
+
def deserialize_string(value)
|
4
|
+
value = value.to_i if (value.to_i == value.to_f) if /^\s*[\d]+(\.0+){0,1}\s*$/.match(value.to_s)
|
5
|
+
return nil if value.to_s.blank? || value.to_s.nil?
|
6
|
+
value.to_s
|
7
|
+
end
|
8
|
+
|
9
|
+
def deserialize_clean_string(value)
|
10
|
+
value = value.to_i if (value.to_i == value.to_f) if /^\s*[\d]+(\.0+){0,1}\s*$/.match(value.to_s)
|
11
|
+
value = value.gsub(/[^A-Za-z0-9 \.,\?'""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]/, '').strip if value.is_a?(String)
|
12
|
+
return nil if value.to_s.blank? || value.to_s.nil?
|
13
|
+
value.to_s
|
14
|
+
end
|
15
|
+
|
16
|
+
def deserialize_boolean value
|
17
|
+
/^y|t/.match(value.strip.downcase) ? true : false
|
18
|
+
end
|
19
|
+
|
20
|
+
def deserialize_date s
|
21
|
+
return nil if (s.nil? || s.blank?)
|
22
|
+
return Date.strptime(s, "%d/%m/%y") if /^[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2}$/.match(s)
|
23
|
+
return DateTime.new(1899,12,30) + s.to_f if s.to_f unless s !~ /^\s*[+-]?((\d+_?)*\d+(\.(\d+_?)*\d+)?|\.(\d+_?)*\d+)(\s*|([eE][+-]?(\d+_?)*\d+)\s*)$/
|
24
|
+
begin
|
25
|
+
result = Date.parse(s)
|
26
|
+
rescue
|
27
|
+
Rseed.logger.error "Could not parse date ".red + "'#{s}'"
|
28
|
+
end
|
29
|
+
|
30
|
+
return result
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Converter
|
3
|
+
include AttributeConverters
|
4
|
+
|
5
|
+
class << self
|
6
|
+
attr_accessor :converter_attributes
|
7
|
+
end
|
8
|
+
|
9
|
+
attr_writer :logger
|
10
|
+
attr_reader :error
|
11
|
+
attr_writer :options
|
12
|
+
|
13
|
+
def name
|
14
|
+
class_name = self.class.to_s
|
15
|
+
m = /^(?<name>.*)Converter$/.match(class_name)
|
16
|
+
m ? m[:name] : class_name
|
17
|
+
end
|
18
|
+
|
19
|
+
def logger
|
20
|
+
@logger.nil? ? Rseed.logger : @logger
|
21
|
+
end
|
22
|
+
|
23
|
+
def options
|
24
|
+
@options ||= {}
|
25
|
+
@options
|
26
|
+
end
|
27
|
+
|
28
|
+
# Used to define an attribute when creating a converter.
|
29
|
+
def self.attribute name, options
|
30
|
+
@converter_attributes ||= []
|
31
|
+
converter_attributes << Attribute.new(name, options)
|
32
|
+
end
|
33
|
+
|
34
|
+
def before_deserialize
|
35
|
+
end
|
36
|
+
|
37
|
+
def after_deserialize
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.mandatory_attributes
|
41
|
+
converter_attributes.reject { |a| a.options[:optional] }
|
42
|
+
end
|
43
|
+
|
44
|
+
# Takes the raw values coming out of an adapter and converts them based on the attribute definitions in the
|
45
|
+
# converter.
|
46
|
+
def deserialize_raw values
|
47
|
+
converted_values = {}
|
48
|
+
self.class.converter_attributes.each do |attribute|
|
49
|
+
converted_values[attribute.name] = attribute.deserialize(values)
|
50
|
+
end
|
51
|
+
|
52
|
+
deserialize converted_values
|
53
|
+
end
|
54
|
+
|
55
|
+
# Dummy convert function
|
56
|
+
def deserialize values
|
57
|
+
logger.debug values
|
58
|
+
end
|
59
|
+
|
60
|
+
# Helpers for converters
|
61
|
+
def remove_nil_from values
|
62
|
+
values.delete_if { |k, v| v.nil? }
|
63
|
+
end
|
64
|
+
|
65
|
+
def remove_blank_from values
|
66
|
+
values.delete_if { |k, v| v.to_s.blank? }
|
67
|
+
end
|
68
|
+
|
69
|
+
def fail_with_error e
|
70
|
+
@error = e
|
71
|
+
false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
require 'colorize'
|
2
|
+
|
3
|
+
module Rseed
|
4
|
+
class CsvAdapter < Rseed::Adapter
|
5
|
+
attr_accessor :file
|
6
|
+
|
7
|
+
def preprocess
|
8
|
+
return false unless file
|
9
|
+
logger.info "Preprocessing CSV file: #{file.to_s.yellow}"
|
10
|
+
@estimated_rows = CSV.read(file).length - 1
|
11
|
+
logger.info "Estimated Rows: #{@estimated_rows}".magenta
|
12
|
+
true
|
13
|
+
end
|
14
|
+
|
15
|
+
def process &block
|
16
|
+
headers = {}
|
17
|
+
header = true
|
18
|
+
data_count = 0
|
19
|
+
row_number = 0
|
20
|
+
|
21
|
+
# Get an estimate of the number of rows in the file
|
22
|
+
CSV.foreach(file, {:encoding => 'windows-1251:utf-8'}) do |row|
|
23
|
+
row_number += 1
|
24
|
+
if (header)
|
25
|
+
column = 0
|
26
|
+
row.each do |column_value|
|
27
|
+
column += 1
|
28
|
+
converter_attributes.each do |attribute|
|
29
|
+
if attribute.matches? column_value
|
30
|
+
logger.debug "Found header for #{attribute.name} at column #{column}".green
|
31
|
+
if (headers[attribute.name].nil?)
|
32
|
+
headers[attribute.name] = column
|
33
|
+
else
|
34
|
+
logger.error "Found duplicate header '#{attribute.name}' on columns #{column} and #{headers[attribute.name]}.".red
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
unless all_headers_found(headers)
|
40
|
+
logger.error "Missing headers".red
|
41
|
+
break
|
42
|
+
end
|
43
|
+
header = false
|
44
|
+
else
|
45
|
+
import_row = {}
|
46
|
+
headers.each_pair do |name, column|
|
47
|
+
value = row[column - 1].to_s
|
48
|
+
import_row[name] = value
|
49
|
+
end
|
50
|
+
data_count += 1
|
51
|
+
yield import_row, record_count: data_count, total_records: @estimated_rows
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def all_headers_found(headers)
|
57
|
+
@missing_headers_mandatory = []
|
58
|
+
@missing_headers_optional = []
|
59
|
+
found_at_least_one = false
|
60
|
+
|
61
|
+
converter_attributes.each do |attribute|
|
62
|
+
if headers[attribute.name].nil?
|
63
|
+
unless attribute.options[:optional]
|
64
|
+
@missing_headers_mandatory << attribute.name
|
65
|
+
else
|
66
|
+
@missing_headers_optional << attribute.name
|
67
|
+
end
|
68
|
+
else
|
69
|
+
found_at_least_one = true
|
70
|
+
end
|
71
|
+
end
|
72
|
+
if found_at_least_one
|
73
|
+
logger.warning "Missing optional headers: #{@missing_headers_optional.join(',')}".yellow unless @missing_headers_optional.empty?
|
74
|
+
logger.warning "Missing mandatory headers: #{@missing_headers_mandatory.join(',')}".red unless @missing_headers_mandatory.empty?
|
75
|
+
end
|
76
|
+
return false unless @missing_headers_mandatory.empty?
|
77
|
+
true
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Rseed
|
2
|
+
class HashAdapter < Rseed::Adapter
|
3
|
+
attr_accessor :data
|
4
|
+
def initialize data = nil
|
5
|
+
@data = data
|
6
|
+
end
|
7
|
+
|
8
|
+
def preprocess
|
9
|
+
return false unless @data.is_a? Array or @data.is_a?(Hash)
|
10
|
+
@data = [@data] if @data.is_a?(Hash)
|
11
|
+
true
|
12
|
+
end
|
13
|
+
|
14
|
+
def process &block
|
15
|
+
meta = {}
|
16
|
+
meta[:total_records] = @data.length
|
17
|
+
@data.each_with_index do |d, i|
|
18
|
+
meta[:record_count] = i + 1
|
19
|
+
yield d, meta
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module Rseed
|
2
|
+
class Processor
|
3
|
+
attr_writer :logger
|
4
|
+
attr_reader :adapter
|
5
|
+
attr_reader :converter
|
6
|
+
|
7
|
+
def initialize(options = {})
|
8
|
+
return nil unless options[:adapter]
|
9
|
+
return nil unless options[:converter]
|
10
|
+
|
11
|
+
adapter = options[:adapter].is_a?(Adapter) ? options[:adapter] : Rseed.const_get("#{options[:adapter].to_s.classify}Adapter").new
|
12
|
+
converter = options[:converter].is_a?(Converter) ? options[:converter] : Rseed.const_get("#{options[:converter].to_s.classify}Converter").new
|
13
|
+
|
14
|
+
@adapter = adapter
|
15
|
+
@converter = converter
|
16
|
+
end
|
17
|
+
|
18
|
+
def logger
|
19
|
+
@logger.nil? ? Rseed.logger : @logger
|
20
|
+
end
|
21
|
+
|
22
|
+
def deserialize options = {}, &block
|
23
|
+
converter.logger = logger
|
24
|
+
adapter.logger = logger
|
25
|
+
adapter.converter = converter
|
26
|
+
|
27
|
+
yield :preprocessing
|
28
|
+
begin
|
29
|
+
if @adapter.preprocess
|
30
|
+
@converter.before_deserialize
|
31
|
+
yield :processing
|
32
|
+
start_time = Time.now
|
33
|
+
adapter.process do |values, meta|
|
34
|
+
result = {}
|
35
|
+
meta ||= {}
|
36
|
+
begin
|
37
|
+
if @converter.deserialize_raw(values)
|
38
|
+
result[:success] = true
|
39
|
+
else
|
40
|
+
result[:success] = false
|
41
|
+
result[:message] = "Failed to convert"
|
42
|
+
result[:error] = @converter.error
|
43
|
+
end
|
44
|
+
rescue Exception => e
|
45
|
+
result[:success] = false
|
46
|
+
result[:message] = "Exception during deserialize"
|
47
|
+
result[:error] = e.message
|
48
|
+
result[:backtrace] = e.backtrace
|
49
|
+
end
|
50
|
+
|
51
|
+
# Calculate the ETA
|
52
|
+
if meta[:record_count] and meta[:total_records]
|
53
|
+
remaining = meta[:total_records] - meta[:record_count]
|
54
|
+
tpr = (Time.now - start_time)/meta[:record_count]
|
55
|
+
meta[:eta] = remaining * tpr
|
56
|
+
end
|
57
|
+
yield :processing, result, meta
|
58
|
+
end
|
59
|
+
@converter.after_deserialize
|
60
|
+
else
|
61
|
+
yield :error, {success: false, message: 'Preprocessing failed', error: @adapter.error}
|
62
|
+
end
|
63
|
+
rescue Exception => e
|
64
|
+
yield :error, {success: false, message: 'Exception during preprocessing', error: e.message, backtrace: e.backtrace}
|
65
|
+
end
|
66
|
+
yield :complete
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
require 'ruby-progressbar'
|
2
|
+
require 'colorize'
|
3
|
+
|
4
|
+
module Rseed
|
5
|
+
class ProgressBarLogger < IO
|
6
|
+
def initialize(progress_bar)
|
7
|
+
@progress_bar = progress_bar
|
8
|
+
end
|
9
|
+
|
10
|
+
def write(*args)
|
11
|
+
@progress_bar.log args.join(',')
|
12
|
+
end
|
13
|
+
|
14
|
+
def close
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def process_with_status_bar processor, options = {}
|
19
|
+
title = options[:title] ? options[:title] : "Seed"
|
20
|
+
title = "#{processor.converter.name.cyan} #{title.blue}"
|
21
|
+
record_count = 0
|
22
|
+
progress_bar = ProgressBar.create(starting_at: nil, total: nil, format: "#{"Preprocessing".magenta} %t <%B>", title: title)
|
23
|
+
processor.logger = Logger.new(ProgressBarLogger.new(progress_bar))
|
24
|
+
processor.deserialize do |status, result, meta|
|
25
|
+
eta = meta ? meta[:eta] : nil
|
26
|
+
eta = eta ? Time.at(eta).utc.strftime("%H:%M:%S") : "??:??"
|
27
|
+
case status
|
28
|
+
when :processing
|
29
|
+
progress_bar.format "#{"Processing".yellow} %t <%B> %c/%C (#{eta.to_s.yellow})"
|
30
|
+
if meta
|
31
|
+
if record_count != meta[:record_count]
|
32
|
+
record_count = meta[:record_count]
|
33
|
+
progress_bar.total ||= meta[:total_records]
|
34
|
+
# Set the progress unless it is the finishing record.
|
35
|
+
progress_bar.progress = record_count unless record_count == progress_bar.total
|
36
|
+
end
|
37
|
+
end
|
38
|
+
when :complete
|
39
|
+
progress_bar.format "#{"Complete".green} %t <%B> %C (%a)"
|
40
|
+
progress_bar.finish
|
41
|
+
when :error
|
42
|
+
processor.logger.error result[:message].to_s.red
|
43
|
+
processor.logger.error result[:error]
|
44
|
+
processor.logger.error result[:backtrace].join('\n')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def from_file file, options = {}
|
50
|
+
p = Processor.new(options)
|
51
|
+
return nil unless p
|
52
|
+
p.adapter.file = file
|
53
|
+
process_with_status_bar p, title: file
|
54
|
+
end
|
55
|
+
|
56
|
+
def from_csv(file, options = {})
|
57
|
+
options[:adapter] = :csv
|
58
|
+
from_file(file, options)
|
59
|
+
end
|
60
|
+
|
61
|
+
def from_hash hash_or_array, options = {}
|
62
|
+
p = Processor.new(adapter: :hash, converter: options[:converter])
|
63
|
+
return nil unless p
|
64
|
+
p.adapter.data = hash_or_array
|
65
|
+
process_with_status_bar p, title: "Hash"
|
66
|
+
end
|
67
|
+
|
68
|
+
module_function :from_file
|
69
|
+
module_function :from_hash
|
70
|
+
module_function :from_csv
|
71
|
+
module_function :process_with_status_bar
|
72
|
+
end
|
data/lib/rseed.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
require "csv"
|
2
|
+
require "rseed/attribute"
|
3
|
+
require "rseed/attribute_converters"
|
4
|
+
require "rseed/version"
|
5
|
+
require "rseed/adapter"
|
6
|
+
require "rseed/hash_adapter"
|
7
|
+
require "rseed/converter"
|
8
|
+
require "rseed/processor"
|
9
|
+
require "rseed/csv_adapter"
|
10
|
+
require "rseed/utilities"
|
11
|
+
|
12
|
+
module Rseed
|
13
|
+
class << self
|
14
|
+
attr_accessor :logger
|
15
|
+
end
|
16
|
+
|
17
|
+
@logger = Logger.new(STDOUT)
|
18
|
+
|
19
|
+
class Railtie < ::Rails::Railtie
|
20
|
+
railtie_name :rseed
|
21
|
+
|
22
|
+
rake_tasks do
|
23
|
+
load "tasks/rseed.rake"
|
24
|
+
end
|
25
|
+
|
26
|
+
generators do
|
27
|
+
require "generators/rseed/converter"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/rseed.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'rseed/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "rseed"
|
8
|
+
spec.version = Rseed::VERSION
|
9
|
+
spec.authors = ["David Monagle"]
|
10
|
+
spec.email = ["david.monagle@intrica.com.au"]
|
11
|
+
spec.description = ""
|
12
|
+
spec.summary = %q{Assist with seeding/import of external data into models.}
|
13
|
+
spec.homepage = "http://www.intrica.com.au"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "colorize"
|
22
|
+
spec.add_dependency "ruby-progressbar"
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
25
|
+
spec.add_development_dependency "rake"
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rseed
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Monagle
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-11-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: colorize
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: ruby-progressbar
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ! '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: ''
|
70
|
+
email:
|
71
|
+
- david.monagle@intrica.com.au
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- .gitignore
|
77
|
+
- Gemfile
|
78
|
+
- LICENSE.txt
|
79
|
+
- README.rdoc
|
80
|
+
- Rakefile
|
81
|
+
- lib/generators/rseed/converter.rb
|
82
|
+
- lib/generators/rseed/templates/converter.rb.erb
|
83
|
+
- lib/generators/rseed/templates/data.csv.erb
|
84
|
+
- lib/rseed.rb
|
85
|
+
- lib/rseed/adapter.rb
|
86
|
+
- lib/rseed/attribute.rb
|
87
|
+
- lib/rseed/attribute_converters.rb
|
88
|
+
- lib/rseed/converter.rb
|
89
|
+
- lib/rseed/csv_adapter.rb
|
90
|
+
- lib/rseed/hash_adapter.rb
|
91
|
+
- lib/rseed/processor.rb
|
92
|
+
- lib/rseed/utilities.rb
|
93
|
+
- lib/rseed/version.rb
|
94
|
+
- lib/tasks/rseed.rake
|
95
|
+
- rseed.gemspec
|
96
|
+
homepage: http://www.intrica.com.au
|
97
|
+
licenses:
|
98
|
+
- MIT
|
99
|
+
metadata: {}
|
100
|
+
post_install_message:
|
101
|
+
rdoc_options: []
|
102
|
+
require_paths:
|
103
|
+
- lib
|
104
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ! '>='
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
110
|
+
requirements:
|
111
|
+
- - ! '>='
|
112
|
+
- !ruby/object:Gem::Version
|
113
|
+
version: '0'
|
114
|
+
requirements: []
|
115
|
+
rubyforge_project:
|
116
|
+
rubygems_version: 2.1.9
|
117
|
+
signing_key:
|
118
|
+
specification_version: 4
|
119
|
+
summary: Assist with seeding/import of external data into models.
|
120
|
+
test_files: []
|