csv_mapper 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +8 -0
- data/.rspec +2 -0
- data/Gemfile +15 -0
- data/MIT-LICENSE +20 -0
- data/README.rdoc +54 -0
- data/Rakefile +26 -0
- data/app/views/controller_actions/import.html.erb +11 -0
- data/app/views/controller_actions/mapper.html.erb +40 -0
- data/csv_mapper.gemspec +21 -0
- data/lib/csv_mapper.rb +37 -0
- data/lib/csv_mapper/controller_actions.rb +73 -0
- data/lib/csv_mapper/engine.rb +5 -0
- data/lib/csv_mapper/file_handler.rb +33 -0
- data/lib/csv_mapper/importer.rb +28 -0
- data/lib/csv_mapper/reader.rb +38 -0
- data/lib/csv_mapper/version.rb +3 -0
- data/spec/dummy/Rakefile +7 -0
- data/spec/dummy/app/controllers/application_controller.rb +3 -0
- data/spec/dummy/app/controllers/people_controller.rb +24 -0
- data/spec/dummy/app/helpers/application_helper.rb +2 -0
- data/spec/dummy/app/models/person.rb +2 -0
- data/spec/dummy/app/views/layouts/application.html.erb +18 -0
- data/spec/dummy/app/views/people/index.erb +21 -0
- data/spec/dummy/config.ru +4 -0
- data/spec/dummy/config/application.rb +45 -0
- data/spec/dummy/config/boot.rb +10 -0
- data/spec/dummy/config/database.yml +22 -0
- data/spec/dummy/config/environment.rb +5 -0
- data/spec/dummy/config/environments/development.rb +26 -0
- data/spec/dummy/config/environments/production.rb +49 -0
- data/spec/dummy/config/environments/test.rb +35 -0
- data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/spec/dummy/config/initializers/inflections.rb +10 -0
- data/spec/dummy/config/initializers/mime_types.rb +5 -0
- data/spec/dummy/config/initializers/secret_token.rb +7 -0
- data/spec/dummy/config/initializers/session_store.rb +8 -0
- data/spec/dummy/config/locales/en.yml +5 -0
- data/spec/dummy/config/routes.rb +6 -0
- data/spec/dummy/public/404.html +26 -0
- data/spec/dummy/public/422.html +26 -0
- data/spec/dummy/public/500.html +26 -0
- data/spec/dummy/public/favicon.ico +0 -0
- data/spec/dummy/public/javascripts/application.js +2 -0
- data/spec/dummy/public/javascripts/controls.js +965 -0
- data/spec/dummy/public/javascripts/dragdrop.js +974 -0
- data/spec/dummy/public/javascripts/effects.js +1123 -0
- data/spec/dummy/public/javascripts/prototype.js +6001 -0
- data/spec/dummy/public/javascripts/rails.js +191 -0
- data/spec/dummy/public/stylesheets/.gitkeep +0 -0
- data/spec/dummy/script/rails +6 -0
- data/spec/dummy_data.csv +3 -0
- data/spec/file_handler_spec.rb +52 -0
- data/spec/importer_spec.rb +9 -0
- data/spec/integration/get_request_spec.rb +1 -0
- data/spec/integration/post_request_spec.rb +13 -0
- data/spec/integration/resource_integration_spec.rb +23 -0
- data/spec/spec_helper.rb +33 -0
- data/spec/test.file +0 -0
- metadata +189 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in csv_mapper.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
group(:development, :test) do
|
7
|
+
gem 'ruby-debug'
|
8
|
+
gem 'rails', '3.0.9'
|
9
|
+
gem 'sqlite3'
|
10
|
+
end
|
11
|
+
|
12
|
+
group(:test) do
|
13
|
+
gem 'rspec-rails', '>= 2.0.0.beta'
|
14
|
+
gem 'capybara', '>= 0.4.0'
|
15
|
+
end
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright 2011 YOURNAME
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
== Description
|
2
|
+
|
3
|
+
This gem adds an import-action in every controller you like to be able to handle csv-Imports.
|
4
|
+
It includes the complete workflow: Csv-Upload, auto-matching and review, importing data into model.
|
5
|
+
|
6
|
+
== Requirements
|
7
|
+
|
8
|
+
Rails 3 & Ruby 1.8.7
|
9
|
+
|
10
|
+
In principle it will also work with Ruby 1.9, whereas changes has to be made as FasterCSV is already integrated named as CSV.
|
11
|
+
|
12
|
+
== Installation
|
13
|
+
|
14
|
+
Gemfile:
|
15
|
+
|
16
|
+
gem 'csv_mapper', :git => git://github.com/masche842/csv_mapper.git
|
17
|
+
|
18
|
+
Set up a new route to point to the import action (get & post will be needed!):
|
19
|
+
|
20
|
+
routes.rb
|
21
|
+
|
22
|
+
resources :myresource do
|
23
|
+
get 'import', :on => :collection
|
24
|
+
post 'import', :on => :collection
|
25
|
+
end
|
26
|
+
|
27
|
+
Include it in your Controller:
|
28
|
+
|
29
|
+
require 'csv_mapper'
|
30
|
+
include CsvMapper::ControllerActions
|
31
|
+
|
32
|
+
Set up the fields to map to:
|
33
|
+
|
34
|
+
csv_mapper_config(
|
35
|
+
:mapping => {
|
36
|
+
"Firstname" => :firstname,
|
37
|
+
"Lastname" => :lastname
|
38
|
+
}
|
39
|
+
)
|
40
|
+
|
41
|
+
See also the implementation in spec/dummy!
|
42
|
+
|
43
|
+
== Usage
|
44
|
+
|
45
|
+
call /myresources/import!
|
46
|
+
|
47
|
+
== Thanks
|
48
|
+
|
49
|
+
This gem is heavily based on Andrew Timberlake's map-fields-gem (http://github.com/internuity/map-fields).
|
50
|
+
Nevertheless I didn't fork it, because the changes are fundamental.
|
51
|
+
|
52
|
+
== License
|
53
|
+
|
54
|
+
MIT License. Copyright 2011 magiclabs* (magiclabs.de)
|
data/Rakefile
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
#encoding: UTF-8
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
begin
|
5
|
+
require 'bundler/setup'
|
6
|
+
rescue LoadError
|
7
|
+
puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
|
8
|
+
end
|
9
|
+
|
10
|
+
require 'rake'
|
11
|
+
require 'rake/rdoctask'
|
12
|
+
|
13
|
+
require 'rspec/core'
|
14
|
+
require 'rspec/core/rake_task'
|
15
|
+
|
16
|
+
RSpec::Core::RakeTask.new(:spec)
|
17
|
+
|
18
|
+
task :default => :spec
|
19
|
+
|
20
|
+
Rake::RDocTask.new(:rdoc) do |rdoc|
|
21
|
+
rdoc.rdoc_dir = 'rdoc'
|
22
|
+
rdoc.title = 'CsvMapperTest'
|
23
|
+
rdoc.options << '--line-numbers' << '--inline-source'
|
24
|
+
rdoc.rdoc_files.include('README.rdoc')
|
25
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
26
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
<h1>CSV importieren</h1>
|
2
|
+
|
3
|
+
<p>Bitte Felder mappen</p>
|
4
|
+
|
5
|
+
<%= form_tag nil, :id => 'map_fields_form', :method => :post do -%>
|
6
|
+
<%= hidden_field_tag :filename, @mapper.filename %>
|
7
|
+
<div class="map_fields">
|
8
|
+
<table cellspacing="0">
|
9
|
+
<thead>
|
10
|
+
<tr>
|
11
|
+
<%- header = @raw_data.first -%>
|
12
|
+
<%- header.size.times do |column_index| -%>
|
13
|
+
<th>
|
14
|
+
<%= select_tag("fields[#{column_index + 1}]", options_for_select(
|
15
|
+
@mapper.map_fields,
|
16
|
+
@mapper.map_fields[header[column_index]]
|
17
|
+
), :include_blank => true, :class => 'field_options') %>
|
18
|
+
</th>
|
19
|
+
<%- end -%>
|
20
|
+
</tr>
|
21
|
+
</thead>
|
22
|
+
<tbody>
|
23
|
+
<%- @raw_data.each do |row| -%>
|
24
|
+
<tr>
|
25
|
+
<%- row.each do |column| -%>
|
26
|
+
<td><%= h(column) -%></td>
|
27
|
+
<%- end -%>
|
28
|
+
</tr>
|
29
|
+
<%- end -%>
|
30
|
+
</tbody>
|
31
|
+
</table>
|
32
|
+
</div>
|
33
|
+
<div class="option">
|
34
|
+
<%= check_box_tag 'ignore_first_row', '1', true, :id => 'ignore_first_row_option' %>
|
35
|
+
<label for="ignore_first_row_option">Ignore the first row (headings)</label>
|
36
|
+
</div>
|
37
|
+
<div class="action">
|
38
|
+
<%= submit_tag 'Import' %>
|
39
|
+
</div>
|
40
|
+
<%- end -%>
|
data/csv_mapper.gemspec
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "csv_mapper/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "csv_mapper"
|
7
|
+
s.version = CsvMapper::VERSION
|
8
|
+
s.authors = ["Marc"]
|
9
|
+
s.email = ["marc@magiclabs.de"]
|
10
|
+
s.homepage = "http://magiclabs.de"
|
11
|
+
s.summary = %q{Provides a controller action and views for uploading, mapping and importing data from a csv}
|
12
|
+
|
13
|
+
s.rubyforge_project = "csv_mapper"
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.add_dependency("fastercsv")
|
21
|
+
end
|
data/lib/csv_mapper.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require "csv_mapper/version"
|
2
|
+
require 'csv_mapper/controller_actions'
|
3
|
+
require 'csv_mapper/engine'
|
4
|
+
require 'csv_mapper/importer'
|
5
|
+
require 'csv_mapper/reader'
|
6
|
+
require 'csv_mapper/file_handler'
|
7
|
+
|
8
|
+
module CsvMapper
|
9
|
+
|
10
|
+
class InconsistentStateError < StandardError
|
11
|
+
end
|
12
|
+
|
13
|
+
class MissingFileContentsError < StandardError
|
14
|
+
end
|
15
|
+
|
16
|
+
@@options = {
|
17
|
+
:col_sep => ";",
|
18
|
+
:row_sep => :auto,
|
19
|
+
:quote_char => '"',
|
20
|
+
:converters => nil,
|
21
|
+
:unconverted_fields => nil,
|
22
|
+
:headers => false,
|
23
|
+
:return_headers => false,
|
24
|
+
:header_converters => nil,
|
25
|
+
:skip_blanks => false,
|
26
|
+
:force_quotes => false
|
27
|
+
}.freeze
|
28
|
+
|
29
|
+
def self.options
|
30
|
+
@@options
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.options=(options)
|
34
|
+
@@options = options
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require 'fastercsv'
|
2
|
+
|
3
|
+
module CsvMapper
|
4
|
+
module ControllerActions
|
5
|
+
|
6
|
+
def self.included(base)
|
7
|
+
base.extend(ClassMethods)
|
8
|
+
base.csv_mapper_config
|
9
|
+
end
|
10
|
+
|
11
|
+
module ClassMethods
|
12
|
+
def csv_mapper_config( options = {} )
|
13
|
+
defaults = {
|
14
|
+
:action => :import,
|
15
|
+
:mapping => {},
|
16
|
+
:file_field => :file
|
17
|
+
}
|
18
|
+
options = defaults.merge(options)
|
19
|
+
write_inheritable_attribute(:map_fields_options, options)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def import
|
24
|
+
resource_name = self.class.name.gsub(/Controller/, '').singularize
|
25
|
+
resource_class = resource_name.constantize
|
26
|
+
if request.post?
|
27
|
+
# already mapped
|
28
|
+
if params[:fields]
|
29
|
+
create_resource_items_from_csv(resource_class)
|
30
|
+
if @csv_import_errors.empty?
|
31
|
+
flash[:notice] = 'Daten erfolgreich importiert!'
|
32
|
+
redirect_to :action => :index
|
33
|
+
else
|
34
|
+
flash[:warning] = @csv_import_errors
|
35
|
+
redirect_to :back
|
36
|
+
end
|
37
|
+
#no mapping yet
|
38
|
+
else
|
39
|
+
@mapper = CsvMapper::Importer.new(params, self.class.read_inheritable_attribute(:map_fields_options))
|
40
|
+
@raw_data = @mapper.raw_data
|
41
|
+
render 'controller_actions/mapper'
|
42
|
+
end
|
43
|
+
else
|
44
|
+
render 'controller_actions/import'
|
45
|
+
end
|
46
|
+
rescue CsvMapper::InconsistentStateError
|
47
|
+
flash[:warning] = 'unbekannter Fehler.'
|
48
|
+
rescue CsvMapper::MissingFileContentsError
|
49
|
+
flash[:warning] = 'Bitte eine CSV-Datei hochladen.'
|
50
|
+
render 'controller_actions/import'
|
51
|
+
rescue FasterCSV::MalformedCSVError => e
|
52
|
+
flash[:warning] = 'Fehlerhaft formatierte CSV-Datei: ' + e
|
53
|
+
render 'controller_actions/import'
|
54
|
+
rescue Errno::ENOENT
|
55
|
+
flash[:warning] = 'Datei nicht mehr auf dem Server. Bitte erneut hochladen!'
|
56
|
+
render 'controller_actions/import'
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
def create_resource_items_from_csv(resource_class)
|
61
|
+
@csv_import_errors = []
|
62
|
+
reader = CsvMapper::Reader.new(params)
|
63
|
+
reader.each do |row|
|
64
|
+
resource = resource_class.new(row)
|
65
|
+
unless resource.save
|
66
|
+
@csv_import_errors.push resource.errors
|
67
|
+
end
|
68
|
+
end
|
69
|
+
reader.remove_file if @csv_import_errors.empty?
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
class CsvMapper::FileHandler
|
2
|
+
|
3
|
+
attr_reader :path, :filename
|
4
|
+
|
5
|
+
def initialize()
|
6
|
+
@path = File.join(Rails.root.to_s, "tmp")
|
7
|
+
end
|
8
|
+
|
9
|
+
def save_temp_file(tempfile)
|
10
|
+
@filename = unique_filename
|
11
|
+
FileUtils.copy_file(tempfile.path, file_path)
|
12
|
+
File.exists?(file_path)
|
13
|
+
end
|
14
|
+
|
15
|
+
def load_file(filename)
|
16
|
+
@filename = filename
|
17
|
+
File.exist?(file_path)
|
18
|
+
end
|
19
|
+
|
20
|
+
def remove_file
|
21
|
+
File.delete(file_path)
|
22
|
+
end
|
23
|
+
|
24
|
+
def file_path
|
25
|
+
File.join(@path, @filename)
|
26
|
+
end
|
27
|
+
|
28
|
+
def unique_filename
|
29
|
+
t = Time.now.strftime("%Y%m%d")
|
30
|
+
"#{t}-#{$$}-#{rand(0x100000000000000).to_s(36)}"
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'fastercsv'
|
2
|
+
|
3
|
+
class CsvMapper::Importer
|
4
|
+
attr_reader :map_fields, :filename
|
5
|
+
|
6
|
+
|
7
|
+
def initialize(params, options)
|
8
|
+
@file_handler = CsvMapper::FileHandler.new()
|
9
|
+
|
10
|
+
if @file_handler.save_temp_file(params[options[:file_field]])
|
11
|
+
set_attributes_from_valid_file(options)
|
12
|
+
else
|
13
|
+
raise CsvMapper::MissingFileContentsError
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def raw_data
|
18
|
+
FasterCSV.read(@file_handler.file_path, CsvMapper.options)
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def set_attributes_from_valid_file(options)
|
24
|
+
@filename = @file_handler.filename
|
25
|
+
@map_fields = options[:mapping]
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'fastercsv'
|
2
|
+
|
3
|
+
class CsvMapper::Reader
|
4
|
+
|
5
|
+
def initialize(params)
|
6
|
+
@file_handler = CsvMapper::FileHandler.new
|
7
|
+
@file_handler.load_file(params[:filename])
|
8
|
+
|
9
|
+
@file_path = @file_handler.file_path
|
10
|
+
@ignore_first_row = params[:ignore_first_row]
|
11
|
+
@mapping = {}
|
12
|
+
params[:fields].each do |k, v|
|
13
|
+
unless v.empty?
|
14
|
+
@mapping[v.downcase.to_sym] = k.to_i - 1
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def each
|
20
|
+
row_number = 1
|
21
|
+
FasterCSV.foreach(@file_path, CsvMapper.options) do |csv_row|
|
22
|
+
unless row_number == 1 && @ignore_first_row
|
23
|
+
row = {}
|
24
|
+
@mapping.each do |k, v|
|
25
|
+
row[k] = csv_row[v]
|
26
|
+
end
|
27
|
+
row.class.send(:define_method, :number) { row_number }
|
28
|
+
yield(row)
|
29
|
+
end
|
30
|
+
row_number += 1
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def remove_file
|
35
|
+
@file_handler.remove_file
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
data/spec/dummy/Rakefile
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
# Add your own tasks in files placed in lib/tasks ending in .rake,
|
2
|
+
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
|
3
|
+
|
4
|
+
require File.expand_path('../config/application', __FILE__)
|
5
|
+
require 'rake'
|
6
|
+
|
7
|
+
Dummy::Application.load_tasks
|