csv_seed 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +8 -0
- data/lib/csv_seed/importer.rb +67 -0
- data/lib/csv_seed/record.rb +73 -0
- data/lib/csv_seed/table.rb +110 -0
- data/lib/csv_seed/uploader.rb +8 -0
- data/lib/csv_seed/version.rb +3 -0
- data/lib/csv_seed.rb +12 -0
- metadata +78 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 61ca14d9500c412955f6f5c8553ade681bcaec17
|
4
|
+
data.tar.gz: a105521118230c11eea2c66be1f788fc63c950bb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c00d8d9934a4af33932d161346de46b88aa6372057ccc476935a9dd14f556343e3e17b95760412d200293e69dd9ef6831871026f7ec1ccb9ecaa6ebe148de3ac
|
7
|
+
data.tar.gz: 681a45c8f395f312159116a2fcbc231ae6e1ad4e143ffaa4554f70cfdc325ea1d33446a093a7be53b1ba4f9001792101237d34be1b0efe8bd2b9728b6e3681eb
|
data/README.md
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
module CsvSeed
|
2
|
+
class Import
|
3
|
+
attr_accessor :name, :tables
|
4
|
+
|
5
|
+
def initialize(name)
|
6
|
+
@name = (name || '')
|
7
|
+
end
|
8
|
+
|
9
|
+
# thor csv:seed --use points --except Activity ActivityForm
|
10
|
+
def run(only, except)
|
11
|
+
# TODO 检查参数
|
12
|
+
@tables = read_csv
|
13
|
+
say_end = []
|
14
|
+
models = only.present? ? only : (@tables.keys - (except or []))
|
15
|
+
run_tables = @tables.values.select {|t| models.include? t.model}
|
16
|
+
# @tables.keep_if {|k| models.include? k}
|
17
|
+
puts "\n\n*** run_tables: #{run_tables.map(&:model).join(', ')}\n\n\n"
|
18
|
+
ActiveRecord::Base.transaction do
|
19
|
+
run_tables.each do |table|
|
20
|
+
say_end += table.execute_commands("destroy_all")
|
21
|
+
end
|
22
|
+
run_tables.each do |table|
|
23
|
+
say_end += table.execute_commands
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
puts "\n"*3
|
28
|
+
say_end.each {|say| puts say}
|
29
|
+
end
|
30
|
+
|
31
|
+
def read_csv()
|
32
|
+
tables = {}
|
33
|
+
current_model = ""
|
34
|
+
Dir.glob(Rails.root.join("db","seeds", @name, "*.csv")).each do |csv|
|
35
|
+
lines = CSV.read(csv)
|
36
|
+
lines.each do |line|
|
37
|
+
next if line[0].nil?
|
38
|
+
|
39
|
+
if line[0][0] == "#"
|
40
|
+
cmd, *args = *line[0][1..-1].split('#')
|
41
|
+
case cmd
|
42
|
+
when "model"
|
43
|
+
current_model = args.first
|
44
|
+
tables[current_model] = Table.new(current_model, tables)
|
45
|
+
when "command"
|
46
|
+
tables[current_model].commands << {cmd: args.first, args: args[1..-1]}
|
47
|
+
when "key"
|
48
|
+
tables[current_model].key = args.first
|
49
|
+
when "uploader"
|
50
|
+
tables[current_model].upload_field_names = args
|
51
|
+
end
|
52
|
+
next
|
53
|
+
end
|
54
|
+
|
55
|
+
# Imported tables should have the 'id' attribute.
|
56
|
+
if line[0] == 'id'
|
57
|
+
tables[current_model].field_names = line.compact[1..-1]
|
58
|
+
next
|
59
|
+
end
|
60
|
+
# 根据字段表长来截,compact会吞掉内容为空的字段。注意这儿有id。
|
61
|
+
tables[current_model].add_record line[0..tables[current_model].field_names.size]
|
62
|
+
end
|
63
|
+
end
|
64
|
+
tables
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
module CsvSeed
|
2
|
+
class Record
|
3
|
+
attr_accessor :table, :id, :real_id, :content
|
4
|
+
|
5
|
+
def initialize(table, attributes_line)
|
6
|
+
@table = table
|
7
|
+
@id = attributes_line[0]
|
8
|
+
@content = Hash[@table.field_names.zip attributes_line[1..-1]]
|
9
|
+
end
|
10
|
+
|
11
|
+
def insert
|
12
|
+
return if @real_id.present?
|
13
|
+
|
14
|
+
# 先去掉外键和上传字段,外键后面统一处理
|
15
|
+
# @table.foreign_keys.keys +
|
16
|
+
params = @content.except *(@table.upload_field_names)
|
17
|
+
# 如果有enum字段,事先要先转为integer
|
18
|
+
params = do_type_cast(params) if @table.table_class.defined_enums.present?
|
19
|
+
r = @table.table_class.new params
|
20
|
+
@table.upload_field_names.each do |field|
|
21
|
+
File.open(Uploader.file_path(@content[field])) do |f|
|
22
|
+
# r[field] = f
|
23
|
+
# 上面这句无论如何不行,没有错也没上传。这是直接赋值,但是 carrierwave 应该改了这个赋值方法。
|
24
|
+
r.send field+'=', f
|
25
|
+
end
|
26
|
+
end
|
27
|
+
print "*** insert model: #{@table.model}, params: #{params}"
|
28
|
+
r.save!
|
29
|
+
# byebug if @table.model == 'Gift'
|
30
|
+
@real_id = r.id
|
31
|
+
@table.count += 1
|
32
|
+
puts ", *** real_id: #{@real_id}\n"
|
33
|
+
|
34
|
+
@table.foreign_keys.each do |k, m|
|
35
|
+
pk = @table.primary_keys[k]
|
36
|
+
puts " model: #{@table.model}, foreign_key: #{k} -> #{m}, fk_content: #{@content[k]}, pk: #{pk}"
|
37
|
+
next if @content[k].nil? # 外键为空
|
38
|
+
next if pk != 'id' && pk != 'polymorphic' # 关系表主键不是 'id' 并且不是 polymorphic
|
39
|
+
m = @content[k.gsub('_id', '_type')] if pk == 'polymorphic'
|
40
|
+
f_record = @table.tables[m].find(@content[k])
|
41
|
+
f_record.insert if !f_record.real_id
|
42
|
+
puts "*** linked #{k}: #{f_record.real_id}"
|
43
|
+
r[k] = f_record.real_id
|
44
|
+
end
|
45
|
+
r.save!
|
46
|
+
end
|
47
|
+
|
48
|
+
# #command#update#2#2#code
|
49
|
+
def update(k, keys, excepts)
|
50
|
+
puts "*** [Record#update] params: #{k}, #{keys}, #{excepts}"
|
51
|
+
condition = @content.select {|k,v| keys.split(',').include? k}
|
52
|
+
obj = @table.table_class.where(condition).first
|
53
|
+
if obj
|
54
|
+
content = excepts.present? ? @content.reject {|k,v| excepts.split(',').include? k} : @content
|
55
|
+
obj.update! content
|
56
|
+
puts "*** id: #{k}, obj_id:#{obj.id}, condition: #{condition}, a record of #{@table.model} has been updated.\n"
|
57
|
+
true
|
58
|
+
else
|
59
|
+
false
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
def do_type_cast(params)
|
65
|
+
result = params.clone
|
66
|
+
@table.table_class.defined_enums.keys.each do |field|
|
67
|
+
# nil表示缺省,缺省并不一定是0。nil.to_i == 0
|
68
|
+
result[field] = params[field].to_i if params[field]
|
69
|
+
end
|
70
|
+
result
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
module CsvSeed
|
2
|
+
class Table
|
3
|
+
attr_accessor :model, :field_names, :foreign_keys, :primary_keys, :records, :count, :commands, :tables, :key, :upload_field_names
|
4
|
+
|
5
|
+
def initialize(model, tables)
|
6
|
+
@model = model
|
7
|
+
@count = 0
|
8
|
+
keys = table_class.reflect_on_all_associations(:belongs_to)
|
9
|
+
@foreign_keys = Hash[keys.map {|k| [k.foreign_key, k.class_name]}]
|
10
|
+
# polymorphic 的时候 association_primary_key 会出错。
|
11
|
+
@primary_keys = Hash[keys.map {|k| [k.foreign_key, k.polymorphic? ? 'polymorphic' : k.association_primary_key]}]
|
12
|
+
@tables = tables
|
13
|
+
@commands = []
|
14
|
+
@field_names = []
|
15
|
+
@upload_field_names = []
|
16
|
+
@key = 'id'
|
17
|
+
@key_h = {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def table_class
|
21
|
+
# puts "*** table_class's model: #{@model}"
|
22
|
+
@model.constantize
|
23
|
+
end
|
24
|
+
|
25
|
+
def add_record(attributes_line)
|
26
|
+
@records ||= {}
|
27
|
+
r = Record.new(self, attributes_line)
|
28
|
+
pk = r.id
|
29
|
+
# 如果有key的时候,这张表一般是用作指示的
|
30
|
+
if @key != 'id'
|
31
|
+
r.real_id = table_class.where(get_params_from_key(r)).first.try(:id)
|
32
|
+
puts "*** [Table#add_record] Model: #{@model}, key: #{@key} -- id: #{r.id}, real_id: #{r.real_id}"
|
33
|
+
puts "*** [Table#add_record] sql: #{table_class.where(get_params_from_key(r)).to_sql}"
|
34
|
+
pk = r.content[@key]
|
35
|
+
end
|
36
|
+
@records[r.id] = r
|
37
|
+
@key_h[pk] = r
|
38
|
+
end
|
39
|
+
|
40
|
+
# 根据 id 或 key 来找
|
41
|
+
def find(k)
|
42
|
+
puts "*** [Table#find] Model: #{@model}, k: #{k}"
|
43
|
+
@records[k] || @key_h[k]
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_params_from_key(record)
|
47
|
+
return {id: 0} if @key == 'id'
|
48
|
+
@key.split(',').each_with_object({}) do |attr_name, h|
|
49
|
+
# 如果外键里有 id 的话,按这个找是找不到记录的。除非是 code 才能找到。
|
50
|
+
h[attr_name.to_sym] = record.content[attr_name]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def execute_commands(only_command = nil)
|
55
|
+
say_later = []
|
56
|
+
done = []
|
57
|
+
@commands.each do |item|
|
58
|
+
cmd, args = *item.values
|
59
|
+
next if only_command.present? and only_command != cmd
|
60
|
+
log_args = args.present? ? " args: #{args}" : ''
|
61
|
+
puts "*** #{self.model}.#{cmd}#{log_args}\n"
|
62
|
+
if cmd == 'destroy_all'
|
63
|
+
say_later << "*** #{self.table_class.count} records have been deleted from #{self.model}\n"
|
64
|
+
self.table_class.destroy_all
|
65
|
+
@records.values.each {|r| r.real_id = nil}
|
66
|
+
done << item
|
67
|
+
next
|
68
|
+
elsif cmd == 'create_all'
|
69
|
+
self.records.values.each {|r| r.insert}
|
70
|
+
done << item
|
71
|
+
say_later << "*** #{self.count} records have been imported into model #{self.model}. \n"
|
72
|
+
next
|
73
|
+
end
|
74
|
+
|
75
|
+
if %w(append delete update).include? cmd
|
76
|
+
# keys 是查询条件包含的字段表列表,以逗号分隔。当cmd为append的时候没有
|
77
|
+
ibegin, iend, keys, excepts = args[0].to_i, args[1].to_i, args[2], args[3]
|
78
|
+
if (ibegin > iend || ibegin * iend <= 0)
|
79
|
+
puts "*** #{cmd} args error: #{args}\n"
|
80
|
+
raise "*** Import Error."
|
81
|
+
end
|
82
|
+
|
83
|
+
# #command#append#5#5
|
84
|
+
if cmd == 'append'
|
85
|
+
# byebug
|
86
|
+
(ibegin..iend).each {|k| self.records[k.to_s].insert}
|
87
|
+
done << item
|
88
|
+
say_later << "*** #{iend - ibegin + 1} records have been imported into model #{self.model}. \n"
|
89
|
+
end
|
90
|
+
|
91
|
+
# #command#update#4#4#code
|
92
|
+
if cmd == 'update'
|
93
|
+
count = 0
|
94
|
+
(ibegin..iend).each do |k|
|
95
|
+
count += 1 if self.records[k.to_s].update(k, keys, excepts)
|
96
|
+
end
|
97
|
+
done << item
|
98
|
+
say_later << "*** #{count} records of model #{self.model} have been updated.\n"
|
99
|
+
end
|
100
|
+
next
|
101
|
+
end
|
102
|
+
|
103
|
+
puts "*** cmd:#{cmd} not supported\n"
|
104
|
+
end
|
105
|
+
@commands -= done
|
106
|
+
puts "*** #{self.model}.commands left: #{@commands}\n"
|
107
|
+
say_later
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
data/lib/csv_seed.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'csv_seed/importer'
|
3
|
+
require 'csv_seed/record'
|
4
|
+
require 'csv_seed/table'
|
5
|
+
require 'csv_seed/uploader'
|
6
|
+
require 'csv_seed/version'
|
7
|
+
|
8
|
+
module CsvSeed
|
9
|
+
def self.load_tasks
|
10
|
+
Dir[File.expand_path('../tasks/*.thor', __FILE__)].each { |ext| load ext }
|
11
|
+
end
|
12
|
+
end
|
metadata
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: csv_seed
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- chaofan
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-12-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.14'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.14'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: thor-rails
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.0.1
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 0.0.1
|
41
|
+
description: Import csv data to projects which are dependant on ActiveRecord.
|
42
|
+
email: jiangchaofan@gmail.com
|
43
|
+
executables: []
|
44
|
+
extensions: []
|
45
|
+
extra_rdoc_files: []
|
46
|
+
files:
|
47
|
+
- README.md
|
48
|
+
- lib/csv_seed.rb
|
49
|
+
- lib/csv_seed/importer.rb
|
50
|
+
- lib/csv_seed/record.rb
|
51
|
+
- lib/csv_seed/table.rb
|
52
|
+
- lib/csv_seed/uploader.rb
|
53
|
+
- lib/csv_seed/version.rb
|
54
|
+
homepage: https://github.com/chaofan/csv_seed
|
55
|
+
licenses:
|
56
|
+
- MIT
|
57
|
+
metadata: {}
|
58
|
+
post_install_message:
|
59
|
+
rdoc_options: []
|
60
|
+
require_paths:
|
61
|
+
- lib
|
62
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
67
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: '0'
|
72
|
+
requirements: []
|
73
|
+
rubyforge_project:
|
74
|
+
rubygems_version: 2.5.1
|
75
|
+
signing_key:
|
76
|
+
specification_version: 4
|
77
|
+
summary: Import csv tables to rails
|
78
|
+
test_files: []
|