csv_seed 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 61ca14d9500c412955f6f5c8553ade681bcaec17
4
+ data.tar.gz: a105521118230c11eea2c66be1f788fc63c950bb
5
+ SHA512:
6
+ metadata.gz: c00d8d9934a4af33932d161346de46b88aa6372057ccc476935a9dd14f556343e3e17b95760412d200293e69dd9ef6831871026f7ec1ccb9ecaa6ebe148de3ac
7
+ data.tar.gz: 681a45c8f395f312159116a2fcbc231ae6e1ad4e143ffaa4554f70cfdc325ea1d33446a093a7be53b1ba4f9001792101237d34be1b0efe8bd2b9728b6e3681eb
data/README.md ADDED
@@ -0,0 +1,8 @@
1
+ # csv_seed
2
+ Import csv data to projects which are dependant on ActiveRecord.
3
+
4
+ Why a new seed gem?
5
+ - Csv is easier to read and update than fixtures or seeds.
6
+ - Associations in csv tables can be applied to real databases.
7
+
8
+ ...
@@ -0,0 +1,67 @@
1
+ module CsvSeed
2
+ class Import
3
+ attr_accessor :name, :tables
4
+
5
+ def initialize(name)
6
+ @name = (name || '')
7
+ end
8
+
9
+ # thor csv:seed --use points --except Activity ActivityForm
10
+ def run(only, except)
11
+ # TODO 检查参数
12
+ @tables = read_csv
13
+ say_end = []
14
+ models = only.present? ? only : (@tables.keys - (except or []))
15
+ run_tables = @tables.values.select {|t| models.include? t.model}
16
+ # @tables.keep_if {|k| models.include? k}
17
+ puts "\n\n*** run_tables: #{run_tables.map(&:model).join(', ')}\n\n\n"
18
+ ActiveRecord::Base.transaction do
19
+ run_tables.each do |table|
20
+ say_end += table.execute_commands("destroy_all")
21
+ end
22
+ run_tables.each do |table|
23
+ say_end += table.execute_commands
24
+ end
25
+ end
26
+
27
+ puts "\n"*3
28
+ say_end.each {|say| puts say}
29
+ end
30
+
31
+ def read_csv()
32
+ tables = {}
33
+ current_model = ""
34
+ Dir.glob(Rails.root.join("db","seeds", @name, "*.csv")).each do |csv|
35
+ lines = CSV.read(csv)
36
+ lines.each do |line|
37
+ next if line[0].nil?
38
+
39
+ if line[0][0] == "#"
40
+ cmd, *args = *line[0][1..-1].split('#')
41
+ case cmd
42
+ when "model"
43
+ current_model = args.first
44
+ tables[current_model] = Table.new(current_model, tables)
45
+ when "command"
46
+ tables[current_model].commands << {cmd: args.first, args: args[1..-1]}
47
+ when "key"
48
+ tables[current_model].key = args.first
49
+ when "uploader"
50
+ tables[current_model].upload_field_names = args
51
+ end
52
+ next
53
+ end
54
+
55
+ # Imported tables should have the 'id' attribute.
56
+ if line[0] == 'id'
57
+ tables[current_model].field_names = line.compact[1..-1]
58
+ next
59
+ end
60
+ # 根据字段表长来截,compact会吞掉内容为空的字段。注意这儿有id。
61
+ tables[current_model].add_record line[0..tables[current_model].field_names.size]
62
+ end
63
+ end
64
+ tables
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,73 @@
1
+ module CsvSeed
2
+ class Record
3
+ attr_accessor :table, :id, :real_id, :content
4
+
5
+ def initialize(table, attributes_line)
6
+ @table = table
7
+ @id = attributes_line[0]
8
+ @content = Hash[@table.field_names.zip attributes_line[1..-1]]
9
+ end
10
+
11
+ def insert
12
+ return if @real_id.present?
13
+
14
+ # 先去掉外键和上传字段,外键后面统一处理
15
+ # @table.foreign_keys.keys +
16
+ params = @content.except *(@table.upload_field_names)
17
+ # 如果有enum字段,事先要先转为integer
18
+ params = do_type_cast(params) if @table.table_class.defined_enums.present?
19
+ r = @table.table_class.new params
20
+ @table.upload_field_names.each do |field|
21
+ File.open(Uploader.file_path(@content[field])) do |f|
22
+ # r[field] = f
23
+ # 上面这句无论如何不行,没有错也没上传。这是直接赋值,但是 carrierwave 应该改了这个赋值方法。
24
+ r.send field+'=', f
25
+ end
26
+ end
27
+ print "*** insert model: #{@table.model}, params: #{params}"
28
+ r.save!
29
+ # byebug if @table.model == 'Gift'
30
+ @real_id = r.id
31
+ @table.count += 1
32
+ puts ", *** real_id: #{@real_id}\n"
33
+
34
+ @table.foreign_keys.each do |k, m|
35
+ pk = @table.primary_keys[k]
36
+ puts " model: #{@table.model}, foreign_key: #{k} -> #{m}, fk_content: #{@content[k]}, pk: #{pk}"
37
+ next if @content[k].nil? # 外键为空
38
+ next if pk != 'id' && pk != 'polymorphic' # 关系表主键不是 'id' 并且不是 polymorphic
39
+ m = @content[k.gsub('_id', '_type')] if pk == 'polymorphic'
40
+ f_record = @table.tables[m].find(@content[k])
41
+ f_record.insert if !f_record.real_id
42
+ puts "*** linked #{k}: #{f_record.real_id}"
43
+ r[k] = f_record.real_id
44
+ end
45
+ r.save!
46
+ end
47
+
48
+ # #command#update#2#2#code
49
+ def update(k, keys, excepts)
50
+ puts "*** [Record#update] params: #{k}, #{keys}, #{excepts}"
51
+ condition = @content.select {|k,v| keys.split(',').include? k}
52
+ obj = @table.table_class.where(condition).first
53
+ if obj
54
+ content = excepts.present? ? @content.reject {|k,v| excepts.split(',').include? k} : @content
55
+ obj.update! content
56
+ puts "*** id: #{k}, obj_id:#{obj.id}, condition: #{condition}, a record of #{@table.model} has been updated.\n"
57
+ true
58
+ else
59
+ false
60
+ end
61
+ end
62
+
63
+ private
64
+ def do_type_cast(params)
65
+ result = params.clone
66
+ @table.table_class.defined_enums.keys.each do |field|
67
+ # nil表示缺省,缺省并不一定是0。nil.to_i == 0
68
+ result[field] = params[field].to_i if params[field]
69
+ end
70
+ result
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,110 @@
1
+ module CsvSeed
2
+ class Table
3
+ attr_accessor :model, :field_names, :foreign_keys, :primary_keys, :records, :count, :commands, :tables, :key, :upload_field_names
4
+
5
+ def initialize(model, tables)
6
+ @model = model
7
+ @count = 0
8
+ keys = table_class.reflect_on_all_associations(:belongs_to)
9
+ @foreign_keys = Hash[keys.map {|k| [k.foreign_key, k.class_name]}]
10
+ # polymorphic 的时候 association_primary_key 会出错。
11
+ @primary_keys = Hash[keys.map {|k| [k.foreign_key, k.polymorphic? ? 'polymorphic' : k.association_primary_key]}]
12
+ @tables = tables
13
+ @commands = []
14
+ @field_names = []
15
+ @upload_field_names = []
16
+ @key = 'id'
17
+ @key_h = {}
18
+ end
19
+
20
+ def table_class
21
+ # puts "*** table_class's model: #{@model}"
22
+ @model.constantize
23
+ end
24
+
25
+ def add_record(attributes_line)
26
+ @records ||= {}
27
+ r = Record.new(self, attributes_line)
28
+ pk = r.id
29
+ # 如果有key的时候,这张表一般是用作指示的
30
+ if @key != 'id'
31
+ r.real_id = table_class.where(get_params_from_key(r)).first.try(:id)
32
+ puts "*** [Table#add_record] Model: #{@model}, key: #{@key} -- id: #{r.id}, real_id: #{r.real_id}"
33
+ puts "*** [Table#add_record] sql: #{table_class.where(get_params_from_key(r)).to_sql}"
34
+ pk = r.content[@key]
35
+ end
36
+ @records[r.id] = r
37
+ @key_h[pk] = r
38
+ end
39
+
40
+ # 根据 id 或 key 来找
41
+ def find(k)
42
+ puts "*** [Table#find] Model: #{@model}, k: #{k}"
43
+ @records[k] || @key_h[k]
44
+ end
45
+
46
+ def get_params_from_key(record)
47
+ return {id: 0} if @key == 'id'
48
+ @key.split(',').each_with_object({}) do |attr_name, h|
49
+ # 如果外键里有 id 的话,按这个找是找不到记录的。除非是 code 才能找到。
50
+ h[attr_name.to_sym] = record.content[attr_name]
51
+ end
52
+ end
53
+
54
+ def execute_commands(only_command = nil)
55
+ say_later = []
56
+ done = []
57
+ @commands.each do |item|
58
+ cmd, args = *item.values
59
+ next if only_command.present? and only_command != cmd
60
+ log_args = args.present? ? " args: #{args}" : ''
61
+ puts "*** #{self.model}.#{cmd}#{log_args}\n"
62
+ if cmd == 'destroy_all'
63
+ say_later << "*** #{self.table_class.count} records have been deleted from #{self.model}\n"
64
+ self.table_class.destroy_all
65
+ @records.values.each {|r| r.real_id = nil}
66
+ done << item
67
+ next
68
+ elsif cmd == 'create_all'
69
+ self.records.values.each {|r| r.insert}
70
+ done << item
71
+ say_later << "*** #{self.count} records have been imported into model #{self.model}. \n"
72
+ next
73
+ end
74
+
75
+ if %w(append delete update).include? cmd
76
+ # keys 是查询条件包含的字段表列表,以逗号分隔。当cmd为append的时候没有
77
+ ibegin, iend, keys, excepts = args[0].to_i, args[1].to_i, args[2], args[3]
78
+ if (ibegin > iend || ibegin * iend <= 0)
79
+ puts "*** #{cmd} args error: #{args}\n"
80
+ raise "*** Import Error."
81
+ end
82
+
83
+ # #command#append#5#5
84
+ if cmd == 'append'
85
+ # byebug
86
+ (ibegin..iend).each {|k| self.records[k.to_s].insert}
87
+ done << item
88
+ say_later << "*** #{iend - ibegin + 1} records have been imported into model #{self.model}. \n"
89
+ end
90
+
91
+ # #command#update#4#4#code
92
+ if cmd == 'update'
93
+ count = 0
94
+ (ibegin..iend).each do |k|
95
+ count += 1 if self.records[k.to_s].update(k, keys, excepts)
96
+ end
97
+ done << item
98
+ say_later << "*** #{count} records of model #{self.model} have been updated.\n"
99
+ end
100
+ next
101
+ end
102
+
103
+ puts "*** cmd:#{cmd} not supported\n"
104
+ end
105
+ @commands -= done
106
+ puts "*** #{self.model}.commands left: #{@commands}\n"
107
+ say_later
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,8 @@
1
+ # TODO 没有 2 个外键情况的处理,如:id, code
2
+ module CsvSeed
3
+ module Uploader
4
+ def self.file_path(path)
5
+ "#{Rails.root}/db/seed/#{path}"
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,3 @@
1
+ module CsvSeed
2
+ VERSION = '0.0.4'
3
+ end
data/lib/csv_seed.rb ADDED
@@ -0,0 +1,12 @@
1
+ require 'csv'
2
+ require 'csv_seed/importer'
3
+ require 'csv_seed/record'
4
+ require 'csv_seed/table'
5
+ require 'csv_seed/uploader'
6
+ require 'csv_seed/version'
7
+
8
+ module CsvSeed
9
+ def self.load_tasks
10
+ Dir[File.expand_path('../tasks/*.thor', __FILE__)].each { |ext| load ext }
11
+ end
12
+ end
metadata ADDED
@@ -0,0 +1,78 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: csv_seed
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.4
5
+ platform: ruby
6
+ authors:
7
+ - chaofan
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-12-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: thor
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.14'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.14'
27
+ - !ruby/object:Gem::Dependency
28
+ name: thor-rails
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: 0.0.1
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: 0.0.1
41
+ description: Import csv data to projects which are dependant on ActiveRecord.
42
+ email: jiangchaofan@gmail.com
43
+ executables: []
44
+ extensions: []
45
+ extra_rdoc_files: []
46
+ files:
47
+ - README.md
48
+ - lib/csv_seed.rb
49
+ - lib/csv_seed/importer.rb
50
+ - lib/csv_seed/record.rb
51
+ - lib/csv_seed/table.rb
52
+ - lib/csv_seed/uploader.rb
53
+ - lib/csv_seed/version.rb
54
+ homepage: https://github.com/chaofan/csv_seed
55
+ licenses:
56
+ - MIT
57
+ metadata: {}
58
+ post_install_message:
59
+ rdoc_options: []
60
+ require_paths:
61
+ - lib
62
+ required_ruby_version: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ required_rubygems_version: !ruby/object:Gem::Requirement
68
+ requirements:
69
+ - - ">="
70
+ - !ruby/object:Gem::Version
71
+ version: '0'
72
+ requirements: []
73
+ rubyforge_project:
74
+ rubygems_version: 2.5.1
75
+ signing_key:
76
+ specification_version: 4
77
+ summary: Import csv tables to rails
78
+ test_files: []