yaml_db 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/lib/tasks/yaml_db_tasks.rake +4 -20
- data/lib/yaml_db.rb +5 -4
- data/lib/yaml_db/csv_db.rb +77 -0
- data/lib/yaml_db/rake_tasks.rb +36 -0
- data/lib/yaml_db/serialization_helper.rb +201 -0
- data/lib/yaml_db/version.rb +1 -1
- metadata +10 -9
- data/init.rb +0 -1
- data/lib/csv_db.rb +0 -78
- data/lib/serialization_helper.rb +0 -196
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 04f954464b2709f0799c94e237e9be6908fae386
|
4
|
+
data.tar.gz: 284dcf617e715dc8fac6359d987c96a6ec3968ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 88e48da7a8e781d54fbcfa6f88d08fb8103f5a39450c4d3226d475edd6a4129882b60fdc4af4c88a1321e9be54bb49f5b701a1be0091013518b22e4dda821bd9
|
7
|
+
data.tar.gz: e20c244eef8ae1d6b20bb63352910d3238fc251d379b6c023d1f13256f1976b32aa5a4b571baa4c1585b2956ff1c728fa89cdcb8c9fdafbae0b9a515ab98b63d
|
data/README.md
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# YamlDb
|
2
2
|
|
3
|
-
YamlDb is a database-independent format for dumping and restoring data. It complements the
|
3
|
+
YamlDb is a database-independent format for dumping and restoring data. It complements the database-independent schema format found in db/schema.rb. The data is saved into db/data.yml.
|
4
4
|
|
5
5
|
This can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps. Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.
|
6
6
|
|
7
|
-
Any database that has an ActiveRecord adapter should work.
|
7
|
+
Any database that has an ActiveRecord adapter should work.
|
8
|
+
|
9
|
+
This gem supports Rails 3.x, 4.x, and 5.0.
|
8
10
|
|
9
11
|
[](https://travis-ci.org/yamldb/yaml_db)
|
10
12
|
|
@@ -42,6 +44,4 @@ One common use would be to switch your data from one database backend to another
|
|
42
44
|
|
43
45
|
## Credits
|
44
46
|
|
45
|
-
Created by Orion Henry and Adam Wiggins.
|
46
|
-
|
47
|
-
Patches contributed by Michael Irwin, Tom Locke, and Tim Galeckas.
|
47
|
+
Created by Orion Henry and Adam Wiggins. Major updates by Ricardo Chimal Jr. and Nate Kidwell.
|
@@ -6,40 +6,24 @@ namespace :db do
|
|
6
6
|
task(:load => [ "db:schema:load", "db:data:load" ])
|
7
7
|
|
8
8
|
namespace :data do
|
9
|
-
def db_dump_data_file (extension = "yml")
|
10
|
-
"#{dump_dir}/data.#{extension}"
|
11
|
-
end
|
12
|
-
|
13
|
-
def dump_dir(dir = "")
|
14
|
-
"#{Rails.root}/db#{dir}"
|
15
|
-
end
|
16
|
-
|
17
9
|
desc "Dump contents of database to db/data.extension (defaults to yaml)"
|
18
10
|
task :dump => :environment do
|
19
|
-
|
20
|
-
helper = format_class.constantize
|
21
|
-
SerializationHelper::Base.new(helper).dump db_dump_data_file helper.extension
|
11
|
+
YamlDb::RakeTasks.data_dump_task
|
22
12
|
end
|
23
13
|
|
24
14
|
desc "Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)"
|
25
15
|
task :dump_dir => :environment do
|
26
|
-
|
27
|
-
dir = ENV['dir'] || "#{Time.now.to_s.gsub(/ /, '_')}"
|
28
|
-
SerializationHelper::Base.new(format_class.constantize).dump_to_dir dump_dir("/#{dir}")
|
16
|
+
YamlDb::RakeTasks.data_dump_dir_task
|
29
17
|
end
|
30
18
|
|
31
19
|
desc "Load contents of db/data.extension (defaults to yaml) into database"
|
32
20
|
task :load => :environment do
|
33
|
-
|
34
|
-
helper = format_class.constantize
|
35
|
-
SerializationHelper::Base.new(helper).load(db_dump_data_file helper.extension)
|
21
|
+
YamlDb::RakeTasks.data_load_task
|
36
22
|
end
|
37
23
|
|
38
24
|
desc "Load contents of db/data_dir into database"
|
39
25
|
task :load_dir => :environment do
|
40
|
-
|
41
|
-
format_class = ENV['class'] || "YamlDb::Helper"
|
42
|
-
SerializationHelper::Base.new(format_class.constantize).load_from_dir dump_dir("/#{dir}")
|
26
|
+
YamlDb::RakeTasks.data_load_dir_task
|
43
27
|
end
|
44
28
|
end
|
45
29
|
end
|
data/lib/yaml_db.rb
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'yaml'
|
3
3
|
require 'active_record'
|
4
|
-
require 'serialization_helper'
|
5
4
|
require 'active_support/core_ext/kernel/reporting'
|
6
5
|
require 'rails/railtie'
|
6
|
+
require 'yaml_db/rake_tasks'
|
7
7
|
require 'yaml_db/version'
|
8
|
+
require 'yaml_db/serialization_helper'
|
8
9
|
|
9
10
|
module YamlDb
|
10
11
|
module Helper
|
11
12
|
def self.loader
|
12
|
-
|
13
|
+
Load
|
13
14
|
end
|
14
15
|
|
15
16
|
def self.dumper
|
16
|
-
|
17
|
+
Dump
|
17
18
|
end
|
18
19
|
|
19
20
|
def self.extension
|
@@ -46,7 +47,7 @@ module YamlDb
|
|
46
47
|
|
47
48
|
each_table_page(table) do |records|
|
48
49
|
rows = SerializationHelper::Utils.unhash_records(records.to_a, column_names)
|
49
|
-
io.write(
|
50
|
+
io.write(Utils.chunk_records(rows))
|
50
51
|
end
|
51
52
|
end
|
52
53
|
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module YamlDb
|
2
|
+
module CsvDb
|
3
|
+
module Helper
|
4
|
+
def self.loader
|
5
|
+
Load
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.dumper
|
9
|
+
Dump
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.extension
|
13
|
+
"csv"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class Load < SerializationHelper::Load
|
18
|
+
def self.load_documents(io, truncate = true)
|
19
|
+
tables = {}
|
20
|
+
curr_table = nil
|
21
|
+
io.each do |line|
|
22
|
+
if /BEGIN_CSV_TABLE_DECLARATION(.+)END_CSV_TABLE_DECLARATION/ =~ line
|
23
|
+
curr_table = $1
|
24
|
+
tables[curr_table] = {}
|
25
|
+
else
|
26
|
+
if tables[curr_table]["columns"]
|
27
|
+
tables[curr_table]["records"] << FasterCSV.parse(line)[0]
|
28
|
+
else
|
29
|
+
tables[curr_table]["columns"] = FasterCSV.parse(line)[0]
|
30
|
+
tables[curr_table]["records"] = []
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
tables.each_pair do |table_name, contents|
|
36
|
+
load_table(table_name, contents, truncate)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class Dump < SerializationHelper::Dump
|
42
|
+
|
43
|
+
def self.before_table(io,table)
|
44
|
+
io.write "BEGIN_CSV_TABLE_DECLARATION#{table}END_CSV_TABLE_DECLARATION\n"
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.dump(io)
|
48
|
+
tables.each do |table|
|
49
|
+
before_table(io, table)
|
50
|
+
dump_table(io, table)
|
51
|
+
after_table(io, table)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.after_table(io,table)
|
56
|
+
io.write ""
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.dump_table_columns(io, table)
|
60
|
+
io.write(table_column_names(table).to_csv)
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.dump_table_records(io, table)
|
64
|
+
|
65
|
+
column_names = table_column_names(table)
|
66
|
+
|
67
|
+
each_table_page(table) do |records|
|
68
|
+
rows = SerializationHelper::Utils.unhash_records(records, column_names)
|
69
|
+
records.each do |record|
|
70
|
+
io.write(record.to_csv)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module YamlDb
|
2
|
+
module RakeTasks
|
3
|
+
def self.data_dump_task
|
4
|
+
SerializationHelper::Base.new(helper).dump(db_dump_data_file(helper.extension))
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.data_dump_dir_task
|
8
|
+
dir = ENV['dir'] || "#{Time.now.strftime('%F_%T')}"
|
9
|
+
SerializationHelper::Base.new(helper).dump_to_dir(dump_dir("/#{dir}"))
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.data_load_task
|
13
|
+
SerializationHelper::Base.new(helper).load(db_dump_data_file(helper.extension))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.data_load_dir_task
|
17
|
+
dir = ENV['dir'] || 'base'
|
18
|
+
SerializationHelper::Base.new(helper).load_from_dir(dump_dir("/#{dir}"))
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def self.db_dump_data_file(extension = 'yml')
|
24
|
+
"#{dump_dir}/data.#{extension}"
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.dump_dir(dir = '')
|
28
|
+
"#{Rails.root}/db#{dir}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.helper
|
32
|
+
format_class = ENV['class'] || 'YamlDb::Helper'
|
33
|
+
format_class.constantize
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,201 @@
|
|
1
|
+
module YamlDb
|
2
|
+
module SerializationHelper
|
3
|
+
|
4
|
+
class Base
|
5
|
+
attr_reader :extension
|
6
|
+
|
7
|
+
def initialize(helper)
|
8
|
+
@dumper = helper.dumper
|
9
|
+
@loader = helper.loader
|
10
|
+
@extension = helper.extension
|
11
|
+
end
|
12
|
+
|
13
|
+
def dump(filename)
|
14
|
+
disable_logger
|
15
|
+
File.open(filename, "w") do |file|
|
16
|
+
@dumper.dump(file)
|
17
|
+
end
|
18
|
+
reenable_logger
|
19
|
+
end
|
20
|
+
|
21
|
+
def dump_to_dir(dirname)
|
22
|
+
Dir.mkdir(dirname)
|
23
|
+
tables = @dumper.tables
|
24
|
+
tables.each do |table|
|
25
|
+
File.open("#{dirname}/#{table}.#{@extension}", "w") do |io|
|
26
|
+
@dumper.before_table(io, table)
|
27
|
+
@dumper.dump_table io, table
|
28
|
+
@dumper.after_table(io, table)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def load(filename, truncate = true)
|
34
|
+
disable_logger
|
35
|
+
@loader.load(File.new(filename, "r"), truncate)
|
36
|
+
reenable_logger
|
37
|
+
end
|
38
|
+
|
39
|
+
def load_from_dir(dirname, truncate = true)
|
40
|
+
Dir.entries(dirname).each do |filename|
|
41
|
+
if filename =~ /^[.]/
|
42
|
+
next
|
43
|
+
end
|
44
|
+
@loader.load(File.new("#{dirname}/#{filename}", "r"), truncate)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def disable_logger
|
49
|
+
@@old_logger = ActiveRecord::Base.logger
|
50
|
+
ActiveRecord::Base.logger = nil
|
51
|
+
end
|
52
|
+
|
53
|
+
def reenable_logger
|
54
|
+
ActiveRecord::Base.logger = @@old_logger
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
class Load
|
59
|
+
def self.load(io, truncate = true)
|
60
|
+
ActiveRecord::Base.connection.transaction do
|
61
|
+
load_documents(io, truncate)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.truncate_table(table)
|
66
|
+
begin
|
67
|
+
ActiveRecord::Base.connection.execute("TRUNCATE #{Utils.quote_table(table)}")
|
68
|
+
rescue Exception
|
69
|
+
ActiveRecord::Base.connection.execute("DELETE FROM #{Utils.quote_table(table)}")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.load_table(table, data, truncate = true)
|
74
|
+
column_names = data['columns']
|
75
|
+
if truncate
|
76
|
+
truncate_table(table)
|
77
|
+
end
|
78
|
+
load_records(table, column_names, data['records'])
|
79
|
+
reset_pk_sequence!(table)
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.load_records(table, column_names, records)
|
83
|
+
if column_names.nil?
|
84
|
+
return
|
85
|
+
end
|
86
|
+
columns = column_names.map{|cn| ActiveRecord::Base.connection.columns(table).detect{|c| c.name == cn}}
|
87
|
+
quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',')
|
88
|
+
quoted_table_name = Utils.quote_table(table)
|
89
|
+
records.each do |record|
|
90
|
+
quoted_values = record.zip(columns).map{|c| ActiveRecord::Base.connection.quote(c.first, c.last)}.join(',')
|
91
|
+
ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})")
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.reset_pk_sequence!(table_name)
|
96
|
+
if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
|
97
|
+
ActiveRecord::Base.connection.reset_pk_sequence!(table_name)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
module Utils
|
104
|
+
|
105
|
+
def self.unhash(hash, keys)
|
106
|
+
keys.map { |key| hash[key] }
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.unhash_records(records, keys)
|
110
|
+
records.each_with_index do |record, index|
|
111
|
+
records[index] = unhash(record, keys)
|
112
|
+
end
|
113
|
+
|
114
|
+
records
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.convert_booleans(records, columns)
|
118
|
+
records.each do |record|
|
119
|
+
columns.each do |column|
|
120
|
+
next if is_boolean(record[column])
|
121
|
+
record[column] = convert_boolean(record[column])
|
122
|
+
end
|
123
|
+
end
|
124
|
+
records
|
125
|
+
end
|
126
|
+
|
127
|
+
def self.convert_boolean(value)
|
128
|
+
['t', '1', true, 1].include?(value)
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.boolean_columns(table)
|
132
|
+
columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
|
133
|
+
columns.map { |c| c.name }
|
134
|
+
end
|
135
|
+
|
136
|
+
def self.is_boolean(value)
|
137
|
+
value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
|
138
|
+
end
|
139
|
+
|
140
|
+
def self.quote_table(table)
|
141
|
+
ActiveRecord::Base.connection.quote_table_name(table)
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
|
146
|
+
class Dump
|
147
|
+
def self.before_table(io, table)
|
148
|
+
|
149
|
+
end
|
150
|
+
|
151
|
+
def self.dump(io)
|
152
|
+
tables.each do |table|
|
153
|
+
before_table(io, table)
|
154
|
+
dump_table(io, table)
|
155
|
+
after_table(io, table)
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def self.after_table(io, table)
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
def self.tables
|
164
|
+
ActiveRecord::Base.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }
|
165
|
+
end
|
166
|
+
|
167
|
+
def self.dump_table(io, table)
|
168
|
+
return if table_record_count(table).zero?
|
169
|
+
|
170
|
+
dump_table_columns(io, table)
|
171
|
+
dump_table_records(io, table)
|
172
|
+
end
|
173
|
+
|
174
|
+
def self.table_column_names(table)
|
175
|
+
ActiveRecord::Base.connection.columns(table).map { |c| c.name }
|
176
|
+
end
|
177
|
+
|
178
|
+
|
179
|
+
def self.each_table_page(table, records_per_page=1000)
|
180
|
+
total_count = table_record_count(table)
|
181
|
+
pages = (total_count.to_f / records_per_page).ceil - 1
|
182
|
+
id = table_column_names(table).first
|
183
|
+
boolean_columns = Utils.boolean_columns(table)
|
184
|
+
quoted_table_name = Utils.quote_table(table)
|
185
|
+
|
186
|
+
(0..pages).to_a.each do |page|
|
187
|
+
query = Arel::Table.new(table).order(id).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*'))
|
188
|
+
records = ActiveRecord::Base.connection.select_all(query.to_sql)
|
189
|
+
records = Utils.convert_booleans(records, boolean_columns)
|
190
|
+
yield records
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def self.table_record_count(table)
|
195
|
+
ActiveRecord::Base.connection.select_one("SELECT COUNT(*) FROM #{Utils.quote_table(table)}").values.first.to_i
|
196
|
+
end
|
197
|
+
|
198
|
+
end
|
199
|
+
|
200
|
+
end
|
201
|
+
end
|
data/lib/yaml_db/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yaml_db
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Wiggins
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2016-07-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -20,7 +20,7 @@ dependencies:
|
|
20
20
|
version: '3.0'
|
21
21
|
- - "<"
|
22
22
|
- !ruby/object:Gem::Version
|
23
|
-
version: '
|
23
|
+
version: '5.1'
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -30,7 +30,7 @@ dependencies:
|
|
30
30
|
version: '3.0'
|
31
31
|
- - "<"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '5.1'
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
35
|
name: rake
|
36
36
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,7 +89,7 @@ dependencies:
|
|
89
89
|
version: '1.3'
|
90
90
|
description: |2
|
91
91
|
|
92
|
-
YamlDb is a database-independent format for dumping and restoring data. It complements the
|
92
|
+
YamlDb is a database-independent format for dumping and restoring data. It complements the database-independent schema format found in db/schema.rb. The data is saved into db/data.yml.
|
93
93
|
This can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps. Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.
|
94
94
|
Any database that has an ActiveRecord adapter should work.
|
95
95
|
email:
|
@@ -99,11 +99,11 @@ extra_rdoc_files:
|
|
99
99
|
- README.md
|
100
100
|
files:
|
101
101
|
- README.md
|
102
|
-
- init.rb
|
103
|
-
- lib/csv_db.rb
|
104
|
-
- lib/serialization_helper.rb
|
105
102
|
- lib/tasks/yaml_db_tasks.rake
|
106
103
|
- lib/yaml_db.rb
|
104
|
+
- lib/yaml_db/csv_db.rb
|
105
|
+
- lib/yaml_db/rake_tasks.rb
|
106
|
+
- lib/yaml_db/serialization_helper.rb
|
107
107
|
- lib/yaml_db/version.rb
|
108
108
|
homepage: https://github.com/yamldb/yaml_db
|
109
109
|
licenses:
|
@@ -125,8 +125,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
125
|
version: '0'
|
126
126
|
requirements: []
|
127
127
|
rubyforge_project:
|
128
|
-
rubygems_version: 2.
|
128
|
+
rubygems_version: 2.5.2
|
129
129
|
signing_key:
|
130
130
|
specification_version: 4
|
131
131
|
summary: yaml_db allows export/import of database into/from yaml files
|
132
132
|
test_files: []
|
133
|
+
has_rdoc:
|
data/init.rb
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require 'yaml_db'
|
data/lib/csv_db.rb
DELETED
@@ -1,78 +0,0 @@
|
|
1
|
-
#require 'FasterCSV'
|
2
|
-
module CsvDb
|
3
|
-
module Helper
|
4
|
-
def self.loader
|
5
|
-
Load
|
6
|
-
end
|
7
|
-
|
8
|
-
def self.dumper
|
9
|
-
Dump
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.extension
|
13
|
-
"csv"
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
class Load < SerializationHelper::Load
|
18
|
-
def self.load_documents(io, truncate = true)
|
19
|
-
tables = {}
|
20
|
-
curr_table = nil
|
21
|
-
io.each do |line|
|
22
|
-
if /BEGIN_CSV_TABLE_DECLARATION(.+)END_CSV_TABLE_DECLARATION/ =~ line
|
23
|
-
curr_table = $1
|
24
|
-
tables[curr_table] = {}
|
25
|
-
else
|
26
|
-
if tables[curr_table]["columns"]
|
27
|
-
tables[curr_table]["records"] << FasterCSV.parse(line)[0]
|
28
|
-
else
|
29
|
-
tables[curr_table]["columns"] = FasterCSV.parse(line)[0]
|
30
|
-
tables[curr_table]["records"] = []
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
tables.each_pair do |table_name, contents|
|
36
|
-
load_table(table_name, contents, truncate)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
class Dump < SerializationHelper::Dump
|
42
|
-
|
43
|
-
def self.before_table(io,table)
|
44
|
-
io.write "BEGIN_CSV_TABLE_DECLARATION#{table}END_CSV_TABLE_DECLARATION\n"
|
45
|
-
end
|
46
|
-
|
47
|
-
def self.dump(io)
|
48
|
-
tables.each do |table|
|
49
|
-
before_table(io, table)
|
50
|
-
dump_table(io, table)
|
51
|
-
after_table(io, table)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def self.after_table(io,table)
|
56
|
-
io.write ""
|
57
|
-
end
|
58
|
-
|
59
|
-
def self.dump_table_columns(io, table)
|
60
|
-
io.write(table_column_names(table).to_csv)
|
61
|
-
end
|
62
|
-
|
63
|
-
def self.dump_table_records(io, table)
|
64
|
-
|
65
|
-
column_names = table_column_names(table)
|
66
|
-
|
67
|
-
each_table_page(table) do |records|
|
68
|
-
rows = SerializationHelper::Utils.unhash_records(records, column_names)
|
69
|
-
records.each do |record|
|
70
|
-
io.write(record.to_csv)
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
|
78
|
-
end
|
data/lib/serialization_helper.rb
DELETED
@@ -1,196 +0,0 @@
|
|
1
|
-
module SerializationHelper
|
2
|
-
|
3
|
-
class Base
|
4
|
-
attr_reader :extension
|
5
|
-
|
6
|
-
def initialize(helper)
|
7
|
-
@dumper = helper.dumper
|
8
|
-
@loader = helper.loader
|
9
|
-
@extension = helper.extension
|
10
|
-
end
|
11
|
-
|
12
|
-
def dump(filename)
|
13
|
-
disable_logger
|
14
|
-
@dumper.dump(File.new(filename, "w"))
|
15
|
-
reenable_logger
|
16
|
-
end
|
17
|
-
|
18
|
-
def dump_to_dir(dirname)
|
19
|
-
Dir.mkdir(dirname)
|
20
|
-
tables = @dumper.tables
|
21
|
-
tables.each do |table|
|
22
|
-
io = File.new "#{dirname}/#{table}.#{@extension}", "w"
|
23
|
-
@dumper.before_table(io, table)
|
24
|
-
@dumper.dump_table io, table
|
25
|
-
@dumper.after_table(io, table)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def load(filename, truncate = true)
|
30
|
-
disable_logger
|
31
|
-
@loader.load(File.new(filename, "r"), truncate)
|
32
|
-
reenable_logger
|
33
|
-
end
|
34
|
-
|
35
|
-
def load_from_dir(dirname, truncate = true)
|
36
|
-
Dir.entries(dirname).each do |filename|
|
37
|
-
if filename =~ /^[.]/
|
38
|
-
next
|
39
|
-
end
|
40
|
-
@loader.load(File.new("#{dirname}/#{filename}", "r"), truncate)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def disable_logger
|
45
|
-
@@old_logger = ActiveRecord::Base.logger
|
46
|
-
ActiveRecord::Base.logger = nil
|
47
|
-
end
|
48
|
-
|
49
|
-
def reenable_logger
|
50
|
-
ActiveRecord::Base.logger = @@old_logger
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
class Load
|
55
|
-
def self.load(io, truncate = true)
|
56
|
-
ActiveRecord::Base.connection.transaction do
|
57
|
-
load_documents(io, truncate)
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def self.truncate_table(table)
|
62
|
-
begin
|
63
|
-
ActiveRecord::Base.connection.execute("TRUNCATE #{SerializationHelper::Utils.quote_table(table)}")
|
64
|
-
rescue Exception
|
65
|
-
ActiveRecord::Base.connection.execute("DELETE FROM #{SerializationHelper::Utils.quote_table(table)}")
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
def self.load_table(table, data, truncate = true)
|
70
|
-
column_names = data['columns']
|
71
|
-
if truncate
|
72
|
-
truncate_table(table)
|
73
|
-
end
|
74
|
-
load_records(table, column_names, data['records'])
|
75
|
-
reset_pk_sequence!(table)
|
76
|
-
end
|
77
|
-
|
78
|
-
def self.load_records(table, column_names, records)
|
79
|
-
if column_names.nil?
|
80
|
-
return
|
81
|
-
end
|
82
|
-
columns = column_names.map{|cn| ActiveRecord::Base.connection.columns(table).detect{|c| c.name == cn}}
|
83
|
-
quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',')
|
84
|
-
quoted_table_name = SerializationHelper::Utils.quote_table(table)
|
85
|
-
records.each do |record|
|
86
|
-
quoted_values = record.zip(columns).map{|c| ActiveRecord::Base.connection.quote(c.first, c.last)}.join(',')
|
87
|
-
ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})")
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
def self.reset_pk_sequence!(table_name)
|
92
|
-
if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
|
93
|
-
ActiveRecord::Base.connection.reset_pk_sequence!(table_name)
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
|
-
end
|
98
|
-
|
99
|
-
module Utils
|
100
|
-
|
101
|
-
def self.unhash(hash, keys)
|
102
|
-
keys.map { |key| hash[key] }
|
103
|
-
end
|
104
|
-
|
105
|
-
def self.unhash_records(records, keys)
|
106
|
-
records.each_with_index do |record, index|
|
107
|
-
records[index] = unhash(record, keys)
|
108
|
-
end
|
109
|
-
|
110
|
-
records
|
111
|
-
end
|
112
|
-
|
113
|
-
def self.convert_booleans(records, columns)
|
114
|
-
records.each do |record|
|
115
|
-
columns.each do |column|
|
116
|
-
next if is_boolean(record[column])
|
117
|
-
record[column] = convert_boolean(record[column])
|
118
|
-
end
|
119
|
-
end
|
120
|
-
records
|
121
|
-
end
|
122
|
-
|
123
|
-
def self.convert_boolean(value)
|
124
|
-
['t', '1', true, 1].include?(value)
|
125
|
-
end
|
126
|
-
|
127
|
-
def self.boolean_columns(table)
|
128
|
-
columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
|
129
|
-
columns.map { |c| c.name }
|
130
|
-
end
|
131
|
-
|
132
|
-
def self.is_boolean(value)
|
133
|
-
value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
|
134
|
-
end
|
135
|
-
|
136
|
-
def self.quote_table(table)
|
137
|
-
ActiveRecord::Base.connection.quote_table_name(table)
|
138
|
-
end
|
139
|
-
|
140
|
-
end
|
141
|
-
|
142
|
-
class Dump
|
143
|
-
def self.before_table(io, table)
|
144
|
-
|
145
|
-
end
|
146
|
-
|
147
|
-
def self.dump(io)
|
148
|
-
tables.each do |table|
|
149
|
-
before_table(io, table)
|
150
|
-
dump_table(io, table)
|
151
|
-
after_table(io, table)
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
def self.after_table(io, table)
|
156
|
-
|
157
|
-
end
|
158
|
-
|
159
|
-
def self.tables
|
160
|
-
ActiveRecord::Base.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }
|
161
|
-
end
|
162
|
-
|
163
|
-
def self.dump_table(io, table)
|
164
|
-
return if table_record_count(table).zero?
|
165
|
-
|
166
|
-
dump_table_columns(io, table)
|
167
|
-
dump_table_records(io, table)
|
168
|
-
end
|
169
|
-
|
170
|
-
def self.table_column_names(table)
|
171
|
-
ActiveRecord::Base.connection.columns(table).map { |c| c.name }
|
172
|
-
end
|
173
|
-
|
174
|
-
|
175
|
-
def self.each_table_page(table, records_per_page=1000)
|
176
|
-
total_count = table_record_count(table)
|
177
|
-
pages = (total_count.to_f / records_per_page).ceil - 1
|
178
|
-
id = table_column_names(table).first
|
179
|
-
boolean_columns = SerializationHelper::Utils.boolean_columns(table)
|
180
|
-
quoted_table_name = SerializationHelper::Utils.quote_table(table)
|
181
|
-
|
182
|
-
(0..pages).to_a.each do |page|
|
183
|
-
query = Arel::Table.new(table, ActiveRecord::Base).order(id).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*'))
|
184
|
-
records = ActiveRecord::Base.connection.select_all(query.to_sql)
|
185
|
-
records = SerializationHelper::Utils.convert_booleans(records, boolean_columns)
|
186
|
-
yield records
|
187
|
-
end
|
188
|
-
end
|
189
|
-
|
190
|
-
def self.table_record_count(table)
|
191
|
-
ActiveRecord::Base.connection.select_one("SELECT COUNT(*) FROM #{SerializationHelper::Utils.quote_table(table)}").values.first.to_i
|
192
|
-
end
|
193
|
-
|
194
|
-
end
|
195
|
-
|
196
|
-
end
|