yaml_db 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -5
- data/lib/tasks/yaml_db_tasks.rake +4 -20
- data/lib/yaml_db.rb +5 -4
- data/lib/yaml_db/csv_db.rb +77 -0
- data/lib/yaml_db/rake_tasks.rb +36 -0
- data/lib/yaml_db/serialization_helper.rb +201 -0
- data/lib/yaml_db/version.rb +1 -1
- metadata +10 -9
- data/init.rb +0 -1
- data/lib/csv_db.rb +0 -78
- data/lib/serialization_helper.rb +0 -196
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 04f954464b2709f0799c94e237e9be6908fae386
|
4
|
+
data.tar.gz: 284dcf617e715dc8fac6359d987c96a6ec3968ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 88e48da7a8e781d54fbcfa6f88d08fb8103f5a39450c4d3226d475edd6a4129882b60fdc4af4c88a1321e9be54bb49f5b701a1be0091013518b22e4dda821bd9
|
7
|
+
data.tar.gz: e20c244eef8ae1d6b20bb63352910d3238fc251d379b6c023d1f13256f1976b32aa5a4b571baa4c1585b2956ff1c728fa89cdcb8c9fdafbae0b9a515ab98b63d
|
data/README.md
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# YamlDb
|
2
2
|
|
3
|
-
YamlDb is a database-independent format for dumping and restoring data. It complements the
|
3
|
+
YamlDb is a database-independent format for dumping and restoring data. It complements the database-independent schema format found in db/schema.rb. The data is saved into db/data.yml.
|
4
4
|
|
5
5
|
This can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps. Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.
|
6
6
|
|
7
|
-
Any database that has an ActiveRecord adapter should work.
|
7
|
+
Any database that has an ActiveRecord adapter should work.
|
8
|
+
|
9
|
+
This gem supports Rails 3.x, 4.x, and 5.0.
|
8
10
|
|
9
11
|
[![Build Status](https://travis-ci.org/yamldb/yaml_db.svg?branch=master)](https://travis-ci.org/yamldb/yaml_db)
|
10
12
|
|
@@ -42,6 +44,4 @@ One common use would be to switch your data from one database backend to another
|
|
42
44
|
|
43
45
|
## Credits
|
44
46
|
|
45
|
-
Created by Orion Henry and Adam Wiggins.
|
46
|
-
|
47
|
-
Patches contributed by Michael Irwin, Tom Locke, and Tim Galeckas.
|
47
|
+
Created by Orion Henry and Adam Wiggins. Major updates by Ricardo Chimal Jr. and Nate Kidwell.
|
@@ -6,40 +6,24 @@ namespace :db do
|
|
6
6
|
task(:load => [ "db:schema:load", "db:data:load" ])
|
7
7
|
|
8
8
|
namespace :data do
|
9
|
-
def db_dump_data_file (extension = "yml")
|
10
|
-
"#{dump_dir}/data.#{extension}"
|
11
|
-
end
|
12
|
-
|
13
|
-
def dump_dir(dir = "")
|
14
|
-
"#{Rails.root}/db#{dir}"
|
15
|
-
end
|
16
|
-
|
17
9
|
desc "Dump contents of database to db/data.extension (defaults to yaml)"
|
18
10
|
task :dump => :environment do
|
19
|
-
|
20
|
-
helper = format_class.constantize
|
21
|
-
SerializationHelper::Base.new(helper).dump db_dump_data_file helper.extension
|
11
|
+
YamlDb::RakeTasks.data_dump_task
|
22
12
|
end
|
23
13
|
|
24
14
|
desc "Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)"
|
25
15
|
task :dump_dir => :environment do
|
26
|
-
|
27
|
-
dir = ENV['dir'] || "#{Time.now.to_s.gsub(/ /, '_')}"
|
28
|
-
SerializationHelper::Base.new(format_class.constantize).dump_to_dir dump_dir("/#{dir}")
|
16
|
+
YamlDb::RakeTasks.data_dump_dir_task
|
29
17
|
end
|
30
18
|
|
31
19
|
desc "Load contents of db/data.extension (defaults to yaml) into database"
|
32
20
|
task :load => :environment do
|
33
|
-
|
34
|
-
helper = format_class.constantize
|
35
|
-
SerializationHelper::Base.new(helper).load(db_dump_data_file helper.extension)
|
21
|
+
YamlDb::RakeTasks.data_load_task
|
36
22
|
end
|
37
23
|
|
38
24
|
desc "Load contents of db/data_dir into database"
|
39
25
|
task :load_dir => :environment do
|
40
|
-
|
41
|
-
format_class = ENV['class'] || "YamlDb::Helper"
|
42
|
-
SerializationHelper::Base.new(format_class.constantize).load_from_dir dump_dir("/#{dir}")
|
26
|
+
YamlDb::RakeTasks.data_load_dir_task
|
43
27
|
end
|
44
28
|
end
|
45
29
|
end
|
data/lib/yaml_db.rb
CHANGED
@@ -1,19 +1,20 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'yaml'
|
3
3
|
require 'active_record'
|
4
|
-
require 'serialization_helper'
|
5
4
|
require 'active_support/core_ext/kernel/reporting'
|
6
5
|
require 'rails/railtie'
|
6
|
+
require 'yaml_db/rake_tasks'
|
7
7
|
require 'yaml_db/version'
|
8
|
+
require 'yaml_db/serialization_helper'
|
8
9
|
|
9
10
|
module YamlDb
|
10
11
|
module Helper
|
11
12
|
def self.loader
|
12
|
-
|
13
|
+
Load
|
13
14
|
end
|
14
15
|
|
15
16
|
def self.dumper
|
16
|
-
|
17
|
+
Dump
|
17
18
|
end
|
18
19
|
|
19
20
|
def self.extension
|
@@ -46,7 +47,7 @@ module YamlDb
|
|
46
47
|
|
47
48
|
each_table_page(table) do |records|
|
48
49
|
rows = SerializationHelper::Utils.unhash_records(records.to_a, column_names)
|
49
|
-
io.write(
|
50
|
+
io.write(Utils.chunk_records(rows))
|
50
51
|
end
|
51
52
|
end
|
52
53
|
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module YamlDb
|
2
|
+
module CsvDb
|
3
|
+
module Helper
|
4
|
+
def self.loader
|
5
|
+
Load
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.dumper
|
9
|
+
Dump
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.extension
|
13
|
+
"csv"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class Load < SerializationHelper::Load
|
18
|
+
def self.load_documents(io, truncate = true)
|
19
|
+
tables = {}
|
20
|
+
curr_table = nil
|
21
|
+
io.each do |line|
|
22
|
+
if /BEGIN_CSV_TABLE_DECLARATION(.+)END_CSV_TABLE_DECLARATION/ =~ line
|
23
|
+
curr_table = $1
|
24
|
+
tables[curr_table] = {}
|
25
|
+
else
|
26
|
+
if tables[curr_table]["columns"]
|
27
|
+
tables[curr_table]["records"] << FasterCSV.parse(line)[0]
|
28
|
+
else
|
29
|
+
tables[curr_table]["columns"] = FasterCSV.parse(line)[0]
|
30
|
+
tables[curr_table]["records"] = []
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
tables.each_pair do |table_name, contents|
|
36
|
+
load_table(table_name, contents, truncate)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class Dump < SerializationHelper::Dump
|
42
|
+
|
43
|
+
def self.before_table(io,table)
|
44
|
+
io.write "BEGIN_CSV_TABLE_DECLARATION#{table}END_CSV_TABLE_DECLARATION\n"
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.dump(io)
|
48
|
+
tables.each do |table|
|
49
|
+
before_table(io, table)
|
50
|
+
dump_table(io, table)
|
51
|
+
after_table(io, table)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.after_table(io,table)
|
56
|
+
io.write ""
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.dump_table_columns(io, table)
|
60
|
+
io.write(table_column_names(table).to_csv)
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.dump_table_records(io, table)
|
64
|
+
|
65
|
+
column_names = table_column_names(table)
|
66
|
+
|
67
|
+
each_table_page(table) do |records|
|
68
|
+
rows = SerializationHelper::Utils.unhash_records(records, column_names)
|
69
|
+
records.each do |record|
|
70
|
+
io.write(record.to_csv)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module YamlDb
|
2
|
+
module RakeTasks
|
3
|
+
def self.data_dump_task
|
4
|
+
SerializationHelper::Base.new(helper).dump(db_dump_data_file(helper.extension))
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.data_dump_dir_task
|
8
|
+
dir = ENV['dir'] || "#{Time.now.strftime('%F_%T')}"
|
9
|
+
SerializationHelper::Base.new(helper).dump_to_dir(dump_dir("/#{dir}"))
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.data_load_task
|
13
|
+
SerializationHelper::Base.new(helper).load(db_dump_data_file(helper.extension))
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.data_load_dir_task
|
17
|
+
dir = ENV['dir'] || 'base'
|
18
|
+
SerializationHelper::Base.new(helper).load_from_dir(dump_dir("/#{dir}"))
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
def self.db_dump_data_file(extension = 'yml')
|
24
|
+
"#{dump_dir}/data.#{extension}"
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.dump_dir(dir = '')
|
28
|
+
"#{Rails.root}/db#{dir}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.helper
|
32
|
+
format_class = ENV['class'] || 'YamlDb::Helper'
|
33
|
+
format_class.constantize
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,201 @@
|
|
1
|
+
module YamlDb
|
2
|
+
module SerializationHelper
|
3
|
+
|
4
|
+
class Base
|
5
|
+
attr_reader :extension
|
6
|
+
|
7
|
+
def initialize(helper)
|
8
|
+
@dumper = helper.dumper
|
9
|
+
@loader = helper.loader
|
10
|
+
@extension = helper.extension
|
11
|
+
end
|
12
|
+
|
13
|
+
def dump(filename)
|
14
|
+
disable_logger
|
15
|
+
File.open(filename, "w") do |file|
|
16
|
+
@dumper.dump(file)
|
17
|
+
end
|
18
|
+
reenable_logger
|
19
|
+
end
|
20
|
+
|
21
|
+
def dump_to_dir(dirname)
|
22
|
+
Dir.mkdir(dirname)
|
23
|
+
tables = @dumper.tables
|
24
|
+
tables.each do |table|
|
25
|
+
File.open("#{dirname}/#{table}.#{@extension}", "w") do |io|
|
26
|
+
@dumper.before_table(io, table)
|
27
|
+
@dumper.dump_table io, table
|
28
|
+
@dumper.after_table(io, table)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def load(filename, truncate = true)
|
34
|
+
disable_logger
|
35
|
+
@loader.load(File.new(filename, "r"), truncate)
|
36
|
+
reenable_logger
|
37
|
+
end
|
38
|
+
|
39
|
+
def load_from_dir(dirname, truncate = true)
|
40
|
+
Dir.entries(dirname).each do |filename|
|
41
|
+
if filename =~ /^[.]/
|
42
|
+
next
|
43
|
+
end
|
44
|
+
@loader.load(File.new("#{dirname}/#{filename}", "r"), truncate)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def disable_logger
|
49
|
+
@@old_logger = ActiveRecord::Base.logger
|
50
|
+
ActiveRecord::Base.logger = nil
|
51
|
+
end
|
52
|
+
|
53
|
+
def reenable_logger
|
54
|
+
ActiveRecord::Base.logger = @@old_logger
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
class Load
|
59
|
+
def self.load(io, truncate = true)
|
60
|
+
ActiveRecord::Base.connection.transaction do
|
61
|
+
load_documents(io, truncate)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def self.truncate_table(table)
|
66
|
+
begin
|
67
|
+
ActiveRecord::Base.connection.execute("TRUNCATE #{Utils.quote_table(table)}")
|
68
|
+
rescue Exception
|
69
|
+
ActiveRecord::Base.connection.execute("DELETE FROM #{Utils.quote_table(table)}")
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def self.load_table(table, data, truncate = true)
|
74
|
+
column_names = data['columns']
|
75
|
+
if truncate
|
76
|
+
truncate_table(table)
|
77
|
+
end
|
78
|
+
load_records(table, column_names, data['records'])
|
79
|
+
reset_pk_sequence!(table)
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.load_records(table, column_names, records)
|
83
|
+
if column_names.nil?
|
84
|
+
return
|
85
|
+
end
|
86
|
+
columns = column_names.map{|cn| ActiveRecord::Base.connection.columns(table).detect{|c| c.name == cn}}
|
87
|
+
quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',')
|
88
|
+
quoted_table_name = Utils.quote_table(table)
|
89
|
+
records.each do |record|
|
90
|
+
quoted_values = record.zip(columns).map{|c| ActiveRecord::Base.connection.quote(c.first, c.last)}.join(',')
|
91
|
+
ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})")
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.reset_pk_sequence!(table_name)
|
96
|
+
if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
|
97
|
+
ActiveRecord::Base.connection.reset_pk_sequence!(table_name)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
module Utils
|
104
|
+
|
105
|
+
def self.unhash(hash, keys)
|
106
|
+
keys.map { |key| hash[key] }
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.unhash_records(records, keys)
|
110
|
+
records.each_with_index do |record, index|
|
111
|
+
records[index] = unhash(record, keys)
|
112
|
+
end
|
113
|
+
|
114
|
+
records
|
115
|
+
end
|
116
|
+
|
117
|
+
def self.convert_booleans(records, columns)
|
118
|
+
records.each do |record|
|
119
|
+
columns.each do |column|
|
120
|
+
next if is_boolean(record[column])
|
121
|
+
record[column] = convert_boolean(record[column])
|
122
|
+
end
|
123
|
+
end
|
124
|
+
records
|
125
|
+
end
|
126
|
+
|
127
|
+
def self.convert_boolean(value)
|
128
|
+
['t', '1', true, 1].include?(value)
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.boolean_columns(table)
|
132
|
+
columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
|
133
|
+
columns.map { |c| c.name }
|
134
|
+
end
|
135
|
+
|
136
|
+
def self.is_boolean(value)
|
137
|
+
value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
|
138
|
+
end
|
139
|
+
|
140
|
+
def self.quote_table(table)
|
141
|
+
ActiveRecord::Base.connection.quote_table_name(table)
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
|
146
|
+
class Dump
|
147
|
+
def self.before_table(io, table)
|
148
|
+
|
149
|
+
end
|
150
|
+
|
151
|
+
def self.dump(io)
|
152
|
+
tables.each do |table|
|
153
|
+
before_table(io, table)
|
154
|
+
dump_table(io, table)
|
155
|
+
after_table(io, table)
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def self.after_table(io, table)
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
def self.tables
|
164
|
+
ActiveRecord::Base.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }
|
165
|
+
end
|
166
|
+
|
167
|
+
def self.dump_table(io, table)
|
168
|
+
return if table_record_count(table).zero?
|
169
|
+
|
170
|
+
dump_table_columns(io, table)
|
171
|
+
dump_table_records(io, table)
|
172
|
+
end
|
173
|
+
|
174
|
+
def self.table_column_names(table)
|
175
|
+
ActiveRecord::Base.connection.columns(table).map { |c| c.name }
|
176
|
+
end
|
177
|
+
|
178
|
+
|
179
|
+
def self.each_table_page(table, records_per_page=1000)
|
180
|
+
total_count = table_record_count(table)
|
181
|
+
pages = (total_count.to_f / records_per_page).ceil - 1
|
182
|
+
id = table_column_names(table).first
|
183
|
+
boolean_columns = Utils.boolean_columns(table)
|
184
|
+
quoted_table_name = Utils.quote_table(table)
|
185
|
+
|
186
|
+
(0..pages).to_a.each do |page|
|
187
|
+
query = Arel::Table.new(table).order(id).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*'))
|
188
|
+
records = ActiveRecord::Base.connection.select_all(query.to_sql)
|
189
|
+
records = Utils.convert_booleans(records, boolean_columns)
|
190
|
+
yield records
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def self.table_record_count(table)
|
195
|
+
ActiveRecord::Base.connection.select_one("SELECT COUNT(*) FROM #{Utils.quote_table(table)}").values.first.to_i
|
196
|
+
end
|
197
|
+
|
198
|
+
end
|
199
|
+
|
200
|
+
end
|
201
|
+
end
|
data/lib/yaml_db/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yaml_db
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Wiggins
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2016-07-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rails
|
@@ -20,7 +20,7 @@ dependencies:
|
|
20
20
|
version: '3.0'
|
21
21
|
- - "<"
|
22
22
|
- !ruby/object:Gem::Version
|
23
|
-
version: '
|
23
|
+
version: '5.1'
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -30,7 +30,7 @@ dependencies:
|
|
30
30
|
version: '3.0'
|
31
31
|
- - "<"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
33
|
+
version: '5.1'
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
35
|
name: rake
|
36
36
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,7 +89,7 @@ dependencies:
|
|
89
89
|
version: '1.3'
|
90
90
|
description: |2
|
91
91
|
|
92
|
-
YamlDb is a database-independent format for dumping and restoring data. It complements the
|
92
|
+
YamlDb is a database-independent format for dumping and restoring data. It complements the database-independent schema format found in db/schema.rb. The data is saved into db/data.yml.
|
93
93
|
This can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps. Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.
|
94
94
|
Any database that has an ActiveRecord adapter should work.
|
95
95
|
email:
|
@@ -99,11 +99,11 @@ extra_rdoc_files:
|
|
99
99
|
- README.md
|
100
100
|
files:
|
101
101
|
- README.md
|
102
|
-
- init.rb
|
103
|
-
- lib/csv_db.rb
|
104
|
-
- lib/serialization_helper.rb
|
105
102
|
- lib/tasks/yaml_db_tasks.rake
|
106
103
|
- lib/yaml_db.rb
|
104
|
+
- lib/yaml_db/csv_db.rb
|
105
|
+
- lib/yaml_db/rake_tasks.rb
|
106
|
+
- lib/yaml_db/serialization_helper.rb
|
107
107
|
- lib/yaml_db/version.rb
|
108
108
|
homepage: https://github.com/yamldb/yaml_db
|
109
109
|
licenses:
|
@@ -125,8 +125,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
125
|
version: '0'
|
126
126
|
requirements: []
|
127
127
|
rubyforge_project:
|
128
|
-
rubygems_version: 2.
|
128
|
+
rubygems_version: 2.5.2
|
129
129
|
signing_key:
|
130
130
|
specification_version: 4
|
131
131
|
summary: yaml_db allows export/import of database into/from yaml files
|
132
132
|
test_files: []
|
133
|
+
has_rdoc:
|
data/init.rb
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require 'yaml_db'
|
data/lib/csv_db.rb
DELETED
@@ -1,78 +0,0 @@
|
|
1
|
-
#require 'FasterCSV'
|
2
|
-
module CsvDb
|
3
|
-
module Helper
|
4
|
-
def self.loader
|
5
|
-
Load
|
6
|
-
end
|
7
|
-
|
8
|
-
def self.dumper
|
9
|
-
Dump
|
10
|
-
end
|
11
|
-
|
12
|
-
def self.extension
|
13
|
-
"csv"
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
class Load < SerializationHelper::Load
|
18
|
-
def self.load_documents(io, truncate = true)
|
19
|
-
tables = {}
|
20
|
-
curr_table = nil
|
21
|
-
io.each do |line|
|
22
|
-
if /BEGIN_CSV_TABLE_DECLARATION(.+)END_CSV_TABLE_DECLARATION/ =~ line
|
23
|
-
curr_table = $1
|
24
|
-
tables[curr_table] = {}
|
25
|
-
else
|
26
|
-
if tables[curr_table]["columns"]
|
27
|
-
tables[curr_table]["records"] << FasterCSV.parse(line)[0]
|
28
|
-
else
|
29
|
-
tables[curr_table]["columns"] = FasterCSV.parse(line)[0]
|
30
|
-
tables[curr_table]["records"] = []
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
tables.each_pair do |table_name, contents|
|
36
|
-
load_table(table_name, contents, truncate)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
class Dump < SerializationHelper::Dump
|
42
|
-
|
43
|
-
def self.before_table(io,table)
|
44
|
-
io.write "BEGIN_CSV_TABLE_DECLARATION#{table}END_CSV_TABLE_DECLARATION\n"
|
45
|
-
end
|
46
|
-
|
47
|
-
def self.dump(io)
|
48
|
-
tables.each do |table|
|
49
|
-
before_table(io, table)
|
50
|
-
dump_table(io, table)
|
51
|
-
after_table(io, table)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
def self.after_table(io,table)
|
56
|
-
io.write ""
|
57
|
-
end
|
58
|
-
|
59
|
-
def self.dump_table_columns(io, table)
|
60
|
-
io.write(table_column_names(table).to_csv)
|
61
|
-
end
|
62
|
-
|
63
|
-
def self.dump_table_records(io, table)
|
64
|
-
|
65
|
-
column_names = table_column_names(table)
|
66
|
-
|
67
|
-
each_table_page(table) do |records|
|
68
|
-
rows = SerializationHelper::Utils.unhash_records(records, column_names)
|
69
|
-
records.each do |record|
|
70
|
-
io.write(record.to_csv)
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
|
-
end
|
76
|
-
|
77
|
-
|
78
|
-
end
|
data/lib/serialization_helper.rb
DELETED
@@ -1,196 +0,0 @@
|
|
1
|
-
module SerializationHelper
|
2
|
-
|
3
|
-
class Base
|
4
|
-
attr_reader :extension
|
5
|
-
|
6
|
-
def initialize(helper)
|
7
|
-
@dumper = helper.dumper
|
8
|
-
@loader = helper.loader
|
9
|
-
@extension = helper.extension
|
10
|
-
end
|
11
|
-
|
12
|
-
def dump(filename)
|
13
|
-
disable_logger
|
14
|
-
@dumper.dump(File.new(filename, "w"))
|
15
|
-
reenable_logger
|
16
|
-
end
|
17
|
-
|
18
|
-
def dump_to_dir(dirname)
|
19
|
-
Dir.mkdir(dirname)
|
20
|
-
tables = @dumper.tables
|
21
|
-
tables.each do |table|
|
22
|
-
io = File.new "#{dirname}/#{table}.#{@extension}", "w"
|
23
|
-
@dumper.before_table(io, table)
|
24
|
-
@dumper.dump_table io, table
|
25
|
-
@dumper.after_table(io, table)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def load(filename, truncate = true)
|
30
|
-
disable_logger
|
31
|
-
@loader.load(File.new(filename, "r"), truncate)
|
32
|
-
reenable_logger
|
33
|
-
end
|
34
|
-
|
35
|
-
def load_from_dir(dirname, truncate = true)
|
36
|
-
Dir.entries(dirname).each do |filename|
|
37
|
-
if filename =~ /^[.]/
|
38
|
-
next
|
39
|
-
end
|
40
|
-
@loader.load(File.new("#{dirname}/#{filename}", "r"), truncate)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def disable_logger
|
45
|
-
@@old_logger = ActiveRecord::Base.logger
|
46
|
-
ActiveRecord::Base.logger = nil
|
47
|
-
end
|
48
|
-
|
49
|
-
def reenable_logger
|
50
|
-
ActiveRecord::Base.logger = @@old_logger
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
class Load
|
55
|
-
def self.load(io, truncate = true)
|
56
|
-
ActiveRecord::Base.connection.transaction do
|
57
|
-
load_documents(io, truncate)
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
def self.truncate_table(table)
|
62
|
-
begin
|
63
|
-
ActiveRecord::Base.connection.execute("TRUNCATE #{SerializationHelper::Utils.quote_table(table)}")
|
64
|
-
rescue Exception
|
65
|
-
ActiveRecord::Base.connection.execute("DELETE FROM #{SerializationHelper::Utils.quote_table(table)}")
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
def self.load_table(table, data, truncate = true)
|
70
|
-
column_names = data['columns']
|
71
|
-
if truncate
|
72
|
-
truncate_table(table)
|
73
|
-
end
|
74
|
-
load_records(table, column_names, data['records'])
|
75
|
-
reset_pk_sequence!(table)
|
76
|
-
end
|
77
|
-
|
78
|
-
def self.load_records(table, column_names, records)
|
79
|
-
if column_names.nil?
|
80
|
-
return
|
81
|
-
end
|
82
|
-
columns = column_names.map{|cn| ActiveRecord::Base.connection.columns(table).detect{|c| c.name == cn}}
|
83
|
-
quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',')
|
84
|
-
quoted_table_name = SerializationHelper::Utils.quote_table(table)
|
85
|
-
records.each do |record|
|
86
|
-
quoted_values = record.zip(columns).map{|c| ActiveRecord::Base.connection.quote(c.first, c.last)}.join(',')
|
87
|
-
ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})")
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
def self.reset_pk_sequence!(table_name)
|
92
|
-
if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!)
|
93
|
-
ActiveRecord::Base.connection.reset_pk_sequence!(table_name)
|
94
|
-
end
|
95
|
-
end
|
96
|
-
|
97
|
-
end
|
98
|
-
|
99
|
-
module Utils
|
100
|
-
|
101
|
-
def self.unhash(hash, keys)
|
102
|
-
keys.map { |key| hash[key] }
|
103
|
-
end
|
104
|
-
|
105
|
-
def self.unhash_records(records, keys)
|
106
|
-
records.each_with_index do |record, index|
|
107
|
-
records[index] = unhash(record, keys)
|
108
|
-
end
|
109
|
-
|
110
|
-
records
|
111
|
-
end
|
112
|
-
|
113
|
-
def self.convert_booleans(records, columns)
|
114
|
-
records.each do |record|
|
115
|
-
columns.each do |column|
|
116
|
-
next if is_boolean(record[column])
|
117
|
-
record[column] = convert_boolean(record[column])
|
118
|
-
end
|
119
|
-
end
|
120
|
-
records
|
121
|
-
end
|
122
|
-
|
123
|
-
def self.convert_boolean(value)
|
124
|
-
['t', '1', true, 1].include?(value)
|
125
|
-
end
|
126
|
-
|
127
|
-
def self.boolean_columns(table)
|
128
|
-
columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
|
129
|
-
columns.map { |c| c.name }
|
130
|
-
end
|
131
|
-
|
132
|
-
def self.is_boolean(value)
|
133
|
-
value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
|
134
|
-
end
|
135
|
-
|
136
|
-
def self.quote_table(table)
|
137
|
-
ActiveRecord::Base.connection.quote_table_name(table)
|
138
|
-
end
|
139
|
-
|
140
|
-
end
|
141
|
-
|
142
|
-
class Dump
|
143
|
-
def self.before_table(io, table)
|
144
|
-
|
145
|
-
end
|
146
|
-
|
147
|
-
def self.dump(io)
|
148
|
-
tables.each do |table|
|
149
|
-
before_table(io, table)
|
150
|
-
dump_table(io, table)
|
151
|
-
after_table(io, table)
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
def self.after_table(io, table)
|
156
|
-
|
157
|
-
end
|
158
|
-
|
159
|
-
def self.tables
|
160
|
-
ActiveRecord::Base.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }
|
161
|
-
end
|
162
|
-
|
163
|
-
def self.dump_table(io, table)
|
164
|
-
return if table_record_count(table).zero?
|
165
|
-
|
166
|
-
dump_table_columns(io, table)
|
167
|
-
dump_table_records(io, table)
|
168
|
-
end
|
169
|
-
|
170
|
-
def self.table_column_names(table)
|
171
|
-
ActiveRecord::Base.connection.columns(table).map { |c| c.name }
|
172
|
-
end
|
173
|
-
|
174
|
-
|
175
|
-
def self.each_table_page(table, records_per_page=1000)
|
176
|
-
total_count = table_record_count(table)
|
177
|
-
pages = (total_count.to_f / records_per_page).ceil - 1
|
178
|
-
id = table_column_names(table).first
|
179
|
-
boolean_columns = SerializationHelper::Utils.boolean_columns(table)
|
180
|
-
quoted_table_name = SerializationHelper::Utils.quote_table(table)
|
181
|
-
|
182
|
-
(0..pages).to_a.each do |page|
|
183
|
-
query = Arel::Table.new(table, ActiveRecord::Base).order(id).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*'))
|
184
|
-
records = ActiveRecord::Base.connection.select_all(query.to_sql)
|
185
|
-
records = SerializationHelper::Utils.convert_booleans(records, boolean_columns)
|
186
|
-
yield records
|
187
|
-
end
|
188
|
-
end
|
189
|
-
|
190
|
-
def self.table_record_count(table)
|
191
|
-
ActiveRecord::Base.connection.select_one("SELECT COUNT(*) FROM #{SerializationHelper::Utils.quote_table(table)}").values.first.to_i
|
192
|
-
end
|
193
|
-
|
194
|
-
end
|
195
|
-
|
196
|
-
end
|