ferry 0.1.0 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +26 -13
- data/bin/ferry +12 -0
- data/doc/ferry_readme_icon.png +0 -0
- data/ferry.gemspec +1 -1
- data/lib/ferry.rb +171 -17
- data/lib/ferry/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 94ff6d9d55c4cca9e5af12fcce865cf305583267
|
4
|
+
data.tar.gz: d9c26d8dd946916e95910713b1237e0159a72f4a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c239c99fb6cddb6497283f1f3414fb125b9b75f6d826b8580191412d307c21c7b9f5485758d6a11e3f09c041a95e1daecc1c0f36f65dce6333877cab53b44f76
|
7
|
+
data.tar.gz: 40fbbd444c3b9e30e485030c581bb571b47532831112bd56ec84da886498170200b75e746844803f03f8d1fbd2f7bcb17c55373cae43ac75877ff8511451e93d
|
data/README.md
CHANGED
@@ -6,15 +6,15 @@ Ferry is a data migration and data manipulation tool that seeks to quickly and e
|
|
6
6
|
## What can I use Ferry for? (Use Cases)
|
7
7
|
See the [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) ROR/Sqlite app for guidance on using Ferry!
|
8
8
|
|
9
|
-
Manipulation Use Cases
|
10
|
-
- RESTful column/ row interaction
|
11
|
-
|
12
9
|
Migration
|
13
10
|
- Exporting data to various file formats (.csv, .sql, .yml)
|
14
11
|
- Importing data from various file formats
|
15
12
|
- Migrating data to third party hosts (Amazon S3, Oracle)
|
16
13
|
- Migrating data to a different database
|
17
14
|
|
15
|
+
Manipulation Use Cases
|
16
|
+
- RESTful column/ row interaction
|
17
|
+
|
18
18
|
### Datebase to CSV
|
19
19
|
Currently, Ferry supports SQLite, PosgreSQL, and MySQL database connections ...
|
20
20
|
Making a simple call like ```ferry to_csv yourdbenvironment``` in any Rails app and Ferry will place a folder in your lib directory with a folder titled ferry_to_csv_yourdbenvironment.
|
@@ -23,30 +23,43 @@ Making a simple call like ```ferry to_csv yourdbenvironment``` in any Rails app
|
|
23
23
|
- [ ] Refactoring before public release October 17th!!!
|
24
24
|
- [ ] TEST!!! EVERYTHING!!!
|
25
25
|
- [ ] Provide working example(s) of using ferry (see [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) app)
|
26
|
-
- [
|
27
|
-
- [
|
28
|
-
- [
|
26
|
+
- [x] ferry --help
|
27
|
+
- [x] CLI tool dev
|
28
|
+
- [x] Simple CSV/YAML export & import
|
29
29
|
- [x] using sqlite3
|
30
30
|
- [x] using psql
|
31
31
|
- [x] using MySQL
|
32
32
|
- [ ] using other dbs ...
|
33
|
-
- [ ]
|
34
|
-
- [ ] Simple YAML export
|
35
|
-
- [ ] Simple CSV import
|
33
|
+
- [ ] Forking processes to make them faster!
|
36
34
|
- [ ] RESTful column interaction
|
35
|
+
- [ ] db switcher
|
36
|
+
- [ ] handling dependency installation (db or gem dependencies)
|
37
|
+
- [ ] 3rd party connections (importing and exporting data to S3 or related services)
|
37
38
|
- [ ] Understanding relationships between generating migrations and migration files in place
|
38
39
|
- [ ] Rolling back on errors / mishaps during migrations and manipulations
|
39
|
-
- [ ]
|
40
|
+
- [ ] error catching and give proper clues to fix errors
|
41
|
+
- [ ] Host documentation site via GitHub pages (ferry.github.io)
|
40
42
|
|
41
43
|
## Installation
|
42
44
|
Add this line to your application's Gemfile:
|
43
|
-
```
|
45
|
+
``` ruby
|
46
|
+
gem 'ferry'
|
47
|
+
```
|
44
48
|
|
45
49
|
And then execute:
|
46
|
-
```
|
50
|
+
``` sh
|
51
|
+
bundle
|
52
|
+
```
|
47
53
|
|
48
54
|
Or install it yourself as:
|
49
|
-
```
|
55
|
+
``` sh
|
56
|
+
gem install ferry
|
57
|
+
```
|
58
|
+
|
59
|
+
To view what Ferry can do for you just run:
|
60
|
+
``` sh
|
61
|
+
ferry --help
|
62
|
+
```
|
50
63
|
|
51
64
|
## Contributing
|
52
65
|
|
data/bin/ferry
CHANGED
@@ -1,5 +1,17 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
require 'ferry'
|
3
|
+
|
4
|
+
# --help
|
5
|
+
if ['--help', '-H', nil].include?(ARGV[0])
|
6
|
+
puts"
|
7
|
+
Choose one of the following commands ...
|
8
|
+
ferry to_csv [your db environment] Exports database to csv file in lib/ferry_to_csv_[your db environment]
|
9
|
+
ferry to_new_db_type [your db environment] [desired adapter] Switches specified database env to a new adapter
|
10
|
+
ferry [--help, -H, ''] Displays this menu!
|
11
|
+
"
|
12
|
+
end
|
13
|
+
|
3
14
|
exporter = Ferry::Exporter.new
|
4
15
|
exporter.to_csv if ARGV[0] == 'to_csv'
|
16
|
+
exporter.to_yaml if ARGV[0] == 'to_yaml'
|
5
17
|
exporter.to_new_db_type if ARGV[0] == 'to_new_db_type'
|
data/doc/ferry_readme_icon.png
CHANGED
Binary file
|
data/ferry.gemspec
CHANGED
@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ["Anthony Corletti", "Logan Watanabe", "Larry Heimann"]
|
10
10
|
spec.email = ["anthcor@gmail.com", "loganwatanabe@gmail.com", "profh@cmu.edu"]
|
11
11
|
spec.summary = "Ferry is a data migration and data manipulation tool"
|
12
|
-
spec.description = "Ferry is a data migration and data manipulation tool that seeks to simplify the increasingly prevalent big data problems
|
12
|
+
spec.description = "Ferry is a data migration and data manipulation tool that seeks to simplify the increasingly prevalent big data problems for developers"
|
13
13
|
spec.homepage = "https://github.com/cmu-is-projects/"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
data/lib/ferry.rb
CHANGED
@@ -15,6 +15,10 @@ module Ferry
|
|
15
15
|
ARGV[2]
|
16
16
|
end
|
17
17
|
|
18
|
+
|
19
|
+
# test to_csv
|
20
|
+
# chekc that all files exist, first and last records match
|
21
|
+
|
18
22
|
def to_csv
|
19
23
|
info = YAML::load(IO.read("config/database.yml"))
|
20
24
|
db_type = info[which_db_env]["adapter"]
|
@@ -53,8 +57,7 @@ module Ferry
|
|
53
57
|
full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
|
54
58
|
if full_table.num_tuples > 0
|
55
59
|
CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
|
56
|
-
|
57
|
-
keys = full_table[0].keys.first(size)
|
60
|
+
keys = full_table[0].keys
|
58
61
|
csv << keys
|
59
62
|
full_table.each do |row|
|
60
63
|
csv << row.values_at(*keys)
|
@@ -65,17 +68,17 @@ module Ferry
|
|
65
68
|
end
|
66
69
|
when "mysql2"
|
67
70
|
puts "operating with mysql2"
|
68
|
-
homedir = "lib/ferry_to_csv_#{
|
71
|
+
homedir = "lib/ferry_to_csv_#{which_db_env}"
|
69
72
|
puts "connected to #{which_db_env} env db"
|
70
73
|
ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
|
71
|
-
puts "connected to #{
|
74
|
+
puts "connected to #{which_db_env} env db"
|
72
75
|
FileUtils.mkdir homedir unless Dir[homedir].present?
|
73
76
|
puts "exporting tables to #{homedir}"
|
74
77
|
mysql_bar = ProgressBar.new("psql_to_csv", 100)
|
75
78
|
ActiveRecord::Base.connection.tables.each do |model|
|
76
79
|
columns = ActiveRecord::Base.connection.execute(
|
77
80
|
"SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
|
78
|
-
WHERE `TABLE_SCHEMA`= '#{info[
|
81
|
+
WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
|
79
82
|
CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
|
80
83
|
col_names=[]
|
81
84
|
columns.each do |col|
|
@@ -90,7 +93,7 @@ module Ferry
|
|
90
93
|
end
|
91
94
|
end
|
92
95
|
when "mongo"
|
93
|
-
puts "mongo is currently
|
96
|
+
puts "mongo is currently unsupported"
|
94
97
|
else
|
95
98
|
puts "Unknown db type or no database associated with this application."
|
96
99
|
end
|
@@ -103,18 +106,169 @@ module Ferry
|
|
103
106
|
puts "current_db_type: #{current_db_type}"
|
104
107
|
puts "to_new_db_type: #{switch_to_db_type}"
|
105
108
|
|
106
|
-
|
107
|
-
|
108
|
-
#
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
#
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
109
|
+
if ['sqlite', 'postgresql', 'mysql'].include?(switch_to_db_type)
|
110
|
+
info[which_db_env]["adapter"] = switch_to_db_type
|
111
|
+
puts "switching #{which_db_env} env to #{switch_to_db_type} ... "
|
112
|
+
File.open("config/database.yml", "w") {|f| f.write info.to_yaml}
|
113
|
+
puts "switched #{which_db_env} env to #{switch_to_db_type}"
|
114
|
+
else
|
115
|
+
puts "#{switch_to_db_type} is currently unsupported"
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
|
120
|
+
def import_csv
|
121
|
+
# importing for different cases
|
122
|
+
end
|
123
|
+
|
124
|
+
|
125
|
+
def to_yaml
|
126
|
+
info = YAML::load(IO.read("config/database.yml"))
|
127
|
+
db_type = info[which_db_env]["adapter"]
|
128
|
+
case db_type
|
129
|
+
when "sqlite3"
|
130
|
+
puts "operating with sqlite3"
|
131
|
+
homedir = "lib/ferry_to_yaml_#{which_db_env}"
|
132
|
+
ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: info[which_db_env]['database'])
|
133
|
+
puts "connected to #{which_db_env} env db"
|
134
|
+
FileUtils.mkdir homedir unless Dir[homedir].present?
|
135
|
+
puts "exporting tables to #{homedir}"
|
136
|
+
sqlite_pbar = ProgressBar.new("sqlite_to_yaml", 100)
|
137
|
+
i = 0
|
138
|
+
ActiveRecord::Base.connection.tables.each do |model|
|
139
|
+
db_object = {}
|
140
|
+
db_output = {}
|
141
|
+
full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
|
142
|
+
if !full_table[0].nil?
|
143
|
+
size = full_table[0].length / 2
|
144
|
+
keys = full_table[0].keys.first(size)
|
145
|
+
db_object["columns"] = keys
|
146
|
+
model_arr=[]
|
147
|
+
full_table.each do |row|
|
148
|
+
model_arr << row.values_at(*keys)
|
149
|
+
end
|
150
|
+
db_object["records"] = model_arr
|
151
|
+
db_output[model] = db_object
|
152
|
+
if i==0
|
153
|
+
File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
|
154
|
+
YAML::dump(db_output, file)
|
155
|
+
end
|
156
|
+
else
|
157
|
+
File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
|
158
|
+
YAML::dump(db_output, file)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
i+=1
|
162
|
+
end
|
163
|
+
end
|
164
|
+
sqlite_pbar.inc
|
165
|
+
sqlite_pbar.finish
|
166
|
+
|
167
|
+
when "postgresql"
|
168
|
+
puts "operating with postgres"
|
169
|
+
homedir = "lib/ferry_to_yaml_#{which_db_env}"
|
170
|
+
ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
|
171
|
+
puts "connected to #{which_db_env} env db"
|
172
|
+
FileUtils.mkdir homedir unless Dir[homedir].present?
|
173
|
+
puts "exporting tables to #{homedir}"
|
174
|
+
psql_pbar = ProgressBar.new("psql_to_yaml", 100)
|
175
|
+
|
176
|
+
i = 0
|
177
|
+
ActiveRecord::Base.connection.tables.each do |model|
|
178
|
+
db_object = {}
|
179
|
+
db_output = {}
|
180
|
+
full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
|
181
|
+
if !full_table[0].nil?
|
182
|
+
keys = full_table[0].keys
|
183
|
+
db_object["columns"] = keys
|
184
|
+
model_arr=[]
|
185
|
+
full_table.each do |row|
|
186
|
+
model_arr << row.values_at(*keys)
|
187
|
+
end
|
188
|
+
db_object["records"] = model_arr
|
189
|
+
db_output[model] = db_object
|
190
|
+
if i==0
|
191
|
+
File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
|
192
|
+
YAML::dump(db_output, file)
|
193
|
+
end
|
194
|
+
else
|
195
|
+
File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
|
196
|
+
YAML::dump(db_output, file)
|
197
|
+
end
|
198
|
+
end
|
199
|
+
i+=1
|
200
|
+
end
|
201
|
+
end
|
202
|
+
psql_pbar.inc
|
203
|
+
psql_pbar.finish
|
204
|
+
|
205
|
+
when "mysql2"
|
206
|
+
puts "operating with mysql2"
|
207
|
+
homedir = "lib/ferry_to_yaml_#{which_db_env}"
|
208
|
+
puts "connected to #{which_db_env} env db"
|
209
|
+
ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
|
210
|
+
puts "connected to #{which_db_env} env db"
|
211
|
+
FileUtils.mkdir homedir unless Dir[homedir].present?
|
212
|
+
puts "exporting tables to #{homedir}"
|
213
|
+
mysql_bar = ProgressBar.new("psql_to_yaml", 100)
|
214
|
+
i=0
|
215
|
+
ActiveRecord::Base.connection.tables.each do |model|
|
216
|
+
db_object = {}
|
217
|
+
columns = ActiveRecord::Base.connection.execute(
|
218
|
+
"SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
|
219
|
+
WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
|
220
|
+
|
221
|
+
col_names=[]
|
222
|
+
columns.each do |col|
|
223
|
+
col_names.append(col[0])
|
224
|
+
end
|
225
|
+
db_object["columns"] = col_names
|
226
|
+
model_arr=[]
|
227
|
+
full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
|
228
|
+
full_table.each do |row|
|
229
|
+
model_arr << row
|
230
|
+
end
|
231
|
+
db_object["records"] = model_arr
|
232
|
+
db_export = {}
|
233
|
+
db_export[model] = db_object
|
234
|
+
|
235
|
+
if i==0
|
236
|
+
File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
|
237
|
+
YAML::dump(db_export, file)
|
238
|
+
end
|
239
|
+
else
|
240
|
+
File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
|
241
|
+
YAML::dump(db_export, file)
|
242
|
+
end
|
243
|
+
end
|
244
|
+
i+=1
|
245
|
+
|
246
|
+
end
|
247
|
+
|
248
|
+
mysql_bar.inc
|
249
|
+
mysql_bar.finish
|
250
|
+
|
251
|
+
when "mongo"
|
252
|
+
puts "mongo is currently not supported"
|
253
|
+
else
|
254
|
+
puts "Unknown db type or no database associated with this application."
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
|
259
|
+
|
260
|
+
|
261
|
+
|
117
262
|
|
263
|
+
|
264
|
+
|
265
|
+
|
266
|
+
|
267
|
+
|
268
|
+
def export_to_service(*args)
|
269
|
+
# exporting to services like AWS
|
118
270
|
end
|
271
|
+
|
272
|
+
|
119
273
|
end
|
120
274
|
end
|
data/lib/ferry/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ferry
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Anthony Corletti
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2014-
|
13
|
+
date: 2014-10-12 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activerecord
|
@@ -125,7 +125,7 @@ dependencies:
|
|
125
125
|
- !ruby/object:Gem::Version
|
126
126
|
version: '0'
|
127
127
|
description: Ferry is a data migration and data manipulation tool that seeks to simplify
|
128
|
-
the increasingly prevalent big data problems
|
128
|
+
the increasingly prevalent big data problems for developers
|
129
129
|
email:
|
130
130
|
- anthcor@gmail.com
|
131
131
|
- loganwatanabe@gmail.com
|