ferry 0.1.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03aaf09ddb58a77b1969d4b3cd8a77d8a6c38aea
-  data.tar.gz: 1404e33c5c9990be48dab7e1f822d69ae0a42214
+  metadata.gz: 91d7f7634081a1b9e9e14f49f90406bb8b73b468
+  data.tar.gz: 06e43c90a6164a35d0550986938f341ced9057a7
 SHA512:
-  metadata.gz: 583d2fd050a3fb623ef3f20e06c1f9a0fbf4e657c4a09a0c98619316bd7b25ad20eea0dfcbb06c03472ca93999304683cb320c6c258319aa4a332cc11064c458
-  data.tar.gz: 537359146f868f6050a2b9ea95babe8c8d05f9c1d8c4fa62ab74e99873d346ec926aa88345a08b7a7744e70a5f68c6443da4b8bd73f729e26730a3a66f5764ff
+  metadata.gz: 4ff5c819939ec96fd0b950495611cda42ce84bcf98819510eb53490d660c052cac28e25ef3991c899e42fb31affcfd91f3b557b7e642304d5fb87a51bf506a35
+  data.tar.gz: d334eb819e200b038cc2fe252621c14ae0ca87c4766e0297b979398ab0f77032af4b54bb9172e792851c0ce6d5ed17077af599aa2ef4e07c6d9542ad56f2c2a6

data/.gitignore

@@ -19,3 +19,5 @@ tmp
 *.o
 *.a
 mkmf.log
+log
+spec/test.db

data/.rspec

@@ -1,3 +1,2 @@
 --color
---warnings
 --require spec_helper

data/Gemfile

@@ -1,3 +1,2 @@
 source 'https://rubygems.org'
-gem 'progressbar', '0.21.0'
 gemspec

data/README.md

@@ -1,44 +1,26 @@
-![ferry](doc/ferry_readme_icon.png)
+![ferry](doc/ferry_readme_icon_2.png)
+
+<!-- ![Build Status](https://travis-ci.org/cmu-is-projects/ferry.png)(https://travis-ci.org/cmu-is-projects/ferry) -->
 
 ## What is Ferry?
-Ferry is a data migration and data manipulation tool that seeks to quickly and easily reduce overhead when dealing with big data problems.
+Ferry is a command-line tool rubygem designed for for data migrations and manipulation, maintained as an open-source project by the students of [Carnegie Mellon's Information Systems department](http://www.cmu.edu/information-systems/) currently [Anthony Corletti](http://github.com/anthcor) and [Logan Watanabe](http://github.com/loganwatanabe). The inspiration for ferry was brought from collective internship experiences and from the growing prevalence of big data migration and manipulation challenges that companies, corporations, universities, and organizations face in today's information age.
 
-## What can I use Ferry for? (Use Cases)
-See the [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) ROR/Sqlite app for guidance on using Ferry!
+## What can I use Ferry for?
+See the [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) app or our [GitHub pages site](http://cmu-is-projects.github.com/ferry) for guidance on using Ferry!
 
-Migration
-  - Exporting data to various file formats (.csv, .sql, .yml)
+Migration and Manipulation use cases
+  - Exporting data to various file formats (.csv, .yml, .sql)
   - Importing data from various file formats
   - Migrating data to third party hosts (Amazon S3, Oracle)
   - Migrating data to a different database
 
-Manipulation Use Cases
-  - RESTful column/ row interaction
-
-### Datebase to CSV
-Currently, Ferry supports SQLite, PosgreSQL, and MySQL database connections  ...
-Making a simple call like ```ferry to_csv yourdbenvironment``` in any Rails app and Ferry will place a folder in your lib directory with a folder titled ferry_to_csv_yourdbenvironment.
-
-## TO-DOs
-- [ ] Refactoring before public release October 17th!!!
-- [ ] TEST!!! EVERYTHING!!!
-  - [ ] Provide working example(s) of using ferry (see [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) app)
-- [x] ferry --help
-- [x] CLI tool dev
-- [x] Simple CSV/YAML export & import
-  - [x] using sqlite3
-  - [x] using psql
-  - [x] using MySQL
-  - [ ] using other dbs ...
-- [ ] Forking processes to make them faster!
-- [ ] RESTful column interaction
-- [ ] db switcher
-  - [ ] handling dependency installation (db or gem dependencies)
-- [ ] 3rd party connections (importing and exporting data to S3 or related services)
-  - [ ] Understanding relationships between generating migrations and migration files in place
-- [ ] Rolling back on errors / mishaps during migrations and manipulations
-  - [ ] error catching and give proper clues to fix errors
-- [ ] Host documentation site via GitHub pages (ferry.github.io)
+## Some current development items
+#### Please feel free to open an issue or pull request with your suggestions
+- Database-switcher guide tool
+- 3rd party connections
+- Allowing user to write own rake tasks (e.g. importing and exporting data to S3 or related services)
+- Rolling back on errors or mishaps during migrations and manipulations
+- Host API and docs via GitHub pages
 
 ## Installation
 Add this line to your application's Gemfile:
@@ -48,17 +30,17 @@ gem 'ferry'
 
 And then execute:
 ``` sh
-bundle
+$ bundle
 ```
 
 Or install it yourself as:
 ``` sh
-gem install ferry
+$ gem install ferry
 ```
 
 To view what Ferry can do for you just run:
 ``` sh
-ferry --help
+$ ferry --help
 ```
 
 ## Contributing

data/bin/ferry

@@ -1,17 +1,54 @@
 #!/usr/bin/env ruby
 require 'ferry'
 
-# --help
-if ['--help', '-H', nil].include?(ARGV[0])
-  puts"
-  Choose one of the following commands ...
-  ferry to_csv [your db environment]                             Exports database to csv file in lib/ferry_to_csv_[your db environment]
-  ferry to_new_db_type [your db environment] [desired adapter]   Switches specified database env to a new adapter
-  ferry [--help, -H, '']                                         Displays this menu!
-  "
-end
-
-exporter = Ferry::Exporter.new
-exporter.to_csv if ARGV[0] == 'to_csv'
-exporter.to_yaml if ARGV[0] == 'to_yaml'
-exporter.to_new_db_type if ARGV[0] == 'to_new_db_type'
+OptionParser.new do |opts|
+  options = {}
+  opts.banner = "See full documentation at http://cmu-is-projects.github.io/ferry"
+
+  opts.on("--to_csv <env,table>", 'Exports table to csv file in db/ferry_to_csv_[your db environment]') do |opt|
+    list = opt.split(",")
+    raise "please enter a field for environment and table" unless list.length == 2
+    options[:input] = list
+    exporter = Ferry::Exporter.new
+    exporter.to_csv(options[:input][0], options[:input][1])
+  end
+
+  opts.on("--to_yaml <env,table>", 'Exports table to yaml file in db/ferry_to_csv_[your db environment]') do |opt|
+    list = opt.split(",")
+    raise "please enter a field for environment and table" unless list.length == 2
+    options[:input] = list
+    exporter = Ferry::Exporter.new
+    exporter.to_yaml(options[:input][0], options[:input][1])
+  end
+
+  opts.on('--import <env,tablename,file>', 'Imports data from specified file into table and environment of your specification') do |opt|
+    list = opt.split(",")
+    raise "please enter a field for environment and table and filename" unless list.length == 3
+    options[:input] = list
+    importer = Ferry::Importer.new
+    importer.import(options[:input][0], options[:input][1], options[:input][2])
+  end
+
+  opts.on('--db_switch <env,adapter>', 'Switches specified database env to a new adapter') do |opt|
+    list = opt.split(",")
+    raise "please enter a field for environment and table" unless list.length == 2
+    options[:input] = list
+    questionmaster = Ferry::Utilities.new
+    if questionmaster.continue?("Are you sure you want to switch to a #{options[:input][1]} database in your #{options[:input][0]} env? It could result in installing software that impacts your project.")
+      switcher = Ferry::Switcher.new
+      switcher.to_new_db_type(options[:input][0], options[:input][1])
+    end
+  end
+
+  opts.on('--init', 'Initializes ferry.rb file in config directory') do |opt|
+    filemaker = Ferry::Utilities.new
+    filemaker.init
+  end
+
+  opts.on_tail('-h', '--help', 'Shows this message') do
+    puts opts
+    exit
+  end
+
+  puts opts if ARGV[0] == nil
+end.parse!

data/ferry.gemspec

@@ -10,22 +10,22 @@ Gem::Specification.new do |spec|
   spec.email         = ["anthcor@gmail.com", "loganwatanabe@gmail.com", "profh@cmu.edu"]
   spec.summary       = "Ferry is a data migration and data manipulation tool"
   spec.description   = "Ferry is a data migration and data manipulation tool that seeks to simplify the increasingly prevalent big data problems for developers"
-  spec.homepage      = "https://github.com/cmu-is-projects/"
+  spec.homepage      = "https://cmu-is-projects.github.io/ferry"
   spec.license       = "MIT"
-
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "activerecord", "~> 4.1.7"
   spec.add_development_dependency "bundler", "~> 1.6"
-  spec.add_development_dependency "progressbar"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "minitest"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "pg"
-  spec.add_development_dependency "sqlite3"
-  # need to look over installation problems with mysql :(
-  # spec.add_development_dependency "mysql"
+  spec.add_development_dependency "rake", "~> 10.3.2"
+  spec.add_development_dependency "minitest", "~> 5.4.1"
+  spec.add_development_dependency "rspec", "~> 3.1.0"
+  spec.add_development_dependency "pg", "~> 0.17.1"
+  spec.add_development_dependency "sqlite3", "~> 1.3.10"
+  spec.add_development_dependency "mysql2", "~> 0.3.16"
+  spec.add_development_dependency "factory_girl", "~> 4.5.0"
+  spec.add_runtime_dependency "progressbar", "~> 0.21.0"
+  spec.add_runtime_dependency "highline", "~> 1.6.21"
 end

data/lib/ferry.rb

@@ -1,274 +1,16 @@
 require 'active_record'
 require 'csv'
+require 'enumerator'
+require 'ferry/exporter'
+require 'ferry/importer'
+require 'ferry/utilities'
+require 'ferry/switcher'
 require 'ferry/version'
+require 'highline/import'
 require 'progressbar'
+require 'optparse'
+require 'pp'
 require 'yaml'
 
 module Ferry
-  class Exporter
-
-    def which_db_env
-      ARGV[1]
-    end
-
-    def switch_to_db_type
-      ARGV[2]
-    end
-
-
-    #  test to_csv
-    #  chekc that all files exist, first and last records match
-
-    def to_csv
-      info = YAML::load(IO.read("config/database.yml"))
-      db_type = info[which_db_env]["adapter"]
-      case db_type
-      when "sqlite3"
-        puts "operating with sqlite3"
-        homedir = "lib/ferry_to_csv_#{which_db_env}"
-        ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: info[which_db_env]['database'])
-        puts "connected to #{which_db_env} env db"
-        FileUtils.mkdir homedir unless Dir[homedir].present?
-        puts "exporting tables to #{homedir}"
-        sqlite_pbar = ProgressBar.new("sqlite_to_csv", 100)
-        ActiveRecord::Base.connection.tables.each do |model|
-          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-          if !full_table[0].nil?
-            CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-              size = full_table[0].length / 2
-              keys = full_table[0].keys.first(size)
-              csv << keys
-              full_table.each do |row|
-                csv << row.values_at(*keys)
-                sqlite_pbar.inc
-              end
-            end
-          end
-        end
-      when "postgresql"
-        puts "operating with postgres"
-        homedir = "lib/ferry_to_csv_#{which_db_env}"
-        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
-        puts "connected to #{which_db_env} env db"
-        FileUtils.mkdir homedir unless Dir[homedir].present?
-        puts "exporting tables to #{homedir}"
-        psql_pbar = ProgressBar.new("psql_to_csv", 100)
-        ActiveRecord::Base.connection.tables.each do |model|
-          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-          if full_table.num_tuples > 0
-            CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-              keys = full_table[0].keys
-              csv << keys
-              full_table.each do |row|
-                csv << row.values_at(*keys)
-                psql_pbar.inc
-              end
-            end
-          end
-        end
-      when "mysql2"
-        puts "operating with mysql2"
-        homedir = "lib/ferry_to_csv_#{which_db_env}"
-        puts "connected to #{which_db_env} env db"
-        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
-        puts "connected to #{which_db_env} env db"
-        FileUtils.mkdir homedir unless Dir[homedir].present?
-        puts "exporting tables to #{homedir}"
-        mysql_bar = ProgressBar.new("psql_to_csv", 100)
-        ActiveRecord::Base.connection.tables.each do |model|
-          columns = ActiveRecord::Base.connection.execute(
-            "SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
-            WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
-          CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-            col_names=[]
-            columns.each do |col|
-              col_names.append(col[0])
-            end
-            csv << col_names
-            full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-            full_table.each do |row|
-              csv << row
-              mysql_bar.inc
-            end
-          end
-        end
-      when "mongo"
-        puts "mongo is currently unsupported"
-      else
-        puts "Unknown db type or no database associated with this application."
-      end
-    end
-
-    def to_new_db_type
-      info = YAML::load(IO.read("config/database.yml"))
-      current_db_type = info[which_db_env]["adapter"]
-      puts "switching the #{which_db_env} database's adapter"
-      puts "current_db_type: #{current_db_type}"
-      puts "to_new_db_type: #{switch_to_db_type}"
-
-      if ['sqlite', 'postgresql', 'mysql'].include?(switch_to_db_type)
-        info[which_db_env]["adapter"] = switch_to_db_type
-        puts "switching #{which_db_env} env to #{switch_to_db_type} ... "
-        File.open("config/database.yml", "w") {|f| f.write info.to_yaml}
-        puts "switched #{which_db_env} env to #{switch_to_db_type}"
-      else
-        puts "#{switch_to_db_type} is currently unsupported"
-      end
-
-    end
-
-    def import_csv
-      # importing for different cases
-    end
-
-
-    def to_yaml
-      info = YAML::load(IO.read("config/database.yml"))
-      db_type = info[which_db_env]["adapter"]
-      case db_type
-      when "sqlite3"
-        puts "operating with sqlite3"
-        homedir = "lib/ferry_to_yaml_#{which_db_env}"
-        ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: info[which_db_env]['database'])
-        puts "connected to #{which_db_env} env db"
-        FileUtils.mkdir homedir unless Dir[homedir].present?
-        puts "exporting tables to #{homedir}"
-        sqlite_pbar = ProgressBar.new("sqlite_to_yaml", 100)
-        i = 0
-        ActiveRecord::Base.connection.tables.each do |model|
-          db_object = {}
-          db_output = {}
-          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-          if !full_table[0].nil?
-            size = full_table[0].length / 2
-            keys = full_table[0].keys.first(size)
-            db_object["columns"] = keys
-            model_arr=[]
-            full_table.each do |row|
-              model_arr << row.values_at(*keys)
-            end
-            db_object["records"] = model_arr
-            db_output[model] = db_object
-            if i==0
-              File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
-                YAML::dump(db_output, file)
-              end
-            else
-              File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
-                YAML::dump(db_output, file)
-              end
-            end
-            i+=1
-          end
-        end
-        sqlite_pbar.inc
-        sqlite_pbar.finish
-
-      when "postgresql"
-        puts "operating with postgres"
-        homedir = "lib/ferry_to_yaml_#{which_db_env}"
-        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
-        puts "connected to #{which_db_env} env db"
-        FileUtils.mkdir homedir unless Dir[homedir].present?
-        puts "exporting tables to #{homedir}"
-        psql_pbar = ProgressBar.new("psql_to_yaml", 100)
-
-        i = 0
-        ActiveRecord::Base.connection.tables.each do |model|
-          db_object = {}
-          db_output = {}
-          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-          if !full_table[0].nil?
-            keys = full_table[0].keys
-            db_object["columns"] = keys
-            model_arr=[]
-            full_table.each do |row|
-              model_arr << row.values_at(*keys)
-            end
-            db_object["records"] = model_arr
-            db_output[model] = db_object
-            if i==0
-              File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
-                YAML::dump(db_output, file)
-              end
-            else
-              File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
-                YAML::dump(db_output, file)
-              end
-            end
-            i+=1
-          end
-        end
-        psql_pbar.inc
-        psql_pbar.finish
-
-      when "mysql2"
-        puts "operating with mysql2"
-        homedir = "lib/ferry_to_yaml_#{which_db_env}"
-        puts "connected to #{which_db_env} env db"
-        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
-        puts "connected to #{which_db_env} env db"
-        FileUtils.mkdir homedir unless Dir[homedir].present?
-        puts "exporting tables to #{homedir}"
-        mysql_bar = ProgressBar.new("psql_to_yaml", 100)
-        i=0
-        ActiveRecord::Base.connection.tables.each do |model|
-          db_object = {}
-          columns = ActiveRecord::Base.connection.execute(
-            "SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
-            WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
-
-            col_names=[]
-            columns.each do |col|
-              col_names.append(col[0])
-            end
-            db_object["columns"] = col_names
-            model_arr=[]
-            full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-            full_table.each do |row|
-              model_arr << row
-            end
-            db_object["records"] = model_arr
-            db_export = {}
-            db_export[model] = db_object
-
-            if i==0
-              File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
-                YAML::dump(db_export, file)
-              end
-            else
-              File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
-                YAML::dump(db_export, file)
-              end
-            end
-            i+=1   
-
-        end
-
-        mysql_bar.inc
-        mysql_bar.finish
-
-      when "mongo"
-        puts "mongo is currently not supported"
-      else
-        puts "Unknown db type or no database associated with this application."
-      end
-    end
-
-
-
-
-
-
-
-
-
-
-
-    def export_to_service(*args)
-      # exporting to services like AWS
-    end
-
-    
-  end
 end