ferry 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 634ec552fec9ef82ef5cb4b7b73f11c1155c9377
-  data.tar.gz: 6fd552539d14caed7f34924511f7dfa9c75eeb4e
+  metadata.gz: 94ff6d9d55c4cca9e5af12fcce865cf305583267
+  data.tar.gz: d9c26d8dd946916e95910713b1237e0159a72f4a
 SHA512:
-  metadata.gz: 453943fab59526c5b1a36f7704a5f6a1f54df57b324558c766c2416e42baa3fcb300c03628a1d84e014b1e23dfceb9a6f4b576fe23151162ab6ca70b9d4d81ad
-  data.tar.gz: 786dbff062dacd6ce0a9366458c7933378d026237b84973319f789deee0b54cd2914a972d239827196f05da1c81136b4e7af95b9cdc9b2fe5f006f9c24072fe8
+  metadata.gz: c239c99fb6cddb6497283f1f3414fb125b9b75f6d826b8580191412d307c21c7b9f5485758d6a11e3f09c041a95e1daecc1c0f36f65dce6333877cab53b44f76
+  data.tar.gz: 40fbbd444c3b9e30e485030c581bb571b47532831112bd56ec84da886498170200b75e746844803f03f8d1fbd2f7bcb17c55373cae43ac75877ff8511451e93d

data/README.md

@@ -6,15 +6,15 @@ Ferry is a data migration and data manipulation tool that seeks to quickly and e
 ## What can I use Ferry for? (Use Cases)
 See the [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) ROR/Sqlite app for guidance on using Ferry!
 
-Manipulation Use Cases
-  - RESTful column/ row interaction
-
 Migration
   - Exporting data to various file formats (.csv, .sql, .yml)
   - Importing data from various file formats
   - Migrating data to third party hosts (Amazon S3, Oracle)
   - Migrating data to a different database
 
+Manipulation Use Cases
+  - RESTful column/ row interaction
+
 ### Datebase to CSV
 Currently, Ferry supports SQLite, PosgreSQL, and MySQL database connections  ...
 Making a simple call like ```ferry to_csv yourdbenvironment``` in any Rails app and Ferry will place a folder in your lib directory with a folder titled ferry_to_csv_yourdbenvironment.
@@ -23,30 +23,43 @@ Making a simple call like ```ferry to_csv yourdbenvironment``` in any Rails app
 - [ ] Refactoring before public release October 17th!!!
 - [ ] TEST!!! EVERYTHING!!!
   - [ ] Provide working example(s) of using ferry (see [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) app)
-- [ ] ferry --help
-- [ ] CLI tool
-- [ ] Simple CSV export
+- [x] ferry --help
+- [x] CLI tool dev
+- [x] Simple CSV/YAML export & import
   - [x] using sqlite3
   - [x] using psql
   - [x] using MySQL
   - [ ] using other dbs ...
-- [ ] Simple CSV import
-- [ ] Simple YAML export
-- [ ] Simple CSV import
+- [ ] Forking processes to make them faster!
 - [ ] RESTful column interaction
+- [ ] db switcher
+  - [ ] handling dependency installation (db or gem dependencies)
+- [ ] 3rd party connections (importing and exporting data to S3 or related services)
   - [ ] Understanding relationships between generating migrations and migration files in place
 - [ ] Rolling back on errors / mishaps during migrations and manipulations
-  - [ ] Host documentation site via GitHub pages (ferry.github.io)
+  - [ ] error catching and give proper clues to fix errors
+- [ ] Host documentation site via GitHub pages (ferry.github.io)
 
 ## Installation
 Add this line to your application's Gemfile:
-```gem 'ferry'```
+``` ruby
+gem 'ferry'
+```
 
 And then execute:
-```bundle```
+``` sh
+bundle
+```
 
 Or install it yourself as:
-```gem install ferry```
+``` sh
+gem install ferry
+```
+
+To view what Ferry can do for you just run:
+``` sh
+ferry --help
+```
 
 ## Contributing
 

data/bin/ferry

@@ -1,5 +1,17 @@
 #!/usr/bin/env ruby
 require 'ferry'
+
+# --help
+if ['--help', '-H', nil].include?(ARGV[0])
+  puts"
+  Choose one of the following commands ...
+  ferry to_csv [your db environment]                             Exports database to csv file in lib/ferry_to_csv_[your db environment]
+  ferry to_new_db_type [your db environment] [desired adapter]   Switches specified database env to a new adapter
+  ferry [--help, -H, '']                                         Displays this menu!
+  "
+end
+
 exporter = Ferry::Exporter.new
 exporter.to_csv if ARGV[0] == 'to_csv'
+exporter.to_yaml if ARGV[0] == 'to_yaml'
 exporter.to_new_db_type if ARGV[0] == 'to_new_db_type'

data/ferry.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Anthony Corletti", "Logan Watanabe", "Larry Heimann"]
   spec.email         = ["anthcor@gmail.com", "loganwatanabe@gmail.com", "profh@cmu.edu"]
   spec.summary       = "Ferry is a data migration and data manipulation tool"
-  spec.description   = "Ferry is a data migration and data manipulation tool that seeks to simplify the increasingly prevalent big data problems that tech companies face"
+  spec.description   = "Ferry is a data migration and data manipulation tool that seeks to simplify the increasingly prevalent big data problems for developers"
   spec.homepage      = "https://github.com/cmu-is-projects/"
   spec.license       = "MIT"
 

data/lib/ferry.rb

@@ -15,6 +15,10 @@ module Ferry
       ARGV[2]
     end
 
+
+    #  test to_csv
+    #  chekc that all files exist, first and last records match
+
     def to_csv
       info = YAML::load(IO.read("config/database.yml"))
       db_type = info[which_db_env]["adapter"]
@@ -53,8 +57,7 @@ module Ferry
           full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
           if full_table.num_tuples > 0
             CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-              size = full_table[0].length / 2
-              keys = full_table[0].keys.first(size)
+              keys = full_table[0].keys
               csv << keys
               full_table.each do |row|
                 csv << row.values_at(*keys)
@@ -65,17 +68,17 @@ module Ferry
         end
       when "mysql2"
         puts "operating with mysql2"
-        homedir = "lib/ferry_to_csv_#{environment}"
+        homedir = "lib/ferry_to_csv_#{which_db_env}"
         puts "connected to #{which_db_env} env db"
         ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
-        puts "connected to #{environment} env db"
+        puts "connected to #{which_db_env} env db"
         FileUtils.mkdir homedir unless Dir[homedir].present?
         puts "exporting tables to #{homedir}"
         mysql_bar = ProgressBar.new("psql_to_csv", 100)
         ActiveRecord::Base.connection.tables.each do |model|
           columns = ActiveRecord::Base.connection.execute(
             "SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
-            WHERE `TABLE_SCHEMA`= '#{info[environment]['database']}' AND `TABLE_NAME`='#{model}';")
+            WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
           CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
             col_names=[]
             columns.each do |col|
@@ -90,7 +93,7 @@ module Ferry
           end
         end
       when "mongo"
-        puts "mongo is currently not supported"
+        puts "mongo is currently unsupported"
       else
         puts "Unknown db type or no database associated with this application."
       end
@@ -103,18 +106,169 @@ module Ferry
       puts "current_db_type: #{current_db_type}"
       puts "to_new_db_type: #{switch_to_db_type}"
 
-      # check for dependencies
-      # if dependencies exist - install them?
-        # how to go about installation ... just let the user fend for themselves?
-      # create new connection if necessary dependencies exist!
-      # transfer old db into new connection
-      # remove the old connection? what to do with it? hmm
-        # possibly keep it / have a method to remove it on user specification
-        # what would the user think
-        # "why would it be any other way?"
-      # update the config file
-      # profit from your new db!
+      if ['sqlite', 'postgresql', 'mysql'].include?(switch_to_db_type)
+        info[which_db_env]["adapter"] = switch_to_db_type
+        puts "switching #{which_db_env} env to #{switch_to_db_type} ... "
+        File.open("config/database.yml", "w") {|f| f.write info.to_yaml}
+        puts "switched #{which_db_env} env to #{switch_to_db_type}"
+      else
+        puts "#{switch_to_db_type} is currently unsupported"
+      end
+
+    end
+
+    def import_csv
+      # importing for different cases
+    end
+
+
+    def to_yaml
+      info = YAML::load(IO.read("config/database.yml"))
+      db_type = info[which_db_env]["adapter"]
+      case db_type
+      when "sqlite3"
+        puts "operating with sqlite3"
+        homedir = "lib/ferry_to_yaml_#{which_db_env}"
+        ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: info[which_db_env]['database'])
+        puts "connected to #{which_db_env} env db"
+        FileUtils.mkdir homedir unless Dir[homedir].present?
+        puts "exporting tables to #{homedir}"
+        sqlite_pbar = ProgressBar.new("sqlite_to_yaml", 100)
+        i = 0
+        ActiveRecord::Base.connection.tables.each do |model|
+          db_object = {}
+          db_output = {}
+          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
+          if !full_table[0].nil?
+            size = full_table[0].length / 2
+            keys = full_table[0].keys.first(size)
+            db_object["columns"] = keys
+            model_arr=[]
+            full_table.each do |row|
+              model_arr << row.values_at(*keys)
+            end
+            db_object["records"] = model_arr
+            db_output[model] = db_object
+            if i==0
+              File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
+                YAML::dump(db_output, file)
+              end
+            else
+              File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
+                YAML::dump(db_output, file)
+              end
+            end
+            i+=1
+          end
+        end
+        sqlite_pbar.inc
+        sqlite_pbar.finish
+
+      when "postgresql"
+        puts "operating with postgres"
+        homedir = "lib/ferry_to_yaml_#{which_db_env}"
+        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
+        puts "connected to #{which_db_env} env db"
+        FileUtils.mkdir homedir unless Dir[homedir].present?
+        puts "exporting tables to #{homedir}"
+        psql_pbar = ProgressBar.new("psql_to_yaml", 100)
+
+        i = 0
+        ActiveRecord::Base.connection.tables.each do |model|
+          db_object = {}
+          db_output = {}
+          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
+          if !full_table[0].nil?
+            keys = full_table[0].keys
+            db_object["columns"] = keys
+            model_arr=[]
+            full_table.each do |row|
+              model_arr << row.values_at(*keys)
+            end
+            db_object["records"] = model_arr
+            db_output[model] = db_object
+            if i==0
+              File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
+                YAML::dump(db_output, file)
+              end
+            else
+              File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
+                YAML::dump(db_output, file)
+              end
+            end
+            i+=1
+          end
+        end
+        psql_pbar.inc
+        psql_pbar.finish
+
+      when "mysql2"
+        puts "operating with mysql2"
+        homedir = "lib/ferry_to_yaml_#{which_db_env}"
+        puts "connected to #{which_db_env} env db"
+        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
+        puts "connected to #{which_db_env} env db"
+        FileUtils.mkdir homedir unless Dir[homedir].present?
+        puts "exporting tables to #{homedir}"
+        mysql_bar = ProgressBar.new("psql_to_yaml", 100)
+        i=0
+        ActiveRecord::Base.connection.tables.each do |model|
+          db_object = {}
+          columns = ActiveRecord::Base.connection.execute(
+            "SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
+            WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
+
+            col_names=[]
+            columns.each do |col|
+              col_names.append(col[0])
+            end
+            db_object["columns"] = col_names
+            model_arr=[]
+            full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
+            full_table.each do |row|
+              model_arr << row
+            end
+            db_object["records"] = model_arr
+            db_export = {}
+            db_export[model] = db_object
+
+            if i==0
+              File.open("#{homedir}/#{which_db_env}_data.yml",'w') do |file|
+                YAML::dump(db_export, file)
+              end
+            else
+              File.open("#{homedir}/#{which_db_env}_data.yml",'a') do |file|
+                YAML::dump(db_export, file)
+              end
+            end
+            i+=1   
+
+        end
+
+        mysql_bar.inc
+        mysql_bar.finish
+
+      when "mongo"
+        puts "mongo is currently not supported"
+      else
+        puts "Unknown db type or no database associated with this application."
+      end
+    end
+
+
+
+
+
 
+
+
+
+
+
+    def export_to_service(*args)
+      # exporting to services like AWS
     end
+
+    
   end
 end

data/lib/ferry/version.rb

@@ -1,3 +1,3 @@
 module Ferry
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ferry
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Anthony Corletti
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-30 00:00:00.000000000 Z
+date: 2014-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -125,7 +125,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Ferry is a data migration and data manipulation tool that seeks to simplify
-  the increasingly prevalent big data problems that tech companies face
+  the increasingly prevalent big data problems for developers
 email:
 - anthcor@gmail.com
 - loganwatanabe@gmail.com