ferry 0.0.8 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 725496d9f272e4289334f35cd5f29dfb877fa77e
-  data.tar.gz: 62516d8f0982f15ebc85c52ea784b4e9e768b7d1
+  metadata.gz: 634ec552fec9ef82ef5cb4b7b73f11c1155c9377
+  data.tar.gz: 6fd552539d14caed7f34924511f7dfa9c75eeb4e
 SHA512:
-  metadata.gz: 2170274c90daecbb1874cb4a498aaf7cbe7beb9a9342f5efdb088527aac875bc77960cb71855a40bb94bbbe9ae7c4166d5f134cba1ee930ceaa68f0ca5d30248
-  data.tar.gz: d0b93db1a4a7f403ab6beb891f2b8bf53711b2c2672d1ff198a9d683326107dbb8e75438ca0a4aafa0b32e067fb3bf9d853ff07d478cec509493080fa99ce45f
+  metadata.gz: 453943fab59526c5b1a36f7704a5f6a1f54df57b324558c766c2416e42baa3fcb300c03628a1d84e014b1e23dfceb9a6f4b576fe23151162ab6ca70b9d4d81ad
+  data.tar.gz: 786dbff062dacd6ce0a9366458c7933378d026237b84973319f789deee0b54cd2914a972d239827196f05da1c81136b4e7af95b9cdc9b2fe5f006f9c24072fe8

data/.gitignore

@@ -7,7 +7,6 @@ Gemfile.lock
 InstalledFiles
 _yardoc
 coverage
-doc/
 lib/bundler/man
 pkg
 rdoc

data/README.md

@@ -1,4 +1,4 @@
-# Ferry
+![ferry](doc/ferry_readme_icon.png)
 
 ## What is Ferry?
 Ferry is a data migration and data manipulation tool that seeks to quickly and easily reduce overhead when dealing with big data problems.
@@ -15,32 +15,38 @@ Migration
   - Migrating data to third party hosts (Amazon S3, Oracle)
   - Migrating data to a different database
 
+### Datebase to CSV
+Currently, Ferry supports SQLite, PosgreSQL, and MySQL database connections  ...
+Making a simple call like ```ferry to_csv yourdbenvironment``` in any Rails app and Ferry will place a folder in your lib directory with a folder titled ferry_to_csv_yourdbenvironment.
+
 ## TO-DOs
-- [ ] Refactoring before public release
-- [x] Define action-items for refactor
-- [x] Provide working example(s) of using ferry (ferry_demo app)
+- [ ] Refactoring before public release October 17th!!!
+- [ ] TEST!!! EVERYTHING!!!
+  - [ ] Provide working example(s) of using ferry (see [ferry_demo](http://github.com/cmu-is-projects/ferry_demo.com) app)
+- [ ] ferry --help
 - [ ] CLI tool
 - [ ] Simple CSV export
-  - [ ] using sqlite3
-  - [ ] using psql
+  - [x] using sqlite3
+  - [x] using psql
+  - [x] using MySQL
+  - [ ] using other dbs ...
+- [ ] Simple CSV import
+- [ ] Simple YAML export
+- [ ] Simple CSV import
 - [ ] RESTful column interaction
   - [ ] Understanding relationships between generating migrations and migration files in place
-- [ ] Tests
 - [ ] Rolling back on errors / mishaps during migrations and manipulations
-  - [ ] Host documentation site via GitHub pages
+  - [ ] Host documentation site via GitHub pages (ferry.github.io)
 
 ## Installation
 Add this line to your application's Gemfile:
-
-    gem 'ferry'
+```gem 'ferry'```
 
 And then execute:
-
-    $ bundle
+```bundle```
 
 Or install it yourself as:
-
-    $ gem install ferry
+```gem install ferry```
 
 ## Contributing
 

data/Rakefile

@@ -1,4 +1 @@
-
 require "bundler/gem_tasks"
-# Bundler::GemHelper.install_tasks
-

data/ferry.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "activerecord"
   spec.add_development_dependency "bundler", "~> 1.6"
-  # spec.add_development_dependency "progressbar"
+  spec.add_development_dependency "progressbar"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
   spec.add_development_dependency "rspec"

data/lib/ferry.rb

@@ -1,7 +1,7 @@
 require 'active_record'
 require 'csv'
 require 'ferry/version'
-# require 'progressbar'
+require 'progressbar'
 require 'yaml'
 
 module Ferry
@@ -17,221 +17,78 @@ module Ferry
 
     def to_csv
       info = YAML::load(IO.read("config/database.yml"))
-      db_type = info[which_db_env||"production"]["adapter"]
-
-
-
-      #issues:
-      #   csv placement in directory:  do we want it in the lib?
-      #   the case where dev/test dbs are different from production db (sqlite for dev/test, pg for prod??)
-      #   empty ARGs
-      #   ARGs with invalid values
-
-
+      db_type = info[which_db_env]["adapter"]
       case db_type
       when "sqlite3"
         puts "operating with sqlite3"
-
-        if(which_db_env)
-          homedir = "lib/ferry_to_csv_#{which_db_env}"
-          ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
-          puts "connected to #{which_db_env} env db"
-          FileUtils.mkdir homedir unless Dir[homedir].present?
-          puts "exporting tables to #{homedir}"
-          # sqlite_pbar = ProgressBar.new("sqlite_to_csv", 100)
-          ActiveRecord::Base.connection.tables.each do |model|
-            full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-            # do not create a csv for an empty table
-            if !full_table[0].nil?
-              CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-                size = full_table[0].length / 2
-                keys = full_table[0].keys.first(size)
-                #first row contains column names
-                csv << keys
-                full_table.each do |row|
-                  csv << row.values_at(*keys)
-                  # sqlite_pbar.inc
-                end
-              end
-            end
-          end
-        else #no db_env is specified
-          info.keys.each do |environment|
-
-            if(environment == 'default')  #in Rails 4.1+ environments inherit from default, which does not have database so we will not include it
-              next
-            end
-
-            homedir = "lib/ferry_to_csv_#{environment}"
-            ActiveRecord::Base.establish_connection(adapter: db_type, database: info[environment]['database'])  #connect to sqlite3 file
-            puts "connected to #{environment} env db"
-            FileUtils.mkdir homedir unless Dir[homedir].present?
-            puts "exporting tables to #{homedir}"
-            # sqlite_pbar = ProgressBar.new("sqlite_to_csv", 100)
-            ActiveRecord::Base.connection.tables.each do |model|                                #for each model in the db
-              full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")    #get all the records
-              if !full_table[0].nil?
-                CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-                  size = full_table[0].length / 2
-                  keys = full_table[0].keys.first(size)
-                  #first row contains column names
-                  csv << keys
-                  full_table.each do |row|
-                    csv << row.values_at(*keys)
-                    # sqlite_pbar.inc
-                  end
-                end
+        homedir = "lib/ferry_to_csv_#{which_db_env}"
+        ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: info[which_db_env]['database'])
+        puts "connected to #{which_db_env} env db"
+        FileUtils.mkdir homedir unless Dir[homedir].present?
+        puts "exporting tables to #{homedir}"
+        sqlite_pbar = ProgressBar.new("sqlite_to_csv", 100)
+        ActiveRecord::Base.connection.tables.each do |model|
+          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
+          if !full_table[0].nil?
+            CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
+              size = full_table[0].length / 2
+              keys = full_table[0].keys.first(size)
+              csv << keys
+              full_table.each do |row|
+                csv << row.values_at(*keys)
+                sqlite_pbar.inc
               end
             end
           end
         end
-
       when "postgresql"
         puts "operating with postgres"
-
-        if(which_db_env)
-          homedir = "lib/ferry_to_csv_#{which_db_env}"
-          ActiveRecord::Base.establish_connection(
-            adapter:  'postgresql', 
-            host:     info[which_db_env]['host'] || 'localhost', 
-            username: info[which_db_env]['username'], 
-            password: info[which_db_env]['password'], 
-            database: info[which_db_env]['database'], 
-            encoding: info[which_db_env]['encoding']
-          )
-          puts "connected to #{which_db_env} env db"
-          FileUtils.mkdir homedir unless Dir[homedir].present?
-          puts "exporting tables to #{homedir}"
-          # psql_pbar = ProgressBar.new("psql_to_csv", 100)
-          ActiveRecord::Base.connection.tables.each do |model|
-            full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-            # do not create a csv for an empty table
-            if full_table.num_tuples > 0
-              CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-                size = full_table[0].length / 2
-                keys = full_table[0].keys.first(size)
-                #first row contains column names
-                csv << keys
-                full_table.each do |row|
-                  csv << row.values_at(*keys)
-                  # psql_pbar.inc
-                end
-              end
-            end
-          end
-        else
-          info.keys.each do |environment|
-
-            if(environment == 'default')  #in Rails 4.1+ environments inherit from default, which does not have database so we will not include it
-              next
-            end
-            homedir = "lib/ferry_to_csv_#{environment}"
-
-            ActiveRecord::Base.establish_connection(
-              adapter:  'postgresql', 
-              host:     info[environment]['host'] || 'localhost', 
-              username: info[environment]['username'], 
-              password: info[environment]['password'], 
-              database: info[environment]['database'], 
-              encoding: info[environment]['encoding']
-            )
-            puts "connected to #{environment} env db"
-            FileUtils.mkdir homedir unless Dir[homedir].present?
-            puts "exporting tables to #{homedir}"
-            # psql_pbar = ProgressBar.new("psql_to_csv", 100)
-            ActiveRecord::Base.connection.tables.each do |model|
-              full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-              # do not create a csv for an empty table
-              if full_table.num_tuples > 0
-                CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-                  size = full_table[0].length / 2
-                  keys = full_table[0].keys.first(size)
-                  #first row contains column names
-                  csv << keys
-                  full_table.each do |row|
-                    csv << row.values_at(*keys)
-                    # psql_pbar.inc
-                  end
-                end
+        homedir = "lib/ferry_to_csv_#{which_db_env}"
+        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
+        puts "connected to #{which_db_env} env db"
+        FileUtils.mkdir homedir unless Dir[homedir].present?
+        puts "exporting tables to #{homedir}"
+        psql_pbar = ProgressBar.new("psql_to_csv", 100)
+        ActiveRecord::Base.connection.tables.each do |model|
+          full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
+          if full_table.num_tuples > 0
+            CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
+              size = full_table[0].length / 2
+              keys = full_table[0].keys.first(size)
+              csv << keys
+              full_table.each do |row|
+                csv << row.values_at(*keys)
+                psql_pbar.inc
               end
             end
           end
         end
       when "mysql2"
         puts "operating with mysql2"
-
-        if(which_db_env)
-          homedir = "lib/ferry_to_csv_#{which_db_env}"
-
-          ActiveRecord::Base.establish_connection(    #this may not work on default rails if production db is not created (must run rake db:create:all)
-            adapter:  'mysql2',
-            host:     info[which_db_env]['host'] || 'localhost', 
-            username: info[which_db_env]['username'], 
-            password: info[which_db_env]['password'], 
-            database: info[which_db_env]['database']
-          )
-          puts "connected to #{which_db_env} env db"
-          FileUtils.mkdir homedir unless Dir[homedir].present?
-          puts "exporting tables to #{homedir}"
-          # psql_pbar = ProgressBar.new("psql_to_csv", 100)
-
-          ActiveRecord::Base.connection.tables.each do |model|                              #for each model in the db
-            columns = ActiveRecord::Base.connection.execute("SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE `TABLE_SCHEMA`= '#{info[which_db_env]['database']}' AND `TABLE_NAME`='#{model}';")
-            CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-
-              col_names=[]
-              columns.each do |col|
-                col_names.append(col[0])  #append the column names to an array, makes for good formatting
-              end
-              csv << col_names  #first csv row is of column names
-
-              full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-              full_table.each do |row|
-                csv << row  #not sure if this will hold, but no 'values_at' method exists
-                # psql_pbar.inc
-              end
-            end
-          end
-        else
-          info.keys.each do |environment|
-
-            if(environment == 'default')  #in Rails 4.1+ environments inherit from default, which does not have database so we will not include it
-              next
+        homedir = "lib/ferry_to_csv_#{environment}"
+        puts "connected to #{which_db_env} env db"
+        ActiveRecord::Base.establish_connection(adapter: db_type, database: info[which_db_env]['database'])
+        puts "connected to #{environment} env db"
+        FileUtils.mkdir homedir unless Dir[homedir].present?
+        puts "exporting tables to #{homedir}"
+        mysql_bar = ProgressBar.new("psql_to_csv", 100)
+        ActiveRecord::Base.connection.tables.each do |model|
+          columns = ActiveRecord::Base.connection.execute(
+            "SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS`
+            WHERE `TABLE_SCHEMA`= '#{info[environment]['database']}' AND `TABLE_NAME`='#{model}';")
+          CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
+            col_names=[]
+            columns.each do |col|
+              col_names.append(col[0])
             end
-
-            homedir = "lib/ferry_to_csv_#{environment}"
-
-              ActiveRecord::Base.establish_connection(    #this may not work on default rails if production db is not created (must run rake db:create:all)
-                adapter:  'mysql2',
-                host:     info[environment]['host'] || 'localhost', 
-                username: info[environment]['username'], 
-                password: info[environment]['password'], 
-                database: info[environment]['database']
-              )
-            puts "connected to #{environment} env db"
-            FileUtils.mkdir homedir unless Dir[homedir].present?
-            puts "exporting tables to #{homedir}"
-            # psql_pbar = ProgressBar.new("psql_to_csv", 100)
-
-            ActiveRecord::Base.connection.tables.each do |model|                              #for each model in the db
-              columns = ActiveRecord::Base.connection.execute("SELECT `COLUMN_NAME` FROM `INFORMATION_SCHEMA`.`COLUMNS` WHERE `TABLE_SCHEMA`= '#{info[environment]['database']}' AND `TABLE_NAME`='#{model}';")
-              CSV.open("#{homedir}/#{model}.csv", "w") do |csv|
-
-                col_names=[]
-                columns.each do |col|
-                  col_names.append(col[0])  #append the column names to an array, makes for good formatting
-                end
-                csv << col_names  #first csv row is of column names
-
-                full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
-                full_table.each do |row|
-                  csv << row  #not sure if this will hold, but no 'values_at' method exists
-                  # psql_pbar.inc
-                end
-              end
+            csv << col_names
+            full_table = ActiveRecord::Base.connection.execute("SELECT * FROM #{model};")
+            full_table.each do |row|
+              csv << row
+              mysql_bar.inc
             end
           end
-        end  
+        end
       when "mongo"
         puts "mongo is currently not supported"
       else
@@ -247,13 +104,17 @@ module Ferry
       puts "to_new_db_type: #{switch_to_db_type}"
 
       # check for dependencies
-      # if dependencies exist - install them
-      # create new connection
+      # if dependencies exist - install them?
+        # how to go about installation ... just let the user fend for themselves?
+      # create new connection if necessary dependencies exist!
       # transfer old db into new connection
-      # drop old connection
+      # remove the old connection? what to do with it? hmm
+        # possibly keep it / have a method to remove it on user specification
+        # what would the user think
+        # "why would it be any other way?"
       # update the config file
-      # profit
-    end
+      # profit from your new db!
 
+    end
   end
 end

data/lib/ferry/version.rb

@@ -1,3 +1,3 @@
 module Ferry
-  VERSION = "0.0.8"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ferry
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors:
 - Anthony Corletti
@@ -40,6 +40,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +142,7 @@ files:
 - README.md
 - Rakefile
 - bin/ferry
+- doc/ferry_readme_icon.png
 - ferry.gemspec
 - lib/ferry.rb
 - lib/ferry/version.rb