ferry 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 135d110be1a1feb5802cb1035af0fab9bf98bea1
-  data.tar.gz: b53e81709fc2a3bbef1314732f8feae82ffe2d41
+  metadata.gz: 23d6e9a00c4fe515bd810869dffb3667e7080f5c
+  data.tar.gz: cd77f263a4774959d6c6fb0253b4992ac54ecd21
 SHA512:
-  metadata.gz: 8274f3a35429634000afd5e6cd106ee24e92a6db2ff7a4e40db9ebd80768ca9f743e1c3534757b23cdd8a161ad1177a7564f118961a6eec083831e61b03629f3
-  data.tar.gz: d9f0d4414f852a925b20e0ee9697a5bec2b203b8974ac6018e440f1f195b28014d474e51fa6c2e11d449c40ff06622f5a9eb0b1d534595f5c74a8004b7c68d25
+  metadata.gz: 24aab9f214d73e21188541171a560b772914ed141f026e5b7f58dcf6fb0d41ef66d171fdc6e9e6a51c13a2df353b3141d6bcae40b41310f79b5d9ea62b5da265
+  data.tar.gz: 2a918e7c071296411a42c99dae4dc56ffb876b527b5b1a8d5c3f765959b2a5c68e5889badd34a992d2ddf5e33f6f224b0bf7d068765a4fcb84798ee445ba20f4

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/README.md

@@ -35,6 +35,84 @@ Or install it yourself as:
 Usage pending. See examples / submit PR's for your ideas.
 
 ## Example(s)
+###### 3 September 2014
+Use Case Ideas
+
+Note: Demo app can initially function with RoR and Postgres.
+Instantiate the command line tool for ferry ...
+  - Init the project with ferry rake namespace (ferry.rake)
+  - Run the tasks containing the methods we write in ferry
+
+Manipulation Use Cases
+  - CRUD for Columns
+  - Copy & Paste Columns
+  - CRUD for Rows
+  - Understanding relationships between generating migrations and migration files in place
+
+Migration
+  - Exporting data to various file formats (.csv, .sql, .yml)
+  - Importing data from various file formats
+  - Migrating data to third party hosts (Amazon S3, Oracle)
+  - Migrating data to a different database
+
+Important things to consider and remember
+  - Rolling back on errors / mishaps during migrations and manipulations
+  - Host documentation site via GitHub pages
+
+
+###### 30 August 2014
+Below is an initial implementation of how ferry will work
+
+```
+# encoding: UTF-8
+require 'consortium'
+
+task :load_wm_design do
+  class WmDesign < Design
+    self.table_name = :wm_design
+  end
+end
+
+namespace :consortium_example do
+  desc "writes design cigs to individual xml files using consortium"
+  task :write_local => [:load_wm_design] do
+    hostname = Socket.gethostname
+    FileUtils.mkdir "consortium_migration_#{hostname}" unless Dir["consortium_migration_#{hostname}"].present?
+    homedir = "consortium_migration_#{hostname}"
+
+    range = Design.where("savedate > ?", 15.hours.ago.strftime("%d.%m.%Y %H").to_datetime)
+
+    consortium_runtime = Benchmark.measure do
+      range.migrate({max_workers: 4, batch_size: 500}) do |collection|
+        collection.each do |design|
+          cons_place_design_content_in_batch(design, homedir, design.composite_id)
+        end
+      end
+    end
+    puts "#{consortium_runtime}"
+  end
+
+  private
+
+  def cons_place_design_content_in_batch(design, homedir, composite_id)
+    begin
+    create_xml_file(homedir, composite_id, design)
+    rescue Exception => e
+      File.rename("#{homedir}/#{composite_id}.xml", "#{homedir}/#{composite_id}.xml.failed")
+      raise e
+    end
+  end
+
+  def create_xml_file(homedir, composite_id, design)
+    design.updated_at ? updated_at = design.updated_at.to_time : updated_at = design.created_at.to_time
+    FileUtils.touch "#{homedir}/#{composite_id}.xml"
+    file = File.open("#{homedir}/#{composite_id}.xml", 'w')
+    file.puts design.content
+    file.close
+    FileUtils.touch "#{homedir}/#{composite_id}.xml", :mtime => updated_at
+  end
+end
+```
 
 ###### 29 July 2014
 Version 0.0.1 is functional with the rake task defined here :: https://github.com/customink/design_content_migration/blob/master/lib/tasks/ferry_example.rake#L10

data/bin/ferry

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'ferry'
+
+var = Ferry::Exporter.new
+var.speak

data/ferry.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "progressbar"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "rspec"
 end

data/lib/ferry.rb

@@ -3,5 +3,46 @@ require "ferry/engine"
 require "ferry/logger"
 
 module Ferry
-  # Your code goes here...
+  #
+  class ActiveRecord::Relation
+    def migrate(options, &block)
+      options[:max_workers] ||= 4
+      options[:batch_size]  ||= 10_000
+
+      log = Logger.new()
+
+      active_workers = []
+      collection = self
+      collection.find_in_batches(batch_size: options[:batch_size]) do |batch|
+        if active_workers.length >= options[:max_workers]
+          log.write "active_workers oversized at capacity of #{active_workers.length}/#{options[:max_workers]}"
+          finished_process = Process.wait
+          log.write "finished_process: #{finished_process}"
+          active_workers.delete finished_process
+          log.write "active_workers capacity now at: #{active_workers.length}/#{options[:max_workers]}"
+        else
+          active_workers << fork do
+            ActiveRecord::Base.connection.reconnect!
+            log.write "kicking off engine on batch(#{batch.first}-#{batch.last})"
+            engine = Engine.new()
+            engine.run({log: log, batch: batch}, &block)
+          end
+        end
+        ActiveRecord::Base.connection.reconnect!
+      end
+    end
+  end
+
+  class Exporter
+    def speak
+      puts "exporting!"
+    end
+  end
+
+  class Importer
+    def speak
+      puts "importing!"
+    end
+  end
+
 end

data/lib/ferry/version.rb

@@ -1,3 +1,3 @@
 module Ferry
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/spec/spec_helper.rb

@@ -0,0 +1,78 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+=end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ferry
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Anthony Corletti
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-30 00:00:00.000000000 Z
+date: 2014-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -82,26 +82,44 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ferry is a data migration and data manipulation tool that seeks to simplify
   the increasingly prevalent big data problems that tech companies face
 email:
 - anthcor@gmail.com
 - loganwatanabe@gmail.com
 - profh@cmu.edu
-executables: []
+executables:
+- ferry
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/ferry
 - ferry.gemspec
 - lib/ferry.rb
 - lib/ferry/engine.rb
 - lib/ferry/logger.rb
 - lib/ferry/version.rb
+- spec/spec_helper.rb
 homepage: https://github.com/cmu-is-projects/
 licenses:
 - MIT
@@ -126,4 +144,5 @@ rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Ferry is a data migration and data manipulation tool
-test_files: []
+test_files:
+- spec/spec_helper.rb