replicate 1.2 → 1.3

data/README.md

@@ -12,30 +12,90 @@ concepts.
 Synopsis
 --------
 
-Installing:
+### Installing
 
     $ gem install replicate
 
-Dumping objects:
+### Dumping objects
 
-    $ replicate -r config/environment -d 'User.find(1)' > user.dump
-    ==> dumped 4 total objects:
-    Profile        1
-    User           1
-    UserEmail      2
-
-Loading objects:
+Evaluate a Ruby expression, dumping all resulting to standard output:
 
-    $ replicate -r config/environment -l < user.dump
-    ==> loaded 4 total objects:
+    $ replicate -r ./config/environment -d "User.find(1)" > user.dump
+    ==> dumped 4 total objects:
     Profile        1
     User           1
     UserEmail      2
 
-Dumping and loading over SSH:
+The `-r ./config/environment` option is used to require environment setup and
+model instantiation code needed by the ruby expression.
+
+### Dumping many objects with a dump script
+
+Dump scripts are normal ruby source files evaluated in the context of the
+dumper. The `dump(object)` method is used to put objects into the dump stream.
+
+```ruby
+# config/replicate/dump-stuff.rb
+require 'config/environment'
+
+%w[rtomayko/tilt rtomayko/bcat].each do |repo_name|
+  repo = Repository.find_by_name_with_owner(repo_name)
+  dump repo
+  dump repo.commit_comments
+  dump repo.issues
+end
+```
+
+Run the dump script:
+
+    $ replicate -d config/replicate/dump-stuff.rb > repos.dump
+    ==> dumped 1479 total objects:
+    AR::Habtm                   101
+    CommitComment                95
+    Issue                       101
+    IssueComment                427
+    IssueEvent                  308
+    Label                         5
+    Language                     19
+    LanguageName                  1
+    Milestone                     3
+    Organization                  4
+    Profile                      82
+    PullRequest                  44
+    PullRequestReviewComment      8
+    Repository                   20
+    Team                          4
+    TeamMember                    6
+    User                         89
+    UserEmail                   162
+
+### Loading many objects:
+
+    $ replicate -r ./config/environment -l < repos.dump
+    ==> loaded 1479 total objects:
+    AR::Habtm                   101
+    CommitComment                95
+    Issue                       101
+    IssueComment                427
+    IssueEvent                  308
+    Label                         5
+    Language                     19
+    LanguageName                  1
+    Milestone                     3
+    Organization                  4
+    Profile                      82
+    PullRequest                  44
+    PullRequestReviewComment      8
+    Repository                   20
+    Team                          4
+    TeamMember                    6
+    User                         89
+    UserEmail                   162
+
+### Dumping and loading over ssh
 
     $ remote_command="replicate -r /app/config/environment -d 'User.find(1234)'"
-    $ ssh example.org "$remote_command" |replicate -r config/environment -l
+    $ ssh example.org "$remote_command" |replicate -r ./config/environment -l
 
 ActiveRecord
 ------------
@@ -47,8 +107,10 @@ MRI 1.8.7 as well as under MRI 1.9.2.
 To use customization macros in your models, require the replicate library after
 ActiveRecord (in e.g., `config/initializers/libraries.rb`):
 
-    require 'active_record'
-    require 'replicate'
+```ruby
+require 'active_record'
+require 'replicate'
+```
 
 ActiveRecord support works sensibly without customization so this isn't strictly
 necessary to use the `replicate` command. The following sections document the
@@ -66,12 +128,14 @@ database being sucked in. It can be useful to mark specific associations for
 automatic inclusion using the `replicate_associations` macro. For instance,
 to always include `EmailAddress` records belonging to a `User`:
 
-    class User < ActiveRecord::Base
-      belongs_to :profile
-      has_many   :email_addresses
+```ruby
+class User < ActiveRecord::Base
+  belongs_to :profile
+  has_many   :email_addresses
 
-      replicate_associations :email_addresses
-    end
+  replicate_associations :email_addresses
+end
+```
 
 ### Natural Keys
 
@@ -81,18 +145,20 @@ exists with matching attributes. To update existing records instead of
 creating new ones, define a natural key for the model using the `replicate_natural_key`
 macro:
 
-    class User < ActiveRecord::Base
-      belongs_to :profile
-      has_many   :email_addresses
+```ruby
+class User < ActiveRecord::Base
+  belongs_to :profile
+  has_many   :email_addresses
 
-      replicate_natural_key :login
-      replicate_associations :email_addresses
-    end
+  replicate_natural_key :login
+  replicate_associations :email_addresses
+end
 
-    class EmailAddress < ActiveRecord::Base
-      belongs_to :user
-      replicate_natural_key :user_id, :email
-    end
+class EmailAddress < ActiveRecord::Base
+  belongs_to :user
+  replicate_natural_key :user_id, :email
+end
+```
 
 Multiple attribute names may be specified to define a compound key. Foreign key
 column attributes (`user_id`) are often included in natural keys.
@@ -109,14 +175,16 @@ instance, you might want to create files on disk or load information into a
 separate data store any time an object enters the database. The best way to go
 about this currently is to override the model's `load_replicant` class method:
 
-    class User < ActiveRecord::Base
-      def self.load_replicant(type, id, attrs)
-        id, object = super
-        object.register_in_redis
-        object.some_other_callback
-        [id, object]
-      end
-    end
+```ruby
+class User < ActiveRecord::Base
+  def self.load_replicant(type, id, attrs)
+    id, object = super
+    object.register_in_redis
+    object.some_other_callback
+    [id, object]
+  end
+end
+```
 
 This interface will be improved in future versions.
 
@@ -132,15 +200,17 @@ The dump side calls `#dump_replicant(dumper)` on each object. The method must
 call `dumper.write()` with the class name, id, and hash of primitively typed
 attributes for the object:
 
-    class User
-      attr_reader   :id
-      attr_accessor :name, :email
+```ruby
+class User
+  attr_reader   :id
+  attr_accessor :name, :email
 
-      def dump_replicant(dumper)
-        attributes { 'name' => name, 'email' => email }
-        dumper.write self.class, id, attributes
-      end
-    end
+  def dump_replicant(dumper)
+    attributes { 'name' => name, 'email' => email }
+    dumper.write self.class, id, attributes
+  end
+end
+```
 
 ### load_replicant
 
@@ -148,15 +218,17 @@ The load side calls `::load_replicant(type, id, attributes)` on the class to
 load each object into the current environment. The method must return an
 `[id, object]` tuple:
 
-    class User
-      def self.load_replicant(type, id, attributes)
-        user = User.new
-        user.name  = attributes['name']
-        user.email = attributes['email']
-        user.save!
-        [user.id, user]
-      end
-    end
+```ruby
+class User
+  def self.load_replicant(type, id, attributes)
+    user = User.new
+    user.name  = attributes['name']
+    user.email = attributes['email']
+    user.save!
+    [user.id, user]
+  end
+end
+```
 
 How it works
 ------------
@@ -169,7 +241,7 @@ combined, `attributes` must consist of only string keys and simply typed values.
 Relationships between objects in the stream are managed as follows:
 
  - An object's attributes may encode references to objects that precede it
-   in the stream using a simple tuple format: [:id, 'User', 1234].
+   in the stream using a simple tuple format: `[:id, 'User', 1234]`.
 
  - The dump side ensures that objects are written to the dump stream in
    "reference order" such that when an object A includes a reference attribute
@@ -177,7 +249,7 @@ Relationships between objects in the stream are managed as follows:
 
  - The load side maintains a mapping of ids from the dumping system to the newly
    replicated objects on the loading system. When the loader encounters a
-   reference value [:id, 'User', 1234] in an object's attributes, it converts it
+   reference value `[:id, 'User', 1234]` in an object's attributes, it converts it
    to the load side id value.
 
 Dumping and loading happens in a streaming fashion. There is no limit on the

data/bin/replicate

@@ -1,9 +1,15 @@
 #!/usr/bin/env ruby
-#/        script/replicate [-r <lib>] --dump "<ruby>" > objects.dump
-#/        script/replicate [-r <lib>] --load < objects.dump
-#/ Dump and load objects between environments. The --dump form writes to stdout
-#/ the objects returned by evaluating "<ruby>", which must be a valid Ruby
-#/ expression. The --load form reads dump data from stdin and loads into the
+#/ Usage: replicate --dump dumpscript.rb > objects.dump
+#/        replicate [-r <lib>] --dump "<ruby>" > objects.dump
+#/        replicate [-r <lib>] --load < objects.dump
+#/ Dump and load objects between environments.
+#
+#/ The --dump form writes to stdout the objects dumped by the script or
+#/ ruby expression(s) given. Dump scripts are normal Ruby source files but
+#/ must call dump(object) on one or more objects. When a Ruby expression is
+#/ given, all resulting objects are dumped automatically.
+#/
+#/ The --load form reads dump data from stdin and creates objects under the
 #/ current environment.
 #/
 #/ Mode selection:
@@ -46,7 +52,7 @@ end
 # load replicate lib and setup AR
 require 'replicate'
 if defined?(ActiveRecord::Base)
-  Replicate::AR
+  require 'replicate/active_record'
   ActiveRecord::Base.replicate_id = keep_id
   ActiveRecord::Base.connection.enable_query_cache!
 end
@@ -55,11 +61,17 @@ end
 # stdout. the database should not be modified at all by this operation.
 if mode == :dump
   usage.call if ARGV.empty? || ARGV[0].empty?
-  objects = eval(ARGV[0])
   Replicate::Dumper.new do |dumper|
     dumper.marshal_to out
     dumper.log_to $stderr, verbose, quiet
-    dumper.dump objects
+    ARGV.each do |code|
+      if File.exist?(code)
+        dumper.load_script code
+      else
+        objects = dumper.instance_eval(code, '<argv>', 0)
+        dumper.dump objects
+      end
+    end
   end
 
 # load mode means we're reading objects from stdin and creating them under

data/lib/replicate/dumper.rb

@@ -49,6 +49,13 @@ module Replicate
       use Replicate::Status, 'dump', out, verbose, quiet
     end
 
+    # Load a dump script. This just evals the source of the file in the context
+    # of the dumper. Dump scripts are useful when you want to dump a lot of
+    # stuff.
+    def load_script(file)
+      instance_eval File.read(file), file, 0
+    end
+
     # Dump one or more objects to the internal array or provided dump
     # stream. This method guarantees that the same object will not be dumped
     # more than once.

data/lib/replicate/status.rb

@@ -27,7 +27,10 @@ module Replicate
     end
 
     def normal_log(type, id, attrs, object)
-      @out.write "==> #{@prefix}ing: #{@count} objects      \r"
+      message = "  %sing: %4d objects" % [@prefix, @count]
+      dots = '.' * (@count % 50)
+      dots = ' ' * 50 if dots.empty?
+      @out.write "#{message} #{dots}\r"
     end
 
     def complete

data/test/dumper_test.rb

@@ -69,4 +69,14 @@ class DumperTest < Test::Unit::TestCase
     end
     assert called
   end
+
+  def test_loading_dump_scripts
+    called = false
+    @dumper.listen do |type, id, attrs, obj|
+      assert !called
+      called = true
+    end
+    @dumper.load_script File.expand_path('../dumpscript.rb', __FILE__)
+    assert called
+  end
 end

data/test/dumpscript.rb

@@ -0,0 +1 @@
+dump Replicate::Object.new('message' => 'hello')

metadata

@@ -1,59 +1,45 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: replicate
-version: !ruby/object:Gem::Version 
-  hash: 11
+version: !ruby/object:Gem::Version
+  version: '1.3'
   prerelease: 
-  segments: 
-  - 1
-  - 2
-  version: "1.2"
 platform: ruby
-authors: 
+authors:
 - Ryan Tomayko
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-09-09 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-09-10 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: activerecord
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70344083570000 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 3
-        - 1
-        version: "3.1"
+      - !ruby/object:Gem::Version
+        version: '3.1'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: sqlite3
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70344083570000
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &70344083569500 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id002
+  prerelease: false
+  version_requirements: *70344083569500
 description: Dump and load relational objects between Ruby environments.
 email: ryan@github.com
-executables: 
+executables:
 - replicate
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - COPYING
 - HACKING
 - README.md
@@ -68,42 +54,34 @@ files:
 - lib/replicate/status.rb
 - test/active_record_test.rb
 - test/dumper_test.rb
+- test/dumpscript.rb
 - test/loader_test.rb
 - test/replicate_test.rb
 homepage: http://github.com/rtomayko/replicate
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.6
 signing_key: 
 specification_version: 2
 summary: Dump and load relational objects between Ruby environments.
-test_files: 
+test_files:
 - test/active_record_test.rb
 - test/dumper_test.rb
 - test/loader_test.rb