gizzmo 0.11.2 → 0.11.3

data/lib/gizzard/thrift.rb

@@ -90,7 +90,7 @@ module Gizzard
     end
   end
 
-  NameServerState = T.make_struct(:NameserverState,
+  NameServerState = T.make_struct(:NameServerState,
     T::Field.new(:shards, list(struct(ShardInfo)), 1),
     T::Field.new(:links, list(struct(LinkInfo)), 2),
     T::Field.new(:forwardings, list(struct(Forwarding)), 3),
@@ -150,6 +150,7 @@ module Gizzard
   end
 
   class Manager < GizzmoService
+    thrift_method :reload_updated_forwardings, void, :throws => exception(GizzardException)
     thrift_method :reload_config, void, :throws => exception(GizzardException)
     thrift_method :rebuild_schema, void, :throws => exception(GizzardException)
 
@@ -179,11 +180,12 @@ module Gizzard
     thrift_method :get_forwardings, list(struct(Forwarding)), :throws => exception(GizzardException)
 
     thrift_method :list_hostnames, list(string), :throws => exception(GizzardException)
+    thrift_method :list_tables, list(i32), :throws => exception(GizzardException)
 
     thrift_method :mark_shard_busy, void, field(:id, struct(ShardId), 1), field(:busy, i32, 2), :throws => exception(GizzardException)
     thrift_method :copy_shard, void, field(:source_id, struct(ShardId), 1), field(:destination_id, struct(ShardId), 2), :throws => exception(GizzardException)
 
-    thrift_method :dump_nameserver, struct(NameServerState), field(:table_id, i32, 1), :throws => exception(GizzardException)
+    thrift_method :dump_nameserver, list(struct(NameServerState)), field(:table_ids, list(i32), 1), :throws => exception(GizzardException)
 
 
     # Job Scheduler Management

data/lib/gizzard/transformation.rb

@@ -1,3 +1,5 @@
+require "set"
+
 module Gizzard
   class Transformation
     require 'gizzard/transformation_op'
@@ -33,7 +35,7 @@ module Gizzard
 
     DEFAULT_DEST_WRAPPER = 'WriteOnlyShard'
 
-    attr_reader :from, :to
+    attr_reader :from, :to, :copy_dest_wrapper
 
     def initialize(from_template, to_template, copy_dest_wrapper = nil)
       copy_dest_wrapper ||= DEFAULT_DEST_WRAPPER
@@ -59,6 +61,27 @@ module Gizzard
       end
     end
 
+    def noop?
+      from.eql? to
+    end
+
+    def eql?(o)
+      o.is_a?(self.class) &&
+      from.eql?(o.from) &&
+      to.eql?(o.to) &&
+      copy_dest_wrapper.eql?(o.copy_dest_wrapper)
+    end
+
+    def <=>(o)
+      to_a = lambda {|t| [t.from, t.to, t.copy_dest_wrapper] }
+
+      to_a.call(self) <=> to_a.call(o)
+    end
+
+    def hash
+      from.hash + to.hash + copy_dest_wrapper.hash
+    end
+
     def inspect
       op_inspect = operations.inject({}) do |h, (phase, ops)|
         h.update phase => ops.map {|job| "    #{job.inspect}" }.join("\n")
@@ -198,13 +221,21 @@ module Gizzard
     end
 
     def involved_shards(phase = :copy)
-      transformation.operations[phase].map do |op|
-        op.involved_shards(@table_prefix, @translations)
-      end.flatten.compact.uniq
+      @involved_shards        ||= {}
+      @involved_shards[phase] ||=
+        Set.new(transformation.operations[phase].map do |op|
+          op.involved_shards(@table_prefix, @translations)
+        end.flatten.compact.uniq)
+    end
+
+    def involved_hosts_array(phase = :copy)
+      @involved_hosts_array        ||= {}
+      @involved_hosts_array[phase] ||= involved_shards(phase).map {|s| s.hostname }.uniq
     end
 
     def involved_hosts(phase = :copy)
-      involved_shards(phase).map {|s| s.hostname }.uniq
+      @involved_hosts        ||= {}
+      @involved_hosts[phase] ||= Set.new(involved_hosts_array(phase))
     end
 
     def inspect

data/lib/gizzard/transformation_scheduler.rb

@@ -1,3 +1,5 @@
+require "set"
+
 module Gizzard
   def self.schedule!(*args)
     Transformation::Scheduler.new(*args).apply!
@@ -27,9 +29,9 @@ module Gizzard
       @jobs_in_progress = []
       @jobs_finished    = []
 
-      @jobs_pending = transformations.map do |transformation, forwardings_to_shards|
+      @jobs_pending = Set.new(transformations.map do |transformation, forwardings_to_shards|
         transformation.bind(base_name, forwardings_to_shards)
-      end.flatten
+      end.flatten)
     end
 
     # to schedule a job:
@@ -65,26 +67,27 @@ module Gizzard
         end
       end
 
-      nameserver.reload_config
+      nameserver.reload_updated_forwardings
 
       log "#{@jobs_finished.length} transformation#{'s' if @jobs_finished.length > 1} applied. Total time elapsed: #{time_elapsed}"
     end
 
     def schedule_jobs(num_to_schedule)
       to_be_busy_hosts = []
+      jobs             = []
 
-      jobs = (1..num_to_schedule).map do
-        job = @jobs_pending.find do |j|
-          (busy_hosts(to_be_busy_hosts) & j.involved_hosts).empty?
-        end
+      @jobs_pending.each do |j|
+        if (busy_hosts(to_be_busy_hosts) & j.involved_hosts).empty?
+          jobs << j
+          to_be_busy_hosts.concat j.involved_hosts_array
 
-        if job
-          to_be_busy_hosts.concat job.involved_hosts
-          @jobs_pending.delete(job)
+          break if jobs.length == num_to_schedule
         end
+      end
+
+      @jobs_pending.subtract(jobs)
 
-        job
-      end.compact.sort_by {|t| t.forwarding }
+      jobs = jobs.sort_by {|t| t.forwarding }
 
       unless jobs.empty?
         log "STARTING:"
@@ -93,7 +96,7 @@ module Gizzard
           j.prepare!(nameserver)
         end
 
-        nameserver.reload_config
+        nameserver.reload_updated_forwardings
 
         copy_jobs = jobs.select {|j| j.copy_required? }
 
@@ -139,9 +142,9 @@ module Gizzard
     def busy_shards
       @busy_shards ||=
         if nameserver.dryrun?
-          []
+          Set.new
         else
-          nameserver.get_busy_shards.map {|s| s.id }
+          nameserver.get_busy_shards.inject(Set.new) {|set, shard| set.add(shard.id) }
         end
     end
 
@@ -152,7 +155,7 @@ module Gizzard
         h.update(host => 1) {|_,a,b| a + b }
       end
 
-      copies_count_map.select {|_, count| count >= @copies_per_host }.map {|(host, _)| host }
+      copies_count_map.select {|_, count| count >= @copies_per_host }.inject(Set.new) {|set,(host, _)| set.add(host) }
     end
 
     def sleep_with_progress(interval)

data/lib/gizzmo.rb

@@ -127,24 +127,6 @@ subcommands = {
     opts.banner = "Usage: #{zero} wrap CLASS_NAME SHARD_ID_TO_WRAP [MORE SHARD_IDS...]"
     separators(opts, DOC_STRINGS["wrap"])
   end,
-  'rebalance' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} rebalance"
-    separators(opts, DOC_STRINGS["rebalance"])
-
-    opts.on("-w", "--write-only=CLASS") do |w|
-      subcommand_options.write_only_shard = w
-    end
-    opts.on("-h", "--shard-hosts=list") do |h|
-      subcommand_options.hosts = h
-    end
-    opts.on("-x", "--exclude-hosts=list") do |x|
-      subcommand_options.exclude_hosts = x
-    end
-  end,
-  'repair' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} repair MASTER SLAVE [MASTER SLAVE...]"
-    separators(opts, DOC_STRINGS["repair"])
-  end,
   'pair' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} pair"
     separators(opts, DOC_STRINGS["pair"])
@@ -165,6 +147,10 @@ subcommands = {
     opts.banner = "Usage: #{zero} hosts"
     separators(opts, DOC_STRINGS["hosts"])
   end,
+  'tables' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} tables"
+    separators(opts, DOC_STRINGS["tables"])
+  end,
   'deleteforwarding' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} deleteforwarding TABLE_ID BASE_ID SHARD_ID"
     separators(opts, DOC_STRINGS["deleteforwarding"])
@@ -308,9 +294,13 @@ subcommands = {
     opts.banner = "Usage: #{zero} topology [options]"
     separators(opts, DOC_STRINGS["topology"])
 
-    opts.on("--forwardings", "Show topology of forwardings instead of counts") do
+    opts.on("--forwardings", "Show topology by forwarding instead of counts") do
       subcommand_options.forwardings = true
     end
+
+    opts.on("--shards", "Show topology by root shard ids instead of counts") do
+      subcommand_options.root_shards = true
+    end
   end,
   'transform-tree' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} transform-tree [options] ROOT_SHARD_ID TEMPLATE"
@@ -323,11 +313,21 @@ subcommands = {
     end
   end,
   'transform' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} transform [options] FROM_TEMPLATE TO_TEMPLATE"
+    opts.banner = "Usage: #{zero} transform [options] FROM_TEMPLATE TO_TEMPLATE ..."
     separators(opts, DOC_STRINGS['transform'])
 
     add_scheduler_opts subcommand_options, opts
 
+    opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
+      subcommand_options.quiet = true
+    end
+  end,
+  'rebalance' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} rebalance [options] WEIGHT TO_TEMPLATE ..."
+    separators(opts, DOC_STRINGS["rebalance"])
+
+    add_scheduler_opts subcommand_options, opts
+
     opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
       subcommand_options.quiet = true
     end

data/test/gizzmo_spec.rb

@@ -1,17 +1,13 @@
 require File.expand_path('../spec_helper.rb', __FILE__)
 
 describe "gizzmo (cli)" do
-  def gizzmo(cmd)
-    `cd #{ROOT_DIR} && ruby -rubygems -Ilib bin/gizzmo -H localhost -P #{MANAGER_PORT} #{cmd} 2>&1`
-  end
-
-  def nameserver
-    @nameserver ||= read_nameserver_db
+  def nameserver_db
+    @nameserver_db ||= read_nameserver_db
   end
 
   before do
     reset_nameserver
-    @nameserver = nil
+    @nameserver_db = nil
   end
 
   describe "basic manipulation commands" do
@@ -19,138 +15,145 @@ describe "gizzmo (cli)" do
       it "creates a single shard" do
         gizzmo "create TestShard localhost/t0_0"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard")]
       end
 
       it "creates multiple shards" do
         gizzmo "create TestShard localhost/t0_0 localhost/t0_1"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard"),
-                                       info("localhost", "t0_1", "TestShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard"),
+                                          info("localhost", "t0_1", "TestShard")]
       end
 
       it "honors source and destination types" do
         gizzmo "create TestShard -s int -d long localhost/t0_0"
         gizzmo "create TestShard --source-type=int --destination-type=long localhost/t0_1"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard", "int", "long"),
-                                       info("localhost", "t0_1", "TestShard", "int", "long")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard", "int", "long"),
+                                          info("localhost", "t0_1", "TestShard", "int", "long")]
       end
     end
 
     describe "delete" do
       it "deletes a shard" do
-        gizzmo "create TestShard localhost/t0_0"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+
         gizzmo "delete localhost/t0_0"
 
-        nameserver[:shards].should == []
+        nameserver_db[:shards].should == []
       end
     end
 
     describe "wrap/unwrap" do
       before do
-        gizzmo "create TestShard localhost/t0_0"
-        gizzmo "create ReplicatingShard localhost/t0_0_replicating"
-        gizzmo "addlink localhost/t0_0_replicating localhost/t0_0 1"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+        ns.create_shard info("localhost", "t0_0_replicating", "ReplicatingShard")
+        ns.add_link id("localhost", "t0_0_replicating"), id("localhost", "t0_0"), 1
 
         gizzmo "wrap BlockedShard localhost/t0_0"
       end
 
       it "wrap wraps a shard" do
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard"),
-                                       info("localhost", "t0_0_blocked", "BlockedShard"),
-                                       info("localhost", "t0_0_replicating", "ReplicatingShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard"),
+                                          info("localhost", "t0_0_blocked", "BlockedShard"),
+                                          info("localhost", "t0_0_replicating", "ReplicatingShard")]
 
-        nameserver[:links].should == [link(id("localhost", "t0_0_blocked"), id("localhost", "t0_0"), 1),
-                                      link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0_blocked"), 1)]
+        nameserver_db[:links].should == [link(id("localhost", "t0_0_blocked"), id("localhost", "t0_0"), 1),
+                                         link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0_blocked"), 1)]
       end
 
       it "unwrap unwraps a shard" do
         gizzmo "unwrap localhost/t0_0_blocked"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard"),
-                                       info("localhost", "t0_0_replicating", "ReplicatingShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard"),
+                                          info("localhost", "t0_0_replicating", "ReplicatingShard")]
 
-        nameserver[:links].should == [link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0"), 1)]
+        nameserver_db[:links].should == [link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0"), 1)]
       end
 
       it "unwrap doesn't unwrap a top level shard or a leaf" do
         gizzmo "unwrap localhost/t0_0"
         gizzmo "unwrap localhost/t0_0_replicating"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard"),
-                                       info("localhost", "t0_0_blocked", "BlockedShard"),
-                                       info("localhost", "t0_0_replicating", "ReplicatingShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard"),
+                                          info("localhost", "t0_0_blocked", "BlockedShard"),
+                                          info("localhost", "t0_0_replicating", "ReplicatingShard")]
 
-        nameserver[:links].should == [link(id("localhost", "t0_0_blocked"), id("localhost", "t0_0"), 1),
-                                      link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0_blocked"), 1)]
+        nameserver_db[:links].should == [link(id("localhost", "t0_0_blocked"), id("localhost", "t0_0"), 1),
+                                         link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0_blocked"), 1)]
       end
     end
 
     describe "markbusy" do
       it "marks shards busy" do
-        gizzmo "create TestShard localhost/t0_0"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+
         gizzmo "markbusy localhost/t0_0"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard", "", "", 1)]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard", "", "", 1)]
       end
     end
 
     describe "markunbusy" do
       it "marks shards as not busy" do
-        gizzmo "create TestShard localhost/t0_0"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
         gizzmo "markbusy localhost/t0_0"
+
         gizzmo "markunbusy localhost/t0_0"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard", "", "", 0)]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard", "", "", 0)]
       end
     end
 
     describe "addforwarding" do
       it "adds a forwarding" do
-        gizzmo "create TestShard localhost/t0_0"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+
         gizzmo "addforwarding 0 0 localhost/t0_0"
 
-        nameserver[:shards].should      == [info("localhost", "t0_0", "TestShard")]
-        nameserver[:forwardings].should == [forwarding(0, 0, id("localhost", "t0_0"))]
+        nameserver_db[:shards].should      == [info("localhost", "t0_0", "TestShard")]
+        nameserver_db[:forwardings].should == [forwarding(0, 0, id("localhost", "t0_0"))]
       end
     end
 
     describe "deleteforwarding" do
       it "removes a forwarding" do
-        gizzmo "create TestShard localhost/t0_0"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+
         gizzmo "addforwarding 0 0 localhost/t0_0"
         gizzmo "deleteforwarding 0 0 localhost/t0_0"
 
-        nameserver[:shards].should      == [info("localhost", "t0_0", "TestShard")]
-        nameserver[:forwardings].should == []
+        nameserver_db[:shards].should      == [info("localhost", "t0_0", "TestShard")]
+        nameserver_db[:forwardings].should == []
       end
     end
 
     describe "addlink" do
       it "links two shards" do
-        gizzmo "create TestShard localhost/t0_0"
-        gizzmo "create ReplicatingShard localhost/t0_0_replicating"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+        ns.create_shard info("localhost", "t0_0_replicating", "ReplicatingShard")
+
         gizzmo "addlink localhost/t0_0_replicating localhost/t0_0 1"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard"),
-                                       info("localhost", "t0_0_replicating", "ReplicatingShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard"),
+                                          info("localhost", "t0_0_replicating", "ReplicatingShard")]
 
-        nameserver[:links].should == [link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0"), 1)]
+        nameserver_db[:links].should == [link(id("localhost", "t0_0_replicating"), id("localhost", "t0_0"), 1)]
       end
     end
 
     describe "unlink" do
       it "unlinks two shards" do
-        gizzmo "create TestShard localhost/t0_0"
-        gizzmo "create ReplicatingShard localhost/t0_0_replicating"
+        ns.create_shard info("localhost", "t0_0", "TestShard")
+        ns.create_shard info("localhost", "t0_0_replicating", "ReplicatingShard")
+
         gizzmo "addlink localhost/t0_0_replicating localhost/t0_0 1"
         gizzmo "unlink localhost/t0_0_replicating localhost/t0_0"
 
-        nameserver[:shards].should == [info("localhost", "t0_0", "TestShard"),
-                                       info("localhost", "t0_0_replicating", "ReplicatingShard")]
+        nameserver_db[:shards].should == [info("localhost", "t0_0", "TestShard"),
+                                          info("localhost", "t0_0_replicating", "ReplicatingShard")]
 
-        nameserver[:links].should == []
+        nameserver_db[:links].should == []
       end
     end
 
@@ -160,9 +163,9 @@ describe "gizzmo (cli)" do
         gizzmo "add-host c1:c1host1:7777"
         gizzmo "add-host c2:c2host1:7777 c2:c2host2:7777"
 
-        nameserver[:hosts].should == [ host("c1host1", 7777, "c1"),
-                                       host("c2host1", 7777, "c2"),
-                                       host("c2host2", 7777, "c2") ]
+        nameserver_db[:hosts].should == [host("c1host1", 7777, "c1"),
+                                         host("c2host1", 7777, "c2"),
+                                         host("c2host2", 7777, "c2")]
       end
     end
 
@@ -171,7 +174,7 @@ describe "gizzmo (cli)" do
         gizzmo "add-host c1:c1host1:7777"
         gizzmo "remove-host c1:c1host1:7777"
 
-        nameserver[:hosts].should == []
+        nameserver_db[:hosts].should == []
       end
     end
   end
@@ -179,11 +182,12 @@ describe "gizzmo (cli)" do
   describe "basic read methods" do
     before do
       3.times do |i|
-        gizzmo "create TestShard -s Int -d Int localhost/t0_#{i}_a 127.0.0.1/t0_#{i}_b"
-        gizzmo "create ReplicatingShard localhost/t0_#{i}_replicating"
-        gizzmo "addlink localhost/t0_#{i}_replicating localhost/t0_#{i}_a 1"
-        gizzmo "addlink localhost/t0_#{i}_replicating 127.0.0.1/t0_#{i}_b 1"
-        gizzmo "addforwarding 0 #{i} localhost/t0_#{i}_replicating"
+        ns.create_shard info("localhost", "t0_#{i}_a", "TestShard", "Int", "Int")
+        ns.create_shard info("127.0.0.1", "t0_#{i}_b", "TestShard", "Int", "Int")
+        ns.create_shard info("localhost", "t0_#{i}_replicating", "ReplicatingShard")
+        ns.add_link id("localhost", "t0_#{i}_replicating"), id("localhost", "t0_#{i}_a"), 1
+        ns.add_link id("localhost", "t0_#{i}_replicating"), id("127.0.0.1", "t0_#{i}_b"), 1
+        ns.set_forwarding forwarding(0, i, id("localhost", "t0_#{i}_replicating"))
       end
     end
 
@@ -202,6 +206,12 @@ describe "gizzmo (cli)" do
       end
     end
 
+    describe "tables" do
+      it "prints a list of table ids in the cluster" do
+        gizzmo("tables").should == "0\n"
+      end
+    end
+
     describe "forwardings" do
       it "lists forwardings and the root of the corresponding shard trees" do
         gizzmo("forwardings").should == <<-EOF
@@ -264,9 +274,17 @@ c2:c2host2:7777 0
 
       it "shows the template for each forwarding" do
         gizzmo("-T 0 topology --forwardings").should == <<-EOF
-                        0	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
-                        1	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
-                        2	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
+[0] 0 = localhost/t0_0_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
+[0] 1 = localhost/t0_1_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
+[0] 2 = localhost/t0_2_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
+        EOF
+      end
+
+      it "shows the template for each root shard" do
+        gizzmo("-T 0 topology --shards").should == <<-EOF
+localhost/t0_0_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
+localhost/t0_1_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
+localhost/t0_2_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1,Int,Int))
         EOF
       end
     end
@@ -276,14 +294,6 @@ c2:c2host2:7777 0
     it "works"
   end
 
-  describe "rebalance" do
-    it "works"
-  end
-
-  describe "repair" do
-    it "works"
-  end
-
   describe "reload" do
     it "works"
   end
@@ -326,15 +336,15 @@ c2:c2host2:7777 0
 
   describe "transform-tree" do
     it "works" do
-      gizzmo "create -s Int -d Int TestShard localhost/s_0_001_a"
-      #gizzmo "create TestShard 127.0.0.1/s_0_001_b"
-      gizzmo "create ReplicatingShard localhost/s_0_001_replicating"
-      gizzmo "addlink localhost/s_0_001_replicating localhost/s_0_001_a 1"
-      #gizzmo "addlink localhost/s_0_001_replicating 127.0.0.1/s_0_001_b 1"
-      gizzmo "addforwarding 0 1 localhost/s_0_001_replicating"
-      gizzmo "-f reload"
-
-      gizzmo('-f transform-tree --no-progress --poll-interval=1 "ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))" localhost/s_0_001_replicating').should == <<-EOF
+      ns.create_shard info("localhost", "s_0_001_a", "TestShard", "Int", "Int")
+      ns.create_shard info("localhost", "s_0_001_replicating", "ReplicatingShard")
+      ns.add_link id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1
+      ns.set_forwarding forwarding(0, 1, id("localhost", "s_0_001_replicating"))
+      ns.reload_config
+
+      gizzmo('-f transform-tree --no-progress --poll-interval=1 \
+"ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))" \
+localhost/s_0_001_replicating').should == <<-EOF
 ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1)) :
   PREPARE
     create_shard(TestShard/127.0.0.1)
@@ -358,28 +368,82 @@ FINISHING:
 1 transformation applied. Total time elapsed: 1 second
       EOF
 
-      nameserver[:shards].should == [ info("127.0.0.1", "s_0_0001", "TestShard"),
-                                      info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_0_001_replicating", "ReplicatingShard") ]
+      nameserver_db[:shards].should == [info("127.0.0.1", "s_0_0001", "TestShard"),
+                                        info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_001_replicating", "ReplicatingShard")]
 
-      nameserver[:links].should == [ link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
-                                     link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1) ]
+      nameserver_db[:links].should == [link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
+                                       link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1)]
     end
   end
 
   describe "transform" do
     it "works" do
+      1.upto(2) do |i|
+        ns.create_shard info("localhost", "s_0_00#{i}_a", "TestShard", "Int", "Int")
+        ns.create_shard info("localhost", "s_0_00#{i}_replicating", "ReplicatingShard")
+        ns.add_link id("localhost", "s_0_00#{i}_replicating"), id("localhost", "s_0_00#{i}_a"), 1
+        ns.set_forwarding forwarding(0, i, id("localhost", "s_0_00#{i}_replicating"))
+      end
+      ns.reload_config
+
+      gizzmo('-f -T0 transform --no-progress --poll-interval=1 \
+"ReplicatingShard -> TestShard(localhost,1,Int,Int)" \
+"ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
+ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1)) :
+  PREPARE
+    create_shard(TestShard/127.0.0.1)
+    create_shard(WriteOnlyShard)
+    add_link(WriteOnlyShard -> TestShard/127.0.0.1)
+    add_link(ReplicatingShard -> WriteOnlyShard)
+  COPY
+    copy_shard(TestShard/127.0.0.1)
+  CLEANUP
+    add_link(ReplicatingShard -> TestShard/127.0.0.1)
+    remove_link(WriteOnlyShard -> TestShard/127.0.0.1)
+    remove_link(ReplicatingShard -> WriteOnlyShard)
+    delete_shard(WriteOnlyShard)
+Applied to 2 shards:
+  [0] 1 = localhost/s_0_001_replicating
+  [0] 2 = localhost/s_0_002_replicating
+
+STARTING:
+  [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
+  [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
+COPIES:
+  localhost/s_0_001_a -> 127.0.0.1/s_0_0001
+  localhost/s_0_002_a -> 127.0.0.1/s_0_0002
+FINISHING:
+  [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
+  [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
+2 transformations applied. Total time elapsed: 1 second
+      EOF
+
+      nameserver_db[:shards].should == [info("127.0.0.1", "s_0_0001", "TestShard"),
+                                        info("127.0.0.1", "s_0_0002", "TestShard"),
+                                        info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_001_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_0_002_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_002_replicating", "ReplicatingShard")]
+
+      nameserver_db[:links].should == [link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
+                                       link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1),
+                                       link(id("localhost", "s_0_002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
+                                       link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1)]
+    end
+
+    it "works with multiple pages" do
       1.upto(2) do |i|
         gizzmo "create TestShard -s Int -d Int localhost/s_0_00#{i}_a"
-        #gizzmo "create TestShard -s Int -d Int 127.0.0.1/s_0_000#{i}_b"
         gizzmo "create ReplicatingShard localhost/s_0_00#{i}_replicating"
         gizzmo "addlink localhost/s_0_00#{i}_replicating localhost/s_0_00#{i}_a 1"
-        #gizzmo "addlink localhost/s_0_00#{i}_replicating 127.0.0.1/s_0_000#{i}_b 1"
         gizzmo "addforwarding 0 #{i} localhost/s_0_00#{i}_replicating"
       end
       gizzmo "-f reload"
 
-      gizzmo('-f -T0 transform --no-progress --poll-interval=1 "ReplicatingShard -> TestShard(localhost,1,Int,Int)" "ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
+      gizzmo('-f -T0 transform --no-progress --poll-interval=1 --max-copies=1 \
+"ReplicatingShard -> TestShard(localhost,1,Int,Int)" \
+"ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
 ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1)) :
   PREPARE
     create_shard(TestShard/127.0.0.1)
@@ -399,43 +463,46 @@ Applied to 2 shards:
 
 STARTING:
   [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
-  [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
 COPIES:
   localhost/s_0_001_a -> 127.0.0.1/s_0_0001
-  localhost/s_0_002_a -> 127.0.0.1/s_0_0002
 FINISHING:
   [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
+STARTING:
   [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
-2 transformations applied. Total time elapsed: 1 second
+COPIES:
+  localhost/s_0_002_a -> 127.0.0.1/s_0_0002
+FINISHING:
+  [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1))
+2 transformations applied. Total time elapsed: 2 seconds
       EOF
 
-      nameserver[:shards].should == [ info("127.0.0.1", "s_0_0001", "TestShard"),
-                                      info("127.0.0.1", "s_0_0002", "TestShard"),
-                                      info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_0_001_replicating", "ReplicatingShard"),
-                                      info("localhost", "s_0_002_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_0_002_replicating", "ReplicatingShard") ]
-
-      nameserver[:links].should == [ link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
-                                     link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1),
-                                     link(id("localhost", "s_0_002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
-                                     link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1) ]
+      nameserver_db[:shards].should == [info("127.0.0.1", "s_0_0001", "TestShard"),
+                                        info("127.0.0.1", "s_0_0002", "TestShard"),
+                                        info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_001_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_0_002_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_002_replicating", "ReplicatingShard")]
+
+      nameserver_db[:links].should == [link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
+                                       link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1),
+                                       link(id("localhost", "s_0_002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
+                                       link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1)]
     end
 
     it "works with multiple forwarding tables" do
       0.upto(1) do |table|
         1.upto(2) do |i|
-          gizzmo "create TestShard -s Int -d Int localhost/s_#{table}_00#{i}_a"
-          #gizzmo "create TestShard -s Int -d Int 127.0.0.1/s_#{table}_000#{i}_b"
-          gizzmo "create ReplicatingShard localhost/s_#{table}_00#{i}_replicating"
-          gizzmo "addlink localhost/s_#{table}_00#{i}_replicating localhost/s_#{table}_00#{i}_a 1"
-          #gizzmo "addlink localhost/s_#{table}_00#{i}_replicating 127.0.0.1/s_#{table}_000#{i}_b 1"
-          gizzmo "addforwarding #{table} #{i} localhost/s_#{table}_00#{i}_replicating"
+          ns.create_shard info("localhost", "s_#{table}_00#{i}_a", "TestShard", "Int", "Int")
+          ns.create_shard info("localhost", "s_#{table}_00#{i}_replicating", "ReplicatingShard")
+          ns.add_link id("localhost", "s_#{table}_00#{i}_replicating"), id("localhost", "s_#{table}_00#{i}_a"), 1
+          ns.set_forwarding forwarding(table, i, id("localhost", "s_#{table}_00#{i}_replicating"))
         end
       end
-      gizzmo "-f reload"
+      ns.reload_config
 
-      gizzmo('-f -T0,1 transform --no-progress --poll-interval=1 "ReplicatingShard -> TestShard(localhost,1,Int,Int)" "ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
+      gizzmo('-f -T0,1 transform --no-progress --poll-interval=1 \
+"ReplicatingShard -> TestShard(localhost,1,Int,Int)" \
+"ReplicatingShard -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1))"').should == <<-EOF
 ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int), TestShard(127.0.0.1,1)) :
   PREPARE
     create_shard(TestShard/127.0.0.1)
@@ -473,27 +540,81 @@ FINISHING:
 4 transformations applied. Total time elapsed: 1 second
       EOF
 
-      nameserver[:shards].should == [ info("127.0.0.1", "s_0_0001", "TestShard"),
-                                      info("127.0.0.1", "s_0_0002", "TestShard"),
-                                      info("127.0.0.1", "s_1_0001", "TestShard"),
-                                      info("127.0.0.1", "s_1_0002", "TestShard"),
-                                      info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_0_001_replicating", "ReplicatingShard"),
-                                      info("localhost", "s_0_002_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_0_002_replicating", "ReplicatingShard"),
-                                      info("localhost", "s_1_001_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_1_001_replicating", "ReplicatingShard"),
-                                      info("localhost", "s_1_002_a", "TestShard", "Int", "Int"),
-                                      info("localhost", "s_1_002_replicating", "ReplicatingShard") ]
-
-      nameserver[:links].should == [ link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
-                                     link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1),
-                                     link(id("localhost", "s_0_002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
-                                     link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1),
-                                     link(id("localhost", "s_1_001_replicating"), id("127.0.0.1", "s_1_0001"), 1),
-                                     link(id("localhost", "s_1_001_replicating"), id("localhost", "s_1_001_a"), 1),
-                                     link(id("localhost", "s_1_002_replicating"), id("127.0.0.1", "s_1_0002"), 1),
-                                     link(id("localhost", "s_1_002_replicating"), id("localhost", "s_1_002_a"), 1) ]
+      nameserver_db[:shards].should == [info("127.0.0.1", "s_0_0001", "TestShard"),
+                                        info("127.0.0.1", "s_0_0002", "TestShard"),
+                                        info("127.0.0.1", "s_1_0001", "TestShard"),
+                                        info("127.0.0.1", "s_1_0002", "TestShard"),
+                                        info("localhost", "s_0_001_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_001_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_0_002_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_0_002_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_1_001_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_1_001_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_1_002_a", "TestShard", "Int", "Int"),
+                                        info("localhost", "s_1_002_replicating", "ReplicatingShard")]
+
+      nameserver_db[:links].should == [link(id("localhost", "s_0_001_replicating"), id("127.0.0.1", "s_0_0001"), 1),
+                                       link(id("localhost", "s_0_001_replicating"), id("localhost", "s_0_001_a"), 1),
+                                       link(id("localhost", "s_0_002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
+                                       link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1),
+                                       link(id("localhost", "s_1_001_replicating"), id("127.0.0.1", "s_1_0001"), 1),
+                                       link(id("localhost", "s_1_001_replicating"), id("localhost", "s_1_001_a"), 1),
+                                       link(id("localhost", "s_1_002_replicating"), id("127.0.0.1", "s_1_0002"), 1),
+                                       link(id("localhost", "s_1_002_replicating"), id("localhost", "s_1_002_a"), 1)]
+    end
+  end
+
+  describe "rebalance" do
+    it "works" do
+      1.upto(8) do |i|
+        ns.create_shard info("localhost","s_0_00#{i}_a","TestShard")
+        ns.create_shard info("localhost","s_0_00#{i}_replicating","ReplicatingShard")
+        ns.add_link id("localhost", "s_0_00#{i}_replicating"), id("localhost", "s_0_00#{i}_a"), 1
+        ns.set_forwarding forwarding(0,i,id("localhost", "s_0_00#{i}_replicating"))
+      end
+      ns.reload_config
+
+      gizzmo('-f -T0 rebalance --no-progress --poll-interval=1 \
+1 "ReplicatingShard -> TestShard(127.0.0.1,1)" \
+1 "ReplicatingShard -> TestShard(localhost,1)"').should match(Regexp.new(Regexp.escape(<<-EOF).gsub("X", "\\d")))
+ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1) :
+  PREPARE
+    create_shard(TestShard/127.0.0.1)
+    create_shard(WriteOnlyShard)
+    add_link(WriteOnlyShard -> TestShard/127.0.0.1)
+    add_link(ReplicatingShard -> WriteOnlyShard)
+  COPY
+    copy_shard(TestShard/127.0.0.1)
+  CLEANUP
+    add_link(ReplicatingShard -> TestShard/127.0.0.1)
+    remove_link(ReplicatingShard -> TestShard/localhost)
+    remove_link(WriteOnlyShard -> TestShard/127.0.0.1)
+    remove_link(ReplicatingShard -> WriteOnlyShard)
+    delete_shard(TestShard/localhost)
+    delete_shard(WriteOnlyShard)
+Applied to 4 shards:
+  [0] X = localhost/s_0_00X_replicating
+  [0] X = localhost/s_0_00X_replicating
+  [0] X = localhost/s_0_00X_replicating
+  [0] X = localhost/s_0_00X_replicating
+
+STARTING:
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+COPIES:
+  localhost/s_0_00X_a -> 127.0.0.1/s_0_000X
+  localhost/s_0_00X_a -> 127.0.0.1/s_0_000X
+  localhost/s_0_00X_a -> 127.0.0.1/s_0_000X
+  localhost/s_0_00X_a -> 127.0.0.1/s_0_000X
+FINISHING:
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  [0] X = localhost/s_0_00X_replicating: ReplicatingShard(1) -> TestShard(localhost,1) => ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+4 transformations applied. Total time elapsed: 1 second
+      EOF
     end
   end
 end