gizzmo 0.11.3 → 0.11.4

data/VERSION

@@ -1 +1 @@
-0.11.3
+0.11.4

data/gizzmo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gizzmo}
-  s.version = "0.11.3"
+  s.version = "0.11.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2011-01-26}
+  s.date = %q{2011-04-19}
   s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
   s.email = %q{kmaxwell@twitter.com}
   s.executables = ["gizzmo", "setup_shards"]

data/lib/gizzard/commands.rb

@@ -27,6 +27,7 @@ module Gizzard
       end
 
       def make_manager(global_options, log)
+        raise "No hosts specified" unless global_options.hosts
         hosts = global_options.hosts.map {|h| [h, global_options.port].join(":") }
 
         Nameserver.new(hosts, :retries => global_options.retry,
@@ -502,6 +503,24 @@ module Gizzard
     end
   end
 
+  class RepairShardsCommand < Command
+    def run
+      shard_id_strings = @argv
+      help!("Requires at least two shard ids") unless shard_id_strings.size >= 2
+      shard_ids = shard_id_strings.map{|s| ShardId.parse(s)}
+      manager.repair_shards(shard_ids)
+    end
+  end
+
+  class DiffShardsCommand < Command
+    def run
+      shard_id_strings = @argv
+      help!("Requires at least two shard ids") unless shard_id_strings.size >= 2
+      shard_ids = shard_id_strings.map{|s| ShardId.parse(s)}
+      manager.diff_shards(shard_ids)
+    end
+  end
+
   class BusyCommand < Command
     def run
       manager.get_busy_shards().each { |shard_info| output shard_info.to_unix }
@@ -793,7 +812,7 @@ module Gizzard
         exit
       end
 
-      base_name = transformations.values.first.values.first.id.table_prefix.split('_').first
+      base_name = transformations.values.find {|v| v.is_a?(Hash) && !v.values.empty? }.values.find {|v| !v.nil?}.id.table_prefix.split('_').first
 
       unless be_quiet
         transformations.sort.each do |transformation, trees|

data/lib/gizzard/nameserver.rb

@@ -1,6 +1,4 @@
 module Gizzard
-  Shard = Struct.new(:info, :children, :weight)
-
   module ParallelMap
     def parallel_map(enumerable, &block)
       enumerable.map do |elem|
@@ -12,7 +10,7 @@ module Gizzard
     end
   end
 
-  class Shard
+  class Shard < Struct.new(:info, :children, :weight)
     class << self
       def canonical_table_prefix(enum, table_id = nil, base_prefix = "shard")
         enum_s         = "%0.4i" % enum
@@ -125,6 +123,16 @@ module Gizzard
       with_retry { c.copy_shard(from_shard_id, to_shard_id) }
     end
 
+    def repair_shards(*shards)
+      c = random_client
+      with_retry { c.repair_shard(*shards) }
+    end
+
+    def diff_shards(*shards)
+      c = random_client
+      with_retry { c.diff_shards(*shards) }
+    end
+
     def respond_to?(method)
       client.respond_to? method or super
     end
@@ -166,10 +174,9 @@ module Gizzard
     def with_retry
       times ||= @retries
       yield
-    rescue ThriftClient::Simple::ThriftException, NoMethodError, Gizzard::GizzardException => e
-      raise if e.is_a? Gizzard::GizzardException and e.message !~ /Communications link failure/
-
-      times -= 1
+    rescue Exception => e
+      puts "retrying #{e.inspect} ..."
+      #times -= 1
       (times < 0) ? raise : (sleep 2; retry)
     end
 

data/lib/gizzard/shard_template.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 module Gizzard
   class ShardTemplate
     include Comparable

data/lib/gizzard/thrift.rb

@@ -184,6 +184,8 @@ module Gizzard
 
     thrift_method :mark_shard_busy, void, field(:id, struct(ShardId), 1), field(:busy, i32, 2), :throws => exception(GizzardException)
     thrift_method :copy_shard, void, field(:source_id, struct(ShardId), 1), field(:destination_id, struct(ShardId), 2), :throws => exception(GizzardException)
+    thrift_method :repair_shard, void, field(:shard_ids, list(struct(ShardId)), 1), :throws => exception(GizzardException)
+    thrift_method :diff_shards, void, field(:shard_ids, list(struct(ShardId)), 1), :throws => exception(GizzardException)
 
     thrift_method :dump_nameserver, list(struct(NameServerState)), field(:table_ids, list(i32), 1), :throws => exception(GizzardException)
 

data/lib/gizzard/transformation.rb

@@ -12,7 +12,9 @@ module Gizzard
       Op::CreateShard      => "create_shard",
       Op::AddLink          => "add_link",
       Op::SetForwarding    => "set_forwarding",
-      Op::CopyShard        => "copy_shard"
+      Op::CopyShard        => "copy_shard",
+      Op::RepairShards     => "repair_shards",
+      Op::DiffShards       => "diff_shards"
     }
 
     OP_INVERSES = {
@@ -30,7 +32,9 @@ module Gizzard
       Op::RemoveForwarding => 4,
       Op::RemoveLink       => 5,
       Op::DeleteShard      => 6,
-      Op::CopyShard        => 7
+      Op::CopyShard        => 7,
+      Op::RepairShards     => 8,
+      Op::DiffShards       => 9
     }
 
     DEFAULT_DEST_WRAPPER = 'WriteOnlyShard'
@@ -89,9 +93,11 @@ module Gizzard
 
       prepare_inspect = op_inspect[:prepare].empty? ? "" : "  PREPARE\n#{op_inspect[:prepare]}\n"
       copy_inspect    = op_inspect[:copy].empty?    ? "" : "  COPY\n#{op_inspect[:copy]}\n"
+      repair_inspect  = op_inspect[:repair].empty?  ? "" : "  REPAIR\n#{op_inspect[:repair]}\n"
+      diff_inspect    = op_inspect[:diff].empty?  ? "" : "  DIFF\n#{op_inspect[:diff]}\n"
       cleanup_inspect = op_inspect[:cleanup].empty? ? "" : "  CLEANUP\n#{op_inspect[:cleanup]}\n"
 
-      op_inspect = [prepare_inspect, copy_inspect, cleanup_inspect].join
+      op_inspect = [prepare_inspect, copy_inspect, repair_inspect, cleanup_inspect].join
 
       "#{from.inspect} => #{to.inspect} :\n#{op_inspect}"
     end
@@ -124,7 +130,7 @@ module Gizzard
     end
 
     def expand_jobs(jobs)
-      expanded = jobs.inject({:prepare => [], :copy => [], :cleanup => []}) do |ops, job|
+      expanded = jobs.inject({:prepare => [], :copy => [], :repair => [], :cleanup => [], :diff => []}) do |ops, job|
         job_ops = job.expand(self.copy_source, involved_in_copy?(job.template), @copy_dest_wrapper)
         ops.update(job_ops) {|k,a,b| a + b }
       end

data/lib/gizzard/transformation_op.rb

@@ -52,6 +52,48 @@ module Gizzard
       end
     end
 
+    class RepairShards < BaseOp
+      attr_reader :from, :to
+      alias template to
+
+      def initialize(*shards)
+        @shards = shards
+      end
+
+      def expand(*args); { :repair => [self] } end
+
+      def involved_shards(table_prefix, translations)
+        shards.map{|s| s.to_shard_id(table_prefix, translations)}
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        nameserver.repair_shards(involved_shards(table_prefix, translations))
+      end
+    end
+
+    class DiffShards < BaseOp
+      attr_reader :from, :to
+      alias template to
+
+      def initialize(from, to)
+        @from = from
+        @to   = to
+      end
+
+      def expand(*args); { :repair => [self] } end
+
+      def involved_shards(table_prefix, translations)
+        [to.to_shard_id(table_prefix, translations)]
+      end
+
+      def apply(nameserver, table_id, base_id, table_prefix, translations)
+        from_shard_id = from.to_shard_id(table_prefix, translations)
+        to_shard_id   = to.to_shard_id(table_prefix, translations)
+
+        nameserver.diff_shards(from_shard_id, to_shard_id)
+      end
+    end
+
     class LinkOp < BaseOp
       attr_reader :from, :to
       alias template to

data/lib/gizzmo.rb

@@ -24,6 +24,8 @@ DOC_STRINGS = {
   "markbusy" => "Mark a shard as busy.",
   "pair" => "Report the replica pairing structure for a list of hosts.",
   "reload" => "Instruct application servers to reload the nameserver state.",
+  "repair-shards" => "Reconcile n shards by detecting differences and rescheduling them",
+  "diff-shards" => "Log differences between n shards",
   "report" => "Show each unique replica structure for a given list of shards. Usually this shard list comes from << gizzmo forwardings | awk '{ print $3 }' >>.",
   "setup-replica" => "Add a replica to be parallel to an existing replica, in write-only mode, ready to be copied to.",
   "wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
@@ -246,6 +248,18 @@ subcommands = {
     opts.banner = "Usage: #{zero} copy SOURCE_SHARD_ID DESTINATION_SHARD_ID"
     separators(opts, DOC_STRINGS["copy"])
   end,
+  'repair-shards' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} repair-shards SHARD_IDS..."
+    separators(opts, DOC_STRINGS["repair-shards"])
+  end,
+  'diff-shards' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} diff-shards SHARD_IDS..."
+    separators(opts, DOC_STRINGS["diff-shards"])
+  end,
+  'diff-shards' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} diff-shards SOURCE_SHARD_ID DESTINATION_SHARD_ID"
+    separators(opts, DOC_STRINGS["diff-shards"])
+  end,
   'busy' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} busy"
     separators(opts, DOC_STRINGS["busy"])

data/test/gizzmo_spec.rb

@@ -318,6 +318,14 @@ localhost/t0_2_replicating	ReplicatingShard(1) -> (TestShard(localhost,1,Int,Int
     it "works"
   end
 
+  describe "repair-shards" do
+    it "works"
+  end
+
+  describe "diff-shards" do
+    it "works"
+  end
+
   describe "setup-migrate" do
     it "works"
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: gizzmo
 version: !ruby/object:Gem::Version 
-  hash: 53
+  hash: 59
   prerelease: false
   segments: 
   - 0
   - 11
-  - 3
-  version: 0.11.3
+  - 4
+  version: 0.11.4
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-26 00:00:00 -08:00
+date: 2011-04-19 00:00:00 -07:00
 default_executable: 
 dependencies: []