gizzmo 0.10.0 → 0.10.1

data/VERSION

@@ -1 +1 @@
-0.10.0
+0.10.1

data/gizzmo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gizzmo}
-  s.version = "0.10.0"
+  s.version = "0.10.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2010-11-02}
+  s.date = %q{2011-01-05}
   s.default_executable = %q{gizzmo}
   s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
   s.email = %q{kmaxwell@twitter.com}
@@ -19,8 +19,7 @@ Gem::Specification.new do |s|
      "README.rdoc"
   ]
   s.files = [
-    ".document",
-     ".gitignore",
+    ".gitignore",
      "LICENSE",
      "README.rdoc",
      "Rakefile",

data/lib/gizzard/commands.rb

@@ -1,5 +1,6 @@
 require "pp"
 require "digest/md5"
+
 module Gizzard
   class Command
     include Thrift
@@ -17,7 +18,7 @@ module Gizzard
     end
 
     def self.classify(string)
-      string.split(/\W+/).map{|s| s.capitalize }.join("")
+      string.split(/\W+/).map { |s| s.capitalize }.join("")
     end
 
     attr_reader :service, :global_options, :argv, :command_options
@@ -41,13 +42,13 @@ module Gizzard
       end
     end
   end
-  
+
   class RetryProxy
     def initialize(retries, object)
       @inner = object
       @retries_left = retries
     end
-    
+
     def method_missing(*args)
       @inner.send(*args)
     rescue
@@ -63,15 +64,15 @@ module Gizzard
 
   class ShardCommand < Command
     def self.make_service(global_options, log)
-      RetryProxy.new global_options.retry.to_i, 
-        Gizzard::Thrift::ShardManager.new(global_options.host, global_options.port, log, global_options.dry)
+      RetryProxy.new global_options.retry.to_i,
+        Gizzard::Thrift::ShardManager.new(global_options.host, global_options.port, log, global_options.framed, global_options.dry)
     end
   end
 
   class JobCommand < Command
     def self.make_service(global_options, log)
       RetryProxy.new global_options.retry.to_i  ,
-        Gizzard::Thrift::JobManager.new(global_options.host, global_options.port + 2, log, global_options.dry)
+        Gizzard::Thrift::JobManager.new(global_options.host, global_options.port + 2, log, global_options.framed, global_options.dry)
     end
   end
 
@@ -147,7 +148,18 @@ module Gizzard
   class ReloadCommand < ShardCommand
     def run
       if global_options.force || ask
-        service.reload_forwardings
+        if @argv
+          # allow hosts to be given on the command line
+          @argv.each do |hostname|
+            output hostname
+            opts = global_options.dup
+            opts.host = hostname
+            s = self.class.make_service(opts, global_options.log || "./gizzmo.log")
+            s.reload_forwardings
+          end
+        else
+          service.reload_forwardings
+        end
       else
         STDERR.puts "aborted"
       end
@@ -201,7 +213,10 @@ module Gizzard
         upward_links = service.list_upward_links(shard_id)
         downward_links = service.list_downward_links(shard_id)
 
-        help! "Shard must not be a root or leaf" if upward_links.length == 0 or downward_links.length == 0
+        if upward_links.length == 0 or downward_links.length == 0
+          STDERR.puts "Shard #{shard_id_string} must not be a root or leaf"
+          next
+        end
 
         upward_links.each do |uplink|
           downward_links.each do |downlink|
@@ -239,11 +254,15 @@ module Gizzard
       shard_ids.each do |shard_id_text|
         shard_id = ShardId.parse(shard_id_text)
         next if !shard_id
-        service.list_upward_links(shard_id).each do |link_info|
-          output link_info.to_unix
+        unless command_options.down
+          service.list_upward_links(shard_id).each do |link_info|
+            output command_options.ids ? link_info.up_id.to_unix : link_info.to_unix
+          end
         end
-        service.list_downward_links(shard_id).each do |link_info|
-          output link_info.to_unix
+        unless command_options.up
+          service.list_downward_links(shard_id).each do |link_info|
+            output command_options.ids ? link_info.down_id.to_unix : link_info.to_unix
+          end
         end
       end
     end
@@ -406,7 +425,7 @@ module Gizzard
 
       puts "gizzmo create #{shard_info.class_name} -s '#{shard_info.source_type}' -d '#{shard_info.destination_type}' #{new_shards.join(" ")}"
       puts "gizzmo wrap #{command_options.write_only_shard} #{new_shards.join(" ")}"
-      shards.map {|(old, new)| puts "gizzmo copy #{old} #{new}" }
+      shards.map { |(old, new)| puts "gizzmo copy #{old} #{new}" }
     end
   end
 
@@ -433,16 +452,16 @@ module Gizzard
 
       overlaps = {}
       ids_by_table.values.each do |arr|
-        key = arr.map{|id| id.hostname }.sort
+        key = arr.map { |id| id.hostname }.sort
         overlaps[key] ||= 0
-        overlaps[key]  += 1
+        overlaps[key] += 1
       end
 
       displayed = {}
-      overlaps.sort_by{|hosts, count| count }.reverse.each do |(host_a, host_b), count|
+      overlaps.sort_by { |hosts, count| count }.reverse.each do |(host_a, host_b), count|
         next if !host_a || !host_b || displayed[host_a] || displayed[host_b]
-        id_a = ids_by_host[host_a].find{|id| service.list_upward_links(id).size > 0 }
-        id_b = ids_by_host[host_b].find{|id| service.list_upward_links(id).size > 0 }
+        id_a = ids_by_host[host_a].find { |id| service.list_upward_links(id).size > 0 }
+        id_b = ids_by_host[host_b].find { |id| service.list_upward_links(id).size > 0 }
         next unless id_a && id_b
         weight_a = service.list_upward_links(id_a).first.weight
         weight_b = service.list_upward_links(id_b).first.weight
@@ -466,7 +485,6 @@ module Gizzard
 
   class ReportCommand < ShardCommand
     def run
-
       things = @argv.map do |shard|
         parse(down(ShardId.parse(shard))).join("\n")
       end
@@ -492,20 +510,20 @@ module Gizzard
         m[e] ||= []
         m[e] << e
         m
-      end.to_a.sort_by{|k, v| v.length}.reverse
+      end.to_a.sort_by { |k, v| v.length }.reverse
     end
 
     def parse(obj, id = nil, depth = 0, sub = true)
       case obj
       when Hash
         id, prefix = parse(obj.keys.first, id, depth, sub)
-        [prefix] + parse(obj.values.first, id, depth + 1, sub)
+        [ prefix ] + parse(obj.values.first, id, depth + 1, sub)
       when String
         host, prefix = obj.split("/")
         host = "db" if host != "localhost" && sub
-        id ||= prefix[/(\w+ward_)?\d+_\d+(_\w+ward)?/]
+        id ||= prefix[/(\w+ward_)?n?\d+_\d+(_\w+ward)?/]
         prefix = ("  " * depth) + host + "/" + ((sub && id) ? prefix.sub(id, "[ID]") : prefix)
-        [id, prefix]
+        [ id, prefix ]
       when Array
         obj.map do |e|
           parse e, id, depth, sub
@@ -517,7 +535,7 @@ module Gizzard
       vals = service.list_downward_links(id).map do |link|
         down(link.down_id)
       end
-      {id.to_unix => vals}
+      { id.to_unix => vals }
     end
   end
 
@@ -575,6 +593,60 @@ module Gizzard
     end
   end
 
+  class SetupReplicaCommand < ShardCommand
+    def run
+      from_shard_id_string, to_shard_id_string = @argv
+      help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
+      from_shard_id = ShardId.parse(from_shard_id_string)
+      to_shard_id = ShardId.parse(to_shard_id_string)
+
+      if service.list_upward_links(to_shard_id).size > 0
+        STDERR.puts "Destination shard #{to_shard_id} has links to it."
+        exit 1
+      end
+
+      link = service.list_upward_links(from_shard_id)[0]
+      replica_shard_id = link.up_id
+      weight = link.weight
+      write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
+      service.create_shard(ShardInfo.new(write_only_shard_id, "WriteOnlyShard", "", "", 0))
+      service.add_link(replica_shard_id, write_only_shard_id, weight)
+      service.add_link(write_only_shard_id, to_shard_id, 1)
+      output to_shard_id.to_unix
+    end
+  end
+
+  class FinishReplicaCommand < ShardCommand
+    def run
+      from_shard_id_string, to_shard_id_string = @argv
+      help!("Requires source & destination shard id") unless from_shard_id_string && to_shard_id_string
+      from_shard_id = ShardId.parse(from_shard_id_string)
+      to_shard_id = ShardId.parse(to_shard_id_string)
+
+      write_only_shard_id = ShardId.new("localhost", "#{to_shard_id.table_prefix}_copy_write_only")
+      link = service.list_upward_links(write_only_shard_id)[0]
+      replica_shard_id = link.up_id
+      weight = link.weight
+
+      # careful. need to validate some basic assumptions.
+      unless global_options.force
+        if service.list_upward_links(from_shard_id).map { |link| link.up_id }.to_a != [ replica_shard_id ]
+          STDERR.puts "Uplink from #{from_shard_id} is not a migration replica."
+          exit 1
+        end
+        if service.list_upward_links(to_shard_id).map { |link| link.up_id }.to_a != [ write_only_shard_id ]
+          STDERR.puts "Uplink from #{to_shard_id} is not a write-only barrier."
+          exit 1
+        end
+      end
+
+      service.remove_link(write_only_shard_id, to_shard_id)
+      service.remove_link(replica_shard_id, write_only_shard_id)
+      service.add_link(replica_shard_id, to_shard_id, weight)
+      service.delete_shard(write_only_shard_id)
+    end
+  end
+
   class SetupMigrateCommand < ShardCommand
     def run
       from_shard_id_string, to_shard_id_string = @argv

data/lib/gizzard/thrift.rb

@@ -52,7 +52,7 @@ module Gizzard
       end
 
       def to_unix
-        [id.to_unix, class_name, busy? ? "busy" : "unbusy"].join("\t")
+        [id.to_unix, class_name, busy? ? "busy" : "ok"].join("\t")
       end
     end
 
@@ -91,8 +91,8 @@ module Gizzard
     end
 
     class GizzmoService < T::ThriftService
-      def initialize(host, port, log_path, dry_run = false)
-        super(host, port)
+      def initialize(host, port, log_path, framed, dry_run = false)
+        super(host, port, framed)
         @dry = dry_run
         begin
           @log = File.open(log_path, "a")

data/lib/gizzmo.rb

@@ -7,11 +7,25 @@ require "gizzard"
 require "yaml"
 
 DOC_STRINGS = {
-  "create" => "Create shard(s) of a given Java/Scala class.  If you don't know the list of available classes, you can just try a bogus class, and the exception will include a list of valid classes.",
-  "wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
+  "addforwarding" => "Add a forwarding from a graph_id / base_source_id to a given shard.",
+  "addlink" => "Add a relationship link between two shards.",
+  "create" => "Create shard(s) of a given Java/Scala class. If you don't know the list of available classes, you can just try a bogus class, and the exception will include a list of valid classes.",
+  "drill" => "Show shard trees for replicas of a given structure signature (from 'report').",
+  "find" => "Show all shards with a given hostname.",
+  "finish-replica" => "Remove the write-only barrier in front of a shard that's finished being copied after 'setup-replica'.",
+  "flush" => "Flush error queue for a given priority.",
+  "forwardings" => "Get a list of all forwardings.",
+  "hosts" => "List hosts used in shard names in the forwarding table and replicas.",
+  "info" => "Show id/class/busy for shards.",
   "inject" => "Inject jobs (as literal json) into the server. Jobs can be linefeed-terminated from stdin, or passed as arguments. Priority is server-defined, but typically lower numbers (like 1) are lower priority.",
+  "links" => "List parent & child links for shards.",
   "lookup" => "Lookup the shard id that holds the record for a given table / source_id.",
-  "flush" => "Flush error queue for a given priority."
+  "markbusy" => "Mark a shard as busy.",
+  "pair" => "Report the replica pairing structure for a list of hosts.",
+  "reload" => "Instruct an appserver to reload its nameserver state.",
+  "report" => "Show each unique replica structure for a given list of shards. Usually this shard list comes from << gizzmo forwardings | awk '{ print $3 }' >>.",
+  "setup-replica" => "Add a replica to be parallel to an existing replica, in write-only mode, ready to be copied to.",
+  "wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
 }
 
 ORIGINAL_ARGV = ARGV.dup
@@ -20,6 +34,7 @@ zero = File.basename($0)
 # Container for parsed options
 global_options     = OpenStruct.new
 global_options.render = []
+global_options.framed = false
 subcommand_options = OpenStruct.new
 
 # Leftover arguments
@@ -61,6 +76,12 @@ def separators(opts, string)
   opts.separator("")
 end
 
+def load_config(options, filename)
+  YAML.load(File.open(filename)).each do |k, v|
+    options.send("#{k}=", v)
+  end
+end
+
 subcommands = {
   'create' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} create [options] CLASS_NAME SHARD_ID [MORE SHARD_IDS...]"
@@ -163,6 +184,16 @@ subcommands = {
   'links' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} links SHARD_ID [MORE SHARD_IDS...]"
     separators(opts, DOC_STRINGS["links"])
+
+    opts.on("--ids", "Show shard ids only") do
+      subcommand_options.ids = true
+    end
+    opts.on("--up", "Show uplinks only") do
+      subcommand_options.up = true
+    end
+    opts.on("--down", "show downlinks only") do
+      subcommand_options.down = true
+    end
   end,
   'info' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} info SHARD_ID [MORE SHARD_IDS...]"
@@ -209,6 +240,14 @@ subcommands = {
     opts.banner = "Usage: #{zero} busy"
     separators(opts, DOC_STRINGS["busy"])
   end,
+  'setup-replica' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} setup-replica SOURCE_SHARD_ID DESTINATION_SHARD_ID"
+    separators(opts, DOC_STRINGS["setup-replica"])
+  end,
+  'finish-replica' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} finish-replica SOURCE_SHARD_ID DESTINATION_SHARD_ID"
+    separators(opts, DOC_STRINGS["finish-replica"])
+  end,
   'setup-migrate' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} setup-migrate SOURCE_SHARD_ID DESTINATION_SHARD_ID"
     separators(opts, DOC_STRINGS["setup-migrate"])
@@ -231,6 +270,10 @@ subcommands = {
   end
 }
 
+if ENV['GIZZMORC']
+  load_config(global_options, ENV['GIZZMORC'])
+end
+
 global = OptionParser.new do |opts|
   opts.banner = "Usage: #{zero} [global-options] SUBCOMMAND [subcommand-options]"
   opts.separator ""
@@ -243,7 +286,7 @@ global = OptionParser.new do |opts|
   opts.separator ""
   opts.separator "You may find it useful to create a ~/.gizzmorc file, which is simply YAML"
   opts.separator "key/value pairs corresponding to options you want by default. A common .gizzmorc"
-  opts.separator "simply contain:"
+  opts.separator "simply contains:"
   opts.separator ""
   opts.separator "    host: localhost"
   opts.separator "    port: 7917"
@@ -270,6 +313,10 @@ global = OptionParser.new do |opts|
     global_options.port = port.to_i
   end
 
+  opts.on("-F", "--framed", "use the thrift framed transport") do |framed|
+    global_options.framed = true
+  end
+
   opts.on("-r", "--retry=TIMES", "TIMES to retry the command") do |r|
     global_options.retry = r
   end
@@ -290,10 +337,8 @@ global = OptionParser.new do |opts|
     global_options.dry = true
   end
 
-  opts.on("-C", "--config=YAML_FILE", "YAML_FILE of option key/values") do |file|
-    YAML.load(File.open(file)).each do |k, v|
-      global_options.send("#{k}=", v)
-    end
+  opts.on("-C", "--config=YAML_FILE", "YAML_FILE of option key/values") do |filename|
+    load_config(global_options, filename)
   end
 
   opts.on("-L", "--log=LOG_FILE", "Path to LOG_FILE") do |file|
@@ -370,7 +415,7 @@ def custom_timeout(seconds)
   end
 end
 
-begin      
+begin
   custom_timeout(global_options.timeout) do
     Gizzard::Command.run(subcommand_name, global_options, argv, subcommand_options, log)
   end

data/lib/vendor/thrift_client/simple.rb

@@ -87,8 +87,14 @@ class ThriftClient
         end
       end
 
-      def pack_request(method_name, arg_struct, request_id=0)
-        [ VERSION_1, CALL, method_name.to_s.size, method_name.to_s, request_id, arg_struct._pack ].pack("nnNa*Na*")
+      def pack_request(method_name, arg_struct, framed, request_id=0)
+        msg = [ VERSION_1, CALL, method_name.to_s.size, method_name.to_s, request_id, arg_struct._pack ].pack("nnNa*Na*")
+        if framed
+          frame = [ msg.length ].pack("N")
+          frame + msg
+        else
+          msg
+        end
       end
 
       def read_value(s, type)
@@ -169,8 +175,13 @@ class ThriftClient
           end
         end
       end
-      
-      def read_response(s, rv_class)
+
+      def read_response(s, rv_class, framed)
+        if framed
+          # unwrap frame size. dont use for now
+          framesize = s.read(4).unpack("N")
+        end
+
         version, message_type, method_name_len = s.read(8).unpack("nnN")
         method_name = s.read(method_name_len)
         seq_id = s.read(4).unpack("N").first
@@ -280,9 +291,10 @@ class ThriftClient
     UnknownStruct = make_struct(:Unknown)
 
     class ThriftService
-      def initialize(host, port)
+      def initialize(host, port, framed = false)
         @host = host
         @port = port
+        @framed = framed
       end
 
       def self._arg_structs
@@ -310,8 +322,8 @@ class ThriftClient
         arg_class, rv_class = cls._arg_structs[method_name.to_sym]
         arg_struct = arg_class.new(*args)
         sock = TCPSocket.new(@host, @port)
-        sock.write(ThriftClient::Simple.pack_request(method_name, arg_struct))
-        rv = ThriftClient::Simple.read_response(sock, rv_class)
+        sock.write(ThriftClient::Simple.pack_request(method_name, arg_struct, @framed))
+        rv = ThriftClient::Simple.read_response(sock, rv_class, @framed)
         sock.close
         rv[2]
       end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 10
-  - 0
-  version: 0.10.0
+  - 1
+  version: 0.10.1
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-02 00:00:00 -07:00
+date: 2011-01-05 00:00:00 -08:00
 default_executable: gizzmo
 dependencies: []
 
@@ -28,7 +28,6 @@ extra_rdoc_files:
 - LICENSE
 - README.rdoc
 files: 
-- .document
 - .gitignore
 - LICENSE
 - README.rdoc

data/.document

@@ -1,5 +0,0 @@
-README.rdoc
-lib/**/*.rb
-bin/*
-features/**/*.feature
-LICENSE