gizzmo 0.7.0 → 0.7.4

data/VERSION

@@ -1 +1 @@
-0.7.0
+0.7.4

data/gizzmo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gizzmo}
-  s.version = "0.7.0"
+  s.version = "0.7.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2010-08-25}
+  s.date = %q{2010-10-04}
   s.default_executable = %q{gizzmo}
   s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
   s.email = %q{kmaxwell@twitter.com}
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
      "gizzmo.gemspec",
      "lib/gizzard.rb",
      "lib/gizzard/commands.rb",
+     "lib/gizzard/digest.rb",
      "lib/gizzard/thrift.rb",
      "lib/gizzmo.rb",
      "lib/vendor/thrift_client/simple.rb",

data/lib/gizzard.rb

@@ -1,4 +1,5 @@
 $: << File.dirname(__FILE__)
 module Gizzard; end
 require "gizzard/thrift"
-require "gizzard/commands"
+require "gizzard/commands"
+require "gizzard/digest"

data/lib/gizzard/commands.rb

@@ -5,13 +5,13 @@ module Gizzard
 
     attr_reader :buffer
 
-    def self.run(command_name, global_options, argv, subcommand_options, log)
+    def self.run(command_name, global_options, argv, subcommand_options, log, service=nil)
       command_class = Gizzard.const_get("#{classify(command_name)}Command")
-      service = command_class.make_service(global_options, log)
+      service = command_class.make_service(global_options, log) if service.nil?
       command = command_class.new(service, global_options, argv, subcommand_options)
       command.run
       if command.buffer && command_name = global_options.render.shift
-        run(command_name, service, global_options, command.buffer, OpenStruct.new)
+        run(command_name, global_options, command.buffer, OpenStruct.new, log, service)
       end
     end
 
@@ -86,7 +86,7 @@ module Gizzard
       end.reject do |forwarding|
         @command_options.table_ids && !@command_options.table_ids.include?(forwarding.table_id)
       end.each do |forwarding|
-        output [ forwarding.table_id, forwarding.base_id, forwarding.shard_id.to_unix ].join("\t")
+        output [ forwarding.table_id, @command_options.hex ? ("%016x" % forwarding.base_id) : forwarding.base_id, forwarding.shard_id.to_unix ].join("\t")
       end
     end
   end
@@ -229,7 +229,7 @@ module Gizzard
       end
     end
   end
-  
+
   class RepairCommand < ShardCommand
     def run
       args = @argv.dup.map{|a| a.split(/\s+/)}.flatten
@@ -289,7 +289,9 @@ module Gizzard
     end
 
     def run
-      puts command_options.inspect
+      help! "No shards specified" if @argv.empty?
+      shards = []
+      command_options.write_only_shard ||= "com.twitter.gizzard.shards.WriteOnlyShard"
       additional_hosts = (command_options.hosts || "").split(/[\s,]+/)
       ids = @argv.map{|arg| ShardId.new(*arg.split("/")) rescue nil }.compact
       by_host = ids.inject({}) do |memo, id|
@@ -297,7 +299,7 @@ module Gizzard
         memo[id.hostname] << id
         memo
       end
-      
+
       additional_hosts.each do |host|
         by_host[host] ||= NamedArray.new(host)
       end
@@ -312,7 +314,6 @@ module Gizzard
       end while longest.length > shortest.length + 1
 
       shard_info = nil
-      puts sets.map{|l|l.length}.inspect
       sets.each do |set|
         host = set.name
         set.each do |id|
@@ -320,11 +321,17 @@ module Gizzard
             shard_info ||= service.get_shard(id)
             old = id.to_unix
             id.hostname = host
-            puts "gizzmo create #{shard_info.class_name} -s '#{shard_info.source_type}' -d '#{shard_info.destination_type}' #{id.to_unix}"
-            puts "gizzmo copy #{old} #{id.to_unix}"
+            shards << [old, id.to_unix]
           end
         end
       end
+
+      new_shards = shards.map{|(old, new)| new }
+
+      puts "gizzmo create #{shard_info.class_name} -s '#{shard_info.source_type}' -d '#{shard_info.destination_type}' #{new_shards.join(" ")}"
+      puts "gizzmo wrap #{command_options.write_only_shard} #{new_shards.join(" ")}"
+      shards.map {|(old, new)| puts "gizzmo copy #{old} #{new}" }
+      shards.map {|(old, new)| puts "gizzmo copy #{old} #{new}" }
     end
   end
 
@@ -359,8 +366,9 @@ module Gizzard
       displayed = {}
       overlaps.sort_by{|hosts, count| count }.reverse.each do |(host_a, host_b), count|
         next if !host_a || !host_b || displayed[host_a] || displayed[host_b]
-        id_a = ids_by_host[host_a].first
-        id_b = ids_by_host[host_b].first
+        id_a = ids_by_host[host_a].find{|id| service.list_upward_links(id).size > 0 }
+        id_b = ids_by_host[host_b].find{|id| service.list_upward_links(id).size > 0 }
+        next unless id_a && id_b
         weight_a = service.list_upward_links(id_a).first.weight
         weight_b = service.list_upward_links(id_b).first.weight
         if weight_a > weight_b
@@ -417,9 +425,15 @@ module Gizzard
 
   class LookupCommand < ShardCommand
     def run
-      table_id, source_id = @argv
-      help!("Requires table id and source id") unless table_id && source_id
-      shard = service.find_current_forwarding(table_id.to_i, source_id.to_i)
+      table_id, source = @argv
+      help!("Requires table id and source") unless table_id && source
+      case @command_options.hash_function
+      when :fnv
+        source_id = Digest.fnv1a_64(source)
+      else
+        source_id = source.to_i
+      end
+      shard = service.find_current_forwarding(table_id.to_i, source_id)
       output shard.id.to_unix
     end
   end
@@ -521,4 +535,16 @@ module Gizzard
       STDERR.print "\n"
     end
   end
+
+  class FlushCommand < JobCommand
+    def run
+      args = @argv[0]
+      help!("Requires --all, or a job priority id.") unless args || command_options.flush_all
+      if command_options.flush_all
+        service.retry_errors()
+      else
+        service.retry_errors_for(args.to_i)
+      end
+    end
+  end
 end

data/lib/gizzard/digest.rb

@@ -0,0 +1,13 @@
+module Gizzard
+  module Digest
+    def self.fnv1a_64(data)
+      prime = 1099511628211
+      rv = 0xcbf29ce484222325
+      data.each_byte do |byte|
+        rv = ((rv ^ (byte & 0xff)) * prime) & 0xffffffffffffffff
+      end
+      # trim to 60 bits for gizzard.
+      rv & 0x0fffffffffffffff
+    end
+  end
+end

data/lib/gizzmo.rb

@@ -9,7 +9,9 @@ require "yaml"
 DOC_STRINGS = {
   "create" => "Create shard(s) of a given Java/Scala class.  If you don't know the list of available classes, you can just try a bogus class, and the exception will include a list of valid classes.",
   "wrap" => "Wrapping creates a new (virtual, e.g. blocking, replicating, etc.) shard, and relinks SHARD_ID_TO_WRAP's parent links to run through the new shard.",
-  "inject" => "Inject jobs (as literal json) into the server. Jobs can be linefeed-terminated from stdin, or passed as arguments. Priority is server-defined, but typically lower numbers (like 1) are lower priority."
+  "inject" => "Inject jobs (as literal json) into the server. Jobs can be linefeed-terminated from stdin, or passed as arguments. Priority is server-defined, but typically lower numbers (like 1) are lower priority.",
+  "lookup" => "Lookup the shard id that holds the record for a given table / source_id.",
+  "flush" => "Flush error queue for a given priority."
 }
 
 ORIGINAL_ARGV = ARGV.dup
@@ -84,6 +86,10 @@ subcommands = {
     opts.banner = "Usage: #{zero} rebalance"
     separators(opts, DOC_STRINGS["rebalance"])
 
+    opts.on("-w", "--write-only=CLASS") do |w|
+      subcommand_options.write_only_shard = w
+    end
+
     opts.on("-h", "--hosts=list") do |h|
       subcommand_options.hosts = h
     end
@@ -128,6 +134,9 @@ subcommands = {
       subcommand_options.table_ids ||= []
       subcommand_options.table_ids +=  table_ids.split(",").map { |s| s.to_i }
     end
+    opts.on("-x", "--hex", "Show base ids in hex") do
+      subcommand_options.hex = true
+    end
   end,
   'unwrap' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} unwrap SHARD_ID_TO_REMOVE [MORE SHARD_IDS]"
@@ -158,7 +167,7 @@ subcommands = {
     separators(opts, DOC_STRINGS["reload"])
   end,
   'addlink' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} link PARENT_SHARD_ID CHILD_SHARD_ID WEIGHT"
+    opts.banner = "Usage: #{zero} addlink PARENT_SHARD_ID WEIGHT"
     separators(opts, DOC_STRINGS["addlink"])
   end,
   'unlink' => OptionParser.new do |opts|
@@ -166,8 +175,12 @@ subcommands = {
     separators(opts, DOC_STRINGS["unlink"])
   end,
   'lookup' => OptionParser.new do |opts|
-    opts.banner = "Usage: #{zero} lookup TABLE_ID SOURCE_ID"
+    opts.banner = "Usage: #{zero} lookup [options] TABLE_ID SOURCE"
     separators(opts, DOC_STRINGS["lookup"])
+
+    opts.on("--fnv", "Use FNV1A_64 hash on source") do
+      subcommand_options.hash_function = :fnv
+    end
   end,
   'copy' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} copy SOURCE_SHARD_ID DESTINATION_SHARD_ID"
@@ -188,6 +201,14 @@ subcommands = {
   'inject' => OptionParser.new do |opts|
     opts.banner = "Usage: #{zero} inject PRIORITY JOBS..."
     separators(opts, DOC_STRINGS["inject"])
+  end,
+  'flush' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} flush --all|PRIORITY"
+    separators(opts, DOC_STRINGS["flush"])
+
+    opts.on("--all", "Flush all error queues.") do
+      subcommand_options.flush_all = true
+    end
   end
 }
 
@@ -227,7 +248,7 @@ global = OptionParser.new do |opts|
   end
 
   opts.on("-P", "--port=PORT", "PORT of remote thrift service") do |port|
-    global_options.port = port
+    global_options.port = port.to_i
   end
 
   opts.on("-r", "--retry=TIMES", "TIMES to retry the command") do |r|
@@ -338,4 +359,4 @@ rescue Errno::EPIPE
   # seeing the backtrace is annoying!
 rescue Interrupt
   exit 1
-end
+end

data/test/test.sh

@@ -6,7 +6,7 @@ function g {
 }
 
 function expect {
-  diff -u - "expected/$1" && echo "    success." || echo "    failed." && exit 1
+  diff -u - "expected/$1" && echo "    success." || (echo "    failed." && exit 1)
 }
 
 function expect-string {
@@ -19,9 +19,9 @@ g find -hlocalhost | expect empty-file.txt
 
 for i in {0..9}
 do
-  g create localhost "table_repl_$i" com.twitter.service.flock.edges.ReplicatingShard
-  g create localhost "table_a_$i" com.twitter.service.flock.edges.SqlShard --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
-  g create localhost "table_b_$i" com.twitter.service.flock.edges.SqlShard --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
+  g create com.twitter.service.flock.edges.ReplicatingShard localhost/table_repl_$i
+  g create com.twitter.service.flock.edges.SqlShard localhost/table_a_$i --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
+  g create com.twitter.service.flock.edges.SqlShard localhost/table_b_$i --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
   g addlink "localhost/table_repl_$i" "localhost/table_a_$i" 2
   g addlink "localhost/table_repl_$i" "localhost/table_b_$i" 1
 done
@@ -52,6 +52,7 @@ g links localhost/replicating_table_b_0 | expect links-for-replicating_table_b_0
 g --subtree --info find -Hlocalhost | expect subtree-info.txt
 
 g lookup 1 100 | expect-string "localhost/forward_1"
+g lookup --fnv 1 "hello" | expect-string "localhost/forward_1"
 
 g unwrap localhost/replicating_table_b_0 | expect unwrapped-replicating_table_b_0.txt
 g links localhost/table_b_0 | expect unwrapped-table_b_0.txt
@@ -72,4 +73,7 @@ do
   g addlink "localhost/table_deep_repl_$last" "localhost/table_deep_repl_$i" 2
 done
 
-g subtree localhost/table_deep_repl_5 | expect deep.txt
+g subtree localhost/table_deep_repl_5 | expect deep.txt
+
+g flush --all
+g flush 1

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 7
-  - 0
-  version: 0.7.0
+  - 4
+  version: 0.7.4
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-25 00:00:00 -07:00
+date: 2010-10-04 00:00:00 -07:00
 default_executable: gizzmo
 dependencies: []
 
@@ -38,6 +38,7 @@ files:
 - gizzmo.gemspec
 - lib/gizzard.rb
 - lib/gizzard/commands.rb
+- lib/gizzard/digest.rb
 - lib/gizzard/thrift.rb
 - lib/gizzmo.rb
 - lib/vendor/thrift_client/simple.rb