gizzmo 0.11.4 → 0.12.0

data/VERSION

@@ -1 +1 @@
-0.11.4
+0.12.0

data/gizzmo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gizzmo}
-  s.version = "0.11.4"
+  s.version = "0.12.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Kyle Maxwell"]
-  s.date = %q{2011-04-19}
+  s.date = %q{2011-05-05}
   s.description = %q{Gizzmo is a command-line client for managing gizzard clusters.}
   s.email = %q{kmaxwell@twitter.com}
   s.executables = ["gizzmo", "setup_shards"]

data/lib/gizzard/commands.rb

@@ -892,4 +892,110 @@ module Gizzard
                         scheduler_options
     end
   end
+
+
+  class CreateTableCommand < Command
+
+    DEFAULT_NUM_SHARDS   = 1024
+    DEFAULT_BASE_NAME    = "shard"
+
+    FORWARDING_SPACE     = 2 ** 60
+    FORWARDING_SPACE_MIN = 0
+    FORWARDING_SPACE_MAX = 2 ** 60 - 1
+
+    def generate_base_ids(num_shards, min_id, max_id)
+      srand(42) # consistent randomization
+
+      id_space  = max_id - min_id + 1
+      step_size = id_space / num_shards
+
+      enums = (0...num_shards).to_a
+      ids   = enums.map {|i| min_id + (i * step_size) }.sort_by { rand }
+
+      enums.zip(ids)
+    end
+
+    def parse_templates_and_weights(arr)
+      templates_and_weights = {}
+
+      arr.each_slice(2) do |(weight_s, to_template_s)|
+        to     = ShardTemplate.parse(to_template_s)
+        weight = weight_s.to_i
+
+        templates_and_weights[to] = weight
+      end
+
+      templates_and_weights
+    end
+
+    # This is all super hacky but I don't have time to generalize right now
+
+    def run
+      help!("must have an even number of arguments") unless @argv.length % 2 == 0
+
+      base_name    = command_options.base_name || DEFAULT_BASE_NAME
+      num_shards   = (command_options.shards || DEFAULT_NUM_SHARDS).to_i
+      max_id       = (command_options.max_id || FORWARDING_SPACE_MAX).to_i
+      min_id       = (command_options.min_id || FORWARDING_SPACE_MIN).to_i
+
+      be_quiet     = global_options.force && command_options.quiet
+
+      templates_and_weights = parse_templates_and_weights(@argv)
+      total_weight = templates_and_weights.values.inject {|a,b| a + b }
+      templates = templates_and_weights.keys
+
+      base_ids = generate_base_ids(num_shards, min_id, max_id)
+
+      templates_and_base_ids = templates_and_weights.inject({}) do |h, (template, weight)|
+        share = (weight.to_f / total_weight * num_shards).floor
+        ids   = []
+        share.times { ids << base_ids.pop }
+
+        h.update template => ids
+      end
+
+      # divvy up the remainder across all templates.
+      base_ids.each_with_index do |base, idx|
+        templates_and_base_ids.values[idx % templates_and_base_ids.length] << base
+      end
+
+      proto     = templates.first
+      transform = Transformation.new(proto, proto)
+
+      op_sets = templates.inject({}) do |h, template|
+        ops = transform.create_tree(template).sort
+        h.update template => ops
+      end
+
+      unless be_quiet
+        puts "Create tables #{global_options.tables.join(", ")}:"
+        templates_and_base_ids.each do |template, base_ids|
+          puts "  #{template.inspect}"
+          puts "  for #{base_ids.length} base ids:"
+          base_ids.each {|(enum, base_id)| puts "    #{base_id}" }
+        end
+        puts ""
+      end
+
+      unless global_options.force
+        print "Continue? (y/n) "; $stdout.flush
+        exit unless $stdin.gets.chomp == "y"
+        puts ""
+      end
+
+      global_options.tables.each do |table_id|
+        templates_and_base_ids.each do |template, base_ids|
+          ops = op_sets[template]
+
+          base_ids.each do |(enum, base_id)|
+            table_prefix = Shard.canonical_table_prefix(enum, table_id, base_name)
+            ops.each do |op|
+              puts "#{op.inspect}: #{table_prefix}"
+              op.apply(manager, table_id, base_id, table_prefix, {})
+            end
+          end
+        end
+      end
+    end
+  end
 end

data/lib/gizzard/nameserver.rb

@@ -87,7 +87,7 @@ module Gizzard
     include ParallelMap
 
     DEFAULT_PORT    = 7917
-    DEFAULT_RETRIES = 20
+    DEFAULT_RETRIES = 10
     PARALLELISM     = 10
 
     attr_reader :hosts, :logfile, :dryrun, :framed
@@ -175,9 +175,8 @@ module Gizzard
       times ||= @retries
       yield
     rescue Exception => e
-      puts "retrying #{e.inspect} ..."
-      #times -= 1
-      (times < 0) ? raise : (sleep 2; retry)
+      times -= 1
+      (times < 0) ? raise : (sleep 0.1; retry)
     end
 
     class Manifest

data/lib/gizzard/shard_template.rb

@@ -126,7 +126,13 @@ module Gizzard
     def shared_host?(other)
       raise ArgumentError, "other is not a ShardTemplate" unless other.is_a? ShardTemplate
 
-      (self.concrete_descendants & other.concrete_descendants).length > 0
+      self.concrete_descendants.each do |s|
+        other.concrete_descendants.each do |o|
+          return true if s.shard_eql? o
+        end
+      end
+
+      false
     end
 
     def hash

data/lib/gizzard/transformation.rb

@@ -147,8 +147,9 @@ module Gizzard
 
     def copies_required?
       return @copies_required unless  @copies_required.nil?
+
       @copies_required = !from.nil? &&
-        to.concrete_descendants.reject {|d| from.shared_host? d }.length > 0
+        to.concrete_descendants.select {|d| !from.shared_host? d }.length > 0
     end
 
     def involved_in_copy?(template)

data/lib/gizzmo.rb

@@ -345,6 +345,30 @@ subcommands = {
     opts.on("-q", "--quiet", "Do not display transformation info (only valid with --force)") do
       subcommand_options.quiet = true
     end
+  end,
+  'create-table' => OptionParser.new do |opts|
+    opts.banner = "Usage: #{zero} create-table [options] WEIGHT TEMPLATE ..."
+    separators(opts, DOC_STRINGS["create-table"])
+
+    opts.on("--shards=COUNT", "Create COUNT shards for each table.") do |count|
+      subcommand_options.shards = count.to_i
+    end
+
+    opts.on("--min-id=NUM", "Set lower bound on the id space to NUM (default min signed long: -1 * 2^63)") do |min_id|
+      subcommand_options.min_id = min_id.to_i
+    end
+
+    opts.on("--max-id=NUM", "Set upper bound on the id space to NUM (default max signed long: 2^63 - 1)") do |max_id|
+      subcommand_options.max_id = max_id.to_i
+    end
+
+    opts.on("--base-name=NAME", "Use NAME as the base prefix for each shard's table prefix (default 'shard')") do |base_name|
+      subcommand_options.base_name = base_name
+    end
+
+    opts.on("-q", "--quiet", "Do not display table creation info (only valid with --force)") do
+      subcommand_options.quiet = true
+    end
   end
 }
 
@@ -388,12 +412,6 @@ global = OptionParser.new do |opts|
     global_options.hosts = hosts.split(",").map {|h| h.strip }
   end
 
-  opts.on("-H", "--host=HOST", "HOST of application servers") do |hosts|
-    global_options.hosts = hosts.split(",").map {|h| h.strip }
-  end
-
-
-
   opts.on("-P", "--port=PORT", "PORT of remote manager service. default 7920") do |port|
     global_options.port = port.to_i
   end

data/test/gizzmo_spec.rb

@@ -440,6 +440,38 @@ FINISHING:
                                        link(id("localhost", "s_0_002_replicating"), id("localhost", "s_0_002_a"), 1)]
     end
 
+    it "properly re-weights shards" do
+      1.upto(2) do |i|
+        gizzmo "create TestShard -s Int -d Int localhost/s_0_00#{i}_a"
+        gizzmo "create ReplicatingShard localhost/s_0_00#{i}_replicating"
+        gizzmo "addlink localhost/s_0_00#{i}_replicating localhost/s_0_00#{i}_a 1"
+        gizzmo "addforwarding 0 #{i} localhost/s_0_00#{i}_replicating"
+      end
+      gizzmo "-f reload"
+
+      gizzmo('-f -T0 transform --no-progress --poll-interval=1 --max-copies=1 \
+"ReplicatingShard -> TestShard(localhost,1,Int,Int)" \
+"ReplicatingShard -> TestShard(localhost,3,Int,Int)"').should == <<-EOF
+ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int) :
+  PREPARE
+    add_link(ReplicatingShard -> TestShard/localhost)
+    remove_link(ReplicatingShard -> TestShard/localhost)
+Applied to 2 shards:
+  [0] 1 = localhost/s_0_001_replicating
+  [0] 2 = localhost/s_0_002_replicating
+
+STARTING:
+  [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+FINISHING:
+  [0] 2 = localhost/s_0_002_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+STARTING:
+  [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+FINISHING:
+  [0] 1 = localhost/s_0_001_replicating: ReplicatingShard(1) -> TestShard(localhost,1,Int,Int) => ReplicatingShard(1) -> TestShard(localhost,3,Int,Int)
+2 transformations applied. Total time elapsed: 2 seconds
+EOF
+    end
+
     it "works with multiple pages" do
       1.upto(2) do |i|
         gizzmo "create TestShard -s Int -d Int localhost/s_0_00#{i}_a"
@@ -625,4 +657,91 @@ FINISHING:
       EOF
     end
   end
+
+  describe "create-table" do
+    it "works" do
+      gizzmo('-f -T0,1 create-table --shards=4 --base-name=s \
+1 "ReplicatingShard -> TestShard(127.0.0.1,1)" \
+1 "ReplicatingShard -> TestShard(localhost,1)"').should match(Regexp.new(Regexp.escape(<<-EOF).gsub("X", "\\d")))
+Create tables 0, 1:
+  ReplicatingShard(1) -> TestShard(127.0.0.1,1)
+  for 2 base ids:
+    288230376151711744
+    576460752303423488
+  ReplicatingShard(1) -> TestShard(localhost,1)
+  for 2 base ids:
+    864691128455135232
+    0
+
+create_shard(ReplicatingShard): s_0_0003
+create_shard(TestShard/127.0.0.1): s_0_0003
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_0_0003
+set_forwarding(ReplicatingShard): s_0_0003
+create_shard(ReplicatingShard): s_0_0002
+create_shard(TestShard/127.0.0.1): s_0_0002
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_0_0002
+set_forwarding(ReplicatingShard): s_0_0002
+create_shard(ReplicatingShard): s_0_0001
+create_shard(TestShard/localhost): s_0_0001
+add_link(ReplicatingShard -> TestShard/localhost): s_0_0001
+set_forwarding(ReplicatingShard): s_0_0001
+create_shard(ReplicatingShard): s_0_0000
+create_shard(TestShard/localhost): s_0_0000
+add_link(ReplicatingShard -> TestShard/localhost): s_0_0000
+set_forwarding(ReplicatingShard): s_0_0000
+create_shard(ReplicatingShard): s_1_0003
+create_shard(TestShard/127.0.0.1): s_1_0003
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_1_0003
+set_forwarding(ReplicatingShard): s_1_0003
+create_shard(ReplicatingShard): s_1_0002
+create_shard(TestShard/127.0.0.1): s_1_0002
+add_link(ReplicatingShard -> TestShard/127.0.0.1): s_1_0002
+set_forwarding(ReplicatingShard): s_1_0002
+create_shard(ReplicatingShard): s_1_0001
+create_shard(TestShard/localhost): s_1_0001
+add_link(ReplicatingShard -> TestShard/localhost): s_1_0001
+set_forwarding(ReplicatingShard): s_1_0001
+create_shard(ReplicatingShard): s_1_0000
+create_shard(TestShard/localhost): s_1_0000
+add_link(ReplicatingShard -> TestShard/localhost): s_1_0000
+set_forwarding(ReplicatingShard): s_1_0000
+      EOF
+
+      nameserver_db[:shards].should == [info("127.0.0.1", "s_0_0002", "TestShard"),
+                                        info("127.0.0.1", "s_0_0003", "TestShard"),
+                                        info("127.0.0.1", "s_1_0002", "TestShard"),
+                                        info("127.0.0.1", "s_1_0003", "TestShard"),
+                                        info("localhost", "s_0_0000", "TestShard"),
+                                        info("localhost", "s_0_0000_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_0_0001", "TestShard"),
+                                        info("localhost", "s_0_0001_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_0_0002_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_0_0003_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_1_0000", "TestShard"),
+                                        info("localhost", "s_1_0000_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_1_0001", "TestShard"),
+                                        info("localhost", "s_1_0001_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_1_0002_replicating", "ReplicatingShard"),
+                                        info("localhost", "s_1_0003_replicating", "ReplicatingShard")]
+
+      nameserver_db[:links].should == [link(id("localhost", "s_0_0000_replicating"), id("localhost", "s_0_0000"), 1),
+                                       link(id("localhost", "s_0_0001_replicating"), id("localhost", "s_0_0001"), 1),
+                                       link(id("localhost", "s_0_0002_replicating"), id("127.0.0.1", "s_0_0002"), 1),
+                                       link(id("localhost", "s_0_0003_replicating"), id("127.0.0.1", "s_0_0003"), 1),
+                                       link(id("localhost", "s_1_0000_replicating"), id("localhost", "s_1_0000"), 1),
+                                       link(id("localhost", "s_1_0001_replicating"), id("localhost", "s_1_0001"), 1),
+                                       link(id("localhost", "s_1_0002_replicating"), id("127.0.0.1", "s_1_0002"), 1),
+                                       link(id("localhost", "s_1_0003_replicating"), id("127.0.0.1", "s_1_0003"), 1)]
+
+      nameserver_db[:forwardings].should == [forwarding(0, 0, id("localhost", "s_0_0000_replicating")),
+                                             forwarding(1, 0, id("localhost", "s_1_0000_replicating")),
+                                             forwarding(0, 288230376151711744, id("localhost", "s_0_0003_replicating")),
+                                             forwarding(1, 288230376151711744, id("localhost", "s_1_0003_replicating")),
+                                             forwarding(0, 576460752303423488, id("localhost", "s_0_0002_replicating")),
+                                             forwarding(1, 576460752303423488, id("localhost", "s_1_0002_replicating")),
+                                             forwarding(0, 864691128455135232, id("localhost", "s_0_0001_replicating")),
+                                             forwarding(1, 864691128455135232, id("localhost", "s_1_0001_replicating"))]
+
+    end
+  end
 end

data/test/spec_helper.rb

@@ -130,7 +130,7 @@ def as_link(h)
 end
 
 def as_forwarding(h)
-  Gizzard::Forwarding.new(h['table_id'].to_i, h['base_id'].to_i, as_shard_id(h, 'shard'))
+  Gizzard::Forwarding.new(h['table_id'].to_i, h['base_source_id'].to_i, as_shard_id(h, 'shard'))
 end
 
 def as_host(h)
@@ -144,14 +144,14 @@ def gizzmo(cmd)
 end
 
 def nameserver
-  @nameserver ||= Gizzard::Nameserver.new('localhost:' + MANAGER_PORT.to_s)
+  @nameserver ||= Gizzard::Nameserver.new('localhost:' + MANAGER_PORT.to_s, :retries => 5)
 end
 
 alias ns nameserver
 
 # setup
 
-mysql_connect!("localhost", '', '')
+mysql_connect!("localhost", 'root', '')
 reset_databases!
 
 unless ENV['EXTERNAL_TEST_SERVER']

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: gizzmo
 version: !ruby/object:Gem::Version 
-  hash: 59
+  hash: 47
   prerelease: false
   segments: 
   - 0
-  - 11
-  - 4
-  version: 0.11.4
+  - 12
+  - 0
+  version: 0.12.0
 platform: ruby
 authors: 
 - Kyle Maxwell
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-19 00:00:00 -07:00
+date: 2011-05-05 00:00:00 -07:00
 default_executable: 
 dependencies: []