gizzmo 0.11.0 → 0.11.1

data/test/test_server/.gitignore

@@ -0,0 +1,13 @@
+/dist/*
+*.log
+*.o
+gen-cpp/
+bin/
+target/
+.DS_Store
+/kestrel/*
+*.hprof.txt
+lib_managed/
+src_managed/
+/project/boot/
+/project/plugins/project/

data/test/test_server/project/build.properties

@@ -0,0 +1,8 @@
+#Project properties
+#Mon Nov 22 21:09:11 PST 2010
+project.organization=com.twitter
+project.name=gizzmotestserver
+sbt.version=0.7.4
+project.version=0.1
+build.scala.versions=2.7.7
+project.initialize=false

data/test/test_server/project/build/Project.scala

@@ -0,0 +1,13 @@
+import sbt._
+import com.twitter.sbt.StandardProject
+
+
+class GizzmoServerProject(info: ProjectInfo) extends StandardProject(info) {
+  override def compileOptions = super.compileOptions ++ Seq(Unchecked)
+  override def filterScalaJars = false
+
+  val scalaTools = "org.scala-lang" % "scala-compiler" % "2.7.7"
+  val gizzard    = "com.twitter" % "gizzard" % "1.6-mc-SNAPSHOT"
+
+  val specs = "org.scala-tools.testing" % "specs" % "1.6.2.1" % "test"
+}

data/test/test_server/project/plugins/Plugins.scala

@@ -0,0 +1,6 @@
+import sbt._
+
+class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
+  val twitterMaven = "twitter.com" at "http://maven.twttr.com/"
+  val defaultProject = "com.twitter" % "standard-project" % "0.7.12"
+}

data/test/test_server/src/main/scala/Main.scala

@@ -0,0 +1,18 @@
+package com.twitter.gizzard.testserver
+
+import com.twitter.gizzard.testserver.config.TestServerConfig
+
+object Main {
+  var service: TestServer = null
+
+  def main(args: Array[String]) {
+    val config = args match {
+      case Array(a)       => TestServerConfig("integration", a.toInt)
+      case Array(a, b, c) => TestServerConfig("integration", a.toInt, b.toInt, c.toInt)
+    }
+
+    service = new TestServer(config)
+
+    service.start()
+  }
+}

data/test/test_server/src/main/scala/TestServer.scala

@@ -0,0 +1,247 @@
+package com.twitter.gizzard.testserver
+
+import java.sql.{ResultSet, SQLException}
+import com.twitter.querulous.evaluator.{QueryEvaluatorFactory, QueryEvaluator}
+import com.twitter.querulous.config.Connection
+import com.twitter.querulous.query.SqlQueryTimeoutException
+import gizzard.GizzardServer
+import nameserver.NameServer
+import shards.{ShardId, ShardInfo, ShardException, ShardTimeoutException}
+import scheduler.{JobScheduler, JsonJob, CopyJob, CopyJobParser, CopyJobFactory, JsonJobParser, PrioritizingJobScheduler}
+
+object config {
+  import com.twitter.gizzard.config._
+  import com.twitter.querulous.config._
+  import com.twitter.util.TimeConversions._
+  import com.twitter.util.Duration
+
+  trait TestDBConnection extends Connection {
+    val username = "root"
+    val password = ""
+    val hostnames = Seq("localhost")
+  }
+
+  object TestQueryEvaluator extends querulous.config.QueryEvaluator {
+    database.pool = new ApachePoolingDatabase {
+      sizeMin = 3
+      sizeMax = 3
+    }
+  }
+
+  trait TestTHsHaServer extends THsHaServer {
+    threadPool.minThreads = 10
+  }
+
+  trait TestServer extends gizzard.config.GizzardServer {
+    def server: TServer
+    def databaseConnection: Connection
+    val queryEvaluator = TestQueryEvaluator
+  }
+
+  trait TestJobScheduler extends Scheduler {
+    val schedulerType = new KestrelScheduler {
+      path = "/tmp"
+      keepJournal = false
+    }
+    errorLimit = 25
+  }
+
+  class TestNameServer(name: String) extends gizzard.config.NameServer {
+    jobRelay.priority = Priority.Low.id
+
+    val replicas = Seq(new Mysql {
+      queryEvaluator = TestQueryEvaluator
+      val connection = new TestDBConnection {
+        val database = "gizzard_test_" + name + "_ns"
+      }
+    })
+  }
+
+  object TestServerConfig {
+    def apply(name: String, sPort: Int, iPort: Int, mPort: Int) = {
+      val queueBase = "gizzard_test_" + name
+
+      new TestServer {
+        val server = new TestTHsHaServer { val name = "TestGizzardService"; val port = sPort }
+        val databaseConnection = new TestDBConnection { val database = "gizzard_test_" + name }
+        val nameServer = new TestNameServer(name)
+        val jobQueues = Map(
+          Priority.High.id -> new TestJobScheduler { val name = queueBase+"_high" },
+          Priority.Low.id  -> new TestJobScheduler { val name = queueBase+"_low" }
+        )
+
+        jobInjector.port = iPort
+        manager.port     = mPort
+      }
+    }
+
+    def apply(name: String, port: Int): TestServer = apply(name, port, port + 1, port + 2)
+  }
+}
+
+
+object Priority extends Enumeration {
+  val High, Low = Value
+}
+
+class TestServer(conf: config.TestServer) extends GizzardServer[TestShard, JsonJob](conf) {
+
+  // shard/nameserver/scheduler wiring
+
+  val readWriteShardAdapter = new TestReadWriteAdapter(_)
+  val jobPriorities         = List(Priority.High.id, Priority.Low.id)
+  val copyPriority          = Priority.Low.id
+  val copyFactory           = new TestCopyFactory(nameServer, jobScheduler(Priority.Low.id))
+
+  shardRepo += ("TestShard" -> new SqlShardFactory(conf.queryEvaluator(), conf.databaseConnection))
+
+  jobCodec += ("Put".r  -> new PutParser(nameServer.findCurrentForwarding(0, _)))
+  jobCodec += ("Copy".r -> new TestCopyParser(nameServer, jobScheduler(Priority.Low.id)))
+
+
+  // service listener
+
+  val testService = new TestServerIFace(nameServer.findCurrentForwarding(0, _), jobScheduler)
+
+  lazy val testThriftServer = {
+    val processor = new thrift.TestServer.Processor(testService)
+    conf.server(processor)
+  }
+
+  def start() {
+    startGizzard()
+    new Thread(new Runnable { def run() { testThriftServer.serve() } }, "TestServerThread").start()
+  }
+
+  def shutdown(quiesce: Boolean) {
+    testThriftServer.stop()
+    shutdownGizzard(quiesce)
+  }
+}
+
+
+// Service Interface
+
+class TestServerIFace(forwarding: Long => TestShard, scheduler: PrioritizingJobScheduler[JsonJob])
+extends thrift.TestServer.Iface {
+  import com.twitter.gizzard.thrift.conversions.Sequences._
+
+  def put(key: Int, value: String) {
+    scheduler.put(Priority.High.id, new PutJob(key, value, forwarding))
+  }
+
+  def get(key: Int) = forwarding(key).get(key).map(asTestResult).map(List(_).toJavaList) getOrElse List[thrift.TestResult]().toJavaList
+
+  private def asTestResult(t: (Int, String, Int)) = new thrift.TestResult(t._1, t._2, t._3)
+}
+
+
+// Shard Definitions
+
+trait TestShard extends shards.Shard {
+  def put(key: Int, value: String): Unit
+  def putAll(kvs: Seq[(Int, String)]): Unit
+  def get(key: Int): Option[(Int, String, Int)]
+  def getAll(key: Int, count: Int): Seq[(Int, String, Int)]
+}
+
+class TestReadWriteAdapter(s: shards.ReadWriteShard[TestShard])
+extends shards.ReadWriteShardAdapter(s) with TestShard {
+  def put(k: Int, v: String)         = s.writeOperation(_.put(k,v))
+  def putAll(kvs: Seq[(Int,String)]) = s.writeOperation(_.putAll(kvs))
+  def get(k: Int)                    = s.readOperation(_.get(k))
+  def getAll(k:Int, c: Int)          = s.readOperation(_.getAll(k,c))
+}
+
+class SqlShardFactory(qeFactory: QueryEvaluatorFactory, conn: Connection)
+extends shards.ShardFactory[TestShard] {
+  def instantiate(info: ShardInfo, weight: Int, children: Seq[TestShard]) =
+    new SqlShard(qeFactory(conn.withHost(info.hostname)), info, weight, children)
+
+  def materialize(info: ShardInfo) {
+    val ddl =
+      """create table if not exists %s (
+           id int(11) not null,
+           value varchar(255) not null,
+           count int(11) not null default 1,
+           primary key (id)
+         ) engine=innodb default charset=utf8"""
+    try {
+      val e = qeFactory(conn.withHost(info.hostname).withoutDatabase)
+      e.execute("create database if not exists " + conn.database)
+      e.execute(ddl.format(conn.database + "." + info.tablePrefix))
+    } catch {
+      case e: SQLException             => throw new ShardException(e.toString)
+      case e: SqlQueryTimeoutException => throw new ShardTimeoutException(e.timeout, info.id)
+    }
+  }
+}
+
+class SqlShard(
+  evaluator: QueryEvaluator,
+  val shardInfo: ShardInfo,
+  val weight: Int,
+  val children: Seq[TestShard])
+extends TestShard {
+  private val table = shardInfo.tablePrefix
+
+  private val putSql = """insert into %s (id, value, count) values (?,?,1) on duplicate key
+                          update value = values(value), count = count+1""".format(table)
+  private val getSql    = "select * from " + table + " where id = ?"
+  private val getAllSql = "select * from " + table + " where id > ? limit ?"
+
+  private def asResult(r: ResultSet) = (r.getInt("id"), r.getString("value"), r.getInt("count"))
+
+  def put(key: Int, value: String) { evaluator.execute(putSql, key, value) }
+  def putAll(kvs: Seq[(Int, String)]) {
+    evaluator.executeBatch(putSql) { b => for ((k,v) <- kvs) b(k,v) }
+  }
+
+  def get(key: Int) = evaluator.selectOne(getSql, key)(asResult)
+  def getAll(key: Int, count: Int) = evaluator.select(getAllSql, key, count)(asResult)
+}
+
+
+// Jobs
+
+class PutParser(forwarding: Long => TestShard) extends JsonJobParser {
+  def apply(map: Map[String, Any]): JsonJob = {
+    new PutJob(map("key").asInstanceOf[Int], map("value").asInstanceOf[String], forwarding)
+  }
+}
+
+class PutJob(key: Int, value: String, forwarding: Long => TestShard) extends JsonJob {
+  def toMap = Map("key" -> key, "value" -> value)
+  def apply() { forwarding(key).put(key, value) }
+}
+
+class TestCopyFactory(ns: NameServer[TestShard], s: JobScheduler[JsonJob])
+extends CopyJobFactory[TestShard] {
+  def apply(src: ShardId, dest: ShardId) = new TestCopy(src, dest, 0, 500, ns, s)
+}
+
+class TestCopyParser(ns: NameServer[TestShard], s: JobScheduler[JsonJob])
+extends CopyJobParser[TestShard] {
+  def deserialize(m: Map[String, Any], src: ShardId, dest: ShardId, count: Int) = {
+    val cursor = m("cursor").asInstanceOf[Int]
+    val count  = m("count").asInstanceOf[Int]
+    new TestCopy(src, dest, cursor, count, ns, s)
+  }
+}
+
+class TestCopy(srcId: ShardId, destId: ShardId, cursor: Int, count: Int,
+               ns: NameServer[TestShard], s: JobScheduler[JsonJob])
+extends CopyJob[TestShard](srcId, destId, count, ns, s) {
+  def copyPage(src: TestShard, dest: TestShard, count: Int) = {
+    val rows = src.getAll(cursor, count).map { case (k,v,c) => (k,v) }
+
+    if (rows.isEmpty) {
+      None
+    } else {
+      dest.putAll(rows)
+      Some(new TestCopy(srcId, destId, rows.last._1, count, ns, s))
+    }
+  }
+
+  def serialize = Map("cursor" -> cursor)
+}

data/test/test_server/src/main/thrift/TestServer.thrift

@@ -0,0 +1,12 @@
+namespace java com.twitter.gizzard.testserver.thrift
+
+struct TestResult {
+  1: i32 key
+  2: string value
+  3: i32 count
+}
+
+service TestServer {
+  void put(1: i32 key, 2: string value)
+  list<TestResult> get(1: i32 key)
+}

data/test/transformation_spec.rb

@@ -0,0 +1,181 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+describe Gizzard::Transformation do
+  Op = Gizzard::Transformation::Op
+
+  def create_shard(t);      Op::CreateShard.new(mk_template(t)) end
+  def delete_shard(t);      Op::DeleteShard.new(mk_template(t)) end
+  def add_link(f, t);       Op::AddLink.new(mk_template(f), mk_template(t)) end
+  def remove_link(f, t);    Op::RemoveLink.new(mk_template(f), mk_template(t)) end
+  def copy_shard(f, t);     Op::CopyShard.new(mk_template(f), mk_template(t)) end
+  def set_forwarding(t);    Op::SetForwarding.new(mk_template(t)) end
+  def remove_forwarding(t); Op::RemoveForwarding.new(mk_template(t)) end
+
+  before do
+    @nameserver = stub!.subject
+    stub(@nameserver).dryrun? { false }
+
+    @config = Gizzard::MigratorConfig.new :prefix => "status", :table_id => 0
+
+    @from_template = mk_template 'ReplicatingShard -> (BlockedShard -> SqlShard(host1), SqlShard(host2))'
+    @to_template   = mk_template 'ReplicatingShard -> (SqlShard(host2), SqlShard(host3))'
+
+    @blocked_template = mk_template 'BlockedShard -> SqlShard(host1)'
+    @host_1_template  = mk_template 'SqlShard(host1)'
+    @host_2_template  = mk_template 'SqlShard(host2)'
+    @host_3_template  = mk_template 'SqlShard(host3)'
+
+    @trans = Gizzard::Transformation.new(@from_template, @to_template)
+  end
+
+  describe "initialization" do
+    it "allows an optional copy wrapper type" do
+      Gizzard::Transformation.new(@from_template, @to_template)
+      Gizzard::Transformation.new(@from_template, @to_template, 'WriteOnlyShard')
+      Gizzard::Transformation.new(@from_template, @to_template, 'BlockedShard')
+      lambda do
+        Gizzard::Transformation.new(@from_template, @to_template, 'InvalidWrapperShard')
+      end.should raise_error(ArgumentError)
+    end
+  end
+
+  # internal method tests
+
+  describe "operations" do
+    it "does a basic replica addition" do
+      from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
+      to   = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2), SqlShard(host3))'
+
+      Gizzard::Transformation.new(from, to, 'BlockedShard').operations.should == {
+        :prepare => [ create_shard('SqlShard(host3)'),
+                      create_shard('BlockedShard'),
+                      add_link('BlockedShard', 'SqlShard(host3)'),
+                      add_link('ReplicatingShard', 'BlockedShard') ],
+        :copy =>    [ copy_shard('SqlShard(host1)', 'SqlShard(host3)') ],
+        :cleanup => [ add_link('ReplicatingShard', 'SqlShard(host3)'),
+                      remove_link('ReplicatingShard', 'BlockedShard'),
+                      remove_link('BlockedShard', 'SqlShard(host3)'),
+                      delete_shard('BlockedShard') ]
+      }
+    end
+
+    it "does a partition migration" do
+      from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
+      to   = mk_template 'ReplicatingShard -> (SqlShard(host3), SqlShard(host4))'
+
+      Gizzard::Transformation.new(from, to).operations.should == {
+        :prepare => [ create_shard('SqlShard(host4)'),
+                      create_shard('WriteOnlyShard'),
+                      create_shard('WriteOnlyShard'),
+                      create_shard('SqlShard(host3)'),
+                      add_link('ReplicatingShard', 'WriteOnlyShard'),
+                      add_link('WriteOnlyShard', 'SqlShard(host4)'),
+                      add_link('WriteOnlyShard', 'SqlShard(host3)'),
+                      add_link('ReplicatingShard', 'WriteOnlyShard') ],
+        :copy =>    [ copy_shard('SqlShard(host1)', 'SqlShard(host4)'),
+                      copy_shard('SqlShard(host1)', 'SqlShard(host3)') ],
+        :cleanup => [ add_link('ReplicatingShard', 'SqlShard(host4)'),
+                      add_link('ReplicatingShard', 'SqlShard(host3)'),
+                      remove_link('ReplicatingShard', 'WriteOnlyShard'),
+                      remove_link('ReplicatingShard', 'WriteOnlyShard'),
+                      remove_link('WriteOnlyShard', 'SqlShard(host3)'),
+                      remove_link('ReplicatingShard', 'SqlShard(host1)'),
+                      remove_link('ReplicatingShard', 'SqlShard(host2)'),
+                      remove_link('WriteOnlyShard', 'SqlShard(host4)'),
+                      delete_shard('SqlShard(host1)'),
+                      delete_shard('SqlShard(host2)'),
+                      delete_shard('WriteOnlyShard'),
+                      delete_shard('WriteOnlyShard') ]
+      }
+    end
+
+    it "migrates the top level shard" do
+      from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
+      to   = mk_template 'FailingOverShard -> (SqlShard(host1), SqlShard(host2))'
+
+      Gizzard::Transformation.new(from, to).operations.should == {
+        :prepare => [ create_shard('FailingOverShard'),
+                      add_link('FailingOverShard', 'SqlShard(host2)'),
+                      add_link('FailingOverShard', 'SqlShard(host1)'),
+                      set_forwarding('FailingOverShard'),
+                      remove_forwarding('ReplicatingShard'),
+                      remove_link('ReplicatingShard', 'SqlShard(host1)'),
+                      remove_link('ReplicatingShard', 'SqlShard(host2)'),
+                      delete_shard('ReplicatingShard') ],
+        :copy =>    [],
+        :cleanup => []
+      }
+    end
+
+    it "wraps a shard" do
+      from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
+      to   = mk_template 'ReplicatingShard -> (ReadOnlyShard -> SqlShard(host1), SqlShard(host2))'
+
+      Gizzard::Transformation.new(from, to).operations.should == {
+        :prepare => [ create_shard('ReadOnlyShard'),
+                      add_link('ReadOnlyShard', 'SqlShard(host1)'),
+                      add_link('ReplicatingShard', 'ReadOnlyShard'),
+                      remove_link('ReplicatingShard', 'SqlShard(host1)') ],
+        :copy =>    [],
+        :cleanup => []
+      }
+    end
+
+    it "raises an argument error if the transformation requires a copy without a valid source" do
+      to = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
+
+      Gizzard::Shard::INVALID_COPY_TYPES.each do |invalid_type|
+        from = mk_template "ReplicatingShard -> #{invalid_type} -> SqlShard(host1)"
+        lambda { Gizzard::Transformation.new(from, to) }.should raise_error(ArgumentError)
+      end
+    end
+  end
+
+  describe "collapse_jobs" do
+    def collapse(jobs); @trans.collapse_jobs(jobs) end
+
+    it "works" do
+      jobs = [ Op::AddLink.new(@host_1_template, @host_2_template),
+               Op::AddLink.new(@host_1_template, @host_3_template) ]
+      collapse(jobs).should == jobs
+
+      collapse([ Op::AddLink.new(@host_1_template, @host_2_template),
+                 Op::RemoveLink.new(@host_1_template, @host_2_template) ]).should == []
+
+      collapse([ Op::RemoveLink.new(@host_1_template, @host_2_template),
+                 Op::AddLink.new(@host_1_template, @host_2_template) ]).should == []
+
+      collapse(@trans.create_tree(@from_template) + @trans.destroy_tree(@from_template)).should == []
+
+      collapse(@trans.create_tree(@to_template) + @trans.destroy_tree(@from_template)).should ==
+        [ Op::CreateShard.new(@host_3_template),
+          Op::AddLink.new(@to_template, @host_3_template),
+          Op::RemoveLink.new(@blocked_template, @host_1_template),
+          Op::DeleteShard.new(@host_1_template),
+          Op::RemoveLink.new(@from_template, @blocked_template),
+          Op::DeleteShard.new(@blocked_template) ]
+    end
+  end
+
+  describe "copy_destination?" do
+    it "returns true if the given template is not a member of the from_template" do
+      @trans.copy_destination?(@host_3_template).should == true
+    end
+
+    it "returns false when there is no from_template (completely new shards, no data to copy)" do
+      @trans = Gizzard::Transformation.new(nil, @to_template)
+      @trans.copy_destination?(@host_1_template).should == false
+      @trans.copy_destination?(@host_2_template).should == false
+      @trans.copy_destination?(@host_3_template).should == false
+    end
+
+    it "returns false if the given template is a member of the from_template (therefore has source data)" do
+      @trans.copy_destination?(@host_1_template).should == false
+      @trans.copy_destination?(@host_2_template).should == false
+    end
+
+    it "returns false if the given template is not concrete" do
+      @trans.copy_destination?(@to_template).should == false
+    end
+  end
+end