gizzmo 0.11.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ /dist/*
2
+ *.log
3
+ *.o
4
+ gen-cpp/
5
+ bin/
6
+ target/
7
+ .DS_Store
8
+ /kestrel/*
9
+ *.hprof.txt
10
+ lib_managed/
11
+ src_managed/
12
+ /project/boot/
13
+ /project/plugins/project/
@@ -0,0 +1,8 @@
1
+ #Project properties
2
+ #Mon Nov 22 21:09:11 PST 2010
3
+ project.organization=com.twitter
4
+ project.name=gizzmotestserver
5
+ sbt.version=0.7.4
6
+ project.version=0.1
7
+ build.scala.versions=2.7.7
8
+ project.initialize=false
@@ -0,0 +1,13 @@
1
+ import sbt._
2
+ import com.twitter.sbt.StandardProject
3
+
4
+
5
+ class GizzmoServerProject(info: ProjectInfo) extends StandardProject(info) {
6
+ override def compileOptions = super.compileOptions ++ Seq(Unchecked)
7
+ override def filterScalaJars = false
8
+
9
+ val scalaTools = "org.scala-lang" % "scala-compiler" % "2.7.7"
10
+ val gizzard = "com.twitter" % "gizzard" % "1.6-mc-SNAPSHOT"
11
+
12
+ val specs = "org.scala-tools.testing" % "specs" % "1.6.2.1" % "test"
13
+ }
@@ -0,0 +1,6 @@
1
+ import sbt._
2
+
3
+ class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
4
+ val twitterMaven = "twitter.com" at "http://maven.twttr.com/"
5
+ val defaultProject = "com.twitter" % "standard-project" % "0.7.12"
6
+ }
@@ -0,0 +1,18 @@
1
+ package com.twitter.gizzard.testserver
2
+
3
+ import com.twitter.gizzard.testserver.config.TestServerConfig
4
+
5
+ object Main {
6
+ var service: TestServer = null
7
+
8
+ def main(args: Array[String]) {
9
+ val config = args match {
10
+ case Array(a) => TestServerConfig("integration", a.toInt)
11
+ case Array(a, b, c) => TestServerConfig("integration", a.toInt, b.toInt, c.toInt)
12
+ }
13
+
14
+ service = new TestServer(config)
15
+
16
+ service.start()
17
+ }
18
+ }
@@ -0,0 +1,247 @@
1
+ package com.twitter.gizzard.testserver
2
+
3
+ import java.sql.{ResultSet, SQLException}
4
+ import com.twitter.querulous.evaluator.{QueryEvaluatorFactory, QueryEvaluator}
5
+ import com.twitter.querulous.config.Connection
6
+ import com.twitter.querulous.query.SqlQueryTimeoutException
7
+ import gizzard.GizzardServer
8
+ import nameserver.NameServer
9
+ import shards.{ShardId, ShardInfo, ShardException, ShardTimeoutException}
10
+ import scheduler.{JobScheduler, JsonJob, CopyJob, CopyJobParser, CopyJobFactory, JsonJobParser, PrioritizingJobScheduler}
11
+
12
+ object config {
13
+ import com.twitter.gizzard.config._
14
+ import com.twitter.querulous.config._
15
+ import com.twitter.util.TimeConversions._
16
+ import com.twitter.util.Duration
17
+
18
+ trait TestDBConnection extends Connection {
19
+ val username = "root"
20
+ val password = ""
21
+ val hostnames = Seq("localhost")
22
+ }
23
+
24
+ object TestQueryEvaluator extends querulous.config.QueryEvaluator {
25
+ database.pool = new ApachePoolingDatabase {
26
+ sizeMin = 3
27
+ sizeMax = 3
28
+ }
29
+ }
30
+
31
+ trait TestTHsHaServer extends THsHaServer {
32
+ threadPool.minThreads = 10
33
+ }
34
+
35
+ trait TestServer extends gizzard.config.GizzardServer {
36
+ def server: TServer
37
+ def databaseConnection: Connection
38
+ val queryEvaluator = TestQueryEvaluator
39
+ }
40
+
41
+ trait TestJobScheduler extends Scheduler {
42
+ val schedulerType = new KestrelScheduler {
43
+ path = "/tmp"
44
+ keepJournal = false
45
+ }
46
+ errorLimit = 25
47
+ }
48
+
49
+ class TestNameServer(name: String) extends gizzard.config.NameServer {
50
+ jobRelay.priority = Priority.Low.id
51
+
52
+ val replicas = Seq(new Mysql {
53
+ queryEvaluator = TestQueryEvaluator
54
+ val connection = new TestDBConnection {
55
+ val database = "gizzard_test_" + name + "_ns"
56
+ }
57
+ })
58
+ }
59
+
60
+ object TestServerConfig {
61
+ def apply(name: String, sPort: Int, iPort: Int, mPort: Int) = {
62
+ val queueBase = "gizzard_test_" + name
63
+
64
+ new TestServer {
65
+ val server = new TestTHsHaServer { val name = "TestGizzardService"; val port = sPort }
66
+ val databaseConnection = new TestDBConnection { val database = "gizzard_test_" + name }
67
+ val nameServer = new TestNameServer(name)
68
+ val jobQueues = Map(
69
+ Priority.High.id -> new TestJobScheduler { val name = queueBase+"_high" },
70
+ Priority.Low.id -> new TestJobScheduler { val name = queueBase+"_low" }
71
+ )
72
+
73
+ jobInjector.port = iPort
74
+ manager.port = mPort
75
+ }
76
+ }
77
+
78
+ def apply(name: String, port: Int): TestServer = apply(name, port, port + 1, port + 2)
79
+ }
80
+ }
81
+
82
+
83
+ object Priority extends Enumeration {
84
+ val High, Low = Value
85
+ }
86
+
87
+ class TestServer(conf: config.TestServer) extends GizzardServer[TestShard, JsonJob](conf) {
88
+
89
+ // shard/nameserver/scheduler wiring
90
+
91
+ val readWriteShardAdapter = new TestReadWriteAdapter(_)
92
+ val jobPriorities = List(Priority.High.id, Priority.Low.id)
93
+ val copyPriority = Priority.Low.id
94
+ val copyFactory = new TestCopyFactory(nameServer, jobScheduler(Priority.Low.id))
95
+
96
+ shardRepo += ("TestShard" -> new SqlShardFactory(conf.queryEvaluator(), conf.databaseConnection))
97
+
98
+ jobCodec += ("Put".r -> new PutParser(nameServer.findCurrentForwarding(0, _)))
99
+ jobCodec += ("Copy".r -> new TestCopyParser(nameServer, jobScheduler(Priority.Low.id)))
100
+
101
+
102
+ // service listener
103
+
104
+ val testService = new TestServerIFace(nameServer.findCurrentForwarding(0, _), jobScheduler)
105
+
106
+ lazy val testThriftServer = {
107
+ val processor = new thrift.TestServer.Processor(testService)
108
+ conf.server(processor)
109
+ }
110
+
111
+ def start() {
112
+ startGizzard()
113
+ new Thread(new Runnable { def run() { testThriftServer.serve() } }, "TestServerThread").start()
114
+ }
115
+
116
+ def shutdown(quiesce: Boolean) {
117
+ testThriftServer.stop()
118
+ shutdownGizzard(quiesce)
119
+ }
120
+ }
121
+
122
+
123
+ // Service Interface
124
+
125
+ class TestServerIFace(forwarding: Long => TestShard, scheduler: PrioritizingJobScheduler[JsonJob])
126
+ extends thrift.TestServer.Iface {
127
+ import com.twitter.gizzard.thrift.conversions.Sequences._
128
+
129
+ def put(key: Int, value: String) {
130
+ scheduler.put(Priority.High.id, new PutJob(key, value, forwarding))
131
+ }
132
+
133
+ def get(key: Int) = forwarding(key).get(key).map(asTestResult).map(List(_).toJavaList) getOrElse List[thrift.TestResult]().toJavaList
134
+
135
+ private def asTestResult(t: (Int, String, Int)) = new thrift.TestResult(t._1, t._2, t._3)
136
+ }
137
+
138
+
139
+ // Shard Definitions
140
+
141
+ trait TestShard extends shards.Shard {
142
+ def put(key: Int, value: String): Unit
143
+ def putAll(kvs: Seq[(Int, String)]): Unit
144
+ def get(key: Int): Option[(Int, String, Int)]
145
+ def getAll(key: Int, count: Int): Seq[(Int, String, Int)]
146
+ }
147
+
148
+ class TestReadWriteAdapter(s: shards.ReadWriteShard[TestShard])
149
+ extends shards.ReadWriteShardAdapter(s) with TestShard {
150
+ def put(k: Int, v: String) = s.writeOperation(_.put(k,v))
151
+ def putAll(kvs: Seq[(Int,String)]) = s.writeOperation(_.putAll(kvs))
152
+ def get(k: Int) = s.readOperation(_.get(k))
153
+ def getAll(k:Int, c: Int) = s.readOperation(_.getAll(k,c))
154
+ }
155
+
156
+ class SqlShardFactory(qeFactory: QueryEvaluatorFactory, conn: Connection)
157
+ extends shards.ShardFactory[TestShard] {
158
+ def instantiate(info: ShardInfo, weight: Int, children: Seq[TestShard]) =
159
+ new SqlShard(qeFactory(conn.withHost(info.hostname)), info, weight, children)
160
+
161
+ def materialize(info: ShardInfo) {
162
+ val ddl =
163
+ """create table if not exists %s (
164
+ id int(11) not null,
165
+ value varchar(255) not null,
166
+ count int(11) not null default 1,
167
+ primary key (id)
168
+ ) engine=innodb default charset=utf8"""
169
+ try {
170
+ val e = qeFactory(conn.withHost(info.hostname).withoutDatabase)
171
+ e.execute("create database if not exists " + conn.database)
172
+ e.execute(ddl.format(conn.database + "." + info.tablePrefix))
173
+ } catch {
174
+ case e: SQLException => throw new ShardException(e.toString)
175
+ case e: SqlQueryTimeoutException => throw new ShardTimeoutException(e.timeout, info.id)
176
+ }
177
+ }
178
+ }
179
+
180
+ class SqlShard(
181
+ evaluator: QueryEvaluator,
182
+ val shardInfo: ShardInfo,
183
+ val weight: Int,
184
+ val children: Seq[TestShard])
185
+ extends TestShard {
186
+ private val table = shardInfo.tablePrefix
187
+
188
+ private val putSql = """insert into %s (id, value, count) values (?,?,1) on duplicate key
189
+ update value = values(value), count = count+1""".format(table)
190
+ private val getSql = "select * from " + table + " where id = ?"
191
+ private val getAllSql = "select * from " + table + " where id > ? limit ?"
192
+
193
+ private def asResult(r: ResultSet) = (r.getInt("id"), r.getString("value"), r.getInt("count"))
194
+
195
+ def put(key: Int, value: String) { evaluator.execute(putSql, key, value) }
196
+ def putAll(kvs: Seq[(Int, String)]) {
197
+ evaluator.executeBatch(putSql) { b => for ((k,v) <- kvs) b(k,v) }
198
+ }
199
+
200
+ def get(key: Int) = evaluator.selectOne(getSql, key)(asResult)
201
+ def getAll(key: Int, count: Int) = evaluator.select(getAllSql, key, count)(asResult)
202
+ }
203
+
204
+
205
+ // Jobs
206
+
207
+ class PutParser(forwarding: Long => TestShard) extends JsonJobParser {
208
+ def apply(map: Map[String, Any]): JsonJob = {
209
+ new PutJob(map("key").asInstanceOf[Int], map("value").asInstanceOf[String], forwarding)
210
+ }
211
+ }
212
+
213
+ class PutJob(key: Int, value: String, forwarding: Long => TestShard) extends JsonJob {
214
+ def toMap = Map("key" -> key, "value" -> value)
215
+ def apply() { forwarding(key).put(key, value) }
216
+ }
217
+
218
+ class TestCopyFactory(ns: NameServer[TestShard], s: JobScheduler[JsonJob])
219
+ extends CopyJobFactory[TestShard] {
220
+ def apply(src: ShardId, dest: ShardId) = new TestCopy(src, dest, 0, 500, ns, s)
221
+ }
222
+
223
+ class TestCopyParser(ns: NameServer[TestShard], s: JobScheduler[JsonJob])
224
+ extends CopyJobParser[TestShard] {
225
+ def deserialize(m: Map[String, Any], src: ShardId, dest: ShardId, count: Int) = {
226
+ val cursor = m("cursor").asInstanceOf[Int]
227
+ val count = m("count").asInstanceOf[Int]
228
+ new TestCopy(src, dest, cursor, count, ns, s)
229
+ }
230
+ }
231
+
232
+ class TestCopy(srcId: ShardId, destId: ShardId, cursor: Int, count: Int,
233
+ ns: NameServer[TestShard], s: JobScheduler[JsonJob])
234
+ extends CopyJob[TestShard](srcId, destId, count, ns, s) {
235
+ def copyPage(src: TestShard, dest: TestShard, count: Int) = {
236
+ val rows = src.getAll(cursor, count).map { case (k,v,c) => (k,v) }
237
+
238
+ if (rows.isEmpty) {
239
+ None
240
+ } else {
241
+ dest.putAll(rows)
242
+ Some(new TestCopy(srcId, destId, rows.last._1, count, ns, s))
243
+ }
244
+ }
245
+
246
+ def serialize = Map("cursor" -> cursor)
247
+ }
@@ -0,0 +1,12 @@
1
+ namespace java com.twitter.gizzard.testserver.thrift
2
+
3
+ struct TestResult {
4
+ 1: i32 key
5
+ 2: string value
6
+ 3: i32 count
7
+ }
8
+
9
+ service TestServer {
10
+ void put(1: i32 key, 2: string value)
11
+ list<TestResult> get(1: i32 key)
12
+ }
@@ -0,0 +1,181 @@
1
+ require File.expand_path('../spec_helper', __FILE__)
2
+
3
+ describe Gizzard::Transformation do
4
+ Op = Gizzard::Transformation::Op
5
+
6
+ def create_shard(t); Op::CreateShard.new(mk_template(t)) end
7
+ def delete_shard(t); Op::DeleteShard.new(mk_template(t)) end
8
+ def add_link(f, t); Op::AddLink.new(mk_template(f), mk_template(t)) end
9
+ def remove_link(f, t); Op::RemoveLink.new(mk_template(f), mk_template(t)) end
10
+ def copy_shard(f, t); Op::CopyShard.new(mk_template(f), mk_template(t)) end
11
+ def set_forwarding(t); Op::SetForwarding.new(mk_template(t)) end
12
+ def remove_forwarding(t); Op::RemoveForwarding.new(mk_template(t)) end
13
+
14
+ before do
15
+ @nameserver = stub!.subject
16
+ stub(@nameserver).dryrun? { false }
17
+
18
+ @config = Gizzard::MigratorConfig.new :prefix => "status", :table_id => 0
19
+
20
+ @from_template = mk_template 'ReplicatingShard -> (BlockedShard -> SqlShard(host1), SqlShard(host2))'
21
+ @to_template = mk_template 'ReplicatingShard -> (SqlShard(host2), SqlShard(host3))'
22
+
23
+ @blocked_template = mk_template 'BlockedShard -> SqlShard(host1)'
24
+ @host_1_template = mk_template 'SqlShard(host1)'
25
+ @host_2_template = mk_template 'SqlShard(host2)'
26
+ @host_3_template = mk_template 'SqlShard(host3)'
27
+
28
+ @trans = Gizzard::Transformation.new(@from_template, @to_template)
29
+ end
30
+
31
+ describe "initialization" do
32
+ it "allows an optional copy wrapper type" do
33
+ Gizzard::Transformation.new(@from_template, @to_template)
34
+ Gizzard::Transformation.new(@from_template, @to_template, 'WriteOnlyShard')
35
+ Gizzard::Transformation.new(@from_template, @to_template, 'BlockedShard')
36
+ lambda do
37
+ Gizzard::Transformation.new(@from_template, @to_template, 'InvalidWrapperShard')
38
+ end.should raise_error(ArgumentError)
39
+ end
40
+ end
41
+
42
+ # internal method tests
43
+
44
+ describe "operations" do
45
+ it "does a basic replica addition" do
46
+ from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
47
+ to = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2), SqlShard(host3))'
48
+
49
+ Gizzard::Transformation.new(from, to, 'BlockedShard').operations.should == {
50
+ :prepare => [ create_shard('SqlShard(host3)'),
51
+ create_shard('BlockedShard'),
52
+ add_link('BlockedShard', 'SqlShard(host3)'),
53
+ add_link('ReplicatingShard', 'BlockedShard') ],
54
+ :copy => [ copy_shard('SqlShard(host1)', 'SqlShard(host3)') ],
55
+ :cleanup => [ add_link('ReplicatingShard', 'SqlShard(host3)'),
56
+ remove_link('ReplicatingShard', 'BlockedShard'),
57
+ remove_link('BlockedShard', 'SqlShard(host3)'),
58
+ delete_shard('BlockedShard') ]
59
+ }
60
+ end
61
+
62
+ it "does a partition migration" do
63
+ from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
64
+ to = mk_template 'ReplicatingShard -> (SqlShard(host3), SqlShard(host4))'
65
+
66
+ Gizzard::Transformation.new(from, to).operations.should == {
67
+ :prepare => [ create_shard('SqlShard(host4)'),
68
+ create_shard('WriteOnlyShard'),
69
+ create_shard('WriteOnlyShard'),
70
+ create_shard('SqlShard(host3)'),
71
+ add_link('ReplicatingShard', 'WriteOnlyShard'),
72
+ add_link('WriteOnlyShard', 'SqlShard(host4)'),
73
+ add_link('WriteOnlyShard', 'SqlShard(host3)'),
74
+ add_link('ReplicatingShard', 'WriteOnlyShard') ],
75
+ :copy => [ copy_shard('SqlShard(host1)', 'SqlShard(host4)'),
76
+ copy_shard('SqlShard(host1)', 'SqlShard(host3)') ],
77
+ :cleanup => [ add_link('ReplicatingShard', 'SqlShard(host4)'),
78
+ add_link('ReplicatingShard', 'SqlShard(host3)'),
79
+ remove_link('ReplicatingShard', 'WriteOnlyShard'),
80
+ remove_link('ReplicatingShard', 'WriteOnlyShard'),
81
+ remove_link('WriteOnlyShard', 'SqlShard(host3)'),
82
+ remove_link('ReplicatingShard', 'SqlShard(host1)'),
83
+ remove_link('ReplicatingShard', 'SqlShard(host2)'),
84
+ remove_link('WriteOnlyShard', 'SqlShard(host4)'),
85
+ delete_shard('SqlShard(host1)'),
86
+ delete_shard('SqlShard(host2)'),
87
+ delete_shard('WriteOnlyShard'),
88
+ delete_shard('WriteOnlyShard') ]
89
+ }
90
+ end
91
+
92
+ it "migrates the top level shard" do
93
+ from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
94
+ to = mk_template 'FailingOverShard -> (SqlShard(host1), SqlShard(host2))'
95
+
96
+ Gizzard::Transformation.new(from, to).operations.should == {
97
+ :prepare => [ create_shard('FailingOverShard'),
98
+ add_link('FailingOverShard', 'SqlShard(host2)'),
99
+ add_link('FailingOverShard', 'SqlShard(host1)'),
100
+ set_forwarding('FailingOverShard'),
101
+ remove_forwarding('ReplicatingShard'),
102
+ remove_link('ReplicatingShard', 'SqlShard(host1)'),
103
+ remove_link('ReplicatingShard', 'SqlShard(host2)'),
104
+ delete_shard('ReplicatingShard') ],
105
+ :copy => [],
106
+ :cleanup => []
107
+ }
108
+ end
109
+
110
+ it "wraps a shard" do
111
+ from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
112
+ to = mk_template 'ReplicatingShard -> (ReadOnlyShard -> SqlShard(host1), SqlShard(host2))'
113
+
114
+ Gizzard::Transformation.new(from, to).operations.should == {
115
+ :prepare => [ create_shard('ReadOnlyShard'),
116
+ add_link('ReadOnlyShard', 'SqlShard(host1)'),
117
+ add_link('ReplicatingShard', 'ReadOnlyShard'),
118
+ remove_link('ReplicatingShard', 'SqlShard(host1)') ],
119
+ :copy => [],
120
+ :cleanup => []
121
+ }
122
+ end
123
+
124
+ it "raises an argument error if the transformation requires a copy without a valid source" do
125
+ to = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
126
+
127
+ Gizzard::Shard::INVALID_COPY_TYPES.each do |invalid_type|
128
+ from = mk_template "ReplicatingShard -> #{invalid_type} -> SqlShard(host1)"
129
+ lambda { Gizzard::Transformation.new(from, to) }.should raise_error(ArgumentError)
130
+ end
131
+ end
132
+ end
133
+
134
+ describe "collapse_jobs" do
135
+ def collapse(jobs); @trans.collapse_jobs(jobs) end
136
+
137
+ it "works" do
138
+ jobs = [ Op::AddLink.new(@host_1_template, @host_2_template),
139
+ Op::AddLink.new(@host_1_template, @host_3_template) ]
140
+ collapse(jobs).should == jobs
141
+
142
+ collapse([ Op::AddLink.new(@host_1_template, @host_2_template),
143
+ Op::RemoveLink.new(@host_1_template, @host_2_template) ]).should == []
144
+
145
+ collapse([ Op::RemoveLink.new(@host_1_template, @host_2_template),
146
+ Op::AddLink.new(@host_1_template, @host_2_template) ]).should == []
147
+
148
+ collapse(@trans.create_tree(@from_template) + @trans.destroy_tree(@from_template)).should == []
149
+
150
+ collapse(@trans.create_tree(@to_template) + @trans.destroy_tree(@from_template)).should ==
151
+ [ Op::CreateShard.new(@host_3_template),
152
+ Op::AddLink.new(@to_template, @host_3_template),
153
+ Op::RemoveLink.new(@blocked_template, @host_1_template),
154
+ Op::DeleteShard.new(@host_1_template),
155
+ Op::RemoveLink.new(@from_template, @blocked_template),
156
+ Op::DeleteShard.new(@blocked_template) ]
157
+ end
158
+ end
159
+
160
+ describe "copy_destination?" do
161
+ it "returns true if the given template is not a member of the from_template" do
162
+ @trans.copy_destination?(@host_3_template).should == true
163
+ end
164
+
165
+ it "returns false when there is no from_template (completely new shards, no data to copy)" do
166
+ @trans = Gizzard::Transformation.new(nil, @to_template)
167
+ @trans.copy_destination?(@host_1_template).should == false
168
+ @trans.copy_destination?(@host_2_template).should == false
169
+ @trans.copy_destination?(@host_3_template).should == false
170
+ end
171
+
172
+ it "returns false if the given template is a member of the from_template (therefore has source data)" do
173
+ @trans.copy_destination?(@host_1_template).should == false
174
+ @trans.copy_destination?(@host_2_template).should == false
175
+ end
176
+
177
+ it "returns false if the given template is not concrete" do
178
+ @trans.copy_destination?(@to_template).should == false
179
+ end
180
+ end
181
+ end