gizzmo 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +11 -16
- data/VERSION +1 -1
- data/bin/setup_shards +173 -0
- data/lib/gizzard.rb +4 -0
- data/lib/gizzard/commands.rb +286 -152
- data/lib/gizzard/migrator.rb +192 -0
- data/lib/gizzard/nameserver.rb +206 -0
- data/lib/gizzard/shard_template.rb +252 -0
- data/lib/gizzard/thrift.rb +187 -135
- data/lib/gizzard/transformation.rb +230 -0
- data/lib/gizzard/transformation_op.rb +181 -0
- data/lib/gizzard/transformation_scheduler.rb +220 -0
- data/lib/gizzmo.rb +87 -20
- data/test/gizzmo_spec.rb +499 -0
- data/test/nameserver_spec.rb +139 -0
- data/test/scheduler_spec.rb +59 -0
- data/test/shard_template_spec.rb +103 -0
- data/test/spec.opts +7 -0
- data/test/spec_helper.rb +139 -0
- data/test/test_server/.gitignore +13 -0
- data/test/test_server/project/build.properties +8 -0
- data/test/test_server/project/build/Project.scala +13 -0
- data/test/test_server/project/plugins/Plugins.scala +6 -0
- data/test/test_server/src/main/scala/Main.scala +18 -0
- data/test/test_server/src/main/scala/TestServer.scala +247 -0
- data/test/test_server/src/main/thrift/TestServer.thrift +12 -0
- data/test/transformation_spec.rb +181 -0
- metadata +32 -5
- data/gizzmo.gemspec +0 -75
@@ -0,0 +1,13 @@
|
|
1
|
+
import sbt._
|
2
|
+
import com.twitter.sbt.StandardProject
|
3
|
+
|
4
|
+
|
5
|
+
class GizzmoServerProject(info: ProjectInfo) extends StandardProject(info) {
|
6
|
+
override def compileOptions = super.compileOptions ++ Seq(Unchecked)
|
7
|
+
override def filterScalaJars = false
|
8
|
+
|
9
|
+
val scalaTools = "org.scala-lang" % "scala-compiler" % "2.7.7"
|
10
|
+
val gizzard = "com.twitter" % "gizzard" % "1.6-mc-SNAPSHOT"
|
11
|
+
|
12
|
+
val specs = "org.scala-tools.testing" % "specs" % "1.6.2.1" % "test"
|
13
|
+
}
|
@@ -0,0 +1,18 @@
|
|
1
|
+
package com.twitter.gizzard.testserver
|
2
|
+
|
3
|
+
import com.twitter.gizzard.testserver.config.TestServerConfig
|
4
|
+
|
5
|
+
object Main {
|
6
|
+
var service: TestServer = null
|
7
|
+
|
8
|
+
def main(args: Array[String]) {
|
9
|
+
val config = args match {
|
10
|
+
case Array(a) => TestServerConfig("integration", a.toInt)
|
11
|
+
case Array(a, b, c) => TestServerConfig("integration", a.toInt, b.toInt, c.toInt)
|
12
|
+
}
|
13
|
+
|
14
|
+
service = new TestServer(config)
|
15
|
+
|
16
|
+
service.start()
|
17
|
+
}
|
18
|
+
}
|
@@ -0,0 +1,247 @@
|
|
1
|
+
package com.twitter.gizzard.testserver
|
2
|
+
|
3
|
+
import java.sql.{ResultSet, SQLException}
|
4
|
+
import com.twitter.querulous.evaluator.{QueryEvaluatorFactory, QueryEvaluator}
|
5
|
+
import com.twitter.querulous.config.Connection
|
6
|
+
import com.twitter.querulous.query.SqlQueryTimeoutException
|
7
|
+
import gizzard.GizzardServer
|
8
|
+
import nameserver.NameServer
|
9
|
+
import shards.{ShardId, ShardInfo, ShardException, ShardTimeoutException}
|
10
|
+
import scheduler.{JobScheduler, JsonJob, CopyJob, CopyJobParser, CopyJobFactory, JsonJobParser, PrioritizingJobScheduler}
|
11
|
+
|
12
|
+
object config {
|
13
|
+
import com.twitter.gizzard.config._
|
14
|
+
import com.twitter.querulous.config._
|
15
|
+
import com.twitter.util.TimeConversions._
|
16
|
+
import com.twitter.util.Duration
|
17
|
+
|
18
|
+
trait TestDBConnection extends Connection {
|
19
|
+
val username = "root"
|
20
|
+
val password = ""
|
21
|
+
val hostnames = Seq("localhost")
|
22
|
+
}
|
23
|
+
|
24
|
+
object TestQueryEvaluator extends querulous.config.QueryEvaluator {
|
25
|
+
database.pool = new ApachePoolingDatabase {
|
26
|
+
sizeMin = 3
|
27
|
+
sizeMax = 3
|
28
|
+
}
|
29
|
+
}
|
30
|
+
|
31
|
+
trait TestTHsHaServer extends THsHaServer {
|
32
|
+
threadPool.minThreads = 10
|
33
|
+
}
|
34
|
+
|
35
|
+
trait TestServer extends gizzard.config.GizzardServer {
|
36
|
+
def server: TServer
|
37
|
+
def databaseConnection: Connection
|
38
|
+
val queryEvaluator = TestQueryEvaluator
|
39
|
+
}
|
40
|
+
|
41
|
+
trait TestJobScheduler extends Scheduler {
|
42
|
+
val schedulerType = new KestrelScheduler {
|
43
|
+
path = "/tmp"
|
44
|
+
keepJournal = false
|
45
|
+
}
|
46
|
+
errorLimit = 25
|
47
|
+
}
|
48
|
+
|
49
|
+
class TestNameServer(name: String) extends gizzard.config.NameServer {
|
50
|
+
jobRelay.priority = Priority.Low.id
|
51
|
+
|
52
|
+
val replicas = Seq(new Mysql {
|
53
|
+
queryEvaluator = TestQueryEvaluator
|
54
|
+
val connection = new TestDBConnection {
|
55
|
+
val database = "gizzard_test_" + name + "_ns"
|
56
|
+
}
|
57
|
+
})
|
58
|
+
}
|
59
|
+
|
60
|
+
object TestServerConfig {
|
61
|
+
def apply(name: String, sPort: Int, iPort: Int, mPort: Int) = {
|
62
|
+
val queueBase = "gizzard_test_" + name
|
63
|
+
|
64
|
+
new TestServer {
|
65
|
+
val server = new TestTHsHaServer { val name = "TestGizzardService"; val port = sPort }
|
66
|
+
val databaseConnection = new TestDBConnection { val database = "gizzard_test_" + name }
|
67
|
+
val nameServer = new TestNameServer(name)
|
68
|
+
val jobQueues = Map(
|
69
|
+
Priority.High.id -> new TestJobScheduler { val name = queueBase+"_high" },
|
70
|
+
Priority.Low.id -> new TestJobScheduler { val name = queueBase+"_low" }
|
71
|
+
)
|
72
|
+
|
73
|
+
jobInjector.port = iPort
|
74
|
+
manager.port = mPort
|
75
|
+
}
|
76
|
+
}
|
77
|
+
|
78
|
+
def apply(name: String, port: Int): TestServer = apply(name, port, port + 1, port + 2)
|
79
|
+
}
|
80
|
+
}
|
81
|
+
|
82
|
+
|
83
|
+
object Priority extends Enumeration {
|
84
|
+
val High, Low = Value
|
85
|
+
}
|
86
|
+
|
87
|
+
class TestServer(conf: config.TestServer) extends GizzardServer[TestShard, JsonJob](conf) {
|
88
|
+
|
89
|
+
// shard/nameserver/scheduler wiring
|
90
|
+
|
91
|
+
val readWriteShardAdapter = new TestReadWriteAdapter(_)
|
92
|
+
val jobPriorities = List(Priority.High.id, Priority.Low.id)
|
93
|
+
val copyPriority = Priority.Low.id
|
94
|
+
val copyFactory = new TestCopyFactory(nameServer, jobScheduler(Priority.Low.id))
|
95
|
+
|
96
|
+
shardRepo += ("TestShard" -> new SqlShardFactory(conf.queryEvaluator(), conf.databaseConnection))
|
97
|
+
|
98
|
+
jobCodec += ("Put".r -> new PutParser(nameServer.findCurrentForwarding(0, _)))
|
99
|
+
jobCodec += ("Copy".r -> new TestCopyParser(nameServer, jobScheduler(Priority.Low.id)))
|
100
|
+
|
101
|
+
|
102
|
+
// service listener
|
103
|
+
|
104
|
+
val testService = new TestServerIFace(nameServer.findCurrentForwarding(0, _), jobScheduler)
|
105
|
+
|
106
|
+
lazy val testThriftServer = {
|
107
|
+
val processor = new thrift.TestServer.Processor(testService)
|
108
|
+
conf.server(processor)
|
109
|
+
}
|
110
|
+
|
111
|
+
def start() {
|
112
|
+
startGizzard()
|
113
|
+
new Thread(new Runnable { def run() { testThriftServer.serve() } }, "TestServerThread").start()
|
114
|
+
}
|
115
|
+
|
116
|
+
def shutdown(quiesce: Boolean) {
|
117
|
+
testThriftServer.stop()
|
118
|
+
shutdownGizzard(quiesce)
|
119
|
+
}
|
120
|
+
}
|
121
|
+
|
122
|
+
|
123
|
+
// Service Interface
|
124
|
+
|
125
|
+
class TestServerIFace(forwarding: Long => TestShard, scheduler: PrioritizingJobScheduler[JsonJob])
|
126
|
+
extends thrift.TestServer.Iface {
|
127
|
+
import com.twitter.gizzard.thrift.conversions.Sequences._
|
128
|
+
|
129
|
+
def put(key: Int, value: String) {
|
130
|
+
scheduler.put(Priority.High.id, new PutJob(key, value, forwarding))
|
131
|
+
}
|
132
|
+
|
133
|
+
def get(key: Int) = forwarding(key).get(key).map(asTestResult).map(List(_).toJavaList) getOrElse List[thrift.TestResult]().toJavaList
|
134
|
+
|
135
|
+
private def asTestResult(t: (Int, String, Int)) = new thrift.TestResult(t._1, t._2, t._3)
|
136
|
+
}
|
137
|
+
|
138
|
+
|
139
|
+
// Shard Definitions
|
140
|
+
|
141
|
+
trait TestShard extends shards.Shard {
|
142
|
+
def put(key: Int, value: String): Unit
|
143
|
+
def putAll(kvs: Seq[(Int, String)]): Unit
|
144
|
+
def get(key: Int): Option[(Int, String, Int)]
|
145
|
+
def getAll(key: Int, count: Int): Seq[(Int, String, Int)]
|
146
|
+
}
|
147
|
+
|
148
|
+
class TestReadWriteAdapter(s: shards.ReadWriteShard[TestShard])
|
149
|
+
extends shards.ReadWriteShardAdapter(s) with TestShard {
|
150
|
+
def put(k: Int, v: String) = s.writeOperation(_.put(k,v))
|
151
|
+
def putAll(kvs: Seq[(Int,String)]) = s.writeOperation(_.putAll(kvs))
|
152
|
+
def get(k: Int) = s.readOperation(_.get(k))
|
153
|
+
def getAll(k:Int, c: Int) = s.readOperation(_.getAll(k,c))
|
154
|
+
}
|
155
|
+
|
156
|
+
class SqlShardFactory(qeFactory: QueryEvaluatorFactory, conn: Connection)
|
157
|
+
extends shards.ShardFactory[TestShard] {
|
158
|
+
def instantiate(info: ShardInfo, weight: Int, children: Seq[TestShard]) =
|
159
|
+
new SqlShard(qeFactory(conn.withHost(info.hostname)), info, weight, children)
|
160
|
+
|
161
|
+
def materialize(info: ShardInfo) {
|
162
|
+
val ddl =
|
163
|
+
"""create table if not exists %s (
|
164
|
+
id int(11) not null,
|
165
|
+
value varchar(255) not null,
|
166
|
+
count int(11) not null default 1,
|
167
|
+
primary key (id)
|
168
|
+
) engine=innodb default charset=utf8"""
|
169
|
+
try {
|
170
|
+
val e = qeFactory(conn.withHost(info.hostname).withoutDatabase)
|
171
|
+
e.execute("create database if not exists " + conn.database)
|
172
|
+
e.execute(ddl.format(conn.database + "." + info.tablePrefix))
|
173
|
+
} catch {
|
174
|
+
case e: SQLException => throw new ShardException(e.toString)
|
175
|
+
case e: SqlQueryTimeoutException => throw new ShardTimeoutException(e.timeout, info.id)
|
176
|
+
}
|
177
|
+
}
|
178
|
+
}
|
179
|
+
|
180
|
+
class SqlShard(
|
181
|
+
evaluator: QueryEvaluator,
|
182
|
+
val shardInfo: ShardInfo,
|
183
|
+
val weight: Int,
|
184
|
+
val children: Seq[TestShard])
|
185
|
+
extends TestShard {
|
186
|
+
private val table = shardInfo.tablePrefix
|
187
|
+
|
188
|
+
private val putSql = """insert into %s (id, value, count) values (?,?,1) on duplicate key
|
189
|
+
update value = values(value), count = count+1""".format(table)
|
190
|
+
private val getSql = "select * from " + table + " where id = ?"
|
191
|
+
private val getAllSql = "select * from " + table + " where id > ? limit ?"
|
192
|
+
|
193
|
+
private def asResult(r: ResultSet) = (r.getInt("id"), r.getString("value"), r.getInt("count"))
|
194
|
+
|
195
|
+
def put(key: Int, value: String) { evaluator.execute(putSql, key, value) }
|
196
|
+
def putAll(kvs: Seq[(Int, String)]) {
|
197
|
+
evaluator.executeBatch(putSql) { b => for ((k,v) <- kvs) b(k,v) }
|
198
|
+
}
|
199
|
+
|
200
|
+
def get(key: Int) = evaluator.selectOne(getSql, key)(asResult)
|
201
|
+
def getAll(key: Int, count: Int) = evaluator.select(getAllSql, key, count)(asResult)
|
202
|
+
}
|
203
|
+
|
204
|
+
|
205
|
+
// Jobs
|
206
|
+
|
207
|
+
class PutParser(forwarding: Long => TestShard) extends JsonJobParser {
|
208
|
+
def apply(map: Map[String, Any]): JsonJob = {
|
209
|
+
new PutJob(map("key").asInstanceOf[Int], map("value").asInstanceOf[String], forwarding)
|
210
|
+
}
|
211
|
+
}
|
212
|
+
|
213
|
+
class PutJob(key: Int, value: String, forwarding: Long => TestShard) extends JsonJob {
|
214
|
+
def toMap = Map("key" -> key, "value" -> value)
|
215
|
+
def apply() { forwarding(key).put(key, value) }
|
216
|
+
}
|
217
|
+
|
218
|
+
class TestCopyFactory(ns: NameServer[TestShard], s: JobScheduler[JsonJob])
|
219
|
+
extends CopyJobFactory[TestShard] {
|
220
|
+
def apply(src: ShardId, dest: ShardId) = new TestCopy(src, dest, 0, 500, ns, s)
|
221
|
+
}
|
222
|
+
|
223
|
+
class TestCopyParser(ns: NameServer[TestShard], s: JobScheduler[JsonJob])
|
224
|
+
extends CopyJobParser[TestShard] {
|
225
|
+
def deserialize(m: Map[String, Any], src: ShardId, dest: ShardId, count: Int) = {
|
226
|
+
val cursor = m("cursor").asInstanceOf[Int]
|
227
|
+
val count = m("count").asInstanceOf[Int]
|
228
|
+
new TestCopy(src, dest, cursor, count, ns, s)
|
229
|
+
}
|
230
|
+
}
|
231
|
+
|
232
|
+
class TestCopy(srcId: ShardId, destId: ShardId, cursor: Int, count: Int,
|
233
|
+
ns: NameServer[TestShard], s: JobScheduler[JsonJob])
|
234
|
+
extends CopyJob[TestShard](srcId, destId, count, ns, s) {
|
235
|
+
def copyPage(src: TestShard, dest: TestShard, count: Int) = {
|
236
|
+
val rows = src.getAll(cursor, count).map { case (k,v,c) => (k,v) }
|
237
|
+
|
238
|
+
if (rows.isEmpty) {
|
239
|
+
None
|
240
|
+
} else {
|
241
|
+
dest.putAll(rows)
|
242
|
+
Some(new TestCopy(srcId, destId, rows.last._1, count, ns, s))
|
243
|
+
}
|
244
|
+
}
|
245
|
+
|
246
|
+
def serialize = Map("cursor" -> cursor)
|
247
|
+
}
|
@@ -0,0 +1,181 @@
|
|
1
|
+
require File.expand_path('../spec_helper', __FILE__)
|
2
|
+
|
3
|
+
describe Gizzard::Transformation do
|
4
|
+
Op = Gizzard::Transformation::Op
|
5
|
+
|
6
|
+
def create_shard(t); Op::CreateShard.new(mk_template(t)) end
|
7
|
+
def delete_shard(t); Op::DeleteShard.new(mk_template(t)) end
|
8
|
+
def add_link(f, t); Op::AddLink.new(mk_template(f), mk_template(t)) end
|
9
|
+
def remove_link(f, t); Op::RemoveLink.new(mk_template(f), mk_template(t)) end
|
10
|
+
def copy_shard(f, t); Op::CopyShard.new(mk_template(f), mk_template(t)) end
|
11
|
+
def set_forwarding(t); Op::SetForwarding.new(mk_template(t)) end
|
12
|
+
def remove_forwarding(t); Op::RemoveForwarding.new(mk_template(t)) end
|
13
|
+
|
14
|
+
before do
|
15
|
+
@nameserver = stub!.subject
|
16
|
+
stub(@nameserver).dryrun? { false }
|
17
|
+
|
18
|
+
@config = Gizzard::MigratorConfig.new :prefix => "status", :table_id => 0
|
19
|
+
|
20
|
+
@from_template = mk_template 'ReplicatingShard -> (BlockedShard -> SqlShard(host1), SqlShard(host2))'
|
21
|
+
@to_template = mk_template 'ReplicatingShard -> (SqlShard(host2), SqlShard(host3))'
|
22
|
+
|
23
|
+
@blocked_template = mk_template 'BlockedShard -> SqlShard(host1)'
|
24
|
+
@host_1_template = mk_template 'SqlShard(host1)'
|
25
|
+
@host_2_template = mk_template 'SqlShard(host2)'
|
26
|
+
@host_3_template = mk_template 'SqlShard(host3)'
|
27
|
+
|
28
|
+
@trans = Gizzard::Transformation.new(@from_template, @to_template)
|
29
|
+
end
|
30
|
+
|
31
|
+
describe "initialization" do
|
32
|
+
it "allows an optional copy wrapper type" do
|
33
|
+
Gizzard::Transformation.new(@from_template, @to_template)
|
34
|
+
Gizzard::Transformation.new(@from_template, @to_template, 'WriteOnlyShard')
|
35
|
+
Gizzard::Transformation.new(@from_template, @to_template, 'BlockedShard')
|
36
|
+
lambda do
|
37
|
+
Gizzard::Transformation.new(@from_template, @to_template, 'InvalidWrapperShard')
|
38
|
+
end.should raise_error(ArgumentError)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
# internal method tests
|
43
|
+
|
44
|
+
describe "operations" do
|
45
|
+
it "does a basic replica addition" do
|
46
|
+
from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
|
47
|
+
to = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2), SqlShard(host3))'
|
48
|
+
|
49
|
+
Gizzard::Transformation.new(from, to, 'BlockedShard').operations.should == {
|
50
|
+
:prepare => [ create_shard('SqlShard(host3)'),
|
51
|
+
create_shard('BlockedShard'),
|
52
|
+
add_link('BlockedShard', 'SqlShard(host3)'),
|
53
|
+
add_link('ReplicatingShard', 'BlockedShard') ],
|
54
|
+
:copy => [ copy_shard('SqlShard(host1)', 'SqlShard(host3)') ],
|
55
|
+
:cleanup => [ add_link('ReplicatingShard', 'SqlShard(host3)'),
|
56
|
+
remove_link('ReplicatingShard', 'BlockedShard'),
|
57
|
+
remove_link('BlockedShard', 'SqlShard(host3)'),
|
58
|
+
delete_shard('BlockedShard') ]
|
59
|
+
}
|
60
|
+
end
|
61
|
+
|
62
|
+
it "does a partition migration" do
|
63
|
+
from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
|
64
|
+
to = mk_template 'ReplicatingShard -> (SqlShard(host3), SqlShard(host4))'
|
65
|
+
|
66
|
+
Gizzard::Transformation.new(from, to).operations.should == {
|
67
|
+
:prepare => [ create_shard('SqlShard(host4)'),
|
68
|
+
create_shard('WriteOnlyShard'),
|
69
|
+
create_shard('WriteOnlyShard'),
|
70
|
+
create_shard('SqlShard(host3)'),
|
71
|
+
add_link('ReplicatingShard', 'WriteOnlyShard'),
|
72
|
+
add_link('WriteOnlyShard', 'SqlShard(host4)'),
|
73
|
+
add_link('WriteOnlyShard', 'SqlShard(host3)'),
|
74
|
+
add_link('ReplicatingShard', 'WriteOnlyShard') ],
|
75
|
+
:copy => [ copy_shard('SqlShard(host1)', 'SqlShard(host4)'),
|
76
|
+
copy_shard('SqlShard(host1)', 'SqlShard(host3)') ],
|
77
|
+
:cleanup => [ add_link('ReplicatingShard', 'SqlShard(host4)'),
|
78
|
+
add_link('ReplicatingShard', 'SqlShard(host3)'),
|
79
|
+
remove_link('ReplicatingShard', 'WriteOnlyShard'),
|
80
|
+
remove_link('ReplicatingShard', 'WriteOnlyShard'),
|
81
|
+
remove_link('WriteOnlyShard', 'SqlShard(host3)'),
|
82
|
+
remove_link('ReplicatingShard', 'SqlShard(host1)'),
|
83
|
+
remove_link('ReplicatingShard', 'SqlShard(host2)'),
|
84
|
+
remove_link('WriteOnlyShard', 'SqlShard(host4)'),
|
85
|
+
delete_shard('SqlShard(host1)'),
|
86
|
+
delete_shard('SqlShard(host2)'),
|
87
|
+
delete_shard('WriteOnlyShard'),
|
88
|
+
delete_shard('WriteOnlyShard') ]
|
89
|
+
}
|
90
|
+
end
|
91
|
+
|
92
|
+
it "migrates the top level shard" do
|
93
|
+
from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
|
94
|
+
to = mk_template 'FailingOverShard -> (SqlShard(host1), SqlShard(host2))'
|
95
|
+
|
96
|
+
Gizzard::Transformation.new(from, to).operations.should == {
|
97
|
+
:prepare => [ create_shard('FailingOverShard'),
|
98
|
+
add_link('FailingOverShard', 'SqlShard(host2)'),
|
99
|
+
add_link('FailingOverShard', 'SqlShard(host1)'),
|
100
|
+
set_forwarding('FailingOverShard'),
|
101
|
+
remove_forwarding('ReplicatingShard'),
|
102
|
+
remove_link('ReplicatingShard', 'SqlShard(host1)'),
|
103
|
+
remove_link('ReplicatingShard', 'SqlShard(host2)'),
|
104
|
+
delete_shard('ReplicatingShard') ],
|
105
|
+
:copy => [],
|
106
|
+
:cleanup => []
|
107
|
+
}
|
108
|
+
end
|
109
|
+
|
110
|
+
it "wraps a shard" do
|
111
|
+
from = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
|
112
|
+
to = mk_template 'ReplicatingShard -> (ReadOnlyShard -> SqlShard(host1), SqlShard(host2))'
|
113
|
+
|
114
|
+
Gizzard::Transformation.new(from, to).operations.should == {
|
115
|
+
:prepare => [ create_shard('ReadOnlyShard'),
|
116
|
+
add_link('ReadOnlyShard', 'SqlShard(host1)'),
|
117
|
+
add_link('ReplicatingShard', 'ReadOnlyShard'),
|
118
|
+
remove_link('ReplicatingShard', 'SqlShard(host1)') ],
|
119
|
+
:copy => [],
|
120
|
+
:cleanup => []
|
121
|
+
}
|
122
|
+
end
|
123
|
+
|
124
|
+
it "raises an argument error if the transformation requires a copy without a valid source" do
|
125
|
+
to = mk_template 'ReplicatingShard -> (SqlShard(host1), SqlShard(host2))'
|
126
|
+
|
127
|
+
Gizzard::Shard::INVALID_COPY_TYPES.each do |invalid_type|
|
128
|
+
from = mk_template "ReplicatingShard -> #{invalid_type} -> SqlShard(host1)"
|
129
|
+
lambda { Gizzard::Transformation.new(from, to) }.should raise_error(ArgumentError)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
describe "collapse_jobs" do
|
135
|
+
def collapse(jobs); @trans.collapse_jobs(jobs) end
|
136
|
+
|
137
|
+
it "works" do
|
138
|
+
jobs = [ Op::AddLink.new(@host_1_template, @host_2_template),
|
139
|
+
Op::AddLink.new(@host_1_template, @host_3_template) ]
|
140
|
+
collapse(jobs).should == jobs
|
141
|
+
|
142
|
+
collapse([ Op::AddLink.new(@host_1_template, @host_2_template),
|
143
|
+
Op::RemoveLink.new(@host_1_template, @host_2_template) ]).should == []
|
144
|
+
|
145
|
+
collapse([ Op::RemoveLink.new(@host_1_template, @host_2_template),
|
146
|
+
Op::AddLink.new(@host_1_template, @host_2_template) ]).should == []
|
147
|
+
|
148
|
+
collapse(@trans.create_tree(@from_template) + @trans.destroy_tree(@from_template)).should == []
|
149
|
+
|
150
|
+
collapse(@trans.create_tree(@to_template) + @trans.destroy_tree(@from_template)).should ==
|
151
|
+
[ Op::CreateShard.new(@host_3_template),
|
152
|
+
Op::AddLink.new(@to_template, @host_3_template),
|
153
|
+
Op::RemoveLink.new(@blocked_template, @host_1_template),
|
154
|
+
Op::DeleteShard.new(@host_1_template),
|
155
|
+
Op::RemoveLink.new(@from_template, @blocked_template),
|
156
|
+
Op::DeleteShard.new(@blocked_template) ]
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
describe "copy_destination?" do
|
161
|
+
it "returns true if the given template is not a member of the from_template" do
|
162
|
+
@trans.copy_destination?(@host_3_template).should == true
|
163
|
+
end
|
164
|
+
|
165
|
+
it "returns false when there is no from_template (completely new shards, no data to copy)" do
|
166
|
+
@trans = Gizzard::Transformation.new(nil, @to_template)
|
167
|
+
@trans.copy_destination?(@host_1_template).should == false
|
168
|
+
@trans.copy_destination?(@host_2_template).should == false
|
169
|
+
@trans.copy_destination?(@host_3_template).should == false
|
170
|
+
end
|
171
|
+
|
172
|
+
it "returns false if the given template is a member of the from_template (therefore has source data)" do
|
173
|
+
@trans.copy_destination?(@host_1_template).should == false
|
174
|
+
@trans.copy_destination?(@host_2_template).should == false
|
175
|
+
end
|
176
|
+
|
177
|
+
it "returns false if the given template is not concrete" do
|
178
|
+
@trans.copy_destination?(@to_template).should == false
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|