mosql 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile.lock +15 -21
- data/LICENSE +22 -0
- data/README.md +8 -3
- data/Rakefile +8 -0
- data/lib/mosql/cli.rb +14 -4
- data/lib/mosql/sql.rb +10 -0
- data/lib/mosql/version.rb +1 -1
- data/test/functional/cli.rb +59 -0
- metadata +5 -2
data/Gemfile.lock
CHANGED
@@ -1,42 +1,37 @@
|
|
1
|
-
GIT
|
2
|
-
remote: git@github.com:stripe-internal/mongoriver
|
3
|
-
revision: d5b5ca1471f9efe7c91b3abe2c26f612a2dd4e9c
|
4
|
-
ref: d5b5ca1471f9efe7c91b3abe2c26f612a2dd4e9c
|
5
|
-
specs:
|
6
|
-
mongoriver (0.0.1)
|
7
|
-
bson_ext
|
8
|
-
log4r
|
9
|
-
mongo (>= 1.7)
|
10
|
-
|
11
1
|
PATH
|
12
2
|
remote: .
|
13
3
|
specs:
|
14
|
-
mosql (0.0
|
4
|
+
mosql (0.1.0)
|
15
5
|
bson_ext
|
16
6
|
json
|
17
7
|
log4r
|
18
8
|
mongo
|
9
|
+
mongoriver
|
19
10
|
pg
|
20
11
|
rake
|
21
12
|
sequel
|
22
13
|
|
23
14
|
GEM
|
24
|
-
remote: https://
|
15
|
+
remote: https://rubygems.org/
|
25
16
|
specs:
|
26
|
-
bson (1.
|
27
|
-
bson_ext (1.
|
28
|
-
bson (~> 1.
|
29
|
-
json (1.7.
|
17
|
+
bson (1.8.2)
|
18
|
+
bson_ext (1.8.2)
|
19
|
+
bson (~> 1.8.2)
|
20
|
+
json (1.7.6)
|
30
21
|
log4r (1.1.10)
|
31
22
|
metaclass (0.0.1)
|
32
23
|
minitest (3.0.0)
|
33
24
|
mocha (0.10.5)
|
34
25
|
metaclass (~> 0.0.1)
|
35
|
-
mongo (1.
|
36
|
-
bson (~> 1.
|
26
|
+
mongo (1.8.2)
|
27
|
+
bson (~> 1.8.2)
|
28
|
+
mongoriver (0.1.0)
|
29
|
+
bson_ext
|
30
|
+
log4r
|
31
|
+
mongo (>= 1.7)
|
37
32
|
pg (0.14.1)
|
38
|
-
rake (10.0.
|
39
|
-
sequel (3.
|
33
|
+
rake (10.0.3)
|
34
|
+
sequel (3.44.0)
|
40
35
|
|
41
36
|
PLATFORMS
|
42
37
|
ruby
|
@@ -44,5 +39,4 @@ PLATFORMS
|
|
44
39
|
DEPENDENCIES
|
45
40
|
minitest
|
46
41
|
mocha
|
47
|
-
mongoriver!
|
48
42
|
mosql!
|
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Nelson Elhage
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
CHANGED
@@ -108,10 +108,15 @@ state.
|
|
108
108
|
|
109
109
|
You likely want to run `mosql` against a secondary node, at least for
|
110
110
|
the initial import, which will cause large amounts of disk activity on
|
111
|
-
the target node. One option is to
|
112
|
-
connection URI:
|
111
|
+
the target node. One option is to specify this in your connect URI:
|
113
112
|
|
114
|
-
mosql --mongo mongodb://node1,node2,node3?
|
113
|
+
mosql --mongo mongodb://node1,node2,node3?slaveOk=true
|
114
|
+
|
115
|
+
(You should be able to specify `?readPreference=secondary`, but the
|
116
|
+
Mongo Ruby driver does not appear to support that usage. I've filed a
|
117
|
+
[bug with 10gen][bug-read-pref] about this omission).
|
118
|
+
|
119
|
+
[bug-read-pref]: https://jira.mongodb.org/browse/RUBY-547
|
115
120
|
|
116
121
|
## Advanced usage
|
117
122
|
|
data/Rakefile
CHANGED
@@ -10,3 +10,11 @@ Rake::TestTask.new do |t|
|
|
10
10
|
file.end_with?('_lib.rb')
|
11
11
|
end
|
12
12
|
end
|
13
|
+
|
14
|
+
Rake::TestTask.new(:test_unit) do |t|
|
15
|
+
t.libs = ["lib"]
|
16
|
+
t.verbose = true
|
17
|
+
t.test_files = FileList['test/unit/**/*.rb'].reject do |file|
|
18
|
+
file.end_with?('_lib.rb')
|
19
|
+
end
|
20
|
+
end
|
data/lib/mosql/cli.rb
CHANGED
@@ -86,6 +86,10 @@ module MoSQL
|
|
86
86
|
opts.on("--reimport", "Force a data re-import") do
|
87
87
|
@options[:reimport] = true
|
88
88
|
end
|
89
|
+
|
90
|
+
opts.on("--no-drop-tables", "Don't drop the table if it exists during the initial import") do
|
91
|
+
@options[:no_drop_tables] = true
|
92
|
+
end
|
89
93
|
end
|
90
94
|
|
91
95
|
optparse.parse!(@args)
|
@@ -183,7 +187,7 @@ module MoSQL
|
|
183
187
|
end
|
184
188
|
|
185
189
|
def initial_import
|
186
|
-
@schemamap.create_schema(@sql.db,
|
190
|
+
@schemamap.create_schema(@sql.db, !options[:no_drop_tables])
|
187
191
|
|
188
192
|
start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
|
189
193
|
|
@@ -207,7 +211,7 @@ module MoSQL
|
|
207
211
|
count = 0
|
208
212
|
batch = []
|
209
213
|
table = @sql.table_for_ns(ns)
|
210
|
-
table.truncate
|
214
|
+
table.truncate unless options[:no_drop_tables]
|
211
215
|
|
212
216
|
start = Time.now
|
213
217
|
sql_time = 0
|
@@ -288,14 +292,20 @@ module MoSQL
|
|
288
292
|
log.debug("resync #{ns}: #{selector['_id']} (update was: #{update.inspect})")
|
289
293
|
sync_object(ns, selector['_id'])
|
290
294
|
else
|
291
|
-
log.debug("upsert #{ns}: _id=#{
|
295
|
+
log.debug("upsert #{ns}: _id=#{selector['_id']}")
|
296
|
+
|
297
|
+
# The update operation replaces the existing object, but
|
298
|
+
# preserves its _id field, so grab the _id off of the
|
299
|
+
# 'query' field -- it's not guaranteed to be present on the
|
300
|
+
# update.
|
301
|
+
update = { '_id' => selector['_id'] }.merge(update)
|
292
302
|
@sql.upsert_ns(ns, update)
|
293
303
|
end
|
294
304
|
when 'd'
|
295
305
|
if options[:ignore_delete]
|
296
306
|
log.debug("Ignoring delete op on #{ns} as instructed.")
|
297
307
|
else
|
298
|
-
@sql.
|
308
|
+
@sql.delete_ns(ns, op['o'])
|
299
309
|
end
|
300
310
|
else
|
301
311
|
log.info("Skipping unknown op #{op.inspect}")
|
data/lib/mosql/sql.rb
CHANGED
@@ -25,6 +25,7 @@ module MoSQL
|
|
25
25
|
@db[@schema.table_for_ns(ns).intern]
|
26
26
|
end
|
27
27
|
|
28
|
+
|
28
29
|
def upsert_ns(ns, obj)
|
29
30
|
h = {}
|
30
31
|
cols = @schema.all_columns(@schema.find_ns(ns))
|
@@ -33,6 +34,15 @@ module MoSQL
|
|
33
34
|
upsert(table_for_ns(ns), h)
|
34
35
|
end
|
35
36
|
|
37
|
+
# obj must contain an _id field. All other fields will be ignored.
|
38
|
+
def delete_ns(ns, obj)
|
39
|
+
cols = @schema.all_columns(@schema.find_ns(ns))
|
40
|
+
row = @schema.transform(ns, obj)
|
41
|
+
sqlid = row[cols.index("_id")]
|
42
|
+
raise "No _id found in transform of #{obj}" if sqlid.nil?
|
43
|
+
table_for_ns(ns).where(:_id => sqlid).delete
|
44
|
+
end
|
45
|
+
|
36
46
|
def upsert(table, item)
|
37
47
|
begin
|
38
48
|
upsert!(table, item)
|
data/lib/mosql/version.rb
CHANGED
@@ -0,0 +1,59 @@
|
|
1
|
+
require File.join(File.dirname(__FILE__), '_lib.rb')
|
2
|
+
require 'mosql/cli'
|
3
|
+
|
4
|
+
class MoSQL::Test::Functional::CLITest < MoSQL::Test::Functional
|
5
|
+
TEST_MAP = <<EOF
|
6
|
+
---
|
7
|
+
mosql_test:
|
8
|
+
collection:
|
9
|
+
:meta:
|
10
|
+
:table: sqltable
|
11
|
+
:columns:
|
12
|
+
- _id: TEXT
|
13
|
+
- var: INTEGER
|
14
|
+
EOF
|
15
|
+
|
16
|
+
def fake_cli
|
17
|
+
# This is a hack. We should refactor cli.rb to be more testable.
|
18
|
+
MoSQL::CLI.any_instance.expects(:setup_signal_handlers)
|
19
|
+
cli = MoSQL::CLI.new([])
|
20
|
+
cli.instance_variable_set(:@mongo, mongo)
|
21
|
+
cli.instance_variable_set(:@schemamap, @map)
|
22
|
+
cli.instance_variable_set(:@sql, @adapter)
|
23
|
+
cli.instance_variable_set(:@options, {})
|
24
|
+
cli
|
25
|
+
end
|
26
|
+
|
27
|
+
before do
|
28
|
+
@map = MoSQL::Schema.new(YAML.load(TEST_MAP))
|
29
|
+
@adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
|
30
|
+
|
31
|
+
@sequel.drop_table?(:sqltable)
|
32
|
+
@map.create_schema(@sequel)
|
33
|
+
|
34
|
+
@cli = fake_cli
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'handle "u" ops without _id' do
|
38
|
+
o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
|
39
|
+
@adapter.upsert_ns('mosql_test.collection', o)
|
40
|
+
|
41
|
+
@cli.handle_op({ 'ns' => 'mosql_test.collection',
|
42
|
+
'op' => 'u',
|
43
|
+
'o2' => { '_id' => o['_id'] },
|
44
|
+
'o' => { 'var' => 27 }
|
45
|
+
})
|
46
|
+
assert_equal(27, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'handle "d" ops with BSON::ObjectIds' do
|
50
|
+
o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
|
51
|
+
@adapter.upsert_ns('mosql_test.collection', o)
|
52
|
+
|
53
|
+
@cli.handle_op({ 'ns' => 'mosql_test.collection',
|
54
|
+
'op' => 'd',
|
55
|
+
'o' => { '_id' => o['_id'] },
|
56
|
+
})
|
57
|
+
assert_equal(0, sequel[:sqltable].where(:_id => o['_id'].to_s).count)
|
58
|
+
end
|
59
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mosql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-02-
|
12
|
+
date: 2013-02-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: sequel
|
@@ -182,6 +182,7 @@ files:
|
|
182
182
|
- .gitignore
|
183
183
|
- Gemfile
|
184
184
|
- Gemfile.lock
|
185
|
+
- LICENSE
|
185
186
|
- README.md
|
186
187
|
- Rakefile
|
187
188
|
- bin/mosql
|
@@ -195,6 +196,7 @@ files:
|
|
195
196
|
- mosql.gemspec
|
196
197
|
- test/_lib.rb
|
197
198
|
- test/functional/_lib.rb
|
199
|
+
- test/functional/cli.rb
|
198
200
|
- test/functional/functional.rb
|
199
201
|
- test/functional/schema.rb
|
200
202
|
- test/functional/sql.rb
|
@@ -226,6 +228,7 @@ summary: MongoDB -> SQL streaming bridge
|
|
226
228
|
test_files:
|
227
229
|
- test/_lib.rb
|
228
230
|
- test/functional/_lib.rb
|
231
|
+
- test/functional/cli.rb
|
229
232
|
- test/functional/functional.rb
|
230
233
|
- test/functional/schema.rb
|
231
234
|
- test/functional/sql.rb
|