mosql 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile.lock CHANGED
@@ -1,42 +1,37 @@
1
- GIT
2
- remote: git@github.com:stripe-internal/mongoriver
3
- revision: d5b5ca1471f9efe7c91b3abe2c26f612a2dd4e9c
4
- ref: d5b5ca1471f9efe7c91b3abe2c26f612a2dd4e9c
5
- specs:
6
- mongoriver (0.0.1)
7
- bson_ext
8
- log4r
9
- mongo (>= 1.7)
10
-
11
1
  PATH
12
2
  remote: .
13
3
  specs:
14
- mosql (0.0.1)
4
+ mosql (0.1.0)
15
5
  bson_ext
16
6
  json
17
7
  log4r
18
8
  mongo
9
+ mongoriver
19
10
  pg
20
11
  rake
21
12
  sequel
22
13
 
23
14
  GEM
24
- remote: https://intgems.stripe.com:446/
15
+ remote: https://rubygems.org/
25
16
  specs:
26
- bson (1.7.1)
27
- bson_ext (1.7.1)
28
- bson (~> 1.7.1)
29
- json (1.7.5)
17
+ bson (1.8.2)
18
+ bson_ext (1.8.2)
19
+ bson (~> 1.8.2)
20
+ json (1.7.6)
30
21
  log4r (1.1.10)
31
22
  metaclass (0.0.1)
32
23
  minitest (3.0.0)
33
24
  mocha (0.10.5)
34
25
  metaclass (~> 0.0.1)
35
- mongo (1.7.1)
36
- bson (~> 1.7.1)
26
+ mongo (1.8.2)
27
+ bson (~> 1.8.2)
28
+ mongoriver (0.1.0)
29
+ bson_ext
30
+ log4r
31
+ mongo (>= 1.7)
37
32
  pg (0.14.1)
38
- rake (10.0.2)
39
- sequel (3.41.0)
33
+ rake (10.0.3)
34
+ sequel (3.44.0)
40
35
 
41
36
  PLATFORMS
42
37
  ruby
@@ -44,5 +39,4 @@ PLATFORMS
44
39
  DEPENDENCIES
45
40
  minitest
46
41
  mocha
47
- mongoriver!
48
42
  mosql!
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Nelson Elhage
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md CHANGED
@@ -108,10 +108,15 @@ state.
108
108
 
109
109
  You likely want to run `mosql` against a secondary node, at least for
110
110
  the initial import, which will cause large amounts of disk activity on
111
- the target node. One option is to use read preferences in your
112
- connection URI:
111
+ the target node. One option is to specify this in your connect URI:
113
112
 
114
- mosql --mongo mongodb://node1,node2,node3?readPreference=secondary
113
+ mosql --mongo mongodb://node1,node2,node3?slaveOk=true
114
+
115
+ (You should be able to specify `?readPreference=secondary`, but the
116
+ Mongo Ruby driver does not appear to support that usage. I've filed a
117
+ [bug with 10gen][bug-read-pref] about this omission).
118
+
119
+ [bug-read-pref]: https://jira.mongodb.org/browse/RUBY-547
115
120
 
116
121
  ## Advanced usage
117
122
 
data/Rakefile CHANGED
@@ -10,3 +10,11 @@ Rake::TestTask.new do |t|
10
10
  file.end_with?('_lib.rb')
11
11
  end
12
12
  end
13
+
14
+ Rake::TestTask.new(:test_unit) do |t|
15
+ t.libs = ["lib"]
16
+ t.verbose = true
17
+ t.test_files = FileList['test/unit/**/*.rb'].reject do |file|
18
+ file.end_with?('_lib.rb')
19
+ end
20
+ end
data/lib/mosql/cli.rb CHANGED
@@ -86,6 +86,10 @@ module MoSQL
86
86
  opts.on("--reimport", "Force a data re-import") do
87
87
  @options[:reimport] = true
88
88
  end
89
+
90
+ opts.on("--no-drop-tables", "Don't drop the table if it exists during the initial import") do
91
+ @options[:no_drop_tables] = true
92
+ end
89
93
  end
90
94
 
91
95
  optparse.parse!(@args)
@@ -183,7 +187,7 @@ module MoSQL
183
187
  end
184
188
 
185
189
  def initial_import
186
- @schemamap.create_schema(@sql.db, true)
190
+ @schemamap.create_schema(@sql.db, !options[:no_drop_tables])
187
191
 
188
192
  start_ts = @mongo['local']['oplog.rs'].find_one({}, {:sort => [['$natural', -1]]})['ts']
189
193
 
@@ -207,7 +211,7 @@ module MoSQL
207
211
  count = 0
208
212
  batch = []
209
213
  table = @sql.table_for_ns(ns)
210
- table.truncate
214
+ table.truncate unless options[:no_drop_tables]
211
215
 
212
216
  start = Time.now
213
217
  sql_time = 0
@@ -288,14 +292,20 @@ module MoSQL
288
292
  log.debug("resync #{ns}: #{selector['_id']} (update was: #{update.inspect})")
289
293
  sync_object(ns, selector['_id'])
290
294
  else
291
- log.debug("upsert #{ns}: _id=#{update['_id']}")
295
+ log.debug("upsert #{ns}: _id=#{selector['_id']}")
296
+
297
+ # The update operation replaces the existing object, but
298
+ # preserves its _id field, so grab the _id off of the
299
+ # 'query' field -- it's not guaranteed to be present on the
300
+ # update.
301
+ update = { '_id' => selector['_id'] }.merge(update)
292
302
  @sql.upsert_ns(ns, update)
293
303
  end
294
304
  when 'd'
295
305
  if options[:ignore_delete]
296
306
  log.debug("Ignoring delete op on #{ns} as instructed.")
297
307
  else
298
- @sql.table_for_ns(ns).where(:_id => op['o']['_id']).delete
308
+ @sql.delete_ns(ns, op['o'])
299
309
  end
300
310
  else
301
311
  log.info("Skipping unknown op #{op.inspect}")
data/lib/mosql/sql.rb CHANGED
@@ -25,6 +25,7 @@ module MoSQL
25
25
  @db[@schema.table_for_ns(ns).intern]
26
26
  end
27
27
 
28
+
28
29
  def upsert_ns(ns, obj)
29
30
  h = {}
30
31
  cols = @schema.all_columns(@schema.find_ns(ns))
@@ -33,6 +34,15 @@ module MoSQL
33
34
  upsert(table_for_ns(ns), h)
34
35
  end
35
36
 
37
+ # obj must contain an _id field. All other fields will be ignored.
38
+ def delete_ns(ns, obj)
39
+ cols = @schema.all_columns(@schema.find_ns(ns))
40
+ row = @schema.transform(ns, obj)
41
+ sqlid = row[cols.index("_id")]
42
+ raise "No _id found in transform of #{obj}" if sqlid.nil?
43
+ table_for_ns(ns).where(:_id => sqlid).delete
44
+ end
45
+
36
46
  def upsert(table, item)
37
47
  begin
38
48
  upsert!(table, item)
data/lib/mosql/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module MoSQL
2
- VERSION = "0.1.0"
2
+ VERSION = "0.1.1"
3
3
  end
@@ -0,0 +1,59 @@
1
+ require File.join(File.dirname(__FILE__), '_lib.rb')
2
+ require 'mosql/cli'
3
+
4
+ class MoSQL::Test::Functional::CLITest < MoSQL::Test::Functional
5
+ TEST_MAP = <<EOF
6
+ ---
7
+ mosql_test:
8
+ collection:
9
+ :meta:
10
+ :table: sqltable
11
+ :columns:
12
+ - _id: TEXT
13
+ - var: INTEGER
14
+ EOF
15
+
16
+ def fake_cli
17
+ # This is a hack. We should refactor cli.rb to be more testable.
18
+ MoSQL::CLI.any_instance.expects(:setup_signal_handlers)
19
+ cli = MoSQL::CLI.new([])
20
+ cli.instance_variable_set(:@mongo, mongo)
21
+ cli.instance_variable_set(:@schemamap, @map)
22
+ cli.instance_variable_set(:@sql, @adapter)
23
+ cli.instance_variable_set(:@options, {})
24
+ cli
25
+ end
26
+
27
+ before do
28
+ @map = MoSQL::Schema.new(YAML.load(TEST_MAP))
29
+ @adapter = MoSQL::SQLAdapter.new(@map, sql_test_uri)
30
+
31
+ @sequel.drop_table?(:sqltable)
32
+ @map.create_schema(@sequel)
33
+
34
+ @cli = fake_cli
35
+ end
36
+
37
+ it 'handle "u" ops without _id' do
38
+ o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
39
+ @adapter.upsert_ns('mosql_test.collection', o)
40
+
41
+ @cli.handle_op({ 'ns' => 'mosql_test.collection',
42
+ 'op' => 'u',
43
+ 'o2' => { '_id' => o['_id'] },
44
+ 'o' => { 'var' => 27 }
45
+ })
46
+ assert_equal(27, sequel[:sqltable].where(:_id => o['_id'].to_s).select.first[:var])
47
+ end
48
+
49
+ it 'handle "d" ops with BSON::ObjectIds' do
50
+ o = { '_id' => BSON::ObjectId.new, 'var' => 17 }
51
+ @adapter.upsert_ns('mosql_test.collection', o)
52
+
53
+ @cli.handle_op({ 'ns' => 'mosql_test.collection',
54
+ 'op' => 'd',
55
+ 'o' => { '_id' => o['_id'] },
56
+ })
57
+ assert_equal(0, sequel[:sqltable].where(:_id => o['_id'].to_s).count)
58
+ end
59
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mosql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-02-05 00:00:00.000000000 Z
12
+ date: 2013-02-14 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sequel
@@ -182,6 +182,7 @@ files:
182
182
  - .gitignore
183
183
  - Gemfile
184
184
  - Gemfile.lock
185
+ - LICENSE
185
186
  - README.md
186
187
  - Rakefile
187
188
  - bin/mosql
@@ -195,6 +196,7 @@ files:
195
196
  - mosql.gemspec
196
197
  - test/_lib.rb
197
198
  - test/functional/_lib.rb
199
+ - test/functional/cli.rb
198
200
  - test/functional/functional.rb
199
201
  - test/functional/schema.rb
200
202
  - test/functional/sql.rb
@@ -226,6 +228,7 @@ summary: MongoDB -> SQL streaming bridge
226
228
  test_files:
227
229
  - test/_lib.rb
228
230
  - test/functional/_lib.rb
231
+ - test/functional/cli.rb
229
232
  - test/functional/functional.rb
230
233
  - test/functional/schema.rb
231
234
  - test/functional/sql.rb