mortar 0.15.7 → 0.15.11

Sign up to get free protection for your applications and to get access to all the features.
@@ -556,6 +556,10 @@ protected
556
556
  return Mortar::Auth.user_s3_safe + "-base"
557
557
  end
558
558
 
559
+ def jdbc_conn(dbtype, host, dbname)
560
+ "jdbc:#{dbtype}://#{host}/#{dbname}?zeroDateTimeBehavior=convertToNull"
561
+ end
562
+
559
563
  end
560
564
 
561
565
  module Mortar::Command
@@ -22,7 +22,6 @@ require "mortar/generators/characterize_generator"
22
22
  #
23
23
  class Mortar::Command::Local < Mortar::Command::Base
24
24
 
25
-
26
25
  # local:configure
27
26
  #
28
27
  # Install dependencies for running this mortar project locally - other mortar:local commands will also perform this step automatically.
@@ -257,5 +256,136 @@ class Mortar::Command::Local < Mortar::Command::Base
257
256
  ctrl.run_luigi(script, luigi_params)
258
257
  end
259
258
 
259
+ # local:sqoop_table dbtype database-name table s3-destination
260
+ #
261
+ # Export data from an RDBMS table to S3.
262
+ #
263
+ # -h, --host HOSTNAME # Database host, localhost assumed if not specified
264
+ # -u, --username USERNAME # User to log into the database with
265
+ # -p, --password PASSWORD # Password to log into the database
266
+ # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
267
+ # -d, --direct # Use a direct import path
268
+ # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
269
+ #
270
+ #Examples:
271
+ #
272
+ # Export from a postgres database
273
+ # $ mortar local:sqoop_table postgres mydb mytable s3://com.dbhost01/archive -u steve -p stevespassword
274
+ def sqoop_table
275
+ dbtype = shift_argument
276
+ unless dbtype
277
+ error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database type.")
278
+ end
279
+ physdb = shift_argument
280
+ unless physdb
281
+ error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database name.")
282
+ end
283
+ dbtable = shift_argument
284
+ unless dbtable
285
+ error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify database table.")
286
+ end
287
+ s3dest = shift_argument
288
+ unless s3dest
289
+ error("Usage: mortar local:sqoop_table dbtype database-name table s3-destination\nMust specify s3 destination.")
290
+ end
291
+ validate_arguments!
292
+
293
+ dbhost = options[:host] || "localhost"
294
+ connstr = jdbc_conn(dbtype, dbhost, physdb)
295
+
296
+ ctrl = Mortar::Local::Controller.new
297
+ ctrl.sqoop_export_table(connstr, dbtable, s3dest, options)
298
+ end
299
+
300
+ # local:sqoop_query dbtype database-name query s3-destination
301
+ #
302
+ # Export the result of an SQL query to S3.
303
+ #
304
+ # -h, --host HOSTNAME # Database host, localhost assumed if not specified
305
+ # -u, --username USERNAME # User to log into the database with
306
+ # -p, --password PASSWORD # Password to log into the database
307
+ # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
308
+ # -d, --direct # Use a direct import path
309
+ # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
310
+ #
311
+ #Examples:
312
+ #
313
+ # Export from a postgres database
314
+ # $ mortar local:sqoop_query postgres mydb "select user_name, id from users" s3://com.dbhost01/archive
315
+ def sqoop_query
316
+ dbtype = shift_argument
317
+ unless dbtype
318
+ error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database type.")
319
+ end
320
+ physdb = shift_argument
321
+ unless physdb
322
+ error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify database name.")
323
+ end
324
+ query = shift_argument
325
+ unless query
326
+ error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify sql query.")
327
+ end
328
+ s3dest = shift_argument
329
+ unless s3dest
330
+ error("Usage: mortar local:sqoop_query dbtype database-name query s3-destination\nMust specify s3 destination.")
331
+ end
332
+ validate_arguments!
333
+
334
+ dbhost = options[:host] || "localhost"
335
+ connstr = jdbc_conn(dbtype, dbhost, physdb)
336
+
337
+ ctrl = Mortar::Local::Controller.new
338
+ ctrl.sqoop_export_query(connstr, query, s3dest, options)
339
+ end
340
+
341
+ # local:sqoop_incremental dbtype database-name table column value s3-destination
342
+ #
343
+ # Export all records where column is > value
344
+ #
345
+ # -h, --host HOSTNAME # Database host, localhost assumed if not specified
346
+ # -u, --username USERNAME # User to log into the database with
347
+ # -p, --password PASSWORD # Password to log into the database
348
+ # -j, --jdbcdriver COM.DRIVER.BAR # Name of the JDBC driver class
349
+ # -d, --direct # Use a direct import path
350
+ # -r, --driverjar JARFILE # Path to the jar containing the jdbc driver
351
+ #
352
+ #Examples:
353
+ #
354
+ # Export from the newest users
355
+ # $ mortar local:sqoop_incremental postgres mydb users user_id 12345 s3://com.dbhost01/archive
356
+ def sqoop_incremental
357
+ dbtype = shift_argument
358
+ unless dbtype
359
+ error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database type.")
360
+ end
361
+ physdb = shift_argument
362
+ unless physdb
363
+ error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database name.")
364
+ end
365
+ table = shift_argument
366
+ unless table
367
+ error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify database table.")
368
+ end
369
+ column = shift_argument
370
+ unless column
371
+ error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify column.")
372
+ end
373
+ max_value = shift_argument
374
+ unless max_value
375
+ error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify value.")
376
+ end
377
+ s3dest = shift_argument
378
+ unless s3dest
379
+ error("Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination\nMust specify s3 destination.")
380
+ end
381
+ validate_arguments!
382
+
383
+ dbhost = options[:host] || "localhost"
384
+ connstr = jdbc_conn(dbtype, dbhost, physdb)
385
+
386
+ ctrl = Mortar::Local::Controller.new
387
+ ctrl.sqoop_export_incremental(connstr, table, column, max_value, s3dest, options)
388
+ end
389
+
260
390
 
261
391
  end
@@ -19,6 +19,7 @@ require "mortar/local/pig"
19
19
  require "mortar/local/java"
20
20
  require "mortar/local/python"
21
21
  require "mortar/local/jython"
22
+ require "mortar/local/sqoop"
22
23
 
23
24
 
24
25
  class Mortar::Local::Controller
@@ -144,6 +145,10 @@ EOF
144
145
  jy = Mortar::Local::Jython.new()
145
146
  jy.install_or_update()
146
147
 
148
+
149
+ sqoop = Mortar::Local::Sqoop.new()
150
+ sqoop.install_or_update()
151
+
147
152
  ensure_local_install_dirs_in_gitignore
148
153
  end
149
154
 
@@ -206,4 +211,27 @@ EOF
206
211
  py.run_luigi_script(luigi_script, user_script_args)
207
212
  end
208
213
 
214
+ def sqoop_export_table(connstr, dbtable, s3dest, options)
215
+ install_and_configure(nil, 'sqoop')
216
+ sqoop = Mortar::Local::Sqoop.new()
217
+ options[:dbtable] = dbtable
218
+ sqoop.export(connstr, s3dest, options)
219
+ end
220
+
221
+ def sqoop_export_query(connstr, query, s3dest, options)
222
+ install_and_configure(nil, 'sqoop')
223
+ sqoop = Mortar::Local::Sqoop.new()
224
+ options[:sqlquery] = sqoop.prep_query(query)
225
+ sqoop.export(connstr, s3dest, options)
226
+ end
227
+
228
+ def sqoop_export_incremental(connstr, dbtable, column, max_value, s3dest, options)
229
+ install_and_configure(nil, 'sqoop')
230
+ sqoop = Mortar::Local::Sqoop.new()
231
+ options[:dbtable] = dbtable
232
+ options[:inc_column] = column
233
+ options[:inc_value] = max_value
234
+ sqoop.export(connstr, s3dest, options)
235
+ end
236
+
209
237
  end
@@ -0,0 +1,159 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require "erb"
18
+ require 'tempfile'
19
+ require "mortar/helpers"
20
+ require "mortar/local/installutil"
21
+
22
+ class Mortar::Local::Sqoop
23
+ include Mortar::Local::InstallUtil
24
+
25
+ def install_or_update
26
+ @command = "#{local_install_directory}/python/bin/python"
27
+ if should_do_install?
28
+ action "Installing sqoop to #{local_install_directory_name}" do
29
+ do_install
30
+ end
31
+ elsif should_do_update?
32
+ action "Updating to latest sqoop in #{local_install_directory_name}" do
33
+ do_install
34
+ end
35
+ end
36
+ true
37
+ end
38
+
39
+ def sqoop_url
40
+ full_host = (host =~ /^http/) ? host : "https://api.#{host}"
41
+ default_url = full_host + "/" + "resource/sqoop"
42
+ return ENV.fetch('SQOOP_DISTRO_URL', default_url)
43
+ end
44
+
45
+ def should_do_install?
46
+ return (not (File.exists?(sqoop_directory)))
47
+ end
48
+
49
+ def should_do_update?
50
+ return is_newer_version('sqoop', sqoop_url)
51
+ end
52
+
53
+ def sqoop_directory
54
+ return "#{local_install_directory}/sqoop"
55
+ end
56
+
57
+ def sqoop_dir_in_tgz
58
+ "sqoop-1.4.4-mortar"
59
+ end
60
+
61
+ def do_install
62
+ local_tgz = File.join(local_install_directory, "sqoop-1.4.4-mortar.tar.gz")
63
+ if File.exists?(local_tgz)
64
+ FileUtils.rm(local_tgz)
65
+ end
66
+ download_file(sqoop_url, local_tgz)
67
+
68
+ if File.exists?(sqoop_directory)
69
+ FileUtils.rm_rf(sqoop_directory)
70
+ end
71
+
72
+ extract_tgz(local_tgz, local_install_directory)
73
+
74
+ FileUtils.mv(File.join(local_install_directory, sqoop_dir_in_tgz), sqoop_directory)
75
+
76
+ # This has been seening coming out of the tgz w/o +x so we do
77
+ # here to be sure it has the necessary permissions
78
+ FileUtils.chmod(0755, "#{sqoop_directory}/bin/sqoop")
79
+ FileUtils.chmod(0755, "#{sqoop_directory}/hadoop/bin/hadoop")
80
+
81
+ File.delete(local_tgz)
82
+ note_install("sqoop")
83
+ end
84
+
85
+ def sqoop_command_script_template_path
86
+ File.expand_path("../../templates/script/sqoop.sh", __FILE__)
87
+ end
88
+
89
+ def hadoop_home
90
+ "#{sqoop_directory}/hadoop"
91
+ end
92
+
93
+ def export(connstr, s3dest, options)
94
+ template_params = sqoop_export_template_parameters(connstr, s3dest, options)
95
+ return run_templated_script(sqoop_command_script_template_path, template_params)
96
+ end
97
+
98
+ def sqoop_export_template_parameters(connstr, s3dest, options)
99
+ pig = Mortar::Local::Pig.new()
100
+ parameters = {
101
+ "sqoop_dir" => sqoop_directory,
102
+ "jdb_conn_string" => connstr,
103
+ "destination" => s3dest,
104
+ "hadoop_home" => hadoop_home,
105
+ "classpath" => pig.template_params_classpath,
106
+ "jdbc_conn" => connstr,
107
+ "s3dest" => s3dest,
108
+ "project_root" => project_root,
109
+ "sqoop_opts" => sqoop_java_options
110
+ }
111
+ parameters["dbtable"] = options[:dbtable] if options[:dbtable]
112
+ parameters["sqlquery"] = options[:sqlquery] if options[:sqlquery]
113
+ parameters["inc_column"] = options[:inc_column] if options[:inc_column]
114
+ parameters["inc_value"] = options[:inc_value] if options[:inc_value]
115
+ if options[:inc_value] and 0 == options[:inc_value].to_i
116
+ parameters[:inc_mode] = "lastmodified"
117
+ elsif options[:inc_value]
118
+ parameters[:inc_mode] = "append"
119
+ end
120
+ parameters["dbuser"] = options[:username] if options[:username]
121
+ parameters["dbpass"] = options[:password] if options[:password]
122
+ parameters["jdbcdriver"] = options[:jdbcdriver] if options[:jdbcdriver]
123
+ parameters["driverjar"] = options[:driverjar] if options[:driverjar]
124
+ parameters["direct_import"] = true if options[:direct]
125
+ return parameters
126
+ end
127
+
128
+ def sqoop_java_options
129
+ opts = {}
130
+ opts['fs.s3n.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
131
+ opts['fs.s3n.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
132
+ opts['fs.s3.awsAccessKeyId'] = ENV['AWS_ACCESS_KEY']
133
+ opts['fs.s3.awsSecretAccessKey'] = ENV['AWS_SECRET_KEY']
134
+ return opts
135
+ end
136
+
137
+ # So this part kind of sucks. In order to partition a query across multiple map
138
+ # reduce tasks sqoop does a query to to find the range of identifiying values,
139
+ # divides this range across the number of tasks to be executed and then modifies
140
+ # the query for each m/r task. To do this Sqoop needs to know at what point in the
141
+ # query that it should place its portion of the where clause. This is done via the
142
+ # $CONDITIONS marker. So that's well and good when you're running sqoop on a cluster
143
+ # but our users will be running on their own machine and don't know or care for this
144
+ # parrallel queries stuff. So to make their lives easier we make a best effort to
145
+ # add the clause for them in a safe way.
146
+ def prep_query(original_query)
147
+ if original_query.include? "$CONDITIONS"
148
+ return original_query
149
+ elsif original_query.downcase.include? "where"
150
+ idxwhere = original_query.downcase.index("where")
151
+ select_where = original_query[0..idxwhere+"where".length-1]
152
+ clause = original_query[idxwhere+"where".length+1..original_query.length]
153
+ return "#{select_where} (#{clause}) AND \$CONDITIONS"
154
+ else
155
+ return "#{original_query} WHERE \$CONDITIONS"
156
+ end
157
+ end
158
+
159
+ end
@@ -0,0 +1,40 @@
1
+ #!/bin/bash
2
+
3
+ set -e
4
+
5
+ export HADOOP_CLASSPATH="<%= @project_root %>/lib/*"
6
+ <% if @driverjar %>
7
+ export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:<%= @driverjar %>"
8
+ <% end %>
9
+ export HADOOP_COMMON_HOME="<%= @hadoop_home %>"
10
+ export HADOOP_MAPRED_HOME="<%= @hadoop_home %>"
11
+ export HADOOP_HOME_WARN_SUPPRESS="t"
12
+
13
+ # Only setting these to get rid of warnings that sqoop is showing
14
+ export HCAT_HOME="<%= @hadoop_home %>"
15
+ export HBASE_HOME="<%= @hadoop_home %>"
16
+
17
+ SQOOP_OPTS="<% @sqoop_opts.each do |k,v| %>-D<%= k %>=<%= v %> <% end %>"
18
+ OPTARGS='<%= "--driver #{@jdbcdriver}" if @jdbcdriver %>'
19
+ OPTARGS="$OPTARGS <%= "--username #{@dbuser}" if @dbuser %>"
20
+ OPTARGS="$OPTARGS <%= "--password #{@dbpass}" if @dbpass %>"
21
+ OPTARGS="$OPTARGS <%= "--direct" if @direct_import %>"
22
+ <% if @inc_column and @inc_value %>
23
+ OPTARGS="$OPTARGS --incremental <%= @inc_mode %>"
24
+ OPTARGS="$OPTARGS --check-column <%= @inc_column %> "
25
+ SQOOP_OPTS="$SQOOP_OPTS -Dsqoop.test.import.rootDir=<%= @s3dest %>/.tmp"
26
+ <% end %>
27
+
28
+ export HADOOP_OPTS="$SQOOP_OPTS";
29
+
30
+ <%= @sqoop_dir %>/bin/sqoop \
31
+ import \
32
+ $SQOOP_OPTS \
33
+ <%= "--table #{@dbtable}" if @dbtable %> \
34
+ <%= "--query '#{@sqlquery}'" if @sqlquery %> \
35
+ -m 1 \
36
+ --connect <%= @jdbc_conn %> \
37
+ --target-dir <%= @s3dest %> \
38
+ $OPTARGS \
39
+ <% if @inc_column and @inc_value %>--last-value '<%= @inc_value %>'<% end %>
40
+
@@ -16,5 +16,5 @@
16
16
 
17
17
  module Mortar
18
18
  # see http://semver.org/
19
- VERSION = "0.15.7"
19
+ VERSION = "0.15.11"
20
20
  end
@@ -206,6 +206,9 @@ PARAMS
206
206
  any_instance_of(Mortar::Local::Jython) do |j|
207
207
  mock(j).install_or_update.returns(true)
208
208
  end
209
+ any_instance_of(Mortar::Local::Sqoop) do |j|
210
+ mock(j).install_or_update.returns(true)
211
+ end
209
212
  any_instance_of(Mortar::Local::Controller) do |j|
210
213
  mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
211
214
  end
@@ -306,6 +309,194 @@ STDERR
306
309
 
307
310
  end
308
311
 
312
+ context "local:sqoop_table" do
313
+ it "requires a db type" do
314
+ stderr, stdout = execute "local:sqoop_table"
315
+ stderr.should == <<-STDERR
316
+ ! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
317
+ ! Must specify database type.
318
+ STDERR
319
+ end
320
+
321
+ it "requires the physical db name" do
322
+ stderr, stdout = execute "local:sqoop_table mysql"
323
+ stderr.should == <<-STDERR
324
+ ! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
325
+ ! Must specify database name.
326
+ STDERR
327
+ end
328
+
329
+ it "requires the table name" do
330
+ stderr, stdout = execute "local:sqoop_table mysql myappdb"
331
+ stderr.should == <<-STDERR
332
+ ! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
333
+ ! Must specify database table.
334
+ STDERR
335
+ end
336
+
337
+ it "requires the s3 destination" do
338
+ stderr, stdout = execute "local:sqoop_table mysql myappdb customers"
339
+ stderr.should == <<-STDERR
340
+ ! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
341
+ ! Must specify s3 destination.
342
+ STDERR
343
+ end
344
+
345
+ it "sends everything to the controller" do
346
+ connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
347
+ dbtable = "customers"
348
+ s3dest = "s3n://a-bucket/a-directory"
349
+ any_instance_of(Mortar::Local::Controller) do |c|
350
+ mock(c).sqoop_export_table(connstr, dbtable, s3dest, {})
351
+ end
352
+ stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} --host foobar.com"
353
+ end
354
+
355
+ it "defaults to 'localhost' if no host specified" do
356
+ connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
357
+ dbtable = "customers"
358
+ s3dest = "s3n://a-bucket/a-directory"
359
+ any_instance_of(Mortar::Local::Controller) do |c|
360
+ mock(c).sqoop_export_table(connstr, dbtable, s3dest, {})
361
+ end
362
+ stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest}"
363
+ end
364
+
365
+ end
366
+
367
+ context "local:sqoop_query" do
368
+ it "requires a db type" do
369
+ stderr, stdout = execute "local:sqoop_query"
370
+ stderr.should == <<-STDERR
371
+ ! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
372
+ ! Must specify database type.
373
+ STDERR
374
+ end
375
+
376
+ it "requires the physical db name" do
377
+ stderr, stdout = execute "local:sqoop_query mysql"
378
+ stderr.should == <<-STDERR
379
+ ! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
380
+ ! Must specify database name.
381
+ STDERR
382
+ end
383
+
384
+ it "requires the table name" do
385
+ stderr, stdout = execute "local:sqoop_query mysql myappdb"
386
+ stderr.should == <<-STDERR
387
+ ! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
388
+ ! Must specify sql query.
389
+ STDERR
390
+ end
391
+
392
+ it "requires the s3 destination" do
393
+ stderr, stdout = execute "local:sqoop_query mysql myappdb customers"
394
+ stderr.should == <<-STDERR
395
+ ! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
396
+ ! Must specify s3 destination.
397
+ STDERR
398
+ end
399
+
400
+ it "sends everything to the controller" do
401
+ connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
402
+ query = "select_*_from_customers"
403
+ s3dest = "s3n://a-bucket/a-directory"
404
+ any_instance_of(Mortar::Local::Controller) do |c|
405
+ mock(c).sqoop_export_query(connstr, query, s3dest, {})
406
+ end
407
+ stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} --host foobar.com"
408
+ stderr.should == ''
409
+ end
410
+
411
+ it "defaults to 'localhost' if no host specified" do
412
+ connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
413
+ query = "select_*_from_customers"
414
+ s3dest = "s3n://a-bucket/a-directory"
415
+ any_instance_of(Mortar::Local::Controller) do |c|
416
+ mock(c).sqoop_export_query(connstr, query, s3dest, {})
417
+ end
418
+ stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest}"
419
+ stderr.should == ''
420
+ end
421
+
422
+ end
423
+
424
+ context "local:sqoop_incremental" do
425
+ it "requires a db type" do
426
+ stderr, stdout = execute "local:sqoop_incremental"
427
+ stderr.should == <<-STDERR
428
+ ! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
429
+ ! Must specify database type.
430
+ STDERR
431
+ end
432
+
433
+ it "requires the physical db name" do
434
+ stderr, stdout = execute "local:sqoop_incremental mysql"
435
+ stderr.should == <<-STDERR
436
+ ! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
437
+ ! Must specify database name.
438
+ STDERR
439
+ end
440
+
441
+ it "requires the table name" do
442
+ stderr, stdout = execute "local:sqoop_incremental mysql myappdb"
443
+ stderr.should == <<-STDERR
444
+ ! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
445
+ ! Must specify database table.
446
+ STDERR
447
+ end
448
+
449
+ it "requires the column name" do
450
+ stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable"
451
+ stderr.should == <<-STDERR
452
+ ! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
453
+ ! Must specify column.
454
+ STDERR
455
+ end
456
+
457
+ it "requires the column value" do
458
+ stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn"
459
+ stderr.should == <<-STDERR
460
+ ! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
461
+ ! Must specify value.
462
+ STDERR
463
+ end
464
+
465
+ it "requires the s3 destination" do
466
+ stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn customers"
467
+ stderr.should == <<-STDERR
468
+ ! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
469
+ ! Must specify s3 destination.
470
+ STDERR
471
+ end
472
+
473
+ it "sends everything to the controller" do
474
+ connstr = "jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull"
475
+ dbtable = "customers"
476
+ column = "customer_id"
477
+ column_value = "12345"
478
+ s3dest = "s3n://a-bucket/a-directory"
479
+ any_instance_of(Mortar::Local::Controller) do |c|
480
+ mock(c).sqoop_export_incremental(connstr, dbtable, column, column_value, s3dest, {})
481
+ end
482
+ stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} --host foobar.com"
483
+ end
484
+
485
+ it "defaults to 'localhost' if no host specified" do
486
+ connstr = "jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull"
487
+ dbtable = "customers"
488
+ column = "customer_id"
489
+ column_value = "12345"
490
+ s3dest = "s3n://a-bucket/a-directory"
491
+ any_instance_of(Mortar::Local::Controller) do |c|
492
+ mock(c).sqoop_export_incremental(connstr, dbtable, column, column_value, s3dest, {})
493
+ end
494
+ stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest}"
495
+ end
496
+
497
+ end
498
+
499
+
309
500
 
310
501
  end
311
502
  end
@@ -146,6 +146,10 @@ module Mortar::Local
146
146
  any_instance_of(Mortar::Local::Jython) do |j|
147
147
  mock(j).install_or_update
148
148
  end
149
+ any_instance_of(Mortar::Local::Sqoop) do |s|
150
+ mock(s).install_or_update
151
+ end
152
+
149
153
  mock(ctrl).ensure_local_install_dirs_in_gitignore
150
154
  ctrl.install_and_configure
151
155
  end
@@ -0,0 +1,57 @@
1
+ #
2
+ # Copyright 2012 Mortar Data Inc.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'spec_helper'
18
+ require 'fakefs/spec_helpers'
19
+ require 'mortar/local/sqoop'
20
+ require 'launchy'
21
+
22
+
23
+ module Mortar::Local
24
+ describe Sqoop do
25
+
26
+ context "prepare query" do
27
+
28
+ it "adds a where clause if none exists" do
29
+ sqoop = Mortar::Local::Sqoop.new
30
+ expect(sqoop.prep_query("SELECT * FROM customers")).to eq("SELECT * FROM customers WHERE \$CONDITIONS")
31
+ end
32
+
33
+ it "wraps existing where clause and appends condition" do
34
+ original = "SELECT * FROM customers WHERE customer_id = 1"
35
+ expected = "SELECT * FROM customers WHERE (customer_id = 1) AND \$CONDITIONS"
36
+ sqoop = Mortar::Local::Sqoop.new
37
+ expect(sqoop.prep_query(original)).to eq(expected)
38
+ end
39
+
40
+ it "wraps a complex where clause and appends condition" do
41
+ original = "SELECT * FROM customers WHERE (customer_id = 1 and customer_name = 'tom') or customer_id=2"
42
+ expected = "SELECT * FROM customers WHERE ((customer_id = 1 and customer_name = 'tom') or customer_id=2) AND \$CONDITIONS"
43
+ sqoop = Mortar::Local::Sqoop.new
44
+ expect(sqoop.prep_query(original)).to eq(expected)
45
+ end
46
+
47
+ it "does nothing if the user was polite enough to supply the clause themselves" do
48
+ query = "SELECT * FROM customers WHERE (customer_id = 1) AND \$CONDITIONS"
49
+ sqoop = Mortar::Local::Sqoop.new
50
+ expect(sqoop.prep_query(query)).to eq(query)
51
+ end
52
+
53
+
54
+ end
55
+
56
+ end
57
+ end
metadata CHANGED
@@ -1,204 +1,229 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: mortar
3
- version: !ruby/object:Gem::Version
4
- version: 0.15.7
3
+ version: !ruby/object:Gem::Version
4
+ hash: 53
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 15
9
+ - 11
10
+ version: 0.15.11
5
11
  platform: ruby
6
- authors:
12
+ authors:
7
13
  - Mortar Data
8
14
  autorequire:
9
15
  bindir: bin
10
16
  cert_chain: []
11
- date: 2014-04-14 00:00:00.000000000 Z
12
- dependencies:
13
- - !ruby/object:Gem::Dependency
17
+
18
+ date: 2014-04-22 00:00:00 Z
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
14
21
  name: rdoc
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ! '>='
18
- - !ruby/object:Gem::Version
19
- version: 4.0.0
20
- type: :runtime
21
22
  prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - ! '>='
25
- - !ruby/object:Gem::Version
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ none: false
25
+ requirements:
26
+ - - ">="
27
+ - !ruby/object:Gem::Version
28
+ hash: 63
29
+ segments:
30
+ - 4
31
+ - 0
32
+ - 0
26
33
  version: 4.0.0
27
- - !ruby/object:Gem::Dependency
28
- name: mortar-api-ruby
29
- requirement: !ruby/object:Gem::Requirement
30
- requirements:
31
- - - ~>
32
- - !ruby/object:Gem::Version
33
- version: 0.8.1
34
34
  type: :runtime
35
+ version_requirements: *id001
36
+ - !ruby/object:Gem::Dependency
37
+ name: mortar-api-ruby
35
38
  prerelease: false
36
- version_requirements: !ruby/object:Gem::Requirement
37
- requirements:
39
+ requirement: &id002 !ruby/object:Gem::Requirement
40
+ none: false
41
+ requirements:
38
42
  - - ~>
39
- - !ruby/object:Gem::Version
40
- version: 0.8.1
41
- - !ruby/object:Gem::Dependency
42
- name: netrc
43
- requirement: !ruby/object:Gem::Requirement
44
- requirements:
45
- - - ~>
46
- - !ruby/object:Gem::Version
47
- version: '0.7'
43
+ - !ruby/object:Gem::Version
44
+ hash: 57
45
+ segments:
46
+ - 0
47
+ - 8
48
+ - 3
49
+ version: 0.8.3
48
50
  type: :runtime
51
+ version_requirements: *id002
52
+ - !ruby/object:Gem::Dependency
53
+ name: netrc
49
54
  prerelease: false
50
- version_requirements: !ruby/object:Gem::Requirement
51
- requirements:
52
- - - ~>
53
- - !ruby/object:Gem::Version
54
- version: '0.7'
55
- - !ruby/object:Gem::Dependency
56
- name: launchy
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
55
+ requirement: &id003 !ruby/object:Gem::Requirement
56
+ none: false
57
+ requirements:
59
58
  - - ~>
60
- - !ruby/object:Gem::Version
61
- version: '2.1'
59
+ - !ruby/object:Gem::Version
60
+ hash: 5
61
+ segments:
62
+ - 0
63
+ - 7
64
+ version: "0.7"
62
65
  type: :runtime
66
+ version_requirements: *id003
67
+ - !ruby/object:Gem::Dependency
68
+ name: launchy
63
69
  prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - ~>
67
- - !ruby/object:Gem::Version
68
- version: '2.1'
69
- - !ruby/object:Gem::Dependency
70
- name: parseconfig
71
- requirement: !ruby/object:Gem::Requirement
72
- requirements:
70
+ requirement: &id004 !ruby/object:Gem::Requirement
71
+ none: false
72
+ requirements:
73
73
  - - ~>
74
- - !ruby/object:Gem::Version
75
- version: 1.0.2
74
+ - !ruby/object:Gem::Version
75
+ hash: 1
76
+ segments:
77
+ - 2
78
+ - 1
79
+ version: "2.1"
76
80
  type: :runtime
81
+ version_requirements: *id004
82
+ - !ruby/object:Gem::Dependency
83
+ name: parseconfig
77
84
  prerelease: false
78
- version_requirements: !ruby/object:Gem::Requirement
79
- requirements:
85
+ requirement: &id005 !ruby/object:Gem::Requirement
86
+ none: false
87
+ requirements:
80
88
  - - ~>
81
- - !ruby/object:Gem::Version
89
+ - !ruby/object:Gem::Version
90
+ hash: 19
91
+ segments:
92
+ - 1
93
+ - 0
94
+ - 2
82
95
  version: 1.0.2
83
- - !ruby/object:Gem::Dependency
84
- name: aws-sdk
85
- requirement: !ruby/object:Gem::Requirement
86
- requirements:
87
- - - ~>
88
- - !ruby/object:Gem::Version
89
- version: '1.0'
90
96
  type: :runtime
97
+ version_requirements: *id005
98
+ - !ruby/object:Gem::Dependency
99
+ name: aws-sdk
91
100
  prerelease: false
92
- version_requirements: !ruby/object:Gem::Requirement
93
- requirements:
94
- - - ~>
95
- - !ruby/object:Gem::Version
96
- version: '1.0'
97
- - !ruby/object:Gem::Dependency
98
- name: nokogiri
99
- requirement: !ruby/object:Gem::Requirement
100
- requirements:
101
+ requirement: &id006 !ruby/object:Gem::Requirement
102
+ none: false
103
+ requirements:
101
104
  - - ~>
102
- - !ruby/object:Gem::Version
103
- version: 1.5.0
105
+ - !ruby/object:Gem::Version
106
+ hash: 15
107
+ segments:
108
+ - 1
109
+ - 0
110
+ version: "1.0"
104
111
  type: :runtime
112
+ version_requirements: *id006
113
+ - !ruby/object:Gem::Dependency
114
+ name: nokogiri
105
115
  prerelease: false
106
- version_requirements: !ruby/object:Gem::Requirement
107
- requirements:
116
+ requirement: &id007 !ruby/object:Gem::Requirement
117
+ none: false
118
+ requirements:
108
119
  - - ~>
109
- - !ruby/object:Gem::Version
120
+ - !ruby/object:Gem::Version
121
+ hash: 3
122
+ segments:
123
+ - 1
124
+ - 5
125
+ - 0
110
126
  version: 1.5.0
111
- - !ruby/object:Gem::Dependency
127
+ type: :runtime
128
+ version_requirements: *id007
129
+ - !ruby/object:Gem::Dependency
112
130
  name: excon
113
- requirement: !ruby/object:Gem::Requirement
114
- requirements:
115
- - - ~>
116
- - !ruby/object:Gem::Version
117
- version: '0.28'
118
- type: :development
119
131
  prerelease: false
120
- version_requirements: !ruby/object:Gem::Requirement
121
- requirements:
132
+ requirement: &id008 !ruby/object:Gem::Requirement
133
+ none: false
134
+ requirements:
122
135
  - - ~>
123
- - !ruby/object:Gem::Version
124
- version: '0.28'
125
- - !ruby/object:Gem::Dependency
126
- name: fakefs
127
- requirement: !ruby/object:Gem::Requirement
128
- requirements:
129
- - - ~>
130
- - !ruby/object:Gem::Version
131
- version: 0.4.2
136
+ - !ruby/object:Gem::Version
137
+ hash: 51
138
+ segments:
139
+ - 0
140
+ - 28
141
+ version: "0.28"
132
142
  type: :development
143
+ version_requirements: *id008
144
+ - !ruby/object:Gem::Dependency
145
+ name: fakefs
133
146
  prerelease: false
134
- version_requirements: !ruby/object:Gem::Requirement
135
- requirements:
147
+ requirement: &id009 !ruby/object:Gem::Requirement
148
+ none: false
149
+ requirements:
136
150
  - - ~>
137
- - !ruby/object:Gem::Version
151
+ - !ruby/object:Gem::Version
152
+ hash: 11
153
+ segments:
154
+ - 0
155
+ - 4
156
+ - 2
138
157
  version: 0.4.2
139
- - !ruby/object:Gem::Dependency
140
- name: gem-release
141
- requirement: !ruby/object:Gem::Requirement
142
- requirements:
143
- - - ! '>='
144
- - !ruby/object:Gem::Version
145
- version: '0'
146
158
  type: :development
159
+ version_requirements: *id009
160
+ - !ruby/object:Gem::Dependency
161
+ name: gem-release
147
162
  prerelease: false
148
- version_requirements: !ruby/object:Gem::Requirement
149
- requirements:
150
- - - ! '>='
151
- - !ruby/object:Gem::Version
152
- version: '0'
153
- - !ruby/object:Gem::Dependency
154
- name: rake
155
- requirement: !ruby/object:Gem::Requirement
156
- requirements:
157
- - - ~>
158
- - !ruby/object:Gem::Version
159
- version: 10.1.1
163
+ requirement: &id010 !ruby/object:Gem::Requirement
164
+ none: false
165
+ requirements:
166
+ - - ">="
167
+ - !ruby/object:Gem::Version
168
+ hash: 3
169
+ segments:
170
+ - 0
171
+ version: "0"
160
172
  type: :development
173
+ version_requirements: *id010
174
+ - !ruby/object:Gem::Dependency
175
+ name: rake
161
176
  prerelease: false
162
- version_requirements: !ruby/object:Gem::Requirement
163
- requirements:
177
+ requirement: &id011 !ruby/object:Gem::Requirement
178
+ none: false
179
+ requirements:
164
180
  - - ~>
165
- - !ruby/object:Gem::Version
181
+ - !ruby/object:Gem::Version
182
+ hash: 73
183
+ segments:
184
+ - 10
185
+ - 1
186
+ - 1
166
187
  version: 10.1.1
167
- - !ruby/object:Gem::Dependency
168
- name: rr
169
- requirement: !ruby/object:Gem::Requirement
170
- requirements:
171
- - - ! '>='
172
- - !ruby/object:Gem::Version
173
- version: '0'
174
188
  type: :development
189
+ version_requirements: *id011
190
+ - !ruby/object:Gem::Dependency
191
+ name: rr
175
192
  prerelease: false
176
- version_requirements: !ruby/object:Gem::Requirement
177
- requirements:
178
- - - ! '>='
179
- - !ruby/object:Gem::Version
180
- version: '0'
181
- - !ruby/object:Gem::Dependency
182
- name: rspec
183
- requirement: !ruby/object:Gem::Requirement
184
- requirements:
185
- - - ! '>='
186
- - !ruby/object:Gem::Version
187
- version: '0'
193
+ requirement: &id012 !ruby/object:Gem::Requirement
194
+ none: false
195
+ requirements:
196
+ - - ">="
197
+ - !ruby/object:Gem::Version
198
+ hash: 3
199
+ segments:
200
+ - 0
201
+ version: "0"
188
202
  type: :development
203
+ version_requirements: *id012
204
+ - !ruby/object:Gem::Dependency
205
+ name: rspec
189
206
  prerelease: false
190
- version_requirements: !ruby/object:Gem::Requirement
191
- requirements:
192
- - - ! '>='
193
- - !ruby/object:Gem::Version
194
- version: '0'
207
+ requirement: &id013 !ruby/object:Gem::Requirement
208
+ none: false
209
+ requirements:
210
+ - - ">="
211
+ - !ruby/object:Gem::Version
212
+ hash: 3
213
+ segments:
214
+ - 0
215
+ version: "0"
216
+ type: :development
217
+ version_requirements: *id013
195
218
  description: Client library and command-line tool to interact with the Mortar service.
196
219
  email: support@mortardata.com
197
- executables:
220
+ executables:
198
221
  - mortar
199
222
  extensions: []
223
+
200
224
  extra_rdoc_files: []
201
- files:
225
+
226
+ files:
202
227
  - README.md
203
228
  - bin/mortar
204
229
  - css/illustrate.css
@@ -247,6 +272,7 @@ files:
247
272
  - lib/mortar/local/jython.rb
248
273
  - lib/mortar/local/pig.rb
249
274
  - lib/mortar/local/python.rb
275
+ - lib/mortar/local/sqoop.rb
250
276
  - lib/mortar/pigversion.rb
251
277
  - lib/mortar/plugin.rb
252
278
  - lib/mortar/project.rb
@@ -293,6 +319,7 @@ files:
293
319
  - lib/mortar/templates/report/illustrate-report.html
294
320
  - lib/mortar/templates/script/runpig.sh
295
321
  - lib/mortar/templates/script/runpython.sh
322
+ - lib/mortar/templates/script/sqoop.sh
296
323
  - lib/mortar/templates/udf/python_udf.py
297
324
  - lib/mortar/updater.rb
298
325
  - lib/mortar/version.rb
@@ -323,6 +350,7 @@ files:
323
350
  - spec/mortar/local/jython_spec.rb
324
351
  - spec/mortar/local/pig_spec.rb
325
352
  - spec/mortar/local/python_spec.rb
353
+ - spec/mortar/local/sqoop_spec.rb
326
354
  - spec/mortar/plugin_spec.rb
327
355
  - spec/mortar/project_spec.rb
328
356
  - spec/mortar/s3_spec.rb
@@ -333,25 +361,38 @@ files:
333
361
  - spec/support/display_message_matcher.rb
334
362
  homepage: http://mortardata.com/
335
363
  licenses: []
336
- metadata: {}
364
+
337
365
  post_install_message:
338
366
  rdoc_options: []
339
- require_paths:
367
+
368
+ require_paths:
340
369
  - lib
341
- required_ruby_version: !ruby/object:Gem::Requirement
342
- requirements:
343
- - - ! '>='
344
- - !ruby/object:Gem::Version
370
+ required_ruby_version: !ruby/object:Gem::Requirement
371
+ none: false
372
+ requirements:
373
+ - - ">="
374
+ - !ruby/object:Gem::Version
375
+ hash: 57
376
+ segments:
377
+ - 1
378
+ - 8
379
+ - 7
345
380
  version: 1.8.7
346
- required_rubygems_version: !ruby/object:Gem::Requirement
347
- requirements:
348
- - - ! '>='
349
- - !ruby/object:Gem::Version
350
- version: '0'
381
+ required_rubygems_version: !ruby/object:Gem::Requirement
382
+ none: false
383
+ requirements:
384
+ - - ">="
385
+ - !ruby/object:Gem::Version
386
+ hash: 3
387
+ segments:
388
+ - 0
389
+ version: "0"
351
390
  requirements: []
391
+
352
392
  rubyforge_project:
353
- rubygems_version: 2.2.2
393
+ rubygems_version: 1.8.24
354
394
  signing_key:
355
- specification_version: 4
395
+ specification_version: 3
356
396
  summary: Client library and CLI to interact with the Mortar service.
357
397
  test_files: []
398
+
checksums.yaml DELETED
@@ -1,15 +0,0 @@
1
- ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- YjQyM2RmYTBiYjJjNjI3NzA2NDkyMWM0ZmNjZmFkZTgwMGQxNDRkZQ==
5
- data.tar.gz: !binary |-
6
- YjY1ZDAzMjlmZjEzNTQyZDcwNjQ3ZjAyYmI0MDNlY2ZjNmQxMmY5MQ==
7
- SHA512:
8
- metadata.gz: !binary |-
9
- ODc3YWI0NDkwOGUyOTRlYjI0MzQ3MzgyYmY1MjA4MzBkNjEyZTM1YmUwZmM0
10
- MTFjOTNlMmJhOGYxNTllMDhmODQ5MmE4N2ZjM2RjNDY2ZDY4ZTU5MGYxNjYz
11
- ZWNjNzYyMTQ5NDc2ODRkM2QyMDE3MjAwOGNjYjg3MzhkYzQyYzA=
12
- data.tar.gz: !binary |-
13
- ODU5MzE3YTRhYmZmMzEzNWVlMjdlZGViNmY3Mzk4MTJiMjE1MTBhNjNiYjUy
14
- ZDUwNjM0NTM1OGY1MzgzM2QwODRlYWRhMmFhMWE4NjQxYTQ0Y2RiMTJjN2Fi
15
- Zjg0MjQzNjNkZmE5Mjk1ODdjZDE3ZjJjNzYzMDMyOWUxNWY4YzY=