mortar 0.15.36 → 0.15.37
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/mortar/command/base.rb +2 -2
- data/lib/mortar/command/config.rb +52 -11
- data/lib/mortar/command/jobs.rb +1 -1
- data/lib/mortar/command/local.rb +1 -136
- data/lib/mortar/command/spark.rb +26 -57
- data/lib/mortar/local/controller.rb +2 -31
- data/lib/mortar/local/pig.rb +1 -1
- data/lib/mortar/pigversion.rb +52 -43
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/base_spec.rb +2 -2
- data/spec/mortar/command/config_spec.rb +63 -0
- data/spec/mortar/command/describe_spec.rb +4 -4
- data/spec/mortar/command/illustrate_spec.rb +7 -7
- data/spec/mortar/command/jobs_spec.rb +61 -13
- data/spec/mortar/command/local_spec.rb +9 -234
- data/spec/mortar/command/validate_spec.rb +2 -2
- data/spec/mortar/local/controller_spec.rb +7 -41
- data/spec/mortar/local/pig_spec.rb +29 -30
- metadata +5 -8
- data/lib/mortar/local/sqoop.rb +0 -162
- data/lib/mortar/templates/script/sqoop.sh +0 -40
- data/spec/mortar/local/sqoop_spec.rb +0 -57
@@ -48,7 +48,7 @@ STDERR
|
|
48
48
|
mock(u).config_parameters.returns([])
|
49
49
|
end
|
50
50
|
any_instance_of(Mortar::Local::Controller) do |u|
|
51
|
-
mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::
|
51
|
+
mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::Pig012), [], false, false).returns(nil)
|
52
52
|
end
|
53
53
|
stderr, stdout = execute("local:illustrate #{script_name} some_alias", p)
|
54
54
|
stderr.should == ""
|
@@ -125,9 +125,9 @@ STDERR
|
|
125
125
|
mock(u).config_parameters.returns([])
|
126
126
|
end
|
127
127
|
any_instance_of(Mortar::Local::Controller) do |u|
|
128
|
-
mock(u).run(pigscript, is_a(Mortar::PigVersion::
|
128
|
+
mock(u).run(pigscript, is_a(Mortar::PigVersion::Pig012Hadoop2), []).returns(nil)
|
129
129
|
end
|
130
|
-
stderr, stdout = execute("local:run pigscripts/#{script_name}.pig -g 0.
|
130
|
+
stderr, stdout = execute("local:run pigscripts/#{script_name}.pig -g 0.12-Hadoop-2", p)
|
131
131
|
stderr.should == ""
|
132
132
|
end
|
133
133
|
end
|
@@ -218,9 +218,6 @@ PARAMS
|
|
218
218
|
any_instance_of(Mortar::Local::Jython) do |j|
|
219
219
|
mock(j).install_or_update.returns(true)
|
220
220
|
end
|
221
|
-
any_instance_of(Mortar::Local::Sqoop) do |j|
|
222
|
-
mock(j).install_or_update.returns(true)
|
223
|
-
end
|
224
221
|
any_instance_of(Mortar::Local::Controller) do |j|
|
225
222
|
mock(j).write_local_readme
|
226
223
|
mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
|
@@ -247,10 +244,10 @@ PARAMS
|
|
247
244
|
mock(u).config_parameters.returns([{"key"=>"k", "value"=>"v"}])
|
248
245
|
end
|
249
246
|
any_instance_of(Mortar::Local::Controller) do |u|
|
250
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
247
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012), 'validate')
|
251
248
|
end
|
252
249
|
any_instance_of(Mortar::Local::Pig) do |u|
|
253
|
-
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::
|
250
|
+
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::Pig012), [{"key"=>"k", "value"=>"v"}])
|
254
251
|
end
|
255
252
|
stderr, stdout = execute("local:validate #{script_name}", p)
|
256
253
|
stderr.should == ""
|
@@ -268,10 +265,10 @@ PARAMS
|
|
268
265
|
mock(u).config_parameters.returns([])
|
269
266
|
end
|
270
267
|
any_instance_of(Mortar::Local::Controller) do |u|
|
271
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
268
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012), 'validate')
|
272
269
|
end
|
273
270
|
any_instance_of(Mortar::Local::Pig) do |u|
|
274
|
-
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::
|
271
|
+
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::Pig012), [])
|
275
272
|
end
|
276
273
|
stderr, stdout = execute("local:validate pigscripts/#{script_name}.pig", p)
|
277
274
|
stderr.should == ""
|
@@ -319,7 +316,7 @@ STDERR
|
|
319
316
|
mock(u).run_stillson_luigi_client_cfg_expansion(luigi_script, config_parameters).returns(true)
|
320
317
|
end
|
321
318
|
any_instance_of(Mortar::Local::Controller) do |u|
|
322
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
319
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012),'luigi')
|
323
320
|
end
|
324
321
|
any_instance_of(Mortar::Command::Local) do |u|
|
325
322
|
mock(u).sync_code_with_cloud().returns("some-git-ref")
|
@@ -350,7 +347,7 @@ STDERR
|
|
350
347
|
mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
|
351
348
|
end
|
352
349
|
any_instance_of(Mortar::Local::Controller) do |u|
|
353
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
350
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012),'luigi')
|
354
351
|
end
|
355
352
|
any_instance_of(Mortar::Command::Local) do |u|
|
356
353
|
mock(u).sync_code_with_cloud().returns("some-git-ref")
|
@@ -364,227 +361,5 @@ STDERR
|
|
364
361
|
|
365
362
|
end
|
366
363
|
|
367
|
-
context "local:sqoop_table" do
|
368
|
-
it "requires a db type" do
|
369
|
-
stderr, stdout = execute "local:sqoop_table"
|
370
|
-
stderr.should == <<-STDERR
|
371
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
372
|
-
! Must specify database type.
|
373
|
-
STDERR
|
374
|
-
end
|
375
|
-
|
376
|
-
it "requires the physical db name" do
|
377
|
-
stderr, stdout = execute "local:sqoop_table mysql"
|
378
|
-
stderr.should == <<-STDERR
|
379
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
380
|
-
! Must specify database name.
|
381
|
-
STDERR
|
382
|
-
end
|
383
|
-
|
384
|
-
it "requires the table name" do
|
385
|
-
stderr, stdout = execute "local:sqoop_table mysql myappdb"
|
386
|
-
stderr.should == <<-STDERR
|
387
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
388
|
-
! Must specify database table.
|
389
|
-
STDERR
|
390
|
-
end
|
391
|
-
|
392
|
-
it "requires the s3 destination" do
|
393
|
-
stderr, stdout = execute "local:sqoop_table mysql myappdb customers"
|
394
|
-
stderr.should == <<-STDERR
|
395
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
396
|
-
! Must specify s3 destination.
|
397
|
-
STDERR
|
398
|
-
end
|
399
|
-
|
400
|
-
it "sends everything to the controller" do
|
401
|
-
connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
|
402
|
-
dbtable = "customers"
|
403
|
-
s3dest = "s3n://a-bucket/a-directory"
|
404
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
405
|
-
mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {:host => "foobar.com"})
|
406
|
-
end
|
407
|
-
stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} --host foobar.com"
|
408
|
-
end
|
409
|
-
|
410
|
-
it "defaults to 'localhost' if no host specified" do
|
411
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
|
412
|
-
dbtable = "customers"
|
413
|
-
s3dest = "s3n://a-bucket/a-directory"
|
414
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
415
|
-
mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {})
|
416
|
-
end
|
417
|
-
stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest}"
|
418
|
-
end
|
419
|
-
|
420
|
-
it "overrides the JDBC connection string if one is specified" do
|
421
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
|
422
|
-
dbtable = "customers"
|
423
|
-
s3dest = "s3n://a-bucket/a-directory"
|
424
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
425
|
-
mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {})
|
426
|
-
end
|
427
|
-
stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
|
428
|
-
end
|
429
|
-
|
430
|
-
end
|
431
|
-
|
432
|
-
context "local:sqoop_query" do
|
433
|
-
it "requires a db type" do
|
434
|
-
stderr, stdout = execute "local:sqoop_query"
|
435
|
-
stderr.should == <<-STDERR
|
436
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
437
|
-
! Must specify database type.
|
438
|
-
STDERR
|
439
|
-
end
|
440
|
-
|
441
|
-
it "requires the physical db name" do
|
442
|
-
stderr, stdout = execute "local:sqoop_query mysql"
|
443
|
-
stderr.should == <<-STDERR
|
444
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
445
|
-
! Must specify database name.
|
446
|
-
STDERR
|
447
|
-
end
|
448
|
-
|
449
|
-
it "requires the table name" do
|
450
|
-
stderr, stdout = execute "local:sqoop_query mysql myappdb"
|
451
|
-
stderr.should == <<-STDERR
|
452
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
453
|
-
! Must specify sql query.
|
454
|
-
STDERR
|
455
|
-
end
|
456
|
-
|
457
|
-
it "requires the s3 destination" do
|
458
|
-
stderr, stdout = execute "local:sqoop_query mysql myappdb customers"
|
459
|
-
stderr.should == <<-STDERR
|
460
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
461
|
-
! Must specify s3 destination.
|
462
|
-
STDERR
|
463
|
-
end
|
464
|
-
|
465
|
-
it "sends everything to the controller" do
|
466
|
-
connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
|
467
|
-
query = "select_*_from_customers"
|
468
|
-
s3dest = "s3n://a-bucket/a-directory"
|
469
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
470
|
-
mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {:host => "foobar.com"})
|
471
|
-
end
|
472
|
-
stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} --host foobar.com"
|
473
|
-
stderr.should == ''
|
474
|
-
end
|
475
|
-
|
476
|
-
it "defaults to 'localhost' if no host specified" do
|
477
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
|
478
|
-
query = "select_*_from_customers"
|
479
|
-
s3dest = "s3n://a-bucket/a-directory"
|
480
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
481
|
-
mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {})
|
482
|
-
end
|
483
|
-
stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest}"
|
484
|
-
stderr.should == ''
|
485
|
-
end
|
486
|
-
|
487
|
-
it "overrides the JDBC connection string if one is specified" do
|
488
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
|
489
|
-
query = "select_*_from_customers"
|
490
|
-
s3dest = "s3n://a-bucket/a-directory"
|
491
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
492
|
-
mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {})
|
493
|
-
end
|
494
|
-
stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
|
495
|
-
stderr.should == ''
|
496
|
-
end
|
497
|
-
|
498
|
-
end
|
499
|
-
|
500
|
-
context "local:sqoop_incremental" do
|
501
|
-
it "requires a db type" do
|
502
|
-
stderr, stdout = execute "local:sqoop_incremental"
|
503
|
-
stderr.should == <<-STDERR
|
504
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
505
|
-
! Must specify database type.
|
506
|
-
STDERR
|
507
|
-
end
|
508
|
-
|
509
|
-
it "requires the physical db name" do
|
510
|
-
stderr, stdout = execute "local:sqoop_incremental mysql"
|
511
|
-
stderr.should == <<-STDERR
|
512
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
513
|
-
! Must specify database name.
|
514
|
-
STDERR
|
515
|
-
end
|
516
|
-
|
517
|
-
it "requires the table name" do
|
518
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb"
|
519
|
-
stderr.should == <<-STDERR
|
520
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
521
|
-
! Must specify database table.
|
522
|
-
STDERR
|
523
|
-
end
|
524
|
-
|
525
|
-
it "requires the column name" do
|
526
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable"
|
527
|
-
stderr.should == <<-STDERR
|
528
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
529
|
-
! Must specify column.
|
530
|
-
STDERR
|
531
|
-
end
|
532
|
-
|
533
|
-
it "requires the column value" do
|
534
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn"
|
535
|
-
stderr.should == <<-STDERR
|
536
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
537
|
-
! Must specify value.
|
538
|
-
STDERR
|
539
|
-
end
|
540
|
-
|
541
|
-
it "requires the s3 destination" do
|
542
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn customers"
|
543
|
-
stderr.should == <<-STDERR
|
544
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
545
|
-
! Must specify s3 destination.
|
546
|
-
STDERR
|
547
|
-
end
|
548
|
-
|
549
|
-
it "sends everything to the controller" do
|
550
|
-
connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
|
551
|
-
dbtable = "customers"
|
552
|
-
column = "customer_id"
|
553
|
-
column_value = "12345"
|
554
|
-
s3dest = "s3n://a-bucket/a-directory"
|
555
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
556
|
-
mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {:host => "foobar.com"})
|
557
|
-
end
|
558
|
-
stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} --host foobar.com"
|
559
|
-
end
|
560
|
-
|
561
|
-
it "defaults to 'localhost' if no host specified" do
|
562
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
|
563
|
-
dbtable = "customers"
|
564
|
-
column = "customer_id"
|
565
|
-
column_value = "12345"
|
566
|
-
s3dest = "s3n://a-bucket/a-directory"
|
567
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
568
|
-
mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {})
|
569
|
-
end
|
570
|
-
stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest}"
|
571
|
-
end
|
572
|
-
|
573
|
-
it "overrides the JDBC connection string if one is specified" do
|
574
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
|
575
|
-
dbtable = "customers"
|
576
|
-
column = "customer_id"
|
577
|
-
column_value = "12345"
|
578
|
-
s3dest = "s3n://a-bucket/a-directory"
|
579
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
580
|
-
mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {})
|
581
|
-
end
|
582
|
-
stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
|
583
|
-
end
|
584
|
-
|
585
|
-
end
|
586
|
-
|
587
|
-
|
588
|
-
|
589
364
|
end
|
590
365
|
end
|
@@ -73,7 +73,7 @@ STDERR
|
|
73
73
|
validate_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
74
74
|
parameters = ["name"=>"key", "value"=>"value" ]
|
75
75
|
|
76
|
-
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.
|
76
|
+
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"validate_id" => validate_id})}
|
77
77
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
78
78
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
79
79
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_PROGRESS, "status_description" => "Starting"})).ordered
|
@@ -103,7 +103,7 @@ STDOUT
|
|
103
103
|
column_number = 32
|
104
104
|
error_type = 'PigError'
|
105
105
|
|
106
|
-
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.
|
106
|
+
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"validate_id" => validate_id})}
|
107
107
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
108
108
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_FAILURE, "status_description" => "Failed",
|
109
109
|
"error_message" => error_message,
|
@@ -135,7 +135,7 @@ module Mortar::Local
|
|
135
135
|
mock(j).check_install.returns(true)
|
136
136
|
end
|
137
137
|
any_instance_of(Mortar::Local::Pig) do |p|
|
138
|
-
mock(p).install_or_update(is_a(Mortar::PigVersion::
|
138
|
+
mock(p).install_or_update(is_a(Mortar::PigVersion::Pig012), nil)
|
139
139
|
end
|
140
140
|
any_instance_of(Mortar::Local::Python) do |p|
|
141
141
|
mock(p).check_or_install.returns(true)
|
@@ -152,53 +152,19 @@ module Mortar::Local
|
|
152
152
|
ENV['MORTAR_PROJECT_NAME'].should eq('myproject')
|
153
153
|
end
|
154
154
|
|
155
|
-
it "install sqoop with command" do
|
156
|
-
ENV.delete('MORTAR_PROJECT_NAME')
|
157
|
-
command = 'command'
|
158
|
-
ctrl = Mortar::Local::Controller.new
|
159
|
-
|
160
|
-
any_instance_of(Mortar::Command::Base) do |b|
|
161
|
-
mock(b).project.returns(Mortar::Project::Project)
|
162
|
-
mock(b).options.returns({:project =>'myproject'})
|
163
|
-
end
|
164
|
-
any_instance_of(Mortar::Local::Java) do |j|
|
165
|
-
mock(j).check_install.returns(true)
|
166
|
-
end
|
167
|
-
any_instance_of(Mortar::Local::Pig) do |p|
|
168
|
-
mock(p).install_or_update(is_a(Mortar::PigVersion::Pig012), command)
|
169
|
-
end
|
170
|
-
any_instance_of(Mortar::Local::Python) do |p|
|
171
|
-
mock(p).check_or_install.returns(true)
|
172
|
-
mock(p).check_virtualenv.returns(true)
|
173
|
-
mock(p).setup_project_python_environment.returns(true)
|
174
|
-
end
|
175
|
-
any_instance_of(Mortar::Local::Jython) do |j|
|
176
|
-
mock(j).install_or_update
|
177
|
-
end
|
178
|
-
any_instance_of(Mortar::Local::Sqoop) do |s|
|
179
|
-
mock(s).install_or_update
|
180
|
-
end
|
181
|
-
|
182
|
-
mock(ctrl).write_local_readme
|
183
|
-
mock(ctrl).ensure_local_install_dirs_in_gitignore
|
184
|
-
ctrl.install_and_configure(Mortar::PigVersion::Pig012.new, command, true)
|
185
|
-
begin
|
186
|
-
ENV['MORTAR_PROJECT_NAME'].should eq('myproject')
|
187
|
-
end
|
188
|
-
end
|
189
155
|
end
|
190
156
|
|
191
157
|
context("run") do
|
192
158
|
|
193
159
|
it "checks for aws keys, checks depenendency installation, runs script" do
|
194
160
|
c = Mortar::Local::Controller.new
|
195
|
-
mock(c).install_and_configure("0.
|
161
|
+
mock(c).install_and_configure("0.12", "run")
|
196
162
|
test_script = "foobar-script"
|
197
163
|
the_parameters = []
|
198
164
|
any_instance_of(Mortar::Local::Pig) do |p|
|
199
|
-
mock(p).run_script(test_script, "0.
|
165
|
+
mock(p).run_script(test_script, "0.12", the_parameters)
|
200
166
|
end
|
201
|
-
c.run(test_script, "0.
|
167
|
+
c.run(test_script, "0.12", the_parameters)
|
202
168
|
end
|
203
169
|
|
204
170
|
end
|
@@ -206,16 +172,16 @@ module Mortar::Local
|
|
206
172
|
context("illustrate") do
|
207
173
|
it "checks for aws keys, checks depenendency installation, runs the illustrate process" do
|
208
174
|
c = Mortar::Local::Controller.new
|
209
|
-
mock(c).install_and_configure("0.
|
175
|
+
mock(c).install_and_configure("0.12", "illustrate")
|
210
176
|
test_script = "foobar-script"
|
211
177
|
script_alias = "some_alias"
|
212
178
|
prune = false
|
213
179
|
no_browser = false
|
214
180
|
the_parameters = []
|
215
181
|
any_instance_of(Mortar::Local::Pig) do |p|
|
216
|
-
mock(p).illustrate_alias(test_script, script_alias, prune, no_browser, "0.
|
182
|
+
mock(p).illustrate_alias(test_script, script_alias, prune, no_browser, "0.12", the_parameters)
|
217
183
|
end
|
218
|
-
c.illustrate(test_script, script_alias, "0.
|
184
|
+
c.illustrate(test_script, script_alias, "0.12", the_parameters, prune, no_browser)
|
219
185
|
end
|
220
186
|
end
|
221
187
|
|
@@ -35,19 +35,19 @@ module Mortar::Local
|
|
35
35
|
# chmods bin/pig, removes tgz, and notes the installation
|
36
36
|
FakeFS do
|
37
37
|
pig = Mortar::Local::Pig.new
|
38
|
-
|
39
|
-
local_pig_archive = File.join(pig.local_install_directory,
|
40
|
-
mock(pig).download_file(pig.pig_archive_url(
|
38
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
39
|
+
local_pig_archive = File.join(pig.local_install_directory, pig012.tgz_name)
|
40
|
+
mock(pig).download_file(pig.pig_archive_url(pig012), local_pig_archive, nil) do
|
41
41
|
# Simulate the tgz file being downloaded, this should be deleted
|
42
42
|
# before the method finishes executing
|
43
43
|
FileUtils.touch(local_pig_archive)
|
44
44
|
end
|
45
45
|
mock(pig).extract_tgz(local_pig_archive, pig.local_install_directory)
|
46
|
-
mock(pig).note_install(
|
47
|
-
mock(pig).command(
|
46
|
+
mock(pig).note_install(pig012.name)
|
47
|
+
mock(pig).command(pig012)
|
48
48
|
begin
|
49
49
|
previous_stdout, $stdout = $stdout, StringIO.new
|
50
|
-
pig.install_pig(
|
50
|
+
pig.install_pig(pig012)
|
51
51
|
ensure
|
52
52
|
$stdout = previous_stdout
|
53
53
|
end
|
@@ -61,17 +61,17 @@ module Mortar::Local
|
|
61
61
|
|
62
62
|
it "does nothing if existing install and no update available" do
|
63
63
|
pig = Mortar::Local::Pig.new
|
64
|
-
|
64
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
65
65
|
|
66
|
-
mock(pig).should_do_pig_install?(
|
67
|
-
mock(pig).should_do_pig_update?(
|
66
|
+
mock(pig).should_do_pig_install?(pig012).returns(false)
|
67
|
+
mock(pig).should_do_pig_update?(pig012, nil).returns(false)
|
68
68
|
mock(pig).should_do_lib_install?.returns(false)
|
69
69
|
mock(pig).should_do_lib_update?.returns(false)
|
70
70
|
FakeFS do
|
71
71
|
FileUtils.mkdir_p(File.dirname(pig.local_install_directory))
|
72
72
|
FileUtils.rm_rf(pig.local_install_directory, :force => true)
|
73
|
-
pig.install_or_update(
|
74
|
-
expect(File.exists?(pig.pig_directory(
|
73
|
+
pig.install_or_update(pig012)
|
74
|
+
expect(File.exists?(pig.pig_directory(pig012))).to be_false
|
75
75
|
expect(File.exists?(pig.lib_directory)).to be_false
|
76
76
|
|
77
77
|
end
|
@@ -79,30 +79,30 @@ module Mortar::Local
|
|
79
79
|
|
80
80
|
it "does install if none has been done before" do
|
81
81
|
pig = Mortar::Local::Pig.new
|
82
|
-
|
82
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
83
83
|
|
84
|
-
mock(pig).should_do_pig_install?(
|
84
|
+
mock(pig).should_do_pig_install?(pig012).returns(true)
|
85
85
|
mock(pig).should_do_lib_install?.returns(true)
|
86
86
|
|
87
|
-
mock(pig).install_pig(
|
87
|
+
mock(pig).install_pig(pig012, nil)
|
88
88
|
mock(pig).install_lib
|
89
89
|
capture_stdout do
|
90
|
-
pig.install_or_update(
|
90
|
+
pig.install_or_update(pig012)
|
91
91
|
end
|
92
92
|
end
|
93
93
|
|
94
94
|
it "does install if one was done before but there is an update" do
|
95
95
|
pig = Mortar::Local::Pig.new
|
96
|
-
|
96
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
97
97
|
|
98
|
-
mock(pig).should_do_pig_install?(
|
99
|
-
mock(pig).should_do_pig_update?(
|
98
|
+
mock(pig).should_do_pig_install?(pig012).returns(false)
|
99
|
+
mock(pig).should_do_pig_update?(pig012, nil).returns(true)
|
100
100
|
mock(pig).should_do_lib_install?.returns(false)
|
101
101
|
mock(pig).should_do_lib_update?.returns(true)
|
102
|
-
mock(pig).install_pig(
|
102
|
+
mock(pig).install_pig(pig012)
|
103
103
|
mock(pig).install_lib
|
104
104
|
capture_stdout do
|
105
|
-
pig.install_or_update(
|
105
|
+
pig.install_or_update(pig012)
|
106
106
|
end
|
107
107
|
end
|
108
108
|
|
@@ -208,7 +208,7 @@ module Mortar::Local
|
|
208
208
|
mock(pig).run_pig_command.with_any_args.returns(true)
|
209
209
|
mock(pig).show_illustrate_output_browser.with_any_args
|
210
210
|
stub(pig).make_pig_param_file.returns('param.file')
|
211
|
-
pig.illustrate_alias(script, 'my_alias', false, false, "0.
|
211
|
+
pig.illustrate_alias(script, 'my_alias', false, false, "0.12", [])
|
212
212
|
end
|
213
213
|
|
214
214
|
it "displays text results if illustrate was successful with no_browser" do
|
@@ -218,7 +218,7 @@ module Mortar::Local
|
|
218
218
|
stub(pig).run_pig_command.with_any_args.returns(true)
|
219
219
|
mock(pig).display.with_any_args
|
220
220
|
stub(pig).make_pig_param_file.returns('param.file')
|
221
|
-
pig.illustrate_alias(script, 'my_alias', false, true, "0.
|
221
|
+
pig.illustrate_alias(script, 'my_alias', false, true, "0.12", [])
|
222
222
|
end
|
223
223
|
|
224
224
|
it "skips results if illustrate was unsuccessful" do
|
@@ -228,7 +228,7 @@ module Mortar::Local
|
|
228
228
|
mock(pig).run_pig_command.with_any_args.returns(false)
|
229
229
|
mock(pig).show_illustrate_output_browser.with_any_args.never
|
230
230
|
stub(pig).make_pig_param_file.returns('param.file')
|
231
|
-
pig.illustrate_alias(script, 'my_alias', false, false, "0.
|
231
|
+
pig.illustrate_alias(script, 'my_alias', false, false, "0.12", [])
|
232
232
|
end
|
233
233
|
|
234
234
|
it "does not require login credentials for illustration" do
|
@@ -238,7 +238,7 @@ module Mortar::Local
|
|
238
238
|
mock(Mortar::Auth).user_s3_safe(true).returns('notloggedin-user-org')
|
239
239
|
mock(pig).run_pig_command.with_any_args.returns(true)
|
240
240
|
mock(pig).show_illustrate_output_browser.with_any_args
|
241
|
-
pig.illustrate_alias(script, 'my_alias', false, false, "0.
|
241
|
+
pig.illustrate_alias(script, 'my_alias', false, false, "0.12", [])
|
242
242
|
end
|
243
243
|
|
244
244
|
end
|
@@ -263,15 +263,14 @@ module Mortar::Local
|
|
263
263
|
it "Uses default pig" do
|
264
264
|
pig = Mortar::Local::Pig.new
|
265
265
|
tpc = pig.template_params_classpath
|
266
|
-
expect(tpc.include?("pig-0.
|
267
|
-
expect(tpc.include?("pig-0.12")).to be_false
|
266
|
+
expect(tpc.include?("pig-0.12")).to be_true
|
267
|
+
expect(tpc.include?("pig-0.12-Hadoop-2")).to be_false
|
268
268
|
end
|
269
269
|
|
270
|
-
it "Works with 0.12 pig" do
|
270
|
+
it "Works with 0.12-Hadoop-2 pig" do
|
271
271
|
pig = Mortar::Local::Pig.new
|
272
|
-
tpc = pig.template_params_classpath(Mortar::PigVersion::
|
273
|
-
expect(tpc.include?("pig-0.12")).to be_true
|
274
|
-
expect(tpc.include?("pig-0.9")).to be_false
|
272
|
+
tpc = pig.template_params_classpath(Mortar::PigVersion::Pig012Hadoop2.new)
|
273
|
+
expect(tpc.include?("pig-0.12-Hadoop-2")).to be_true
|
275
274
|
end
|
276
275
|
end
|
277
276
|
|