mortar 0.15.36 → 0.15.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/mortar/command/base.rb +2 -2
- data/lib/mortar/command/config.rb +52 -11
- data/lib/mortar/command/jobs.rb +1 -1
- data/lib/mortar/command/local.rb +1 -136
- data/lib/mortar/command/spark.rb +26 -57
- data/lib/mortar/local/controller.rb +2 -31
- data/lib/mortar/local/pig.rb +1 -1
- data/lib/mortar/pigversion.rb +52 -43
- data/lib/mortar/version.rb +1 -1
- data/spec/mortar/command/base_spec.rb +2 -2
- data/spec/mortar/command/config_spec.rb +63 -0
- data/spec/mortar/command/describe_spec.rb +4 -4
- data/spec/mortar/command/illustrate_spec.rb +7 -7
- data/spec/mortar/command/jobs_spec.rb +61 -13
- data/spec/mortar/command/local_spec.rb +9 -234
- data/spec/mortar/command/validate_spec.rb +2 -2
- data/spec/mortar/local/controller_spec.rb +7 -41
- data/spec/mortar/local/pig_spec.rb +29 -30
- metadata +5 -8
- data/lib/mortar/local/sqoop.rb +0 -162
- data/lib/mortar/templates/script/sqoop.sh +0 -40
- data/spec/mortar/local/sqoop_spec.rb +0 -57
@@ -48,7 +48,7 @@ STDERR
|
|
48
48
|
mock(u).config_parameters.returns([])
|
49
49
|
end
|
50
50
|
any_instance_of(Mortar::Local::Controller) do |u|
|
51
|
-
mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::
|
51
|
+
mock(u).illustrate(pigscript, "some_alias", is_a(Mortar::PigVersion::Pig012), [], false, false).returns(nil)
|
52
52
|
end
|
53
53
|
stderr, stdout = execute("local:illustrate #{script_name} some_alias", p)
|
54
54
|
stderr.should == ""
|
@@ -125,9 +125,9 @@ STDERR
|
|
125
125
|
mock(u).config_parameters.returns([])
|
126
126
|
end
|
127
127
|
any_instance_of(Mortar::Local::Controller) do |u|
|
128
|
-
mock(u).run(pigscript, is_a(Mortar::PigVersion::
|
128
|
+
mock(u).run(pigscript, is_a(Mortar::PigVersion::Pig012Hadoop2), []).returns(nil)
|
129
129
|
end
|
130
|
-
stderr, stdout = execute("local:run pigscripts/#{script_name}.pig -g 0.
|
130
|
+
stderr, stdout = execute("local:run pigscripts/#{script_name}.pig -g 0.12-Hadoop-2", p)
|
131
131
|
stderr.should == ""
|
132
132
|
end
|
133
133
|
end
|
@@ -218,9 +218,6 @@ PARAMS
|
|
218
218
|
any_instance_of(Mortar::Local::Jython) do |j|
|
219
219
|
mock(j).install_or_update.returns(true)
|
220
220
|
end
|
221
|
-
any_instance_of(Mortar::Local::Sqoop) do |j|
|
222
|
-
mock(j).install_or_update.returns(true)
|
223
|
-
end
|
224
221
|
any_instance_of(Mortar::Local::Controller) do |j|
|
225
222
|
mock(j).write_local_readme
|
226
223
|
mock(j).ensure_local_install_dirs_in_gitignore.returns(true)
|
@@ -247,10 +244,10 @@ PARAMS
|
|
247
244
|
mock(u).config_parameters.returns([{"key"=>"k", "value"=>"v"}])
|
248
245
|
end
|
249
246
|
any_instance_of(Mortar::Local::Controller) do |u|
|
250
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
247
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012), 'validate')
|
251
248
|
end
|
252
249
|
any_instance_of(Mortar::Local::Pig) do |u|
|
253
|
-
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::
|
250
|
+
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::Pig012), [{"key"=>"k", "value"=>"v"}])
|
254
251
|
end
|
255
252
|
stderr, stdout = execute("local:validate #{script_name}", p)
|
256
253
|
stderr.should == ""
|
@@ -268,10 +265,10 @@ PARAMS
|
|
268
265
|
mock(u).config_parameters.returns([])
|
269
266
|
end
|
270
267
|
any_instance_of(Mortar::Local::Controller) do |u|
|
271
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
268
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012), 'validate')
|
272
269
|
end
|
273
270
|
any_instance_of(Mortar::Local::Pig) do |u|
|
274
|
-
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::
|
271
|
+
mock(u).run_pig_command(" -check #{pigscript.path}", is_a(Mortar::PigVersion::Pig012), [])
|
275
272
|
end
|
276
273
|
stderr, stdout = execute("local:validate pigscripts/#{script_name}.pig", p)
|
277
274
|
stderr.should == ""
|
@@ -319,7 +316,7 @@ STDERR
|
|
319
316
|
mock(u).run_stillson_luigi_client_cfg_expansion(luigi_script, config_parameters).returns(true)
|
320
317
|
end
|
321
318
|
any_instance_of(Mortar::Local::Controller) do |u|
|
322
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
319
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012),'luigi')
|
323
320
|
end
|
324
321
|
any_instance_of(Mortar::Command::Local) do |u|
|
325
322
|
mock(u).sync_code_with_cloud().returns("some-git-ref")
|
@@ -350,7 +347,7 @@ STDERR
|
|
350
347
|
mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
|
351
348
|
end
|
352
349
|
any_instance_of(Mortar::Local::Controller) do |u|
|
353
|
-
mock(u).install_and_configure(is_a(Mortar::PigVersion::
|
350
|
+
mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig012),'luigi')
|
354
351
|
end
|
355
352
|
any_instance_of(Mortar::Command::Local) do |u|
|
356
353
|
mock(u).sync_code_with_cloud().returns("some-git-ref")
|
@@ -364,227 +361,5 @@ STDERR
|
|
364
361
|
|
365
362
|
end
|
366
363
|
|
367
|
-
context "local:sqoop_table" do
|
368
|
-
it "requires a db type" do
|
369
|
-
stderr, stdout = execute "local:sqoop_table"
|
370
|
-
stderr.should == <<-STDERR
|
371
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
372
|
-
! Must specify database type.
|
373
|
-
STDERR
|
374
|
-
end
|
375
|
-
|
376
|
-
it "requires the physical db name" do
|
377
|
-
stderr, stdout = execute "local:sqoop_table mysql"
|
378
|
-
stderr.should == <<-STDERR
|
379
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
380
|
-
! Must specify database name.
|
381
|
-
STDERR
|
382
|
-
end
|
383
|
-
|
384
|
-
it "requires the table name" do
|
385
|
-
stderr, stdout = execute "local:sqoop_table mysql myappdb"
|
386
|
-
stderr.should == <<-STDERR
|
387
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
388
|
-
! Must specify database table.
|
389
|
-
STDERR
|
390
|
-
end
|
391
|
-
|
392
|
-
it "requires the s3 destination" do
|
393
|
-
stderr, stdout = execute "local:sqoop_table mysql myappdb customers"
|
394
|
-
stderr.should == <<-STDERR
|
395
|
-
! Usage: mortar local:sqoop_table dbtype database-name table s3-destination
|
396
|
-
! Must specify s3 destination.
|
397
|
-
STDERR
|
398
|
-
end
|
399
|
-
|
400
|
-
it "sends everything to the controller" do
|
401
|
-
connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
|
402
|
-
dbtable = "customers"
|
403
|
-
s3dest = "s3n://a-bucket/a-directory"
|
404
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
405
|
-
mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {:host => "foobar.com"})
|
406
|
-
end
|
407
|
-
stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} --host foobar.com"
|
408
|
-
end
|
409
|
-
|
410
|
-
it "defaults to 'localhost' if no host specified" do
|
411
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
|
412
|
-
dbtable = "customers"
|
413
|
-
s3dest = "s3n://a-bucket/a-directory"
|
414
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
415
|
-
mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {})
|
416
|
-
end
|
417
|
-
stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest}"
|
418
|
-
end
|
419
|
-
|
420
|
-
it "overrides the JDBC connection string if one is specified" do
|
421
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
|
422
|
-
dbtable = "customers"
|
423
|
-
s3dest = "s3n://a-bucket/a-directory"
|
424
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
425
|
-
mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {})
|
426
|
-
end
|
427
|
-
stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
|
428
|
-
end
|
429
|
-
|
430
|
-
end
|
431
|
-
|
432
|
-
context "local:sqoop_query" do
|
433
|
-
it "requires a db type" do
|
434
|
-
stderr, stdout = execute "local:sqoop_query"
|
435
|
-
stderr.should == <<-STDERR
|
436
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
437
|
-
! Must specify database type.
|
438
|
-
STDERR
|
439
|
-
end
|
440
|
-
|
441
|
-
it "requires the physical db name" do
|
442
|
-
stderr, stdout = execute "local:sqoop_query mysql"
|
443
|
-
stderr.should == <<-STDERR
|
444
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
445
|
-
! Must specify database name.
|
446
|
-
STDERR
|
447
|
-
end
|
448
|
-
|
449
|
-
it "requires the table name" do
|
450
|
-
stderr, stdout = execute "local:sqoop_query mysql myappdb"
|
451
|
-
stderr.should == <<-STDERR
|
452
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
453
|
-
! Must specify sql query.
|
454
|
-
STDERR
|
455
|
-
end
|
456
|
-
|
457
|
-
it "requires the s3 destination" do
|
458
|
-
stderr, stdout = execute "local:sqoop_query mysql myappdb customers"
|
459
|
-
stderr.should == <<-STDERR
|
460
|
-
! Usage: mortar local:sqoop_query dbtype database-name query s3-destination
|
461
|
-
! Must specify s3 destination.
|
462
|
-
STDERR
|
463
|
-
end
|
464
|
-
|
465
|
-
it "sends everything to the controller" do
|
466
|
-
connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
|
467
|
-
query = "select_*_from_customers"
|
468
|
-
s3dest = "s3n://a-bucket/a-directory"
|
469
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
470
|
-
mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {:host => "foobar.com"})
|
471
|
-
end
|
472
|
-
stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} --host foobar.com"
|
473
|
-
stderr.should == ''
|
474
|
-
end
|
475
|
-
|
476
|
-
it "defaults to 'localhost' if no host specified" do
|
477
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
|
478
|
-
query = "select_*_from_customers"
|
479
|
-
s3dest = "s3n://a-bucket/a-directory"
|
480
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
481
|
-
mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {})
|
482
|
-
end
|
483
|
-
stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest}"
|
484
|
-
stderr.should == ''
|
485
|
-
end
|
486
|
-
|
487
|
-
it "overrides the JDBC connection string if one is specified" do
|
488
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
|
489
|
-
query = "select_*_from_customers"
|
490
|
-
s3dest = "s3n://a-bucket/a-directory"
|
491
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
492
|
-
mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {})
|
493
|
-
end
|
494
|
-
stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
|
495
|
-
stderr.should == ''
|
496
|
-
end
|
497
|
-
|
498
|
-
end
|
499
|
-
|
500
|
-
context "local:sqoop_incremental" do
|
501
|
-
it "requires a db type" do
|
502
|
-
stderr, stdout = execute "local:sqoop_incremental"
|
503
|
-
stderr.should == <<-STDERR
|
504
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
505
|
-
! Must specify database type.
|
506
|
-
STDERR
|
507
|
-
end
|
508
|
-
|
509
|
-
it "requires the physical db name" do
|
510
|
-
stderr, stdout = execute "local:sqoop_incremental mysql"
|
511
|
-
stderr.should == <<-STDERR
|
512
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
513
|
-
! Must specify database name.
|
514
|
-
STDERR
|
515
|
-
end
|
516
|
-
|
517
|
-
it "requires the table name" do
|
518
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb"
|
519
|
-
stderr.should == <<-STDERR
|
520
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
521
|
-
! Must specify database table.
|
522
|
-
STDERR
|
523
|
-
end
|
524
|
-
|
525
|
-
it "requires the column name" do
|
526
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable"
|
527
|
-
stderr.should == <<-STDERR
|
528
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
529
|
-
! Must specify column.
|
530
|
-
STDERR
|
531
|
-
end
|
532
|
-
|
533
|
-
it "requires the column value" do
|
534
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn"
|
535
|
-
stderr.should == <<-STDERR
|
536
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
537
|
-
! Must specify value.
|
538
|
-
STDERR
|
539
|
-
end
|
540
|
-
|
541
|
-
it "requires the s3 destination" do
|
542
|
-
stderr, stdout = execute "local:sqoop_incremental mysql myappdb mytable mycolumn customers"
|
543
|
-
stderr.should == <<-STDERR
|
544
|
-
! Usage: mortar local:sqoop_incremental dbtype database-name table column value s3-destination
|
545
|
-
! Must specify s3 destination.
|
546
|
-
STDERR
|
547
|
-
end
|
548
|
-
|
549
|
-
it "sends everything to the controller" do
|
550
|
-
connstr = Shellwords.escape("jdbc:mysql://foobar.com/mydb?zeroDateTimeBehavior=convertToNull")
|
551
|
-
dbtable = "customers"
|
552
|
-
column = "customer_id"
|
553
|
-
column_value = "12345"
|
554
|
-
s3dest = "s3n://a-bucket/a-directory"
|
555
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
556
|
-
mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {:host => "foobar.com"})
|
557
|
-
end
|
558
|
-
stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} --host foobar.com"
|
559
|
-
end
|
560
|
-
|
561
|
-
it "defaults to 'localhost' if no host specified" do
|
562
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydb?zeroDateTimeBehavior=convertToNull")
|
563
|
-
dbtable = "customers"
|
564
|
-
column = "customer_id"
|
565
|
-
column_value = "12345"
|
566
|
-
s3dest = "s3n://a-bucket/a-directory"
|
567
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
568
|
-
mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {})
|
569
|
-
end
|
570
|
-
stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest}"
|
571
|
-
end
|
572
|
-
|
573
|
-
it "overrides the JDBC connection string if one is specified" do
|
574
|
-
connstr = Shellwords.escape("jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull")
|
575
|
-
dbtable = "customers"
|
576
|
-
column = "customer_id"
|
577
|
-
column_value = "12345"
|
578
|
-
s3dest = "s3n://a-bucket/a-directory"
|
579
|
-
any_instance_of(Mortar::Local::Controller) do |c|
|
580
|
-
mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {})
|
581
|
-
end
|
582
|
-
stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} -c jdbc:mysql://localhost/mydbother?zeroDateTimeBehavior=convertToNull"
|
583
|
-
end
|
584
|
-
|
585
|
-
end
|
586
|
-
|
587
|
-
|
588
|
-
|
589
364
|
end
|
590
365
|
end
|
@@ -73,7 +73,7 @@ STDERR
|
|
73
73
|
validate_id = "c571a8c7f76a4fd4a67c103d753e2dd5"
|
74
74
|
parameters = ["name"=>"key", "value"=>"value" ]
|
75
75
|
|
76
|
-
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.
|
76
|
+
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => parameters) {Excon::Response.new(:body => {"validate_id" => validate_id})}
|
77
77
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
78
78
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_GATEWAY_STARTING, "status_description" => "GATEWAY_STARTING"})).ordered
|
79
79
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_PROGRESS, "status_description" => "Starting"})).ordered
|
@@ -103,7 +103,7 @@ STDOUT
|
|
103
103
|
column_number = 32
|
104
104
|
error_type = 'PigError'
|
105
105
|
|
106
|
-
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.
|
106
|
+
mock(Mortar::Auth.api).post_validate("myproject", "my_script", is_a(String), :pig_version => "0.12", :project_script_path => be_a_kind_of(String), :parameters => []) {Excon::Response.new(:body => {"validate_id" => validate_id})}
|
107
107
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_QUEUED, "status_description" => "Pending"})).ordered
|
108
108
|
mock(Mortar::Auth.api).get_validate(validate_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Validate::STATUS_FAILURE, "status_description" => "Failed",
|
109
109
|
"error_message" => error_message,
|
@@ -135,7 +135,7 @@ module Mortar::Local
|
|
135
135
|
mock(j).check_install.returns(true)
|
136
136
|
end
|
137
137
|
any_instance_of(Mortar::Local::Pig) do |p|
|
138
|
-
mock(p).install_or_update(is_a(Mortar::PigVersion::
|
138
|
+
mock(p).install_or_update(is_a(Mortar::PigVersion::Pig012), nil)
|
139
139
|
end
|
140
140
|
any_instance_of(Mortar::Local::Python) do |p|
|
141
141
|
mock(p).check_or_install.returns(true)
|
@@ -152,53 +152,19 @@ module Mortar::Local
|
|
152
152
|
ENV['MORTAR_PROJECT_NAME'].should eq('myproject')
|
153
153
|
end
|
154
154
|
|
155
|
-
it "install sqoop with command" do
|
156
|
-
ENV.delete('MORTAR_PROJECT_NAME')
|
157
|
-
command = 'command'
|
158
|
-
ctrl = Mortar::Local::Controller.new
|
159
|
-
|
160
|
-
any_instance_of(Mortar::Command::Base) do |b|
|
161
|
-
mock(b).project.returns(Mortar::Project::Project)
|
162
|
-
mock(b).options.returns({:project =>'myproject'})
|
163
|
-
end
|
164
|
-
any_instance_of(Mortar::Local::Java) do |j|
|
165
|
-
mock(j).check_install.returns(true)
|
166
|
-
end
|
167
|
-
any_instance_of(Mortar::Local::Pig) do |p|
|
168
|
-
mock(p).install_or_update(is_a(Mortar::PigVersion::Pig012), command)
|
169
|
-
end
|
170
|
-
any_instance_of(Mortar::Local::Python) do |p|
|
171
|
-
mock(p).check_or_install.returns(true)
|
172
|
-
mock(p).check_virtualenv.returns(true)
|
173
|
-
mock(p).setup_project_python_environment.returns(true)
|
174
|
-
end
|
175
|
-
any_instance_of(Mortar::Local::Jython) do |j|
|
176
|
-
mock(j).install_or_update
|
177
|
-
end
|
178
|
-
any_instance_of(Mortar::Local::Sqoop) do |s|
|
179
|
-
mock(s).install_or_update
|
180
|
-
end
|
181
|
-
|
182
|
-
mock(ctrl).write_local_readme
|
183
|
-
mock(ctrl).ensure_local_install_dirs_in_gitignore
|
184
|
-
ctrl.install_and_configure(Mortar::PigVersion::Pig012.new, command, true)
|
185
|
-
begin
|
186
|
-
ENV['MORTAR_PROJECT_NAME'].should eq('myproject')
|
187
|
-
end
|
188
|
-
end
|
189
155
|
end
|
190
156
|
|
191
157
|
context("run") do
|
192
158
|
|
193
159
|
it "checks for aws keys, checks depenendency installation, runs script" do
|
194
160
|
c = Mortar::Local::Controller.new
|
195
|
-
mock(c).install_and_configure("0.
|
161
|
+
mock(c).install_and_configure("0.12", "run")
|
196
162
|
test_script = "foobar-script"
|
197
163
|
the_parameters = []
|
198
164
|
any_instance_of(Mortar::Local::Pig) do |p|
|
199
|
-
mock(p).run_script(test_script, "0.
|
165
|
+
mock(p).run_script(test_script, "0.12", the_parameters)
|
200
166
|
end
|
201
|
-
c.run(test_script, "0.
|
167
|
+
c.run(test_script, "0.12", the_parameters)
|
202
168
|
end
|
203
169
|
|
204
170
|
end
|
@@ -206,16 +172,16 @@ module Mortar::Local
|
|
206
172
|
context("illustrate") do
|
207
173
|
it "checks for aws keys, checks depenendency installation, runs the illustrate process" do
|
208
174
|
c = Mortar::Local::Controller.new
|
209
|
-
mock(c).install_and_configure("0.
|
175
|
+
mock(c).install_and_configure("0.12", "illustrate")
|
210
176
|
test_script = "foobar-script"
|
211
177
|
script_alias = "some_alias"
|
212
178
|
prune = false
|
213
179
|
no_browser = false
|
214
180
|
the_parameters = []
|
215
181
|
any_instance_of(Mortar::Local::Pig) do |p|
|
216
|
-
mock(p).illustrate_alias(test_script, script_alias, prune, no_browser, "0.
|
182
|
+
mock(p).illustrate_alias(test_script, script_alias, prune, no_browser, "0.12", the_parameters)
|
217
183
|
end
|
218
|
-
c.illustrate(test_script, script_alias, "0.
|
184
|
+
c.illustrate(test_script, script_alias, "0.12", the_parameters, prune, no_browser)
|
219
185
|
end
|
220
186
|
end
|
221
187
|
|
@@ -35,19 +35,19 @@ module Mortar::Local
|
|
35
35
|
# chmods bin/pig, removes tgz, and notes the installation
|
36
36
|
FakeFS do
|
37
37
|
pig = Mortar::Local::Pig.new
|
38
|
-
|
39
|
-
local_pig_archive = File.join(pig.local_install_directory,
|
40
|
-
mock(pig).download_file(pig.pig_archive_url(
|
38
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
39
|
+
local_pig_archive = File.join(pig.local_install_directory, pig012.tgz_name)
|
40
|
+
mock(pig).download_file(pig.pig_archive_url(pig012), local_pig_archive, nil) do
|
41
41
|
# Simulate the tgz file being downloaded, this should be deleted
|
42
42
|
# before the method finishes executing
|
43
43
|
FileUtils.touch(local_pig_archive)
|
44
44
|
end
|
45
45
|
mock(pig).extract_tgz(local_pig_archive, pig.local_install_directory)
|
46
|
-
mock(pig).note_install(
|
47
|
-
mock(pig).command(
|
46
|
+
mock(pig).note_install(pig012.name)
|
47
|
+
mock(pig).command(pig012)
|
48
48
|
begin
|
49
49
|
previous_stdout, $stdout = $stdout, StringIO.new
|
50
|
-
pig.install_pig(
|
50
|
+
pig.install_pig(pig012)
|
51
51
|
ensure
|
52
52
|
$stdout = previous_stdout
|
53
53
|
end
|
@@ -61,17 +61,17 @@ module Mortar::Local
|
|
61
61
|
|
62
62
|
it "does nothing if existing install and no update available" do
|
63
63
|
pig = Mortar::Local::Pig.new
|
64
|
-
|
64
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
65
65
|
|
66
|
-
mock(pig).should_do_pig_install?(
|
67
|
-
mock(pig).should_do_pig_update?(
|
66
|
+
mock(pig).should_do_pig_install?(pig012).returns(false)
|
67
|
+
mock(pig).should_do_pig_update?(pig012, nil).returns(false)
|
68
68
|
mock(pig).should_do_lib_install?.returns(false)
|
69
69
|
mock(pig).should_do_lib_update?.returns(false)
|
70
70
|
FakeFS do
|
71
71
|
FileUtils.mkdir_p(File.dirname(pig.local_install_directory))
|
72
72
|
FileUtils.rm_rf(pig.local_install_directory, :force => true)
|
73
|
-
pig.install_or_update(
|
74
|
-
expect(File.exists?(pig.pig_directory(
|
73
|
+
pig.install_or_update(pig012)
|
74
|
+
expect(File.exists?(pig.pig_directory(pig012))).to be_false
|
75
75
|
expect(File.exists?(pig.lib_directory)).to be_false
|
76
76
|
|
77
77
|
end
|
@@ -79,30 +79,30 @@ module Mortar::Local
|
|
79
79
|
|
80
80
|
it "does install if none has been done before" do
|
81
81
|
pig = Mortar::Local::Pig.new
|
82
|
-
|
82
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
83
83
|
|
84
|
-
mock(pig).should_do_pig_install?(
|
84
|
+
mock(pig).should_do_pig_install?(pig012).returns(true)
|
85
85
|
mock(pig).should_do_lib_install?.returns(true)
|
86
86
|
|
87
|
-
mock(pig).install_pig(
|
87
|
+
mock(pig).install_pig(pig012, nil)
|
88
88
|
mock(pig).install_lib
|
89
89
|
capture_stdout do
|
90
|
-
pig.install_or_update(
|
90
|
+
pig.install_or_update(pig012)
|
91
91
|
end
|
92
92
|
end
|
93
93
|
|
94
94
|
it "does install if one was done before but there is an update" do
|
95
95
|
pig = Mortar::Local::Pig.new
|
96
|
-
|
96
|
+
pig012 = Mortar::PigVersion::Pig012.new
|
97
97
|
|
98
|
-
mock(pig).should_do_pig_install?(
|
99
|
-
mock(pig).should_do_pig_update?(
|
98
|
+
mock(pig).should_do_pig_install?(pig012).returns(false)
|
99
|
+
mock(pig).should_do_pig_update?(pig012, nil).returns(true)
|
100
100
|
mock(pig).should_do_lib_install?.returns(false)
|
101
101
|
mock(pig).should_do_lib_update?.returns(true)
|
102
|
-
mock(pig).install_pig(
|
102
|
+
mock(pig).install_pig(pig012)
|
103
103
|
mock(pig).install_lib
|
104
104
|
capture_stdout do
|
105
|
-
pig.install_or_update(
|
105
|
+
pig.install_or_update(pig012)
|
106
106
|
end
|
107
107
|
end
|
108
108
|
|
@@ -208,7 +208,7 @@ module Mortar::Local
|
|
208
208
|
mock(pig).run_pig_command.with_any_args.returns(true)
|
209
209
|
mock(pig).show_illustrate_output_browser.with_any_args
|
210
210
|
stub(pig).make_pig_param_file.returns('param.file')
|
211
|
-
pig.illustrate_alias(script, 'my_alias', false, false, "0.
|
211
|
+
pig.illustrate_alias(script, 'my_alias', false, false, "0.12", [])
|
212
212
|
end
|
213
213
|
|
214
214
|
it "displays text results if illustrate was successful with no_browser" do
|
@@ -218,7 +218,7 @@ module Mortar::Local
|
|
218
218
|
stub(pig).run_pig_command.with_any_args.returns(true)
|
219
219
|
mock(pig).display.with_any_args
|
220
220
|
stub(pig).make_pig_param_file.returns('param.file')
|
221
|
-
pig.illustrate_alias(script, 'my_alias', false, true, "0.
|
221
|
+
pig.illustrate_alias(script, 'my_alias', false, true, "0.12", [])
|
222
222
|
end
|
223
223
|
|
224
224
|
it "skips results if illustrate was unsuccessful" do
|
@@ -228,7 +228,7 @@ module Mortar::Local
|
|
228
228
|
mock(pig).run_pig_command.with_any_args.returns(false)
|
229
229
|
mock(pig).show_illustrate_output_browser.with_any_args.never
|
230
230
|
stub(pig).make_pig_param_file.returns('param.file')
|
231
|
-
pig.illustrate_alias(script, 'my_alias', false, false, "0.
|
231
|
+
pig.illustrate_alias(script, 'my_alias', false, false, "0.12", [])
|
232
232
|
end
|
233
233
|
|
234
234
|
it "does not require login credentials for illustration" do
|
@@ -238,7 +238,7 @@ module Mortar::Local
|
|
238
238
|
mock(Mortar::Auth).user_s3_safe(true).returns('notloggedin-user-org')
|
239
239
|
mock(pig).run_pig_command.with_any_args.returns(true)
|
240
240
|
mock(pig).show_illustrate_output_browser.with_any_args
|
241
|
-
pig.illustrate_alias(script, 'my_alias', false, false, "0.
|
241
|
+
pig.illustrate_alias(script, 'my_alias', false, false, "0.12", [])
|
242
242
|
end
|
243
243
|
|
244
244
|
end
|
@@ -263,15 +263,14 @@ module Mortar::Local
|
|
263
263
|
it "Uses default pig" do
|
264
264
|
pig = Mortar::Local::Pig.new
|
265
265
|
tpc = pig.template_params_classpath
|
266
|
-
expect(tpc.include?("pig-0.
|
267
|
-
expect(tpc.include?("pig-0.12")).to be_false
|
266
|
+
expect(tpc.include?("pig-0.12")).to be_true
|
267
|
+
expect(tpc.include?("pig-0.12-Hadoop-2")).to be_false
|
268
268
|
end
|
269
269
|
|
270
|
-
it "Works with 0.12 pig" do
|
270
|
+
it "Works with 0.12-Hadoop-2 pig" do
|
271
271
|
pig = Mortar::Local::Pig.new
|
272
|
-
tpc = pig.template_params_classpath(Mortar::PigVersion::
|
273
|
-
expect(tpc.include?("pig-0.12")).to be_true
|
274
|
-
expect(tpc.include?("pig-0.9")).to be_false
|
272
|
+
tpc = pig.template_params_classpath(Mortar::PigVersion::Pig012Hadoop2.new)
|
273
|
+
expect(tpc.include?("pig-0.12-Hadoop-2")).to be_true
|
275
274
|
end
|
276
275
|
end
|
277
276
|
|