mortar 0.15.19 → 0.15.20

data/lib/mortar/command/local.rb

@@ -40,7 +40,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     Dir.chdir(project_root)
 
     ctrl = Mortar::Local::Controller.new
-    ctrl.install_and_configure(pig_version)
+    ctrl.install_and_configure(pig_version, nil, true)
   end
 
   # local:run SCRIPT
@@ -253,7 +253,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     ctrl = Mortar::Local::Controller.new
     luigi_params = pig_parameters.sort_by { |p| p['name'] }
     luigi_params = luigi_params.map { |arg| ["--#{arg['name']}", "#{arg['value']}"] }.flatten
-    ctrl.run_luigi(script, luigi_params)
+    ctrl.run_luigi(pig_version, script, luigi_params)
   end
 
   # local:sqoop_table dbtype database-name table s3-destination
@@ -294,7 +294,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     connstr = jdbc_conn(dbtype, dbhost, physdb)
 
     ctrl = Mortar::Local::Controller.new
-    ctrl.sqoop_export_table(connstr, dbtable, s3dest, options)
+    ctrl.sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
   end
 
   # local:sqoop_query dbtype database-name query s3-destination
@@ -335,7 +335,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     connstr = jdbc_conn(dbtype, dbhost, physdb)
 
     ctrl = Mortar::Local::Controller.new
-    ctrl.sqoop_export_query(connstr, query, s3dest, options)
+    ctrl.sqoop_export_query(pig_version, connstr, query, s3dest, options)
   end
 
   # local:sqoop_incremental dbtype database-name table column value s3-destination
@@ -384,7 +384,7 @@ class Mortar::Command::Local < Mortar::Command::Base
     connstr = jdbc_conn(dbtype, dbhost, physdb)
 
     ctrl = Mortar::Local::Controller.new
-    ctrl.sqoop_export_incremental(connstr, table, column, max_value, s3dest, options)
+    ctrl.sqoop_export_incremental(pig_version, connstr, table, column, max_value, s3dest, options)
   end
 
 

data/lib/mortar/local/controller.rb

@@ -114,11 +114,12 @@ EOF
   
   # Main entry point to perform installation and configuration necessary
   # to run pig on the users local machine
-  def install_and_configure(pig_version=nil, command=nil)
+  def install_and_configure(pig_version=nil, command=nil, install_sqoop=false)
     #To support old watchtower plugins we'll accept nil pig_version
     if pig_version.nil?
       pig_version = Mortar::PigVersion::Pig09.new
     end
+
     java = Mortar::Local::Java.new()
     unless java.check_install
       error(NO_JAVA_ERROR_MESSAGE)
@@ -145,9 +146,10 @@ EOF
     jy = Mortar::Local::Jython.new()
     jy.install_or_update()
 
-
-    sqoop = Mortar::Local::Sqoop.new()
-    sqoop.install_or_update()
+    if install_sqoop
+      sqoop = Mortar::Local::Sqoop.new()
+      sqoop.install_or_update()
+    end
 
     ensure_local_install_dirs_in_gitignore
   end
@@ -205,31 +207,31 @@ EOF
     pig.launch_repl(pig_version, pig_parameters)
   end
 
-  def run_luigi(luigi_script, user_script_args)
-    install_and_configure(nil, 'luigi')
+  def run_luigi(pig_version, luigi_script, user_script_args)
+    install_and_configure(pig_version, 'luigi')
     py = Mortar::Local::Python.new()
     py.run_luigi_script(luigi_script, user_script_args)
   end
 
-  def sqoop_export_table(connstr, dbtable, s3dest, options)
+  def sqoop_export_table(pig_version, connstr, dbtable, s3dest, options)
     require_aws_keys
-    install_and_configure(nil, 'sqoop')
+    install_and_configure(pig_version, 'sqoop', true)
     sqoop = Mortar::Local::Sqoop.new()
     options[:dbtable] = dbtable
     sqoop.export(connstr, s3dest, options)
   end
 
-  def sqoop_export_query(connstr, query, s3dest, options)
+  def sqoop_export_query(pig_version, connstr, query, s3dest, options)
     require_aws_keys
-    install_and_configure(nil, 'sqoop')
+    install_and_configure(pig_version, 'sqoop', true)
     sqoop = Mortar::Local::Sqoop.new()
     options[:sqlquery] = sqoop.prep_query(query)
     sqoop.export(connstr, s3dest, options)
   end
 
-  def sqoop_export_incremental(connstr, dbtable, column, max_value, s3dest, options)
+  def sqoop_export_incremental(pig_version, connstr, dbtable, column, max_value, s3dest, options)
     require_aws_keys
-    install_and_configure(nil, 'sqoop')
+    install_and_configure(pig_version, 'sqoop', true)
     sqoop = Mortar::Local::Sqoop.new()
     options[:dbtable] = dbtable
     options[:inc_column] = column

data/lib/mortar/project.rb

@@ -93,10 +93,6 @@ module Mortar
         path
       end
 
-      def fixtures_path
-        path = File.join(@root_path, "fixtures")
-      end
-
       def embedded_project?()
         File.exists?(File.join(@root_path, ".mortar-project-remote"))
       end

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.19"
+  VERSION = "0.15.20"
 end

data/spec/mortar/command/local_spec.rb

@@ -299,7 +299,7 @@ STDERR
             mock(u).run_luigi_script(luigi_script, %W{--myoption 2 --myotheroption 3})
           end
           any_instance_of(Mortar::Local::Controller) do |u|
-            mock(u).install_and_configure(nil,'luigi')
+            mock(u).install_and_configure(is_a(Mortar::PigVersion::Pig09),'luigi')
           end
           stderr, stdout = execute("local:luigi some_luigi_script -p myoption=2 -p myotheroption=3", p)
           stderr.should == ""
@@ -347,7 +347,7 @@ STDERR
         dbtable = "customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
-          mock(c).sqoop_export_table(connstr, dbtable, s3dest, {})
+          mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {:host => "foobar.com"})
         end
         stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest} --host foobar.com"
       end
@@ -357,7 +357,7 @@ STDERR
         dbtable = "customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
-          mock(c).sqoop_export_table(connstr, dbtable, s3dest, {})
+          mock(c).sqoop_export_table(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, s3dest, {})
         end
         stderr, stdout = execute "local:sqoop_table mysql mydb #{dbtable} #{s3dest}"
       end
@@ -402,7 +402,7 @@ STDERR
         query = "select_*_from_customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
-          mock(c).sqoop_export_query(connstr, query, s3dest, {})
+          mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {:host => "foobar.com"})
         end
         stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest} --host foobar.com"
         stderr.should == ''
@@ -413,7 +413,7 @@ STDERR
         query = "select_*_from_customers"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
-          mock(c).sqoop_export_query(connstr, query, s3dest, {})
+          mock(c).sqoop_export_query(is_a(Mortar::PigVersion::Pig09), connstr, query, s3dest, {})
         end
         stderr, stdout = execute "local:sqoop_query mysql mydb #{query} #{s3dest}"
         stderr.should == ''
@@ -477,7 +477,7 @@ STDERR
         column_value = "12345"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
-          mock(c).sqoop_export_incremental(connstr, dbtable, column, column_value, s3dest, {})
+          mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {:host => "foobar.com"})
         end
         stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest} --host foobar.com"
       end
@@ -489,7 +489,7 @@ STDERR
         column_value = "12345"
         s3dest = "s3n://a-bucket/a-directory"
         any_instance_of(Mortar::Local::Controller) do |c|
-          mock(c).sqoop_export_incremental(connstr, dbtable, column, column_value, s3dest, {})
+          mock(c).sqoop_export_incremental(is_a(Mortar::PigVersion::Pig09), connstr, dbtable, column, column_value, s3dest, {})
         end
         stderr, stdout = execute "local:sqoop_incremental mysql mydb #{dbtable} #{column} #{column_value} #{s3dest}"
       end

data/spec/mortar/local/controller_spec.rb

@@ -146,12 +146,35 @@ module Mortar::Local
         any_instance_of(Mortar::Local::Jython) do |j|
           mock(j).install_or_update
         end
+
+        mock(ctrl).ensure_local_install_dirs_in_gitignore
+        ctrl.install_and_configure
+      end
+
+      it "install sqoop with command" do
+        command = 'command'
+        ctrl = Mortar::Local::Controller.new
+
+        any_instance_of(Mortar::Local::Java) do |j|
+          mock(j).check_install.returns(true)
+        end
+        any_instance_of(Mortar::Local::Pig) do |p|
+          mock(p).install_or_update(is_a(Mortar::PigVersion::Pig012), command)
+        end
+        any_instance_of(Mortar::Local::Python) do |p|
+          mock(p).check_or_install.returns(true)
+          mock(p).check_virtualenv.returns(true)
+          mock(p).setup_project_python_environment.returns(true)
+        end
+        any_instance_of(Mortar::Local::Jython) do |j|
+          mock(j).install_or_update
+        end
         any_instance_of(Mortar::Local::Sqoop) do |s|
           mock(s).install_or_update
         end
 
         mock(ctrl).ensure_local_install_dirs_in_gitignore
-        ctrl.install_and_configure
+        ctrl.install_and_configure(Mortar::PigVersion::Pig012.new, command, true)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.15.19
+  version: 0.15.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-27 00:00:00.000000000 Z
+date: 2014-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.3
+        version: 0.8.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.3
+        version: 0.8.4
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
@@ -208,17 +208,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 description: Client library and command-line tool to interact with the Mortar service.
 email: support@mortardata.com
 executables:
@@ -245,7 +245,6 @@ files:
 - lib/mortar/command/clusters.rb
 - lib/mortar/command/config.rb
 - lib/mortar/command/describe.rb
-- lib/mortar/command/fixtures.rb
 - lib/mortar/command/generate.rb
 - lib/mortar/command/help.rb
 - lib/mortar/command/illustrate.rb
@@ -282,7 +281,6 @@ files:
 - lib/mortar/templates/characterize/README.md
 - lib/mortar/templates/characterize/controlscripts/lib/__init__.py
 - lib/mortar/templates/characterize/controlscripts/lib/characterize_control.py
-- lib/mortar/templates/characterize/fixtures/gitkeep
 - lib/mortar/templates/characterize/gitignore
 - lib/mortar/templates/characterize/macros/characterize_macro.pig
 - lib/mortar/templates/characterize/macros/gitkeep
@@ -304,7 +302,6 @@ files:
 - lib/mortar/templates/pigscript/python_udf.py
 - lib/mortar/templates/project/README.md
 - lib/mortar/templates/project/controlscripts/lib/__init__.py
-- lib/mortar/templates/project/fixtures/gitkeep
 - lib/mortar/templates/project/gitignore
 - lib/mortar/templates/project/lib/gitkeep
 - lib/mortar/templates/project/macros/gitkeep
@@ -333,7 +330,6 @@ files:
 - spec/mortar/command/clusters_spec.rb
 - spec/mortar/command/config_spec.rb
 - spec/mortar/command/describe_spec.rb
-- spec/mortar/command/fixtures_spec.rb
 - spec/mortar/command/generate_spec.rb
 - spec/mortar/command/illustrate_spec.rb
 - spec/mortar/command/jobs_spec.rb

data/lib/mortar/command/fixtures.rb

@@ -1,100 +0,0 @@
-#
-# Copyright 2012 Mortar Data Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require "mortar/command/base"
-
-# create a reusable fixture.
-#
-class Mortar::Command::Fixtures < Mortar::Command::Base
-
-  WARNING_NUM_ROWS = 50
-
-  #fixtures:head [INPUT_URL] [NUM_ROWS] [FIXTURE_NAME]
-  #
-  #Create a reusable fixture [FIXTURE_NAME] made up of [NUM_ROWS]
-  #number of rows from the head of the input file(s) at [INPUT_URL].
-  #
-  # Examples:
-  #
-  # $ mortar fixtures:head s3n://tbmmsd/*.tsv.* 100 samll_song_sample
-  #
-  def head
-    input_url = shift_argument
-    num_rows = shift_argument
-    fixture_name = shift_argument
-    unless input_url && num_rows && fixture_name
-      error("Usage: mortar fixtures:head INPUT_URL NUM_ROWS FIXTURE_NAME\nMust specifiy INPUT_URL, NUM_ROWS, and FIXTURE_NAME.")
-    end
-    if does_fixture_exist(fixture_name)
-      error("Fixture #{fixture_name} already exists.")
-    end
-    unless num_rows.to_i < WARNING_NUM_ROWS
-      warning("Creating fixtures with more than #{WARNING_NUM_ROWS} rows is not recommended.  Large local fixtures may cause slowness when using Mortar.")
-      display
-    end
-    validate_arguments!
-    validate_git_based_project!
-
-    fixture_id = nil
-    action("Requesting fixture creation") do
-      fixture_id = api.post_fixture_limit(project.name, fixture_name, input_url, num_rows).body['fixture_id']
-    end
-
-    poll_for_fixture_results(fixture_id)
-  end
-
-
-
-  private
-
-  def does_fixture_exist(fixture_name)
-    fixture_path = File.join(project.fixtures_path, fixture_name)
-    File.exists?(fixture_path)
-  end
-
-  def poll_for_fixture_results(fixture_id)
-    fixture_result = nil
-    display
-    ticking(polling_interval) do |ticks|
-      fixture_result = api.get_fixture(fixture_id).body
-      is_finished =
-        Mortar::API::Fixtures::STATUSES_COMPLETE.include?(fixture_result["status_code"])
-
-      redisplay("Status: %s %s" % [
-        fixture_result['status_description'] + (is_finished ? "" : "..."),
-        is_finished ? "" : spinner(ticks)],
-        is_finished) # only display newline on last message
-      if is_finished
-        display
-         break
-      end
-    end
-
-    case fixture_result['status_code']
-    when Mortar::API::Fixtures::STATUS_FAILED
-      error_message = "Fixture generation failed with #{fixture_result['error_type'] || 'error'}"
-      error_context = get_error_message_context(fixture_result['error_message'] || "")
-      error_message += ":\n\n#{fixture_result['error_message']}\n\n#{error_context}"
-      error(error_message)
-    when Mortar::API::Fixtures::STATUS_CREATED
-      fixture_result['sample_s3_urls'].each do |u|
-        download_to_file(u['url'], "fixtures/#{fixture_result['name']}/#{u['name']}")
-        display
-      end
-    end
-  end
-
-end

data/spec/mortar/command/fixtures_spec.rb

@@ -1,136 +0,0 @@
-#
-# Copyright 2012 Mortar Data Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-require 'spec_helper'
-require 'fakefs/spec_helpers'
-require 'mortar/command/fixtures'
-require 'mortar/api/fixtures'
-
-module Mortar::Command
-  describe Fixtures do
-    before(:each) do
-      stub_core
-      @git = Mortar::Git::Git.new
-    end
-
-    context("index") do
-
-      it "errors when missing command" do
-        with_git_initialized_project do |p|
-          stderr, stdout = execute("fixtures:head s3n://tbmmsd/*.tsv.* 5", p, @git)
-          stderr.should == <<-STDERR
- !    Usage: mortar fixtures:head INPUT_URL NUM_ROWS FIXTURE_NAME
- !    Must specifiy INPUT_URL, NUM_ROWS, and FIXTURE_NAME.
-STDERR
-        end
-      end
-
-      it "requests and reports on a successful fixtures:head" do
-        with_git_initialized_project do |p|
-          fixture_id = "12345abcde"
-          name = "My_pet_fixture"
-          url = "s3://my_pet_fixture"
-          num_rows = "60"
-
-          sample_s3_urls = [ {'url' => "url1",
-                              'name' => "url1_name"}]
-
-          mock(Mortar::Auth.api).post_fixture_limit(p.name, name, url, num_rows) {Excon::Response.new(:body => {"fixture_id" => fixture_id})}
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_PENDING, "status_description" => "Pending"})).ordered
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_CREATING, "status_description" => "Creating"})).ordered
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_SAVING, "status_description" => "Uploading"})).ordered
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_CREATED, "status_description" => "Success", "name" => name, "sample_s3_urls" => sample_s3_urls})).ordered
-
-          any_instance_of(Mortar::Command::Fixtures) do |base|
-            mock(base).download_to_file(sample_s3_urls[0]['url'], "fixtures/#{name}/#{sample_s3_urls[0]['name']}")
-          end
-
-          stderr, stdout = execute("fixtures:head #{url} #{num_rows} #{name} --polling_interval 0.05", p, @git)
-
-          stdout.should == <<-STDOUT
-WARNING: Creating fixtures with more than 50 rows is not recommended.  Large local fixtures may cause slowness when using Mortar.
-
-Requesting fixture creation... done
-
-\r\e[0KStatus: Pending... /\r\e[0KStatus: Creating... -\r\e[0KStatus: Uploading... \\\r\e[0KStatus: Success \n\n
-STDOUT
-
-        end
-      end
-
-      it "requests and reports on a failed fixtures:head" do
-        with_git_initialized_project do |p|
-          fixture_id = "12345abcde"
-          name = "My_pet_fixture"
-          url = "s3://my_pet_fixture"
-          num_rows = "60"
-
-          sample_s3_urls = [ {'url' => "url1",
-                              'name' => "url1_name"}]
-
-          mock(Mortar::Auth.api).post_fixture_limit(p.name, name, url, num_rows) {Excon::Response.new(:body => {"fixture_id" => fixture_id})}
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_PENDING, "status_description" => "Pending"})).ordered
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_CREATING, "status_description" => "Creating"})).ordered
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_SAVING, "status_description" => "Uploading"})).ordered
-          mock(Mortar::Auth.api).get_fixture(fixture_id).returns(Excon::Response.new(:body => {"status_code" => Mortar::API::Fixtures::STATUS_FAILED, 
-            "status_description" => "Failed", 
-            "name" => name,
-            "error_message" => "This is an error message.",
-            "error_type" => "UserError" })).ordered
-
-          stderr, stdout = execute("fixtures:head #{url} #{num_rows} #{name} --polling_interval 0.05", p, @git)
-
-          stdout.should == <<-STDOUT
-WARNING: Creating fixtures with more than 50 rows is not recommended.  Large local fixtures may cause slowness when using Mortar.
-
-Requesting fixture creation... done
-
-\r\e[0KStatus: Pending... /\r\e[0KStatus: Creating... -\r\e[0KStatus: Uploading... \\\r\e[0KStatus: Failed \n
-STDOUT
-
-stderr.should == <<-STDERR
- !    Fixture generation failed with UserError:
- !    
- !    This is an error message.
-STDERR
-
-        end
-      end
-
-
-      it "tries to create a fixture in an existing directory" do
-        with_git_initialized_project do |p|
-          fixture_id = "12345abcde"
-          name = "My_pet_fixture"
-          url = "s3://my_pet_fixture"
-          num_rows = "60"
-
-          fixtures_dir = File.join(Dir.pwd, "fixtures", name)
-          FileUtils.mkdir_p(fixtures_dir)
-
-          stderr, stdout = execute("fixtures:head #{url} #{num_rows} #{name} --polling_interval 0.05", p, @git)
-
-          stderr.should == <<-STDERR
- !    Fixture #{name} already exists.
-STDERR
-        end
-      end
-
-
-    
-    end
-  end
-end