mortar 0.15.11 → 0.15.12

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YmJiOGIwNzRlYjIxZjAxNjk3NjJjZTQ1OGExYzM1MThlOTM0ZjZlYw==
+  data.tar.gz: !binary |-
+    YzdlNDlhMTNkNmM4MjRjMmE5ODIyMDAzN2Q4MmFlZmJjYzRiNTdlYw==
+SHA512:
+  metadata.gz: !binary |-
+    MGM2NzRjMDZlOWViZWZjNjI5ZDE2MGQ0MDc1MTk0MWMwOGY3MmEzMTI3N2Nh
+    OWM5OGQzM2YwN2Q1OGQ4ZWNiZGQ5MDRkYmExMDdiNDRkYWI1ZGQ1YWM2Mjg3
+    YTQyNmY1MjAyMzFkNDYzN2VjZTVmZDdkMGExNjg2YTk2N2UxOWU=
+  data.tar.gz: !binary |-
+    NTM1NWNjN2U4OWNiYjM4ZWI5OTZmMDFjODZmMDAyZWNiNjJiYzk3YmZlZmQ2
+    NTYyZWRiN2E2YjJiOGFmMjRjNTdlNjA5Yzk2MzE3YWU3NTMxNTk5MTUxNTBl
+    MDRiOTFjNTQwNzAyYjdkODU5NTBlN2Q1NzVkOWExZjYxNTgwNzQ=

data/lib/mortar/command/base.rb

@@ -152,7 +152,7 @@ class Mortar::Command::Base
 
   def validate_project_structure()
     present_dirs = Dir.glob("*").select { |path| File.directory? path }
-    required_dirs = ["controlscripts", "pigscripts", "macros", "udfs", "fixtures"]
+    required_dirs = ["pigscripts", "macros", "udfs"]
     missing_dirs = required_dirs - present_dirs
 
     if missing_dirs.length > 0

data/lib/mortar/generators/project_generator.rb

@@ -40,26 +40,11 @@ module Mortar
             generate_file "pigscript.pig", "#{project_name}.pig"
           end
 
-          mkdir "controlscripts" 
-
-          inside "controlscripts" do
-            mkdir "lib"
-            inside "lib" do
-              copy_file "__init__.py", "__init__.py"
-            end
-          end
-          
           mkdir "macros"
           
           inside "macros" do
             copy_file "gitkeep", ".gitkeep"
           end
-
-          mkdir "fixtures"
-
-          inside "fixtures" do
-            copy_file "gitkeep", ".gitkeep"
-          end
           
           mkdir "udfs"
           
@@ -85,46 +70,6 @@ module Mortar
             copy_file "gitkeep", ".gitkeep"
           end
 
-          mkdir "vendor"
-
-          inside "vendor" do
-            mkdir "controlscripts"
-            inside "controlscripts" do
-              mkdir "lib"
-              inside "lib" do
-                copy_file "__init__.py", "__init__.py"
-              end
-            end
-
-            mkdir "pigscripts"
-            inside "pigscripts" do
-              copy_file "gitkeep", ".gitkeep"
-            end
-
-            mkdir "macros"
-            inside "macros" do
-              copy_file "gitkeep", ".gitkeep"
-            end
-
-            mkdir "udfs"
-            inside "udfs" do
-              mkdir "python"
-              inside "python" do
-                copy_file "gitkeep", ".gitkeep"
-              end
-
-              mkdir "jython"
-              inside "jython" do
-                copy_file "gitkeep", ".gitkeep"
-              end
-
-              mkdir "java"
-              inside "java" do
-                copy_file "gitkeep", ".gitkeep"
-              end
-            end
-          end
-          
         rescue => e 
           #If we can't set up the project correctly and the project folder
           #didn't exist before - remove it.

data/lib/mortar/git.rb

@@ -151,13 +151,6 @@ module Mortar
               manifest.puts "" # ensure file ends with a newline
             end
 
-            if File.directory?('controlscripts') and not contents.include?('controlscripts')
-              manifest.puts "controlscripts"
-            end
-
-            if File.directory?("vendor") and not contents.include?("vendor")
-              manifest.puts "vendor"
-            end
           end
         else
           create_mortar_project_manifest('.')
@@ -169,18 +162,9 @@ module Mortar
       #
       def create_mortar_project_manifest(path)
         File.open("#{path}/.mortar-project-manifest", 'w') do |manifest|
-          if File.directory? "#{path}/controlscripts"
-            manifest.puts "controlscripts"
-          end
-          if File.directory? "#{path}/fixtures"
-            manifest.puts "fixtures"
-          end
           manifest.puts "pigscripts"
           manifest.puts "macros"
           manifest.puts "udfs"
-          if File.directory? "#{path}/vendor"
-            manifest.puts "vendor"
-          end
           if File.directory? "#{path}/lib"
             manifest.puts "lib"
           end

data/lib/mortar/local/controller.rb

@@ -212,6 +212,7 @@ EOF
   end
 
   def sqoop_export_table(connstr, dbtable, s3dest, options)
+    require_aws_keys
     install_and_configure(nil, 'sqoop')
     sqoop = Mortar::Local::Sqoop.new()
     options[:dbtable] = dbtable
@@ -219,6 +220,7 @@ EOF
   end
 
   def sqoop_export_query(connstr, query, s3dest, options)
+    require_aws_keys
     install_and_configure(nil, 'sqoop')
     sqoop = Mortar::Local::Sqoop.new()
     options[:sqlquery] = sqoop.prep_query(query)
@@ -226,6 +228,7 @@ EOF
   end
 
   def sqoop_export_incremental(connstr, dbtable, column, max_value, s3dest, options)
+    require_aws_keys
     install_and_configure(nil, 'sqoop')
     sqoop = Mortar::Local::Sqoop.new()
     options[:dbtable] = dbtable

data/lib/mortar/local/sqoop.rb

@@ -103,6 +103,7 @@ class Mortar::Local::Sqoop
       "destination" => s3dest,
       "hadoop_home" => hadoop_home,
       "classpath" => pig.template_params_classpath,
+      "postgres_jar" => "#{pig.lib_directory}/lib-cluster/postgresql.jar",
       "jdbc_conn" => connstr,
       "s3dest" => s3dest,
       "project_root" => project_root,

data/lib/mortar/templates/script/sqoop.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-export HADOOP_CLASSPATH="<%= @project_root %>/lib/*"
+export HADOOP_CLASSPATH="<%= @project_root %>/lib/*:<%= @postgres_jar %>"
 <% if @driverjar %>
 export HADOOP_CLASSPATH="$HADOOP_CLASSPATH:<%= @driverjar %>"
 <% end %>

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.11"
+  VERSION = "0.15.12"
 end

data/spec/mortar/command/generate_spec.rb

@@ -32,14 +32,11 @@ describe Mortar::Command::Generate do
       stderr, stdout = execute("generate:project Test")
       File.exists?("Test").should be_true
       File.exists?("Test/macros").should be_true
-      File.exists?("Test/fixtures").should be_true
       File.exists?("Test/pigscripts").should be_true
-      File.exists?("Test/controlscripts").should be_true
       File.exists?("Test/udfs").should be_true
       File.exists?("Test/README.md").should be_true
       File.exists?("Test/Gemfile").should be_false
       File.exists?("Test/macros/.gitkeep").should be_true
-      File.exists?("Test/fixtures/.gitkeep").should be_true
       File.exists?("Test/pigscripts/Test.pig").should be_true
       File.exists?("Test/udfs/python/Test.py").should be_true
 

data/spec/mortar/command/projects_spec.rb

@@ -113,13 +113,11 @@ STDOUT
         stderr, stdout = execute("projects:create #{project_name}", nil, @git)
         Dir.pwd.end_with?("some_new_project").should be_true
         File.exists?("macros").should be_true
-        File.exists?("fixtures").should be_true
         File.exists?("pigscripts").should be_true
         File.exists?("udfs").should be_true
         File.exists?("README.md").should be_true
         File.exists?("Gemfile").should be_false
         File.exists?("macros/.gitkeep").should be_true
-        File.exists?("fixtures/.gitkeep").should be_true
         File.exists?("pigscripts/some_new_project.pig").should be_true
         File.exists?("udfs/python/some_new_project.py").should be_true
 
@@ -132,13 +130,8 @@ Sending request to register project: some_new_project... done
 \e[1;32m      create\e[0m  .gitignore
 \e[1;32m      create\e[0m  pigscripts
 \e[1;32m      create\e[0m  pigscripts/some_new_project.pig
-\e[1;32m      create\e[0m  controlscripts
-\e[1;32m      create\e[0m  controlscripts/lib
-\e[1;32m      create\e[0m  controlscripts/lib/__init__.py
 \e[1;32m      create\e[0m  macros
 \e[1;32m      create\e[0m  macros/.gitkeep
-\e[1;32m      create\e[0m  fixtures
-\e[1;32m      create\e[0m  fixtures/.gitkeep
 \e[1;32m      create\e[0m  udfs
 \e[1;32m      create\e[0m  udfs/python
 \e[1;32m      create\e[0m  udfs/python/some_new_project.py
@@ -148,21 +141,6 @@ Sending request to register project: some_new_project... done
 \e[1;32m      create\e[0m  udfs/java/.gitkeep
 \e[1;32m      create\e[0m  lib
 \e[1;32m      create\e[0m  lib/.gitkeep
-\e[1;32m      create\e[0m  vendor
-\e[1;32m      create\e[0m  vendor/controlscripts
-\e[1;32m      create\e[0m  vendor/controlscripts/lib
-\e[1;32m      create\e[0m  vendor/controlscripts/lib/__init__.py
-\e[1;32m      create\e[0m  vendor/pigscripts
-\e[1;32m      create\e[0m  vendor/pigscripts/.gitkeep
-\e[1;32m      create\e[0m  vendor/macros
-\e[1;32m      create\e[0m  vendor/macros/.gitkeep
-\e[1;32m      create\e[0m  vendor/udfs
-\e[1;32m      create\e[0m  vendor/udfs/python
-\e[1;32m      create\e[0m  vendor/udfs/python/.gitkeep
-\e[1;32m      create\e[0m  vendor/udfs/jython
-\e[1;32m      create\e[0m  vendor/udfs/jython/.gitkeep
-\e[1;32m      create\e[0m  vendor/udfs/java
-\e[1;32m      create\e[0m  vendor/udfs/java/.gitkeep
 \n\r\e[0KStatus: ACTIVE  \n\nYour project is ready for use.  Type 'mortar help' to see the commands you can perform on the project.\n
 NOTE: You'll need to change to the new directory to use your project:
     cd some_new_project
@@ -188,13 +166,11 @@ STDOUT
         Dir.pwd.end_with?("some_new_project").should be_true
         File.exists?(".mortar-project-remote").should be_true
         File.exists?("macros").should be_true
-        File.exists?("fixtures").should be_true
         File.exists?("pigscripts").should be_true
         File.exists?("udfs").should be_true
         File.exists?("README.md").should be_true
         File.exists?("Gemfile").should be_false
         File.exists?("macros/.gitkeep").should be_true
-        File.exists?("fixtures/.gitkeep").should be_true
         File.exists?("pigscripts/some_new_project.pig").should be_true
         File.exists?("udfs/python/some_new_project.py").should be_true
 

data/spec/spec_helper.rb

@@ -171,13 +171,12 @@ def with_blank_project_with_name(name, &block)
   FileUtils.mkdir_p(project_path)
   
   # setup project subdirectories
-  FileUtils.mkdir_p(File.join(project_path, "controlscripts"))
   FileUtils.mkdir_p(File.join(project_path, "pigscripts"))
   FileUtils.mkdir_p(File.join(project_path, "macros"))
   FileUtils.mkdir_p(File.join(project_path, "udfs"))
   FileUtils.mkdir_p(File.join(project_path, "udfs/python"))
   FileUtils.mkdir_p(File.join(project_path, "udfs/jython"))
-  FileUtils.mkdir_p(File.join(project_path, "fixtures"))
+  FileUtils.mkdir_p(File.join(project_path, "udfs/java"))
 
   Dir.chdir(project_path)
   
@@ -348,11 +347,11 @@ def create_and_validate_git_snapshot(git)
   git.git("branch").include?(snapshot_branch).should be_true
 
   snapshotted_paths = Dir.glob("**/*")
-  snapshotted_paths.should include("controlscripts")
   snapshotted_paths.should include("pigscripts")
   snapshotted_paths.should include("macros")
   snapshotted_paths.should include("udfs/python")
   snapshotted_paths.should include("udfs/jython")
+  snapshotted_paths.should include("udfs/java")
 
   Dir.chdir(curdir)
   FileUtils.remove_entry_secure(snapshot_dir)

metadata

@@ -1,229 +1,204 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mortar
-version: !ruby/object:Gem::Version 
-  hash: 53
-  prerelease: 
-  segments: 
-  - 0
-  - 15
-  - 11
-  version: 0.15.11
+version: !ruby/object:Gem::Version
+  version: 0.15.12
 platform: ruby
-authors: 
+authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2014-04-22 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2014-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rdoc
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 63
-        segments: 
-        - 4
-        - 0
-        - 0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: mortar-api-ruby
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: mortar-api-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 57
-        segments: 
-        - 0
-        - 8
-        - 3
+      - !ruby/object:Gem::Version
         version: 0.8.3
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: netrc
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 0
-        - 7
-        version: "0.7"
+      - !ruby/object:Gem::Version
+        version: 0.8.3
+- !ruby/object:Gem::Dependency
+  name: netrc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: launchy
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: launchy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 2
-        - 1
-        version: "2.1"
+      - !ruby/object:Gem::Version
+        version: '2.1'
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: parseconfig
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: parseconfig
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 19
-        segments: 
-        - 1
-        - 0
-        - 2
+      - !ruby/object:Gem::Version
         version: 1.0.2
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: aws-sdk
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 15
-        segments: 
-        - 1
-        - 0
-        version: "1.0"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 5
-        - 0
+      - !ruby/object:Gem::Version
         version: 1.5.0
   type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: excon
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 51
-        segments: 
-        - 0
-        - 28
-        version: "0.28"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.28'
   type: :development
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  name: fakefs
   prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 0
-        - 4
-        - 2
+      - !ruby/object:Gem::Version
+        version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
         version: 0.4.2
   type: :development
-  version_requirements: *id009
-- !ruby/object:Gem::Dependency 
-  name: gem-release
   prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id010
-- !ruby/object:Gem::Dependency 
-  name: rake
   prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 73
-        segments: 
-        - 10
-        - 1
-        - 1
+      - !ruby/object:Gem::Version
         version: 10.1.1
   type: :development
-  version_requirements: *id011
-- !ruby/object:Gem::Dependency 
-  name: rr
   prerelease: false
-  requirement: &id012 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.1.1
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id012
-- !ruby/object:Gem::Dependency 
-  name: rspec
   prerelease: false
-  requirement: &id013 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id013
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Client library and command-line tool to interact with the Mortar service.
 email: support@mortardata.com
-executables: 
+executables:
 - mortar
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - README.md
 - bin/mortar
 - css/illustrate.css
@@ -361,38 +336,25 @@ files:
 - spec/support/display_message_matcher.rb
 homepage: http://mortardata.com/
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 57
-      segments: 
-      - 1
-      - 8
-      - 7
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.8.7
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []
-