mortar 0.9.3 → 0.9.4

data/lib/mortar/plugin.rb

@@ -55,6 +55,7 @@ module Mortar
       # TODO: Deal with the bundler as a runtime dependency issue
       # before moving these require statements to the top.
       begin
+        require 'bundler'
         require 'bundler/cli'
         require 'bundler/friendly_errors'
         require 'monkey_patch/bundler/bundler'

data/lib/mortar/templates/characterize/README.md

@@ -0,0 +1,7 @@
+# Welcome to Mortar!
+
+Mortar is a platform-as-a-service for Hadoop.  With Mortar, you can run jobs on Hadoop using Apache Pig and Python without any special training.  You create your project using the Mortar Development Framework, deploy code using the Git revision control system, and Mortar does the rest.
+
+# Getting Started
+
+For help getting started with Mortar, check out the [Mortar Help](http://help.mortardata.com/) site.

data/lib/mortar/templates/characterize/controlscripts/lib/characterize_control.py

@@ -0,0 +1,27 @@
+from org.apache.pig.scripting import Pig
+import os
+import sys
+
+if __name__ == "__main__":
+    params       = Pig.getParameters()
+    loader       = params["LOADER"]
+    input_source = params["INPUT_SRC"]
+    output_path  = params["OUTPUT_PATH"]
+    infer_types  = params["INFER_TYPES"]
+
+    if os.path.exists("../%s" % output_path):
+        sys.stderr.write("ERROR: Output directory file: %s exists...Aborting!\n" % output_path)
+    else:
+        Pig.compileFromFile("../pigscripts/characterize.pig").bind({
+            "LOADER"      : loader,
+            "INPUT_SRC"   : input_source,
+            "OUTPUT_PATH" : output_path,
+            "INFER_TYPES" : infer_types
+        }).runSingle()
+
+        for root, _, files in os.walk("../%s" % output_path):
+            for f in files:
+                if f[0] != '.':
+                    fullpath = os.path.join(root, f)
+                    copypath = os.path.join(root, f + '.csv')
+                    os.system ("cp %s %s" % (fullpath, copypath))

data/lib/mortar/templates/characterize/gitignore

@@ -0,0 +1,8 @@
+Gemfile.lock
+*.pyc
+*.class
+*.log
+tmp
+.mortar-local
+logs
+illustrate-output

data/lib/mortar/templates/characterize/pigscripts/pigscript.pig

@@ -0,0 +1,38 @@
+/**
+ * <%= project_name %>
+ */
+ 
+/** 
+ * Parameters - set default values here; you can override with -p on the command-line.
+ */
+ 
+<%= '%default'%> INPUT_PATH 's3n://mortar-example-data/tutorial/excite.log.bz2'
+<%= '%default'%> OUTPUT_PATH 's3n://my-output-bucket/$MORTAR_EMAIL_S3_ESCAPED/<%= project_name %>'
+
+/**
+ * User-Defined Functions (UDFs)
+ */
+
+REGISTER '../udfs/python/<%= project_name %>.py' USING streaming_python AS <%= project_name_alias %>;
+
+-- This is an example of loading up input data
+my_input_data = LOAD '$INPUT_PATH' 
+               USING PigStorage('\t') 
+                  AS (field0:chararray, field1:chararray, field2:chararray);
+
+-- This is an example pig operation
+filtered = FILTER my_input_data
+               BY field0 IS NOT NULL;
+
+-- This is an example call to a python user-defined function
+with_udf_output = FOREACH filtered 
+                 GENERATE field0..field2, 
+                          <%= project_name_alias %>.example_udf(field0) AS example_udf_field;
+
+-- remove any existing data
+rmf $OUTPUT_PATH;
+
+-- store the results
+STORE with_udf_output 
+ INTO '$OUTPUT_PATH' 
+USING PigStorage('\t');

data/lib/mortar/templates/characterize/udfs/python/python_udf.py

@@ -0,0 +1,13 @@
+from pig_util import outputSchema
+
+# 
+# This is where we write python UDFs (User-Defined Functions) that we can call from pig.
+# Pig needs to know the schema of the data coming out of the function, 
+# which we specify using the @outputSchema decorator.
+#
+@outputSchema('example_udf:int')
+def example_udf(input_str):
+    """
+    A simple example function that just returns the length of the string passed in.
+    """
+    return len(input_str) if input_str else None

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.9.3"
+  VERSION = "0.9.4"
 end

data/spec/mortar/command/local_spec.rb

@@ -18,6 +18,7 @@ require 'spec_helper'
 require 'fakefs/spec_helpers'
 require 'mortar/command/local'
 require 'launchy'
+require 'fileutils'
 
 module Mortar::Command
   describe Local do
@@ -104,6 +105,33 @@ STDERR
     # run
     end
 
+    context("characterize") do
+      # TODO - oleiman: some tests for characterize should go here
+
+      it "should clean up after itself" do
+        @tmpdir = Dir.mktmpdir
+        Dir.chdir(@tmpdir)
+
+        File.open("test.params", 'w') do |file|
+          file.write(<<PARAMS
+LOADER=org.apache.pig.piggybank.storage.JsonLoader()
+INPUT_SRC=s3n://twitter-gardenhose-mortar/example
+OUTPUT_PATH=twitter_char
+INFER_TYPES=true
+PARAMS
+          )
+        end
+        stderr, stdout = execute("generate:project Test")
+        stderr, stdout = execute("local:characterize -f test.params --project_root Test")
+        File.exists?("Test/pigscripts/characterize.pig").should be_false
+        File.exists?("Test/macros/characterize_macro.pig").should be_false
+        File.exists?("Test/udfs/jython/top_5_tuple.py").should be_false
+        File.exists?("Test/controlscripts/lib/characterize_control.py").should be_false
+        File.delete("test.params")           
+      end
+
+    end
+
     context("configure") do
 
       it "errors if the project root doesn't exist or we can't cd there" do
@@ -142,8 +170,7 @@ STDERR
         end
         any_instance_of(Mortar::Local::Python) do |j|
           mock(j).check_or_install.returns(true)
-        end
-        any_instance_of(Mortar::Local::Python) do |j|
+          mock(j).check_virtualenv.returns(true)
           mock(j).setup_project_python_environment.returns(true)
         end
         any_instance_of(Mortar::Local::Jython) do |j|

data/spec/mortar/command/projects_spec.rb

@@ -130,14 +130,11 @@ STDOUT
 \e[1;32m      create\e[0m  .gitignore
 \e[1;32m      create\e[0m  pigscripts
 \e[1;32m      create\e[0m  pigscripts/some_new_project.pig
-\e[1;32m      create\e[0m  pigscripts/characterize.pig
 \e[1;32m      create\e[0m  controlscripts
 \e[1;32m      create\e[0m  controlscripts/lib
 \e[1;32m      create\e[0m  controlscripts/lib/__init__.py
-\e[1;32m      create\e[0m  controlscripts/lib/characterize_control.py
 \e[1;32m      create\e[0m  macros
 \e[1;32m      create\e[0m  macros/.gitkeep
-\e[1;32m      create\e[0m  macros/characterize_macro.pig
 \e[1;32m      create\e[0m  fixtures
 \e[1;32m      create\e[0m  fixtures/.gitkeep
 \e[1;32m      create\e[0m  udfs
@@ -145,7 +142,6 @@ STDOUT
 \e[1;32m      create\e[0m  udfs/python/some_new_project.py
 \e[1;32m      create\e[0m  udfs/jython
 \e[1;32m      create\e[0m  udfs/jython/.gitkeep
-\e[1;32m      create\e[0m  udfs/jython/top_5_tuple.py
 \e[1;32m      create\e[0m  udfs/java
 \e[1;32m      create\e[0m  udfs/java/.gitkeep
 \e[1;32m      create\e[0m  vendor

data/spec/mortar/local/installutil_spec.rb

@@ -106,7 +106,11 @@ module Mortar::Local
 
       it "returns an epoch" do
         excon_response = Excon::Response.new(:headers => {"Last-Modified" => "Mon, 11 Mar 2013 15:03:55 GMT"})
-        mock(Excon).head("http://foo/bar").returns(excon_response)
+        mock(Excon).head("http://foo/bar", 
+                         :headers => {"User-Agent"=>Mortar::USER_AGENT},
+                         :query => {}
+                        ).returns(excon_response)
+        
         actual_epoch = @installutil.url_date("http://foo/bar")
         expect(actual_epoch).to eq(1363014235)
       end
@@ -122,5 +126,69 @@ module Mortar::Local
 
     end
 
+    context "download_file" do
+
+      it "follows redirect" do
+        excon_first_response = Excon::Response.new(:headers => {"Location" => "http://foo/bar2"}, :status => 302)
+        mock(Excon).get("http://foo/bar", 
+                        :headers => {'User-Agent' => Mortar::USER_AGENT},
+                        :query => {}
+                        ).returns(excon_first_response)
+        
+        excon_second_response = Excon::Response.new(:status => 200, :body => "content")
+        mock(Excon).get("http://foo/bar2", 
+                        :headers => {'User-Agent' => Mortar::USER_AGENT},
+                        :query => {}
+                        ).returns(excon_second_response)
+        
+        local_install_directory = @installutil.local_install_directory
+        expected_file_path = File.join(local_install_directory, 'bar2')
+        FakeFS do
+          FileUtils.mkdir_p(local_install_directory)
+          @installutil.download_file('http://foo/bar', expected_file_path)
+          expect(File.exists?(expected_file_path)).to be_true
+        end
+      end
+
+    end
+
+    context "get_resource" do
+      it "too many redirects" do
+        response = Excon::Response.new(:headers => {"Location" => "http://foo/bar"}, :status => 302)
+        mock(Excon).get("http://foo/bar", 
+                        :headers => {'User-Agent' => Mortar::USER_AGENT},
+                        :query => {}
+                       ).times(5).returns(response)
+        
+        lambda { @installutil.get_resource("http://foo/bar") }.should raise_error(RuntimeError, /Too many redirects/)
+      end
+
+      it "too many errors" do
+        response = Excon::Response.new(:status => 500)
+        mock(@installutil).make_call_sleep_seconds().times(5).returns(0)
+        mock(Excon).get("http://foo/bar", 
+                        :headers => {'User-Agent' => Mortar::USER_AGENT},
+                        :query => {}
+                       ).times(5).returns(response)
+        
+        lambda { @installutil.get_resource("http://foo/bar") }.should raise_error(RuntimeError, /Server Error/)
+      end
+
+    end
+
+    context "head_resource" do
+      it "too many redirects" do
+        ENV['MORTAR_TEST_NAME'] = "Unit Testor"
+        response = Excon::Response.new(:headers => {"Location" => "http://foo/bar"}, :status => 302)
+        mock(Excon).head("http://foo/bar", 
+                         :headers => {'User-Agent' => Mortar::USER_AGENT},
+                         :query => { :test_name => "Unit Testor" }
+                        ).times(5).returns(response)
+        
+        lambda { @installutil.head_resource("http://foo/bar") }.should raise_error(RuntimeError, /Too many redirects/)
+      end
+    end
+
+
   end
 end

data/spec/mortar/local/pig_spec.rb

@@ -34,8 +34,8 @@ module Mortar::Local
         # chmods bin/pig, removes tgz, and notes the installation
         FakeFS do
           pig = Mortar::Local::Pig.new
-          local_pig_archive = File.join(pig.local_install_directory, pig.pig_archive_file)
-          mock(pig).download_file(pig.pig_archive_url, pig.local_install_directory) do
+          local_pig_archive = File.join(pig.local_install_directory, Mortar::Local::Pig::PIG_TGZ_NAME)
+          mock(pig).download_file(pig.pig_archive_url, local_pig_archive) do
             # Simulate the tgz file being downloaded, this should be deleted
             # before the method finishes executing
             FileUtils.touch(local_pig_archive)

metadata

@@ -1,167 +1,180 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: mortar
-version: !ruby/object:Gem::Version
-  version: 0.9.3
+version: !ruby/object:Gem::Version 
+  hash: 51
   prerelease: 
+  segments: 
+  - 0
+  - 9
+  - 4
+  version: 0.9.4
 platform: ruby
-authors:
+authors: 
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-23 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: mortar-api-ruby
-  requirement: !ruby/object:Gem::Requirement
+
+date: 2013-08-02 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rdoc
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.6.3
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 63
+        segments: 
+        - 4
+        - 0
+        - 0
+        version: 4.0.0
   type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: mortar-api-ruby
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 0
+        - 6
+        - 3
         version: 0.6.3
-- !ruby/object:Gem::Dependency
-  name: netrc
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.7'
   type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: netrc
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.7'
-- !ruby/object:Gem::Dependency
-  name: launchy
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.1'
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 0
+        - 7
+        version: "0.7"
   type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: launchy
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.1'
-- !ruby/object:Gem::Dependency
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 2
+        - 1
+        version: "2.1"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
   name: excon
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.15'
-  type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.15'
-- !ruby/object:Gem::Dependency
-  name: fakefs
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 0.4.2
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 0
+        - 15
+        version: "0.15"
   type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: fakefs
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 0
+        - 4
+        - 2
         version: 0.4.2
-- !ruby/object:Gem::Dependency
-  name: gem-release
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: gem-release
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: rake
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rr
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: rr
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :development
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: rspec
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id010
 description: Client library and command-line tool to interact with the Mortar service.
 email: support@mortardata.com
-executables:
+executables: 
 - mortar
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - README.md
 - bin/mortar
 - css/illustrate.css
@@ -192,6 +205,7 @@ files:
 - lib/mortar/command/validate.rb
 - lib/mortar/command/version.rb
 - lib/mortar/errors.rb
+- lib/mortar/generators/characterize_generator.rb
 - lib/mortar/generators/controlscript_generator.rb
 - lib/mortar/generators/generator_base.rb
 - lib/mortar/generators/macro_generator.rb
@@ -208,22 +222,37 @@ files:
 - lib/mortar/local/python.rb
 - lib/mortar/plugin.rb
 - lib/mortar/project.rb
+- lib/mortar/templates/characterize/README.md
+- lib/mortar/templates/characterize/controlscripts/lib/__init__.py
+- lib/mortar/templates/characterize/controlscripts/lib/characterize_control.py
+- lib/mortar/templates/characterize/fixtures/gitkeep
+- lib/mortar/templates/characterize/gitignore
+- lib/mortar/templates/characterize/macros/characterize_macro.pig
+- lib/mortar/templates/characterize/macros/gitkeep
+- lib/mortar/templates/characterize/pigscripts/characterize.pig
+- lib/mortar/templates/characterize/pigscripts/pigscript.pig
+- lib/mortar/templates/characterize/udfs/java/gitkeep
+- lib/mortar/templates/characterize/udfs/jython/gitkeep
+- lib/mortar/templates/characterize/udfs/jython/top_5_tuple.py
+- lib/mortar/templates/characterize/udfs/python/python_udf.py
+- lib/mortar/templates/characterize/vendor/controlscripts/lib/__init__.py
+- lib/mortar/templates/characterize/vendor/macros/gitkeep
+- lib/mortar/templates/characterize/vendor/pigscripts/gitkeep
+- lib/mortar/templates/characterize/vendor/udfs/java/gitkeep
+- lib/mortar/templates/characterize/vendor/udfs/jython/gitkeep
+- lib/mortar/templates/characterize/vendor/udfs/python/gitkeep
 - lib/mortar/templates/controlscript/controlscript.py
 - lib/mortar/templates/macro/macro.pig
 - lib/mortar/templates/pigscript/pigscript.pig
 - lib/mortar/templates/pigscript/python_udf.py
 - lib/mortar/templates/project/README.md
 - lib/mortar/templates/project/controlscripts/lib/__init__.py
-- lib/mortar/templates/project/controlscripts/lib/characterize_control.py
 - lib/mortar/templates/project/fixtures/gitkeep
 - lib/mortar/templates/project/gitignore
-- lib/mortar/templates/project/macros/characterize_macro.pig
 - lib/mortar/templates/project/macros/gitkeep
-- lib/mortar/templates/project/pigscripts/characterize.pig
 - lib/mortar/templates/project/pigscripts/pigscript.pig
 - lib/mortar/templates/project/udfs/java/gitkeep
 - lib/mortar/templates/project/udfs/jython/gitkeep
-- lib/mortar/templates/project/udfs/jython/top_5_tuple.py
 - lib/mortar/templates/project/udfs/python/python_udf.py
 - lib/mortar/templates/project/vendor/controlscripts/lib/__init__.py
 - lib/mortar/templates/project/vendor/macros/gitkeep
@@ -268,26 +297,38 @@ files:
 - spec/support/display_message_matcher.rb
 homepage: http://mortardata.com/
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 57
+      segments: 
+      - 1
+      - 8
+      - 7
       version: 1.8.7
-required_rubygems_version: !ruby/object:Gem::Requirement
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []
+