mortar 0.15.4 → 0.15.5

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Zjc5OGFiMTk0NDQxMjg2ZWZhNjI5ZDA2MDU3YTIwNTY4ZDg4ZGU5NQ==
+  data.tar.gz: !binary |-
+    MzZiOGExYzYwZjE0ZjU3MmUyY2JmY2QwNmM0YmFkNjliMGY4N2I1Zg==
+SHA512:
+  metadata.gz: !binary |-
+    NWQ2ZTI2NTY1MmFjZDllOTVlZGFkNDExYTk1ODYxMjdhNTgyZDhjNzk2MDlm
+    YjU1MmZiNDU2ZDc0MmI3Mjc1ZDQ0NzM1MWU0ZGI1NGE3NDI0NjMwNzlmNjUx
+    NGFhNzM0ODQ2ZTZlNDQ0ZDcxMzc4YzBlN2U3MTdjNTY5MWZmOGE=
+  data.tar.gz: !binary |-
+    MDliMjQzZGI2YjkyYTg3Y2M3M2ZmOTNjYzI3ZGY0OGZlYzMwMzA2NDdiODE5
+    ZDJkNGVlYTc2MTg2NDA2M2VkYzU5ZmQwZjM1MjhlODY4NjUxMDA3NzkzMzdl
+    OTAyYWI5ZTAyYzY1Y2I2NDI0NTI5Y2QzNWM3NjU2MTg2YTgwYWM=

data/lib/mortar/command/base.rb

@@ -65,10 +65,20 @@ class Mortar::Command::Base
       end
       
       @project = Mortar::Project::Project.new(project_name, project_dir, remote)
+
+      #Every time we get the project, we're going to check if its a forked version and
+      #if it is we'll check for new updates to the base project.
+      begin
+        if git.is_fork_repo_updated(git_organization)
+          warning("The repository this project was forked from has been updated.  To get the latest changes commit all of your work and do:\n\n\tgit merge #{git.fork_base_remote_name}/master\n\nYou may have conflicts that will need to be resolved manually.\n\n")
+        end
+      rescue 
+        #Do nothing.  We'll repeat this call often enough that we don't care if it fails.
+      end
     end
     @project
   end
-  
+
   def api
     Mortar::Auth.api
   end

data/lib/mortar/command/projects.rb

@@ -229,7 +229,7 @@ class Mortar::Command::Projects < Mortar::Command::Base
     end
     is_public = options[:public]
     ask_public(is_public)
-    git.clone(git_url, name, "base")
+    git.clone(git_url, name, git.fork_base_remote_name)
     Dir.chdir(name)
     # register a nil project id because it hasn't been created yet
     register_project(name, is_public, nil) do |project_result|

data/lib/mortar/command/s3.rb

@@ -0,0 +1,62 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "mortar/command/base"
+require "mortar/s3"
+
+# Work with your data on Amazon S3
+class Mortar::Command::S3 < Mortar::Command::Base
+
+  # s3:get S3_PATH OUTPUT_PATH
+  #
+  # Download files from a path in S3 to your local computer.
+  #
+  #
+  # Examples:
+  #
+  #     Download from your bucket with:
+  #         $ mortar s3:get s3://mortar-example/out out/
+  def get
+    do_get("get")
+  end
+
+  # s3:getmerge S3_PATH OUTPUT_PATH
+  #
+  # Download and concatenate files from a path in S3 to your local computer. Merges together all of the hadoop output into one file called 'output' at specified OUTPUT_PATH
+  #
+  # Examples:
+  #
+  #     Download from your bucket with:
+  #         $ mortar s3:getmerge s3://mortar-example/out out/
+  def getmerge
+    do_get("getmerge", true)
+  end
+
+  private 
+  
+  # get functionality
+  def do_get(command, concat = nil)
+    s3_path = shift_argument
+    output_path = shift_argument
+    unless s3_path and output_path
+      error("Usage: mortar s3:#{command} S3_PATH OUTPUT_PATH\nMust specify S3_PATH and OUTPUT_PATH.")
+    end
+    s3 = Mortar::S3::S3.new
+    bucket, key = s3.get_bucket_and_key(s3_path)
+    s3.do_download(bucket, key, output_path, concat)
+  end
+
+end

data/lib/mortar/command/version.rb

@@ -53,7 +53,7 @@ class Mortar::Command::Version < Mortar::Command::Base
         shell_url = ENV.fetch("MORTAR_INSTALL", "http://install.mortardata.com")
         dir = "/opt/mortar/installer"
         begin
-          cmd = "sudo mkdir -p #{dir} && sudo curl -sS -L -o #{dir}/install.sh #{shell_url} && sudo bash #{dir}/install.sh#{version_number}"
+          cmd = "echo 'Upgrading will prompt for your sudo password.\n' && sudo mkdir -p #{dir} && sudo curl -sS -L -o #{dir}/install.sh #{shell_url} && sudo bash #{dir}/install.sh#{version_number}"
           Kernel.system cmd
         ensure
         end

data/lib/mortar/git.rb

@@ -536,6 +536,52 @@ module Mortar
       def clone(git_url, path="", remote_name="origin")
         git("clone -o %s %s \"%s\"" % [remote_name, git_url, path], true, false)
       end
+
+      def fork_base_remote_name
+        "base"
+      end
+
+      def is_fork_repo_updated(git_organization)
+        if remotes(git_organization).has_key?(fork_base_remote_name)
+          fetch(fork_base_remote_name)
+          latest_commit = get_latest_hash(fork_base_remote_name)
+          last_commit = get_last_recorded_fork_hash
+          unless latest_commit == last_commit || contains_hash(latest_commit)
+            File.open(mortar_fork_meta_file, "wb") do |f|
+              f.write(latest_commit)
+            end
+            return true
+          end
+        end
+        return false
+      end
+
+      def get_last_recorded_fork_hash
+        if File.exists?(mortar_fork_meta_file)
+          File.open(mortar_fork_meta_file, "r") do |f|
+            file_contents = f.read()
+            file_contents.strip
+          end
+        end
+      end
+
+      def mortar_fork_meta_file
+        ".mortar-fork"
+      end
+
+      def fetch(remote)
+        git("fetch #{remote}")
+      end
+
+      def get_latest_hash(remote)
+        git("log --pretty=\"%H\" -n 1 #{remote}")
+      end
+
+      def contains_hash(hash)
+        git("log --pretty=\"%H\"").include?(hash)
+      end
+
+
     end
   end
 end

data/lib/mortar/s3.rb

@@ -0,0 +1,177 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Portions of this code from heroku (https://github.com/heroku/heroku/) Copyright Heroku 2008 - 2014,
+# used under an MIT license (https://github.com/heroku/heroku/blob/master/LICENSE).
+#
+require 'mortar/local/controller'
+require "mortar/command/base"
+require "mortar/helpers"
+require 'aws-sdk'
+require 'uri'
+require 'pathname'
+
+
+module Mortar
+  module S3
+    class S3
+      include Helpers
+
+      # gets aws::s3 object
+      def get_s3
+        ctrl = Mortar::Local::Controller.new
+        ctrl.require_aws_keys
+        return AWS::S3.new(
+          :access_key_id => ENV['AWS_ACCESS_KEY'],
+          :secret_access_key => ENV['AWS_SECRET_KEY'])
+      end
+
+      # returns bucket and key from s3 path
+      def get_bucket_and_key(s3_path)
+        uri = ''
+        begin
+          uri = URI.parse(s3_path)
+        rescue URI::InvalidURIError => msg
+          error("#{msg}.\nIt is strongly suggested that your bucket name does not contain an underscore character.\nPlease see http://blog.mortardata.com/post/58920122308/s3-hadoop-performance at tip #4.") 
+        end
+
+
+        unless uri != '' && is_valid_s3_path(uri)
+          error("Requested S3 path, #{s3_path}, is invalid. Please ensure that your S3 path begins with 's3://'.  Example: s3://my-bucket/my-key.")
+        end
+        return uri.host,uri.path[1, uri.path.length]
+      end
+
+
+      # checks if string is a valid s3 path
+      def is_valid_s3_path(uri)
+        uri.scheme == 's3' && uri.host && !uri.path[1, uri.path.length].to_s.empty?
+      end
+
+
+      def remove_slash(str)
+        if str[-1, 1] == "/"
+          str = str[0, str.length-1]
+        end
+        return str
+      end
+
+      def get_file_name(key_str)
+          if !key_str.rindex("/") 
+            return key_str
+          else
+            return Pathname.new(key_str).basename.to_s
+          end
+      end
+
+      def is_file(path, key)
+        if is_not_folder(path,key)
+          res = Pathname.new(path).basename.to_s
+          if !is_hidden_file(res)
+            return true 
+          end
+        end 
+        return false 
+      end
+
+      def is_not_folder(path, key)
+        suffix = ["/", "_$folder$"]
+        # not a folder
+        path[-1,1] != suffix[0]  && !path.index(suffix[1]) and !path[key.length+1, path.length].index("/")
+        #path[-1,1] != suffix[0] && !path.index(suffix[1]) and Pathname.new(path).basename.to_s != '' 
+      end
+
+      def is_hidden_file(str)
+        str[0,1] == "." || str[0,1] == "_"
+      end
+
+      def remove_file(file_path)
+        if File.file?(file_path)
+          FileUtils.remove(file_path)
+        end
+      end
+
+      # checks s3 bucket to see if bucket and key exists for given aws keys
+      def check_bucket(s3, bucket)
+        s3.buckets[bucket].exists?
+      end
+
+
+      # gets s3 object, where each item is a file in bucket and key
+      def get_s3_objects(s3, bucket, key)
+        buck = s3.buckets[bucket]
+        # removes slash at end if it exists
+        key = remove_slash(key) 
+        if buck.objects[key].exists?
+          [buck.objects[key]]
+        else
+          valid_items = Array.new
+          buck.objects.with_prefix(key).each do |obj|
+            if is_file(obj.key, key) 
+              valid_items.push(obj)
+            end
+          end
+          return valid_items
+        end
+      end
+
+      def write_s3_to_file(s3_object, file_name, write_mode)
+        File.open(file_name, write_mode) do |file|
+          s3_object.read do |chunk|
+            file.write(chunk)
+          end
+        end
+      end
+
+
+      def download_s3(s3, bucket, key, output_path, concat = nil)
+          s3_objects = get_s3_objects(s3, bucket,key)
+          concat_file_name = "output"
+          concat_target = "#{output_path}/#{concat_file_name}"
+          if concat
+            remove_file(concat_target)
+          end
+          if !s3_objects.empty?
+            s3_objects.each do |s3_obj|
+              key_str = s3_obj.key
+              out_file = concat ? concat_file_name : get_file_name(key_str)
+              output_target = "#{output_path}/#{out_file}"
+              display "Writing s3://#{bucket}/#{key_str} to #{output_target}"
+              write_s3_to_file(s3_obj, output_target, concat ? "a" : "w+")
+            end
+          else
+            error("No contents were found at path s3://#{bucket}/#{key}.  Please specify again.")
+          end
+      end
+
+      # Implementation of the download command
+      def do_download(bucket, key, output_path, concat=nil)
+        s3 = get_s3
+        # check and messages for concatting file
+        unless check_bucket(s3, bucket)
+         error("Requested S3 bucket #{bucket} in path, s3://#{bucket}/#{key}, does not exist.")
+        end
+        # creates output directory
+        unless File.directory?(output_path)
+          FileUtils.mkdir_p(output_path)
+        end
+        download_s3(s3, bucket, key, output_path, concat)
+
+        display "All done! "
+      end
+
+    end
+  end
+end

data/lib/mortar/templates/characterize/gitignore

@@ -3,6 +3,7 @@ Gemfile.lock
 *.class
 *.log
 tmp
+.mortar-fork
 .mortar-local
 logs
 illustrate-output

data/lib/mortar/templates/project/gitignore

@@ -3,6 +3,7 @@ Gemfile.lock
 *.class
 *.log
 tmp
+.mortar-fork
 .mortar-local
 logs
 illustrate-output

data/lib/mortar/version.rb

@@ -16,5 +16,5 @@
 
 module Mortar
   # see http://semver.org/
-  VERSION = "0.15.4"
+  VERSION = "0.15.5"
 end

data/spec/mortar/command/base_spec.rb

@@ -83,6 +83,24 @@ other\tgit@github.com:other.git (push)
         @base.project.name.should == 'myproject-staging'
       end
 
+      it "errors out on checking for updates to forked project and nobody cares" do
+        stub(@base.git).has_dot_git? {true}
+        stub(@base.git).remotes {{ 'mortar' => 'myproject' }}
+        mock(@base.git).git("config mortar.remote", false).returns("")
+        mock(@base.git).is_fork_repo_updated.with_any_args.returns { raise StandardError.new("meessage") }
+        mock(@base).warning.times(0)
+        @base.project.name.should == 'myproject'
+      end
+
+      it "finds an updated forked project and displays warning" do
+        stub(@base.git).has_dot_git? {true}
+        stub(@base.git).remotes {{ 'mortar' => 'myproject' }}
+        mock(@base.git).git("config mortar.remote", false).returns("")
+        mock(@base.git).is_fork_repo_updated.with_any_args.returns(true)
+        mock(@base).warning.times(1).with_any_args
+        @base.project.name.should == 'myproject'
+      end
+
     end
 
     context "method_added" do

data/spec/mortar/command/s3_spec.rb

@@ -0,0 +1,92 @@
+#
+# Copyright 2014 Mortar Data Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'spec_helper'
+require 'fakefs/spec_helpers'
+require 'mortar/command/s3'
+require 'aws-sdk'
+require "mortar/s3"
+require 's3_faker'
+
+module Mortar::Command
+  describe S3 do
+    context "s3:getmerge" do
+      good_bucket = "good-bucket"
+      bad_bucket = "bad-bucket"
+      key = "direc"
+      s3_path = "s3://#{good_bucket}/#{key}"
+      s3_path_bad = "s3://#{bad_bucket}/key"
+      output_path = "out"
+      concat_file = "#{output_path}/output"
+      keys = [
+          FakeObject.new("key1", true),
+          FakeObject.new("direc/key2", true),
+          FakeObject.new("direc/key3", true),
+          FakeObject.new("direc/another_direc/key", true)
+        ]
+      before(:each) do
+        ENV["AWS_ACCESS_KEY"] = "foo"
+        ENV["AWS_SECRET_KEY"] = "bar"
+        buckets = [
+          {:bucket => good_bucket, :keys =>keys, :does_exist => true},
+          {:bucket => bad_bucket, :keys=> [], :does_exist => false}]
+        stub_s3(buckets, ENV["AWS_ACCESS_KEY"], ENV["AWS_SECRET_KEY"])
+
+      end
+
+      context "s3:get" do
+        it "should specify an s3 bucket" do 
+          stderr, stdout = execute("s3:get")
+          stderr.should == <<-STDERR
+ !    Usage: mortar s3:get S3_PATH OUTPUT_PATH
+ !    Must specify S3_PATH and OUTPUT_PATH.
+STDERR
+        end
+
+        it "should throw errors at invalid  s3_bucket and fetch urls" do
+          stderr, stdout = execute("s3:get s3://#{bad_bucket} out")
+          stderr.should eq(" !    Requested S3 path, s3://#{bad_bucket}, is invalid. Please ensure that your S3 path begins with 's3://'.  Example: s3://my-bucket/my-key.\n")
+        end
+
+
+        it "should throw errors at invalid  s3_bucket and fetch urls" do
+          stderr, stdout = execute("s3:get #{s3_path_bad} out")
+          stderr.should eq(" !    Requested S3 bucket #{bad_bucket} in path, #{s3_path_bad}, does not exist.\n")
+        end
+
+        it "should have no errors when ran" do
+          any_instance_of(Mortar::S3::S3) do |s3|
+            mock(s3).do_download(good_bucket, key, output_path, nil)
+          end
+          stderr, stdout = execute("s3:get #{s3_path} #{output_path}")
+          stderr.should eq("")
+        end
+      end
+
+      context "s3:getmerge" do
+        it "should have aws-sdk keys properly set for aws-sdk functionality" do
+          any_instance_of(Mortar::S3::S3) do |s3|
+            mock(s3).do_download(good_bucket, key, output_path, true)
+          end
+          stderr, stdout = execute("s3:getmerge #{s3_path} #{output_path}")
+          stderr.should eq("")
+        end
+      end
+    end
+  end
+end
+
+

data/spec/mortar/command/version_spec.rb

@@ -27,7 +27,7 @@ module Mortar::Command
     base_url  = "http://install.mortardata.com"
     base_version = "0.15.1"
     tmp_dir_dumm = "/opt/mortar/installer"
-    curl_command = "sudo mkdir -p #{tmp_dir_dumm} && sudo curl -sS -L -o #{tmp_dir_dumm}/install.sh #{base_url} && sudo bash #{tmp_dir_dumm}/install.sh"
+    curl_command = "echo 'Upgrading will prompt for your sudo password.\n' && sudo mkdir -p #{tmp_dir_dumm} && sudo curl -sS -L -o #{tmp_dir_dumm}/install.sh #{base_url} && sudo bash #{tmp_dir_dumm}/install.sh"
 
     context("version in prod") do
       mortar_install_env = ENV['MORTAR_INSTALL']
@@ -63,7 +63,7 @@ module Mortar::Command
 
     context("version dev") do
       dev_url = "dev_url.com"
-      dev_curl =  "sudo mkdir -p #{tmp_dir_dumm} && sudo curl -sS -L -o #{tmp_dir_dumm}/install.sh #{dev_url} && sudo bash #{tmp_dir_dumm}/install.sh" 
+      dev_curl =  "echo 'Upgrading will prompt for your sudo password.\n' && sudo mkdir -p #{tmp_dir_dumm} && sudo curl -sS -L -o #{tmp_dir_dumm}/install.sh #{dev_url} && sudo bash #{tmp_dir_dumm}/install.sh" 
       before(:each) do
         ENV['MORTAR_INSTALL'] = dev_url
       end

data/spec/mortar/git_spec.rb

@@ -405,7 +405,56 @@ STASH
         end
       end
     end
-    
+
+    context "checking updates for forked projects" do
+      it "skips non forked projects" do
+        with_git_initialized_project do |p|
+          @git.is_fork_repo_updated("mortarcode-dev").should be_false
+        end
+      end
+
+      it "returns true with no .mortar-fork file and new commits" do
+        with_forked_git_project do |p|
+          mock(@git).fetch(@git.fork_base_remote_name)
+          new_commit_hash = "ANewCommitHash"
+          mock(@git).get_latest_hash(@git.fork_base_remote_name) {new_commit_hash}
+          
+          @git.is_fork_repo_updated("mortarcode-dev").should be_true
+          File.exists?(@git.mortar_fork_meta_file).should be_true
+          File.open(@git.mortar_fork_meta_file, "r") do |f|
+            file_contents = f.read()
+            file_contents.strip
+            file_contents.eql?(new_commit_hash).should be_true
+          end
+        end
+      end
+
+      it "returns false with up to date .mortar-fork file and new commits" do
+        with_forked_git_project do |p|
+          mock(@git).fetch(@git.fork_base_remote_name)
+          new_commit_hash = "ANewCommitHash"
+          mock(@git).get_latest_hash(@git.fork_base_remote_name) {new_commit_hash}
+          File.open(@git.mortar_fork_meta_file, "wb") do |f|
+            f.write(new_commit_hash)
+          end
+          File.exists?(@git.mortar_fork_meta_file).should be_true
+          
+          @git.is_fork_repo_updated("mortarcode-dev").should be_false
+        end
+      end
+
+      it "returns false with no .mortar-fork file and no new commits" do
+        with_forked_git_project do |p|
+          mock(@git).fetch(@git.fork_base_remote_name)
+          current_latest_hash = @git.get_latest_hash("")
+          mock(@git).get_latest_hash(@git.fork_base_remote_name) {current_latest_hash}
+          
+          @git.is_fork_repo_updated("mortarcode-dev").should be_false
+        end
+      end
+    end 
+
+      
 =begin
     #TODO: Fix this.
 

data/spec/mortar/s3_spec.rb

@@ -0,0 +1,157 @@
+require 'spec_helper'
+require 'mortar/s3'
+require 's3_faker'
+
+module Mortar
+  describe S3 do
+    before(:each) do
+      @s3 = Mortar::S3::S3.new
+    end
+    context("download s3 commands") do
+      keys = [
+        FakeObject.new("key1", true),
+        FakeObject.new("direc/key2", true),
+        FakeObject.new("direc/key3", true),
+        FakeObject.new("direc/.hidden_key4", true),
+        FakeObject.new("direc/another_direc/key", true)
+      ]
+
+      before(:each) do
+        ENV["AWS_ACCESS_KEY"] = "foo"
+        ENV["AWS_SECRET_KEY"] = "bar"
+        buckets = [
+          {:bucket => "good_bucket", :keys =>keys, :does_exist => true},
+          {:bucket => "bad_bucket", :keys=> [], :does_exist => false}]
+        stub_s3(buckets, ENV["AWS_ACCESS_KEY"], ENV["AWS_SECRET_KEY"])
+
+      end
+
+      context("string functions") do
+        it "should check if the valid path" do
+          bucket, key = @s3.get_bucket_and_key('s3://bucket/key')
+          bucket.should eq('bucket') 
+          key.should eq('key') 
+          bucket, key = @s3.get_bucket_and_key('s3://bucket/deeper/level/key')
+          bucket.should eq('bucket') 
+          key.should eq('deeper/level/key') 
+        end
+
+        it "should throw appropriate error if path is not valid" do
+          bad_bucket = 'bad'
+          previous_stderr, $stderr = $stderr, StringIO.new
+          begin
+            expect { @s3.get_bucket_and_key(bad_bucket) }.to raise_error(SystemExit)
+            $stderr.string.should eq(" !    Requested S3 path, #{bad_bucket}, is invalid. Please ensure that your S3 path begins with 's3://'.  Example: s3://my-bucket/my-key.\n")
+          ensure
+            $stderr = previous_stderr
+          end        
+        end
+
+        it "should throw error if bucket has underscore" do
+          path = "s3://will_fail/key"
+          previous_stderr, $stderr = $stderr, StringIO.new
+          begin
+            expect { @s3.get_bucket_and_key(path) }.to raise_error(SystemExit)
+            $stderr.string.should eq(" !    the scheme s3 does not accept registry part: will_fail (or bad hostname?).\n !    It is strongly suggested that your bucket name does not contain an underscore character.\n !    Please see http://blog.mortardata.com/post/58920122308/s3-hadoop-performance at tip #4.\n")
+          ensure
+            $stderr = previous_stderr
+          end
+        end
+
+        it "should remove / if at the end" do
+          str = "with/"
+          str = @s3.remove_slash(str)
+          str.should eq("with")
+        end
+        it "should not remove / if not at the end" do
+          str = "without"
+          @s3.remove_slash(str)
+          str.should eq("without")
+        end
+
+        it "should get just the file" do
+          key = "folder"
+          path = "folder/file"
+          @s3.is_file(path, key).should eq(true)
+
+        end
+
+        it "should get nothing" do
+          key = "folder"
+          path = "folder/file_$folder$"
+          @s3.is_file(path, key).should eq(false)
+          path = "folder/folder/"
+          @s3.is_file(path, key).should eq(false)
+        end
+
+      end
+
+
+      context("s3 functions") do
+        s3 = nil
+        before (:each) do
+          any_instance_of(Mortar::Local::Controller) do |ctrl|
+            mock(ctrl).require_aws_keys
+          end
+          s3 = @s3.get_s3
+        end
+
+        it "should check if bucket and key exists" do
+          bucket = "good_bucket"
+          key = "key1"
+          @s3.check_bucket(s3, bucket).should eq(true)
+
+
+          bucket = "bad_bucket"
+          key = "bad_key"
+          @s3.check_bucket(s3, bucket).should eq(false)
+        end
+
+        it "should get s3_objects" do
+          bucket = "good_bucket"
+          key = "key1"
+          result = @s3.get_s3_objects(s3, bucket, key)
+          result.length.should eq(1)
+          result[0].key.should eq(key)
+
+          bucket = "good_bucket"
+          key = "direc"
+          result = @s3.get_s3_objects(s3, bucket, key)
+          result.length.should eq(2)
+        end
+
+        it "should return emtpy list if s3_objects are empty" do
+          bucket = "good_bucket"
+          key = "dire"
+          result = @s3.get_s3_objects(s3, bucket, key)
+          result.length.should eq(0)
+        end
+
+        it "should check content that is written" do
+          bucket = "good_bucket"
+          key = "key1"
+          mock(@s3).write_s3_to_file(keys[0], 'output/key1', 'w+')
+          @s3.download_s3(s3, bucket, key, 'output')
+        end
+
+        it "should check all files are written" do
+          bucket = "good_bucket"
+          key = "direc"
+          mock(@s3).write_s3_to_file(keys[1], 'output/key2', 'w+')
+          mock(@s3).write_s3_to_file(keys[2], 'output/key3', 'w+')
+          @s3.download_s3(s3, bucket, key, 'output')
+        end
+
+        it "should concat all files to one and delete out file when already exists" do
+          bucket = "good_bucket"
+          key = "direc"
+          mock(File).file?("output/output"){true}
+          mock(FileUtils).remove("output/output")
+          mock(@s3).write_s3_to_file(keys[1], 'output/output', 'a')
+          mock(@s3).write_s3_to_file(keys[2], 'output/output', 'a')
+          @s3.download_s3(s3, bucket, key, 'output', true)
+        end
+      end
+    end
+  end
+end

data/spec/s3_faker.rb

@@ -0,0 +1,99 @@
+require 'aws-sdk'
+def stub_s3(buckets, access_key_id, secret_access_key)
+  stub(AWS::S3).new( 
+      :access_key_id => access_key_id,
+      :secret_access_key => secret_access_key).returns(FakeS3.new(buckets))
+end
+
+class FakeS3
+  attr_accessor :buckets
+
+  def initialize(buckets)
+    @buckets = FakeS3Collection.new
+    buckets.each do |buck|
+     @buckets.add(buck[:bucket],  FakeBucket.new(buck[:bucket],buck[:keys], buck[:does_exist]))
+    end
+
+  end
+
+end
+
+# class that mocks bucket and s3 object collection
+class FakeS3Collection 
+  include Enumerable
+  attr_accessor :objects
+  def initialize 
+    @objects = {} 
+  end
+  def [](key)
+    if @objects.has_key?(key)
+      return @objects[key]
+    else
+      return FakeObject.new("", false)
+    end
+  end
+
+  def add (key, val)
+    @objects[key] = val
+  end
+  
+  def with_prefix(prefix)
+    res = FakeS3Collection.new
+    @objects.keys.each do |key| 
+      if key.index(prefix) == 0
+        res.add(key, @objects[key])
+      end
+    end
+    return res
+  end
+
+  def each(options = {}, &block)
+    @objects.keys.each do |key|
+      if @objects[key].does_exist
+        block.call(@objects[key])
+      end
+    end
+  end
+
+end
+
+ 
+# Spec class to mock a fake bucket
+class FakeBucket
+  attr_accessor :name, :keys, :does_exist, :objects
+
+  def initialize(name, keys, does_exist)
+    @name = name
+    @keys = keys
+    @objects = FakeS3Collection.new
+    keys.each do |k|
+      @objects.add(k.key,k) 
+    end
+    @does_exist = does_exist
+  end
+  
+
+  def exists?
+    return @does_exist 
+  end
+
+end
+
+
+
+# Spec class to mock a fake s3 object
+class FakeObject
+  attr_accessor :key, :does_exist
+  def initialize(key, does_exist)
+    @key = key
+    @does_exist = does_exist
+  end 
+
+  def exists?
+    @does_exist
+  end
+
+  def read
+    return "Content #{@key}"
+  end
+end

data/spec/spec_helper.rb

@@ -200,6 +200,15 @@ def with_blank_project_with_name(name, &block)
   end
 end
 
+def with_forked_git_project(&block)
+  with_git_initialized_project do |project|
+    git = Mortar::Git::Git.new
+    remote = git.fork_base_remote_name
+    `git remote add #{remote} git@github.com:mortarcode-dev/fork_#{project.name}.git`
+    block.call(project)
+  end
+end
+
 def with_git_initialized_project(&block)
   remote_prefix = "4dbbd83cae8d5bf8a4000000_"
   with_git_initialized_project_with_remote_prefix(remote_prefix, &block)

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: mortar
 version: !ruby/object:Gem::Version
-  version: 0.15.4
-  prerelease: 
+  version: 0.15.5
 platform: ruby
 authors:
 - Mortar Data
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-25 00:00:00.000000000 Z
+date: 2014-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: mortar-api-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: netrc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: parseconfig
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,15 +76,41 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.0
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -102,7 +118,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -110,7 +125,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fakefs
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -118,7 +132,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -126,7 +139,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: gem-release
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -134,7 +146,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -142,23 +153,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 10.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 10.1.1
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -166,7 +174,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -174,7 +181,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -182,7 +188,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -222,6 +227,7 @@ files:
 - lib/mortar/command/pigscripts.rb
 - lib/mortar/command/plugins.rb
 - lib/mortar/command/projects.rb
+- lib/mortar/command/s3.rb
 - lib/mortar/command/validate.rb
 - lib/mortar/command/version.rb
 - lib/mortar/conf/luigi/logging.ini
@@ -244,6 +250,7 @@ files:
 - lib/mortar/pigversion.rb
 - lib/mortar/plugin.rb
 - lib/mortar/project.rb
+- lib/mortar/s3.rb
 - lib/mortar/templates/characterize/README.md
 - lib/mortar/templates/characterize/controlscripts/lib/__init__.py
 - lib/mortar/templates/characterize/controlscripts/lib/characterize_control.py
@@ -304,6 +311,7 @@ files:
 - spec/mortar/command/local_spec.rb
 - spec/mortar/command/pigscripts_spec.rb
 - spec/mortar/command/projects_spec.rb
+- spec/mortar/command/s3_spec.rb
 - spec/mortar/command/validate_spec.rb
 - spec/mortar/command/version_spec.rb
 - spec/mortar/command_spec.rb
@@ -317,32 +325,33 @@ files:
 - spec/mortar/local/python_spec.rb
 - spec/mortar/plugin_spec.rb
 - spec/mortar/project_spec.rb
+- spec/mortar/s3_spec.rb
 - spec/mortar/updater_spec.rb
+- spec/s3_faker.rb
 - spec/spec.opts
 - spec/spec_helper.rb
 - spec/support/display_message_matcher.rb
 homepage: http://mortardata.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Client library and CLI to interact with the Mortar service.
 test_files: []