mobilize-hdfs 1.0.06 → 1.0.07

data/README.md

@@ -228,16 +228,16 @@ same cluster as your first.
 Meta
 ----
 
-* Code: `git clone git://github.com/ngmoco/mobilize-hdfs.git`
-* Home: <https://github.com/ngmoco/mobilize-hdfs>
-* Bugs: <https://github.com/ngmoco/mobilize-hdfs/issues>
+* Code: `git clone git://github.com/dena/mobilize-hdfs.git`
+* Home: <https://github.com/dena/mobilize-hdfs>
+* Bugs: <https://github.com/dena/mobilize-hdfs/issues>
 * Gems: <http://rubygems.org/gems/mobilize-hdfs>
 
 <a name='section_Author'></a>
 Author
 ------
 
-Cassio Paes-Leme :: cpaesleme@ngmoco.com :: @cpaesleme
+Cassio Paes-Leme :: cpaesleme@dena.com :: @cpaesleme
 
-[mobilize-base]: https://github.com/ngmoco/mobilize-base
-[mobilize-ssh]: https://github.com/ngmoco/mobilize-ssh
+[mobilize-base]: https://github.com/dena/mobilize-base
+[mobilize-ssh]: https://github.com/dena/mobilize-ssh

data/lib/mobilize-hdfs/handlers/hdfs.rb

@@ -37,7 +37,7 @@ module Mobilize
       cluster, cluster_path = Hdfs.resolve_path(path)
       gateway_node = Hadoop.gateway_node(cluster)
       #need to direct stderr to dev null since hdfs throws errors at being headed off
-      command = "((#{Hadoop.exec_path(cluster)} fs -fs '#{Hdfs.namenode_path(path)}' -cat #{cluster_path}"
+      command = "((#{Hadoop.exec_path(cluster)} fs -fs '#{Hdfs.root(cluster)}' -cat '#{cluster_path}'"
       command += " | head -c #{Hadoop.read_limit}) > out.txt 2> /dev/null) && cat out.txt"
       response = Ssh.run(gateway_node,command,user)
       if response.length==Hadoop.read_limit
@@ -53,7 +53,8 @@ module Mobilize
       elsif Hadoop.clusters.keys.include?(path.split("/").first)
         return path.split("/").ie{|p| [p.first,"/#{p[1..-1].join("/")}"]}
       else
-        return [nil,nil]
+        #default cluster, slash on the front
+        return [Hadoop.output_cluster,"/#{path.to_s}"]
       end
     end
 
@@ -111,18 +112,21 @@ module Mobilize
       source_path = params['source']
       target_path = params['target']
       user = params['user']
-      #check for source in hdfs format
-      source_cluster, source_cluster_path = Hdfs.resolve_path(source_path)
-      if source_cluster.nil?
-        #not hdfs
+      begin
+        #check for source in gsheet format
         gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
         #return blank response if there are no slots available
         return nil unless gdrive_slot
         source_dst = s.source_dsts(gdrive_slot).first
+        in_string = source_dst.read(user)
         Gdrive.unslot_worker_by_path(stage_path)
-      else
+      rescue
+        #try hdfs
+        source_cluster, source_cluster_path = Hdfs.resolve_path(source_path)
         source_path = "#{source_cluster}#{source_cluster_path}"
         source_dst = Dataset.find_or_create_by_handler_and_path("hdfs",source_path)
+        in_string = source_dst.read(user)
+        raise "No data found at hdfs://#{source_path}" unless in_string.to_s.length>0
       end
 
       #determine cluster for target
@@ -137,7 +141,6 @@ module Mobilize
       end
 
       target_path = "#{target_cluster}#{target_cluster_path}"
-      in_string = source_dst.read(user)
       out_string = Hdfs.write(target_path,in_string,user)
 
       out_url = "hdfs://#{Hadoop.output_cluster}#{Hadoop.output_dir}hdfs/#{stage_path}/out"

data/lib/mobilize-hdfs/version.rb

@@ -1,5 +1,5 @@
 module Mobilize
   module Hdfs
-    VERSION = "1.0.06"
+    VERSION = "1.0.07"
   end
 end

data/mobilize-hdfs.gemspec

@@ -10,11 +10,11 @@ Gem::Specification.new do |gem|
   gem.email         = ["cpaesleme@ngmoco.com"]
   gem.description   = %q{Adds hdfs read, write, and copy support to mobilize-ssh}
   gem.summary       = %q{Adds hdfs read, write, and copy support to mobilize-ssh}
-  gem.homepage      = "http://github.com/ngmoco/mobilize-hdfs"
+  gem.homepage      = "http://github.com/dena/mobilize-hdfs"
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
-  gem.add_runtime_dependency "mobilize-ssh","1.1.06"
+  gem.add_runtime_dependency "mobilize-ssh","1.1.07"
 end

data/test/hdfs_job_rows.yml

@@ -5,7 +5,7 @@
   stage1: hdfs.write target:"/user/mobilize/test/test_hdfs_1.out", 
             source:"Runner_mobilize(test)/test_hdfs_1.in"
   stage2: hdfs.copy source:"/user/mobilize/test/test_hdfs_1.out", 
-            target:"test_cluster_2/user/mobilize/test/test_hdfs_copy.out",
-  stage3: hdfs.read source:"/user/mobilize/test/test_hdfs_1_copy.out"
+            target:"test_cluster_2/user/mobilize/test/test_hdfs_1_copy.out",
+  stage3: hdfs.read source:"test_cluster_2/user/mobilize/test/test_hdfs_1_copy.out"
   stage4: gsheet.write source:"hdfs://test_cluster_2/user/mobilize/test/test_hdfs_1_copy.out", 
             target:"Runner_mobilize(test)/test_hdfs_1_copy.out"

data/test/mobilize-hdfs_test.rb

@@ -22,7 +22,7 @@ describe "Mobilize" do
 
     puts "add test_source data"
     hdfs_1_sheet = Mobilize::Gsheet.find_or_create_by_path("#{r.path.split("/")[0..-2].join("/")}/test_hdfs_1.in",gdrive_slot)
-    hdfs_1_tsv = ([(["test"]*10).join("\t")]*10).join("\n")
+    hdfs_1_tsv = ([%w{test0 test1 test2 test3 test4 test5 test6 test7 test8 test9}.join("\t")]*10).join("\n")
     hdfs_1_sheet.write(hdfs_1_tsv,u.name)
 
     jobs_sheet = r.gsheet(gdrive_slot)
@@ -41,7 +41,7 @@ describe "Mobilize" do
     puts "jobtracker posted data to test sheet"
     test_destination_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/test_hdfs_1_copy.out",gdrive_slot)
 
-    assert test_destination_sheet.to_tsv.length == 499
+    assert test_destination_sheet.read(u.name).length == 599
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mobilize-hdfs
 version: !ruby/object:Gem::Version
-  version: 1.0.06
+  version: 1.0.07
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-04 00:00:00.000000000 Z
+date: 2013-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mobilize-ssh
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.06
+        version: 1.1.07
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.1.06
+        version: 1.1.07
 description: Adds hdfs read, write, and copy support to mobilize-ssh
 email:
 - cpaesleme@ngmoco.com
@@ -50,7 +50,7 @@ files:
 - test/mobilize-hdfs_test.rb
 - test/redis-test.conf
 - test/test_helper.rb
-homepage: http://github.com/ngmoco/mobilize-hdfs
+homepage: http://github.com/dena/mobilize-hdfs
 licenses: []
 post_install_message: 
 rdoc_options: []