filbunke 1.2.1 → 1.3.0

data/VERSION

@@ -1 +1 @@
-1.2.1
+1.3.0

data/doc/examples/filbunke_config.yml

@@ -15,7 +15,8 @@ repositories:
         file_url_password: 'contentpassword'
         file_umask: 0644
         directory_umask: 0755
+        hadoop_binary: '/usr/bin/hadoop'
         callbacks:
           http_evict_cache:
             host: 'localhost'
-            port: 3010
+            port: 3010    

data/filbunke.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{filbunke}
-  s.version = "1.2.1"
+  s.version = "1.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Wouter de Bie"]
-  s.date = %q{2010-11-18}
+  s.date = %q{2010-11-19}
   s.default_executable = %q{filbunked}
   s.description = %q{Filbunke client and library}
   s.email = %q{wouter@deltaprojects.se}

data/lib/filbunke/client.rb

@@ -32,25 +32,25 @@ module Filbunke
 
         local_file_path = ::File.join(repository.local_path, file.path)
         if file_needs_update?(file, local_file_path)
-          if file.url =~ /^http:\/\//
-            if update_http_file!(file, local_file_path) then
-              @callbacks.each do |callback|
-                callback.on_update(file)
-              end
-              yield file
-            else
-              @logger.log "Unable to get file #{file.url} ==> #{file.path}!"
-              failure = true
+          
+          if update_file!(file, local_file_path) then
+            @callbacks.each do |callback|
+              callback.on_update(file)
             end
+            yield file
           else
-            raise "Unsupported protocol for file: #{file.inspect}"
+            @logger.log "Unable to get file #{file.url} ==> #{file.path}!"
+            failure = true
           end
+
         else
           @logger.log "File exists with correct hash: #{local_file_path}"
         end
       end
       failure ? last_checkpoint : (new_checkpoint || last_checkpoint)
     end
+    
+
 
     def update_files!(last_checkpoint)
       with_updated_files(last_checkpoint) {}
@@ -108,6 +108,22 @@ module Filbunke
     
     private
 
+    def update_file!(file, local_file_path)
+        
+      if file.state == "DELETED" then
+        delete_file!(local_file_path)
+        return true
+      end
+      
+      if file.url =~ /^http:\/\//
+        update_http_file!(file, local_file_path)
+      elsif file.url =~ /^hdfs:\/\//
+        update_hdfs_file!(file, local_file_path)
+      else
+        raise "Unsupported protocol for file: #{file.inspect}"
+      end
+    end
+
     def file_needs_update?(file, local_file_path)
       return true if file.hash.nil? || file.hash == ""
 
@@ -144,13 +160,7 @@ module Filbunke
     end
 
     def update_http_file!(file, local_file_path)
-      begin
-        
-        if file.state == "DELETED" then
-          delete_file!(local_file_path)
-          return true
-        end
-        
+      begin 
         uri = URI.parse(file.url)
         file_http=Net::HTTP.new(uri.host, uri.port)
         file_http.start do |http|
@@ -171,6 +181,26 @@ module Filbunke
         return false
       end
     end
+    
+    def update_hdfs_file!(file, local_file_path)
+      begin
+        ::FileUtils.mkdir_p(::File.dirname(local_file_path))
+      
+        hdfs_cmd = "#{@repository.hadoop_binary} fs -copyToLocal #{file.url} #{local_file_path}.tmp &> /dev/null && mv #{local_file_path}.tmp #{local_file_path}"
+
+        system hdfs_cmd
+      
+        if $?.exitstatus == 0 then
+          return true
+        else
+          @logger.log "Failed to update file #{file.url}!"
+          return false
+        end
+      rescue StandardError => e
+        @logger.log "Failed to update file #{file.url}: #{e.message}"
+        return false
+      end
+    end
 
     def write_file!(file_path, contents)
       if ::File.exists?(file_path)

data/lib/filbunke/daemon.rb

@@ -21,7 +21,8 @@ module Filbunke
                                       repository_config["file_umask"].to_i,
                                       repository_config["directory_umask"].to_i,
                                       repository_config["file_url_username"],
-                                      repository_config["file_url_password"])
+                                      repository_config["file_url_password"],
+                                      repository_config["hadoop_binary"])
           callbacks = []
           repository_config["callbacks"].each do |callback_name, callback_config|
             require ::File.join(@config["callback_path"], callback_name.to_s)

data/lib/filbunke/repository.rb

@@ -1,9 +1,9 @@
 module Filbunke
   class Repository
 
-    attr_accessor :name, :host, :port, :local_path, :file_umask, :directory_umask, :user, :pass
+    attr_accessor :name, :host, :port, :local_path, :file_umask, :directory_umask, :user, :pass, :hadoop_binary
 
-    def initialize(name, host, port, local_path, file_umask, directory_umask, user = nil, pass = nil)
+    def initialize(name, host, port, local_path, file_umask, directory_umask, user = nil, pass = nil, hadoop_binary = nil)
       @name = name
       @host = host
       @port = port
@@ -12,6 +12,7 @@ module Filbunke
       @directory_umask = directory_umask
       @user = user
       @pass = pass
+      @hadoop_binary = hadoop_binary
     end
 
   end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 2
-  - 1
-  version: 1.2.1
+  - 3
+  - 0
+  version: 1.3.0
 platform: ruby
 authors: 
 - Wouter de Bie
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-18 00:00:00 +01:00
+date: 2010-11-19 00:00:00 +01:00
 default_executable: filbunked
 dependencies: 
 - !ruby/object:Gem::Dependency