filbunke 1.8.3 → 1.8.4

data/Rakefile

@@ -7,9 +7,9 @@ begin
     gem.name = "filbunke"
     gem.summary = %Q{Filbunke client}
     gem.description = %Q{Filbunke client and library}
-    gem.email = "wouter@deltaprojects.se"
+    gem.email = "info@deltaprojects.com"
     gem.homepage = "http://github.com/deltaprojects/filbunke"
-    gem.authors = ["Wouter de Bie", "Björn Sperber"]
+    gem.authors = ["Björn Sperber", "Karl Ravn"]
     gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
     gem.files.exclude 'pkg'
     gem.executables = ['filbunked']

data/VERSION

@@ -1 +1 @@
-1.8.3
+1.8.4

data/filbunke.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{filbunke}
-  s.version = "1.8.3"
+  s.version = "1.8.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Wouter de Bie", "Bj\303\266rn Sperber"]
-  s.date = %q{2012-09-18}
+  s.authors = ["Bj\303\266rn Sperber", "Karl Ravn"]
+  s.date = %q{2012-10-01}
   s.default_executable = %q{filbunked}
   s.description = %q{Filbunke client and library}
-  s.email = %q{wouter@deltaprojects.se}
+  s.email = %q{info@deltaprojects.com}
   s.executables = ["filbunked"]
   s.files = [
     "LICENSE",
@@ -38,14 +38,13 @@ Gem::Specification.new do |s|
   ]
   s.homepage = %q{http://github.com/deltaprojects/filbunke}
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.6}
+  s.rubygems_version = %q{1.6.2}
   s.summary = %q{Filbunke client}
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
-    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<thoughtbot-shoulda>, [">= 0"])
       s.add_runtime_dependency(%q<json>, [">= 1.1.7"])
       s.add_runtime_dependency(%q<typhoeus>, ["= 0.3.3"])

data/lib/filbunke/callbacks.rb

@@ -9,5 +9,11 @@ module Filbunke
     def on_update(file, local_file_path)
     end
 
+    def on_no_change(file, local_file_path)
+    end
+    
+    def on_delete(file, local_file_path)
+    end
+
   end
 end

data/lib/filbunke/client.rb

@@ -1,7 +1,7 @@
 module Filbunke
   class Client
 
-    attr_reader :repository, :run_every
+    attr_reader :repository
 
     UPDATES_ACTION = 'updates'
     FILES_ACTION = 'files'
@@ -12,10 +12,9 @@ module Filbunke
     HASH_KEY = 'hash'
 
 
-    def initialize(repository, logger, run_every, callbacks = [], failed_request_log_file_name = nil)
+    def initialize(repository, logger, callbacks = [], failed_request_log_file_name = nil)
       @repository = repository
       @logger = logger
-      @run_every = run_every
       @callbacks = callbacks
       @failed_request_log_file_name = failed_request_log_file_name
     end
@@ -38,7 +37,7 @@ module Filbunke
         local_file_path = ::File.join(repository.local_path, file.path)
         if file.state == "DELETED" then
           delete_file!(local_file_path)
-          
+          run_callbacks_delete(file, local_file_path)
         else
         
           if file_needs_update?(file, local_file_path)
@@ -53,6 +52,7 @@ module Filbunke
 
           else
             @logger.log "File exists with correct hash: #{local_file_path}"
+            run_callbacks_no_change(file, local_file_path)
           end
         end
       end
@@ -144,7 +144,6 @@ module Filbunke
       elsif file.url =~ /^hdfs:\/\//
         success = update_hdfs_file!(file, local_file_path)
         run_callbacks(file, local_file_path) if success
-
       else
         raise "Unsupported protocol for file: #{file.inspect}"
       end
@@ -156,6 +155,18 @@ module Filbunke
       end
     end
 
+    def run_callbacks_no_change(file, local_file_path)
+      @callbacks.each do |callback|
+        callback.on_no_change(file, local_file_path)
+      end
+    end
+
+    def run_callbacks_delete(file, local_file_path)
+      @callbacks.each do |callback|
+        callback.on_delete(file, local_file_path)
+      end
+    end
+
     def file_needs_update?(file, local_file_path)
       return true if file.hash.nil? || file.hash == ""
 
@@ -203,8 +214,6 @@ module Filbunke
           begin
             if response.code.to_i == 200
               write_file!(local_file_path, response.body)
-            elsif response.code.to_i == 404
-              delete_file!(local_file_path)
             else
               @logger.log "Failed to update file #{file.url}, error code = #{response.code}"
               success = false
@@ -254,16 +263,19 @@ module Filbunke
     end
 
     def write_file!(file_path, contents)
-      if ::File.exists?(file_path)
-        @logger.log("Updating: #{file_path}")
-      else
-        @logger.log("Writing: #{file_path}")
-      end
       ::FileUtils.mkdir_p(::File.dirname(file_path))
-      ::File.open(file_path, 'w') do |file|
-        file.write(contents);
-        file.close
-      end
+      @logger.log("Updating: #{file_path}")
+      begin
+        ::File.open("#{file_path}.tmp", 'w') do |file|
+          file.write(contents);
+          file.close
+        end
+        ::FileUtils.mv "#{file_path}.tmp", file_path
+        return true
+      rescue StandardError => e
+        @logger.log "Failed to move file #{file.url}: #{e.message}"
+        return false
+      end  
     end
 
     def delete_file!(file_path)

data/lib/filbunke/daemon.rb

@@ -11,42 +11,6 @@ module Filbunke
     def setup_clients!(local = false)
       @logger = Logger.new(@config["log_file"], local)
       @logger.log("Initializing filbunked")
-
-      if(@config["version"] && @config["version"].to_i == 2)
-        version_2_configure
-      else
-        version_1_configure
-      end
-    end
-
-    def version_2_configure
-      @config["repositories"].each do |repository_name, local_repository_config|
-        @logger.log("Initializing repository (version 2): #{repository_name}")
-        repository_file = File.read("#{@config["repository_files_path"]}#{repository_name}.yml")
-        static_repository_config = YAML.load(repository_file)
-        @clients << begin
-          repository = Repository.new(static_repository_config["filbunke_server_repository"],
-                                      static_repository_config["filbunke_server_host"],
-                                      static_repository_config["filbunke_server_port"],
-                                      local_repository_config["local_path"],
-                                      @config["file_umask"].to_i,
-                                      @config["directory_umask"].to_i,
-                                      static_repository_config["file_url_username"],
-                                      static_repository_config["file_url_password"],
-                                      @config["hadoop_binary"])
-          callbacks = []
-          local_repository_config["callbacks"].each do |callback_name, callback_config|
-            require ::File.join(@config["callback_path"], callback_name.to_s)
-            callback_class =  Module.const_get(callback_name.split("_").map(&:capitalize).join)
-            callbacks << callback_class.new(@logger, callback_config)
-          end if local_repository_config["callbacks"]
-          run_every = local_repository_config["run_every"] ? local_repository_config["run_every"].to_i : @config["run_every"].to_i
-          Client.new(repository, @logger, run_every, callbacks)
-        end
-      end
-    end
-
-    def version_1_configure
       @config["repositories"].each do |repository_name, repository_config|
         @logger.log("Initializing repository: #{repository_name}")
         @clients << begin
@@ -65,32 +29,26 @@ module Filbunke
             callback_class =  Module.const_get(callback_name.split("_").map(&:capitalize).join)
             callbacks << callback_class.new(@logger, callback_config)
           end
-          Client.new(repository, @logger, @config["run_every"].to_i, callbacks)
+          Client.new(repository, @logger, callbacks)
         end
       end
     end
 
-
-
     def run!
       version = ::File.read(::File.expand_path(::File.join(::File.dirname(__FILE__), "../../VERSION"))).chomp
       @logger.log("Starting filbunked version #{version}")
-      counter = 0
       while true
-        counter++
         begin
           @clients.each do |client|
-            if(counter % client.run_every == 0)
-              new_checkpoint = client.update_files!(checkpoint_for_repository(client.repository))
-              update_checkpoint_for_repository(client.repository, new_checkpoint)
-            end
+            new_checkpoint = client.update_files!(checkpoint_for_repository(client.repository))
+            update_checkpoint_for_repository(client.repository, new_checkpoint)
           end
         rescue StandardError => e
           @logger.log("Died.. #{e.message}")
           @logger.log(e.backtrace.join("\n"))
           exit 1
         end
-        sleep 1
+        sleep @config["run_every"]
       end
     end
 

metadata

@@ -1,30 +1,33 @@
 --- !ruby/object:Gem::Specification 
 name: filbunke
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 63
+  prerelease: 
   segments: 
   - 1
   - 8
-  - 3
-  version: 1.8.3
+  - 4
+  version: 1.8.4
 platform: ruby
 authors: 
-- Wouter de Bie
 - "Bj\xC3\xB6rn Sperber"
+- Karl Ravn
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2012-09-18 00:00:00 +02:00
+date: 2012-10-01 00:00:00 +02:00
 default_executable: filbunked
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: thoughtbot-shoulda
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -34,9 +37,11 @@ dependencies:
   name: json
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 29
         segments: 
         - 1
         - 1
@@ -48,9 +53,11 @@ dependencies:
   name: typhoeus
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
+        hash: 21
         segments: 
         - 0
         - 3
@@ -62,9 +69,11 @@ dependencies:
   name: open4
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 21
         segments: 
         - 1
         - 0
@@ -73,7 +82,7 @@ dependencies:
   type: :runtime
   version_requirements: *id004
 description: Filbunke client and library
-email: wouter@deltaprojects.se
+email: info@deltaprojects.com
 executables: 
 - filbunked
 extensions: []
@@ -111,23 +120,27 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Filbunke client