filbunke 0.3.2 → 1.0.0

data/VERSION

@@ -1 +1 @@
-0.3.2
+1.0.0

data/bin/filbunked

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'filbunke'
+require 'yaml'
+require 'getoptlong'
+
+raw_opts = GetoptLong.new(
+        [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+        [ '--config', '-c', GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+opts = {}
+raw_opts.each do |opt, arg|
+  opts[opt] = arg
+end
+
+if opts['--help'] || opts['--config'].blank?
+  puts "Usage: filbunked -c <config_file.yml>"
+  exit 1
+else
+  config_file = File.read(opts['--config'])
+  config = YAML.load(config_file)
+  daemon = Filbunke::Daemon.new(config)
+  Process::UID.grant_privilege(Etc.getpwnam(config["user"]).uid)
+  daemon.run!
+end

data/doc/examples/filbunke_config.yml

@@ -0,0 +1,21 @@
+---
+pid_file: '/var/run/filbunked.pid'
+user: 'bjorns'
+log_file: '/tmp/filbunke.log'
+checkpoint_path: '/var/tmp/filbunke_checkpoints/'
+run_every: 5
+callback_path: '/Users/bjorns/Documents/Development/filbunke-cdn/client/doc/examples/'
+repositories:
+    cache:
+        local_path: '/v5/cachetest'
+        filbunke_server_host: 'localhost'
+        filbunke_server_port: 8080
+        filbunke_server_repository: 'cache'
+        file_url_username: 'content'
+        file_url_password: 'JeSuisContent#1'
+        file_umask: 0644
+        directory_umask: 0755
+        callbacks:
+          http_evict_cache:
+            host: 'localhost'
+            port: 3010

data/doc/examples/http_evict_cache.rb

@@ -0,0 +1,22 @@
+require 'net/http'
+
+class HttpEvictCache < Filbunke::Callbacks
+
+  def on_update(file)
+    begin
+      evict_http = Net::HTTP.new(@config["host"], @config["port"].to_i)
+      evict_http.start do |http|
+        evict_path = "/cache/evict/#{file.path.chomp('.yml')}"
+        request = Net::HTTP::Get.new(evict_path)
+        response = http.request(request)
+        if response.code.to_i != 200
+          @logger.log "Failed to evict cache entry: #{file.path}"
+        end
+      end
+    rescue StandardError => e
+      @logger.log("Failed to evict cache entry: #{file.path}")
+      @logger.log(e.backtrace.join("\n"))
+    end
+  end
+
+end

data/filbunke.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{filbunke}
-  s.version = "0.3.2"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Wouter de Bie"]
-  s.date = %q{2010-11-03}
+  s.date = %q{2010-11-10}
   s.description = %q{Filbunke client and library}
   s.email = %q{wouter@deltaprojects.se}
   s.files = [
@@ -18,10 +18,16 @@ Gem::Specification.new do |s|
      "Rakefile",
      "VERSION",
      "bin/.gitignore",
+     "bin/filbunked",
+     "doc/examples/filbunke_config.yml",
+     "doc/examples/http_evict_cache.rb",
      "filbunke.gemspec",
      "lib/filbunke.rb",
+     "lib/filbunke/callbacks.rb",
      "lib/filbunke/client.rb",
+     "lib/filbunke/daemon.rb",
      "lib/filbunke/file.rb",
+     "lib/filbunke/logger.rb",
      "lib/filbunke/repository.rb",
      "test/helper.rb",
      "test/test_filbunke.rb"

data/lib/filbunke/callbacks.rb

@@ -0,0 +1,13 @@
+module Filbunke
+  class Callbacks
+
+    def initialize(logger, config = {})
+      @config = config
+      @logger = logger
+    end
+
+    def on_update(file)
+    end
+
+  end
+end

data/lib/filbunke/client.rb

@@ -1,4 +1,5 @@
 require 'active_support'
+require 'active_support/all' rescue nil
 require 'json'
 require 'net/http'
 require 'fileutils'
@@ -6,24 +7,31 @@ require 'fileutils'
 module Filbunke
   class Client
 
+    attr_reader :repository
+
     UPDATES_ACTION = 'updates'
     FILES_ACTION = 'files'
     URL_KEY = 'url'
     FROM_CHECKPOINT_KEY = 'from_checkpoint'
     HASH_KEY = 'hash'
 
-    def initialize(repository)
+    def initialize(repository, logger, callbacks = [])
       @repository = repository
+      @logger = logger
+      @callbacks = callbacks
     end
 
-    def with_updated_files(last_checkpoint, local_path)
+    def with_updated_files(last_checkpoint)
       updates = get_updated_file_list(last_checkpoint)
       updated_files = updates["files"] || []
       updated_files.each do |raw_file|
         file = File.new(raw_file)
         if file.url.start_with?("http://")
-          local_file_path = File.join(local_path, file.path)
+          local_file_path = ::File.join(repository.local_path, file.path)
           update_http_file!(file, local_file_path)
+          @callbacks.each do |callback|
+            callback.on_update(file)
+          end
           yield file
         else
           raise "Unsupported protocol for file: #{file.inspect}"
@@ -32,6 +40,10 @@ module Filbunke
       updates["checkpoint"] || last_checkpoint
     end
 
+    def update_files!(last_checkpoint)
+      with_updated_files(last_checkpoint) {}
+    end
+
     def register_updated_file!(path, url, hash = nil)
       register_http = Net::HTTP.new(@repository.host, @repository.port)
       register_http.start do |http|
@@ -91,15 +103,16 @@ module Filbunke
     end
 
     def write_file!(file_path, contents)
-      FileUtils.mkdir_p(File.dirname(file_path))
-      File.open(file_path, 'w') do |file|
+      @logger.log("Writing: #{file_path}")
+      ::FileUtils.mkdir_p(::File.dirname(file_path))
+      ::File.open(file_path, 'w') do |file|
         file.write(contents);
         file.close
       end
     end
 
     def delete_file!(file_path)
-      File.delete(file_path)
+      ::File.delete(file_path)
     end
 
   end

data/lib/filbunke/daemon.rb

@@ -0,0 +1,76 @@
+module Filbunke
+  class Daemon
+
+    def initialize(config)
+      @config = config
+      @clients = []
+      setup_clients!
+      write_pid!(config["pid_file"])
+    end
+
+    def setup_clients!
+      @logger = Logger.new(@config["log_file"])
+      @logger.log("Initializing filbunked")
+      @config["repositories"].each do |repository_name, repository_config|
+        @logger.log("Initializing repository: #{repository_name}")
+        @clients << begin
+          repository = Repository.new(repository_config["filbunke_server_repository"],
+                                      repository_config["filbunke_server_host"],
+                                      repository_config["filbunke_server_port"],
+                                      repository_config["local_path"],
+                                      repository_config["file_umask"].to_i,
+                                      repository_config["directory_umask"].to_i,
+                                      repository_config["file_url_username"],
+                                      repository_config["file_url_password"])
+          callbacks = []
+          repository_config["callbacks"].each do |callback_name, callback_config|
+            require ::File.join(@config["callback_path"], callback_name.to_s)
+            callback_class = callback_name.to_s.camelize.constantize
+            callbacks << callback_class.new(@logger, callback_config)
+          end
+          Client.new(repository, @logger, callbacks)
+        end
+      end
+    end
+
+    def run!
+      @logger.log("Starting filbunked")
+      while true
+        @clients.each do |client|
+          @logger.log("Updating repository: #{client.repository.name}")
+          new_checkpoint = client.update_files!(checkpoint_for_repository(client.repository))
+          update_checkpoint_for_repository(client.repository, new_checkpoint)
+        end
+        sleep @config["run_every"]
+      end
+    end
+
+    private
+
+    def checkpoint_for_repository(repository)
+      @logger.log("Getting checkpoint for: #{repository.name}")
+      ::File.open(checkpoint_file_name_for_repository(repository), 'r') {|f| f.readline.to_i } rescue 0
+    end
+
+    def update_checkpoint_for_repository(repository, checkpoint)
+      @logger.log("Updating checkpoint for: #{repository.name}")
+      f = ::File.new(checkpoint_file_name_for_repository(repository), 'w', repository.file_umask)
+      f.write(checkpoint)
+      f.close
+    end
+
+    def checkpoint_file_name_for_repository(repository)
+      FileUtils.mkdir_p(@config["checkpoint_path"], :mode => repository.directory_umask)
+      ::File.join(@config["checkpoint_path"], repository.name + ".checkpoint")
+    end
+
+    def write_pid!(pid_file_path)
+      ::File.open(pid_file_path, 'w') do |f|
+        f.write(Process.pid.to_i)
+        f.close
+      end
+    end
+
+  end
+
+end

data/lib/filbunke/logger.rb

@@ -0,0 +1,14 @@
+module Filbunke
+  class Logger
+
+    def initialize(log_file_name)
+      @log_file = ::File.open(log_file_name, "a")
+    end
+
+    def log(msg)
+      @log_file.write("#{Time.now}: #{msg}\n")
+      @log_file.flush
+    end
+
+  end
+end

data/lib/filbunke/repository.rb

@@ -1,12 +1,15 @@
 module Filbunke
   class Repository
 
-    attr_accessor :host, :port, :name, :user, :pass
+    attr_accessor :name, :host, :port, :local_path, :file_umask, :directory_umask, :user, :pass
 
-    def initialize(name, host, port, user = nil, pass = nil)
+    def initialize(name, host, port, local_path, file_umask, directory_umask, user = nil, pass = nil)
       @name = name
       @host = host
       @port = port
+      @local_path = local_path
+      @file_umask = file_umask
+      @directory_umask = directory_umask
       @user = user
       @pass = pass
     end

data/lib/filbunke.rb

@@ -1,3 +1,6 @@
 require 'filbunke/client.rb'
 require 'filbunke/file.rb'
-require 'filbunke/repository.rb'
+require 'filbunke/repository.rb'
+require 'filbunke/callbacks.rb'
+require 'filbunke/daemon.rb'
+require 'filbunke/logger.rb'

metadata

@@ -3,10 +3,10 @@ name: filbunke
 version: !ruby/object:Gem::Version 
   prerelease: false
   segments: 
+  - 1
   - 0
-  - 3
-  - 2
-  version: 0.3.2
+  - 0
+  version: 1.0.0
 platform: ruby
 authors: 
 - Wouter de Bie
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-03 00:00:00 +01:00
+date: 2010-11-10 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -57,10 +57,16 @@ files:
 - Rakefile
 - VERSION
 - bin/.gitignore
+- bin/filbunked
+- doc/examples/filbunke_config.yml
+- doc/examples/http_evict_cache.rb
 - filbunke.gemspec
 - lib/filbunke.rb
+- lib/filbunke/callbacks.rb
 - lib/filbunke/client.rb
+- lib/filbunke/daemon.rb
 - lib/filbunke/file.rb
+- lib/filbunke/logger.rb
 - lib/filbunke/repository.rb
 - test/helper.rb
 - test/test_filbunke.rb