lacquer 0.4.0 → 0.6.6

data/lib/lacquer/cache_control.rb

@@ -0,0 +1,75 @@
+module Lacquer
+  def self.cache_control
+    @cache_control ||= CacheControl.new
+  end
+
+  class CacheControl
+    include Lacquer::CacheUtils
+    attr_accessor :store
+
+    def initialize
+      self.store = []
+    end
+
+    def register(group, options = {})
+      options[:group]  = group
+      options[:args]   = Array(options[:args]).compact
+      store            << options
+    end
+
+    def configure
+      yield self
+    end
+
+    def purge(group, *args)
+      clear_cache_for(*urls_for(group, *args))      
+    end
+
+    def urls_for(group, *args)
+      args.map! { |arg| arg.to_param }
+      urls_by(group).map { |options| options[:url] % args }
+    end
+
+    def to_vcl_conditions(urls = store)
+      urls.map { |opt| %Q[req.url ~ "#{(opt[:url] % opt[:args])}"] }.join(" || ")
+    end
+
+    def to_vcl_override_ttl_urls
+      urls_grouped_by_expires.map do |expires_in, list|
+        <<-CODE.gsub(/^[ \t]{4}*/, '')
+        if(#{to_vcl_conditions(list)}) {
+          unset beresp.http.Set-Cookie;
+          set beresp.ttl = #{expires_in};
+          return(deliver);
+        }
+        CODE
+      end.join("\n")
+    end
+
+    def to_vcl_pass_urls
+      <<-CODE.gsub(/^[ \t]{4}*/, '')
+      if(#{to_vcl_conditions(urls_by(:pass))}) {
+        return(pass);
+      }
+      CODE
+    end
+
+    def to_vcl_pipe_urls
+      <<-CODE.gsub(/^[ \t]{4}*/, '')
+      if(#{to_vcl_conditions(urls_by(:pipe))}) {
+        return(pipe);
+      }
+      CODE
+    end
+
+    protected
+
+    def urls_grouped_by_expires
+      store.group_by { |opt| opt[:expires_in] }.select { |expires_in, list| expires_in }
+    end
+
+    def urls_by(group)
+      store.select { |opt| opt[:group] == group }
+    end
+  end
+end

data/lib/lacquer/cache_utils.rb

@@ -26,20 +26,22 @@ module Lacquer
     #
     # clear_cache_for(root_path, blog_posts_path, '/other/content/*')
     def clear_cache_for(*paths)
-      paths.each do |path|
-        case Lacquer.configuration.job_backend
-        when :delayed_job
-          require 'lacquer/delayed_job_job'
-          Delayed::Job.enqueue(Lacquer::DelayedJobJob.new(path))
-        when :resque
-          require 'lacquer/resque_job'
-          Resque.enqueue(Lacquer::ResqueJob, path)
-        when :none
-          Varnish.new.purge(path)
-        end
+      return unless Lacquer.configuration.enable_cache
+      case Lacquer.configuration.job_backend
+      when :delayed_job
+        require 'lacquer/delayed_job_job'
+        Delayed::Job.enqueue(Lacquer::DelayedJobJob.new(paths))
+      when :resque
+        require 'lacquer/resque_job'
+        Resque.enqueue(Lacquer::ResqueJob, paths)
+      when :sidekiq
+        require 'lacquer/sidekiq_worker'
+        Lacquer::SidekiqWorker.perform_async(paths)
+      when :none
+        Varnish.new.purge(*paths)
       end
     end
-    
+
     # Sends cache control headers with page.
     # These are the headers that varnish responds to
     # to set cache properly.

data/lib/lacquer/capistrano/v2/hooks.rb

@@ -0,0 +1,21 @@
+# Capistrano tasks for Lacquer
+
+Capistrano::Configuration.instance(:must_exit).load do
+  _cset(:lacquer_roles) { :web }
+
+  after "deploy:web:disable", "lacquer:global_purge"
+  after "deploy:web:enable", "lacquer:global_purge"
+  after "deploy:rollback", "lacquer:global_purge"
+  after "deploy:rollback", "lacquer:restart"
+  after "deploy:update", "lacquer:restart"
+
+  namespace :lacquer do
+    %w( start stop restart global_purge status ).each do |name|
+      desc "#{name} varnish"
+      task name.to_sym, :roles => lacquer_roles do
+        next if find_servers_for_task(current_task).empty?
+        run "cd #{current_path} && #{rake} lacquer:varnishd:#{name}"
+      end
+    end
+  end
+end

data/lib/lacquer/capistrano/v3/tasks/lacquer.rake

@@ -0,0 +1,24 @@
+namespace :load do
+  task :defaults do
+    set :lacquer_roles, -> { :web }
+  end
+end
+
+namespace :lacquer do
+  %w( start stop restart purge global_purge status ).each do |name|
+    desc "#{name} varnish"
+    task name.to_sym do
+      on roles(fetch(:lacquer_roles)) do
+        within release_path do
+          with rails_env: fetch(:rails_env) do
+            execute :bundle, "exec rake", "lacquer:varnishd:#{name}"
+          end
+        end
+      end
+    end
+  end
+
+  after "deploy:rollback", "lacquer:global_purge"
+  after "deploy:rollback", "lacquer:restart"
+  after "deploy:updated", "lacquer:restart"
+end

data/lib/lacquer/capistrano.rb

@@ -0,0 +1,7 @@
+require 'capistrano/version'
+
+if defined?(Capistrano::VERSION) && Gem::Version.new(Capistrano::VERSION).release >= Gem::Version.new('3.0.0')
+  load File.expand_path("../capistrano/v3/tasks/lacquer.rake", __FILE__)
+else
+  require 'lacquer/capistrano/v2/hooks'
+end

data/lib/lacquer/configuration.rb

@@ -20,6 +20,15 @@ module Lacquer
     # Error handler
     attr_accessor :command_error_handler
 
+    # Purge Command
+    attr_accessor :purge_command
+
+    # Pass Command (in vcl_fetch)
+    attr_accessor :pass_command
+
+    # Use sudo for start up
+    attr_accessor :use_sudo
+
     def initialize
       @enable_cache = true
       @varnish_servers = []
@@ -27,6 +36,9 @@ module Lacquer
       @job_backend = :none
       @retries = 5
       @command_error_handler = nil
+      @purge_command = "url.purge"
+      @pass_command = "pass"
+      @use_sudo = false
     end
 
     # Returns a hash of all configurable options

data/lib/lacquer/delayed_job_job.rb

@@ -1,7 +1,7 @@
 module Lacquer
-  class DelayedJobJob < Struct.new(:url)
+  class DelayedJobJob < Struct.new(:urls)
     def perform
-      Varnish.new.purge(url)
+      Varnish.new.purge(*urls)
     end
   end
 end

data/lib/lacquer/railtie.rb

@@ -0,0 +1,9 @@
+require 'lacquer'
+require 'rails'
+module Lacquer
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'lacquer/tasks.rb'
+    end
+  end
+end

data/lib/lacquer/recipes.rb

@@ -0,0 +1,62 @@
+# Capistrano Recipes for managing varnishd
+#
+# Add these callbacks to have the varnishd process restart when the server
+# is restarted:
+#
+#   after "deploy:stop",    "delayed_job:stop"
+#   after "deploy:start",   "delayed_job:start"
+#   after "deploy:restart", "delayed_job:restart"
+#
+# If you've got varnishd running on a servers, you can also specify
+# which servers have varnishd running and should be restarted after deploy.
+#
+#   set :varnishd_role, :varnish
+#
+
+Capistrano::Configuration.instance.load do
+  namespace :varnishd do
+    def rails_env
+      fetch(:rails_env, false) ? "RAILS_ENV=#{fetch(:rails_env)}" : ''
+    end
+
+    def rake
+      fetch(:rake, 'rake')
+    end
+
+    def roles
+      fetch(:varnishd_role, :varnish)
+    end
+
+    desc "Link varnishd.yml from shared/config to app/config"
+    task :link_varnishd_yml, :roles => lambda { roles } do
+      run "cd #{release_path} && ln -nfs #{shared_path}/config/varnishd.yml  #{release_path}/config/varnishd.yml"
+    end
+    after 'deploy:update_code', 'varnishd:link_varnishd_yml'
+
+    desc "Link varnish.vcl from shared/config to app/config"
+    task :link_varnish_vcl, :roles => lambda { roles } do
+      run "cd #{release_path} && ln -nfs #{shared_path}/config/varnish.vcl  #{release_path}/config/varnish.vcl"
+    end
+    after 'deploy:update_code', 'varnishd:link_varnishd_vcl'
+
+    desc "Purge ALL urls from Varnish"
+    task :global_purge, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{rake} varnishd:global_purge"
+    end
+
+    desc "Stop the varnishd process"
+    task :stop, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{rake} varnishd:stop"
+    end
+
+    desc "Start the varnishd process"
+    task :start, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{rake} varnishd:start"
+    end
+
+    desc "Restart the delayed_job process"
+    task :restart, :roles => lambda { roles } do
+      run "cd #{current_path};#{rails_env} #{rake} varnishd:restart"
+    end
+  end
+end

data/lib/lacquer/resque_job.rb

@@ -2,8 +2,8 @@ module Lacquer
   class ResqueJob
     @queue = :lacquer
 
-    def self.perform(url)
-      Varnish.new.purge(url)
+    def self.perform(urls)
+      Varnish.new.purge(*urls)
     end
   end
 end

data/lib/lacquer/sidekiq_worker.rb

@@ -0,0 +1,11 @@
+module Lacquer
+  class SidekiqWorker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: :lacquer
+
+    def perform(urls)
+      Varnish.new.purge(*urls)
+    end
+  end
+end

data/lib/lacquer/tasks.rb

@@ -0,0 +1,48 @@
+namespace :lacquer do
+  namespace :varnishd do
+    desc "Start varnishd daemon using Lacquer's settings"
+    task :start => :environment do
+      Lacquer::Varnishd.new.start
+    end
+
+    desc "Stop varnishd daemon using Lacquer's settings"
+    task :stop => :environment do
+      Lacquer::Varnishd.new.stop
+    end
+
+    desc "Running status of varnishd daemon using Lacquer's settings"
+    task :status => :environment do
+      if Lacquer::Varnishd.new.running?
+        puts "Varnishd is running"
+      else
+        puts "Varnishd is not running"
+      end
+    end
+
+    desc "Restart varnishd daemon using Lacquer's settings"
+    task :restart => :environment do
+      varnishd = Lacquer::Varnishd.new
+      if varnishd.running?
+        varnishd.stop
+        sleep(1)
+      end
+      varnishd.start
+    end
+
+    desc "Reload VCL configuration through varnishadm with Lacquer's settings"
+    task :reload => :environment do
+      varnishd = Lacquer::Varnishd.new
+      varnishd.reload
+    end
+
+    desc "Purge a urls from Varnish"
+    task :purge => :environment do
+      Lacquer::Varnish.new.purge(ENV['PURGE'].to_s)
+    end
+
+    desc "Purge ALL urls from Varnish"
+    task :global_purge => :environment do
+      Lacquer::Varnish.new.purge('.*')
+    end
+  end
+end

data/lib/lacquer/varnish.rb

@@ -12,9 +12,14 @@ module Lacquer
     end
 
     # Sends the command 'url.purge *path*'
-    def purge(path)
-      send_command('url.purge ' << path).all? do |result|
-        result =~ /200/
+    def purge(*paths)
+      paths.all? do |path|
+        ActiveSupport::Notifications.instrument('purge.lacquer', path: path) do |payload|
+          send_command(Lacquer.configuration.purge_command + " " + path.gsub('\\', '\\\\\\')).all? do |result|
+            payload[:result] = result
+            result =~ /200/
+          end
+        end
       end
     end
 
@@ -29,8 +34,32 @@ module Lacquer
             'Host' => server[:host],
             'Port' => server[:port],
             'Timeout' => server[:timeout] || 5)
+
+          if(server[:secret])
+            connection.waitfor("Match" => /^107/) do |authentication_request|
+              matchdata = /^107 \d{2}\s*(.{32}).*$/m.match(authentication_request) # Might be a bit ugly regex, but it works great!
+              salt = matchdata[1]
+              if(salt.empty?)
+                raise VarnishError, "Bad authentication request"
+              end
+
+              digest = OpenSSL::Digest.new('sha256')
+              digest << salt
+              digest << "\n"
+              digest << server[:secret]
+              digest << salt
+              digest << "\n"
+
+              connection.cmd("String" => "auth #{digest.to_s}", "Match" => /\d{3}/) do |auth_response|
+                if(!(/^200/ =~ auth_response))
+                  raise AuthenticationError, "Could not authenticate"
+                end
+              end
+            end
+          end
+
           connection.cmd('String' => command, 'Match' => /\n\n/) {|r| response = r.split("\n").first.strip}
-          connection.close
+          connection.close if connection.respond_to?(:close)
         rescue Exception => e
           if retries < Lacquer.configuration.retries
             retry
@@ -41,6 +70,8 @@ module Lacquer
                :error_message => "Error while trying to connect to #{server[:host]}:#{server[:port]}: #{e}",
                :parameters    => server,
                :response      => response })
+            elsif e.kind_of?(Lacquer::AuthenticationError)
+              raise e
             else
               raise VarnishError.new("Error while trying to connect to #{server[:host]}:#{server[:port]} #{e}")
             end

data/lib/lacquer/varnishd.rb

@@ -0,0 +1,152 @@
+module Lacquer
+  class Varnishd
+    attr_accessor :listen, :telnet, :sbin_path, :bin_path, :storage, :working_dir, :user, :backend, :params, :use_sudo, :pid_path
+
+    cattr_accessor :started_check_delay, :vcl_script_filename
+    self.started_check_delay = 1
+    self.vcl_script_filename = 'config/varnish.vcl'
+
+    def self.root_path
+      Rails.root
+    end
+
+    def self.env
+      Rails.env
+    end
+
+    def self.config_file
+      root_path.join('config/varnishd.yml')
+    end
+
+    def self.config
+      YAML.load(File.read(config_file))[env].stringify_keys
+    end
+
+    def initialize(settings = self.class.config)
+      self.listen, self.telnet, self.backend, self.sbin_path, self.bin_path, self.storage, self.working_dir, self.user, self.params, self.use_sudo, self.pid_path =
+        settings.values_at("listen", "telnet", "backend", "sbin_path", "bin_path", "storage", "working_dir", "user", "params", "use_sudo", "pid_path")
+    end
+
+    def render_vcl
+      require 'erubis'
+      eruby = Erubis::Eruby.new(erb_vcl_script_filename.read)
+      eruby.result(binding)
+    end
+
+    def generate_vcl
+      if erb_vcl_script_filename.exist?
+        log "#{erb_vcl_script_filename} found rendering to #{vcl_script_filename}"
+        File.open(vcl_script_filename, "w") do |vcl|
+          vcl.write(render_vcl)
+        end
+      end
+    end
+
+    def start
+      if running?
+        log("Already running")
+        return
+      end
+      generate_vcl
+      execute("#{varnishd_cmd} #{args} #{params_args}")
+      sleep(self.class.started_check_delay)
+      log("Failed to start varnishd daemon") unless running?
+    end
+
+    def stop
+      if running?
+        execute("#{'sudo ' if use_sudo}kill #{pid}")
+        pid_file.delete
+      else
+        log("pid file not found or varnishd not running")
+      end
+    end
+
+    def reload
+      if running?
+        generate_vcl
+        reload_id = "reload#{Time.now.usec}"
+        load_cmd = "#{varnishadm_cmd} vcl.load #{reload_id} #{options['-f']}"
+        use_cmd = "#{varnishadm_cmd} vcl.use #{reload_id}"
+        execute "#{load_cmd} && #{use_cmd}"
+      else
+        start
+      end
+    end
+
+    def running?
+      !!pid && !!execute("ps p #{pid}").include?(pid.to_s) # works with sudo
+    end
+
+    def args
+      options.map { |k, v| "#{k} #{v}" }.join(" ")
+    end
+
+    def params_args
+      params.map { |k, v| "-p #{k}=#{v}" }.join(" ")
+    end
+
+    def options
+      opt = {}
+      opt["-P"] = pid_file
+      opt["-a"] = listen
+      opt["-T"] = telnet        if telnet.present?
+      opt["-n"] = working_dir   if working_dir.present?
+      opt["-u"] = user          if user.present?
+      opt["-s"] = eval(%Q("#{storage}"))
+      opt["-f"] = vcl_script_filename
+      opt
+    end
+
+    def params
+      @params || {}
+    end
+
+    protected
+
+    def varnishd_cmd
+      "#{'sudo ' if use_sudo}#{Pathname.new(sbin_path).join('varnishd')}"
+    end
+
+    def varnishadm_cmd
+      "#{'sudo ' if use_sudo}#{Pathname.new(bin_path).join('varnishadm')} -T #{options['-T']}"
+    end
+
+    def pid_file
+      pid_computed_path.join("varnishd.#{self.class.env}.pid")
+    end
+
+    def pid_computed_path
+      if self.pid_path
+        Pathname.new self.pid_path
+      else
+        self.class.root_path.join('log/')
+      end
+    end
+
+    def vcl_script_filename
+      self.class.root_path.join(self.class.vcl_script_filename)
+    end
+
+    def erb_vcl_script_filename
+      vcl_script_filename.sub_ext('.vcl.erb')
+    end
+
+    def log(message)
+      puts "** [#{self.class.name}] #{message}"
+    end
+
+    def execute(cmd)
+      log(cmd)
+      `#{cmd}`
+    end
+
+    def pid
+      if pid_file.exist?
+        pid_file.read
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/lacquer/version.rb

@@ -0,0 +1,3 @@
+module Lacquer
+  VERSION = '0.6.6'
+end

data/lib/lacquer.rb

@@ -1,13 +1,22 @@
 require "rubygems"
 require "net/telnet"
+
+require "openssl"
+require "digest/sha2"
+
 require "active_support/core_ext"
 
 require "lacquer/configuration"
 require "lacquer/cache_utils"
+require "lacquer/cache_control"
 require "lacquer/varnish"
+require "lacquer/varnishd"
+
+require "lacquer/railtie" if defined?(Rails::Railtie)
 
 module Lacquer
   class VarnishError < Exception; end # @private
+  class AuthenticationError < VarnishError; end # @private
 
   class << self
     attr_accessor :configuration

data/rails/init.rb

@@ -1 +1 @@
-require 'lacquer'
+require 'lacquer'

data/recipes/lacquer.rb

@@ -0,0 +1 @@
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'lacquer', 'recipes'))

data/spec/config/generate.vcl.erb

@@ -0,0 +1,15 @@
+backend default {
+  .host = "<%= backend.split(':').first %>";
+  .port = "<%= backend.split(':').last %>";
+}
+
+sub vcl_recv {
+  set req.backend = default;
+  unset req.http.Cookie;
+  set req.grace = 30m;
+}
+
+sub vcl_fetch {
+  unset beresp.http.Set-Cookie;
+  set beresp.grace = 30m;
+}

data/spec/config/varnish.vcl

@@ -0,0 +1,15 @@
+backend default {
+  .host = "127.0.0.1";
+  .port = "3000";
+}
+
+sub vcl_recv {
+  set req.backend = default;
+  unset req.http.Cookie;
+  set req.grace = 30m;
+}
+
+sub vcl_fetch {
+  unset beresp.http.Set-Cookie;
+  set beresp.grace = 30m;
+}

data/spec/config/varnishd.yml

@@ -0,0 +1,12 @@
+test:
+  listen: :80
+  telnet: localhost:6082
+  sbin_path: /usr/local/sbin
+  bin_path: /usr/local/bin
+  backend: 0.0.0.0:3000
+  storage: "file,#{Rails.root}/log/varnish,1GB"
+  working_dir: ""
+  user: "deploy"
+  use_sudo: true
+  params:
+    overflow_max: 2000