middle_squid 1.0

data/lib/middle_squid/builder.rb

@@ -0,0 +1,159 @@
+module MiddleSquid
+  # Small DSL to configure MiddleSquid.
+  #
+  # @example
+  #   database '/home/proxy/blacklist.db'
+  #
+  #   adv     = blacklist 'adv', aliases: ['ads']
+  #   tracker = blacklist 'tracker'
+  #
+  #   run lambda {|uri, extras|
+  #     if adv.include? uri
+  #       redirect_to 'http://your.webserver/block_pages/advertising.html'
+  #     end
+  #
+  #     if tracker.include? uri
+  #       redirect_to 'http://your.webserver/block_pages/tracker.html'
+  #     end
+  #   }
+  class Builder
+    # Returns the blacklists registered by {#blacklist}.
+    #
+    # @return [Array<BlackList>]
+    attr_reader :blacklists
+
+    # Returns the custom actions created by {#define_action}.
+    #
+    # @return [Hash<Symbol, Proc>]
+    attr_reader :custom_actions
+
+    # Returns the object passed to {#run}.
+    #
+    # @return [#call]
+    attr_reader :handler
+
+    # Returns the adapter selected by {#use}.
+    #
+    # @!attribute [r] adapter
+    # @return [Adapter]
+    def adapter
+      @adapter ||= Adapters::Squid.new
+    end
+
+    def initialize
+      @blacklists = []
+      @custom_actions = {}
+    end
+
+    # @return [Builder]
+    def self.from_file(file)
+      obj = self.new
+      content = File.read file
+
+      obj.instance_eval content, file
+      obj
+    end
+
+    # Select the active adapter.
+    # By default {Adapters::Squid} with no options will be used.
+    #
+    # @example Squid in concurrency mode
+    #   use Adapters::Squid, concurrency: true
+    # @param adapter [Class]
+    # @param options [Hash] adapter configuration
+    # @return [Adapter]
+    # @raise [ArgumentError] if the adapter is not a subclass of {Adapter}
+    # @see Adapters List of predefined adapters
+    def use(adapter, **options)
+      raise ArgumentError, 'Not an adapter.' unless adapter < Adapter
+
+      @adapter = adapter.new(options)
+    end
+
+    # Setup the blacklist database.
+    # It will be created if the file does not exists.
+    # Read/write access is required.
+    #
+    # Run {CLI#index middle_squid index} to add your blacklists to the database.
+    #
+    # @example
+    #   database '/home/proxy/blacklist.db'
+    #
+    #   run lambda {|uri, extras| }
+    # @param path [String] path to the SQLite database
+    def database(path)
+      Database.setup path
+    end
+
+    # Returns a new registered blacklist instance.
+    #
+    # @note You need to call {#database} in order to use the blacklists.
+    # @example Block advertising
+    #   adv = blacklist 'adv'
+    #
+    #   run lambda {|uri, extras|
+    #     do_something if adv.include? uri
+    #   }
+    # @example Group blacklists
+    #   adv = blacklist 'adv'
+    #   tracker = blacklist 'tracker'
+    #
+    #   group = [adv, tracker]
+    #
+    #   run lambda {|uri, extras|
+    #     do_something if group.any? {|bl| bl.include? uri }
+    #   }
+    # @example Create an alias
+    #   adv = blacklist 'adv', aliases: ['ads']
+    #
+    #   run lambda {|uri, extras|
+    #     do_something if adv.include? uri
+    #   }
+    # @return [BlackList]
+    # @see BlackList#initialize BlackList#initialize
+    def blacklist(*args)
+      bl = BlackList.new *args
+      @blacklists << bl
+      bl
+    end
+
+    # Register a custom action or helper.
+    #
+    # @example Don't Repeat Yourself
+    #   define_action :block do
+    #     redirect_to 'http://goodsite.com/'
+    #   end
+    #
+    #   run lambda {|uri, extras|
+    #     block if uri.host == 'badsite.com'
+    #     # ...
+    #     block if uri.host == 'terriblesite.com'
+    #   }
+    # @param name  [Symbol] method name
+    # @param block [Proc]   method body
+    # @see Actions List of predefined actions
+    # @see Helpers List of predefined helpers
+    def define_action(name, &block)
+      raise ArgumentError, 'no block given' unless block_given?
+
+      @custom_actions[name] = block
+    end
+    alias :define_helper :define_action
+
+    # Takes any object that responds to the +call+ method with two arguments:
+    # the URI to process and an array of extra data.
+    #
+    # @example
+    #   run lambda {|uri, extras|
+    #     # executed when the adapter has received a query from an underlying software (eg. Squid)
+    #   }
+    # @param handler [#call<URI, Array>]
+    # @raise [ArgumentError] if the handler does not respond to +#call+
+    # @see Runner Execution context (Runner)
+    def run(handler)
+      raise ArgumentError, 'the handler must respond to #call' unless handler.respond_to? :call
+
+      @handler = handler
+    end
+  end
+end

data/lib/middle_squid/cli.rb

@@ -0,0 +1,119 @@
+module MiddleSquid
+  class CLI < Thor
+    package_name 'MiddleSquid'
+
+    class_option :'config-file',
+      aliases:  '-C',
+      default: '~/middle_squid.rb',
+      desc:     'configuration file'
+
+    # <START>
+    desc 'start', 'Run the given configuration file'
+
+    # Runs the given configuration file.
+    #
+    # @example
+    #   middle_squid start -C middle_squid_config.rb
+    def start
+      config_file = File.expand_path options[:'config-file']
+
+      builder = Builder.from_file config_file
+
+      MiddleSquid::Runner.new builder
+    end
+    # </START>
+
+    # <INDEX>
+    option :append,  type: :boolean, default: false, aliases: '-a',
+      desc: 'keep the entries already in the database'
+
+    option :domains, type: :boolean, default: true,
+      desc: 'index domain lists'
+
+    option :full,    type: :boolean, default: false,
+      desc: 'index all blacklist categories'
+
+    option :quiet,   type: :boolean, default: false, aliases: '-q',
+      desc: 'disable status output'
+
+    option :urls,    type: :boolean, default: true,
+      desc: 'index urls lists'
+
+    desc 'index SOURCES...', 'Populate the database from one or more blacklists'
+
+    # Populates the database from one or more blacklists.
+    #
+    # *Flags:*
+    #
+    # [\-a, \--append, \--no-append]
+    #   Whether to keep the entries already in the database.
+    #
+    # [\--domains, \--no-domains]
+    #   Whether to index domain lists.
+    #
+    #   <b>Enabled by default.</b>
+    #
+    # [\--full, \--no-full]
+    #   Whether to index all blacklist categories.
+    #   By default MiddleSquid will only read the categories registered
+    #   using {Builder#blacklist} in the configuration script.
+    #
+    #   Enable if you want to reuse the same database in multiple configurations
+    #   set to use different blacklist categories and you need to index everything.
+    #
+    # [\-q, \--quiet, \--no-quiet]
+    #   Whether to disable status output.
+    #
+    # [\--urls, \--no-urls]
+    #   Whether to index url lists.
+    #
+    #   <b>Enabled by default.</b>
+    #
+    # @note {Builder#database} must be called in your configuration script in order to initialize the database.
+    # @example
+    #   middle_squid index ShallaBlackList -C middle_squid_config.rb
+    def index(*directories)
+      config_file = File.expand_path options[:'config-file']
+      directories.map! {|rel| File.expand_path rel }
+
+      builder = Builder.from_file config_file
+
+      entries = []
+      entries << :url if options[:urls]
+      entries << :domain if options[:domains]
+
+      indexer = MiddleSquid::Indexer.new
+      indexer.blacklists = builder.blacklists
+
+      indexer.append     = options[:append]
+      indexer.entries    = entries
+      indexer.full_index = options[:full]
+      indexer.quiet      = options[:quiet]
+
+      indexer.index directories
+    end
+    # </INDEX>
+
+    # <VERSION>
+    desc 'version', 'Show current version and copyright notice'
+
+    # Shows MiddleSquid's version and copyright notice.
+    def version
+      puts "MiddleSquid #{MiddleSquid::VERSION}"
+      puts <<GPL
+Copyright (C) 2014 by Christian Fillion
+
+  This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of
+the License, or (at your option) any later version.
+
+  This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+GPL
+    end
+    # </VERSION>
+  end
+end

data/lib/middle_squid/core_ext/hash.rb

@@ -0,0 +1,29 @@
+class Hash
+  IGNORED_HEADERS = [
+    'Connection',
+    'Content-Encoding',
+    'Content-Length',
+    'Host',
+    'Transfer-Encoding',
+    'Version',
+  ].freeze
+
+  DASH = '-'.freeze
+  UNDERSCORE = '_'.freeze
+
+  # @return [Hash]
+  def sanitize_headers!
+    clean = {}
+    each {|key, value|
+      key = key.tr UNDERSCORE, DASH
+      key = key.split(DASH).map(&:capitalize).join(DASH)
+
+      next if IGNORED_HEADERS.include? key
+
+      clean[key] = value
+    }
+
+    clear
+    merge! clean
+  end
+end

data/lib/middle_squid/database.rb

@@ -0,0 +1,47 @@
+module MiddleSquid::Database
+  @@db = nil
+
+  # Setup the database. Use {Builder#database} instead.
+  #
+  # @param path [String]
+  def self.setup(path)
+    @@db.close if @@db
+
+    @@db = SQLite3::Database.new path
+
+    @@db.execute <<-SQL
+    CREATE TABLE IF NOT EXISTS domains (
+      category TEXT, host TEXT
+    )
+    SQL
+
+    @@db.execute <<-SQL
+    CREATE UNIQUE INDEX IF NOT EXISTS unique_domains ON domains (
+      category, host
+    )
+    SQL
+
+    @@db.execute <<-SQL
+    CREATE TABLE IF NOT EXISTS urls (
+      category TEXT, host TEXT, path TEXT
+    )
+    SQL
+
+    @@db.execute <<-SQL
+    CREATE UNIQUE INDEX IF NOT EXISTS unique_urls ON urls (
+      category, host, path
+    )
+    SQL
+
+    # minimize downtime due to locks when the database is rebuilding
+    # see http://www.sqlite.org/wal.html
+    @@db.execute 'PRAGMA journal_mode=WAL'
+  end
+
+  # @return [SQLite3::Database]
+  def db
+    raise "The database is not initialized. Did you call Builder#database in your configuration file?" unless @@db
+
+    @@db
+  end
+end

data/lib/middle_squid/exceptions.rb

@@ -0,0 +1,4 @@
+module MiddleSquid
+  Error = Class.new RuntimeError
+  InvalidURIError = Class.new Addressable::URI::InvalidURIError
+end

data/lib/middle_squid/helpers.rb

@@ -0,0 +1,74 @@
+module MiddleSquid::Helpers
+  #
+  # @!group Predefined Helpers
+  #
+
+  # Download a resource with the same headers and body as a rack request.
+  #
+  # @note
+  #   This method must be called inside an active fiber. {Actions#intercept} does it automatically.
+  # @example Transparent Proxying
+  #   run lambda {|uri, extras|
+  #     # you should use 'accept' instead of doing this
+  #     intercept {|req, res|
+  #       download_like req, uri
+  #     }
+  #   }
+  # @example Body Modification
+  #   run lambda {|uri, extras|
+  #     intercept {|req, res|
+  #       status, headers, body = download_like req, uri
+  #
+  #       content_type = headers['Content-Type'].to_s
+  #
+  #       if content_type.include? 'text/html'
+  #         body.gsub! 'green', 'blue'
+  #       end
+  #
+  #       [status, headers, body]
+  #     }
+  #   }
+  # @param request [Rack::Request] the request to imitate
+  # @param uri [URI] the resource to fetch
+  # @return [Array] a rack triplet (status code, response headers and body)
+  # @see Actions#intercept
+  def download_like(request, uri)
+    fiber = Fiber.current
+
+    method = request.request_method.downcase.to_sym
+
+    headers = {'Content-Type' => request.env['CONTENT_TYPE']}
+    request.env.
+      select {|k| k.start_with? 'HTTP_' }.
+      each {|key, val| headers[key[5..-1]] = val }
+
+    headers.sanitize_headers!
+
+    options = {
+      :head => headers,
+      :body => request.body.read,
+    }
+
+    http = EM::HttpRequest.new(uri.to_s).send method, options
+
+    http.callback {
+      status = http.response_header.status
+      headers = http.response_header
+      body = http.response
+
+      headers.sanitize_headers!
+
+      fiber.resume [status, headers, body]
+    }
+
+    http.errback {
+      fiber.resume [520, {}, "[MiddleSquid] #{http.error}"]
+    }
+
+    Fiber.yield
+  end
+
+  #
+  # @!endgroup
+  #
+end

data/lib/middle_squid/indexer.rb

@@ -0,0 +1,194 @@
+module MiddleSquid
+  # Used internally to build the blacklist database.
+  #
+  # @see CLI#index <code>middle_squid index</code> command
+  class Indexer
+    include Database
+    
+    # @return [Boolean]
+    attr_accessor :append
+
+    # @return [Array<Symbol>]
+    attr_accessor :entries
+
+    # @return [Boolean]
+    attr_accessor :full_index
+
+    # @return [Boolean]
+    attr_accessor :quiet
+
+    def initialize
+      @append = false
+      @entries = [:url, :domain]
+      @full_index = true
+      @quiet = false
+
+      @aliases = {}
+      @cats_in_use  = []
+      @indexed_cats = []
+
+      @total = {
+        :url    => 0,
+        :domain  => 0,
+        :ignored  => 0,
+        :duplicate => 0,
+      }
+    end
+
+    # @param list [Array<BlackList>]
+    def blacklists=(list)
+      @cats_in_use.clear
+      @aliases.clear
+
+      list.each {|bl|
+        @cats_in_use << bl.category
+
+        bl.aliases.each {|name|
+          @aliases[name] = bl.category
+        }
+      }
+
+      @cats_in_use.uniq!
+    end
+
+    # @param directories [Array<String>]
+    def index(directories)
+      if !@full_index && @cats_in_use.empty?
+        warn 'ERROR: the loaded configuration does not use any blacklist'
+        puts 'nothing to do in minimal indexing mode'
+        return
+      end
+
+      start_time = Time.now
+
+      db.transaction
+
+      truncate unless @append
+      directories.each {|dir|
+        walk_in dir
+      }
+      cats_summary
+      stats
+      commit_or_rollback
+
+      end_time = Time.now
+      puts "finished after #{end_time - start_time} seconds"
+    ensure
+      db.rollback if db.transaction_active?
+    end
+
+    private
+    def puts(*args)
+      super *args unless @quiet
+    end
+
+    def truncate
+      puts 'truncating database'
+
+      db.execute 'DELETE FROM domains' 
+      db.execute 'DELETE FROM urls' 
+    end
+
+    def walk_in(directory)
+      puts "reading #{directory}"
+
+      unless File.directory? directory
+        warn "WARNING: #{directory}: no such directory"
+        return
+      end
+
+      files = Dir.glob File.join(directory, '*/*')
+      files.sort! # fixes travis build
+
+      files.each {|file|
+        index_file file
+      }
+    end
+
+    def index_file(path)
+      pn = Pathname.new path
+      return unless pn.file?
+
+      dirname = pn.dirname.basename.to_s
+      category = @aliases.has_key?(dirname) \
+        ? @aliases[dirname]
+        : dirname
+
+      if !@full_index
+        return unless @cats_in_use.include? category
+      end
+
+      @indexed_cats << category
+
+      puts "indexing #{dirname}/#{pn.basename}"
+
+      File.foreach(path) {|line|
+        type = append_to category, line
+        @total[type] += 1
+      }
+    end
+
+    def append_to(category, line)
+      # remove trailing whitespace
+      line.strip!
+
+      # ignore regex lists
+      return :ignored unless line[0] =~ /\w/
+
+      # fix invalid bytes
+      line.scrub! ''
+
+      # fix for dirty lists
+      line.tr! '\\', '/'
+
+      uri = MiddleSquid::URI.parse "http://#{line}"
+      host, path = uri.cleanhost, uri.cleanpath
+
+      if path.empty?
+        return :ignored unless @entries.include? :domain
+
+        db.execute 'INSERT INTO domains (category, host) VALUES (?, ?)',
+          [category, host]
+
+        :domain
+      else
+        return :ignored unless @entries.include? :url
+
+        db.execute 'INSERT INTO urls (category, host, path) VALUES (?, ?, ?)',
+          [category, host, path]
+
+        :url
+      end
+    rescue SQLite3::ConstraintException
+      :duplicate
+    end
+
+    def cats_summary
+      @indexed_cats.uniq!
+      missing_cats = @cats_in_use - @indexed_cats
+
+      puts
+      puts "indexed #{@indexed_cats.size} categorie(s): #{@indexed_cats}"
+      warn "WARNING: could not find #{missing_cats}" unless missing_cats.empty?
+    end
+
+    def stats
+      puts "found #{@total[:domain]} domain(s)"
+      puts "found #{@total[:url]} url(s)"
+      puts "found #{@total[:duplicate]} duplicate(s)"
+      puts "found #{@total[:ignored]} ignored expression(s)"
+      puts
+    end
+
+    def commit_or_rollback
+      if @total[:domain] > 0 || @total[:url] > 0
+        puts 'committing changes'
+        db.commit
+      else
+        warn 'ERROR: nothing to commit'
+        puts 'reverting changes'
+        db.rollback
+      end
+    end
+  end
+end

data/lib/middle_squid/runner.rb

@@ -0,0 +1,37 @@
+module MiddleSquid
+  # Used internally to start the configured adapter and the internal HTTP server.
+  # The handler passed to {Builder#run} will be called in this context.
+  #
+  # @see Builder Configuration DSL
+  # @see CLI#start <code>middle_squid start</code> command
+  class Runner
+    include Actions
+    include Helpers
+
+    # Returns the internal HTTP server.
+    #
+    # @return [Server]
+    attr_reader :server
+
+    # @raise [Error] if the handler is undefined
+    def initialize(builder)
+      raise Error, 'Invalid handler. Did you call Builder#run in your configuration file?' unless builder.handler
+
+      define_singleton_method :_handler_wrapper, builder.handler
+
+      builder.custom_actions.each {|name, body|
+        define_singleton_method name, body
+      }
+
+      adapter = builder.adapter
+      adapter.handler = method :_handler_wrapper
+
+      @server = Server.new
+
+      EM.run {
+        adapter.start
+        @server.start
+      }
+    end
+  end
+end