middle_squid 1.0

data/lib/middle_squid/server.rb

@@ -0,0 +1,84 @@
+module MiddleSquid
+  # Manages the internal HTTP server.
+  class Server
+    DEFAULT_HOST = '127.0.0.1'.freeze
+    DEFAULT_PORT = 0
+
+    TOKEN_TIMEOUT = 10
+
+    Thin::Logging.logger = Logger.new STDERR
+    Thin::Logging.level = Logger::WARN
+
+    # @return [String]
+    attr_reader :host
+
+    # @return [Fixnum]
+    attr_reader :port
+
+    def initialize
+      @tokens = {}
+      @thin = Thin::Server.new DEFAULT_HOST, DEFAULT_PORT, method(:handler),
+        :backend => Backends::Thin,
+        :signals => false
+    end
+
+    def start
+      @thin.start
+
+      sockname = EM.get_sockname @thin.backend.signature
+      @port, @host = Socket.unpack_sockaddr_in sockname
+    end
+
+    def stop
+      @thin.stop
+      @port = @host = nil
+    end
+
+    # Creates a temporary token.
+    #
+    # @param block [#call] called when the token is requested
+    # @return [String] random token
+    def token_for(block)
+      token = SecureRandom.uuid
+      @tokens[token] = block
+
+      EM.add_timer(TOKEN_TIMEOUT) {
+        @tokens.delete token
+      }
+
+      token
+    end
+
+    private
+    def handler(env)
+      callback = @tokens[env['PATH_INFO'][1..-1]]
+
+      return [
+        404,
+        {'Content-Type' => 'text/plain'},
+        ['[MiddleSquid] Invalid Token']
+      ] unless callback
+
+      request  = Rack::Request.new env
+      response = Thin::AsyncResponse.new env
+
+      Fiber.new {
+        retval = callback.call request, response
+
+        if retval.is_a?(Array) && retval.size == 3
+          status, headers, body = retval
+
+          headers.sanitize_headers!
+
+          response.status = status
+          response.headers.merge! headers
+          response.write body
+        end
+
+        response.done
+      }.resume
+
+      response.finish
+    end
+  end
+end

data/lib/middle_squid/uri.rb

@@ -0,0 +1,31 @@
+module MiddleSquid
+  # Provides distinguishable host and path for the blacklists.
+  #
+  # @see http://rubydoc.info/gems/addressable/Addressable/URI
+  class URI < Addressable::URI
+    DOT   = '.'.freeze
+    SLASH = '/'.freeze
+
+    # @return [String]
+    def cleanhost
+      clean = normalized_host.force_encoding Encoding::UTF_8
+      clean.sub! /\Awww\./, ''
+      clean.sub! /\.+\z/, ''
+      clean.insert 0, DOT
+      clean
+    end
+
+    # @return [String]
+    def cleanpath
+      dirty = normalized_path.force_encoding Encoding::UTF_8
+      p = Pathname.new(dirty).cleanpath
+
+      file = p.basename('.*').to_s.downcase
+      p = p.dirname if %w[index default].include? file
+
+      clean = p.to_s[1..-1]
+      clean << SLASH unless clean.empty?
+      clean
+    end
+  end
+end

data/lib/middle_squid/version.rb

@@ -0,0 +1,3 @@
+module MiddleSquid
+  VERSION = '1.0'
+end

data/lib/middle_squid.rb

@@ -0,0 +1,46 @@
+require 'middle_squid/version'
+
+require 'fiber'
+require 'pathname'
+require 'securerandom'
+
+require 'addressable/uri'
+require 'eventmachine'
+require 'em-http-request'
+require 'sqlite3'
+require 'thin'
+require 'thin/async'
+require 'thor'
+
+# Copyright (C) 2014 by Christian Fillion
+#
+# @see Builder Configuration syntax (DSL)
+# @see Actions List of predefined actions
+# @see Helpers List of predefined helpers
+# @see Adapters Available adapters
+module MiddleSquid
+  require 'middle_squid/actions'
+  require 'middle_squid/database'
+  require 'middle_squid/helpers'
+
+  module Adapters
+    require 'middle_squid/adapter'
+    require 'middle_squid/adapters/squid'
+  end
+
+  module Backends
+    require 'middle_squid/backends/keyboard'
+    require 'middle_squid/backends/thin'
+  end
+
+  require 'middle_squid/blacklist'
+  require 'middle_squid/builder'
+  require 'middle_squid/cli'
+  require 'middle_squid/exceptions'
+  require 'middle_squid/indexer'
+  require 'middle_squid/runner'
+  require 'middle_squid/server'
+  require 'middle_squid/uri'
+
+  require 'middle_squid/core_ext/hash'
+end

data/middle_squid.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path '../lib', __FILE__
+$LOAD_PATH.unshift lib unless $LOAD_PATH.include? lib
+
+require 'middle_squid/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'middle_squid'
+  spec.version       = MiddleSquid::VERSION
+  spec.authors       = ['Christian Fillion']
+  spec.email         = ['middle_squid@cfillion.tk']
+  spec.summary       = 'A redirector, url mangler and webpage interceptor for the Squid HTTP proxy'
+  spec.homepage      = 'https://github.com/cfillion/middle_squid'
+  spec.license       = 'GPL-3.0+'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'coveralls', '~> 0.7'
+  spec.add_development_dependency 'minitest', '~> 5.4'
+  spec.add_development_dependency 'rack-test', '~> 0.6'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'simplecov', '~> 0.9'
+  spec.add_development_dependency 'thin-async-test', '~> 1.0'
+  spec.add_development_dependency 'webmock', '~> 1.18'
+
+  spec.add_runtime_dependency 'addressable', '~> 2.3'
+  spec.add_runtime_dependency 'em-http-request', '~> 1.1'
+  spec.add_runtime_dependency 'eventmachine', '~> 1.0'
+  spec.add_runtime_dependency 'sqlite3', '~> 1.3'
+  spec.add_runtime_dependency 'thin', '~> 1.6'
+  spec.add_runtime_dependency 'thin_async', '~> 0.1'
+  spec.add_runtime_dependency 'thor', '~> 0.19'
+end

data/middle_squid_wrapper.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+GEM_HOME=$(ruby -e 'puts Gem.user_dir')
+exec $GEM_HOME/bin/middle_squid $*

data/test/helper.rb

@@ -0,0 +1,26 @@
+require 'simplecov'
+require 'coveralls'
+
+Coveralls::Output.silent = true
+
+SimpleCov.formatters = [
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+
+SimpleCov.start {
+  project_name 'MiddleSquid'
+  add_filter '/test/'
+
+  add_group 'Adapters', '/adapter'
+  add_group 'Backends', '/backend'
+}
+
+require 'minitest/autorun'
+require 'rack/test'
+require 'thin/async/test'
+require 'webmock/minitest'
+
+require 'middle_squid'
+
+MiddleSquid::Database.setup ':memory:'

data/test/resources/backslash/cat/list

@@ -0,0 +1 @@
+google.com\path\to\file

data/test/resources/black/ads/domains

@@ -0,0 +1,2 @@
+ads.google.com
+doubleclick.net

data/test/resources/black/ads/urls

@@ -0,0 +1 @@
+google.com/adsense

data/test/resources/black/tracker/domains

@@ -0,0 +1,2 @@
+xiti.com
+google-analytics.com

data/test/resources/black/tracker/urls

@@ -0,0 +1,2 @@
+feedproxy.google.com/~r/
+cloudfront-labs.amazonaws.com/x.png

data/test/resources/copy_of_duplicates/cat/copy_of_list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/copy_of_duplicates/cat/list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/copy_of_duplicates/copy_of_cat/copy_of_list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/copy_of_duplicates/copy_of_cat/list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/duplicates/cat/copy_of_list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/duplicates/cat/list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/duplicates/copy_of_cat/copy_of_list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/duplicates/copy_of_cat/list

@@ -0,0 +1,2 @@
+host.com
+host.com/path

data/test/resources/empty_path/cat/list

@@ -0,0 +1 @@
+host.com/

data/test/resources/expressions/cat/list

@@ -0,0 +1,3 @@
+(/ads/|/ad/|/banner/|/sponsor/|/event.ng/|/Advertisement/|adverts/)
+[0-9]
+^[0-9]+

data/test/resources/gray/isp/domains

@@ -0,0 +1,2 @@
+000webhost.com
+comcast.com

data/test/resources/gray/isp/urls

@@ -0,0 +1 @@
+www.telus.com/content/internet/

data/test/resources/gray/news/domains

@@ -0,0 +1,2 @@
+reddit.com
+news.ycombinator.com

data/test/resources/hello.rb

@@ -0,0 +1,2 @@
+puts "hello #{self}"
+run proc {}

data/test/resources/invalid_byte/cat/list

@@ -0,0 +1 @@
+host.com/path_with_�_invalid_byte

data/test/resources/mixed/cat/list

@@ -0,0 +1,2 @@
+url.com/path
+domain.com

data/test/resources/trailing_space/cat/list

@@ -0,0 +1,2 @@
+ before.com/path
+after.com/path 

data/test/test_actions.rb

@@ -0,0 +1,76 @@
+require File.expand_path '../helper', __FILE__
+
+class TestActions < MiniTest::Test
+  FakeClass = Class.new { include MiddleSquid::Actions }
+
+  def setup
+    @obj = FakeClass.new
+  end
+
+  def test_accept
+    action = catch :action do
+      @obj.accept
+    end
+
+    assert_equal [:accept, {}], action
+  end
+
+  def test_redirect_301
+    action = catch :action do
+      @obj.redirect_to 'url'
+    end
+
+    assert_equal [:redirect, {
+      :status => 301,
+      :url => 'url'
+    }], action
+  end
+
+  def test_redirect_custom_status
+    action = catch :action do
+      @obj.redirect_to 'new url', status: 418
+    end
+
+    assert_equal [:redirect, {
+      :status => 418,
+      :url => 'new url'
+    }], action
+  end
+
+  def test_replace
+    action = catch :action do
+      @obj.replace_by 'http://duckduckgo.com/'
+    end
+
+    assert_equal [:replace, {
+      :url => 'http://duckduckgo.com/'
+    }], action
+  end
+
+  def test_intercept
+    mock = MiniTest::Mock.new
+    mock.expect :token_for, 'qwfpgjluy', [Proc]
+    mock.expect :host, '127.0.0.1'
+    mock.expect :port, 8901
+    @obj.define_singleton_method(:server) { mock }
+
+    action = catch :action do
+      EM.run {
+        @obj.intercept {}
+        EM.next_tick { EM.stop }
+      }
+    end
+
+    assert_equal [:replace, {
+      :url => 'http://127.0.0.1:8901/qwfpgjluy'
+    }], action
+
+    mock.verify
+  end
+
+  def test_intercept_requires_a_block
+    assert_raises ArgumentError do
+      @obj.intercept
+    end
+  end
+end

data/test/test_adapter.rb

@@ -0,0 +1,61 @@
+require File.expand_path '../helper', __FILE__
+
+class TestAdapter < MiniTest::Test
+  def setup
+    @obj = MiddleSquid::Adapter.new
+  end
+  
+  def test_handler
+    bag = []
+
+    @obj.handler = proc {|*args| bag << args }
+
+    @obj.define_singleton_method(:output) { |*args| bag << args }
+
+    @obj.handle 'http://test.com', ['hello', 'world']
+
+    uri, extras = bag.shift
+    assert_instance_of MiddleSquid::URI, uri
+    assert_equal 'http://test.com', uri.to_s
+    assert_equal ['hello', 'world'], extras
+
+    assert_equal [:accept, {}], bag.shift # default action
+
+    assert_empty bag
+  end
+
+  def test_default_output
+    assert_raises NotImplementedError do
+      @obj.output :accept, {}
+    end
+  end
+
+  def test_overridden_output
+    bag = []
+
+    @obj.handler = proc { throw :action, [:type, :options] }
+
+    @obj.define_singleton_method(:output) { |*args| bag << args }
+
+    @obj.handle 'http://test.com', []
+
+    assert_equal [:type, :options], bag.shift
+    assert_empty bag
+  end
+
+  def test_empty_url
+    error = assert_raises MiddleSquid::InvalidURIError do
+      @obj.handle nil, []
+    end
+
+    assert_equal "invalid URL received: ''", error.message
+  end
+
+  def test_invalid_url
+    error = assert_raises MiddleSquid::InvalidURIError do
+      @obj.handle 'hello world', []
+    end
+
+    assert_equal "invalid URL received: 'hello world'", error.message
+  end
+end

data/test/test_blacklist.rb

@@ -0,0 +1,189 @@
+require File.expand_path '../helper', __FILE__
+
+class TestBlackList < MiniTest::Test
+  include MiddleSquid::Database
+
+  DOMAINS = %w{
+    .cfillion.tk
+    .duckduckgo.com
+    .sub.stackoverflow.com
+    .analytics.google.com
+    .anidb.net
+  }
+
+  URLS = [
+    %w[.google.com analytics/],
+    %w[.youtube.com watch/],
+    %w[.github.com user/],
+    %w[.host.net file.html/],
+    %w[.host.net index.html/],
+  ]
+
+  def setup
+    db.transaction
+
+    db.execute 'DELETE FROM domains' 
+    db.execute 'DELETE FROM urls' 
+
+    DOMAINS.each_with_index {|domain, index|
+      db.execute 'INSERT INTO domains (category, host) VALUES (?, ?)',
+        [index % 2 == 0 ? 'even' : 'odd', domain]
+    }
+
+    URLS.each_with_index {|url, index|
+      db.execute 'INSERT INTO urls (category, host, path) VALUES (?, ?, ?)',
+        [index % 2 == 0 ? 'even' : 'odd', *url]
+    }
+
+    db.commit
+  end
+
+  def test_category
+    bl = MiddleSquid::BlackList.new 'cat_name'
+    assert_equal 'cat_name', bl.category
+  end
+
+  def test_aliases_default
+    bl = MiddleSquid::BlackList.new 'cat_name'
+    assert_equal [], bl.aliases
+  end
+
+  def test_aliases_custom
+    bl = MiddleSquid::BlackList.new 'cat_name', aliases: ['name_cat']
+    assert_equal ['name_cat'], bl.aliases
+  end
+
+  def test_unmatch
+    uri = MiddleSquid::URI.parse('http://anidb.net/a9002')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_domain? uri
+    refute odd.include_url? uri
+    refute odd.include? uri
+  end
+
+  def test_domain
+    uri = MiddleSquid::URI.parse('http://cfillion.tk/love-of-babble/')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_domain? uri
+    refute even.include_url? uri
+    assert even.include? uri
+  end
+
+  def test_domain_trailing_dots
+    uri = MiddleSquid::URI.parse('http://cfillion.tk...')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_domain? uri
+    assert even.include? uri
+  end
+
+  def test_domain_partial_match
+    uri1 = MiddleSquid::URI.parse('http://www.anidb.net/')
+    uri2 = MiddleSquid::URI.parse('http://abcde.anidb.net/')
+    uri3 = MiddleSquid::URI.parse('http://bus.sub.stackoverflow.com/')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_domain?(uri1), 'www'
+    assert even.include? uri1
+
+    assert even.include_domain?(uri2), 'any subdomain'
+    assert even.include? uri2
+
+    assert even.include_domain?(uri3), 'sub-sub-domain'
+    assert even.include? uri3
+  end
+
+  def test_domain_right_match
+    uri = MiddleSquid::URI.parse('http://google.com')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_domain? uri
+    refute odd.include? uri
+  end
+
+  def test_domain_left_match
+    uri = MiddleSquid::URI.parse('http://duckduckgo/')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_domain? uri
+    refute odd.include? uri
+  end
+
+  def test_domain_wrong_subdomain
+    uri = MiddleSquid::URI.parse('http://fakeanalytics.google.com/')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_domain? uri
+    refute odd.include? uri
+  end
+
+  def test_url
+    uri = MiddleSquid::URI.parse('http://google.com/analytics')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_url? uri
+    refute even.include_domain? uri
+    assert even.include? uri
+  end
+
+  def test_url_wrong_extension
+    uri = MiddleSquid::URI.parse('http://host.net/file.php')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_url? uri
+    refute odd.include? uri
+  end
+
+  def test_url_partial_path
+    uri = MiddleSquid::URI.parse('http://github.com/us')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_url? uri
+    refute odd.include? uri
+  end
+
+  def test_url_left_match
+    uri = MiddleSquid::URI.parse('http://host.net/file.html_ohno')
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    refute odd.include_url? uri
+    refute odd.include? uri
+  end
+
+  def test_url_longer_path
+    uri = MiddleSquid::URI.parse('http://github.com/user/repository')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_url? uri
+    assert even.include? uri
+  end
+
+  def test_url_query_string
+    uri = MiddleSquid::URI.parse('http://github.com/user?tab=repositories')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_url? uri
+    assert even.include? uri
+  end
+
+  def test_url_cheap_tricks
+    uri = MiddleSquid::URI.parse('http://google.com//maps/../analytics/./')
+    even = MiddleSquid::BlackList.new 'even'
+
+    assert even.include_url? uri
+    assert even.include? uri
+  end
+
+  def test_group_demo
+    uri = MiddleSquid::URI.parse('http://youtube.com/watch?v=test')
+
+    even = MiddleSquid::BlackList.new 'even'
+    odd = MiddleSquid::BlackList.new 'odd'
+
+    group = [even, odd]
+
+    assert group.any? {|bl| bl.include? uri }
+  end
+end