middle_squid 1.0

data/test/test_indexer.rb

@@ -0,0 +1,457 @@
+require File.expand_path '../helper', __FILE__
+
+class TestIndexer < MiniTest::Test
+  include MiddleSquid::Database
+
+  make_my_diffs_pretty!
+
+  def setup
+    @obj = MiddleSquid::Indexer.new
+    @path = File.expand_path '../resources', __FILE__
+
+    db.transaction
+
+    db.execute 'DELETE FROM domains' 
+    db.execute 'DELETE FROM urls' 
+
+    db.execute 'INSERT INTO domains (category, host) VALUES (?, ?)',
+      ['test', '.anidb.net']
+
+    db.execute 'INSERT INTO urls (category, host, path) VALUES (?, ?, ?)',
+      ['test', '.test.com', 'path']
+
+    db.commit
+  end
+
+  def has_test_data?
+    has_domain = !!db.get_first_row(
+      "SELECT 1 FROM domains WHERE category = 'test' AND host = '.anidb.net' AND rowid = 1 LIMIT 1"
+    )
+
+    has_url = !!db.get_first_row(
+      "SELECT 1 FROM urls WHERE category = 'test' AND host = '.test.com' AND path = 'path/' AND rowid = 1 LIMIT 1"
+    )
+
+    has_domain || has_url
+  end
+
+  def test_minimal_no_blacklist_used
+    @obj.full_index = false
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    assert_equal "nothing to do in minimal indexing mode\n", stdout
+
+    assert_match 'ERROR', stderr
+
+    assert has_test_data?
+  end
+
+  def test_empty_rollback
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'empty')]
+    end
+
+    assert_match 'indexing cat/emptylist', stdout
+    assert_match 'reverting changes', stdout
+
+    assert_match 'ERROR: nothing to commit', stderr
+
+    assert has_test_data?
+  end
+
+  def test_full_index
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['ads', '.ads.google.com'],
+      ['ads', '.doubleclick.net'],
+      ['tracker', '.xiti.com'],
+      ['tracker', '.google-analytics.com'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['ads', '.google.com', 'adsense/'],
+      ['tracker', '.feedproxy.google.com', '~r/'],
+      ['tracker', '.cloudfront-labs.amazonaws.com', 'x.png/'],
+    ], urls
+
+    assert_match 'indexing ads/urls', stdout
+    assert_match 'indexing ads/domains', stdout
+    assert_match 'indexing tracker/urls', stdout
+    assert_match 'indexing tracker/domains', stdout
+    assert_match 'indexed 2 categorie(s): ["ads", "tracker"]', stdout
+    assert_match 'found 4 domain(s)', stdout
+    assert_match 'found 3 url(s)', stdout
+    assert_match 'found 0 duplicate(s)', stdout
+    assert_match 'found 0 ignored expression(s)', stdout
+    assert_match 'committing changes', stdout
+
+    assert_empty stderr
+  end
+
+  def test_index_multiple
+    stdout, stderr = capture_io do
+      @obj.index [
+        File.join(@path, 'black'),
+        File.join(@path, 'gray'),
+      ]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['ads', '.ads.google.com'],
+      ['ads', '.doubleclick.net'],
+      ['tracker', '.xiti.com'],
+      ['tracker', '.google-analytics.com'],
+      ['isp', '.000webhost.com'],
+      ['isp', '.comcast.com'],
+      ['news', '.reddit.com'],
+      ['news', '.news.ycombinator.com'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['ads', '.google.com', 'adsense/'],
+      ['tracker', '.feedproxy.google.com', '~r/'],
+      ['tracker', '.cloudfront-labs.amazonaws.com', 'x.png/'],
+      ['isp', '.telus.com', 'content/internet/'],
+    ], urls
+
+    assert_match 'indexed 4 categorie(s): ["ads", "tracker", "isp", "news"]', stdout
+    assert_match 'found 8 domain(s)', stdout
+    assert_match 'found 4 url(s)', stdout
+    assert_match 'found 0 duplicate(s)', stdout
+
+    assert_empty stderr
+  end
+
+  def test_ignore_subdirectories
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'subdirectory')]
+    end
+
+    refute_match 'cat/ignore', stdout
+    assert has_test_data?
+  end
+
+  def test_minimal_indexing
+    @obj.blacklists = [MiddleSquid::BlackList.new('ads')]
+    @obj.full_index = false
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['ads', '.ads.google.com'],
+      ['ads', '.doubleclick.net'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['ads', '.google.com', 'adsense/'],
+    ], urls
+
+    refute_match 'tracker', stdout
+    assert_match 'indexed 1 categorie(s): ["ads"]', stdout
+
+    assert_empty stderr
+  end
+
+  def test_not_found
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, '404')]
+    end
+
+    assert has_test_data?
+
+    assert_match "reading #{File.join @path, '404'}", stdout
+
+    assert_match "WARNING: #{File.join @path, '404'}: no such directory\n", stderr
+    assert_match "ERROR: nothing to commit", stderr
+  end
+
+  def test_multiple_not_found
+    stdout, stderr = capture_io do
+      @obj.index [
+        File.join(@path, '404'),
+        File.join(@path, 'gray'),
+      ]
+    end
+
+    refute has_test_data?
+
+    assert_match "reading #{File.join @path, '404'}", stdout
+    assert_match "reading #{File.join @path, 'gray'}", stdout
+
+    assert_match "WARNING: #{File.join @path, '404'}: no such directory\n", stderr
+  end
+
+  def test_mixed_content
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'mixed')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['cat', '.domain.com'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['cat', '.url.com', 'path/'],
+    ], urls
+  end
+
+  def test_backslash
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'backslash')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_empty domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['cat', '.google.com', 'path/to/file/'],
+    ], urls
+  end
+
+  def test_invalid_byte
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'invalid_byte')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_empty domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['cat', '.host.com', 'path_with__invalid_byte/'],
+    ], urls
+  end
+
+  def test_empty_path_as_domain
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'empty_path')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['cat', '.host.com'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_empty urls
+  end
+
+  def test_duplicates
+    stdout, stderr = capture_io do
+      @obj.index [
+        File.join(@path, 'duplicates'),
+        File.join(@path, 'copy_of_duplicates'),
+      ]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['cat', '.host.com'],
+      ['copy_of_cat', '.host.com'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['cat', '.host.com', 'path/'],
+      ['copy_of_cat', '.host.com', 'path/'],
+    ], urls
+
+    assert_match 'found 12 duplicate(s)', stdout
+    assert_match 'found 0 ignored expression(s)', stdout
+
+    assert_empty stderr
+  end
+
+  def test_missing_category
+    bl = MiddleSquid::BlackList.new '404'
+    @obj.blacklists = [bl, bl] # should not cause duplicate output
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    refute has_test_data?
+
+    assert_match 'WARNING: could not find ["404"]', stderr
+  end
+
+  def test_expressions
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'expressions')]
+    end
+
+    assert has_test_data?
+
+    assert_match 'found 3 ignored expression(s)', stdout
+    assert_match 'ERROR: nothing to commit', stderr
+  end
+
+  def test_aliases
+    @obj.full_index = false
+    @obj.blacklists = [
+      MiddleSquid::BlackList.new('cat_name', aliases: ['ads'])
+    ]
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['cat_name', '.ads.google.com'],
+      ['cat_name', '.doubleclick.net'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['cat_name', '.google.com', 'adsense/'],
+    ], urls
+
+    refute_match 'tracker', stdout
+    assert_match 'indexing ads/', stdout
+    assert_match 'indexed 1 categorie(s): ["cat_name"]', stdout
+
+    assert_empty stderr
+  end
+
+  def test_domains_only
+    @obj.entries = [:domain]
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_equal [
+      ['ads', '.ads.google.com'],
+      ['ads', '.doubleclick.net'],
+      ['tracker', '.xiti.com'],
+      ['tracker', '.google-analytics.com'],
+    ], domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_empty urls
+
+    assert_match 'found 4 domain(s)', stdout
+    assert_match 'found 0 url(s)', stdout
+    assert_match 'found 3 ignored expression(s)', stdout
+
+    assert_empty stderr
+  end
+
+  def test_urls_only
+    @obj.entries = [:url]
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_empty domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['ads', '.google.com', 'adsense/'],
+      ['tracker', '.feedproxy.google.com', '~r/'],
+      ['tracker', '.cloudfront-labs.amazonaws.com', 'x.png/'],
+    ], urls
+
+    assert_match 'found 0 domain(s)', stdout
+    assert_match 'found 3 url(s)', stdout
+    assert_match 'found 4 ignored expression(s)', stdout
+
+    assert_empty stderr
+  end
+
+  def test_append
+    @obj.append = true
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    assert has_test_data?, 'should not be truncated'
+    refute_match 'truncating', stdout
+
+    assert_match 'found 4 domain(s)', stdout
+    assert_match 'found 3 url(s)', stdout
+  end
+
+  def test_quiet
+    @obj.quiet = true
+
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    assert_empty stdout
+    assert_empty stderr
+  end
+
+  def test_quiet_errors
+    @obj.quiet = true
+
+    stdout, stderr = capture_io do
+      @obj.index []
+    end
+
+    assert_empty stdout
+    assert_match 'nothing to commit', stderr
+  end
+
+  def test_strip_spaces
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'trailing_space')]
+    end
+
+    refute has_test_data?
+
+    domains = db.execute 'SELECT category, host FROM domains'
+    assert_empty domains
+
+    urls = db.execute 'SELECT category, host, path FROM urls'
+    assert_equal [
+      ['cat', '.before.com', 'path/'],
+      ['cat', '.after.com', 'path/'],
+    ], urls
+  end
+end

data/test/test_keyboard.rb

@@ -0,0 +1,79 @@
+require File.expand_path '../helper', __FILE__
+
+class TestKeyboard < MiniTest::Test
+  def test_receive_line
+    bag = []
+
+    input = MiddleSquid::Backends::Keyboard.new nil, proc {|*args|
+      bag << args
+    }
+
+    input.receive_line 'hello world'
+
+    assert_equal ['hello world'], bag.shift
+    assert_empty bag
+  end
+
+  def test_encoding_fix
+    bag = []
+
+    input = MiddleSquid::Backends::Keyboard.new nil, proc {|line| bag << line }
+
+    input.receive_line 'hello world'.force_encoding(Encoding::ASCII_8BIT)
+
+    assert_equal Encoding::UTF_8, bag.shift.encoding
+    assert_empty bag
+  end
+
+  def test_input_eof
+    input = MiddleSquid::Backends::Keyboard.new nil, proc {|line| nil }
+
+    Timeout::timeout 1 do
+      EM.run { input.receive_data "\x00" }
+    end
+  end
+
+  def test_line_buffer
+    bag = []
+
+    input = MiddleSquid::Backends::Keyboard.new nil, proc {|*args| bag << args }
+    input.receive_data 'h'
+    input.receive_data 'e'
+    input.receive_data 'l'
+    input.receive_data 'l'
+    input.receive_data 'o'
+    input.receive_data ' '
+    input.receive_data 'w'
+    input.receive_data 'o'
+    input.receive_data 'r'
+    input.receive_data 'l'
+    input.receive_data 'd'
+    input.receive_data "\n"
+
+    assert_equal ['hello world'], bag.shift
+    assert_empty bag
+  end
+
+  def test_buffer_is_always_cleared
+    bag = []
+
+    input = MiddleSquid::Backends::Keyboard.new nil, proc {|*args|
+      bag << args
+      throw :skip
+    }
+
+    catch :skip do
+      input.receive_data 'a'
+      input.receive_data "\n"
+    end
+
+    catch :skip do
+      input.receive_data 'b'
+      input.receive_data "\n"
+    end
+
+    assert_equal ['a'], bag.shift
+    assert_equal ['b'], bag.shift
+    assert_empty bag
+  end
+end

data/test/test_runner.rb

@@ -0,0 +1,56 @@
+require File.expand_path '../helper', __FILE__
+
+class TestRunner < MiniTest::Test
+  def test_invalid_config
+    fake_builder = Class.new
+    def fake_builder.handler; nil; end
+
+    error = assert_raises MiddleSquid::Error do
+      MiddleSquid::Runner.new fake_builder
+    end
+
+    assert_equal 'Invalid handler. Did you call Builder#run in your configuration file?', error.message
+  end
+
+  def test_run
+    handler = proc { self }
+
+    custom_actions = { :my_action => proc { self } }
+
+    fake_adapter = MiniTest::Mock.new
+    fake_adapter.expect :handler=, nil do |wrapper|
+      assert_respond_to wrapper, :call
+      assert_instance_of MiddleSquid::Runner, wrapper.call
+    end
+    fake_adapter.expect :start, nil, []
+
+    fake_builder = Class.new
+    fake_builder.define_singleton_method(:handler) { handler }
+    fake_builder.define_singleton_method(:adapter) { fake_adapter }
+    fake_builder.define_singleton_method(:custom_actions) { custom_actions }
+
+    EM.run {
+      @obj = MiddleSquid::Runner.new fake_builder
+      verify_server
+      EM.next_tick { EM.stop }
+    }
+
+    fake_adapter.verify
+
+    verify_custom_actions
+  end
+
+  def verify_server
+    assert_instance_of MiddleSquid::Server, @obj.server
+    assert_equal '127.0.0.1', @obj.server.host
+    assert @obj.server.port > 0, 'port'
+  end
+
+  def verify_custom_actions
+    assert_equal @obj, @obj.my_action
+
+    assert_raises NoMethodError do
+      @obj.not_found
+    end
+  end
+end

data/test/test_server.rb

@@ -0,0 +1,86 @@
+require File.expand_path '../helper', __FILE__
+
+class TestServer < MiniTest::Test
+  include Rack::Test::Methods
+
+  def setup
+    @server = MiddleSquid::Server.new
+    @app = Thin::Async::Test.new @server.method(:handler)
+  end
+
+  def app
+    @app
+  end
+
+  def token_for(&block)
+    token = nil
+
+    EM.run {
+      token = @server.token_for block
+      EM.next_tick { EM.stop }
+    }
+
+    token
+  end
+
+  def test_start_stop
+    EM.run {
+      assert_nil @server.host
+      assert_nil @server.port
+
+      @server.start
+
+      assert_equal '127.0.0.1', @server.host
+      assert_instance_of Fixnum, @server.port
+      assert @server.port > 0
+
+      @server.stop
+
+      assert_nil @server.host
+      assert_nil @server.port
+
+      EM.next_tick { EM.stop }
+    }
+  end
+
+  def test_not_found
+    get '/not_found'
+
+    assert_equal 404, last_response.status
+    assert_equal 'text/plain', last_response['Content-Type']
+    assert_equal '[MiddleSquid] Invalid Token', last_response.body
+  end
+
+  def test_token
+    bag = []
+
+    tk = token_for {|*args|
+      bag << args
+      bag << Fiber.current
+    }
+
+    get '/' + tk
+
+    assert_equal 200, last_response.status
+    assert_empty last_response.body
+
+    (req, res), fiber = bag
+
+    assert_instance_of Rack::Request, req
+    assert_instance_of Thin::AsyncResponse, res
+
+    refute_same fiber, Fiber.current
+  end
+
+  def test_rack_reply
+    tk = token_for {
+      [418, {'Hello'=>'World'}, ['hello world']]
+    }
+
+    get '/' + tk
+
+    assert_equal 418, last_response.status
+    assert_equal 'World', last_response['HELLO']
+    assert_equal 'hello world', last_response.body
+  end
+end