middle_squid 1.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8afc13446c5543d835fc77291d1a08a124fc222f
-  data.tar.gz: 93d0646721e4cf660b75670f71cc18eef08a371a
+  metadata.gz: 516bbcd321414bdfd91755211912e7edb9a7533a
+  data.tar.gz: ff8669daab5db30a2f25810ae40a13feed93fbd7
 SHA512:
-  metadata.gz: 830df42d71fab5086d17f6393ef1a4fdf7d22c8a67a95463439836bc07ed26fafca22772d07e6123d4d7735f5218ae1fc7382930868c28ce4a2d3c4ec497520f
-  data.tar.gz: 95401e707dbe31ef3430c51fe8e65bbed16ec6555b1a4b142e806dfab53dde70f0c7b2fe9feecadd3de58949db0b4a3d88760d88652b3f27f1cc379d3e284bf5
+  metadata.gz: a62589c10bccafeb51c082f53552dd97bb566489c6be203f1eed1b3aff846f4111e4a4a5cc931b16cb502b6228596ef2650f33073fbc2094dac3d99effb16ef7
+  data.tar.gz: e473a55bc36ad604a4f1e7efd9c492d42a2ee08971f493484e1028f7ad8f83c1fcd40c1f61647261f4145f0c09f27384b1bfd9ba1b06d940df6323ca09e48466

data/README.md

@@ -147,7 +147,7 @@ tar xzf shallalist.tar.gz
 mv BL ShallaBlackList
 
 # Construct the blacklist database
-/usr/local/bin/middle_squid_wrapper.sh index ShallaBlackList -C /etc/squid/middle_squid.rb
+/usr/local/bin/middle_squid_wrapper.sh index ShallaBlackList -C /home/proxy/middle_squid_config.rb
 
 exit
 ```
@@ -208,6 +208,13 @@ MiddleSquid's documentation is hosted at
 
 ## Changelog
 
+### v1.0.1 (2014-11-06)
+
+- send download errors as text/plain
+- fix a crash when reading invalid UTF-8 byte sequences
+- cleanup `index`'s output (everything is now sent to stderr)
+- show the indexing progress
+
 ### v1.0 (2014-10-05)
 
 First public release.

data/lib/middle_squid/helpers.rb

@@ -29,7 +29,7 @@ module MiddleSquid::Helpers
   #     }
   #   }
   # @param request [Rack::Request] the request to imitate
-  # @param uri [URI] the resource to fetch
+  # @param uri [#to_s] the resource to fetch
   # @return [Array] a rack triplet (status code, response headers and body)
   # @see Actions#intercept
   def download_like(request, uri)
@@ -62,7 +62,11 @@ module MiddleSquid::Helpers
     }
 
     http.errback {
-      fiber.resume [520, {}, "[MiddleSquid] #{http.error}"]
+      fiber.resume [
+        520,
+        {'Content-Type' => 'text/plain'},
+        "[MiddleSquid] #{http.error}"
+      ]
     }
 
     Fiber.yield

data/lib/middle_squid/indexer.rb

@@ -54,8 +54,8 @@ module MiddleSquid
     # @param directories [Array<String>]
     def index(directories)
       if !@full_index && @cats_in_use.empty?
-        warn 'ERROR: the loaded configuration does not use any blacklist'
-        puts 'nothing to do in minimal indexing mode'
+        oops 'the loaded configuration does not use any blacklist'
+        info 'nothing to do in minimal indexing mode'
         return
       end
 
@@ -72,28 +72,41 @@ module MiddleSquid
       commit_or_rollback
 
       end_time = Time.now
-      puts "finished after #{end_time - start_time} seconds"
+      info "finished after #{end_time - start_time} seconds"
     ensure
       db.rollback if db.transaction_active?
     end
 
     private
-    def puts(*args)
-      super *args unless @quiet
+    def output(string, always: false)
+      $stderr.print string if always || !@quiet
+    end
+
+    def oops(msg)
+      output "ERROR: #{msg}\n", always: true
+    end
+
+    def warn(msg)
+      output "WARNING: #{msg}\n", always: true
+    end
+
+    def info(line = "")
+      line << "\n"
+      output line
     end
 
     def truncate
-      puts 'truncating database'
+      info 'truncating database'
 
       db.execute 'DELETE FROM domains' 
       db.execute 'DELETE FROM urls' 
     end
 
     def walk_in(directory)
-      puts "reading #{directory}"
+      info "reading #{directory}"
 
       unless File.directory? directory
-        warn "WARNING: #{directory}: no such directory"
+        warn "#{directory}: no such directory"
         return
       end
 
@@ -120,24 +133,38 @@ module MiddleSquid
 
       @indexed_cats << category
 
-      puts "indexing #{dirname}/#{pn.basename}"
+      total_size = File.size path
+      current_pos = percent = reported = 0
+
+      status = "\rindexing #{dirname}/#{pn.basename} [%d%%]"
+      output status % percent
 
       File.foreach(path) {|line|
+        current_pos += line.bytesize
+        percent = (current_pos.to_f / total_size * 100).to_i
+
+        if percent != reported
+          output status % percent
+          reported = percent
+        end
+
         type = append_to category, line
         @total[type] += 1
       }
+
+      output "\n"
     end
 
     def append_to(category, line)
+      # fix invalid UTF-8 byte sequences
+      line.scrub! ''
+
       # remove trailing whitespace
       line.strip!
 
       # ignore regex lists
       return :ignored unless line[0] =~ /\w/
 
-      # fix invalid bytes
-      line.scrub! ''
-
       # fix for dirty lists
       line.tr! '\\', '/'
 
@@ -167,26 +194,26 @@ module MiddleSquid
       @indexed_cats.uniq!
       missing_cats = @cats_in_use - @indexed_cats
 
-      puts
-      puts "indexed #{@indexed_cats.size} categorie(s): #{@indexed_cats}"
-      warn "WARNING: could not find #{missing_cats}" unless missing_cats.empty?
+      info
+      info "indexed #{@indexed_cats.size} categorie(s): #{@indexed_cats}"
+      warn "could not find #{missing_cats}" unless missing_cats.empty?
     end
 
     def stats
-      puts "found #{@total[:domain]} domain(s)"
-      puts "found #{@total[:url]} url(s)"
-      puts "found #{@total[:duplicate]} duplicate(s)"
-      puts "found #{@total[:ignored]} ignored expression(s)"
-      puts
+      info "found #{@total[:domain]} domain(s)"
+      info "found #{@total[:url]} url(s)"
+      info "found #{@total[:duplicate]} duplicate(s)"
+      info "found #{@total[:ignored]} ignored expression(s)"
+      info
     end
 
     def commit_or_rollback
       if @total[:domain] > 0 || @total[:url] > 0
-        puts 'committing changes'
+        info 'committing changes'
         db.commit
       else
-        warn 'ERROR: nothing to commit'
-        puts 'reverting changes'
+        oops 'nothing to commit'
+        info 'reverting changes'
         db.rollback
       end
     end

data/lib/middle_squid/version.rb

@@ -1,3 +1,3 @@
 module MiddleSquid
-  VERSION = '1.0'
+  VERSION = '1.0.1'
 end

data/test/resources/empty_config.rb

@@ -0,0 +1 @@
+run proc {}

data/test/resources/invalid_byte/cat/list

@@ -1 +1,2 @@
 host.com/path_with_�_invalid_byte
+host.com/invalid�byte

data/test/test_cli.rb

@@ -48,28 +48,30 @@ class TestCLI < MiniTest::Test
       MiddleSquid::CLI.start(%W[index #{list} -C #{conf} --full])
     end
 
-    assert_match /\Ahello #<MiddleSquid:.+>$/, stdout
-    assert_match "reading #{list}", stdout
+    assert_match /\Ahello #<MiddleSquid:.+>\Z/, stdout
+
+    assert_match "reading #{list}", stderr
   end
 
   def test_index_relative_path
     absolute = File.expand_path '../resources', __FILE__
     path = Pathname.new(absolute).relative_path_from(Pathname.new(Dir.home))
 
-    conf = File.join '~', path, 'hello.rb'
+    conf = File.join '~', path, 'empty_config.rb'
     list = File.join '~', path, 'black'
 
     stdout, stderr = capture_io do
       MiddleSquid::CLI.start(%W[index #{list} -C #{conf} --full])
     end
 
-    assert_match /\Ahello #<MiddleSquid:.+>$/, stdout
-    assert_match "reading #{absolute}/black", stdout
+    assert_empty stdout
+
+    assert_match "reading #{absolute}/black", stderr
   end
 
   def test_index_multiple
     path = File.expand_path '../resources', __FILE__
-    config = File.join path, 'hello.rb'
+    config = File.join path, 'empty_config.rb'
     list_1 = File.join path, 'black'
     list_2 = File.join path, 'gray'
 
@@ -77,8 +79,10 @@ class TestCLI < MiniTest::Test
       MiddleSquid::CLI.start(%W[index #{list_1} #{list_2} -C #{config} --full])
     end
 
-    assert_match "reading #{list_1}", stdout
-    assert_match "reading #{list_2}", stdout
+    assert_empty stdout
+
+    assert_match "reading #{list_1}", stderr
+    assert_match "reading #{list_2}", stderr
   end
 
   def test_version

data/test/test_helper.rb

@@ -70,7 +70,20 @@ class TestHelpers < MiniTest::Test
     assert_requested stub
 
     assert_equal 520, status
-    assert_empty headers
+    assert_equal({'Content-Type' => 'text/plain'}, headers)
     assert_equal '[MiddleSquid] WebMock timeout error', body
   end
+
+  def test_download_from_string
+    uri = 'http://test.com/' # not a URI object
+
+    stub = stub_request(:get, uri).
+      to_return(:status => 200, :body => '')
+
+    download_wrapper uri,
+      'REQUEST_METHOD' => 'GET',
+      'rack.input' => StringIO.new
+
+    assert_requested stub
+  end
 end

data/test/test_indexer.rb

@@ -42,22 +42,22 @@ class TestIndexer < MiniTest::Test
       @obj.index [File.join(@path, 'black')]
     end
 
-    assert_equal "nothing to do in minimal indexing mode\n", stdout
-
-    assert_match 'ERROR', stderr
+    assert_equal <<-OUT, stderr
+ERROR: the loaded configuration does not use any blacklist
+nothing to do in minimal indexing mode
+    OUT
 
     assert has_test_data?
   end
 
-  def test_empty_rollback
+  def test_empty_file
     stdout, stderr = capture_io do
       @obj.index [File.join(@path, 'empty')]
     end
 
-    assert_match 'indexing cat/emptylist', stdout
-    assert_match 'reverting changes', stdout
-
+    assert_match 'indexing cat/emptylist', stderr
     assert_match 'ERROR: nothing to commit', stderr
+    assert_match 'reverting changes', stderr
 
     assert has_test_data?
   end
@@ -84,18 +84,18 @@ class TestIndexer < MiniTest::Test
       ['tracker', '.cloudfront-labs.amazonaws.com', 'x.png/'],
     ], urls
 
-    assert_match 'indexing ads/urls', stdout
-    assert_match 'indexing ads/domains', stdout
-    assert_match 'indexing tracker/urls', stdout
-    assert_match 'indexing tracker/domains', stdout
-    assert_match 'indexed 2 categorie(s): ["ads", "tracker"]', stdout
-    assert_match 'found 4 domain(s)', stdout
-    assert_match 'found 3 url(s)', stdout
-    assert_match 'found 0 duplicate(s)', stdout
-    assert_match 'found 0 ignored expression(s)', stdout
-    assert_match 'committing changes', stdout
-
-    assert_empty stderr
+    assert_match 'indexing ads/urls', stderr
+    assert_match 'indexing ads/urls', stderr
+    assert_match 'indexing ads/domains', stderr
+    assert_match 'indexing ads/domains', stderr
+    assert_match 'indexing tracker/urls', stderr
+    assert_match 'indexing tracker/domains', stderr
+    assert_match 'indexed 2 categorie(s): ["ads", "tracker"]', stderr
+    assert_match 'found 4 domain(s)', stderr
+    assert_match 'found 3 url(s)', stderr
+    assert_match 'found 0 duplicate(s)', stderr
+    assert_match 'found 0 ignored expression(s)', stderr
+    assert_match 'committing changes', stderr
   end
 
   def test_index_multiple
@@ -128,12 +128,10 @@ class TestIndexer < MiniTest::Test
       ['isp', '.telus.com', 'content/internet/'],
     ], urls
 
-    assert_match 'indexed 4 categorie(s): ["ads", "tracker", "isp", "news"]', stdout
-    assert_match 'found 8 domain(s)', stdout
-    assert_match 'found 4 url(s)', stdout
-    assert_match 'found 0 duplicate(s)', stdout
-
-    assert_empty stderr
+    assert_match 'indexed 4 categorie(s): ["ads", "tracker", "isp", "news"]', stderr
+    assert_match 'found 8 domain(s)', stderr
+    assert_match 'found 4 url(s)', stderr
+    assert_match 'found 0 duplicate(s)', stderr
   end
 
   def test_ignore_subdirectories
@@ -141,7 +139,8 @@ class TestIndexer < MiniTest::Test
       @obj.index [File.join(@path, 'subdirectory')]
     end
 
-    refute_match 'cat/ignore', stdout
+    refute_match 'cat/ignore', stderr
+
     assert has_test_data?
   end
 
@@ -166,10 +165,8 @@ class TestIndexer < MiniTest::Test
       ['ads', '.google.com', 'adsense/'],
     ], urls
 
-    refute_match 'tracker', stdout
-    assert_match 'indexed 1 categorie(s): ["ads"]', stdout
-
-    assert_empty stderr
+    refute_match 'tracker', stderr
+    assert_match 'indexed 1 categorie(s): ["ads"]', stderr
   end
 
   def test_not_found
@@ -179,7 +176,7 @@ class TestIndexer < MiniTest::Test
 
     assert has_test_data?
 
-    assert_match "reading #{File.join @path, '404'}", stdout
+    assert_match "reading #{File.join @path, '404'}", stderr
 
     assert_match "WARNING: #{File.join @path, '404'}: no such directory\n", stderr
     assert_match "ERROR: nothing to commit", stderr
@@ -195,8 +192,8 @@ class TestIndexer < MiniTest::Test
 
     refute has_test_data?
 
-    assert_match "reading #{File.join @path, '404'}", stdout
-    assert_match "reading #{File.join @path, 'gray'}", stdout
+    assert_match "reading #{File.join @path, '404'}", stderr
+    assert_match "reading #{File.join @path, 'gray'}", stderr
 
     assert_match "WARNING: #{File.join @path, '404'}: no such directory\n", stderr
   end
@@ -248,6 +245,7 @@ class TestIndexer < MiniTest::Test
     urls = db.execute 'SELECT category, host, path FROM urls'
     assert_equal [
       ['cat', '.host.com', 'path_with__invalid_byte/'],
+      ['cat', '.host.com', 'invalidbyte/'],
     ], urls
   end
 
@@ -289,10 +287,8 @@ class TestIndexer < MiniTest::Test
       ['copy_of_cat', '.host.com', 'path/'],
     ], urls
 
-    assert_match 'found 12 duplicate(s)', stdout
-    assert_match 'found 0 ignored expression(s)', stdout
-
-    assert_empty stderr
+    assert_match 'found 12 duplicate(s)', stderr
+    assert_match 'found 0 ignored expression(s)', stderr
   end
 
   def test_missing_category
@@ -315,7 +311,7 @@ class TestIndexer < MiniTest::Test
 
     assert has_test_data?
 
-    assert_match 'found 3 ignored expression(s)', stdout
+    assert_match 'found 3 ignored expression(s)', stderr
     assert_match 'ERROR: nothing to commit', stderr
   end
 
@@ -342,11 +338,9 @@ class TestIndexer < MiniTest::Test
       ['cat_name', '.google.com', 'adsense/'],
     ], urls
 
-    refute_match 'tracker', stdout
-    assert_match 'indexing ads/', stdout
-    assert_match 'indexed 1 categorie(s): ["cat_name"]', stdout
-
-    assert_empty stderr
+    refute_match 'tracker', stderr
+    assert_match 'indexing ads/', stderr
+    assert_match 'indexed 1 categorie(s): ["cat_name"]', stderr
   end
 
   def test_domains_only
@@ -369,11 +363,9 @@ class TestIndexer < MiniTest::Test
     urls = db.execute 'SELECT category, host, path FROM urls'
     assert_empty urls
 
-    assert_match 'found 4 domain(s)', stdout
-    assert_match 'found 0 url(s)', stdout
-    assert_match 'found 3 ignored expression(s)', stdout
-
-    assert_empty stderr
+    assert_match 'found 4 domain(s)', stderr
+    assert_match 'found 0 url(s)', stderr
+    assert_match 'found 3 ignored expression(s)', stderr
   end
 
   def test_urls_only
@@ -395,11 +387,9 @@ class TestIndexer < MiniTest::Test
       ['tracker', '.cloudfront-labs.amazonaws.com', 'x.png/'],
     ], urls
 
-    assert_match 'found 0 domain(s)', stdout
-    assert_match 'found 3 url(s)', stdout
-    assert_match 'found 4 ignored expression(s)', stdout
-
-    assert_empty stderr
+    assert_match 'found 0 domain(s)', stderr
+    assert_match 'found 3 url(s)', stderr
+    assert_match 'found 4 ignored expression(s)', stderr
   end
 
   def test_append
@@ -410,10 +400,10 @@ class TestIndexer < MiniTest::Test
     end
 
     assert has_test_data?, 'should not be truncated'
-    refute_match 'truncating', stdout
 
-    assert_match 'found 4 domain(s)', stdout
-    assert_match 'found 3 url(s)', stdout
+    refute_match 'truncating', stderr
+    assert_match 'found 4 domain(s)', stderr
+    assert_match 'found 3 url(s)', stderr
   end
 
   def test_quiet
@@ -454,4 +444,19 @@ class TestIndexer < MiniTest::Test
       ['cat', '.after.com', 'path/'],
     ], urls
   end
+
+  def test_progress
+    stdout, stderr = capture_io do
+      @obj.index [File.join(@path, 'black')]
+    end
+
+    lines = stderr.lines.select {|l| l =~ /indexing/ }
+
+    assert_equal [
+      "\rindexing ads/domains [0%]\rindexing ads/domains [48%]\rindexing ads/domains [100%]\n",
+      "\rindexing ads/urls [0%]\rindexing ads/urls [100%]\n",
+      "\rindexing tracker/domains [0%]\rindexing tracker/domains [30%]\rindexing tracker/domains [100%]\n",
+      "\rindexing tracker/urls [0%]\rindexing tracker/urls [40%]\rindexing tracker/urls [100%]\n",
+    ], lines
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: middle_squid
 version: !ruby/object:Gem::Version
-  version: '1.0'
+  version: 1.0.1
 platform: ruby
 authors:
 - Christian Fillion
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-06 00:00:00.000000000 Z
+date: 2014-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -270,6 +270,7 @@ files:
 - test/resources/duplicates/copy_of_cat/copy_of_list
 - test/resources/duplicates/copy_of_cat/list
 - test/resources/empty/cat/emptylist
+- test/resources/empty_config.rb
 - test/resources/empty_path/cat/list
 - test/resources/expressions/cat/list
 - test/resources/gray/isp/domains
@@ -335,6 +336,7 @@ test_files:
 - test/resources/duplicates/copy_of_cat/copy_of_list
 - test/resources/duplicates/copy_of_cat/list
 - test/resources/empty/cat/emptylist
+- test/resources/empty_config.rb
 - test/resources/empty_path/cat/list
 - test/resources/expressions/cat/list
 - test/resources/gray/isp/domains