sq 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41ba49d5ce6c8cf47cbf2b2ac14b6cec6a937bd5
-  data.tar.gz: bea18aa8b03dd5770bd957befc8bb4f86f4f97a5
+  metadata.gz: 12b002f2a938d9096d85360736c3949d8ca60935
+  data.tar.gz: 023f25cd005f28611644a9b6cecc89f703a50b61
 SHA512:
-  metadata.gz: d005f3c16ead6a5cfd85a5f5e0f155ebe7a041eaf8a295ad809645ccb28140dee885e11e26c28d8c795766b14318586ac02ba4e4c3bfa81c3ac64070618635a3
-  data.tar.gz: d648efad30b62583ae3da4d06dd658c95664321075cbe6917fc17031bbbf3554ecd58088b8469f7ba1f11d822f27a5ca4f7b6109400cddf2225796f43eb90427
+  metadata.gz: 34ad6b35d57baa6b039ccca9a245f1675205b50a82317645414fafc91742c43c5a14ac230f36cdffe4103d8e06a6e452f04d77e19a54f3b1363a996e405f5d99
+  data.tar.gz: deda0c2bf3f2168db8ea07b52b348b310a4b6d810f4d1605ac41f3506f545648de6f21cc85882742ec2fcab273417c42486b082fb290ef6488e3c64bf7244566

data/lib/sq.rb

@@ -4,14 +4,20 @@ require 'uri'
 require 'open-uri'
 require 'nokogiri'
 require 'fileutils'
+require 'ruby-progressbar'
 require File.expand_path(File.dirname __FILE__) + '/version'
 
 module SQ
   class << self
+    # return the user-agent used by SQ
     def user_agent
       "SQ/#{version} +github.com/bfontaine/sq"
     end
 
+    # query an URI and return a list of PDFs. Each PDF is an hash with two
+    # keys: :uri is its absolute URI, :name is its name (last part of its URI).
+    # @uri [String]
+    # @regex [Regexp]
     def query(uri, regex=/./)
       uri = 'http://' + uri unless uri =~ /^https?:\/\//
 
@@ -29,10 +35,17 @@ module SQ
       end
     end
 
+    # query an URI and download all PDFs which match the regex. It returns the
+    # number of downloaded PDFs.
+    # @uri   [String]
+    # @regex [Regexp] Regex to use to match PDF URIs
+    # @opts  [Hash]   Supported options: :verbose, :directory (specify the
+    #                 directory to use for output instead of the current one)
     def process(uri, regex=/./, opts={})
       uris = self.query(uri, regex)
+      count = uris.count
 
-      puts "Found #{uris.count} PDFs." if opts[:verbose]
+      puts "Found #{count} PDFs:" if opts[:verbose]
 
       return 0 if uris.empty?
 
@@ -40,17 +53,20 @@ module SQ
 
       unless Dir.exists?(out)
         puts "-> mkdir #{out}" if opts[:verbose]
-        Dir.mkdir(out)
+        FileUtils.mkdir_p(out)
       end
 
+      p = ProgressBar.create(:title => "PDFs", :total => count)
+
       uris.each do |u|
-        puts "Downloading #{u[:name]}..." if opts[:verbose]
         open("#{out}/#{u[:name]}", 'wb') do |f|
           open(u[:uri], 'rb') do |resp|
             f.write(resp.read)
+            p.log u[:name] if opts[:verbose]
+            p.increment
           end
         end
-      end
+      end.count
     end
   end
 end

data/lib/version.rb

@@ -3,7 +3,7 @@
 module SQ
   class << self
     def version
-      '0.0.2'
+      '0.1.0'
     end
   end
 end

data/tests/process_tests.rb

@@ -0,0 +1,69 @@
+# -*- coding: UTF-8 -*-
+
+require 'tmpdir'
+require 'fileutils'
+require File.dirname(__FILE__) + '/fake_responses'
+
+class SQ_process_test < Test::Unit::TestCase
+
+  def setup
+    @url  = 'example.com'
+    @http = "http://#{@url}"
+
+    @prev_path = Dir.pwd
+    @test_path = Dir.mktmpdir('sq-tests')
+    Dir.chdir @test_path
+  end
+
+  def teardown
+    Dir.chdir @prev_path
+    FileUtils.rm_rf @test_path
+  end
+
+  def test_no_links
+    assert_equal(0, SQ.process("#{@url}/no-links", /./))
+  end
+
+  def test_one_link_no_dir
+    assert_equal(1, SQ.process("#{@url}/one", /./))
+    assert(File.exists?('bar.pdf'), 'bar.pdf exists')
+    assert_equal('%PDFbar', File.read('bar.pdf'))
+  end
+
+  def test_one_link_existing_dir
+    dir = 'foo'
+    Dir.mkdir dir
+    assert_equal(1, SQ.process("#{@url}/one", /./, :directory => dir))
+    assert(File.exists?("#{dir}/bar.pdf"), "#{dir}/bar.pdf exists")
+    assert_equal('%PDFbar', File.read("#{dir}/bar.pdf"))
+  end
+
+  def test_one_link_existing_subdir
+    dir = 'foo/bar/qux'
+    FileUtils.mkdir_p dir
+    assert_equal(1, SQ.process("#{@url}/one", /./, :directory => dir))
+    assert(File.exists?("#{dir}/bar.pdf"), "#{dir}/bar.pdf exists")
+    assert_equal('%PDFbar', File.read("#{dir}/bar.pdf"))
+  end
+
+  def test_one_link_unexisting_subdir
+    dir = 'foo/bar/qux'
+    assert_equal(1, SQ.process("#{@url}/one", /./, :directory => dir))
+    assert(Dir.exists?(dir), "#{dir} exists")
+    assert(File.exists?("#{dir}/bar.pdf"), "#{dir}/bar.pdf exists")
+    assert_equal('%PDFbar', File.read("#{dir}/bar.pdf"))
+  end
+
+  def test_two_links_unexisting_subdir
+    dir = 'foo/bar/qux'
+    assert_equal(2, SQ.process("#{@url}/two", /./, :directory => dir))
+    assert(Dir.exists?(dir), "#{dir} exists")
+
+    assert(File.exists?("#{dir}/bar.pdf"), "#{dir}/bar.pdf exists")
+    assert_equal('%PDFbar', File.read("#{dir}/bar.pdf"))
+
+    assert(File.exists?("#{dir}/foo.pdf"), "#{dir}/foo.pdf exists")
+    assert_equal('%PDFfoo', File.read("#{dir}/foo.pdf"))
+  end
+
+end

data/tests/query_tests.rb

@@ -0,0 +1,53 @@
+# -*- coding: UTF-8 -*-
+
+require File.dirname(__FILE__) + '/fake_responses'
+
+class SQ_query_test < Test::Unit::TestCase
+
+  def setup
+    @url  = 'example.com'
+    @http = "http://#{@url}"
+  end
+
+  def test_no_links
+    assert_equal([], SQ.query("#{@url}/no-links", /./))
+  end
+
+  def test_no_href
+    assert_equal([], SQ.query("#{@url}/no-href", /./))
+  end
+
+  def test_no_pdfs
+    assert_equal([], SQ.query("#{@url}/no-pdf", /./))
+  end
+
+  def test_no_match
+    assert_equal([], SQ.query("#{@url}/bar", /foo/))
+  end
+
+  def test_full_match
+    pdfs = [
+      {:uri => "#{@http}/bar1.pdf", :name => 'bar1.pdf'},
+      {:uri => "#{@http}/bar2.pdf", :name => 'bar2.pdf'}
+    ]
+    assert_equal(pdfs, SQ.query("#{@url}/bar", /./))
+    assert_equal(pdfs, SQ.query("#{@http}/bar", /./))
+  end
+
+  def test_absolute_path
+    pdfs = [
+      {:uri => "#{@http}/bar1.pdf", :name => 'bar1.pdf'}
+    ]
+    assert_equal(pdfs, SQ.query("#{@url}/ab/so/lu/te", /./))
+  end
+
+  def test_malformed_html
+    pdfs = [
+      {:uri => "#{@http}/bar1.pdf", :name => 'bar1.pdf'}
+    ]
+    assert_equal(pdfs, SQ.query("#{@url}/malformed1", /./))
+    assert_equal(pdfs, SQ.query("#{@url}/malformed2", /./))
+    assert_equal(pdfs, SQ.query("#{@url}/malformed3", /./))
+  end
+
+end

data/tests/tests.rb

@@ -16,12 +16,17 @@ end
 
 class SQTests < Test::Unit::TestCase
 
-  # == UD#version == #
+  # == SQ#version == #
 
   def test_sq_version
     assert(SQ.version =~ /^\d+\.\d+\.\d+/)
   end
 
+  # == SQ#user_agent == #
+  def test_sq_ua
+    assert(SQ.user_agent =~ /^SQ\/\d+\.\d+\.\d+/)
+  end
+
 end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sq
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Baptiste Fontaine
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-13 00:00:00.000000000 Z
+date: 2014-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -52,6 +52,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -117,6 +131,8 @@ extra_rdoc_files: []
 files:
 - lib/sq.rb
 - lib/version.rb
+- tests/process_tests.rb
+- tests/query_tests.rb
 - tests/tests.rb
 - bin/sq
 homepage: https://github.com/bfontaine/sq
@@ -144,4 +160,6 @@ signing_key:
 specification_version: 4
 summary: Bulk PDFs downloader
 test_files:
+- tests/process_tests.rb
+- tests/query_tests.rb
 - tests/tests.rb