scrape 0.2 → 0.2.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    scrape (0.2)
+    scrape (0.2.1)
 
 GEM
   remote: http://rubygems.org/

data/README.md

@@ -36,9 +36,20 @@ Simply install the gem
 gem install scrape
 ```
 
+or you can download the source by cloning the repository
+
+```
+git clone https://github.com/evilmarty/scrape.git
+```
+
+## Contribute
+
+Please fork the repository and make a pull request on Github.
+
+If you discover an issue please [lodge it](https://github.com/evilmarty/scrape/issues).
+
 ## TODO
 
 * Fix bugs
-* Add support for Robots.txt
 * Depth limiting
 * Better docs

data/bin/scrape

@@ -2,7 +2,6 @@
 
 $: << File.expand_path('../../lib', __FILE__)
 
-require "scrape"
 require "scrape/cli"
 
-Scrape::CLI.new(File.basename($0)).run(ARGV)
+Scrape::CLI.new(File.basename($0), ARGV).run

data/lib/scrape/cli.rb

@@ -1,24 +1,26 @@
 require "optparse"
+require "scrape"
 
 class Scrape::CLI
-  attr_reader :command, :pwd
+  attr_reader :command, :app, :options
 
-  def initialize command, pwd = Dir.pwd
-    @command, @pwd = command, pwd
-  end
+  def initialize command, argv = ""
+    @command = command
+    @options = {:file => File.join(Dir.pwd, 'Scrapefile'), :ignore_robots_txt => false}
 
-  def run argv
-    options = {:file => File.join(pwd, 'Scrapefile')}
     opts = OptionParser.new do |opts|
       opts.banner = "Scrape #{Scrape::VERSION} - Usage: #{command} [options]"
       opts.separator ""
       opts.separator "Specific options:"
 
       opts.on "-f", "--scrapefile [FILE]", "Use FILE as scrapefile" do |file|
-        options[:file] = File.expand_path file
+        @options[:file] = File.expand_path file.strip
       end
       opts.on "-i", "--ignore-robots-txt", "Ignore robots.txt" do
-        options[:ignore_robots_txt] = true
+        @options[:ignore_robots_txt] = true
+      end
+      opts.on "-u", "--user-agent [AGENT]", "Change the user agent" do |agent|
+        Scrape.user_agent = agent.strip
       end
       opts.on_tail "-h", "--help", "Show this message" do
         puts opts
@@ -31,8 +33,15 @@ class Scrape::CLI
     end
     opts.parse argv
 
-    Scrape::Application.new(options.delete(:file), options).run
+    @app = Scrape::Application.new options[:file], options
+  end
 
+  def run
+    app.run
+    exit
+  rescue SystemExit, Interrupt
+    puts ""
+    exit
   rescue Scrape::FileNotFound
     puts "#{command} aborted!"
     puts "No Scrapefile found"

data/lib/scrape/version.rb

@@ -1,3 +1,3 @@
 module Scrape
-  VERSION = '0.2' unless defined? ::Scrape::VERSION
+  VERSION = '0.2.1' unless defined? ::Scrape::VERSION
 end

data/test/unit/cli_test.rb

@@ -0,0 +1,43 @@
+require "test_helper"
+require "scrape/cli"
+
+class CLITest < Scrape::TestCase
+  test "should use default file when none specified" do
+    cli = Scrape::CLI.new "test", ""
+    assert_equal File.join(Dir.pwd, 'Scrapefile'), cli.options[:file]
+  end
+
+  test "should use the specified file" do
+    cli = Scrape::CLI.new "test", "-f /tmp/test1.scrape"
+    assert_equal "/tmp/test1.scrape", cli.options[:file]
+  end
+
+  test "should not ignore robots.txt file when not specified" do
+    cli = Scrape::CLI.new "test", ""
+    assert_equal false, cli.options[:ignore_robots_txt]
+  end
+
+  test "should ignore robots.txt file when specified" do
+    cli = Scrape::CLI.new "test", "-i"
+    assert_equal true, cli.options[:ignore_robots_txt]
+  end
+
+  test "should set the user agent when specified" do
+    user_agent = Scrape.user_agent
+    cli = Scrape::CLI.new "test", "-u Test"
+    assert_equal "Test", Scrape.user_agent
+    Scrape.user_agent = user_agent
+  end
+
+  test "should exit when help is displayed" do
+    assert_raises SystemExit do
+      capture_io{ Scrape::CLI.new "test", "-h" }
+    end
+  end
+
+  test "should exit when version is displayed" do
+    assert_raises SystemExit do
+      capture_io{ Scrape::CLI.new "test", "-v" }
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrape
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -77,6 +77,7 @@ files:
 - test/support/test3.scrape
 - test/test_helper.rb
 - test/unit/application_test.rb
+- test/unit/cli_test.rb
 - test/unit/default_loader_test.rb
 - test/unit/match_test.rb
 - test/unit/robots_txt_rules_test.rb