scrapouille 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0edc7fb52550844caa8d189373a577b113462c8
-  data.tar.gz: 5b76257e9540d9f92e46d502d8346518fba1589b
+  metadata.gz: 9aa9621a90f3ebb7e73aa8d9ea6e46826d78bc7f
+  data.tar.gz: 95f49609599be7100ff1b3ebd05c90ae6364a0b1
 SHA512:
-  metadata.gz: 4b1924dab7efceb32eea3a684e6a42a294d746ed98f8bd30096640e100e4c54419d8f3dfad1af5492a60d19ae1cb77594cb4c28a0f3e8e3f0451a4176e05adeb
-  data.tar.gz: ca5405c1da48f22add8cb924df2df458d7ab3470b2e017f87ac4d17dde677fe0794ae03751ab172fd41eac6eb623ebc7108c0c5405d63b63b77c45f8f70dcb9d
+  metadata.gz: 87ad778ca5d8be90001921f492cce90271a82a64529c26dbfb305fb578eacea0f2214d45d14ecab7731623fe5d600de3a213d7669e0347fb9427ed48ae4d7fc0
+  data.tar.gz: 01d1ec1ee5757269cf47c222472d7f63194189246c0d877a6bd15262bf06473e1642705d62e9baf3cdd056eb2a5255d042ebf630e0dda2652d4ebbe23f65407e

data/README.md

@@ -1,35 +1,61 @@
-Declarative HTML scraper
+# Scrapouille
 
-# Usage
+Scrapouille is a declarative XPath driven HTML scraper with an interactive mode as a bonus
+
+Why XPath ? XPath is powerful enough to get any data on a HTML document (see http://www.w3schools.com/xpath/xpath_axes.asp)
+
+Scrapouille run XPath queries using the **nokogiri** gem
+
+### Install
+
+    gem install 'scrapouille'
 
 ### Test
 
     rake
 
-### Scrap
+# Usage
+
+### Interactive mode
+
+From the command line you can interact with a remote web page as if it was local
+
+    $ scrapouille http://tennis.com/player.html        # launch scrapouille on the command line with a provided URI
+    > //div[@class='player-name']/h1/child::text()     # You will get a prompt. Enter a xpath query
+    Richard Gasquest                                   # Get the result string
+
+**Behind the scene - during the session - the remote web page is stored in a `Tempfile` for fast xpath interaction**
+
+You can also directly interact with a local file
+
+    $ scrapouille /Users/simon/web/player.html         # launch scrapouille on the command line with a provided filepath
+    > //div[@class='player-name']/h1/child::text()     # enter your xpath query
+    Richard Gasquest                                   # Get the result String
+
+### Scraping programatically
 
 Define a scraper
 
-    ```ruby
-    scraper = Scrapouille.new do
-      scrap 'fullname', at: "//div[@class='player-name']/h1/child::text()"
-      scrap 'image_url', at: "//div[@id='basic']//img/attribute::src"
-      scrap 'rank', at: "//div[@class='position']/text()" do |c|
-        Integer(c.sub('#', ''))
-      end
-    end
-    ```
+```ruby
+scraper = Scrapouille.new do
+  scrap 'fullname', at: "//div[@class='player-name']/h1/child::text()"
+  scrap 'image_url', at: "//div[@id='basic']//img/attribute::src"
+  scrap 'rank', at: "//div[@class='position']/text()" do |c|
+    Integer(c.sub('#', ''))
+  end
+end
+```
 
-Use you scraper instance on an URI (as defined by `open-uri`: filepath, http, ...)
+Use the scraper instance on an URI (as defined by `open-uri`: filepath, http, ...)
 
-    ```ruby
-    results = scraper.scrap!('http://tennis-player.com/richard-gasquet')
-    results['fullname'] # => 'Richard Gasquest'
-    ```
+```ruby
+results = scraper.scrap!('http://tennis-player.com/richard-gasquet')
+results['fullname'] # => 'Richard Gasquest'
+```
 
-You can test your xpath expression with a local HTML filepath
+You can also run your scraper using a local HTML filepath for testing purposes
 
-    ```ruby
-    scraper.scrap!(File.join('..', 'player.html'))
-    ```
+```ruby
+scraper.scrap!(File.join('..', 'player.html'))
+```
 

data/Rakefile

@@ -4,7 +4,6 @@ require 'rake/testtask'
 
 Rake::TestTask.new do |t|
   t.libs << 'test'
-  t.test_files = FileList['test/**/*_spec.rb']
 end
 
 task default: :test

data/bin/scrapouille

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'scrapouille'
+require 'readline'
+require 'open-uri'
+require 'tempfile'
+
+abort 'Missing uri parameter' if ARGV.empty?
+
+uri = URI(ARGV.shift)
+
+if uri.scheme && uri.scheme.start_with?('http')
+  file = Tempfile.new('scrapouille-')
+  puts 'Fetching remote content ...'
+  file.write(open(uri).read)
+  uri = file.path
+  puts "Tempfile created at #{file.path}"
+else
+  path = uri.to_s
+  abort "Cannot find file '#{path}'" unless File.exists?(path)
+end
+
+while provided_xpath = Readline.readline("> ", true)
+  scraper = Scrapouille.new
+  to_eval = "scrap :item, at: \"#{provided_xpath}\""
+  scraper.instance_eval(to_eval)
+  begin
+    puts scraper.scrap!(uri.to_s)[:item]
+  rescue => e
+    puts e.message
+  end
+end
+
+if file
+  puts "Closing and deleting temp file"
+  file.close
+  file.unlink
+end

data/scrapouille.gemspec

@@ -2,10 +2,10 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "scrapouille"
-  spec.version       = "0.0.2"
+  spec.version       = "0.0.3"
   spec.authors       = ["simcap"]
-  spec.summary       = %q{Simpe declarative HTML scrapper}
-  spec.description   = %q{Simpe declarative HTML scrapper}
+  spec.summary       = %q{Interactive and declarative XPath driven HTML scraper}
+  spec.description   = %q{Interactive and declarative XPath driven HTML scraper}
   spec.homepage      = "https://github.com/simcap/scrapouille"
 
   spec.files         = `git ls-files -z`.split("\x0")
@@ -17,4 +17,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
 end

data/test/helper.rb

@@ -1,4 +1,4 @@
 $LOAD_PATH << File.join(__dir__, '../lib')
+require 'scrapouille'
 
 require 'minitest/autorun'
-require 'scrapouille'

data/test/{scraping_spec.rb → test_scraping.rb}

@@ -1,6 +1,6 @@
 require 'helper'
 
-class ScrapingTest < MiniTest::Unit::TestCase
+class TestScraping < MiniTest::Unit::TestCase
 
   def test_one_player_scrapping
     scraper = Scrapouille.new do 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrapouille
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - simcap
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-17 00:00:00.000000000 Z
+date: 2014-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -52,9 +52,24 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Simpe declarative HTML scrapper
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Interactive and declarative XPath driven HTML scraper
 email: 
-executables: []
+executables:
+- scrapouille
 extensions: []
 extra_rdoc_files: []
 files:
@@ -64,11 +79,12 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/scrapouille
 - lib/scrapouille.rb
 - scrapouille.gemspec
 - test/fixtures/tennis-player.html
 - test/helper.rb
-- test/scraping_spec.rb
+- test/test_scraping.rb
 homepage: https://github.com/simcap/scrapouille
 licenses: []
 metadata: {}
@@ -91,8 +107,8 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Simpe declarative HTML scrapper
+summary: Interactive and declarative XPath driven HTML scraper
 test_files:
 - test/fixtures/tennis-player.html
 - test/helper.rb
-- test/scraping_spec.rb
+- test/test_scraping.rb