scrapula 0.6.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3d346e08f162bc19ef64bc931986df984fc56aa2
+  data.tar.gz: e64475c7d6cc3cfe5075981e5152931e23423458
+SHA512:
+  metadata.gz: cf881c26643ea5a11fc1a6f2f1a651f9f2321f9a046483586e63366a1dd280246c4ca29e04b74c6f8ddb97655ed4893270b11c7bc95fbd1ed30ddddab85b3900
+  data.tar.gz: f7951f02b12b0affea85431b2a30d5a9b656e97534b89c8ec411be8f5d68cf090f2edf5d4ac8313ef3f2aa9fdfdc8a6f1efb3537043b54fff8b2cc11379b7de4

data/.gitignore

@@ -0,0 +1,3 @@
+coverage/
+tmp/
+*.gem

data/.rspec

@@ -0,0 +1 @@
+--color --require spec_helper

data/.simplecov

@@ -0,0 +1 @@
+SimpleCov.start

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# CHANGELOG
+
+## 0.6.3 (2015-09-17) Juan A. Martín Lucas <scrapula@jaml.site>
+
+ * Published to RubyGems.org.
+ * Prepare the `scrapula.gemspec` for publishing to RubyGems.org.
+ * Improve the project description in the README.
+
+## 0.6.2 (2015-09-12) Juan A. Martín Lucas <scrapula@jaml.site>
+
+ * "S" API shortcuts (e.g: `S.g` => `Scrapula.get`)
+
+## 0.6.1 (2015-09-10) Juan A. Martín Lucas <scrapula@jaml.site>
+
+  * Implement `Scraper#respond_to?` tests (for specifying its expected behaviour)

data/Gemfile

@@ -0,0 +1,24 @@
+source 'http://rubygems.org'
+
+gemspec
+
+group :development do
+	gem 'rake'
+
+  gem 'guard'
+  gem 'guard-bundler'
+  gem 'guard-rspec'
+end
+
+group :test do
+	gem 'rspec'
+
+  gem 'webmock'
+  gem 'vcr'
+
+  gem 'simplecov'
+end
+
+group :development, :test do
+  gem 'byebug'
+end

data/Gemfile.lock

@@ -0,0 +1,127 @@
+PATH
+  remote: .
+  specs:
+    scrapula (0.6.3)
+      mechanize (~> 2.7, >= 2.7.3)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    byebug (5.0.0)
+      columnize (= 0.9.0)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    coderay (1.1.0)
+    columnize (0.9.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    ffi (1.9.8)
+    formatador (0.2.5)
+    guard (2.12.5)
+      formatador (>= 0.2.4)
+      listen (~> 2.7)
+      lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-bundler (2.1.0)
+      bundler (~> 1.0)
+      guard (~> 2.2)
+      guard-compat (~> 1.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.5.0)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    hitimes (1.2.2)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.2)
+    listen (2.10.0)
+      celluloid (~> 0.16.0)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.9)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    method_source (0.8.2)
+    mime-types (2.6.2)
+    mini_portile (0.6.2)
+    nenv (0.2.0)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    notiffany (0.0.6)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    ntlm-http (0.1.1)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.4.2)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.3)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.2)
+    safe_yaml (1.0.4)
+    shellany (0.0.1)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    thor (0.19.1)
+    timers (4.0.1)
+      hitimes
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    vcr (2.9.2)
+    webmock (1.17.4)
+      addressable (>= 2.2.7)
+      crack (>= 0.3.2)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  guard
+  guard-bundler
+  guard-rspec
+  rake
+  rspec
+  scrapula!
+  simplecov
+  vcr
+  webmock

data/Guardfile

@@ -0,0 +1,12 @@
+guard :bundler do
+  watch 'Gemfile'
+end
+
+guard :rspec, cmd: 'rspec --format progress --color -r ./spec/spec_helper.rb', all_on_start: true, all_after_pass: true do
+
+  watch 'spec/spec_helper.rb'
+
+  watch(%r{^lib/(.+)\.rb$}) {|m| "spec/lib/#{m[1]}_spec.rb"}
+  watch(%r{^spec/.+_spec\.rb$})
+
+end

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2012-2015 Juan A. Martín Lucas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,108 @@
+[![Build Status](TODO)
+
+Scrapula
+========
+Scrapula is a library for scraping web pages that simplifies some of the
+common actions that are involved.
+
+It has a very simple API that can be used in several ways and contexts, and
+another, shorter, that facilitates processing pages when characters are
+scarce, like irb / pry, or quick and dirty scripts.
+
+Requirements
+============
+It uses [Mechanize](http://mechanize.rubyforge.org/) and [Nokogiri](http://nokogiri.org) to obtain and extract the information and [RSpec](https://www.relishapp.com/rspec) for testing.
+
+Configuration
+=============
+If you want to show the output of some steps:
+
+```ruby
+Scrapula.verbose = true
+```
+
+API
+===
+
+Perform requests:
+
+```ruby
+page = Scrapula.get 'example.net' #=> Scrapula::Page object
+
+page = Scrapula.post 'example.net', { q: 'a query' }   #=> Scrapula::Page object
+```
+
+Extract information from the page:
+
+```ruby
+# Using a CSS selector (all elements)
+page.search! 'a'
+
+# Using a CSS selector (fist element)
+page.at! 'h1'
+
+# Using XPath (fist element)
+page.at! '//'
+```
+
+Perform a GET request:
+
+```ruby
+Scrapula.get 'example.net
+```
+
+S interface
+-----------
+This API is not required by default, so it is up to you to use it:
+```ruby
+require 'scrapula/s'
+```
+
+It provides the method and its shortcut For all HTTP verbs:
+
+```ruby
+S.get 'example.net'
+S.g 'example.net'
+
+S.post 'example.net'
+S.p 'example.net'
+
+S.put 'example.net'
+S.u 'example.net'
+
+S.patch 'example.net'
+S.a 'example.net'
+
+S.delete 'example.net'
+S.d 'example.net'
+
+S.head 'example.net'
+S.h 'example.net'
+```
+
+Additionally, GET requests, can be performed with through the shortest invocation:
+
+```ruby
+S 'example.net'
+```
+
+Examples
+--------
+
+There are more examples in the `examples` folder.
+
+Changelog
+=========
+
+You can read previous changes in `CHANGELOG.md`
+
+Contributing
+============
+
+Authors
+=======
+Juan A. Martín Lucas (https://github.com/j-a-m-l)
+
+License
+=======
+This project is licensed under the MIT license. See [LICENSE]() for details.

data/ROADMAP.md

@@ -0,0 +1,42 @@
+# ROADMAP
+
+This is a proposal and should not be interpreted literally. If you want to collaborate in some feature before its version, do it.
+
+## 3.0.0
+
+ * Other agents than Mechanize
+ * Uncommon HTTP methods: OPTIONS, CONNECT, TRACE
+
+## 2.0.0
+
+ * Page | Scraping schema definition
+ * Page | Scraping schema validation
+ * Scraping operations: `regex!`
+ * Scraping operations: `int!`
+ * Scraping operations: `decimal!`
+ * Scraping operations: `number!`
+ * Scraping operations: `date!`
+ * Scraping operations: `time!`
+ * Scraping operations: `datetime!`
+ * Non-standard behaviour of some known non-standard metas, like og:image
+
+## 1.0.0
+
+ * Documented with TODO
+
+## 0.9.0
+
+ * Nested block syntax
+ * Scraping operations: `attribute!`
+ * Scraping operations: `inner_html!` ?
+ * Scraping operations: `xhtml!` ?
+ * Scraping operations: `title!` ?
+
+## 0.8.0
+
+ * Add pending HTTP methods: POST, PUT, PATCH, DELETE and HEAD
+ * Request parameters
+
+## 0.7.0
+
+ * Timeouts

data/Rakefile

@@ -0,0 +1,30 @@
+require 'rspec/core/rake_task'
+
+require './lib/scrapula/version'
+name = 'scrapula'
+version = Scrapula::VERSION
+
+task :default => :coverage
+
+desc "Run specs!"
+RSpec::Core::RakeTask.new do |t|
+	t.rspec_opts = ['--color', '--debug']
+	t.pattern = "./spec/**/*_spec.rb"
+end
+
+# Alias
+task :test => :spec
+
+desc "Happy coverage"
+task :coverage do
+	ENV['COVERAGE'] = 'true'
+	Rake::Task[:spec].execute
+end
+
+task :build do
+	system "gem build #{name}.gemspec"
+end
+ 
+task :install => :build do
+	system "gem install #{name}-#{version}"
+end

data/examples/block_syntax.rb

@@ -0,0 +1,20 @@
+require_relative '../lib/scrapula'
+
+page = Scrapula.get 'reddit.com'
+
+hs1 = page.scrape do
+  h1 'h1'
+  h2 'h2'
+  h3 'h3'
+  h4 'h4'
+  h5 'h5'
+end
+
+# Alternative way
+hs2 = page.scrape
+  # TODO
+  txt!({ h1: 'h1', h2: 'h2', h3: 'h3', h4: 'h4', h5: 'h5' })
+end
+
+puts hs1
+puts hs2

data/examples/find_nodes.rb

@@ -0,0 +1,6 @@
+require_relative '../lib/scrapula'
+
+page = Scrapula.get 'reddit.com'
+h1 = page.at! 'h1'
+
+puts h1

data/examples/get_first_and_scrape_later.rb

@@ -0,0 +1,13 @@
+require_relative '../lib/scrapula'
+
+query = 'ruby scraping'
+page = Scrapula.get 'stackoverflow.com/search', q: query
+
+# Extract the number of results (with newline characters and spaces)
+number_of_results = page.text! '.results-header h2'
+
+# Apply the operation (method) strip to the result
+first_result = page.txt! '.search-result[data-position="1"] .result-link', [:strip]
+
+puts "Stack Overflow returns #{number_of_results.strip} for \"#{query}\""
+puts "Stack Overflow returns \"#{first_result}\" as first result for \"#{query}\""