skrape 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/Gemfile +3 -0
  4. data/LICENSE.txt +22 -0
  5. data/README.md +57 -0
  6. data/Rakefile +1 -0
  7. data/lib/skrape/version.rb +3 -0
  8. data/lib/skrape.rb +37 -0
  9. data/skrape.gemspec +26 -0
  10. data/spec/skrape_spec.rb +39 -0
  11. data/spec/spec_helper.rb +13 -0
  12. data/spec/test_data/example_com_raw_response +63 -0
  13. metadata +114 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: a431f8709c3f19242645f7f2221392417f9a089a
4
+ data.tar.gz: 878dd97f1c89db56dbbf78db16433ab05deb5e3c
5
+ SHA512:
6
+ metadata.gz: 37f91a440aeb8e90e82e33b069888e9e25dfedbceaef6243666cc6e824bf7da169cac57d28e8724a9393766f0549860fbb85d5a7562f4e4278abbad476140e43
7
+ data.tar.gz: 543871c6fa15105834452047c1a0076a853d2d7b92487c8d59fafd30a87bee2dd3df3bfdd9069cce1cf319f764f54eeefd2bbac5dbaccb0190828140ce71725b
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Mike Williamson
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,57 @@
1
+ # Skrape
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'skrape'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install skrape
18
+
19
+ ## Usage
20
+
21
+ Skrape provides a cute DSL for extracting information from pages on the
22
+ web. You give it a url and a block and it gives you back a hash.
23
+
24
+ Lets say you have a page like this:
25
+
26
+ <html><body><h1>I am a title</h1></body></html>
27
+
28
+ And you want a hash like this:
29
+
30
+ {title: "I am a title"}
31
+
32
+ You can run Skrape like this:
33
+
34
+ results = Skrape::Page.new("http://example.com").extract do
35
+ extract_title with: 'h1'
36
+ end
37
+
38
+ Skrape will give you the elements text by default, but in cases where
39
+ you are after something else, or need to do some massaging of the data,
40
+ you can pass a block:
41
+
42
+ results = Skrape::Page.new(url).extract do
43
+ extract_link_href with: 'a', and_run: proc {|link| link.attr('href').value }
44
+ end
45
+
46
+ The element(s) will be passed into the block as a
47
+ Nokogiri::XML::NodeSet for you to play with. Whatever text you return
48
+ will be added to the hash of things to return.
49
+
50
+
51
+ ## Contributing
52
+
53
+ 1. Fork it ( http://github.com/sleepycat/skrape/fork )
54
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
55
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
56
+ 4. Push to the branch (`git push origin my-new-feature`)
57
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/lib/skrape/version.rb ADDED
@@ -0,0 +1,3 @@
1
+ module Skrape
2
+ VERSION = "0.0.1"
3
+ end
data/lib/skrape.rb ADDED
@@ -0,0 +1,37 @@
1
+ require "skrape/version"
2
+ require 'nokogiri'
3
+ require 'open-uri'
4
+ require 'pry'
5
+
6
+ module Skrape
7
+
8
+ class NoElementsFoundError < StandardError; end
9
+
10
+ class Page
11
+
12
+ def initialize url
13
+ @extracted_info = {}
14
+ agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/32.0.1700.102 Chrome/32.0.1700.102 Safari/537.36"
15
+ @document = Nokogiri::HTML(open(url, "User-Agent" => agent))
16
+ end
17
+
18
+ def extract
19
+ block = Proc.new
20
+ instance_eval &block
21
+ @extracted_info
22
+ end
23
+
24
+ def method_missing name, args
25
+ feature_name = name.to_s.gsub('extract_', '').to_sym
26
+ element = @document.css args[:with]
27
+ raise NoElementsFoundError, "the css selector for '#{feature_name}' did not return anything" if element.empty?
28
+ if args[:and_run]
29
+ @extracted_info[feature_name] = args[:and_run].call(element)
30
+ else
31
+ @extracted_info[feature_name] = element.text
32
+ end
33
+ end
34
+
35
+ end
36
+
37
+ end
data/skrape.gemspec ADDED
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'skrape/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "skrape"
8
+ spec.version = Skrape::VERSION
9
+ spec.authors = ["Mike Williamson"]
10
+ spec.email = ["mike@korora.ca"]
11
+ spec.summary = "A tiny DSL for web scraping."
12
+ spec.description = "A cute little DSL for picking information off of web pages."
13
+ spec.homepage = "https://github.com/sleepycat/skrape"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.5"
22
+ spec.add_development_dependency "rspec", "~> 2.14"
23
+ spec.add_development_dependency "webmock", "~> 1.7"
24
+ spec.add_development_dependency "pry", "~> 0.9"
25
+ spec.add_dependency "nokogiri", "~> 1.6"
26
+ end
data/spec/skrape_spec.rb ADDED
@@ -0,0 +1,39 @@
1
+ require 'spec_helper'
2
+
3
+ describe Skrape do
4
+
5
+ describe Skrape::Page do
6
+
7
+ let(:url){ "http://example.com" }
8
+ let(:example_response){ File.new('spec/test_data/example_com_raw_response') }
9
+
10
+ before(:each) do
11
+ stub_request(:get, url).to_return(example_response)
12
+ end
13
+
14
+ it "returns the text of the element identifed by the CSS selector" do
15
+ results = Skrape::Page.new(url).extract do
16
+ extract_title with: 'h1'
17
+ end
18
+ expect(results[:title]).to eq "Example Domain"
19
+ end
20
+
21
+ it "accepts a block so you can do more sophisticated things" do
22
+ results = Skrape::Page.new(url).extract do
23
+ extract_link_href with: 'a', and_run: proc {|link| link.attr('href').value }
24
+ end
25
+ expect(results[:link_href]).to eq "http://www.iana.org/domains/example"
26
+ end
27
+
28
+ it "raises a helpful error when the CSS selector returns nothing" do
29
+ expect{
30
+ Skrape::Page.new(url).extract do
31
+ extract_nothing with: 'foo'
32
+ end
33
+ }.to raise_error Skrape::NoElementsFoundError
34
+ end
35
+
36
+ end
37
+
38
+ end
39
+
data/spec/spec_helper.rb ADDED
@@ -0,0 +1,13 @@
1
+ require_relative '../lib/skrape'
2
+ require 'webmock/rspec'
3
+
4
+ RSpec.configure do |config|
5
+ # Use color in STDOUT
6
+ config.color_enabled = true
7
+
8
+ # Use color not only in STDOUT but also in pagers and files
9
+ config.tty = true
10
+
11
+ # Use the specified formatter
12
+ config.formatter = :documentation # :progress, :html, :textmate
13
+ end
data/spec/test_data/example_com_raw_response ADDED
@@ -0,0 +1,63 @@
1
+ HTTP/1.1 200 OK
2
+ Accept-Ranges: bytes
3
+ Cache-Control: max-age=604800
4
+ Content-Type: text/html
5
+ Date: Tue, 11 Feb 2014 15:50:08 GMT
6
+ Etag: "359670651"
7
+ Expires: Tue, 18 Feb 2014 15:50:08 GMT
8
+ Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT
9
+ Server: ECS (iad/19AB)
10
+ X-Cache: HIT
11
+ x-ec-custom-error: 1
12
+ Content-Length: 1270
13
+
14
+ <!doctype html>
15
+ <html>
16
+ <head>
17
+ <title>Example Domain</title>
18
+
19
+ <meta charset="utf-8" />
20
+ <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
21
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
22
+ <style type="text/css">
23
+ body {
24
+ background-color: #f0f0f2;
25
+ margin: 0;
26
+ padding: 0;
27
+ font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
28
+
29
+ }
30
+ div {
31
+ width: 600px;
32
+ margin: 5em auto;
33
+ padding: 50px;
34
+ background-color: #fff;
35
+ border-radius: 1em;
36
+ }
37
+ a:link, a:visited {
38
+ color: #38488f;
39
+ text-decoration: none;
40
+ }
41
+ @media (max-width: 700px) {
42
+ body {
43
+ background-color: #fff;
44
+ }
45
+ div {
46
+ width: auto;
47
+ margin: 0 auto;
48
+ border-radius: 0;
49
+ padding: 1em;
50
+ }
51
+ }
52
+ </style>
53
+ </head>
54
+
55
+ <body>
56
+ <div>
57
+ <h1>Example Domain</h1>
58
+ <p>This domain is established to be used for illustrative examples in documents. You may use this
59
+ domain in examples without prior coordination or asking for permission.</p>
60
+ <p><a href="http://www.iana.org/domains/example">More information...</a></p>
61
+ </div>
62
+ </body>
63
+ </html>
metadata ADDED
@@ -0,0 +1,114 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: skrape
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Mike Williamson
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2014-02-12 00:00:00 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ prerelease: false
17
+ requirement: &id001 !ruby/object:Gem::Requirement
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: "1.5"
22
+ type: :development
23
+ version_requirements: *id001
24
+ - !ruby/object:Gem::Dependency
25
+ name: rspec
26
+ prerelease: false
27
+ requirement: &id002 !ruby/object:Gem::Requirement
28
+ requirements:
29
+ - - ~>
30
+ - !ruby/object:Gem::Version
31
+ version: "2.14"
32
+ type: :development
33
+ version_requirements: *id002
34
+ - !ruby/object:Gem::Dependency
35
+ name: webmock
36
+ prerelease: false
37
+ requirement: &id003 !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - ~>
40
+ - !ruby/object:Gem::Version
41
+ version: "1.7"
42
+ type: :development
43
+ version_requirements: *id003
44
+ - !ruby/object:Gem::Dependency
45
+ name: pry
46
+ prerelease: false
47
+ requirement: &id004 !ruby/object:Gem::Requirement
48
+ requirements:
49
+ - - ~>
50
+ - !ruby/object:Gem::Version
51
+ version: "0.9"
52
+ type: :development
53
+ version_requirements: *id004
54
+ - !ruby/object:Gem::Dependency
55
+ name: nokogiri
56
+ prerelease: false
57
+ requirement: &id005 !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: "1.6"
62
+ type: :runtime
63
+ version_requirements: *id005
64
+ description: A cute little DSL for picking information off of web pages.
65
+ email:
66
+ - mike@korora.ca
67
+ executables: []
68
+
69
+ extensions: []
70
+
71
+ extra_rdoc_files: []
72
+
73
+ files:
74
+ - .gitignore
75
+ - Gemfile
76
+ - LICENSE.txt
77
+ - README.md
78
+ - Rakefile
79
+ - lib/skrape.rb
80
+ - lib/skrape/version.rb
81
+ - skrape.gemspec
82
+ - spec/skrape_spec.rb
83
+ - spec/spec_helper.rb
84
+ - spec/test_data/example_com_raw_response
85
+ homepage: https://github.com/sleepycat/skrape
86
+ licenses:
87
+ - MIT
88
+ metadata: {}
89
+
90
+ post_install_message:
91
+ rdoc_options: []
92
+
93
+ require_paths:
94
+ - lib
95
+ required_ruby_version: !ruby/object:Gem::Requirement
96
+ requirements:
97
+ - &id006
98
+ - ">="
99
+ - !ruby/object:Gem::Version
100
+ version: "0"
101
+ required_rubygems_version: !ruby/object:Gem::Requirement
102
+ requirements:
103
+ - *id006
104
+ requirements: []
105
+
106
+ rubyforge_project:
107
+ rubygems_version: 2.2.2
108
+ signing_key:
109
+ specification_version: 4
110
+ summary: A tiny DSL for web scraping.
111
+ test_files:
112
+ - spec/skrape_spec.rb
113
+ - spec/spec_helper.rb
114
+ - spec/test_data/example_com_raw_response