urifetch 0.0.1.rc6
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.rspec +1 -0
- data/.rvmrc +1 -0
- data/Gemfile +20 -0
- data/Gemfile.lock +51 -0
- data/LICENSE.txt +20 -0
- data/README.md +39 -0
- data/Rakefile +52 -0
- data/lib/.DS_Store +0 -0
- data/lib/urifetch/handler.rb +31 -0
- data/lib/urifetch/response.rb +15 -0
- data/lib/urifetch/strategy/layout.rb +35 -0
- data/lib/urifetch/strategy.rb +77 -0
- data/lib/urifetch/version.rb +14 -0
- data/lib/urifetch.rb +79 -0
- data/spec/helpers/url_helpers.rb +15 -0
- data/spec/spec_helper.rb +12 -0
- data/spec/urifetch/handler_spec.rb +19 -0
- data/spec/urifetch/response_spec.rb +28 -0
- data/spec/urifetch/strategy/layout_spec.rb +51 -0
- data/spec/urifetch/strategy_spec.rb +67 -0
- data/spec/urifetch_spec.rb +45 -0
- data/urifetch.gemspec +86 -0
- metadata +174 -0
data/.document
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use 1.9.2-p290
|
data/Gemfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
# Add dependencies required to use your gem here.
|
3
|
+
# Example:
|
4
|
+
# gem "activesupport", ">= 2.3.5"
|
5
|
+
|
6
|
+
# Add dependencies to develop your gem here.
|
7
|
+
# Include everything needed to run rake, tests, features, etc.
|
8
|
+
|
9
|
+
gem 'addressable'
|
10
|
+
gem 'hashie'
|
11
|
+
gem 'nokogiri'
|
12
|
+
|
13
|
+
group :development do
|
14
|
+
gem "rspec"
|
15
|
+
gem "bundler", "1.1.rc"
|
16
|
+
gem "jeweler"
|
17
|
+
gem "rcov"
|
18
|
+
gem 'pry'
|
19
|
+
gem "rdoc"
|
20
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
addressable (2.2.6)
|
5
|
+
coderay (0.9.8)
|
6
|
+
diff-lcs (1.1.3)
|
7
|
+
git (1.2.5)
|
8
|
+
hashie (1.2.0)
|
9
|
+
jeweler (1.6.4)
|
10
|
+
bundler (~> 1.0)
|
11
|
+
git (>= 1.2.5)
|
12
|
+
rake
|
13
|
+
json (1.6.4)
|
14
|
+
method_source (0.6.7)
|
15
|
+
ruby_parser (>= 2.3.1)
|
16
|
+
nokogiri (1.5.0)
|
17
|
+
pry (0.9.7.4)
|
18
|
+
coderay (~> 0.9.8)
|
19
|
+
method_source (~> 0.6.7)
|
20
|
+
ruby_parser (>= 2.3.1)
|
21
|
+
slop (~> 2.1.0)
|
22
|
+
rake (0.9.2.2)
|
23
|
+
rcov (0.9.11)
|
24
|
+
rdoc (3.12)
|
25
|
+
json (~> 1.4)
|
26
|
+
rspec (2.3.0)
|
27
|
+
rspec-core (~> 2.3.0)
|
28
|
+
rspec-expectations (~> 2.3.0)
|
29
|
+
rspec-mocks (~> 2.3.0)
|
30
|
+
rspec-core (2.3.1)
|
31
|
+
rspec-expectations (2.3.0)
|
32
|
+
diff-lcs (~> 1.1.2)
|
33
|
+
rspec-mocks (2.3.0)
|
34
|
+
ruby_parser (2.3.1)
|
35
|
+
sexp_processor (~> 3.0)
|
36
|
+
sexp_processor (3.0.9)
|
37
|
+
slop (2.1.0)
|
38
|
+
|
39
|
+
PLATFORMS
|
40
|
+
ruby
|
41
|
+
|
42
|
+
DEPENDENCIES
|
43
|
+
addressable
|
44
|
+
bundler (= 1.1.rc)
|
45
|
+
hashie
|
46
|
+
jeweler
|
47
|
+
nokogiri
|
48
|
+
pry
|
49
|
+
rcov
|
50
|
+
rdoc
|
51
|
+
rspec
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 Philip Vieira
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# Urifetch
|
2
|
+
|
3
|
+
Match URL's to request strategies and retrieve them in a usable format. Urifetch allows you to fetch data from any URL using pattern matching. The library allows for dynamically adding match handlers as well as build your own strategies.
|
4
|
+
|
5
|
+
## Install
|
6
|
+
|
7
|
+
##### Simply install them gem through RubyGems
|
8
|
+
> gem install urifetch
|
9
|
+
|
10
|
+
#### Or through Bundler
|
11
|
+
Gemfile:
|
12
|
+
> gem 'urifetch'
|
13
|
+
Run:
|
14
|
+
> ~$ bundle install
|
15
|
+
|
16
|
+
## Basic Usage
|
17
|
+
|
18
|
+
#### Require the library
|
19
|
+
> require 'urifetch'
|
20
|
+
|
21
|
+
#### Start using
|
22
|
+
|
23
|
+
> response = Urifetch.fetch_from("http://www.github.com")
|
24
|
+
> response.status
|
25
|
+
> => ["200","ok"]
|
26
|
+
> response.data
|
27
|
+
> => { title: 'Google' }
|
28
|
+
|
29
|
+
|
30
|
+
## Contributing to Urifetch
|
31
|
+
|
32
|
+
##### Coming soon...
|
33
|
+
|
34
|
+
|
35
|
+
## Copyright
|
36
|
+
|
37
|
+
Copyright (c) 2012 Philip Vieira. See LICENSE.txt for
|
38
|
+
further details.
|
39
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:test, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
|
16
|
+
require './lib/urifetch/version'
|
17
|
+
Jeweler::Tasks.new do |gem|
|
18
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
19
|
+
gem.version = Uriversal::Version::STRING
|
20
|
+
gem.name = "urifetch"
|
21
|
+
gem.homepage = "http://github.com/zeeraw/Urifetch"
|
22
|
+
gem.license = "MIT"
|
23
|
+
gem.summary = "Match URL's to request strategies and retrieve them in a usable format."
|
24
|
+
gem.description = "Urifetch allows you to fetch data from any URL using pattern matching. The library allows for dynamically adding match handlers as well as build your own strategies."
|
25
|
+
gem.email = "philip@vallin.se"
|
26
|
+
gem.authors = ["Philip Vieira"]
|
27
|
+
# dependencies defined in Gemfile
|
28
|
+
end
|
29
|
+
Jeweler::RubygemsDotOrgTasks.new
|
30
|
+
|
31
|
+
require 'rspec/core'
|
32
|
+
require 'rspec/core/rake_task'
|
33
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
34
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
35
|
+
end
|
36
|
+
|
37
|
+
RSpec::Core::RakeTask.new(:rcov) do |spec|
|
38
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
39
|
+
spec.rcov = true
|
40
|
+
end
|
41
|
+
|
42
|
+
task :default => :spec
|
43
|
+
|
44
|
+
require 'rdoc/task'
|
45
|
+
Rake::RDocTask.new do |rdoc|
|
46
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
47
|
+
|
48
|
+
rdoc.rdoc_dir = 'rdoc'
|
49
|
+
rdoc.title = "urifetch #{version}"
|
50
|
+
rdoc.rdoc_files.include('README*')
|
51
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
52
|
+
end
|
data/lib/.DS_Store
ADDED
Binary file
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
|
4
|
+
class Handler
|
5
|
+
|
6
|
+
def initialize(strategy_key,&block)
|
7
|
+
@handlers = {}
|
8
|
+
raise ArgumentError "strategy ('#{strategy.class}') needs to be a 'Symbol'" unless strategy_key.kind_of?(Symbol)
|
9
|
+
@strategy_key = strategy_key || DEFAULT_STRATEGY
|
10
|
+
instance_eval(&block) if block_given?
|
11
|
+
end
|
12
|
+
|
13
|
+
def find(url,previous_match=nil)
|
14
|
+
previous_match = url.match(DEFAULT_MATCH_STRING) if previous_match.nil?
|
15
|
+
@handlers.each do |key,val|
|
16
|
+
m = url.match(key)
|
17
|
+
return val.find(url,m) if m != 0 and !m.nil?
|
18
|
+
end
|
19
|
+
return Urifetch::Strategy.apply(@strategy_key, with: previous_match)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def match(string,strategy,&block)
|
25
|
+
raise ArgumentError "matcher ('#{key.class}') needs to be either 'String' or 'Regexp'" unless [String,Regexp].include?(string.class)
|
26
|
+
@handlers[string] = Handler.new(strategy,&block)
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
class Response
|
4
|
+
|
5
|
+
attr_reader :status, :strategy, :data
|
6
|
+
|
7
|
+
def initialize(args={})
|
8
|
+
@status = args[:status] || ['0','']
|
9
|
+
@strategy = args[:strategy] || Strategy.new(:test,"".match(//))
|
10
|
+
@data = args[:data] || Hashie::Mash.new
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
class Strategy
|
4
|
+
|
5
|
+
class Layout
|
6
|
+
|
7
|
+
attr_reader :before, :success, :failure
|
8
|
+
|
9
|
+
def initialize(args={},&block)
|
10
|
+
@success = nil
|
11
|
+
@failure = nil
|
12
|
+
@before = nil
|
13
|
+
instance_exec(args,&block) if block_given?
|
14
|
+
end
|
15
|
+
|
16
|
+
def after_success(&block)
|
17
|
+
raise ArgumentError unless block_given?
|
18
|
+
@success = block
|
19
|
+
end
|
20
|
+
|
21
|
+
def after_failure(&block)
|
22
|
+
raise ArgumentError unless block_given?
|
23
|
+
@failure = block
|
24
|
+
end
|
25
|
+
|
26
|
+
def before_request(&block)
|
27
|
+
raise ArgumentError unless block_given?
|
28
|
+
@before = block
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
require 'open-uri'
|
4
|
+
require 'nokogiri'
|
5
|
+
|
6
|
+
class Strategy
|
7
|
+
|
8
|
+
autoload :Layout, 'urifetch/strategy/layout'
|
9
|
+
|
10
|
+
@@layouts = Hashie::Mash.new
|
11
|
+
def self.layouts
|
12
|
+
@@layouts
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :layout, :match_data, :layout_key, :uri
|
16
|
+
|
17
|
+
def initialize(layout_key,match_data)
|
18
|
+
@layout_key = layout_key
|
19
|
+
@match_data = match_data
|
20
|
+
@layout = @@layouts[layout_key]
|
21
|
+
raise "no matching layouts found on #{layout_key}" unless @layout
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.layout(layout_key,&block)
|
25
|
+
layouts[layout_key] = Layout.new(layout_key,&block)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.apply(layout_key,args={})
|
29
|
+
m_data = args[:with] || "".match(//)
|
30
|
+
Strategy.new(layout_key,m_data)
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.apply!(layout_key,args={})
|
34
|
+
apply(layout_key,args).execute!
|
35
|
+
end
|
36
|
+
|
37
|
+
def execute!
|
38
|
+
run_before!
|
39
|
+
begin
|
40
|
+
@uri = Addressable::URI.heuristic_parse(match_data.string)
|
41
|
+
request = open(@uri.to_s)
|
42
|
+
status = request.status
|
43
|
+
run_on_success!(request)
|
44
|
+
rescue OpenURI::HTTPError => error
|
45
|
+
status = (error.message.split(" ",2))
|
46
|
+
run_on_failure!(error)
|
47
|
+
rescue SocketError => error
|
48
|
+
status = (["400","Bad Request"])
|
49
|
+
run_on_failure!(error)
|
50
|
+
rescue Errno::ENOENT => error
|
51
|
+
status = (["404","File not Found"])
|
52
|
+
run_on_failure!(error)
|
53
|
+
end
|
54
|
+
return Response.new(status: status, strategy: self, data: @data)
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def run_before!
|
60
|
+
instance_exec(&layout.before) unless layout.before.nil?
|
61
|
+
end
|
62
|
+
|
63
|
+
def run_on_success!(request)
|
64
|
+
instance_exec(request,&layout.success) unless layout.before.nil?
|
65
|
+
end
|
66
|
+
|
67
|
+
def run_on_failure!(error)
|
68
|
+
instance_exec(error,&layout.failure) unless layout.before.nil?
|
69
|
+
end
|
70
|
+
|
71
|
+
def data
|
72
|
+
@data ||= Hashie::Mash.new
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
data/lib/urifetch.rb
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'hashie/mash'
|
2
|
+
require 'addressable/uri'
|
3
|
+
|
4
|
+
module Urifetch
|
5
|
+
|
6
|
+
DEFAULT_MATCH_STRING = /(.*)/i
|
7
|
+
DEFAULT_STRATEGY = :default
|
8
|
+
|
9
|
+
autoload :Handler, 'urifetch/handler'
|
10
|
+
autoload :Strategy, 'urifetch/strategy'
|
11
|
+
autoload :Response, 'urifetch/response'
|
12
|
+
|
13
|
+
@@handler = Handler.new(DEFAULT_STRATEGY)
|
14
|
+
|
15
|
+
def self.fetch_from(url,args={})
|
16
|
+
find_strategy_from(url).execute!
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.find_strategy_from(url,args={})
|
20
|
+
if valid_url?(url)
|
21
|
+
@@handler.find(url)
|
22
|
+
else
|
23
|
+
raise ArgumentError, "invalid url"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.valid_url?(url)
|
28
|
+
!(url =~ /^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$/ix).nil?
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.register(&block)
|
32
|
+
@@handler.instance_eval(&block)
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
Urifetch.register do
|
38
|
+
end
|
39
|
+
|
40
|
+
Urifetch::Strategy.layout(:test) do
|
41
|
+
|
42
|
+
before_request do
|
43
|
+
end
|
44
|
+
|
45
|
+
after_success do |request|
|
46
|
+
end
|
47
|
+
|
48
|
+
after_failure do |error|
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
Urifetch::Strategy.layout(:default) do
|
54
|
+
|
55
|
+
before_request do
|
56
|
+
end
|
57
|
+
|
58
|
+
after_success do |request|
|
59
|
+
doc = Nokogiri::HTML(request)
|
60
|
+
|
61
|
+
# Title
|
62
|
+
title = doc.css('title').first
|
63
|
+
data.title = title.nil? ? match_data[0] : title.content.strip
|
64
|
+
|
65
|
+
# Favicon
|
66
|
+
favicon = doc.css('link[rel="shortcut icon"], link[rel="icon shortcut"], link[rel="shortcut"], link[rel="icon"]').first
|
67
|
+
favicon = favicon.nil? ? nil : favicon['href'].strip
|
68
|
+
if favicon
|
69
|
+
if favicon.match(/^https?:\/\//i).nil?
|
70
|
+
favicon = uri.scheme + "://" + uri.host + uri.path.gsub(/((.*)(\/(.*\..*$)))/i,'\2/') + favicon
|
71
|
+
end
|
72
|
+
data.favicon = favicon
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
after_failure do |error|
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
3
|
+
require 'rspec'
|
4
|
+
require 'urifetch'
|
5
|
+
require 'helpers/url_helpers'
|
6
|
+
|
7
|
+
# Requires supporting files with custom matchers and macros, etc,
|
8
|
+
# in ./support/ and its subdirectories.
|
9
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
10
|
+
|
11
|
+
RSpec.configure do |config|
|
12
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Handler do
|
4
|
+
|
5
|
+
describe '#initialize' do
|
6
|
+
|
7
|
+
it 'should return Urifetch::Handler' do
|
8
|
+
Urifetch::Handler.new(:default).should be_a_kind_of Urifetch::Handler
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should have a default value of a Hash on @handlers' do
|
12
|
+
@handlers = Urifetch::Handler.new(:default).instance_variable_get(:@handlers)
|
13
|
+
@handlers.should_not be_nil
|
14
|
+
@handlers.should be_a_kind_of Hash
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Response do
|
4
|
+
|
5
|
+
describe '#initialize' do
|
6
|
+
|
7
|
+
before do
|
8
|
+
@response = Urifetch::Response.new()
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should contain status as an array' do
|
12
|
+
@response.status.should_not be_nil
|
13
|
+
@response.status.should be_a_kind_of(Array)
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should contain the strategy used when matching' do
|
17
|
+
@response.strategy.should_not be_nil
|
18
|
+
@response.strategy.should be_a_kind_of(Urifetch::Strategy)
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'should contain data as a hashie/mash' do
|
22
|
+
@response.data.should_not be_nil
|
23
|
+
@response.data.should be_a_kind_of(Hashie::Mash)
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Strategy::Layout do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@layout = Urifetch::Strategy::Layout.new do
|
7
|
+
|
8
|
+
after_success do
|
9
|
+
some_method
|
10
|
+
end
|
11
|
+
|
12
|
+
after_failure do
|
13
|
+
some_method
|
14
|
+
end
|
15
|
+
|
16
|
+
before_request do
|
17
|
+
some_method
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe 'after_success' do
|
24
|
+
|
25
|
+
it 'should overwrite the value proc' do
|
26
|
+
proc = @layout.instance_variable_get(:@success)
|
27
|
+
@layout.after_success do
|
28
|
+
some_method
|
29
|
+
end
|
30
|
+
proc.should_not eq(@layout.instance_variable_get(:@success))
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should overwrite the value proc' do
|
34
|
+
proc = @layout.instance_variable_get(:@failure)
|
35
|
+
@layout.after_failure do
|
36
|
+
some_method
|
37
|
+
end
|
38
|
+
proc.should_not eq(@layout.instance_variable_get(:@failure))
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should overwrite the value proc' do
|
42
|
+
proc = @layout.instance_variable_get(:@before)
|
43
|
+
@layout.before_request do
|
44
|
+
some_method
|
45
|
+
end
|
46
|
+
proc.should_not eq(@layout.instance_variable_get(:@before))
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Strategy do
|
4
|
+
|
5
|
+
describe 'self.layouts' do
|
6
|
+
|
7
|
+
it 'should return an Hashie::Mash' do
|
8
|
+
Urifetch::Strategy.layouts.should be_a_kind_of Hashie::Mash
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
12
|
+
|
13
|
+
describe 'self.layout' do
|
14
|
+
|
15
|
+
it 'should return a Urifetch::Strategy::Layout instance and make it accessible through .layouts' do
|
16
|
+
@layout = Urifetch::Strategy.layout(:test) do; end
|
17
|
+
@layout.should be_an_instance_of Urifetch::Strategy::Layout
|
18
|
+
Urifetch::Strategy.layouts[:test].should == @layout
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
describe 'self.apply' do
|
24
|
+
|
25
|
+
it 'should return an instance of Urifetch::Strategy' do
|
26
|
+
Urifetch::Strategy.apply(:test).should be_an_instance_of Urifetch::Strategy
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
describe 'self.apply!' do
|
32
|
+
|
33
|
+
it 'should return an instance of Urifetch::Response' do
|
34
|
+
Urifetch::Strategy.apply!(:test).should be_an_instance_of Urifetch::Response
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
before do
|
40
|
+
@strategy = Urifetch::Strategy.new(:test,"".match(//i))
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '#initialize' do
|
44
|
+
|
45
|
+
it 'should return an error if layout was not found' do
|
46
|
+
-> { Urifetch::Strategy.new(:not_found,"".match(//i)) }.should raise_error(RuntimeError, /no matching layouts found/i)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'should contain match data as a MatchData object' do
|
50
|
+
@strategy.match_data.should be_a_kind_of MatchData
|
51
|
+
end
|
52
|
+
|
53
|
+
it 'should have a layout as a Urifetch::Strategy::Layout' do
|
54
|
+
@strategy.layout.should be_an_instance_of Urifetch::Strategy::Layout
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'execute!' do
|
60
|
+
|
61
|
+
it 'should return an instance of Urifetch::Response' do
|
62
|
+
@strategy.execute!.should be_an_instance_of Urifetch::Response
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch do
|
4
|
+
|
5
|
+
describe 'self.fetch_from' do
|
6
|
+
|
7
|
+
it 'should return a Urifetch::Response' do
|
8
|
+
Urifetch.fetch_from(Urifetch::Test::UrlHelpers.generate_valid_url).should be_a_kind_of(Urifetch::Response)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should raise an error if url is invalid according to configuration' do
|
12
|
+
-> { Urifetch.fetch_from(Urifetch::Test::UrlHelpers.generate_invalid_url) }.should raise_error ArgumentError, "invalid url"
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
describe 'self.find_strategy_from' do
|
18
|
+
|
19
|
+
it 'should an instance of Urifetch::Strategy' do
|
20
|
+
Urifetch.find_strategy_from("http://www.google.com").should be_an_instance_of Urifetch::Strategy
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'strategy' do
|
26
|
+
|
27
|
+
describe 'default' do
|
28
|
+
|
29
|
+
it 'should return title and favicon for valid path' do
|
30
|
+
@response = Urifetch.fetch_from("http://www.youtube.com")
|
31
|
+
@response.data.should have_key(:title)
|
32
|
+
@response.data.should have_key(:favicon)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should return title and NOT favicon for valid path' do
|
36
|
+
@response = Urifetch.fetch_from("http://www.google.com")
|
37
|
+
@response.data.should have_key(:title)
|
38
|
+
@response.data.should_not have_key(:favicon)
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
data/urifetch.gemspec
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "urifetch"
|
8
|
+
s.version = "0.0.1.rc6"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Philip Vieira"]
|
12
|
+
s.date = "2012-01-14"
|
13
|
+
s.description = "Urifetch allows you to fetch data from any URL using pattern matching. The library allows for dynamically adding match handlers as well as build your own strategies."
|
14
|
+
s.email = "philip@vallin.se"
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE.txt",
|
17
|
+
"README.md"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
".rspec",
|
22
|
+
".rvmrc",
|
23
|
+
"Gemfile",
|
24
|
+
"Gemfile.lock",
|
25
|
+
"LICENSE.txt",
|
26
|
+
"README.md",
|
27
|
+
"Rakefile",
|
28
|
+
"lib/.DS_Store",
|
29
|
+
"lib/urifetch.rb",
|
30
|
+
"lib/urifetch/handler.rb",
|
31
|
+
"lib/urifetch/response.rb",
|
32
|
+
"lib/urifetch/strategy.rb",
|
33
|
+
"lib/urifetch/strategy/layout.rb",
|
34
|
+
"lib/urifetch/version.rb",
|
35
|
+
"spec/helpers/url_helpers.rb",
|
36
|
+
"spec/spec_helper.rb",
|
37
|
+
"spec/urifetch/handler_spec.rb",
|
38
|
+
"spec/urifetch/response_spec.rb",
|
39
|
+
"spec/urifetch/strategy/layout_spec.rb",
|
40
|
+
"spec/urifetch/strategy_spec.rb",
|
41
|
+
"spec/urifetch_spec.rb",
|
42
|
+
"urifetch.gemspec"
|
43
|
+
]
|
44
|
+
s.homepage = "http://github.com/zeeraw/Urifetch"
|
45
|
+
s.licenses = ["MIT"]
|
46
|
+
s.require_paths = ["lib"]
|
47
|
+
s.rubygems_version = "1.8.10"
|
48
|
+
s.summary = "Match URL's to request strategies and retrieve them in a usable format."
|
49
|
+
|
50
|
+
if s.respond_to? :specification_version then
|
51
|
+
s.specification_version = 3
|
52
|
+
|
53
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
54
|
+
s.add_runtime_dependency(%q<addressable>, [">= 0"])
|
55
|
+
s.add_runtime_dependency(%q<hashie>, [">= 0"])
|
56
|
+
s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
|
57
|
+
s.add_development_dependency(%q<rspec>, [">= 0"])
|
58
|
+
s.add_development_dependency(%q<bundler>, ["= 1.1.rc"])
|
59
|
+
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
60
|
+
s.add_development_dependency(%q<rcov>, [">= 0"])
|
61
|
+
s.add_development_dependency(%q<pry>, [">= 0"])
|
62
|
+
s.add_development_dependency(%q<rdoc>, [">= 0"])
|
63
|
+
else
|
64
|
+
s.add_dependency(%q<addressable>, [">= 0"])
|
65
|
+
s.add_dependency(%q<hashie>, [">= 0"])
|
66
|
+
s.add_dependency(%q<nokogiri>, [">= 0"])
|
67
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
68
|
+
s.add_dependency(%q<bundler>, ["= 1.1.rc"])
|
69
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
70
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
71
|
+
s.add_dependency(%q<pry>, [">= 0"])
|
72
|
+
s.add_dependency(%q<rdoc>, [">= 0"])
|
73
|
+
end
|
74
|
+
else
|
75
|
+
s.add_dependency(%q<addressable>, [">= 0"])
|
76
|
+
s.add_dependency(%q<hashie>, [">= 0"])
|
77
|
+
s.add_dependency(%q<nokogiri>, [">= 0"])
|
78
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
79
|
+
s.add_dependency(%q<bundler>, ["= 1.1.rc"])
|
80
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
81
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
82
|
+
s.add_dependency(%q<pry>, [">= 0"])
|
83
|
+
s.add_dependency(%q<rdoc>, [">= 0"])
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
metadata
ADDED
@@ -0,0 +1,174 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: urifetch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.rc6
|
5
|
+
prerelease: 6
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Philip Vieira
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-01-14 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: addressable
|
16
|
+
requirement: &70123223672060 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *70123223672060
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: hashie
|
27
|
+
requirement: &70123223669420 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70123223669420
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: nokogiri
|
38
|
+
requirement: &70123223667980 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *70123223667980
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rspec
|
49
|
+
requirement: &70123223667020 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :development
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *70123223667020
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: bundler
|
60
|
+
requirement: &70123223665120 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - =
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: 1.1.rc
|
66
|
+
type: :development
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *70123223665120
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: jeweler
|
71
|
+
requirement: &70123223663280 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *70123223663280
|
80
|
+
- !ruby/object:Gem::Dependency
|
81
|
+
name: rcov
|
82
|
+
requirement: &70123223662420 !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
84
|
+
requirements:
|
85
|
+
- - ! '>='
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0'
|
88
|
+
type: :development
|
89
|
+
prerelease: false
|
90
|
+
version_requirements: *70123223662420
|
91
|
+
- !ruby/object:Gem::Dependency
|
92
|
+
name: pry
|
93
|
+
requirement: &70123223661260 !ruby/object:Gem::Requirement
|
94
|
+
none: false
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
type: :development
|
100
|
+
prerelease: false
|
101
|
+
version_requirements: *70123223661260
|
102
|
+
- !ruby/object:Gem::Dependency
|
103
|
+
name: rdoc
|
104
|
+
requirement: &70123223660520 !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: *70123223660520
|
113
|
+
description: Urifetch allows you to fetch data from any URL using pattern matching.
|
114
|
+
The library allows for dynamically adding match handlers as well as build your own
|
115
|
+
strategies.
|
116
|
+
email: philip@vallin.se
|
117
|
+
executables: []
|
118
|
+
extensions: []
|
119
|
+
extra_rdoc_files:
|
120
|
+
- LICENSE.txt
|
121
|
+
- README.md
|
122
|
+
files:
|
123
|
+
- .document
|
124
|
+
- .rspec
|
125
|
+
- .rvmrc
|
126
|
+
- Gemfile
|
127
|
+
- Gemfile.lock
|
128
|
+
- LICENSE.txt
|
129
|
+
- README.md
|
130
|
+
- Rakefile
|
131
|
+
- lib/.DS_Store
|
132
|
+
- lib/urifetch.rb
|
133
|
+
- lib/urifetch/handler.rb
|
134
|
+
- lib/urifetch/response.rb
|
135
|
+
- lib/urifetch/strategy.rb
|
136
|
+
- lib/urifetch/strategy/layout.rb
|
137
|
+
- lib/urifetch/version.rb
|
138
|
+
- spec/helpers/url_helpers.rb
|
139
|
+
- spec/spec_helper.rb
|
140
|
+
- spec/urifetch/handler_spec.rb
|
141
|
+
- spec/urifetch/response_spec.rb
|
142
|
+
- spec/urifetch/strategy/layout_spec.rb
|
143
|
+
- spec/urifetch/strategy_spec.rb
|
144
|
+
- spec/urifetch_spec.rb
|
145
|
+
- urifetch.gemspec
|
146
|
+
homepage: http://github.com/zeeraw/Urifetch
|
147
|
+
licenses:
|
148
|
+
- MIT
|
149
|
+
post_install_message:
|
150
|
+
rdoc_options: []
|
151
|
+
require_paths:
|
152
|
+
- lib
|
153
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
154
|
+
none: false
|
155
|
+
requirements:
|
156
|
+
- - ! '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
segments:
|
160
|
+
- 0
|
161
|
+
hash: 1351382682644594782
|
162
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
163
|
+
none: false
|
164
|
+
requirements:
|
165
|
+
- - ! '>'
|
166
|
+
- !ruby/object:Gem::Version
|
167
|
+
version: 1.3.1
|
168
|
+
requirements: []
|
169
|
+
rubyforge_project:
|
170
|
+
rubygems_version: 1.8.10
|
171
|
+
signing_key:
|
172
|
+
specification_version: 3
|
173
|
+
summary: Match URL's to request strategies and retrieve them in a usable format.
|
174
|
+
test_files: []
|