urifetch 0.0.1.rc6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rspec +1 -0
- data/.rvmrc +1 -0
- data/Gemfile +20 -0
- data/Gemfile.lock +51 -0
- data/LICENSE.txt +20 -0
- data/README.md +39 -0
- data/Rakefile +52 -0
- data/lib/.DS_Store +0 -0
- data/lib/urifetch/handler.rb +31 -0
- data/lib/urifetch/response.rb +15 -0
- data/lib/urifetch/strategy/layout.rb +35 -0
- data/lib/urifetch/strategy.rb +77 -0
- data/lib/urifetch/version.rb +14 -0
- data/lib/urifetch.rb +79 -0
- data/spec/helpers/url_helpers.rb +15 -0
- data/spec/spec_helper.rb +12 -0
- data/spec/urifetch/handler_spec.rb +19 -0
- data/spec/urifetch/response_spec.rb +28 -0
- data/spec/urifetch/strategy/layout_spec.rb +51 -0
- data/spec/urifetch/strategy_spec.rb +67 -0
- data/spec/urifetch_spec.rb +45 -0
- data/urifetch.gemspec +86 -0
- metadata +174 -0
data/.document
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use 1.9.2-p290
|
data/Gemfile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
# Add dependencies required to use your gem here.
|
3
|
+
# Example:
|
4
|
+
# gem "activesupport", ">= 2.3.5"
|
5
|
+
|
6
|
+
# Add dependencies to develop your gem here.
|
7
|
+
# Include everything needed to run rake, tests, features, etc.
|
8
|
+
|
9
|
+
gem 'addressable'
|
10
|
+
gem 'hashie'
|
11
|
+
gem 'nokogiri'
|
12
|
+
|
13
|
+
group :development do
|
14
|
+
gem "rspec"
|
15
|
+
gem "bundler", "1.1.rc"
|
16
|
+
gem "jeweler"
|
17
|
+
gem "rcov"
|
18
|
+
gem 'pry'
|
19
|
+
gem "rdoc"
|
20
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
addressable (2.2.6)
|
5
|
+
coderay (0.9.8)
|
6
|
+
diff-lcs (1.1.3)
|
7
|
+
git (1.2.5)
|
8
|
+
hashie (1.2.0)
|
9
|
+
jeweler (1.6.4)
|
10
|
+
bundler (~> 1.0)
|
11
|
+
git (>= 1.2.5)
|
12
|
+
rake
|
13
|
+
json (1.6.4)
|
14
|
+
method_source (0.6.7)
|
15
|
+
ruby_parser (>= 2.3.1)
|
16
|
+
nokogiri (1.5.0)
|
17
|
+
pry (0.9.7.4)
|
18
|
+
coderay (~> 0.9.8)
|
19
|
+
method_source (~> 0.6.7)
|
20
|
+
ruby_parser (>= 2.3.1)
|
21
|
+
slop (~> 2.1.0)
|
22
|
+
rake (0.9.2.2)
|
23
|
+
rcov (0.9.11)
|
24
|
+
rdoc (3.12)
|
25
|
+
json (~> 1.4)
|
26
|
+
rspec (2.3.0)
|
27
|
+
rspec-core (~> 2.3.0)
|
28
|
+
rspec-expectations (~> 2.3.0)
|
29
|
+
rspec-mocks (~> 2.3.0)
|
30
|
+
rspec-core (2.3.1)
|
31
|
+
rspec-expectations (2.3.0)
|
32
|
+
diff-lcs (~> 1.1.2)
|
33
|
+
rspec-mocks (2.3.0)
|
34
|
+
ruby_parser (2.3.1)
|
35
|
+
sexp_processor (~> 3.0)
|
36
|
+
sexp_processor (3.0.9)
|
37
|
+
slop (2.1.0)
|
38
|
+
|
39
|
+
PLATFORMS
|
40
|
+
ruby
|
41
|
+
|
42
|
+
DEPENDENCIES
|
43
|
+
addressable
|
44
|
+
bundler (= 1.1.rc)
|
45
|
+
hashie
|
46
|
+
jeweler
|
47
|
+
nokogiri
|
48
|
+
pry
|
49
|
+
rcov
|
50
|
+
rdoc
|
51
|
+
rspec
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2012 Philip Vieira
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# Urifetch
|
2
|
+
|
3
|
+
Match URL's to request strategies and retrieve them in a usable format. Urifetch allows you to fetch data from any URL using pattern matching. The library allows for dynamically adding match handlers as well as build your own strategies.
|
4
|
+
|
5
|
+
## Install
|
6
|
+
|
7
|
+
##### Simply install them gem through RubyGems
|
8
|
+
> gem install urifetch
|
9
|
+
|
10
|
+
#### Or through Bundler
|
11
|
+
Gemfile:
|
12
|
+
> gem 'urifetch'
|
13
|
+
Run:
|
14
|
+
> ~$ bundle install
|
15
|
+
|
16
|
+
## Basic Usage
|
17
|
+
|
18
|
+
#### Require the library
|
19
|
+
> require 'urifetch'
|
20
|
+
|
21
|
+
#### Start using
|
22
|
+
|
23
|
+
> response = Urifetch.fetch_from("http://www.github.com")
|
24
|
+
> response.status
|
25
|
+
> => ["200","ok"]
|
26
|
+
> response.data
|
27
|
+
> => { title: 'Google' }
|
28
|
+
|
29
|
+
|
30
|
+
## Contributing to Urifetch
|
31
|
+
|
32
|
+
##### Coming soon...
|
33
|
+
|
34
|
+
|
35
|
+
## Copyright
|
36
|
+
|
37
|
+
Copyright (c) 2012 Philip Vieira. See LICENSE.txt for
|
38
|
+
further details.
|
39
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'bundler'
|
5
|
+
begin
|
6
|
+
Bundler.setup(:test, :development)
|
7
|
+
rescue Bundler::BundlerError => e
|
8
|
+
$stderr.puts e.message
|
9
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
10
|
+
exit e.status_code
|
11
|
+
end
|
12
|
+
require 'rake'
|
13
|
+
|
14
|
+
require 'jeweler'
|
15
|
+
|
16
|
+
require './lib/urifetch/version'
|
17
|
+
Jeweler::Tasks.new do |gem|
|
18
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
19
|
+
gem.version = Uriversal::Version::STRING
|
20
|
+
gem.name = "urifetch"
|
21
|
+
gem.homepage = "http://github.com/zeeraw/Urifetch"
|
22
|
+
gem.license = "MIT"
|
23
|
+
gem.summary = "Match URL's to request strategies and retrieve them in a usable format."
|
24
|
+
gem.description = "Urifetch allows you to fetch data from any URL using pattern matching. The library allows for dynamically adding match handlers as well as build your own strategies."
|
25
|
+
gem.email = "philip@vallin.se"
|
26
|
+
gem.authors = ["Philip Vieira"]
|
27
|
+
# dependencies defined in Gemfile
|
28
|
+
end
|
29
|
+
Jeweler::RubygemsDotOrgTasks.new
|
30
|
+
|
31
|
+
require 'rspec/core'
|
32
|
+
require 'rspec/core/rake_task'
|
33
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
34
|
+
spec.pattern = FileList['spec/**/*_spec.rb']
|
35
|
+
end
|
36
|
+
|
37
|
+
RSpec::Core::RakeTask.new(:rcov) do |spec|
|
38
|
+
spec.pattern = 'spec/**/*_spec.rb'
|
39
|
+
spec.rcov = true
|
40
|
+
end
|
41
|
+
|
42
|
+
task :default => :spec
|
43
|
+
|
44
|
+
require 'rdoc/task'
|
45
|
+
Rake::RDocTask.new do |rdoc|
|
46
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
47
|
+
|
48
|
+
rdoc.rdoc_dir = 'rdoc'
|
49
|
+
rdoc.title = "urifetch #{version}"
|
50
|
+
rdoc.rdoc_files.include('README*')
|
51
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
52
|
+
end
|
data/lib/.DS_Store
ADDED
Binary file
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
|
4
|
+
class Handler
|
5
|
+
|
6
|
+
def initialize(strategy_key,&block)
|
7
|
+
@handlers = {}
|
8
|
+
raise ArgumentError "strategy ('#{strategy.class}') needs to be a 'Symbol'" unless strategy_key.kind_of?(Symbol)
|
9
|
+
@strategy_key = strategy_key || DEFAULT_STRATEGY
|
10
|
+
instance_eval(&block) if block_given?
|
11
|
+
end
|
12
|
+
|
13
|
+
def find(url,previous_match=nil)
|
14
|
+
previous_match = url.match(DEFAULT_MATCH_STRING) if previous_match.nil?
|
15
|
+
@handlers.each do |key,val|
|
16
|
+
m = url.match(key)
|
17
|
+
return val.find(url,m) if m != 0 and !m.nil?
|
18
|
+
end
|
19
|
+
return Urifetch::Strategy.apply(@strategy_key, with: previous_match)
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def match(string,strategy,&block)
|
25
|
+
raise ArgumentError "matcher ('#{key.class}') needs to be either 'String' or 'Regexp'" unless [String,Regexp].include?(string.class)
|
26
|
+
@handlers[string] = Handler.new(strategy,&block)
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
class Response
|
4
|
+
|
5
|
+
attr_reader :status, :strategy, :data
|
6
|
+
|
7
|
+
def initialize(args={})
|
8
|
+
@status = args[:status] || ['0','']
|
9
|
+
@strategy = args[:strategy] || Strategy.new(:test,"".match(//))
|
10
|
+
@data = args[:data] || Hashie::Mash.new
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
class Strategy
|
4
|
+
|
5
|
+
class Layout
|
6
|
+
|
7
|
+
attr_reader :before, :success, :failure
|
8
|
+
|
9
|
+
def initialize(args={},&block)
|
10
|
+
@success = nil
|
11
|
+
@failure = nil
|
12
|
+
@before = nil
|
13
|
+
instance_exec(args,&block) if block_given?
|
14
|
+
end
|
15
|
+
|
16
|
+
def after_success(&block)
|
17
|
+
raise ArgumentError unless block_given?
|
18
|
+
@success = block
|
19
|
+
end
|
20
|
+
|
21
|
+
def after_failure(&block)
|
22
|
+
raise ArgumentError unless block_given?
|
23
|
+
@failure = block
|
24
|
+
end
|
25
|
+
|
26
|
+
def before_request(&block)
|
27
|
+
raise ArgumentError unless block_given?
|
28
|
+
@before = block
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Urifetch
|
2
|
+
|
3
|
+
require 'open-uri'
|
4
|
+
require 'nokogiri'
|
5
|
+
|
6
|
+
class Strategy
|
7
|
+
|
8
|
+
autoload :Layout, 'urifetch/strategy/layout'
|
9
|
+
|
10
|
+
@@layouts = Hashie::Mash.new
|
11
|
+
def self.layouts
|
12
|
+
@@layouts
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :layout, :match_data, :layout_key, :uri
|
16
|
+
|
17
|
+
def initialize(layout_key,match_data)
|
18
|
+
@layout_key = layout_key
|
19
|
+
@match_data = match_data
|
20
|
+
@layout = @@layouts[layout_key]
|
21
|
+
raise "no matching layouts found on #{layout_key}" unless @layout
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.layout(layout_key,&block)
|
25
|
+
layouts[layout_key] = Layout.new(layout_key,&block)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.apply(layout_key,args={})
|
29
|
+
m_data = args[:with] || "".match(//)
|
30
|
+
Strategy.new(layout_key,m_data)
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.apply!(layout_key,args={})
|
34
|
+
apply(layout_key,args).execute!
|
35
|
+
end
|
36
|
+
|
37
|
+
def execute!
|
38
|
+
run_before!
|
39
|
+
begin
|
40
|
+
@uri = Addressable::URI.heuristic_parse(match_data.string)
|
41
|
+
request = open(@uri.to_s)
|
42
|
+
status = request.status
|
43
|
+
run_on_success!(request)
|
44
|
+
rescue OpenURI::HTTPError => error
|
45
|
+
status = (error.message.split(" ",2))
|
46
|
+
run_on_failure!(error)
|
47
|
+
rescue SocketError => error
|
48
|
+
status = (["400","Bad Request"])
|
49
|
+
run_on_failure!(error)
|
50
|
+
rescue Errno::ENOENT => error
|
51
|
+
status = (["404","File not Found"])
|
52
|
+
run_on_failure!(error)
|
53
|
+
end
|
54
|
+
return Response.new(status: status, strategy: self, data: @data)
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def run_before!
|
60
|
+
instance_exec(&layout.before) unless layout.before.nil?
|
61
|
+
end
|
62
|
+
|
63
|
+
def run_on_success!(request)
|
64
|
+
instance_exec(request,&layout.success) unless layout.before.nil?
|
65
|
+
end
|
66
|
+
|
67
|
+
def run_on_failure!(error)
|
68
|
+
instance_exec(error,&layout.failure) unless layout.before.nil?
|
69
|
+
end
|
70
|
+
|
71
|
+
def data
|
72
|
+
@data ||= Hashie::Mash.new
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
data/lib/urifetch.rb
ADDED
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'hashie/mash'
|
2
|
+
require 'addressable/uri'
|
3
|
+
|
4
|
+
module Urifetch
|
5
|
+
|
6
|
+
DEFAULT_MATCH_STRING = /(.*)/i
|
7
|
+
DEFAULT_STRATEGY = :default
|
8
|
+
|
9
|
+
autoload :Handler, 'urifetch/handler'
|
10
|
+
autoload :Strategy, 'urifetch/strategy'
|
11
|
+
autoload :Response, 'urifetch/response'
|
12
|
+
|
13
|
+
@@handler = Handler.new(DEFAULT_STRATEGY)
|
14
|
+
|
15
|
+
def self.fetch_from(url,args={})
|
16
|
+
find_strategy_from(url).execute!
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.find_strategy_from(url,args={})
|
20
|
+
if valid_url?(url)
|
21
|
+
@@handler.find(url)
|
22
|
+
else
|
23
|
+
raise ArgumentError, "invalid url"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.valid_url?(url)
|
28
|
+
!(url =~ /^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$/ix).nil?
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.register(&block)
|
32
|
+
@@handler.instance_eval(&block)
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
Urifetch.register do
|
38
|
+
end
|
39
|
+
|
40
|
+
Urifetch::Strategy.layout(:test) do
|
41
|
+
|
42
|
+
before_request do
|
43
|
+
end
|
44
|
+
|
45
|
+
after_success do |request|
|
46
|
+
end
|
47
|
+
|
48
|
+
after_failure do |error|
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
Urifetch::Strategy.layout(:default) do
|
54
|
+
|
55
|
+
before_request do
|
56
|
+
end
|
57
|
+
|
58
|
+
after_success do |request|
|
59
|
+
doc = Nokogiri::HTML(request)
|
60
|
+
|
61
|
+
# Title
|
62
|
+
title = doc.css('title').first
|
63
|
+
data.title = title.nil? ? match_data[0] : title.content.strip
|
64
|
+
|
65
|
+
# Favicon
|
66
|
+
favicon = doc.css('link[rel="shortcut icon"], link[rel="icon shortcut"], link[rel="shortcut"], link[rel="icon"]').first
|
67
|
+
favicon = favicon.nil? ? nil : favicon['href'].strip
|
68
|
+
if favicon
|
69
|
+
if favicon.match(/^https?:\/\//i).nil?
|
70
|
+
favicon = uri.scheme + "://" + uri.host + uri.path.gsub(/((.*)(\/(.*\..*$)))/i,'\2/') + favicon
|
71
|
+
end
|
72
|
+
data.favicon = favicon
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
after_failure do |error|
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
3
|
+
require 'rspec'
|
4
|
+
require 'urifetch'
|
5
|
+
require 'helpers/url_helpers'
|
6
|
+
|
7
|
+
# Requires supporting files with custom matchers and macros, etc,
|
8
|
+
# in ./support/ and its subdirectories.
|
9
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
10
|
+
|
11
|
+
RSpec.configure do |config|
|
12
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Handler do
|
4
|
+
|
5
|
+
describe '#initialize' do
|
6
|
+
|
7
|
+
it 'should return Urifetch::Handler' do
|
8
|
+
Urifetch::Handler.new(:default).should be_a_kind_of Urifetch::Handler
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should have a default value of a Hash on @handlers' do
|
12
|
+
@handlers = Urifetch::Handler.new(:default).instance_variable_get(:@handlers)
|
13
|
+
@handlers.should_not be_nil
|
14
|
+
@handlers.should be_a_kind_of Hash
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Response do
|
4
|
+
|
5
|
+
describe '#initialize' do
|
6
|
+
|
7
|
+
before do
|
8
|
+
@response = Urifetch::Response.new()
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should contain status as an array' do
|
12
|
+
@response.status.should_not be_nil
|
13
|
+
@response.status.should be_a_kind_of(Array)
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should contain the strategy used when matching' do
|
17
|
+
@response.strategy.should_not be_nil
|
18
|
+
@response.strategy.should be_a_kind_of(Urifetch::Strategy)
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'should contain data as a hashie/mash' do
|
22
|
+
@response.data.should_not be_nil
|
23
|
+
@response.data.should be_a_kind_of(Hashie::Mash)
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Strategy::Layout do
|
4
|
+
|
5
|
+
before do
|
6
|
+
@layout = Urifetch::Strategy::Layout.new do
|
7
|
+
|
8
|
+
after_success do
|
9
|
+
some_method
|
10
|
+
end
|
11
|
+
|
12
|
+
after_failure do
|
13
|
+
some_method
|
14
|
+
end
|
15
|
+
|
16
|
+
before_request do
|
17
|
+
some_method
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe 'after_success' do
|
24
|
+
|
25
|
+
it 'should overwrite the value proc' do
|
26
|
+
proc = @layout.instance_variable_get(:@success)
|
27
|
+
@layout.after_success do
|
28
|
+
some_method
|
29
|
+
end
|
30
|
+
proc.should_not eq(@layout.instance_variable_get(:@success))
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should overwrite the value proc' do
|
34
|
+
proc = @layout.instance_variable_get(:@failure)
|
35
|
+
@layout.after_failure do
|
36
|
+
some_method
|
37
|
+
end
|
38
|
+
proc.should_not eq(@layout.instance_variable_get(:@failure))
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'should overwrite the value proc' do
|
42
|
+
proc = @layout.instance_variable_get(:@before)
|
43
|
+
@layout.before_request do
|
44
|
+
some_method
|
45
|
+
end
|
46
|
+
proc.should_not eq(@layout.instance_variable_get(:@before))
|
47
|
+
end
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch::Strategy do
|
4
|
+
|
5
|
+
describe 'self.layouts' do
|
6
|
+
|
7
|
+
it 'should return an Hashie::Mash' do
|
8
|
+
Urifetch::Strategy.layouts.should be_a_kind_of Hashie::Mash
|
9
|
+
end
|
10
|
+
|
11
|
+
end
|
12
|
+
|
13
|
+
describe 'self.layout' do
|
14
|
+
|
15
|
+
it 'should return a Urifetch::Strategy::Layout instance and make it accessible through .layouts' do
|
16
|
+
@layout = Urifetch::Strategy.layout(:test) do; end
|
17
|
+
@layout.should be_an_instance_of Urifetch::Strategy::Layout
|
18
|
+
Urifetch::Strategy.layouts[:test].should == @layout
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
describe 'self.apply' do
|
24
|
+
|
25
|
+
it 'should return an instance of Urifetch::Strategy' do
|
26
|
+
Urifetch::Strategy.apply(:test).should be_an_instance_of Urifetch::Strategy
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
describe 'self.apply!' do
|
32
|
+
|
33
|
+
it 'should return an instance of Urifetch::Response' do
|
34
|
+
Urifetch::Strategy.apply!(:test).should be_an_instance_of Urifetch::Response
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
before do
|
40
|
+
@strategy = Urifetch::Strategy.new(:test,"".match(//i))
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '#initialize' do
|
44
|
+
|
45
|
+
it 'should return an error if layout was not found' do
|
46
|
+
-> { Urifetch::Strategy.new(:not_found,"".match(//i)) }.should raise_error(RuntimeError, /no matching layouts found/i)
|
47
|
+
end
|
48
|
+
|
49
|
+
it 'should contain match data as a MatchData object' do
|
50
|
+
@strategy.match_data.should be_a_kind_of MatchData
|
51
|
+
end
|
52
|
+
|
53
|
+
it 'should have a layout as a Urifetch::Strategy::Layout' do
|
54
|
+
@strategy.layout.should be_an_instance_of Urifetch::Strategy::Layout
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'execute!' do
|
60
|
+
|
61
|
+
it 'should return an instance of Urifetch::Response' do
|
62
|
+
@strategy.execute!.should be_an_instance_of Urifetch::Response
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Urifetch do
|
4
|
+
|
5
|
+
describe 'self.fetch_from' do
|
6
|
+
|
7
|
+
it 'should return a Urifetch::Response' do
|
8
|
+
Urifetch.fetch_from(Urifetch::Test::UrlHelpers.generate_valid_url).should be_a_kind_of(Urifetch::Response)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should raise an error if url is invalid according to configuration' do
|
12
|
+
-> { Urifetch.fetch_from(Urifetch::Test::UrlHelpers.generate_invalid_url) }.should raise_error ArgumentError, "invalid url"
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
describe 'self.find_strategy_from' do
|
18
|
+
|
19
|
+
it 'should an instance of Urifetch::Strategy' do
|
20
|
+
Urifetch.find_strategy_from("http://www.google.com").should be_an_instance_of Urifetch::Strategy
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'strategy' do
|
26
|
+
|
27
|
+
describe 'default' do
|
28
|
+
|
29
|
+
it 'should return title and favicon for valid path' do
|
30
|
+
@response = Urifetch.fetch_from("http://www.youtube.com")
|
31
|
+
@response.data.should have_key(:title)
|
32
|
+
@response.data.should have_key(:favicon)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should return title and NOT favicon for valid path' do
|
36
|
+
@response = Urifetch.fetch_from("http://www.google.com")
|
37
|
+
@response.data.should have_key(:title)
|
38
|
+
@response.data.should_not have_key(:favicon)
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
data/urifetch.gemspec
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "urifetch"
|
8
|
+
s.version = "0.0.1.rc6"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Philip Vieira"]
|
12
|
+
s.date = "2012-01-14"
|
13
|
+
s.description = "Urifetch allows you to fetch data from any URL using pattern matching. The library allows for dynamically adding match handlers as well as build your own strategies."
|
14
|
+
s.email = "philip@vallin.se"
|
15
|
+
s.extra_rdoc_files = [
|
16
|
+
"LICENSE.txt",
|
17
|
+
"README.md"
|
18
|
+
]
|
19
|
+
s.files = [
|
20
|
+
".document",
|
21
|
+
".rspec",
|
22
|
+
".rvmrc",
|
23
|
+
"Gemfile",
|
24
|
+
"Gemfile.lock",
|
25
|
+
"LICENSE.txt",
|
26
|
+
"README.md",
|
27
|
+
"Rakefile",
|
28
|
+
"lib/.DS_Store",
|
29
|
+
"lib/urifetch.rb",
|
30
|
+
"lib/urifetch/handler.rb",
|
31
|
+
"lib/urifetch/response.rb",
|
32
|
+
"lib/urifetch/strategy.rb",
|
33
|
+
"lib/urifetch/strategy/layout.rb",
|
34
|
+
"lib/urifetch/version.rb",
|
35
|
+
"spec/helpers/url_helpers.rb",
|
36
|
+
"spec/spec_helper.rb",
|
37
|
+
"spec/urifetch/handler_spec.rb",
|
38
|
+
"spec/urifetch/response_spec.rb",
|
39
|
+
"spec/urifetch/strategy/layout_spec.rb",
|
40
|
+
"spec/urifetch/strategy_spec.rb",
|
41
|
+
"spec/urifetch_spec.rb",
|
42
|
+
"urifetch.gemspec"
|
43
|
+
]
|
44
|
+
s.homepage = "http://github.com/zeeraw/Urifetch"
|
45
|
+
s.licenses = ["MIT"]
|
46
|
+
s.require_paths = ["lib"]
|
47
|
+
s.rubygems_version = "1.8.10"
|
48
|
+
s.summary = "Match URL's to request strategies and retrieve them in a usable format."
|
49
|
+
|
50
|
+
if s.respond_to? :specification_version then
|
51
|
+
s.specification_version = 3
|
52
|
+
|
53
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
54
|
+
s.add_runtime_dependency(%q<addressable>, [">= 0"])
|
55
|
+
s.add_runtime_dependency(%q<hashie>, [">= 0"])
|
56
|
+
s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
|
57
|
+
s.add_development_dependency(%q<rspec>, [">= 0"])
|
58
|
+
s.add_development_dependency(%q<bundler>, ["= 1.1.rc"])
|
59
|
+
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
60
|
+
s.add_development_dependency(%q<rcov>, [">= 0"])
|
61
|
+
s.add_development_dependency(%q<pry>, [">= 0"])
|
62
|
+
s.add_development_dependency(%q<rdoc>, [">= 0"])
|
63
|
+
else
|
64
|
+
s.add_dependency(%q<addressable>, [">= 0"])
|
65
|
+
s.add_dependency(%q<hashie>, [">= 0"])
|
66
|
+
s.add_dependency(%q<nokogiri>, [">= 0"])
|
67
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
68
|
+
s.add_dependency(%q<bundler>, ["= 1.1.rc"])
|
69
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
70
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
71
|
+
s.add_dependency(%q<pry>, [">= 0"])
|
72
|
+
s.add_dependency(%q<rdoc>, [">= 0"])
|
73
|
+
end
|
74
|
+
else
|
75
|
+
s.add_dependency(%q<addressable>, [">= 0"])
|
76
|
+
s.add_dependency(%q<hashie>, [">= 0"])
|
77
|
+
s.add_dependency(%q<nokogiri>, [">= 0"])
|
78
|
+
s.add_dependency(%q<rspec>, [">= 0"])
|
79
|
+
s.add_dependency(%q<bundler>, ["= 1.1.rc"])
|
80
|
+
s.add_dependency(%q<jeweler>, [">= 0"])
|
81
|
+
s.add_dependency(%q<rcov>, [">= 0"])
|
82
|
+
s.add_dependency(%q<pry>, [">= 0"])
|
83
|
+
s.add_dependency(%q<rdoc>, [">= 0"])
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
metadata
ADDED
@@ -0,0 +1,174 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: urifetch
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1.rc6
|
5
|
+
prerelease: 6
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Philip Vieira
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-01-14 00:00:00.000000000Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: addressable
|
16
|
+
requirement: &70123223672060 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *70123223672060
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: hashie
|
27
|
+
requirement: &70123223669420 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70123223669420
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: nokogiri
|
38
|
+
requirement: &70123223667980 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *70123223667980
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rspec
|
49
|
+
requirement: &70123223667020 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :development
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *70123223667020
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: bundler
|
60
|
+
requirement: &70123223665120 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - =
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: 1.1.rc
|
66
|
+
type: :development
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *70123223665120
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: jeweler
|
71
|
+
requirement: &70123223663280 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *70123223663280
|
80
|
+
- !ruby/object:Gem::Dependency
|
81
|
+
name: rcov
|
82
|
+
requirement: &70123223662420 !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
84
|
+
requirements:
|
85
|
+
- - ! '>='
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0'
|
88
|
+
type: :development
|
89
|
+
prerelease: false
|
90
|
+
version_requirements: *70123223662420
|
91
|
+
- !ruby/object:Gem::Dependency
|
92
|
+
name: pry
|
93
|
+
requirement: &70123223661260 !ruby/object:Gem::Requirement
|
94
|
+
none: false
|
95
|
+
requirements:
|
96
|
+
- - ! '>='
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: '0'
|
99
|
+
type: :development
|
100
|
+
prerelease: false
|
101
|
+
version_requirements: *70123223661260
|
102
|
+
- !ruby/object:Gem::Dependency
|
103
|
+
name: rdoc
|
104
|
+
requirement: &70123223660520 !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
type: :development
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: *70123223660520
|
113
|
+
description: Urifetch allows you to fetch data from any URL using pattern matching.
|
114
|
+
The library allows for dynamically adding match handlers as well as build your own
|
115
|
+
strategies.
|
116
|
+
email: philip@vallin.se
|
117
|
+
executables: []
|
118
|
+
extensions: []
|
119
|
+
extra_rdoc_files:
|
120
|
+
- LICENSE.txt
|
121
|
+
- README.md
|
122
|
+
files:
|
123
|
+
- .document
|
124
|
+
- .rspec
|
125
|
+
- .rvmrc
|
126
|
+
- Gemfile
|
127
|
+
- Gemfile.lock
|
128
|
+
- LICENSE.txt
|
129
|
+
- README.md
|
130
|
+
- Rakefile
|
131
|
+
- lib/.DS_Store
|
132
|
+
- lib/urifetch.rb
|
133
|
+
- lib/urifetch/handler.rb
|
134
|
+
- lib/urifetch/response.rb
|
135
|
+
- lib/urifetch/strategy.rb
|
136
|
+
- lib/urifetch/strategy/layout.rb
|
137
|
+
- lib/urifetch/version.rb
|
138
|
+
- spec/helpers/url_helpers.rb
|
139
|
+
- spec/spec_helper.rb
|
140
|
+
- spec/urifetch/handler_spec.rb
|
141
|
+
- spec/urifetch/response_spec.rb
|
142
|
+
- spec/urifetch/strategy/layout_spec.rb
|
143
|
+
- spec/urifetch/strategy_spec.rb
|
144
|
+
- spec/urifetch_spec.rb
|
145
|
+
- urifetch.gemspec
|
146
|
+
homepage: http://github.com/zeeraw/Urifetch
|
147
|
+
licenses:
|
148
|
+
- MIT
|
149
|
+
post_install_message:
|
150
|
+
rdoc_options: []
|
151
|
+
require_paths:
|
152
|
+
- lib
|
153
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
154
|
+
none: false
|
155
|
+
requirements:
|
156
|
+
- - ! '>='
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: '0'
|
159
|
+
segments:
|
160
|
+
- 0
|
161
|
+
hash: 1351382682644594782
|
162
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
163
|
+
none: false
|
164
|
+
requirements:
|
165
|
+
- - ! '>'
|
166
|
+
- !ruby/object:Gem::Version
|
167
|
+
version: 1.3.1
|
168
|
+
requirements: []
|
169
|
+
rubyforge_project:
|
170
|
+
rubygems_version: 1.8.10
|
171
|
+
signing_key:
|
172
|
+
specification_version: 3
|
173
|
+
summary: Match URL's to request strategies and retrieve them in a usable format.
|
174
|
+
test_files: []
|