morito 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: bc8084f99cd40ba2b8f121d0af27cc5c7161ad15
4
+ data.tar.gz: 03c0f154b98907acc7606097724a05b31e6e57d5
5
+ SHA512:
6
+ metadata.gz: 3dee2cc8743162be7c3ac36c17f15d67d035dcd911a1c344f20adca5f48f2117cff77a77b92cde2d97ef49f47de63c92c2220568231f9694f54ee808803485cd
7
+ data.tar.gz: a8062db1f264247a2ff2f3630b94142f046227f5b56bdba63cc45b028395d6c01047da745cd205a585ef6d83221685343c8c06e17c15f312e77074d4950d7e9b
data/.gitignore ADDED
@@ -0,0 +1,22 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in morito.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 negipo
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,33 @@
1
+ # Morito
2
+
3
+ robots.txt parser client
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'morito'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install morito
18
+
19
+ ## Usage
20
+
21
+ ```ruby
22
+ client = Morito::Client.new('some user agent')
23
+ client.allowed?('http://example.com/some/path') # => true / false
24
+ client.allowed?('http://example.com/some/path', cache: false) # => true / false without cache
25
+ ```
26
+
27
+ ## Contributing
28
+
29
+ 1. Fork it ( https://github.com/negipo/morito/fork )
30
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
31
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
32
+ 4. Push to the branch (`git push origin my-new-feature`)
33
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+
@@ -0,0 +1,13 @@
1
+ module Morito
2
+ class Client
3
+ def initialize(user_agent)
4
+ @user_agent = user_agent
5
+ end
6
+
7
+ def allowed?(requesting_url, cache: true)
8
+ uri = URI.parse(requesting_url)
9
+ robots_txt_body = Morito::Connector.new(uri, cache: cache).get
10
+ Morito::Processor.new(robots_txt_body).allowed?(@user_agent, uri.path)
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,46 @@
1
+ require 'faraday'
2
+ require 'uri'
3
+
4
+ module Morito
5
+ class Connector
6
+ PATH = '/robots.txt'
7
+
8
+ attr_reader :uri
9
+
10
+ def initialize(uri, cache: true)
11
+ @uri = uri
12
+ @cache = cache
13
+ end
14
+
15
+ def get
16
+ if @cache
17
+ self.class.with_cache(base_url) do
18
+ raw_get
19
+ end
20
+ else
21
+ raw_get
22
+ end
23
+ end
24
+
25
+ private
26
+
27
+ def self.with_cache(base_url, &block)
28
+ @cache ||= {}
29
+ @cache[base_url] ||= yield
30
+ end
31
+
32
+ def raw_get
33
+ @get ||= connection.get(PATH).body
34
+ end
35
+
36
+ def connection
37
+ Faraday.new(url: base_url) do |connection|
38
+ connection.adapter :net_http
39
+ end
40
+ end
41
+
42
+ def base_url
43
+ "#{@uri.scheme}://#{@uri.host}:#{@uri.port}"
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,58 @@
1
+ module Morito
2
+ class Processor
3
+ def initialize(body)
4
+ @body = body || ''
5
+ end
6
+
7
+ def allowed?(user_agent, path)
8
+ disallows = disallows_for(user_agent)
9
+
10
+ if disallows.empty?
11
+ true
12
+ else
13
+ regexp = Regexp.new("\\A(?:#{disallows.join('|')})")
14
+ regexp !~ path
15
+ end
16
+ end
17
+
18
+ private
19
+
20
+ def disallows_for(user_agent)
21
+ build
22
+ @disallows['*'] + @disallows[user_agent]
23
+ end
24
+
25
+ def build
26
+ return if @disallows
27
+ @disallows = Hash.new {|h, k| h[k] = [] }
28
+
29
+ parser = LineParser.new
30
+ @body.split(/\n+/).each do |line|
31
+ parser.parse(line)
32
+ @disallows[parser.user_agent] << parser.disallow if parser.disallow?
33
+ end
34
+
35
+ @disallows
36
+ end
37
+
38
+ class LineParser
39
+ attr_reader :user_agent, :disallow
40
+
41
+ def parse(line)
42
+ case line
43
+ when /\AUser-agent:\s+(.+?)\s*(?:#.+)?\z/i
44
+ @user_agent = $1
45
+ @disallow = nil
46
+ when /\ADisallow:\s+(.+?)\s*(?:#.+)?\z/i
47
+ @disallow = $1
48
+ else
49
+ @disallow = nil
50
+ end
51
+ end
52
+
53
+ def disallow?
54
+ @user_agent && @disallow
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,3 @@
1
+ module Morito
2
+ VERSION = "0.0.1"
3
+ end
data/lib/morito.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'morito/version'
2
+ require 'morito/processor'
3
+ require 'morito/connector'
4
+ require 'morito/client'
data/morito.gemspec ADDED
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'morito/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "morito"
8
+ spec.version = Morito::VERSION
9
+ spec.authors = ["negipo"]
10
+ spec.email = ["negipo@gmail.com"]
11
+ spec.summary = %q{A client to handle robots.txt}
12
+ spec.homepage = "https://github.com/negipo/morito"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+ spec.required_ruby_version = '>= 2.0'
20
+
21
+ spec.add_dependency "faraday"
22
+ spec.add_development_dependency "bundler", "~> 1.6"
23
+ spec.add_development_dependency "rake"
24
+ spec.add_development_dependency "pry"
25
+ spec.add_development_dependency "rspec"
26
+ spec.add_development_dependency "webmock"
27
+ end
@@ -0,0 +1,50 @@
1
+ require 'spec_helper'
2
+
3
+ describe Morito::Client do
4
+ before do
5
+ stub_request(:any, //).to_return(body: body)
6
+ end
7
+
8
+ let(:client) { described_class.new(user_agent) }
9
+ let(:body) do
10
+ <<-EOS
11
+ User-agent: *
12
+ Disallow: /private
13
+
14
+ User-agent: restricted agent # Comment
15
+ Disallow: /
16
+ EOS
17
+ end
18
+
19
+ describe '#allowed?' do
20
+ subject { client.allowed?(requesting_url) }
21
+
22
+ context 'with public path' do
23
+ let(:requesting_url) { 'http://example.com/public/path' }
24
+
25
+ context 'with some agent' do
26
+ let(:user_agent) { 'some agent' }
27
+ it { should == true }
28
+ end
29
+
30
+ context 'with restricted agent' do
31
+ let(:user_agent) { 'restricted agent' }
32
+ it { should == false }
33
+ end
34
+ end
35
+
36
+ context 'with private path' do
37
+ let(:requesting_url) { 'http://example.com/private/path' }
38
+
39
+ context 'with some agent' do
40
+ let(:user_agent) { 'some agent' }
41
+ it { should == false }
42
+ end
43
+
44
+ context 'with restricted agent' do
45
+ let(:user_agent) { 'restricted agent' }
46
+ it { should == false }
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,7 @@
1
+ require 'webmock/rspec'
2
+ require 'morito'
3
+
4
+ RSpec.configure do |config|
5
+ config.run_all_when_everything_filtered = true
6
+ config.filter_run :focus
7
+ end
metadata ADDED
@@ -0,0 +1,143 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: morito
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - negipo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-06-13 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: faraday
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.6'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: pry
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rspec
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: webmock
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description:
98
+ email:
99
+ - negipo@gmail.com
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - ".gitignore"
105
+ - Gemfile
106
+ - LICENSE.txt
107
+ - README.md
108
+ - Rakefile
109
+ - lib/morito.rb
110
+ - lib/morito/client.rb
111
+ - lib/morito/connector.rb
112
+ - lib/morito/processor.rb
113
+ - lib/morito/version.rb
114
+ - morito.gemspec
115
+ - spec/morito/client_spec.rb
116
+ - spec/spec_helper.rb
117
+ homepage: https://github.com/negipo/morito
118
+ licenses:
119
+ - MIT
120
+ metadata: {}
121
+ post_install_message:
122
+ rdoc_options: []
123
+ require_paths:
124
+ - lib
125
+ required_ruby_version: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - ">="
128
+ - !ruby/object:Gem::Version
129
+ version: '2.0'
130
+ required_rubygems_version: !ruby/object:Gem::Requirement
131
+ requirements:
132
+ - - ">="
133
+ - !ruby/object:Gem::Version
134
+ version: '0'
135
+ requirements: []
136
+ rubyforge_project:
137
+ rubygems_version: 2.2.2
138
+ signing_key:
139
+ specification_version: 4
140
+ summary: A client to handle robots.txt
141
+ test_files:
142
+ - spec/morito/client_spec.rb
143
+ - spec/spec_helper.rb