morito 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bc8084f99cd40ba2b8f121d0af27cc5c7161ad15
-  data.tar.gz: 03c0f154b98907acc7606097724a05b31e6e57d5
+  metadata.gz: 50f1f0b304f54a4d042d91ab748679af6aa8064d
+  data.tar.gz: ada9db82752169ccfd43f7337bb97ec51d36827a
 SHA512:
-  metadata.gz: 3dee2cc8743162be7c3ac36c17f15d67d035dcd911a1c344f20adca5f48f2117cff77a77b92cde2d97ef49f47de63c92c2220568231f9694f54ee808803485cd
-  data.tar.gz: a8062db1f264247a2ff2f3630b94142f046227f5b56bdba63cc45b028395d6c01047da745cd205a585ef6d83221685343c8c06e17c15f312e77074d4950d7e9b
+  metadata.gz: 7d864fa9b215e1b7b076b4fdba050e993ef52a402e4792dd8c882cc9f057eeb73c888eaeab624731393a2675b421801ffe7a32811719a1539f74f5d09ffcbdd3
+  data.tar.gz: 50a9ed68a538e2f8b15caa8b26221a770a5507abc51e310d0453223eb1b313e217f6954d4c4cf432eb8c0a5c6e6f3bfd4cdcce4d93e27a9562a796088cfb83a4

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.1.1
+env:
+  - CODECLIMATE_REPO_TOKEN=8c8730934142da9cee2f45467b27dc2e0e19befc98ae5f9d607c4bcb32741c54

data/README.md

@@ -1,6 +1,6 @@
-# Morito
+# Morito [![Build Status](https://travis-ci.org/negipo/morito.svg?branch=master)](https://travis-ci.org/negipo/morito) [![Code Climate](https://codeclimate.com/github/negipo/morito.png)](https://codeclimate.com/github/negipo/morito) [![Code Climate](https://codeclimate.com/github/negipo/morito/coverage.png)](https://codeclimate.com/github/negipo/morito)
 
-robots.txt parser client
+A client to handle robots.txt
 
 ## Installation
 
@@ -24,6 +24,10 @@ client.allowed?('http://example.com/some/path') # => true / false
 client.allowed?('http://example.com/some/path', cache: false) # => true / false without cache
 ```
 
+## TODO
+
+- Currently does not handle `Allow:` description.
+
 ## Contributing
 
 1. Fork it ( https://github.com/negipo/morito/fork )

data/Rakefile

@@ -1,2 +1,5 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
 
+RSpec::Core::RakeTask.new(:spec)
+task default: :spec

data/lib/morito/client.rb

@@ -1,3 +1,5 @@
+require 'uri'
+
 module Morito
   class Client
     def initialize(user_agent)
@@ -6,8 +8,29 @@ module Morito
 
     def allowed?(requesting_url, cache: true)
       uri = URI.parse(requesting_url)
-      robots_txt_body = Morito::Connector.new(uri, cache: cache).get
+      robots_txt_body = robots_txt_body(uri, cache: cache)
       Morito::Processor.new(robots_txt_body).allowed?(@user_agent, uri.path)
     end
+
+    private
+
+    def robots_txt_body(uri, cache: true)
+      if cache
+        with_cache(uri.host) do
+          robots_txt_body_without_cache(uri)
+        end
+      else
+        robots_txt_body_without_cache(uri)
+      end
+    end
+
+    def robots_txt_body_without_cache(uri)
+      Morito::Connector.new(uri).get
+    end
+
+    def with_cache(host, &block)
+      @cache ||= {}
+      @cache[host] ||= yield
+    end
   end
 end

data/lib/morito/connector.rb

@@ -1,38 +1,19 @@
 require 'faraday'
-require 'uri'
 
 module Morito
   class Connector
     PATH = '/robots.txt'
 
-    attr_reader :uri
-
-    def initialize(uri, cache: true)
+    def initialize(uri)
       @uri = uri
-      @cache = cache
     end
 
     def get
-      if @cache
-        self.class.with_cache(base_url) do
-          raw_get
-        end
-      else
-        raw_get
-      end
+      @get ||= connection.get(PATH).body
     end
 
     private
 
-    def self.with_cache(base_url, &block)
-      @cache ||= {}
-      @cache[base_url] ||= yield
-    end
-
-    def raw_get
-      @get ||= connection.get(PATH).body
-    end
-
     def connection
       Faraday.new(url: base_url) do |connection|
         connection.adapter :net_http

data/lib/morito/processor.rb

@@ -19,7 +19,12 @@ module Morito
 
     def disallows_for(user_agent)
       build
-      @disallows['*'] + @disallows[user_agent]
+
+      if !@disallows[user_agent].empty?
+        @disallows[user_agent]
+      else
+        @disallows['*']
+      end
     end
 
     def build

data/lib/morito/version.rb

@@ -1,3 +1,3 @@
 module Morito
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/morito.gemspec

@@ -24,4 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "webmock"
+  spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "codeclimate-test-reporter"
 end

data/spec/morito/client_spec.rb

@@ -1,50 +1,81 @@
 require 'spec_helper'
 
 describe Morito::Client do
-  before do
-    stub_request(:any, //).to_return(body: body)
-  end
-
   let(:client) { described_class.new(user_agent) }
-  let(:body) do
-    <<-EOS
+
+  describe '#allowed?' do
+    subject { client.allowed?(requesting_url, cache: cache) }
+
+    let(:cache) { true }
+
+    context 'with response' do
+      let(:body) do
+        <<-EOS
 User-agent: *
 Disallow: /private
 
+# Comment
 User-agent: restricted agent # Comment
 Disallow: /
+
+User-agent: allowed agent
+Disallow: /super_private
 EOS
-  end
+      end
 
-  describe '#allowed?' do
-    subject { client.allowed?(requesting_url) }
+      before do
+        stub_request(:any, //).to_return(body: body)
+      end
 
-    context 'with public path' do
-      let(:requesting_url) { 'http://example.com/public/path' }
+      context 'with public path' do
+        let(:requesting_url) { 'http://example.com/public/path' }
+
+        context 'with some agent' do
+          let(:user_agent) { 'some agent' }
+          it { should == true }
+        end
 
-      context 'with some agent' do
-        let(:user_agent) { 'some agent' }
-        it { should == true }
+        context 'with restricted agent' do
+          let(:user_agent) { 'restricted agent' }
+          it { should == false }
+        end
+
+        context 'without cache' do
+          let(:user_agent) { 'some agent' }
+          let(:cache) { false }
+          it { should == true }
+        end
       end
 
-      context 'with restricted agent' do
-        let(:user_agent) { 'restricted agent' }
-        it { should == false }
+      context 'with private path' do
+        let(:requesting_url) { 'http://example.com/private/path' }
+
+        context 'with some agent' do
+          let(:user_agent) { 'some agent' }
+          it { should == false }
+        end
+
+        context 'with restricted agent' do
+          let(:user_agent) { 'restricted agent' }
+          it { should == false }
+        end
+
+        context 'with allowed agent' do
+          let(:user_agent) { 'allowed agent' }
+          it { should == true }
+        end
       end
     end
 
-    context 'with private path' do
-      let(:requesting_url) { 'http://example.com/private/path' }
+    context 'with 404' do
+      let(:requesting_url) { 'http://example.com/public/path' }
+      let(:user_agent) { 'restricted agent' }
 
-      context 'with some agent' do
-        let(:user_agent) { 'some agent' }
-        it { should == false }
+      before do
+        stub_request(:any, //).to_return(body: '', status: 404)
       end
 
-      context 'with restricted agent' do
-        let(:user_agent) { 'restricted agent' }
-        it { should == false }
-      end
+      it { should == true }
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,4 +1,14 @@
+require 'simplecov'
+SimpleCov.start
+
 require 'webmock/rspec'
+
+if ENV['CI']
+  WebMock.disable_net_connect!(allow: %w(codeclimate.com))
+  require 'codeclimate-test-reporter'
+  CodeClimate::TestReporter.start
+end
+
 require 'morito'
 
 RSpec.configure do |config|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: morito
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - negipo
@@ -94,6 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - negipo@gmail.com
@@ -102,6 +130,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md