morito 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50f1f0b304f54a4d042d91ab748679af6aa8064d
-  data.tar.gz: ada9db82752169ccfd43f7337bb97ec51d36827a
+  metadata.gz: 1c55decebb4c9e07810ea06f3f21d8f40088faa5
+  data.tar.gz: 1268725db5c75afe009299dbd5f3c29e3780ba80
 SHA512:
-  metadata.gz: 7d864fa9b215e1b7b076b4fdba050e993ef52a402e4792dd8c882cc9f057eeb73c888eaeab624731393a2675b421801ffe7a32811719a1539f74f5d09ffcbdd3
-  data.tar.gz: 50a9ed68a538e2f8b15caa8b26221a770a5507abc51e310d0453223eb1b313e217f6954d4c4cf432eb8c0a5c6e6f3bfd4cdcce4d93e27a9562a796088cfb83a4
+  metadata.gz: f36208dbab6355df0c691c3f4cef8783abbf10ae211c7b8440df5bd951380d4db4c0f452391f95e3b7a8e9b54c93f1a5aedea890edcaf495784ef2a1e0c120ee
+  data.tar.gz: 58dae02a4a09fa4529f6889a8da4597431a857285938c03e887813797347069ac34995dbde8ca756eec89537e4026c26f5929c4570bbc289fe27d55fa60af63c

data/README.md

@@ -26,7 +26,8 @@ client.allowed?('http://example.com/some/path', cache: false) # => true / false
 
 ## TODO
 
-- Currently does not handle `Allow:` description.
+- Handle `Allow:` description.
+- Pattern matching like `Disallow: /*?`
 
 ## Contributing
 

data/lib/morito/client.rb

@@ -9,11 +9,19 @@ module Morito
     def allowed?(requesting_url, cache: true)
       uri = URI.parse(requesting_url)
       robots_txt_body = robots_txt_body(uri, cache: cache)
-      Morito::Processor.new(robots_txt_body).allowed?(@user_agent, uri.path)
+      Morito::Processor.new(robots_txt_body).allowed?(@user_agent, path(uri))
     end
 
     private
 
+    def path(uri)
+      if uri.query
+        "#{uri.path}?#{uri.query}"
+      else
+        uri.path
+      end
+    end
+
     def robots_txt_body(uri, cache: true)
       if cache
         with_cache(uri.host) do

data/lib/morito/processor.rb

@@ -41,7 +41,7 @@ module Morito
     end
 
     class LineParser
-      attr_reader :user_agent, :disallow
+      attr_reader :user_agent
 
       def parse(line)
         case line
@@ -55,6 +55,10 @@ module Morito
         end
       end
 
+      def disallow
+        Regexp.escape(@disallow).gsub('\*', '.*').gsub('\$', '$')
+      end
+
       def disallow?
         @user_agent && @disallow
       end

data/lib/morito/version.rb

@@ -1,3 +1,3 @@
 module Morito
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/morito/client_spec.rb

@@ -13,6 +13,7 @@ describe Morito::Client do
         <<-EOS
 User-agent: *
 Disallow: /private
+Disallow: /*?$
 
 # Comment
 User-agent: restricted agent # Comment
@@ -65,6 +66,20 @@ EOS
           it { should == true }
         end
       end
+
+      context 'with patternized path' do
+        let(:user_agent) { 'some agent' }
+
+        context 'for eol' do
+          let(:requesting_url) { 'http://example.com/some/path?' }
+          it { should == false }
+        end
+
+        context 'for no eol' do
+          let(:requesting_url) { 'http://example.com/some/path?param=1' }
+          it { should == true }
+        end
+      end
     end
 
     context 'with 404' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: morito
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - negipo