plagiarism2 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bff4f9ef7060e99ba754a8e85d88ded43bc92234
-  data.tar.gz: fb32b027c73191b7724644a4f169db41bd93135b
+  metadata.gz: a5ae907124afcc6b114cb2fd1a575b7d0a4c5c85
+  data.tar.gz: b6c4a942896b1933ce60a7e3216b8c839bf422a5
 SHA512:
-  metadata.gz: c919a74db75b832676a998db8a5084cd7b93c2da18d171178ad98091f7d363bb4058399a397e8bacc401fd61f00b64d7133ab4c8a7421c5483e79eef99e4d4bc
-  data.tar.gz: a38c0840f60715c6c32bdcb8673d2217128d5d945c4bbb6133addc8b1a119bace2aa951f5fa3675cf04bbcf01953651a4cb67ec97fa1beb7b4a7021032557d13
+  metadata.gz: bb93fde868664ee036d0fdaa13ee7de53235a372ec41daf5d72109f490b6985094e44afbd8cd189b77693f1a9283d2216010b7c040a5792505874b58e321ffdd
+  data.tar.gz: 93f3cf1b8fed1d7071a80ce4c92383beec0a1b49de3d27f6956c645cc269b8140e4b16d86bfd63384def1ab4ffa8feb9a70b8bf24dfe610f61f120e7b7ea65ef

data/README.md

@@ -22,20 +22,24 @@ Add in your config
 
 ```ruby
 Plagiarism.configure do |config|
-  config.strategies = [xxx] # => [:google, :bing, :duck, :yahoo]
+  config.strategies = [xxx] # => [:google, :bing, :duck, :yahoo, :free_google]
   config.whitelists = ['www.ring.md']
 end
 ```
 
+> There is a limit in using duck, yahoo and free google. If you spam request, they will mark you as spam
+
 + Using bing engine, you have to set access key (you can get it from [here](https://datamarket.azure.com/dataset/bing/searchweb))
 
 ```ruby
+config.strategies = :bing
 config.bing_key = xxx
 ```
 
-+ Using google engine, you have to set two keys (you can get it from [here](https://developers.google.com/custom-search/json-api/v1/using_rest))
++ Using google engine (not free google), you have to set two keys (you can get it from [here](https://developers.google.com/custom-search/json-api/v1/using_rest))
 
 ```ruby
+config.strategies = :google
 config.google_key = xxx
 config.google_cx = xx
 ```

data/lib/plagiarism/cli.rb

@@ -6,11 +6,22 @@ module Plagiarism
     include Thor::Actions
     CONFIG_PATH = '~/.plagiarism.yml'
 
+    class << self
+      def load_config(path)
+        config = YAML.load_file(File.expand_path path)
+        Config.strategies = config['strategies'].split(',').map &:strip
+        Config.whitelists = config['whitelists'].split(',').map &:strip
+        Config.bing_key = config['bing_key']
+        Config.google_key = config['google_key']
+        Config.google_cx = config['google_cx']
+      end
+    end
+
     desc 'init', 'Create file for plagiarism to load config'
     method_option :path, aliases: '-p', desc: 'where put the config', type: :string, default: CONFIG_PATH
     def init
       create_file options[:path], <<-STRING
-strategies: "duck, yahoo"
+strategies: "yahoo"
 whitelists: "www.ring.md, blog.ring.md"
 bing_key: xxx
 google_key: xxx
@@ -22,14 +33,16 @@ google_cx: xxx
     method_option :content, aliases: '-c', desc: 'content which need to checked', type: :string, required: true
     method_option :path, aliases: '-p', desc: 'where put the config', type: :string, default: CONFIG_PATH
     def unique
-      config = YAML.load_file(File.expand_path options[:path])
-      Config.strategies = config['strategies'].split(',').map &:strip
-      Config.whitelists = config['whitelists'].split(',').map &:strip
-      Config.bing_key = config['bing_key']
-      Config.google_key = config['google_key']
-      Config.google_cx = config['google_cx']
-
+      Cli.load_config(options[:path])
       puts Plagiarism.unique? options[:content]
     end
+
+    desc 'match', 'Get the first match from search engines'
+    method_option :content, aliases: '-c', desc: 'content which need to get', type: :string, required: true
+    method_option :path, aliases: '-p', desc: 'where put the config', type: :string, default: CONFIG_PATH
+    def match
+      Cli.load_config(options[:path])
+      puts Plagiarism.match options[:content]
+    end
   end
 end

data/lib/plagiarism/strategries/bing.rb

@@ -9,11 +9,11 @@ module Plagiarism
           Typhoeus.get(URL, params: params.merge('$format' => :json, 'Query' => "'#{content}'"), userpwd: ":#{Config.bing_key}")
         end
 
-        def exists?(response)
+        def iterate(response)
           results = JSON.parse(response)['d']['results'] rescue []
           results.all? do |r|
             uri = URI.parse URI::encode(r['Url'])
-            uri.host =~ whitelists_regex
+            yield uri
           end
         end
 

data/lib/plagiarism/strategries/duck.rb

@@ -9,17 +9,16 @@ module Plagiarism
           Typhoeus.get(URL, params: params.merge(q: content))
         end
 
-        def exists?(response)
+        def iterate(response)
           doc = Nokogiri::HTML response
           doc.css('.results_links_deep:not(.result--no-result)').all? do |row|
             href = row.at_css('.result__a').attributes['href'].value rescue ''
             uri = URI.parse URI::encode(href)
-            uri.host =~ whitelists_regex
+            yield uri
           end
         end
 
       end
-
     end
   end
 end

data/lib/plagiarism/strategries/engine.rb

@@ -10,6 +10,14 @@ module Plagiarism
           raise
         end
 
+        def iterate(r)
+          raise
+        end
+
+        def exists?(response)
+          iterate(response) { |uri| uri.host =~ whitelists_regex }
+        end
+
         def valid_segments(ps, params)
           ps.segment.count do |sentence|
             typhoeus = fetch("\"#{sentence}\"", params)
@@ -17,10 +25,6 @@ module Plagiarism
           end
         end
 
-        def exists?(response)
-          raise
-        end
-
         def whitelists_regex
           whitelists = Config.whitelists.map { |w| Regexp.new w }
           Regexp.union whitelists
@@ -31,11 +35,25 @@ module Plagiarism
         @content, @params = c, p
       end
 
+      def retrieve_link(response)
+        raise
+      end
+
       def unique?
         ps = PragmaticSegmenter::Segmenter.new(text: content)
         valid_segments = self.class.valid_segments(ps, params)
         valid_segments / ps.segment.size >= THRESHOLD
       end
+
+      def match
+        typhoeus = self.class.fetch("\"#{content}\"", params)
+        typhoeus.success? && retrieve_link(typhoeus.response_body)
+      end
+
+      def retrieve_link(response)
+        self.class.iterate(response) { |uri| uri.host !~ self.class.whitelists_regex and return uri.to_s }
+      end
+
     end
   end
 end

data/lib/plagiarism/strategries/free_google.rb

@@ -0,0 +1,25 @@
+module Plagiarism
+  module Strategies
+    class FreeGoogle < Engine
+      URL = 'https://ajax.googleapis.com/ajax/services/search/web'
+      VERSION = '1.0'
+
+      class << self
+
+        def fetch(content, params)
+          Typhoeus.get URL, params: params.merge(v: VERSION, q: content, rsz: :large)
+        end
+
+        def iterate(response)
+          results = JSON.parse(response)['responseData']['results'] rescue []
+          results.all? do |r|
+            uri = URI.parse URI::encode(r['unescapedUrl'])
+            yield uri
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/plagiarism/strategries/google.rb

@@ -16,11 +16,11 @@ module Plagiarism
           )
         end
 
-        def exists?(response)
+        def iterate(response)
           results = JSON.parse(response)['items'] || []
           results.all? do |r|
             uri = URI.parse URI::encode(r['link'])
-            uri.host =~ whitelists_regex
+            yield uri
           end
         end
 

data/lib/plagiarism/strategries/yahoo.rb

@@ -9,12 +9,12 @@ module Plagiarism
           Typhoeus.get(URL, params: params.merge(p: content))
         end
 
-        def exists?(response)
+        def iterate(response)
           doc = Nokogiri::HTML response
           doc.css('.searchCenterMiddle li').all? do |row|
-            href = row.at_css('.compTitle div').content.strip rescue ''
-            uri = URI.parse URI::encode(href =~ /^http/ ? href : 'https://' + href)
-            uri.host =~ whitelists_regex
+            href = row.at_css('.compTitle a').attributes['href'].value rescue ''
+            uri = URI.parse URI::encode(href)
+            yield uri
           end
         end
 

data/lib/plagiarism/strategy.rb

@@ -1,5 +1,6 @@
 require 'plagiarism/strategries/engine'
 require 'plagiarism/strategries/google'
+require 'plagiarism/strategries/free_google'
 require 'plagiarism/strategries/bing'
 require 'plagiarism/strategries/duck'
 require 'plagiarism/strategries/yahoo'
@@ -8,8 +9,8 @@ module Plagiarism
   module Strategy
     extend self
 
-    def get(name = :google)
-      Strategies.const_get(name.to_s.sub(/\S/, &:upcase))
+    def get(name = :free_google)
+      Strategies.const_get(name.to_s.split('_').map(&:capitalize).join(''))
     end
 
     def unique?(content, params)
@@ -18,5 +19,13 @@ module Plagiarism
         klass.new(content, params).unique?
       end
     end
+
+    def match(content, params)
+      Config.strategies.find do |strategy|
+        klass = get(strategy)
+        link = klass.new(content, params).match
+        link and return (link == true ? '' : link)
+      end
+    end
   end
 end

data/lib/plagiarism/version.rb

@@ -1,3 +1,3 @@
 module Plagiarism
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/plagiarism.rb

@@ -19,4 +19,8 @@ module Plagiarism
   def unique?(content, params = {})
     Strategy.unique?(content, params)
   end
+
+  def match(content, params = {})
+    Strategy.match(content, params)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: plagiarism2
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - MQuy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-02 00:00:00.000000000 Z
+date: 2016-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -146,6 +146,7 @@ files:
 - lib/plagiarism/strategries/bing.rb
 - lib/plagiarism/strategries/duck.rb
 - lib/plagiarism/strategries/engine.rb
+- lib/plagiarism/strategries/free_google.rb
 - lib/plagiarism/strategries/google.rb
 - lib/plagiarism/strategries/yahoo.rb
 - lib/plagiarism/strategy.rb