seospider 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 984a688994b3020f4b7187b54694925197c3eac8
+  data.tar.gz: d252376604c47e26e2c5495496134c28b46b34ee
+SHA512:
+  metadata.gz: 66624790fe10be37ae65053c814063ddbd54d610e4d5e1f008b880682efcdbebb97ca7b68db6c6d6b14cd4cddfe5b7ff79f6e97425c5e83a007d049fbaffeb5d
+  data.tar.gz: 7598dde6a43ae5a8fd57fadc56cfe4065ce649033366570ae0940b9e811bbaa12761c07fef527dcfbe25706c90411931ec77cd19cdcdae51ea1716988d4e8d38

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in seospider.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 MingQian Zhang
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Seospider
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'seospider'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install seospider
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/ss

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require 'seospider'
+require 'optparse'
+require 'pp'
+
+usage = "Usage: 
+ss -u url
+"
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = usage
+
+  opts.on("-u URL", "--url URL", "URL to check") do |v|
+    options[:url] = v
+  end
+
+end.parse!
+
+options[:url] = ARGV[0] if options[:url].nil?
+
+page = Seospider.get options[:url]
+=begin
+(page.keys - [:headers, :body]).each do |k|
+  puts [k, page[k]].join(': ')
+end
+
+page[:headers].each_pair do |k,v|
+  puts [k,v].join(': ')
+end
+=end
+
+puts YAML.dump(page)

data/install.rb

@@ -0,0 +1,15 @@
+path = File.expand_path('..',__FILE__)
+gem_name = File.basename(Dir["#{path}/*.gemspec"].first).sub(/\.gemspec$/,'')
+
+puts "Installing gem #{gem_name}"
+puts "Gem source path is #{path}"
+puts `
+cd #{path}
+rm #{gem_name}-*.gem
+echo '#{gem_name}-*.gem deleted'
+gem build #{gem_name}.gemspec
+gem uninstall #{gem_name} -x -a -I
+gem install #{gem_name}-*.gem
+
+`
+puts "#{gem_name} gem installed successfully!!!"

data/lib/seospider/client.rb

@@ -0,0 +1,10 @@
+require 'httparty'
+
+module Seospider
+  class Client
+    include HTTParty
+
+    follow_redirects false
+
+  end
+end

data/lib/seospider/parser.rb

@@ -0,0 +1,111 @@
+require 'nokogiri'
+require 'seospider/client'
+require 'uri'
+require 'webrobots'
+
+module Seospider
+  class Parser
+    attr_reader :result
+
+    def initialize(attrs)
+      @url = attrs[:url]
+      @uri = URI(@url)
+      @user_agent = attrs[:user_agent] || 'Baiduspider'
+      @webrobots = WebRobots.new(@user_agent)
+      @debug = false
+
+      time_start = Time.now
+      r = Client.get(@url)
+      time_end = Time.now
+      response_time = time_end - time_start
+
+      @html = r.body.encode!('UTF-8','UTF-8',:invalid => :replace)
+      @doc = Nokogiri::HTML(@html)
+
+      @result = {:url => @url,
+        :status => r.response.code.to_i,
+        :location => r.headers['location'],
+        :response_time => response_time,
+        :canonical => '',
+        :title => '',
+        :meta_keywords => '',
+        :meta_description => '',
+        :meta_robots => '',
+        :h1 => '',
+        :h2 => '',
+        :h3 => '',
+        :links => '',
+        :headers => r.headers.to_hash
+      }
+      
+      parse
+    end
+
+    def parse
+      self.methods.each do |m|
+        next unless m =~ /^_parse_/
+
+        begin
+          value = self.send m
+        rescue Exception => e
+          warn "#{e.class} - #{e.message} -- #{m}" if @debug
+          value = nil
+        end
+
+        @result[m.to_s.sub('_parse_','').to_sym] = value
+      end
+
+      @result
+    end
+
+    def _parse_title
+      @doc.search('title').first.content
+    end
+
+    def _parse_meta_description
+      @doc.search('meta[@name="description"]').first['content']
+    end
+
+    def _parse_meta_keywords
+      @doc.search('meta[@name="keywords"]').first['content']
+    end
+
+    def _parse_meta_robots
+      @doc.search('meta[@name="robots"]').first['content']
+    end
+
+    def _parse_links
+      links = []
+      @doc.search('a').each do |a|
+        href = a['href']
+        text = a.content.strip
+        rel = a['rel']
+
+        url = @uri.merge(URI.escape(href.to_s))
+        if url.host == @uri.host
+          disallow = @webrobots.disallowed?(url.to_s)
+        else
+          disallow = nil
+        end
+
+        links << {href: href, text: text, rel: rel, disallow: disallow}
+      end
+      links
+    end
+
+    def _parse_canonical
+      @doc.search('link[@rel="canonical"]').first['href']
+    end
+
+    (1..6).each do |i|
+      define_method "_parse_h#{i}" do
+        result = []
+        @doc.search("h#{i}").each do |h|
+          result << h.content.strip
+        end
+        result
+      end
+    end
+
+  end
+end

data/lib/seospider/version.rb

@@ -0,0 +1,3 @@
+module Seospider
+  VERSION = "0.0.1"
+end

data/lib/seospider.rb

@@ -0,0 +1,8 @@
+require 'seospider/version'
+require 'seospider/parser'
+
+module Seospider
+  def self.get(url)
+      Parser.new(url: url).result
+  end
+end

data/seospider.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'seospider/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "seospider"
+  spec.version       = Seospider::VERSION
+  spec.authors       = ["MingQian Zhang"]
+  spec.email         = ["zmingqian@qq.com"]
+  spec.description   = %q{SEO Spider : HTTP Client for SEO}
+  spec.summary       = %q{SEO SPider}
+  spec.homepage      = "https://github.com/mqzhang/seospider"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'httparty'
+  spec.add_runtime_dependency 'webrobots'
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: seospider
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- MingQian Zhang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrobots
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'SEO Spider : HTTP Client for SEO'
+email:
+- zmingqian@qq.com
+executables:
+- ss
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/ss
+- install.rb
+- lib/seospider.rb
+- lib/seospider/client.rb
+- lib/seospider/parser.rb
+- lib/seospider/version.rb
+- seospider.gemspec
+homepage: https://github.com/mqzhang/seospider
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: SEO SPider
+test_files: []