webmaster_tools 0.1.0.rc1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changes
+
+## v0.1.x - ???
+ * add tests
+ * add docu
+ * check different auth optinos
+ * integrate official API
+
+## v0.1.0 - 05-03-2012
+
+_inital release_

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in webmaster_tools.gemspec
+gemspec

data/LICENCE

@@ -0,0 +1,26 @@
+Copyright (c) 2012, SoundCloud, Tobias Bielohlawek
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+- Neither the name of the SoundCloud nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,52 @@
+# Webmaster Tools
+
+* API very limited
+* subset of tools
+* based on mechanize
+
+## Usage
+
+Simple usage case to get error counts
+
+```ruby
+require 'webmaster_tools'
+
+client = WebmasterTools.new(<username>, <password>)
+
+pp client.crawl_error_counts(<url>)
+
+```
+
+## Interface
+
+### crawl
+
+#### info
+#### errors
+#### stats
+
+### other
+
+#### submit removal request
+
+
+## Dependencies
+
+Depends on [mechanize](http://mechanize.rubyforge.org/) to access the Webinterface
+
+
+## Contributing
+
+We'll check out your contribution if you:
+
+- Provide a comprehensive suite of tests for your fork.
+- Have a clear and documented rationale for your changes.
+- Package these up in a pull request.
+
+We'll do our best to help you out with any contribution issues you may have.
+
+
+## License
+
+The license is included as LICENSE in this directory.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/VERSION

@@ -0,0 +1 @@
+0.1.0.rc1

data/lib/webmaster_tools.rb

@@ -0,0 +1,96 @@
+# Copyright (c) 2012, SoundCloud Ltd., Tobias Bielohlawek
+
+require 'mechanize'
+
+# WebmasterTools
+# Parameters:
+#  required:
+#   :username  -
+#   :password  -
+#
+#  optional:
+#   :url -
+#   :security_token -
+class WebmasterTools
+  LOGIN     = "https://accounts.google.com/ServiceLogin?service=sitemaps"
+  REMOVAL   = "https://www.google.com/webmasters/tools/removals-request?hl=en&siteUrl=%s&urlt=%s"
+  INFO      = "https://www.google.com/webmasters/tools/sitemaps-dl?hl=en&siteUrl=%s&security_token=%s"
+  DASHBOARD = "https://www.google.com/webmasters/tools/dashboard?hl=en&siteUrl=%s"
+  ERRORS    = "https://www.google.com/webmasters/tools/crawl-errors?hl=en&siteUrl=%s"
+  STATS     = "https://www.google.com/webmasters/tools/crawl-stats?hl=en&siteUrl=%s"
+
+  def initialize(username, password)
+    login(username, password)
+  end
+
+  def login(username, password)
+    page = agent.get(LOGIN)
+    page = agent.submit(page.form.tap do |form|
+      form.Email  = username
+      form.Passwd = password
+    end)
+  end
+
+  def dashboard(url)
+    url   = norm_url(url)
+    page  = agent.get(DASHBOARD % url)
+    {
+      :indexed => page.search("#sitemap tbody .rightmost").text.gsub(/\D/, '').to_i
+    }
+  end
+
+  def crawl_info(url, token)
+    url   = norm_url(url)
+    page  = agent.get(INFO % [url, token])
+
+    lines = page.content.split("\n").map do |line|
+      line.split(",")
+    end
+    head  = lines.shift.map { |key| key.downcase.gsub(' ', '_').to_sym }
+
+    $lines = lines.map do |line|
+      Hash[head.zip(line)]
+    end
+  end
+
+  def crawl_stats(url)
+    url   = norm_url(url)
+    types = %w(pages kilobytes milliseconds).map(&:to_sym)
+    head  = %w(high avg low).map(&:to_sym)
+
+    page  = agent.get(STATS % url)
+
+    Hash[types.zip(page.search(".hostload-activity tr td").map do |node|
+      node.text.gsub(/\D/, '').to_i
+    end.each_slice(3).map do |slice|
+      Hash[head.zip(slice)]
+    end)]
+  end
+
+  def crawl_error_counts(url)
+    url  = norm_url(url)
+    page = agent.get(ERRORS % url)
+
+    page.search(".categories a").inject({}) do |hash, n|
+      key, value = n.text.split("\n")
+      hash[key.downcase.gsub(' ', '_').to_sym] = value.gsub(/\D/, '').to_i
+      hash
+    end
+  end
+
+  def remove_url(url, file)
+    url  = norm_url(url)
+    page = agent.get(REMOVAL % [url, url + file])
+    page = agent.submit page.form
+  end
+
+  private
+  def agent
+    @agent ||= Mechanize.new
+  end
+
+  def norm_url(url)
+    schema, host = url.scan(/^(https?:\/\/)?(.+?)\/?$/).flatten
+    CGI::escape "#{schema || 'http://'}#{host}/"
+  end
+end

data/webmaster_tools.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "webmaster_tools"
+  s.version     = File.read("VERSION").to_s.strip
+  s.authors     = ["Tobias Bielohlawek"]
+  s.email       = ["tobi@soundcloud.com"]
+  s.homepage    = "http://github.com/rngtng/webmaster_tools"
+  s.summary     = %q{Gives access to Webmaster Tools Interface data programmatically which is not provided by the official API}
+  s.description = %q{Webmaster Tools extends the official API to give programmatically access to various crawl information and functions which are available via the Interface}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  %w(mechanize).each do |gem|
+    s.add_runtime_dependency *gem.split(' ')
+  end
+
+  %w(rake rspec).each do |gem|
+    s.add_development_dependency *gem.split(' ')
+  end
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: webmaster_tools
+version: !ruby/object:Gem::Version
+  version: 0.1.0.rc1
+  prerelease: 6
+platform: ruby
+authors:
+- Tobias Bielohlawek
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-05 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: &70232871533880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70232871533880
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70232871533360 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70232871533360
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70232871532900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70232871532900
+description: Webmaster Tools extends the official API to give programmatically access
+  to various crawl information and functions which are available via the Interface
+email:
+- tobi@soundcloud.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- CHANGELOG.md
+- Gemfile
+- LICENCE
+- README.md
+- Rakefile
+- VERSION
+- lib/webmaster_tools.rb
+- webmaster_tools.gemspec
+homepage: http://github.com/rngtng/webmaster_tools
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Gives access to Webmaster Tools Interface data programmatically which is
+  not provided by the official API
+test_files: []