mediawiki_robot 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in mediawiki_robot.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/lib/mediawiki_robot.rb

@@ -0,0 +1,6 @@
+require 'mediawiki_robot/enhanced_gateway'
+require 'mediawiki_robot/robot'
+
+module MediawikiRobot
+
+end

data/lib/mediawiki_robot/enhanced_gateway.rb

@@ -0,0 +1,121 @@
+#!/usr/bin/env ruby
+
+require 'media_wiki'
+require 'media_wiki/gateway'
+require 'media_wiki/config'
+
+module MediawikiRobot
+
+  class EnhancedGateway < MediaWiki::Gateway
+
+    # overrides the usual 'login' to provide retries upon timeout
+    def login(user, pass)
+      done = false
+      while not done
+        begin
+          super(user, pass)
+          done = true
+        rescue SocketError
+          puts "MediaWiki::Gateway::login -- caught SocketError, retrying..."
+        rescue Errno::ETIMEDOUT
+          puts "MediaWiki::Gateway::login -- caught Errno::ETIMEDOUT, retrying..."
+        #rescue MediaWiki::Exception
+        #  puts "MediaWiki::Gateway::login -- caught MediaWiki::Exception, retrying..."
+        end
+      end
+    end
+
+    def make_api_request_with_retry(form_data)
+      res = nil
+      while res.nil?
+        begin
+          res = make_api_request(form_data)
+        rescue SocketError
+          puts "MediaWiki::Gateway::make_api_request_with_retry -- caught SocketError, retrying..."
+        rescue Errno::ETIMEDOUT
+          puts "MediaWiki::Gateway::make_api_request_with_retry -- caught Errno::ETIMEDOUT, retrying..."
+        rescue MediaWiki::Exception
+          puts "MediaWiki::Gateway::make_api_request_with_retry -- caught MediaWiki::Exception, retrying..."
+        end
+      end
+
+      return res
+    end
+
+    def get_with_retry(title)
+      res = nil
+      while res.nil?
+        begin
+          res = get(title)
+        rescue SocketError
+          puts "MediaWiki::Gateway::get_with_retry -- caught SocketError, retrying..."
+        rescue Errno::ETIMEDOUT
+          puts "MediaWiki::Gateway::get_with_retry -- caught Errno::ETIMEDOUT, retrying..."
+        rescue MediaWiki::Exception
+          puts "MediaWiki::Gateway::get_with_retry -- caught MediaWiki::Exception, retrying..."
+        end
+      end
+
+      return res
+    end
+   
+    def recent_changes(num_changes, end_time)
+      form_data =
+        {'action' => 'query',
+        'list' => 'recentchanges'}
+      form_data['rclimit'] = num_changes if !num_changes.nil?
+      form_data['rcend'] = end_time.strftime("%Y%m%d%H%M%S") if !end_time.nil?
+
+      res = make_api_request_with_retry(form_data)
+
+      changes = REXML::XPath.match(res, "//rc").map { |x| { :type        => x.attributes["type"], 
+                                                            :title       => x.attributes["title"], 
+                                                            :timestamp   => x.attributes["timestamp"], 
+                                                            :revision_id => x.attributes["revid"] } }
+      return changes
+    end
+
+    def get_all_pages_in_category(category_title) # e.g., 'Category:Articles_with_hCards'
+      form_data =
+        {'action' => 'query',
+        'list'    => 'categorymembers',
+        'cmtitle' => category_title,
+        'cmlimit' => '5000'}
+
+      res = make_api_request_with_retry(form_data)
+
+      titles = REXML::XPath.match(res, "//cm").map { |x| x.attributes["title"] }
+      return titles
+    end
+
+    def get_page_revision(title)
+      form_data =
+        {'action' => 'query',
+        'titles'  => title,
+        'prop'    => 'revisions'}
+
+      res = make_api_request_with_retry(form_data)
+
+      rev_ids = REXML::XPath.match(res, "////rev").map { |x| x.attributes["revid"] }
+      return rev_ids[0]
+    end
+   
+    def exists(page_title)
+      # used to be:
+      #    return ! get_with_retry(page_title).nil?
+      # but that was slower.
+
+      form_data =
+        {'action' => 'query',
+        'titles'  => page_title,
+        'prop'    => 'revisions'}
+
+      res = make_api_request_with_retry(form_data)
+
+      rev_ids = REXML::XPath.match(res, "////rev").map { |x| x.attributes["revid"] }
+      return !( rev_ids.nil? or rev_ids.empty? )
+    end
+
+  end
+
+end

data/lib/mediawiki_robot/robot.rb

@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+require 'thread'
+require 'time'
+require 'mediawiki_robot/enhanced_gateway'
+
+module MediawikiRobot
+
+  class Robot
+
+    attr_accessor :mw
+
+    API_SUFFIX = '/api.php'
+    MAX_SIMULTANEOUS_THREADS = 10
+
+    def initialize(mw_opts)
+      api_url = mw_opts[:base_url] + mw_opts[:normal_prefix] + API_SUFFIX
+      @mw = MediawikiRobot::EnhancedGateway.new(api_url, {:ignorewarnings=>1})
+
+      @thread = nil
+
+      @mainloop_sleep_secs = 5.0 # make this non-constant so that it can be overridden for testing
+    end
+
+    def start
+      @thread = Thread.new { main_loop }
+    end
+
+    def stop
+      @thread.kill
+    end
+
+    def is_running
+      return false if @thread.nil?
+      return true unless @thread.status.nil? or @thread.status == false
+      return false
+    end
+ 
+  private
+
+    def handle_single_change(change)
+      raise "not implemented"
+    end
+ 
+    def handle_changes(changes)
+
+      while not changes.empty?
+
+        # spin up a bunch of threads to pull down these batches of changes in parallel
+        threads = []
+        for i in 1..MAX_SIMULTANEOUS_THREADS do
+          if not changes.empty?
+
+            threads << Thread.new(changes.shift) do |cur_change| 
+              handle_single_change(cur_change)
+            end
+
+          end
+        end
+        threads.each { |aThread|  aThread.join }
+
+      end
+
+    end
+
+    def main_loop
+
+      # find the timestamp of the first change.  We only want NEW changes
+      num_recent_changes = 500
+      prev_time          = nil
+      changes = @mw.recent_changes(num_recent_changes, prev_time)
+
+      if !changes.nil? and !changes.empty?
+        timestamp_of_first_change = changes[0][:timestamp].gsub(/[-T:]/, ' ').gsub(/Z/,'')
+        prev_time = Time.strptime(timestamp_of_first_change, "%Y %m %d %H %M %S")
+        prev_time = prev_time + 1 # move 1 sec past the last change
+      end
+
+      while true
+
+        changes = @mw.recent_changes(num_recent_changes, prev_time)
+
+        if !changes.nil? and !changes.empty?
+          timestamp_of_first_change = changes[0][:timestamp].gsub(/[-T:]/, ' ').gsub(/Z/,'')
+          prev_time = Time.strptime(timestamp_of_first_change, "%Y %m %d %H %M %S")
+          prev_time = prev_time + 1 # move 1 sec past the last change
+
+          handle_changes(changes)
+        end
+ 
+        sleep @mainloop_sleep_secs
+      end
+    end
+    
+  end
+
+end

data/lib/mediawiki_robot/version.rb

@@ -0,0 +1,3 @@
+module MediawikiRobot
+  VERSION = "0.0.1"
+end

data/mediawiki_robot.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "mediawiki_robot/version"
+
+Gem::Specification.new do |s|
+  s.name        = "mediawiki_robot"
+  s.version     = MediawikiRobot::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Jim Lindstrom"]
+  s.email       = ["jim.lindstrom@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{MediaWiki robot framework}
+  s.description = %q{Gem for creating robots that monitor/maintain MediaWiki-based wikis.}
+
+  s.rubyforge_project = "mediawiki_robot"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'mediawiki-gateway'
+  s.add_dependency 'active_support'
+end

data/specs/mediawiki_robot_enhanced_gateway_spec.rb

@@ -0,0 +1,139 @@
+# mediawiki_enhanced_gateway_spec.rb
+
+require 'rubygems'
+require 'mediawiki_robot/enhanced_gateway'
+
+DO_MESSY_TESTS = false # These tests muck with the 'recent_changes' list and so I try to minimize them
+
+API_SUFFIX     = '/api.php'
+
+def rand_alphanumeric_str(len)
+  o =  [('a'..'z'),('A'..'Z'),('0'..'9')].map{|i| i.to_a}.flatten;  
+  (0..len).map{ o[rand(o.length)]  }.join;
+end
+
+describe MediawikiRobot::EnhancedGateway do
+   
+  before(:each) do
+
+    @mw_opts = 
+      {:base_url      => 'http://jimlindstrom.com',
+      :normal_prefix  => '/mediawiki',
+      :special_prefix => '/mediawiki'}
+    @robot_acct =
+      {:user => "robot",
+      :pass  => "robotpass"}
+
+    api_url = @mw_opts[:base_url] + @mw_opts[:normal_prefix] + API_SUFFIX
+    @gateway = MediawikiRobot::EnhancedGateway.new(api_url, {:ignorewarnings=>1})
+
+    @bogus_user = rand_alphanumeric_str(10)
+    @bogus_password = rand_alphanumeric_str(10)
+
+    @nonexistant_page = rand_alphanumeric_str(20)
+    @existing_page = "Main_Page"
+
+    @existing_category = "Category:Wikipedia_protected_templates"
+    @page_in_existing_category = "Template:Navbox"
+  end
+
+  describe "#login" do
+    it "throws MediaWiki::Unauthorized if bad password" do
+      lambda {
+        @gateway.login(@robot_acct[:user], @bogus_password)
+      }.should raise_error(MediaWiki::Unauthorized)
+    end
+  
+    it "throws MediaWiki::Unauthorized if bad username" do
+      lambda {
+        @gateway.login(@bogus_user, @robot_acct[:pass])
+      }.should raise_error(MediaWiki::Unauthorized)
+    end
+  
+    it "doesn't throw anything if good credentials" do
+      lambda {
+        @gateway.login(@robot_acct[:user], @robot_acct[:pass])
+      }.should_not raise_error(MediaWiki::Unauthorized)
+    end
+  end
+
+  describe "#exists" do
+    it "returns false if page does not exist" do
+      @gateway.exists(@nonexistant_page).should == false
+    end
+  
+    it "returns true if page exists" do
+      if DO_MESSY_TESTS
+        @gateway.login(user,pass)
+        @gateway.exists(@nonexistant_page).should == false
+        @gateway.create(@nonexistant_page, "Testing page creation")
+        @gateway.exists(@nonexistant_page).should == true
+        @gateway.delete(@nonexistant_page)
+        @gateway.exists(@nonexistant_page).should == false
+      else
+        @gateway.exists(@existing_page).should == true 
+      end
+    end
+  end
+  
+  describe "#recent_changes" do
+    it "returns the requested number of items" do
+      pages = @gateway.recent_changes(50,nil).length.should == 50
+    end
+
+    it "returns no items if no edits since the start time" do
+      t = Time.new
+      starttime = t.getgm
+      pages = @gateway.recent_changes(nil,starttime).length.should == 0 
+    end
+
+    it "returns the items edited since the start time" do
+      if DO_MESSY_TESTS
+        t = Time.new
+        starttime = t.getgm
+        @gateway.login(@robot_acct[:user], @robot_acct[:pass])
+        @gateway.create(@nonexistant_page, "Testing page creation")
+        @gateway.delete(@nonexistant_page)
+        pages = @gateway.recent_changes(nil,starttime).length.should >= 1 
+      else
+        t = Time.gm(2010, 7, 8, 9, 10, 11)
+        starttime = t.getgm
+        @gateway.recent_changes(nil,starttime).length.should >= 1 
+      end
+    end
+
+    it "returns a list of items that are hashes containing 'type', 'title', 'timestamp', and 'revision_id' keys" do
+      if DO_MESSY_TESTS
+        t = Time.new
+        starttime = t.getgm
+        @gateway.login(@robot_acct[:user], @robot_acct[:pass])
+        @gateway.create(@nonexistant_page, "Testing page creation")
+        @gateway.recent_changes(nil,starttime).should == [ { :type => "new", :title => @nonexistant_page } ] # FIXME: needs to take into account revision_id
+        @gateway.delete(@nonexistant_page)
+      else
+        @gateway.login(@robot_acct[:user], @robot_acct[:pass])
+        @gateway.recent_changes(10,nil)[0].keys.should == [:type, :title, :timestamp, :revision_id]
+      end
+    end
+  end
+  
+  describe "#get" do
+    it "returns nil if doesn't exist" do
+      @gateway.get(@nonexistant_page).nil?.should == true
+    end
+    it "returns a String object" do
+      @gateway.get(@existing_page).class.should == String
+    end
+  end
+   
+  describe "#get_all_pages_in_category" do
+    it "returns a list" do
+      @gateway.get_all_pages_in_category(@existing_category).class.should == Array
+    end
+    it "returns a list of all the person pages on the mediawiki" do
+      @gateway.get_all_pages_in_category(@existing_category).index(@page_in_existing_category).nil?.should == false
+      @gateway.get_all_pages_in_category(@existing_category).length.should > 1
+    end
+  end
+  
+end

data/specs/mediawiki_robot_robot_spec.rb

@@ -0,0 +1,109 @@
+# mediawiki_robot_spec.rb
+
+require 'rubygems'
+require 'mediawiki_robot'
+
+def wait_on_condition(max_retries, wait_time, condition)
+
+  retries = max_retries
+  while retries > 0 and condition.call==false do
+    sleep wait_time
+    retries -= 1
+  end
+
+end
+
+def rand_alphanumeric_str(len)
+  o =  [('a'..'z'),('A'..'Z'),('0'..'9')].map{|i| i.to_a}.flatten;  
+  (0..len).map{ o[rand(o.length)]  }.join;
+end
+
+class RobotWithTestHarness < MediawikiRobot::Robot
+
+  def initialize(mw_opts)
+    super(mw_opts)
+
+    @last_change = nil
+    @mainloop_sleep_secs = 0.1
+  end
+
+  def handle_single_change(change)
+    @last_change = change
+  end
+
+  def get_last_change
+    return @last_change
+  end
+
+end
+
+describe MediawikiRobot::Robot do
+  
+  before(:each) do
+
+    @mw_opts = 
+      {:base_url      => 'http://jimlindstrom.com',
+      :normal_prefix  => '/mediawiki',
+      :special_prefix => '/mediawiki'}
+    @robot_acct =
+      {:user => "robot",
+      :pass  => "robotpass"}
+
+    @robot = RobotWithTestHarness.new(@mw_opts)
+  end
+ 
+  describe "#start" do
+    it "starts the main loop" do
+      @robot.start
+
+      wait_on_condition(10, 0.1, lambda { @robot.is_running } )
+
+      @robot.is_running.should == true
+      @robot.stop
+    end
+
+    it "doesn't call 'handle_single_change' unless a page on the mediawiki is changed" do
+      @robot.start
+      @robot.mw.login(@robot_acct[:user], @robot_acct[:pass])
+
+      last_change = @robot.get_last_change
+      last_change.nil?.should == true
+
+      @robot.stop
+    end
+
+    it "calls 'handle_single_change' when a page on the mediawiki is changed" do
+      @robot.start
+      @robot.mw.login(@robot_acct[:user], @robot_acct[:pass])
+
+      rand_page = rand_alphanumeric_str(20)
+      @robot.mw.create(rand_page, "Testing robot functionality")
+
+      sleep 0.5
+
+      last_change = @robot.get_last_change
+      last_change.nil?.should == false
+      last_change.delete(:revision_id)
+      last_change.delete(:timestamp)
+      last_change.should == {:type=>"new", :title=>rand_page }
+    
+      @robot.mw.delete(rand_page)
+
+      @robot.stop
+    end
+  end
+ 
+  describe "#stop" do
+    it "stops the main loop" do
+      @robot.start
+      sleep 0.1
+      @robot.stop
+
+      wait_on_condition(10, 0.1, lambda { !@robot.is_running } )
+
+      @robot.is_running.should == false
+    end
+  end
+ 
+end
+

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: mediawiki_robot
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jim Lindstrom
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-06-14 00:00:00.000000000 -04:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mediawiki-gateway
+  requirement: &79825670 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *79825670
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: &79825460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *79825460
+description: Gem for creating robots that monitor/maintain MediaWiki-based wikis.
+email:
+- jim.lindstrom@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- lib/mediawiki_robot.rb
+- lib/mediawiki_robot/enhanced_gateway.rb
+- lib/mediawiki_robot/robot.rb
+- lib/mediawiki_robot/version.rb
+- mediawiki_robot.gemspec
+- specs/mediawiki_robot_enhanced_gateway_spec.rb
+- specs/mediawiki_robot_robot_spec.rb
+has_rdoc: true
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: mediawiki_robot
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: MediaWiki robot framework
+test_files: []