ruby-mediawiki 0.1

data/README

@@ -0,0 +1,60 @@
+Ruby-MediaWiki 0.1
+------------------
+------------------
+
+A library to retrieve and modify content managed by the popular
+MediaWiki software.
+
+
+AUTHORS
+=======
+
+Sven Klemm
+Stephan Maka
+Mike Gerber
+Michael Witrant
+
+(patches go to stephan@spaceboyz.net)
+
+
+LICENSE
+=======
+
+Distributed under the GNU General Public License version 3, see the
+COPYING file.
+
+
+USAGE
+=====
+
+Ruby-MediaWiki relies on a configuration file with URLs, user account
+details and per-agent settings in YAML. The sample mediawikirc.sample
+should be rather self-explanatory.
+
+There are various environment variables for easy usage:
+
+* MEDIAWIKI_RC    - location of the configuration file
+                    (default: ~/.mediawikirc)
+* MEDIAWIKI_WIKI  - which section of the configuration file to use
+                    (default is the one set in the `default' key)
+* MEDIAWIKI_DEBUG - if set, set the log level to DEBUG
+
+Quick start:
+------------
+ruby-mediawiki % export RUBYLIB=`pwd`/lib
+ruby-mediawiki % export MEDIAWIKI_RC=mediawikirc.sample 
+ruby-mediawiki % export MEDIAWIKI_WIKI=wpde
+ruby-mediawiki % ruby apps/wikicat.rb Arschgeweih 
+
+
+RESOURCES
+=========
+
+German description:
+  https://wiki.c3d2.de/Ruby-MediaWiki
+
+RDoc to Wiki:
+  https://wiki.c3d2.de/Ruby-MediaWiki/Documentation
+
+SVN repository:
+  svn://svn.c3d2.de/ruby-mediawiki/trunk

data/apps/comment_sync.rb

@@ -0,0 +1,178 @@
+#!/usr/bin/env ruby
+=begin
+    This file is part of Ruby-MediaWiki.
+
+    Ruby-MediaWiki is free software: you can redistribute it and/or
+    modify it under the terms of the GNU General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    Ruby-MediaWiki is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Ruby-MediaWiki.  If not, see
+    <http://www.gnu.org/licenses/>.
+=end
+
+$:.unshift('../lib')
+require 'postgres'
+require 'yaml'
+require 'mediawiki'
+require 'mediawiki/dotfile'
+require 'mediawiki/table'
+
+class SQL_Parser
+  # returns a hash with the tablename as key
+  def self.parse( sql )
+    tables = {}
+    current = nil
+    sql.each_line do | line |
+      line.gsub!(/,$/, '')
+      next if line.match(/^ *$/) or line.match(/BEGIN/) or line.match(/COMMIT/)
+      if match = line.match(/CREATE TABLE ([a-z_]+) *\(/)
+        current = match[1]
+        tables[match[1]] = {:field_names => [], :field => {}}
+      elsif line.match(/\) *(WITHOUT OIDS)?;/)
+        current = nil
+      elsif current
+        if match = line.match(/^ +([a-z_0-9]+) +(.*)$/)
+          tables[current][:field_names].push( match[1] )
+          tables[current][:field][match[1]] = match[2].gsub(/(NOT NULL|UNIQUE|DEFAULT|CHECK).*/, '')
+        elsif match = line.match(/^ *(FOREIGN KEY|PRIMARY KEY|CHECK|UNIQUE)/)
+        else
+          puts "unparsed line: #{line}"
+        end
+      else
+       puts "unrecognized line: #{line.inspect}"
+      end
+    end
+    tables
+  end
+end
+
+# reads and writes comment to/from the database
+class SQL_Comments
+
+  # setup database connection
+  def initialize( config )
+    @connection = PGconn.connect( config['host'], config['port'], nil, nil, config['database'], config['username'], config['password'])
+  end
+
+  def escape( data )
+    data.gsub("\\",'').gsub("'", "''")
+  end
+
+  def escape_ident( data )
+    data.gsub("\\",'').gsub('"', '')
+  end
+
+  def get_table_description( table )
+    @connection.exec("SELECT description
+                        FROM pg_description
+                             INNER JOIN pg_class ON ( pg_class.oid = pg_description.objoid )
+                       WHERE pg_description.objsubid = 0 AND
+                             pg_class.relname = '#{escape(table)}';"
+                     ).entries.flatten[0]
+  end
+
+  def set_table_description( table, text )
+    @connection.exec("COMMENT ON TABLE \"#{escape_ident(table)}\" IS #{text.to_s == "" ? 'NULL' : "'#{escape(text.strip)}'"};")
+  end
+
+  def set_column_description( table, column, text )
+    @connection.exec("COMMENT ON COLUMN \"#{escape_ident(table)}\".\"#{escape_ident(column)}\" IS #{text.to_s == "" ? 'NULL' : "'#{escape(text.strip)}'"};")
+  end
+
+  def get_column_description( table, column )
+    @connection.exec("SELECT description
+                        FROM pg_description
+                             INNER JOIN pg_class ON ( pg_class.oid = pg_description.objoid )
+                             INNER JOIN pg_attribute ON
+                                ( pg_attribute.attrelid = pg_description.objoid AND
+                                  pg_description.objsubid = pg_attribute.attnum )
+                       WHERE pg_class.relname = '#{escape(table)}' AND
+                             pg_attribute.attname = '#{escape(column)}'"
+                     ).entries.flatten[0]
+  end
+
+end
+
+
+class Comment_Synchronizer
+
+  def initialize( wiki, config, sql)
+    @wiki = MediaWiki.dotfile
+    @sql = SQL_Comments.new(config)
+    @tables = SQL_Parser.parse( sql )
+  end
+
+  # update wiki with content from database
+  def update_wiki
+    # generate index page
+    page = @wiki.article('Database/Tables')
+    page.text = "This is a list of all tables used in pentabarf\n\n"
+    @tables.keys.sort.each do | table |
+      page.text += "*[[Database/Tables/#{table.capitalize}|#{table}]]\n"
+    end
+    page.text += "[[Category:Database]]"
+    page.submit('generating list of tables')
+
+    # create one page for each table
+    @tables.keys.sort.each do | table |
+      page = @wiki.article("Database/Tables/#{table.capitalize}")
+
+      # create wiki page
+      page.text = @sql.get_table_description( table ).to_s + "\n"
+      # table header
+      page.text += "==Columns==\n"
+      page.text += "[[Database/Tables|<< List of Tables]]\n"
+      t = MediaWiki::Table.new
+      t.style = 'border="1" cellspacing="0" cellpadding="3" style="border-collapse: collapse;"'
+      t.header_style = 'bgcolor="lightblue"'
+      t.header = ['field name', 'datatype', 'description']
+
+      @tables[table][:field_names].each do | field_name |
+        t.data.push([field_name, @tables[table][:field][field_name], @sql.get_column_description( table, field_name)])
+      end
+      page.text += t.text
+      page.text += "\n[[Category:Database]]\n"
+      page.submit("updating page for table #{table}")
+
+    end
+  end
+
+  # write documentation from wiki into database
+  def update_database
+    # fetch content from all wiki pages and put it in the database
+    @tables.keys.sort.each do | table |
+
+      # get the section containing the general description
+      page = @wiki.article("Database/Tables/#{table.capitalize}", 0)
+      @sql.set_table_description( table, page.text )
+
+      # get the section containing the column table
+      page = @wiki.article("Database/Tables/#{table.capitalize}", 1)
+      state = :description
+      column = ''
+      # parse wiki text
+      MediaWiki::Table.parse( page.text ).each do | column |
+        next if column[0] == 'field name' # this is the header column
+        @sql.set_column_description( table, column[0], column[2] )
+      end
+    end
+  end
+
+
+end
+
+  db_config = YAML.load_file('db_config.yml')['development']
+  #db_config = {'host' => 'localhost', 'port' => 5432, 'database' => 'pentabarf', 'username' => 'joe', 'password' => 'secret'}
+  sql = `svnlook cat /var/subversion/pentabarf /trunk/sql/tables.sql`
+
+  bot = Comment_Synchronizer.new( :pentabarf, db_config, sql )
+  bot.update_database
+  bot.update_wiki
+

data/apps/date_determinator.rb

@@ -0,0 +1,185 @@
+#!/usr/bin/env ruby
+=begin
+    This file is part of Ruby-MediaWiki.
+
+    Ruby-MediaWiki is free software: you can redistribute it and/or
+    modify it under the terms of the GNU General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    Ruby-MediaWiki is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Ruby-MediaWiki.  If not, see
+    <http://www.gnu.org/licenses/>.
+=end
+
+require 'yaml'
+$:.unshift('../lib')
+require 'mediawiki/dotfile'
+
+include MediaWiki
+
+class DateUser
+  attr_reader :name
+
+  def initialize(name, hsh)
+    @name = name
+    @hsh = hsh
+  end
+
+  def dates
+    @hsh.keys
+  end
+
+  def day(d)
+    @hsh[d]
+  end
+
+  def day_class(d)
+    if day(d) == nil
+      nil
+    elsif day(d) =~ /^[yj]/i
+      true
+    elsif day(d) =~ /^n/i
+      false
+    else
+      ""
+    end
+  end
+
+  def day_style(d)
+    case day_class(d)
+      when nil then ""
+      when true then 'bgcolor="#7fff7f"'
+      when false then 'bgcolor="#ff7f7f"'
+      else 'bgcolor="#bfbfbf"'
+    end
+  end
+end
+
+class DateData
+  def initialize(yaml)
+    @users = []
+    @error = nil
+    begin
+      YAML::load(yaml).each { |user,dates|
+        @users << DateUser.new(user, dates)
+      }
+    rescue Exception => e
+      puts "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+      @error = e.to_s
+    end
+  end
+
+  def table
+    return @error if @error
+    
+    s = ''
+
+    ##
+    # Collect dates
+    ##
+    dates = []
+    dates_yes = {}
+    @users.each { |user| dates += user.dates }
+
+    ##
+    # Sort dates
+    ##
+    dates.uniq!
+    dates.sort! { |a,b|
+      if a =~ /^(\d+)\.(\d+)\./
+        a_day = $1
+        a_month = $2
+        if b =~ /^(\d+)\.(\d+)\./
+          b_day = $1
+          b_month = $2
+          if a_month.to_i == b_month.to_i
+            a_day.to_i <=> b_day.to_i
+          else
+            a_month.to_i <=> b_month.to_i
+          end
+        else
+          a <=> b
+        end
+      else
+        a <=> b
+      end
+    }
+
+    ##
+    # Construct header
+    ##
+    s += "{| border=\"1\" cellpadding=\"2\" cellspacing=\"0\" style=\"border-collapse:collapse;\"\n|-\n! \n"
+    dates.each { |date|
+      s += "!#{date}\n"
+      dates_yes[date] = 0
+    }
+
+    ##
+    # Construct rows
+    ##
+    @users.each { |user|
+      s += "|-\n|[[User:#{user.name}|#{user.name}]]\n"
+      dates.each { |date|
+        s += "| #{user.day_style(date)} | #{user.day(date)}\n"
+        dates_yes[date] += 1 if user.day_class(date) == true
+      }
+    }
+
+    ##
+    # Build summary
+    ##
+    s += "|-\n|'''sum(ja)'''\n"
+    dates.each { |date|
+      s += "|'''#{dates_yes[date]}'''\n"
+    }
+    s += "|}"
+  end
+end
+
+
+
+wiki, conf = MediaWiki.dotfile('date determinator')
+conf['pages'].each { |name|
+  page = wiki.article(name)
+
+  datasets = {}
+  current_data_name = nil
+  current_data_yaml = ''
+  page.text.split(/\n/).each { |line|
+    if line =~ /BEGIN DATA "(.+?)"/
+      current_data_name = $1
+      current_data_yaml = ''
+    elsif line =~ /END DATA/ and current_data_name
+      datasets[current_data_name] = DateData.new(current_data_yaml)
+      current_data_name = nil
+      current_data_yaml = ''
+    elsif current_data_name
+      current_data_yaml += "#{line}\n"
+    end
+  }
+
+
+  text_old = page.text.dup
+
+  signature = /(<!-- BEGIN TABLE ")(.+?)(" -->)(.+?)(<!-- END TABLE -->)/m
+  page.text.gsub!(signature) { |part|
+    begin1,name,begin2,obsolete,end1 = part.match(signature).to_a[1..-1]
+    table = datasets[name] ? datasets[name].table : "DATA #{name} not found!"
+    p table
+    "#{begin1}#{name}#{begin2}\n#{table}\n#{end1}"
+  }
+  puts page.text
+
+  if page.text != text_old
+    puts "submitting"
+    page.submit('Date Determinator run')
+  end
+
+  puts "done"
+}

data/apps/iso_639_leecher.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+=begin
+    This file is part of Ruby-MediaWiki.
+
+    Ruby-MediaWiki is free software: you can redistribute it and/or
+    modify it under the terms of the GNU General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    Ruby-MediaWiki is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Ruby-MediaWiki.  If not, see
+    <http://www.gnu.org/licenses/>.
+=end
+
+$:.unshift('../lib')
+
+require 'mediawiki/dotfile'
+require 'mediawiki/table'
+require 'momomoto/momomoto'
+require 'momomoto/tables'
+require 'momomoto/bot_login'
+
+  ENV['MEDIAWIKI_WIKI'] = 'wikipedia_de'
+
+  db_config = YAML.load_file('db_config.yml')['development']
+  Momomoto::Base.connect(db_config)
+  Momomoto::Bot_login.authorize('ui_tagger')
+
+  wiki = MediaWiki.dotfile
+  page = wiki.article('ISO 639', 3)
+  t = MediaWiki::Table.parse( page.text )
+
+  t.shift
+
+  t.each do | row |
+    row[2].split( '/' ).each do | iso_code |
+      id = Momomoto::Language.find({:iso_639_code=>iso_code})
+      next unless id.length == 1
+      match = row[0].match /\[\[(.*\|)?(.+)\]\]/
+      name = match[2]
+
+      local = Momomoto::Language_localized.find({:language_id=>id.language_id,:translated_id=>144})
+      if local.length == 0 && name != ''
+        local.create
+        local.language_id = id.language_id
+        local.translated_id = 144
+        local.name = name
+        local.write
+      end
+
+    end
+  end
+