wikiavro 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c64ef5f767535d9432d8f589411f76dedb6a4b04
+  data.tar.gz: b846af138fe797fa09e9f248bcdb3298e988b5fb
+SHA512:
+  metadata.gz: b5f7cba59679bc913ceaddf95e97f8ed7c63d9f54660d81680db419da09abf2c61ddadd2f2a75b0e199fbfe9f710363edcd889550f76f44acdd6a34e591f69a2
+  data.tar.gz: 8b63a6ae13ce31cc31eb2f5698a0bd586c9645dfbce1be6ae52e45f90315af3015ed1e1f17badaa4f06425483369a369af6adefd0dae2c8ef0555d8f05193f01

data/bin/wikiavro

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+
+require 'wikiavro'
+require 'xml'
+require 'optparse'
+
+Version = WikiAvro::Version
+
+logger = WikiAvro::MediaWiki::RevisionProgress.new 10000
+overwrite = false
+deflate = false
+ns = nil
+page = nil
+rev = nil
+lqt = nil
+
+opt_parser = OptionParser.new do |opts|
+  opts.banner = 'Usage: wikiavro [options] [INFILE]'
+
+  opts.on('-q', '--quiet', "Don't report progress") do |v|
+    logger = WikiAvro::MediaWiki::NoProgress.new
+  end
+
+  opts.on('-v LEVEL', '--verbose=LEVEL', Integer,
+          'Report progress after every LEVEL revisions parsed') do |lvl|
+    logger = WikiAvro::MediaWiki::RevisionProgress.new lvl.to_i
+  end
+
+  opts.on('-o', '--overwrite') do |o|
+    overwrite = o
+  end
+
+  opts.on('-d', '--deflate', 'Enable Avro internal compression') do |d|
+    deflate = d
+  end
+
+  opts.on('-n OUTFILE', '--namespaces=OUTFILE') do |path|
+    ns = path
+  end
+
+  opts.on('-p OUTFILE', '--pages=OUTFILE') do |path|
+    page = path
+  end
+
+  opts.on('-r OUTFILE', '--revisions=OUTFILE') do |path|
+    rev = path
+  end
+
+  opts.on('-l OUTFILE', '--liquidthreads=OUTFILE') do |path|
+    lqt = path
+  end
+end
+
+opt_parser.parse!
+
+abort('You must specify --namespaces') if ns.nil?
+abort('You must specify --pages') if page.nil?
+abort('You must specify --revisions') if rev.nil?
+abort('You must specify --liquidthreads') if lqt.nil?
+
+if !overwrite
+  [ns, page, rev, lqt].each do |path|
+    if File.exists? path
+      abort("#{path} already exists! pass --overwrite to proceed anyway")
+    end
+  end
+end
+
+ns = WikiAvro::Avro::NamespaceWriter.new ns, deflate
+page = WikiAvro::Avro::PageWriter.new page, deflate
+rev = WikiAvro::Avro::RevisionWriter.new rev, deflate
+lqt = WikiAvro::Avro::LqtWriter.new lqt, deflate
+
+writer = WikiAvro::MediaWiki::WikiWriter.new :logger => logger,
+                                             :namespace => ns,
+                                             :page => page,
+                                             :revision => rev,
+                                             :lqt => lqt
+
+mw = WikiAvro::MediaWiki::WikiDump.new
+
+xml = XML::Reader.io(ARGF)
+xml.read
+
+mw.parse(writer, nil, xml)
+
+ns.close
+page.close
+rev.close
+lqt.close

data/lib/wikiavro.rb

@@ -0,0 +1,6 @@
+module WikiAvro
+  Version = '0.0.1'
+end
+
+require 'wikiavro/mediawiki'
+require 'wikiavro/avro'

data/lib/wikiavro/avro.rb

@@ -0,0 +1,199 @@
+require 'avro'
+
+module WikiAvro::Avro
+  NAMESPACE_SCHEMA = <<-EOS
+{
+  "namespace": "org.rationalwiki",
+  "name": "Namespace",
+  "type": "record",
+  "fields": [
+    {"name": "key", "type": "int"},
+    {"name": "case", "type": "string"},
+    {"name": "name", "type": "string"}
+  ]
+}
+EOS
+
+  PAGE_SCHEMA = <<-EOS
+{
+  "namespace": "org.rationalwiki",
+  "name": "Page",
+  "type": "record",
+  "fields": [
+    {"name": "id", "type": "long"},
+    {"name": "ns", "type": "long"},
+    {"name": "title", "type": "string"},
+    {"name": "redirect", "type": ["null", "string"]},
+    {"name": "sha1", "type": ["null", "string"]}
+  ]
+}
+EOS
+
+  REVISION_SCHEMA = <<-EOS
+{
+  "namespace": "org.rationalwiki",
+  "name": "Revision",
+  "type": "record",
+  "fields": [
+    {"name": "id", "type": "long"},
+    {"name": "page_id", "type": "long"},
+    {"name": "n", "type": "long"},
+    {"name": "timestamp", "type": "string"},
+    {"name": "contributor", "type": ["null", {
+      "namespace": "org.rationalwiki",
+      "name": "Contributor",
+      "type": "record",
+      "fields": [
+	{"name": "id", "type": ["null", "long"]},
+	{"name": "username", "type": ["null", "string"]},
+	{"name": "ip", "type": ["null", "string"]}
+      ]
+    }]},
+    {"name": "minor", "type": "boolean"},
+    {"name": "comment", "type": ["null", "string"]},
+    {"name": "bytes", "type": "long"},
+    {"name": "textid", "type": ["null", "string"]},
+    {"name": "text", "type": ["null", "string"]}
+  ]
+}
+EOS
+
+  LQT_SCHEMA = <<-EOS
+{
+  "namespace": "org.rationalwiki",
+  "name": "Threading",
+  "type": "record",
+  "fields": [
+    {"name": "subject", "type": "string"},
+    {"name": "parent", "type": ["null", "long"]},
+    {"name": "ancestor", "type": ["null", "long"]},
+    {"name": "page", "type": "string"},
+    {"name": "id", "type": "long"},
+    {"name": "summary_page", "type": ["null", "string"]},
+    {"name": "author", "type": "string"},
+    {"name": "edit_status", "type": "string"},
+    {"name": "type", "type": "string"},
+    {"name": "signature", "type": ["null", "string"]}
+  ]
+}
+EOS
+
+  class AvroWriter
+    def schema
+      raise NotImplementedError
+    end
+
+    def close
+      @writer.close
+    end
+
+    protected
+
+    def encode(data)
+      @writer << data
+    end
+
+    def initialize(path, deflate=false)
+      if !deflate
+        @writer = Avro::DataFile.open(path, 'w', schema)
+      else
+        @writer = Avro::DataFile.open(path, 'w', schema, 'deflate')
+      end
+    end
+  end
+
+  class NamespaceWriter < AvroWriter
+    def schema
+      NAMESPACE_SCHEMA
+    end
+
+    def write(key, casetype, name)
+      encode 'key' => key.to_i,
+             'case' => casetype,
+             'name' => name || ''
+    end
+  end
+
+  class PageWriter < AvroWriter
+    def schema
+      PAGE_SCHEMA
+    end
+
+    def write(ns, id, title, redirect, sha1)
+      encode 'id' => id.to_i,
+             'ns' => ns.to_i,
+             'title' => title,
+             'redirect' => redirect,
+             'sha1' => sha1
+    end
+  end
+
+  class RevisionWriter < AvroWriter
+    def schema
+      REVISION_SCHEMA
+    end
+
+    def write(id, page_id, n, timestamp, contributor, minor,
+              comment, text_deleted, bytes, textid, text)
+      if !contributor[:deleted].nil? && !(contributor[:id].nil? &&
+                                          contributor[:username].nil? &&
+                                          contributor[:ip].nil?)
+        raise 'deleted contributor has content'
+      end
+
+      if contributor[:deleted].nil?
+        contributor.delete :deleted
+        contributor = {
+          'username' => contributor[:username],
+          'id' => contributor[:id].to_i,
+          'ip' => contributor[:ip]
+        }
+      else
+        contributor = nil
+      end
+
+      if comment[:deleted].nil?
+        comment = comment[:comment]
+      else
+        raise 'deleted comment has content' if comment[:comment]
+        comment = nil
+      end
+
+      text = nil if !text_deleted.nil?
+
+      encode 'id' => id.to_i,
+             'page_id' => page_id.to_i,
+             'n' => n.to_i,
+             'timestamp' => timestamp,
+             'contributor' => contributor,
+             'minor' => minor.nil?,
+             'comment' => comment,
+             'bytes' => bytes.to_i,
+             'textid' => textid,
+             'text' => text
+    end
+  end
+
+  class LqtWriter < AvroWriter
+    def schema
+      LQT_SCHEMA
+    end
+
+    def write(subject, parent, ancestor, page, id, summary_page,
+              author, edit_status, type, signature)
+      parent = parent.to_i if parent
+      ancestor = ancestor.to_i if ancestor
+
+      encode 'subject' => subject,
+             'parent' => parent,
+             'ancestor' => ancestor,
+             'page' => page,
+             'id' => id.to_i,
+             'summary_page' => summary_page,
+             'author' => author,
+             'edit_status' => edit_status,
+             'type' => type,
+             'signature' => signature
+    end
+  end
+end

data/lib/wikiavro/mediawiki.rb

@@ -0,0 +1,671 @@
+require 'wikiavro/xml'
+
+# RW declares schema 0.6 but has redirects as in 0.7
+#
+# RW has <sha1> tags right after <redirect> - they are indented to a
+# different level too. The <sha1>s are missing within <revision>s
+# where they should be.
+#
+# Schema claims discussionthreadinginfo, but actual tag is
+# DiscussionThreading. Schema does not describe ThreadSummaryPage or
+# ThreadSignature. Schema does not describe which LQT tags are
+# omissible. Schema says thread info should always come after
+# revisions, but it does not.
+
+module WikiAvro::MediaWiki
+  class NamespacePrinter
+    def write(key, casetype, name)
+      puts "namespace #{key}: \"#{name}\" #{casetype}"
+    end
+  end
+
+  class PagePrinter
+    def write(ns, id, title, redirect, sha1)
+      puts "page \"#{title}\": #{id} #{ns} #{redirect} #{sha1}"
+    end
+  end
+
+  class RevisionPrinter
+    def write(id, page_id, n, timestamp, contributor, minor,
+              comment, text_deleted, bytes, textid, text)
+      puts "rev #{page_id} #{n}: #{timestamp} " +
+           "#{bytes} #{contributor[:username]}"
+    end
+  end
+
+  class LqtPrinter
+    def write(threadSubject, threadParent, threadAncestor,
+              threadPage, threadID, threadSummaryPage,
+              threadAuthor, threadEditStatus, threadType,
+              threadSignature)
+      puts "thread #{threadSubject} #{threadParent} #{threadAncestor} " +
+           "#{threadAuthor} #{threadEditStatus} #{threadType}"
+    end
+  end
+
+  class NullWriter
+    def method_missing(target, *args, &block)
+      # All methods return nil
+    end
+
+    def initialize
+    end
+  end
+
+  class WikiWriter
+    def initialize(writers)
+      null_writer = NullWriter.new
+      @namespace = (writers[:namespace] or NamespacePrinter.new)
+      @logger = (writers[:logger] or NoProgress.new)
+      @page = (writers[:page] or NullWriter.new)
+      @revision = (writers[:revision] or NullWriter.new)
+      @lqt = (writers[:lqt] or NullWriter.new)
+    end
+
+    def namespace(key, casetype, name)
+      @namespace.write(key, casetype, name)
+    end
+
+    def page(ns, id, title, redirect, sha1)
+      @logger.report_pages(1)
+      @page.write(ns, id, title, redirect, sha1)
+    end
+
+    def revision(id, page_id, n, timestamp, contributor, minor,
+                 comment, text_deleted, bytes, textid, text)
+      @logger.report_revisions(1)
+      @revision.write(id, page_id, n, timestamp, contributor, minor,
+                      comment, text_deleted, bytes, textid, text)
+    end
+
+    def lqt(threadSubject, threadParent, threadAncestor,
+            threadPage, threadID, threadSummaryPage,
+            threadAuthor, threadEditStatus, threadType,
+            threadSignature)
+      @lqt.write(threadSubject, threadParent, threadAncestor,
+                 threadPage, threadID, threadSummaryPage,
+                 threadAuthor, threadEditStatus, threadType,
+                 threadSignature)
+    end
+
+    def done
+      @logger.report_done
+    end
+
+    def skipped(name)
+#      puts "wikiwriter: skipped element #{name}"
+#      raise 'what?'
+      @logger.report_skipped_element(name)
+    end
+  end
+
+  class NoProgress
+    def report_pages(n)
+    end
+
+    def report_revisions(n)
+    end
+
+    def report_done
+    end
+
+    def report_skipped_element(name)
+    end
+  end
+
+  class FinalProgress
+    def f(n)
+      parts = []
+      while n >= 1
+        parts.unshift(n % 1000)
+        n /= 1000
+      end
+      head = parts.shift
+      if parts.empty?
+        if head
+          head.to_s
+        else
+          n
+        end
+      else
+        [head, parts.map {|p| sprintf('%03d', p)}.join(',')].join(',')
+      end
+    end
+
+    def total_skipped
+      @skipped_counts.values.reduce(0, :+)
+    end
+
+    def report_pages(n)
+      @pages += n
+    end
+
+    def report_revisions(n)
+      @revisions += n
+    end
+
+    def report_skipped_element(name)
+      @skipped_counts[name] += 1
+    end
+
+    def show_skipped
+      @skipped_counts.each do |name, count|
+        puts "#{name}: #{count}"
+      end
+    end
+
+    def report_done
+      duration = Time.now - @start_time
+      avg_rate = @revisions / duration
+      h = (duration / 60 / 60).floor
+      m = (duration % (60 * 60) / 60).floor
+      s = (duration % 60).floor
+      # FIXME: Print to STDERR or some log
+      skipped = total_skipped
+      if skipped > 0
+        puts "Couldn't process #{skipped} elements! Detailed breakdown:"
+        show_skipped
+      end
+      puts "Done! Took #{h}h#{m}m#{s}s. Averaged #{f avg_rate.round(0)} rps."
+    end
+
+    def initialize
+      @start_time = Time.now
+      @pages = 0
+      @revisions = 0
+      @skipped_counts = Hash.new 0
+    end
+  end
+
+  class RevisionProgress < FinalProgress
+    def announce_progress
+      now = Time.now
+      rps = (@revisions - @previous_revisions) / (now - @previous_time)
+      puts "Page #{f @pages}, rev #{f @revisions} (#{f rps.round(0)} rps)"
+      skipped = total_skipped
+      puts "#{f skipped} unprocessable elements so far."
+      show_skipped
+      @previous_time = now
+      @previous_revisions = @revisions
+    end
+
+    def report_revisions(n)
+      super(n)
+
+      if @revisions - @previous_revisions >= @interval
+        announce_progress
+      end
+    end
+
+    def report_done
+      announce_progress
+      super
+    end
+
+    def initialize(interval)
+      super()
+      @interval = interval
+      @previous_time = @start_time
+      @previous_revisions = 0
+    end
+  end
+
+  class Namespace < WikiAvro::XML::Leaf
+    def name
+      'namespace'
+    end
+
+    def reset
+      # everything is overwritten each cycle anyway
+    end
+
+    def parse_attributes(w, p, r)
+      @key = r['key']
+      @case = r['case']
+    end
+
+    def parse_content(w, p, r)
+      name = r.read_string
+      WikiAvro::XML.skip_tag(w, r, false)
+      w.namespace(@key, @case, name)
+    end
+  end
+
+  class NamespaceStream < WikiAvro::XML::Stream
+    def initialize
+      super([Namespace.new])
+    end
+  end
+
+  class Sitename < WikiAvro::XML::Inserter
+    def initialize
+      super('sitename')
+    end
+  end
+
+  class Base < WikiAvro::XML::Inserter
+    def initialize
+      super('base')
+    end
+  end
+
+  class Generator < WikiAvro::XML::Inserter
+    def initialize
+      super('generator')
+    end
+  end
+
+  class Case < WikiAvro::XML::Inserter
+    def initialize
+      super('case')
+    end
+  end
+
+  class Namespaces < WikiAvro::XML::Element
+    def name
+      'namespaces'
+    end
+
+    def initialize
+      super([NamespaceStream.new])
+    end
+  end
+
+  class SiteInfo < WikiAvro::XML::Element
+    attr_accessor :sitename
+    attr_accessor :base
+    attr_accessor :generator
+    attr_accessor :case
+
+    def name
+      'siteinfo'
+    end
+
+    def reset
+      @sitename = nil
+      @base = nil
+      @generator = nil
+      @case = nil
+    end
+
+    def initialize
+      super([Sitename.new, Base.new, Generator.new,
+             Case.new, Namespaces.new])
+    end
+  end
+
+  class Title < WikiAvro::XML::Inserter
+    def initialize
+      super('title')
+    end
+  end
+
+  class Ns < WikiAvro::XML::Inserter
+    def initialize
+      super('ns')
+    end
+  end
+
+  class Id < WikiAvro::XML::Inserter
+    def initialize
+      super('id')
+    end
+  end
+
+  class Redirect < WikiAvro::XML::Leaf
+    def name
+      'redirect'
+    end
+
+    def parse_attributes(w, p, r)
+#      puts "redirect: #{r['title']}"
+      p.redirect = r['title']
+    end
+  end
+
+  class Sha1 < WikiAvro::XML::Inserter
+    def initialize
+      super('sha1')
+    end
+  end
+
+  class PageFlags < WikiAvro::XML::Group
+    def initialize
+      super [{:element => Redirect.new, :min => 0, :max => 1},
+             {:element => Sha1.new, :min => 0, :max => 1}]
+    end
+  end
+
+  class Timestamp < WikiAvro::XML::Inserter
+    def initialize
+      super('timestamp')
+    end
+  end
+
+  class Username < WikiAvro::XML::Inserter
+    def initialize
+      super('username')
+    end
+  end
+
+  class Ip < WikiAvro::XML::Inserter
+    def initialize
+      super('ip')
+    end
+  end
+
+  class ContributorGroup < WikiAvro::XML::Group
+    def optional?
+      true
+    end
+
+    def initialize
+      super [{:element => Username.new, :min => 0, :max => 1},
+             {:element => Id.new, :min => 0, :max => 1},
+             {:element => Ip.new, :min => 0, :max => 1}]
+    end
+  end
+
+  class Contributor < WikiAvro::XML::Element
+    def name
+      'contributor'
+    end
+
+    attr_accessor :id
+    attr_accessor :username
+    attr_accessor :ip
+
+    def reset
+      @id = nil
+      @username = nil
+      @ip = nil
+      @deleted = nil
+    end
+
+    def parse_attributes(w, p, r)
+      @deleted = r['deleted']
+    end
+
+    def handle_content(w, p, r)
+      p.contributor = {:deleted => @deleted, :id => id,
+                       :username => username, :ip => ip}
+    end
+
+    def initialize
+      super([ContributorGroup.new])
+    end
+  end
+
+  class Minor < WikiAvro::XML::Inserter
+    def initialize
+      super('minor')
+    end
+  end
+
+  class Comment < WikiAvro::XML::Leaf
+    def name
+      'comment'
+    end
+
+    def parse_attributes(w, p, r)
+      deleted = r['deleted']
+      comment = r.read_string
+      p.comment = {:deleted => deleted,
+                   :comment => comment}
+    end
+  end
+
+  class RevisionFlags < WikiAvro::XML::Group
+    def initialize
+      super [{:element => Minor.new, :min => 0, :max => 1},
+             {:element => Comment.new, :min => 0, :max => 1}]
+    end
+  end
+
+  class Text < WikiAvro::XML::Inserter
+    def parse_attributes(w, p, r)
+      p.text_deleted = r['deleted']
+      p.textid = r['id']
+      p.bytes = r['bytes']
+    end
+
+    def initialize
+      super('text')
+    end
+  end
+
+  class Revision < WikiAvro::XML::Element
+    attr_accessor :id
+    attr_accessor :timestamp
+    attr_accessor :contributor
+    attr_accessor :minor
+    attr_accessor :comment
+    attr_accessor :text_deleted
+    attr_accessor :bytes
+    attr_accessor :textid
+    attr_accessor :text
+
+    def name
+      'revision'
+    end
+
+    def reset
+      id = nil
+      timestamp = nil
+      contributor = nil
+      minor = nil
+      comment = nil
+      text_deleted = nil
+      bytes = nil
+      textid = nil
+      text = nil
+    end
+
+    def handle_content(w, p, r)
+      p.revision_count += 1
+      n = p.revision_count
+      w.revision(id, p.id, n, timestamp, contributor, minor,
+                 comment, text_deleted, bytes, textid, text)
+    end
+
+    def initialize
+      super([Id.new, Timestamp.new, Contributor.new,
+             RevisionFlags.new, Text.new])
+    end
+  end
+
+  class RevStream < WikiAvro::XML::Stream
+    def initialize
+      super([Revision.new])
+    end
+  end
+
+  class ThreadSubject < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadSubject', 'threadSubject')
+    end
+  end
+
+  class ThreadParent < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadParent', 'threadParent')
+    end
+  end
+
+  class ThreadAncestor < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadAncestor', 'threadAncestor')
+    end
+  end
+
+  class ThreadParentGroup < WikiAvro::XML::Group
+    def optional?
+      true
+    end
+
+    def initialize
+      super [{:element => ThreadParent.new, :min => 0, :max => 1},
+             {:element => ThreadAncestor.new, :min => 0, :max => 1}]
+    end
+  end
+
+  class ThreadPage < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadPage', 'threadPage')
+    end
+  end
+
+  class ThreadID < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadID', 'threadID')
+    end
+  end
+
+  class ThreadSummaryPage < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadSummaryPage', 'threadSummaryPage')
+    end
+  end
+
+  class ThreadSummaryPageGroup < WikiAvro::XML::Group
+    def optional?
+      true
+    end
+
+    def initialize
+      super [{:element => ThreadSummaryPage.new, :min => 0, :max => 1}]
+    end
+  end
+
+  class ThreadAuthor < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadAuthor', 'threadAuthor')
+    end
+  end
+
+  class ThreadEditStatus < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadEditStatus', 'threadEditStatus')
+    end
+  end
+
+  class ThreadType < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadType', 'threadType')
+    end
+  end
+
+  class ThreadSignature < WikiAvro::XML::Inserter
+    def initialize
+      super('ThreadSignature', 'threadSignature')
+    end
+  end
+
+  class DiscussionThreading < WikiAvro::XML::Element
+    attr_accessor :threadSubject, :threadParent, :threadAncestor,
+                  :threadPage, :threadID, :threadSummaryPage,
+                  :threadAuthor, :threadEditStatus, :threadType,
+                  :threadSignature
+    def name
+      'DiscussionThreading'
+    end
+
+    def reset
+      threadSubject = nil
+      threadParent = nil
+      threadAncestor = nil
+      threadPage = nil
+      threadID = nil
+      threadSummaryPage = nil
+      threadAuthor = nil
+      threadEditStatus = nil
+      threadType = nil
+      threadSignature = nil
+    end
+
+    def handle_content(w, p, r)
+      w.lqt(threadSubject, threadParent, threadAncestor,
+            threadPage, threadID, threadSummaryPage,
+            threadAuthor, threadEditStatus, threadType,
+            threadSignature)
+    end
+
+    def initialize
+      super([ThreadSubject.new, ThreadParentGroup.new, ThreadPage.new,
+             ThreadID.new, ThreadSummaryPageGroup.new, ThreadAuthor.new,
+             ThreadEditStatus.new, ThreadType.new, ThreadSignature.new])
+    end
+  end
+
+  class DiscussionThreadingGroup < WikiAvro::XML::Group
+    def optional?
+      true
+    end
+
+    def initialize
+      super [{:element => DiscussionThreading.new, :min => 0, :max => 1}]
+    end
+  end
+
+  class Page < WikiAvro::XML::Element
+    attr_accessor :title
+    attr_accessor :ns
+    attr_accessor :id
+    attr_accessor :redirect
+    attr_accessor :sha1
+    attr_accessor :revision_count
+
+    def name
+      'page'
+    end
+
+    def reset
+      title = nil
+      ns = nil
+      id = nil
+      redirect = nil
+      sha1 = nil
+      revision_count = nil
+      @revision_count = 0
+    end
+
+    def handle_content(w, p, r)
+      w.page(ns, id, title, redirect, sha1)
+    end
+
+    def initialize
+      super([Title.new, Ns.new, Id.new, PageFlags.new,
+             RevStream.new, DiscussionThreadingGroup.new,
+             RevStream.new])
+    end
+  end
+
+  class PageStream < WikiAvro::XML::Stream
+    def initialize
+      super([Page.new])
+    end
+  end
+
+  class WikiDump < WikiAvro::XML::Element
+    attr_reader :version
+
+    def name
+      'mediawiki'
+    end
+
+    protected
+
+    def parse_attributes(w, p, r)
+      @version = r['version']
+      warn 'dump version != 0.6' if @version != '0.6'
+    end
+
+    def handle_content(w, p, r)
+      w.done
+    end
+
+    def initialize
+      super([SiteInfo.new, PageStream.new])
+    end
+  end
+end

data/lib/wikiavro/xml.rb

@@ -0,0 +1,282 @@
+# Parser functions will assume to possibly start on their opening tag,
+# and stop parsing right after their end tag. There're probably loads
+# of bugs waiting for when you nest tags of the same name.
+
+module WikiAvro::XML
+  def self.to_tag(reader)
+#    puts 'to_tag: moving to tag'
+    loop do
+      case reader.node_type
+      when XML::Reader::TYPE_ELEMENT
+#        puts "to_tag: got tag #{reader.name}"
+        return true
+      when XML::Reader::TYPE_END_ELEMENT
+#        puts "to_tag: got end tag #{reader.name}"
+        return false
+      end
+
+      break if !reader.read
+    end
+
+    # XML::Reader will probably raise its own exception before we ever
+    # could get here
+    raise EOFError.new('no opening tag')
+  end
+
+  # Do not call this while you are on the opening tag
+  def self.exit_tag(writer, reader, name)
+    nest = 1
+
+#    puts "exit_tag: exiting #{name}"
+
+    loop do
+      case reader.node_type
+      when XML::Reader::TYPE_ELEMENT
+#        puts "exit_tag: entered #{reader.name}"
+        writer.skipped(reader.name)
+        nest += 1 if reader.name == name
+      when XML::Reader::TYPE_END_ELEMENT
+#        puts "exit_tag: exited #{reader.name}"
+        nest -= 1 if reader.name == name
+      end
+      reader.read
+      if nest == 0
+#        puts "exit_tag: successful exit, now at #{reader.name}"
+        break
+      end
+    end
+  end
+
+  # Call this to skip when reader is on the opening tag
+  def self.skip_tag(writer, reader, skipping)
+    nest = 1
+    name = reader.name
+
+#    puts "skip_tag: skipping #{name}"
+
+    if reader.empty_element?
+#      puts "skip_tag: element was empty; skipped"
+      writer.skipped(name) if skipping
+      reader.read
+      return
+    end
+
+    while reader.read
+      case reader.node_type
+      when XML::Reader::TYPE_ELEMENT
+#        puts "skip_tag: entered #{reader.name}"
+        writer.skipped(reader.name) if skipping
+        nest += 1 if reader.name == name
+      when XML::Reader::TYPE_END_ELEMENT
+#        puts "skip_tag: exited #{reader.name}"
+        nest -= 1 if reader.name == name
+      end
+      if nest == 0
+        reader.read
+        break
+      end
+    end
+  end
+
+  class MissingElement < Exception
+  end
+
+  def self.to_element(writer, reader, name)
+    while WikiAvro::XML::to_tag(reader)
+#      puts "to_element: saw #{reader.name}"
+      if reader.name == name
+        return
+      else
+#        puts "to_element: skipping #{reader.name}"
+        WikiAvro::XML.skip_tag(writer, reader, true)
+#        puts "to_element: skipped"
+      end
+
+      break if !reader.read
+    end
+
+    raise MissingElement.new(name)
+  end
+
+  class Element
+    attr_reader :attr
+
+    def name
+      raise NotImplementedError.new('name')
+    end
+
+    def optional?
+      false
+    end
+
+    def parse(output, parent, reader)
+      if parent.nil? && reader.name != self.name
+        raise RuntimeError.new('reader.name != self.name')
+      else
+        WikiAvro::XML::to_element(output, reader, self.name)
+      end
+
+      reset
+      @attr = parse_attributes(output, parent, reader)
+      parse_content(output, parent, reader)
+      handle_content(output, parent, reader)
+    end
+
+    protected
+
+    # Instances will be reused. Subclasses that keep state which needs
+    # to be discarded after each parse should implement this.
+    def reset
+    end
+
+    def parse_attributes(w, p, r)
+      # no attributes parsed
+    end
+
+    # parse_content should move the reader away from the children's
+    # parent's opening tag. It should leave reader positioned after
+    # the closing tag.
+    def parse_content(w, p, r)
+      if r.empty_element?
+        @children.each do |c|
+          raise MissingElement.new(c.name) if !c.optional?
+        end
+        r.read
+        return
+      end
+
+      # Move away from our opening tag
+      r.read
+      @children.each do |c|
+#        puts "element: parsing #{c.class}"
+        c.parse(w, self, r)
+#        puts "parsed #{c.class}"
+      end
+
+      if r.empty_element? && r.name == self.name
+#        puts "got empty: #{r.name}"
+        r.read
+#        puts "now got this: #{r.name}"
+      else
+#        puts "mopping up #{self.name}"
+        WikiAvro::XML.exit_tag(w, r, self.name)
+      end
+    end
+
+    def handle_content(w, p, r)
+      # nothing done
+    end
+
+    private
+
+    def initialize(children)
+      @children = children
+    end
+  end
+
+  class Leaf < Element
+    def initialize
+      super([])
+    end
+  end
+
+  class Inserter < Leaf
+    attr_reader :name
+
+    def parse_content(w, p, r)
+      p.send(@writer, r.read_string)
+#      puts "inserter: exiting #{@name}"
+      WikiAvro::XML.skip_tag(w, r, false)
+#      puts "exited"
+    end
+
+    def initialize(name, target=name)
+      super()
+      @name = name
+      @writer = (target + '=').to_sym
+    end
+  end
+
+  class Stream
+    def optional?
+      true
+    end
+
+    def parse(output, parent, reader)
+#      puts "stream: parsing #{self.class}"
+
+      while WikiAvro::XML::to_tag(reader)
+        e = @elements[reader.name]
+
+        if e.nil?
+#          puts "stream: rejected #{reader.name}"
+          return
+        else
+#          puts "stream: accepted #{reader.name}"
+          e.parse(output, parent, reader)
+          reader.read
+        end
+      end
+
+#      puts "stream: ran to parent end"
+    end
+
+    private
+
+    def initialize(elements)
+      @elements = {}
+      elements.each { |e| @elements[e.name] = e }
+    end
+  end
+
+  class TooManyElements < Exception
+  end
+
+  class TooFewElements < Exception
+  end
+
+  class Group
+    # remember to override this if untrue, especially if it might be
+    # within an empty element
+    def optional?
+      false
+    end
+
+    def parse(output, parent, reader)
+      @n.keys.each {|k| @n[k] = 0}
+
+      while WikiAvro::XML::to_tag(reader)
+        e = @elements[reader.name]
+
+        if e.nil?
+          @elements.each do |k, v|
+            raise TooFewElements.new(k) if @n[k] < v[:min]
+            # this ought to be a redundant check
+            raise TooManyElements.new(k) if @n[k] > v[:max]
+          end
+#          puts "group: rejected #{reader.name}"
+          return
+        else
+#          puts "group: accepted #{reader.name}"
+          name = reader.name
+          @n[name] += 1
+          raise TooManyElements.new(name) if @n[name] > e[:max]
+          e[:element].parse(output, parent, reader)
+          reader.read
+        end
+      end
+
+#      puts 'group: ran to parent end'
+    end
+
+    def initialize(elements)
+      @elements = {}
+      @n = Hash.new 0
+      elements.each do |e|
+        name = e[:element].name
+        @elements[name] = e
+        @n[name] = 0
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: wikiavro
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Someon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: libxml-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: 
+email: someon@openmailbox.org
+executables:
+- wikiavro
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/wikiavro
+- lib/wikiavro.rb
+- lib/wikiavro/avro.rb
+- lib/wikiavro/mediawiki.rb
+- lib/wikiavro/xml.rb
+homepage: 
+licenses:
+- GPL-3.0+
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Convert MediaWiki XML dumps to Avro
+test_files: []