wp 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ab7e33f36703b9dad5ab221b6232041779eca3a5
+  data.tar.gz: b5475589d63447669a7319aba5923519d80a1a16
+SHA512:
+  metadata.gz: 6f9614e78691b8c6a0d09ef2247d18e42bbb09e63855e49cdeb0803870071814dde4512bd0c9f21d0ba18b62eadfd74b4346f767e36dc54f92454c0f12c7b59a
+  data.tar.gz: fceac62168a01353bd64daa27055a24c9e5350c7266e853f5289bf643f3a8bedd7ca2302fbb7d94411604132b718eba9df4050c2dc053bcd703f6aeba3ba2e83

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Chris Gahan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+Writeme!

data/bin/wp

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+require 'cgi'
+
+########################################################### 
+# TODOs:
+# [ ] Handle redirects (in Wiki#article)
+# [ ] Render templates
+# [ ] Remove "edit" links
+# [ ] Don't break on pages with & in them
+# [x] handle redirects
+# [ ] store redirects in leveldb
+#     (update leveldb as the redirects are discovered)
+# [ ] Random titles (lookup 30 random letter pairs)
+
+########################################################### 
+
+def console_viewer(html)
+  # IO.popen("lynx -stdin", "w") do |lynx|
+  IO.popen("w3m -T text/html", "w") do |lynx|
+    lynx.puts html # "<h1>#{article.title}</h1>\n#{html}"
+  end
+end
+
+########################################################### 
+
+require 'slop' # lazy loaded
+
+opts = Slop.parse(help: true, strict: true) do
+  banner 'Usage: wp [options] <query...>'
+
+  on 's', 'server', 'Launch server'
+  on 'p=', 'port', 'Webserver port (default: 4567)'
+  on 'd', 'download', 'Download the latest dump'
+  # on 'u', 'update', 'Update the database (takes a LONG time)'
+  on 'i=', 'import', 'Import dump'
+  on 'o=', 'outfile', 'Where to output the file'
+  on 'c', 'config', 'Interactive config'
+
+  # command 'server' do
+  #   on "p", "port", 'Port (default: 3000)', default: 3000
+
+  #   run do |opts, args|
+  #     server(opts.port)
+  #   end
+  # end
+end
+
+########################################################### 
+
+require 'wp/wiki'
+
+if opts.server?
+  # Run Webserver
+
+  require 'wp/webserver'
+
+elsif opts.reindex?
+  #
+
+elsif opts.update?
+  #
+
+else
+  # CLI interface
+  query = ARGV.join(" ")
+  wiki  = Wiki.new
+  
+  titles = wiki.search(query, 20)
+
+  if titles.size == 1
+    title = titles.first
+  else
+    titles.each.with_index do |title, i|
+      puts "#{i+1}. #{title}"
+    end
+
+    print "> "
+
+    loop do
+      case STDIN.gets.strip
+      when ""
+        exit 1
+      when /^(\d+)$/
+        n = $1.to_i - 1
+        title = titles[n]
+        break
+      end
+    end
+
+  end
+
+  a = wiki.article(title)
+
+  html = []
+  html << "<title>#{a.title}</title>"
+  html << "<center><h1>#{a.title}</h1></center>"
+  html << "<center>(redirected from: <b>#{a.redirected_from}</b>)</center>" if a.redirected_from
+  html << a.html
+
+  console_viewer html.join("\n")
+  # ws = Thread.new { require 'wp/webserver' }
+
+  # system("luakit", "http://localhost:4567/#{CGI.escape title}")
+end
+

data/lib/wp.rb

@@ -0,0 +1,28 @@
+
+require 'wp/wiki'
+
+#####################################################################
+
+def commatize(thing)
+  thing.to_s.reverse.each_slice(3).to_a.join(",").reverse
+end
+
+#####################################################################
+
+class Downloader
+
+  def mech
+    require 'mechanize'
+    @mech ||= Mechanize.new
+  end
+
+end
+
+#####################################################################
+
+def console_viewer(html)
+  IO.popen("lynx -stdin", "w") do |lynx|
+  # IO.popen("w3m -T text/html", "w") do |lynx|
+    lynx.puts html # "<h1>#{article.title}</h1>\n#{html}"
+  end
+end

data/lib/wp/config.rb

@@ -0,0 +1,79 @@
+require 'epitools'
+
+module Pedia
+  class Config
+
+    # Constants #########################################################
+
+    CONFIG_DIR = File.expand_path "~/.config/pedia"
+    CACHE_DIR = File.expand_path "~/.cache/pedia"
+    CONFIG_FILE = File.join CONFIG_DIR, "config.yml"
+
+    # DSL ###############################################################
+
+    class Option < TypedStruct["name value default prompt type:symbol"]
+
+      def value
+        @value ||= @default
+      end
+
+      def init
+        case type
+        when :dir
+          File.mkdir value
+        end
+      end
+
+    end
+
+
+    def self.option(*args)
+      opt = Option.new(*args)
+
+      attr_accessor opt.name
+
+      @@spec ||= {}
+      @@spec[opt.name] = opt
+    end
+
+    # Options ###########################################################
+
+    option name:    "db_path",
+           prompt:  "Where should the database be stored (approx. 8 gigs)?",
+           default: CACHE_DIR,
+           type: :dir
+
+    option name:    "xml_path",
+           prompt:  "Where should the XML be stored (approx. 8 gigs)?",
+           default: CACHE_DIR,
+           type: :dir
+
+
+    # Methods ###########################################################
+
+    def initialize(filename="config.yml")
+      Dir.mkdir CONFIG_DIR unless File.exists? CONFIG_DIR
+
+      unless File.exists? CONFIG_FILE
+
+      end
+
+      yaml = YAML.load open(CONFIG_FILE)
+
+      @options = {}
+
+      yaml.each do |key, val|
+        p [key, val]
+        if opt = @@spec[key]
+          @options[key] = opt.with(value: val)
+        end
+      end
+    end
+
+    
+    def options
+      @options
+    end
+
+  end # class Config
+end # module Pedia

data/lib/wp/old/wiki.rb

@@ -0,0 +1,33 @@
+require 'leveldb'
+
+# https://github.com/nricciar/wikicloth
+require 'wikicloth'
+
+class Wiki
+
+  def initialize(path="enwiki")
+  end
+
+  def [](key)
+    Zlib.inflate @db.get(key)
+  end
+
+  def titles(max=30)
+    search("", max)
+  end
+
+  def search(prefix, max=30)
+    @db.fwmkeys(prefix, max)
+  end
+
+  def html(title)
+    WikiCloth::Parser.new(data: self[title]).to_html
+  end
+
+end
+
+
+if $0 == __FILE__
+  wiki = Wiki.new
+  pp wiki.titles
+end

data/lib/wp/old/xmlreader.rb

@@ -0,0 +1,67 @@
+require 'nokogiri'
+
+# <title>AccessibleComputing</title>
+# <ns>0</ns>
+# <id>10</id>
+# <redirect title="Computer accessibility" />
+# <revision>
+#   <sha1>lo15ponaybcg2sf49sstw9gdjmdetnk</sha1>
+#   <format>text/x-wiki</format>
+#   <text>...ARTICLE...</text>
+# </revision>
+
+
+class XMLReader
+
+  include Enumerable
+
+  def initialize(filename)
+    if filename[/\.bz2/]
+      io = IO.popen ["bunzip2", "-c", filename]
+    else
+      io = open(filename)
+    end
+
+    @reader = Nokogiri::XML::Reader io
+  end
+
+  def each
+    info = {}
+
+    @reader.each do |node|
+
+      case node.name
+      when "page"
+        if node.open?
+          info = {}
+        else
+          yield info
+        end
+      when "title", "text"
+        info[node.name] = node.text if node.open?
+      when "redirect"
+        info["redirect"] = node.attribute("title")
+      end
+
+    end
+  end
+
+end
+
+
+
+
+class Nokogiri::XML::Reader
+  def open?
+    node_type == Nokogiri::XML::Reader::TYPE_ELEMENT
+  end
+
+  def closed?
+    node_type == Nokogiri::XML::Reader::TYPE_END_ELEMENT
+  end
+
+  def text
+    inner_xml
+  end
+end
+

data/lib/wp/webserver.rb

@@ -0,0 +1,115 @@
+require 'sinatra/base'
+require 'wp/wiki'
+require "sinatra/reloader" # if development?
+
+$wiki = Wiki.new
+
+class Webserver < Sinatra::Base
+
+  def wiki
+    $wiki
+  end
+
+  def link_to(title, desc=nil)
+    desc ||= title
+    "<a href=\"/#{title}\">#{desc}</a>"
+  end
+
+  def highlight(title, query)
+    title.gsub(query) { |m| "<span style='background: yellow'>#{m}</span>"}
+  end
+
+  def header(query="")
+    %{
+      <form action="search">
+        <input type="text" name="q" value="#{query}">
+        <input type="submit" value="search">
+      </form>
+    }
+  end
+
+  get "/" do
+    letter = ('A'..'Z').to_a.sample
+    titles = wiki.search(letter, 20)
+
+    links = titles.map { |title| "<li>#{link_to title}</li>" }
+
+    %{
+      #{header}
+
+      <h1>#{letter}</h1>
+      <ul>
+        #{links.join "\n"}
+      </ul>
+    }
+  end
+
+  get "/search" do
+    query = params[:q]
+    titles = wiki.search query
+
+    links = titles.map do |title|
+      desc = highlight title, query
+      "<li>#{link_to title, desc}</li>"
+    end
+
+    %{
+      #{header query }
+
+      <h1>Query: #{query}</h1>
+      <ul>
+        #{links.join "\n"}
+      </ul>
+    }
+  end
+
+  get %r{/(.+)} do
+    title = params[:captures].first
+
+    if article = wiki.article(title)
+      %{
+        #{header article.title}
+
+        <h1>#{article.title}</h1>
+
+        #{article.html}
+
+        <!--
+        #{article.xml.to_s}
+        -->
+      }
+    else
+      "No article found."
+    end
+  end
+
+  run!
+
+end
+
+
+
+class WikiParser < WikiCloth::Parser
+
+  url_for do |page|
+    "javascript:alert('You clicked on: #{page}');"
+  end
+
+  link_attributes_for do |page|
+    { :href => url_for(page) }
+  end
+
+  template do |template|
+    "Hello {{{1}}}" if template == "hello"
+  end
+
+  external_link do |url,text|
+    "<a href=\"#{url}\" target=\"_blank\" class=\"exlink\">#{text.blank? ? url : text}</a>"
+  end
+
+end
+
+# @wiki = WikiParser.new({ 
+#   :params => { "PAGENAME" => "Testing123" }, 
+#   :data => "{{hello|world}} From {{ PAGENAME }} -- [www.google.com]"; 
+# })

data/lib/wp/wiki.rb

@@ -0,0 +1,189 @@
+require 'leveldb'
+require 'snappy'
+require 'nokogiri'
+require 'wikicloth'
+
+#####################################################################
+
+def bz2_stream(thing, offset=nil, length=nil)
+  require 'open3'
+
+  open(thing, "rb") do |bz2|
+
+    Open3.popen2("bunzip2", "-c") do |inp, outp, th|
+      Thread.new do
+        IO.copy_stream(bz2, inp, length, offset)
+        inp.close
+      end
+
+      yield outp
+    end
+
+  end
+
+end
+
+#####################################################################
+
+class Article < Struct.new(:title, :text, :redirect, :xml, :redirected_from)
+
+  def initialize(page)
+    self.xml = page
+    self.title = page.at("title").text
+
+    if redirect = page.at("redirect")
+      self.redirect = redirect["title"]
+    else
+      self.text = page.at("text").text
+    end
+  end
+
+
+  def redirect?
+    !!redirect
+  end
+
+
+  def to_html
+    # WikiCloth::Parser.new(data: text).to_html
+    WikiParser.new(data: text).to_html
+  end
+
+  alias_method :html, :to_html
+
+end
+
+#####################################################################
+
+class XMLReader
+
+  include Enumerable
+
+  def initialize(input)
+    @doc = Nokogiri::XML.fragment input
+  end
+
+
+  def each
+    @doc.search("page").each do |page|
+      yield Article.new(page)
+    end
+  end
+
+end
+
+#####################################################################
+
+class Wiki
+  #@@index_url = "http://dumps.wikimedia.org/enwiki/20130604/enwiki-20130604-pages-articles-multistream-index.txt.bz2"
+  @@index_url    = "/d/wiki/enwiki-20130604-pages-articles-multistream-index.txt.bz2"
+  @@articles_url = "/d/wiki/enwiki-20130604-pages-articles-multistream.xml.bz2"
+
+  def initialize
+  end
+
+
+  def db
+    @db ||= LevelDB::DB.new File.expand_path("~/.cache/wp/enwiki-index")
+  end
+
+
+  def search(title, amount=40)
+    db.each(from: "title/#{title}").take(amount).map do |key, val|
+      key.split("/", 2).last
+    end
+  end
+
+  def random(amount=30)
+    letters = [*'A'..'Z'] + [*'0'..'9']
+
+    (1..30).map do 
+      prefix = letters.sample + letters.sample.downcase
+      key, val = db.each(from: prefix).first
+      key.split("/", 2).last
+    end
+  end
+
+  def article(title, redirected_from=nil)
+    if title =~ %r{^title/(.+)}
+      title = $1
+    end
+
+    return nil unless offset = db["title/#{title}"]
+      
+    offset = offset.to_i
+    length = db["length/#{offset}"].to_i
+
+    bz2_stream(@@articles_url, offset, length) do |io|
+      xml = XMLReader.new io.read
+
+      xml.each do |article|
+
+        if title == article.title
+          if article.redirect?
+            return article(article.redirect, title)
+          else
+            article.redirected_from = redirected_from
+            return article
+          end
+        end
+
+      end
+    end
+  end
+
+  def import_index(index_url=@@index_url)
+    bz2_stream(index_url) do |io|
+
+      last_offset = nil
+
+      io.each_line.with_index do |line,i|
+        line.chomp!
+
+        offset, n, title = line.split(":", 3)
+        db["title/#{title}"] = offset
+
+        offset = offset.to_i
+
+        last_offset = offset if last_offset.nil?
+
+        if last_offset != offset
+          length = offset - last_offset
+          db["length/#{last_offset}"] = length.to_s
+
+          last_offset = offset
+        end
+
+        print "\e[1G#{commatize i} - #{title}\e[J" if i % 11337 == 0
+
+      end
+
+    end
+  end
+
+
+end
+
+#####################################################################
+
+class WikiParser < WikiCloth::Parser
+
+  url_for do |page|
+    "javascript:alert('You clicked on: #{page}');"
+  end
+
+  link_attributes_for do |page|
+    { :href => url_for(page) }
+  end
+
+  template do |template|
+    "Hello {{{1}}}" if template == "hello"
+  end
+
+  external_link do |url,text|
+    "<a href=\"#{url}\" target=\"_blank\" class=\"exlink\">#{text.blank? ? url : text}</a>"
+  end
+
+end
+
+#####################################################################

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: wp
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- epitron
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: leveldb-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: wikicloth
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Imports Wikipedia dumps to a local database and lets you browse them
+  from the commandline or with a local sinatra server.
+email: chris@ill-logic.com
+executables:
+- wp
+extensions: []
+extra_rdoc_files:
+- README.md
+- LICENSE
+files:
+- bin/wp
+- lib/wp.rb
+- lib/wp/wiki.rb
+- lib/wp/webserver.rb
+- lib/wp/config.rb
+- lib/wp/old/wiki.rb
+- lib/wp/old/xmlreader.rb
+- README.md
+- LICENSE
+homepage: http://github.com/epitron/wp/
+licenses:
+- WTFPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: A local Wikipedia with commandline and web interfaces.
+test_files: []
+has_rdoc: