wikiscript 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e86e3e1b50b44067a5815a13155b04aa441e8023
-  data.tar.gz: c38ed306346a9b3d2c7c365fba00c41bee4624b4
+  metadata.gz: fe20e2b58b4703144ef8a83d1cb41edf92f5ef3e
+  data.tar.gz: ae8a0d31b16099c33fee4ac450b27433852b8fb8
 SHA512:
-  metadata.gz: 290c18216f59d7c2e6f2e1b181651413790fd2f7ff709c7f6f9c9cf1708b540480efa8827c57613df6862b1da755cd584c53f71ff93b07fe34a7a2ef4a67420d
-  data.tar.gz: 9e0c5c402b17263d38065eaac1a9e1febee6f3a73dff382aae8d81a8d0436881a2437a784dcee83f69c50d1fb5c6c5f385458a90ebb2cff8a31d31fffbd951a5
+  metadata.gz: 61fe785e9ea9150f6a5711ff0038cb135aba18f719abf26d6783bf50c465678f7376c315bcce3091b325d8ac9ca4714e084f93503e21dae583cddb9c904863d5
+  data.tar.gz: c437c92ba5df517d21231060044ab690aba2b1b11a368d37018c65eb493cea467bbd3331f161f217c1764fc24025fcf97a57ab3889e5aa2fc6b53d57e460f13b

data/NOTES.md

@@ -3,4 +3,33 @@
 
 ## Alternatives
 
-TBD
+
+
+- [wikipedia-client](https://rubygems.org/gems/wikipedia-client) - by Ken Pratt et al - ruby client for the Wikipedia API 
+  - <https://github.com/kenpratt/wikipedia-client>
+  - <https://www.rubydoc.info/gems/wikipedia-client>
+
+<!-- break -->
+
+- [infoboxer](https://rubygems.org/gems/infoboxer) - by Victor Shepelev et al - pure-Ruby Wikipedia (and generic MediaWiki) client and parser, targeting information extraction  
+  - <https://github.com/molybdenum-99/infoboxer>
+  - <https://www.rubydoc.info/gems/infoboxer>
+
+<!-- break -->
+
+More
+
+- <https://github.com/molybdenum-99/reality>
+- https://github.com/molybdenum-99/mediawiktory
+
+**Python**
+
+- <https://pypi.org/project/wptools/> - Wikipedia tools (for Humans)
+  - <https://github.com/siznax/wptools/>
+
+
+## Wikipedia
+
+  - Wikipedia API reference: <http://en.wikipedia.org/w/api.php>
+
+

data/lib/wikiscript.rb

@@ -51,11 +51,11 @@ module Wikiscript
     }x
 
 
-  def self.unlink( value )
+  def self.unlink( text )
     ## replace ALL wiki links with title (or link)
     ##  e.g. [[Santiago]] ([[La Florida, Chile|La Florida]])
     ##   =>    Santiago (La Florida)
-    value = value.gsub( LINK_PATTERN ) do |_|
+    text = text.gsub( LINK_PATTERN ) do |_|
       link  = $~[:link]
       title = $~[:title]
 
@@ -66,14 +66,16 @@ module Wikiscript
       end
     end
 
-    value.strip
+    text.strip
+  end
+  class << self
+    alias_method :flatten_links, :unlink
   end
 
-
-  def self.parse_link( value )     ## todo/change: find a better name - use match_link/etc. - why? why not?
+  def self.parse_link( text )     ## todo/change: find a better name - use match_link/etc. - why? why not?
     ##  find first matching link
     ##   return [nil,nil] if nothing found
-    if (m = LINK_PATTERN.match( value ))
+    if (m = LINK_PATTERN.match( text ))
       link  = m[:link]
       title = m[:title]
 
@@ -85,6 +87,17 @@ module Wikiscript
     end
   end
 
+  ############################
+  ## more convenience shortcuts / helpers
+  def self.parse( text )        PageReader.parse( text );  end
+  def self.parse_table( text )  TableReader.parse_table( text );  end
+
+  def self.get( title, lang: Wikiscript.lang )      Page.get( title, lang: lang ); end
+  class << self
+    alias_method :fetch,    :get
+    alias_method :download, :get
+  end
+
 end # module Wikiscript
 
 

data/lib/wikiscript/client.rb

@@ -15,30 +15,30 @@ module Wikiscript
       @worker = Fetcher::Worker.new
     end
 
-    ## change to: wikitext why? why not? or to raw? why? why not?
-    def text( title )
+    ## change to: wikitext or raw why? why not? or to raw? why? why not?
+    def text( title, lang: Wikiscript.lang )
       ## todo/fix: convert spaces to _ if not present for wikipedia page title - why ?? why not ???
-      get( action: 'raw', title: title )
-    end
 
-private
-    def site_base
-      ## replace lang w/ lang config if present e.g.
+      ## note: replace lang w/ lang config if present e.g.
       ##   http://{lang}.wikipedia.org/w/index.php
       # becomes
       #   http://en.wikipedia.org/w/index.php  or
       #   http://de.wikipedia.org/w/index.php  etc
+      base_url = SITE_BASE.gsub( "{lang}", lang )
+      params   = { action: 'raw',
+                   title:  title }
 
-      SITE_BASE.gsub( "{lang}", Wikiscript.lang )
+      get( base_url, params )
     end
 
+private
     def build_query( h )
       h.map do |k,v|
         "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
       end.join( '&' )
     end
 
-    def get( params )
+    def get( base_url, params )
       # note: lets us passing in params as hash e.g.
       #   action: 'raw', title: 'Austria'
       #   key and values will get CGI escaped
@@ -46,7 +46,7 @@ private
 
       ## uri = URI.parse( "#{SITE_BASE}?#{params}" )
       ## fix: pass in uri (add to fetcher check for is_a? URI etc.)
-      uri_string = "#{site_base}?#{query}"
+      uri_string = "#{base_url}?#{query}"
 
       response = @worker.get_response( uri_string )
 
@@ -69,7 +69,8 @@ private
         t
       else
         logger.error "fetch HTTP - #{response.code} #{response.message}"
-        nil
+        exit 1    ### exit for now on error - why? why not?
+        ## nil
       end
     end
 

data/lib/wikiscript/page.rb

@@ -6,15 +6,24 @@ module Wikiscript
 
     include LogUtils::Logging
 
-    attr_reader :title
+    attr_reader :title, :lang
 
-    def initialize( title, text: nil )
+
+    def self.get( title, lang: Wikiscript.lang )    ## todo/check: add a fetch/download alias - why? why not?
+      o = new( title: title, lang: lang )
+      ## o.text   ## "force" download / fetch
+      o
+    end
+
+
+    def initialize( text=nil, title: nil, lang: Wikiscript.lang )
       ## todo: check title
       ## replace title spaces w/ _ ????
       ##  to allow "pretty" titles - why? why not??
 
-      @title = title
       @text  = text
+      @title = title
+      @lang  = lang
     end
 
     def text
@@ -22,7 +31,7 @@ module Wikiscript
     end
 
     def download_text
-      Client.new.text( @title )
+      Client.new.text( @title, lang: @lang )
     end
 
     def parse   ## todo/change: use/find a different name e.g. doc/elements/etc. - why? why not?

data/lib/wikiscript/page_reader.rb

@@ -53,9 +53,11 @@ class PageReader
       elsif inside_table
          table_txt << line << "\n"
       else
-        puts "** !!! ERROR !!! unknown line type in wiki page:"
-        pp line
-        exit 1
+        ## note: skip unknown line types for now
+        
+        ## puts "** !!! ERROR !!! unknown line type in wiki page:"
+        ## pp line
+        ## exit 1
       end
     end
     page

data/lib/wikiscript/table_reader.rb

@@ -65,21 +65,31 @@ class TableReader
            row = []
            rows << row
          end
-         ## add each value one-by-one for now (to keep (same) row reference)
-         ##   note: also strip leading (optional) attributes
-         values.each do |value|
-           row <<  strip_emphases( strip_attributes( value.strip ))
+         if values.empty?
+           ## note: support continuing column text in next line
+           row << String.new
+         else
+           ## add each value one-by-one for now (to keep (same) row reference)
+           ##   note: also strip leading (optional) attributes
+           values.each do |value|
+             row <<  strip_emphases( strip_attributes( value.strip ))
+           end
          end
       elsif inside_table && line.start_with?( '|' )   ## table data
          values = line.sub( '|', '' ).strip.split( '||' )
-         ## add each value one-by-one for now (to keep (same) row reference)
-         values.each do |value|
-           row <<  strip_emphases( strip_attributes( value.strip ))
+         if values.empty?
+           ## note: support continuing column text in next line
+           row << String.new
+         else
+           ## add each value one-by-one for now (to keep (same) row reference)
+           values.each do |value|
+             row << strip_emphases( strip_attributes( value.strip ))
+           end
          end
       elsif inside_table
-        puts "!! ERROR !! unknown line type inside table:"
-        puts line
-        exit 1
+        ## note: support continuing column text in next line
+        ##  todo/check: for now doesn't support multi-line just simple continuing line - fix later if needed!!!
+        row[-1] << line
       else
         puts "!! ERROR !! unknown line type outside (before or after) table:"
         puts line

data/lib/wikiscript/version.rb

@@ -1,6 +1,6 @@
 
 module Wikiscript
-   VERSION = '0.2.0'
+   VERSION = '0.3.0'
 
    def self.banner
      "wikiscript/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"

data/test/test_page.rb

@@ -1,5 +1,9 @@
 # encoding: utf-8
 
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_page.rb
+
 
 require 'helper'
 
@@ -11,7 +15,12 @@ class TestPage < MiniTest::Test
   end
 
   def test_austria_en
-    page = Wikiscript::Page.new( 'Austria' )
+    page = Wikiscript::Page.get( 'Austria' )
+# [debug] GET /w/index.php?action=raw&title=Austria uri=http://en.wikipedia.org/w/index.php?action=raw&title=Austria, redirect_limit=5
+# [debug] 301 TLS Redirect location=https://en.wikipedia.org/w/index.php?action=raw&title=Austria
+# [debug] GET /w/index.php?action=raw&title=Austria uri=https://en.wikipedia.org/w/index.php?action=raw&title=Austria, redirect_limit=4
+# [debug] 200 OK
+
     text = page.text
 
     ## print first 600 chars
@@ -21,20 +30,25 @@ class TestPage < MiniTest::Test
     assert /{{Infobox country/ =~ text
     assert /common_name = Austria/ =~ text
     assert /capital = \[\[Vienna\]\]/ =~ text
-    assert /The origins of modern-day Austria date back to the time/ =~ text
+    # assert /The origins of modern-day Austria date back to the time/ =~ text
   end
 
   def test_sankt_poelten_en
-    page = Wikiscript::Page.new( 'Sankt_Pölten' )
+    page = Wikiscript::Page.get( 'Sankt_Pölten' )
+# [debug] GET /w/index.php?action=raw&title=Sankt_P%C3%B6lten uri=http://en.wikipedia.org/w/index.php?action=raw&title=Sankt_P%C3%B6lten, redirect_limit=5
+# [debug] 301 TLS Redirect location=https://en.wikipedia.org/w/index.php?action=raw&title=Sankt_P%C3%B6lten
+# [debug] GET /w/index.php?action=raw&title=Sankt_P%C3%B6lten uri=https://en.wikipedia.org/w/index.php?action=raw&title=Sankt_P%C3%B6lten, redirect_limit=4
+# [debug] 200 OK
+
     text = page.text
 
     ## print first 600 chars
     pp text[0..600]
 
     ## check for some snippets
-    assert /{{Infobox Town AT/ =~ text
-    assert /Name\s+=\s+Sankt Pölten/ =~ text
-    assert /'''Sankt Pölten''' \(''St. Pölten''\) is the capital city of/ =~ text
+    assert /{{Infobox settlement/ =~ text
+    assert /name\s+=\s+Sankt Pölten/ =~ text
+    # assert /'''Sankt Pölten''' \(''St. Pölten''\) is the capital city of/ =~ text
   end
 
 end # class TestPage

data/test/test_page_de.rb

@@ -1,6 +1,11 @@
 # encoding: utf-8
 
 
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_page_de.rb
+
+
 require 'helper'
 
 
@@ -11,7 +16,13 @@ class TestPageDe < MiniTest::Test
   end
 
   def test_st_poelten_de
-    page = Wikiscript::Page.new( 'St._Pölten' )
+    page = Wikiscript::Page.get( 'St._Pölten' )
+#    [debug] GET /w/index.php?action=raw&title=St._P%C3%B6lten uri=http://de.wikipedia.org/w/index.php?action=raw&title=St._P%C3%B6lten, redirect_limit=5
+#    [debug] 301 TLS Redirect location=https://de.wikipedia.org/w/index.php?action=raw&title=St._P%C3%B6lten
+#    [debug] GET /w/index.php?action=raw&title=St._P%C3%B6lten uri=https://de.wikipedia.org/w/index.php?action=raw&title=St._P%C3%B6lten, redirect_limit=4
+#    [debug] 200 OK
+
+
     text = page.text
 
     ## print first 600 chars

data/test/test_page_reader.rb

@@ -11,7 +11,7 @@ require 'helper'
 class TestPageReader < MiniTest::Test
 
   def test_basic
-    el = Wikiscript::PageReader.parse( <<TXT )
+    el = Wikiscript.parse( <<TXT )
 =Heading 1==
 ==Heading 2==
 ===Heading 3===
@@ -44,7 +44,7 @@ TXT
   end
 
   def test_parse
-    page = Wikiscript::Page.new( 'Test', text: <<TXT )
+    page = Wikiscript::Page.new( <<TXT )
 =Heading 1==
 ==Heading 2==
 ===Heading 3===

data/test/test_table_reader.rb

@@ -10,7 +10,7 @@ require 'helper'
 class TestTableReader < MiniTest::Test
 
   def test_basic
-    tables = Wikiscript::TableReader.parse( <<TXT )
+    table = Wikiscript.parse_table( <<TXT )
 {|
 |-
 ! header1
@@ -27,8 +27,6 @@ class TestTableReader < MiniTest::Test
 |}
 TXT
 
-    table = tables[0]
-    assert_equal 1, tables.size     ## one table
     assert_equal 3, table.size      ## three rows
     assert_equal ['header1', 'header2', 'header3'], table[0]
     assert_equal ['row1cell1', 'row1cell2', 'row1cell3'], table[1]
@@ -36,7 +34,7 @@ TXT
   end
 
   def test_basic_ii     ##  with optional (missing) row divider before headers
-    tables = Wikiscript::TableReader.parse( <<TXT )
+    table = Wikiscript.parse_table( <<TXT )
 {|
 ! header1 !! header2 !! header3
 |-
@@ -46,16 +44,48 @@ TXT
 |}
 TXT
 
-    table = tables[0]
-    assert_equal 1, tables.size     ## one table
     assert_equal 3, table.size      ## three rows
     assert_equal ['header1', 'header2', 'header3'], table[0]
     assert_equal ['row1cell1', 'row1cell2', 'row1cell3'], table[1]
     assert_equal ['row2cell1', 'row2cell2', 'row2cell3'], table[2]
   end
 
+  def test_basic_iii  # with continuing header column lines
+    table = Wikiscript.parse_table( <<TXT )
+{|
+|-
+!
+header1
+!
+header2
+!
+header3
+|-
+|
+row1cell1
+|
+row1cell2
+|
+row1cell3
+|-
+|
+row2cell1
+|
+row2cell2
+|
+row2cell3
+|}
+TXT
+
+    assert_equal 3, table.size      ## three rows
+    assert_equal ['header1', 'header2', 'header3'], table[0]
+    assert_equal ['row1cell1', 'row1cell2', 'row1cell3'], table[1]
+    assert_equal ['row2cell1', 'row2cell2', 'row2cell3'], table[2]
+  end
+
+
   def test_strip_attributes_and_emphases
-    tables = Wikiscript::TableReader.parse( <<TXT )
+    table = Wikiscript.parse_table( <<TXT )
 {|
 |-
 ! style="width:200px;"|Club
@@ -69,8 +99,6 @@ TXT
 |}
 TXT
 
-table = tables[0]
-assert_equal 1, tables.size     ## one table
 assert_equal 4, table.size      ## four rows
 assert_equal ['Club', 'City'], table[0]
 assert_equal ['[[Biu Chun Rangers]]',            '[[Sham Shui Po]]'], table[1]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wikiscript
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-20 00:00:00.000000000 Z
+date: 2019-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils