wikiscript 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: aa972f9f2c4580091fb3dac93cf76f94f6054ba1
-  data.tar.gz: 138954a7a0281032b68aae0f29b9d239c9f7f90a
+SHA256:
+  metadata.gz: 9c6ea507fa295074d82e5d81eb88828c4473b35e5aee7f04b79cfbc774bb9e72
+  data.tar.gz: 56a5e69536495a90e0c1635799c8023ca57fdfbc554f818d40f86046972f13fb
 SHA512:
-  metadata.gz: 74af5f6bd6b915a1887d9022763294f0e6e634415d1bbdf3b18ae9fc0f9a11b01812b904a42a4ed687a60cafbd50230afd8a804dab8bb6a8b28475f99437be2b
-  data.tar.gz: 44f92623eb560825a834de570b8cfc604a92bfb907b38c2390b2d3a52cc74bcf8f229d21369c27bd73c0b99a9db9c65f5df3bf9b6d304e9876694135e7517ddc
+  metadata.gz: 5d0bc2f5ec90d5a3c3f6c0c72c145f33a4faf1acccb28a3487059c715a8c60b5a5ed43fdd54318fe30669b1f64601a48eab0bfbba1f30da494f0b89a526cf054
+  data.tar.gz: ad591ed36382240029ddf9cf3e120fb938d3a253b8766cf04e0875b222b2349ee7473a1b0c851ee337afcd501d8830cea1da39d30eb78e60f9aa539d33408a59

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+### 0.4.0
+
 ### 0.0.1 / 2014-07-03
 
 * Everything is new. First release.

data/Manifest.txt

@@ -1,18 +1,11 @@
 CHANGELOG.md
-LICENSE.md
 Manifest.txt
-NOTES.md
 README.md
 Rakefile
 lib/wikiscript.rb
 lib/wikiscript/client.rb
+lib/wikiscript/outline_reader.rb
 lib/wikiscript/page.rb
 lib/wikiscript/page_reader.rb
 lib/wikiscript/table_reader.rb
 lib/wikiscript/version.rb
-test/helper.rb
-test/test_link.rb
-test/test_page.rb
-test/test_page_de.rb
-test/test_page_reader.rb
-test/test_table_reader.rb

data/README.md

@@ -12,16 +12,203 @@ Read-only access to wikikpedia pages.
 Example - Get wikitext source (via `en.wikipedia.org/w/index.php?action=raw&title=<title>`):
 
 
+``` ruby
+page = Wikiscript::Page.get( '2022_FIFA_World_Cup' )  # same as Wikiscript.get
+page.text
 ```
->> page = Wikiscript::Page.new( '2014_FIFA_World_Cup_squads' )
->> page.text
 
-The [[2014 FIFA World Cup]] is an international [[association football|football]]
-tournament which is currently being held in Brazil from 12 June to 13 July 2014.
-The 32 national teams involved in the tournament were required to register
-a squad of 23 players, including three goalkeepers...
+prints
+
+```
+The '''2022 FIFA World Cup''' is scheduled to be the 22nd edition of the [[FIFA World Cup]],
+the quadrennial international men's [[association football]] championship contested by the
+[[List of men's national association football teams|national teams]] of the member associations of [[FIFA]].
+It is scheduled to take place in [[Qatar]] in 2022. This will be the first World Cup ever to be held
+in the [[Arab world]] and the first in a Muslim-majority country...
+```
+
+Or build your own page from scratch (no download):
+
+``` ruby
+page = Wikiscript::Page.new( <<TXT, title: '2022_FIFA_World_Cup' )
+The '''2022 FIFA World Cup''' is scheduled to be the 22nd edition of the [[FIFA World Cup]],
+the quadrennial international men's [[association football]] championship contested by the
+[[List of men's national association football teams|national teams]] of the member associations of [[FIFA]].
+It is scheduled to take place in [[Qatar]] in 2022. This will be the first World Cup ever to be held
+in the [[Arab world]] and the first in a Muslim-majority country...
+TXT
+page.text
+```
+
+prints
+
+```
+The '''2022 FIFA World Cup''' is scheduled to be the 22nd edition of the [[FIFA World Cup]],
+the quadrennial international men's [[association football]] championship contested by the
+[[List of men's national association football teams|national teams]] of the member associations of [[FIFA]].
+It is scheduled to take place in [[Qatar]] in 2022. This will be the first World Cup ever to be held
+in the [[Arab world]] and the first in a Muslim-majority country...
+```
+
+
+### Tables
+
+Parse wiki tables into an array.  Example:
+
+``` ruby
+table = Wikiscript.parse_table( <<TXT )
+{|
+|-
+! header1
+! header2
+! header3
+|-
+| row1cell1
+| row1cell2
+| row1cell3
+|-
+| row2cell1
+| row2cell2
+| row2cell3
+|}
+TXT
+
+# -or-
+
+table = Wikiscript.parse_table( <<TXT )
+{|
+! header1 !! header2 !! header3
+|-
+| row1cell1 || row1cell2 || row1cell3
+|-
+| row2cell1 || row2cell2 || row2cell3
+|}
+TXT
+
+# -or-
+
+table = Wikiscript.parse_table( <<TXT )
+{|
+|-
+!
+header1
+!
+header2
+!
+header3
+|-
+|
+row1cell1
+|
+row1cell2
+|
+row1cell3
+|-
+|
+row2cell1
+|
+row2cell2
+|
+row2cell3
+|}
+TXT
+```
+
+resulting in:
+
+``` ruby
+pp table
+#=> [["header1",   "header2",   "header3"],
+#    ["row1cell1", "row1cell2", "row1cell3"],
+#    ["row2cell1", "row2cell2", "row2cell3"]]
 ```
 
+Note: `parse_table` will strip/remove (leading) style attributes (e.g. `àttribute="value" |` and (inline) bold and italic emphases (e.g. `''`) from the (cell) text. Example:
+
+``` ruby
+table = Wikiscript.parse_table( <<TXT )
+{|
+|-
+! style="width:200px;"|Club
+! style="width:150px;"|City
+|-
+|[[Biu Chun Rangers]]||[[Sham Shui Po]]
+|-
+|bgcolor=#ffff44 |''[[Eastern Sports Club|Eastern]]''||[[Mong Kok]]
+|-
+|[[HKFC Soccer Section]]||[[Happy Valley, Hong Kong|Happy Valley]]
+|}
+TXT
+```
+
+resulting in:
+
+``` ruby
+pp table
+#=> [["Club",                            "City"],
+#    ["[[Biu Chun Rangers]]",            "[[Sham Shui Po]]"],
+#    ["[[Eastern Sports Club|Eastern]]", "[[Mong Kok]]"],
+#    ["[[HKFC Soccer Section]]",         "[[Happy Valley, Hong Kong|Happy Valley]]"]]
+```
+
+### Links
+
+Split links into two parts. Note: The alternate link title is optional. Example:
+
+``` ruby
+link, title = Wikiscript.parse_link( '[[La Florida, Chile|La Florida]]' )
+link   #=> "La Florida, Chile"
+title  #=> "La Florida"
+
+link, title = Wikiscript.parse_link( '[[ La Florida, Chile]]' )
+link   #=> "La Florida, Chile"
+title  #=> nil
+
+link, title = Wikiscript.parse_link( 'La Florida' )
+link   #=> nil
+title  #=> nil
+```
+
+### Document Element Structure
+
+Get the document's element structure.
+Note: For now only section headings (`h1`, `h2`, `h3`, ...) and tables are supported.
+Example:
+
+``` ruby
+nodes = Wikiscript.parse( <<TXT )
+=Heading 1==
+==Heading 2==
+===Heading 3===
+
+{|
+|-
+! header1
+! header2
+! header3
+|-
+| row1cell1
+| row1cell2
+| row1cell3
+|-
+| row2cell1
+| row2cell2
+| row2cell3
+|}
+TXT
+
+pp nodes
+#=> [[:h1, "Heading 1"],
+#    [:h2, "Heading 2"],
+#    [:h3, "Heading 3"],
+#    [:table, [["header1", "header2", "header3"],
+#              ["row1cell1", "row1cell2", "row1cell3"],
+#              ["row2cell1", "row2cell2", "row2cell3"]]]
+```
+
+
+That's all for now. More functionality will get added over time.
+
 
 
 ## Install

data/Rakefile

@@ -5,26 +5,26 @@ Hoe.spec 'wikiscript' do
 
   self.version = Wikiscript::VERSION
 
-  self.summary = 'wikiscript - scripts for wikipedia (get wikitext for page etc.)'
+  self.summary = "wikiscript - scripts for wikipedia (get wikitext for page, parse tables 'n' links, etc.)"
   self.description = summary
 
-  self.urls = ['https://github.com/wikiscript/wikiscript']
+  self.urls = { home: 'https://github.com/wikiscript/wikiscript' }
 
   self.author = 'Gerald Bauer'
-  self.email = 'opensport@googlegroups.com'
+  self.email = 'gerald.bauer@gmail.com'
 
   # switch extension to .markdown for gihub formatting
   self.readme_file  = 'README.md'
   self.history_file = 'CHANGELOG.md'
 
   self.extra_deps = [
+    ['cocos'],
     ['logutils' ],
-    ['fetcher']
   ]
 
   self.licenses = ['Public Domain']
 
   self.spec_extras = {
-    required_ruby_version: '>= 2.2.2'
+    required_ruby_version: '>= 3.1.0'
   }
 end

data/lib/wikiscript/client.rb

@@ -1,19 +1,13 @@
-# encoding: utf-8
 
 module Wikiscript
 
   class Client
+    include Logging
 
-    include LogUtils::Logging
-
-    SITE_BASE = 'http://{lang}.wikipedia.org/w/index.php'
+    SITE_BASE = 'https://{lang}.wikipedia.org/w/index.php'
 
     ### API_BASE  = 'http://en.wikipedia.org/w/api.php'
 
-    def initialize( opts={} )
-      @opts   = opts
-      @worker = Fetcher::Worker.new
-    end
 
     ## change to: wikitext or raw why? why not? or to raw? why? why not?
     def text( title, lang: Wikiscript.lang )
@@ -24,7 +18,7 @@ module Wikiscript
       # becomes
       #   http://en.wikipedia.org/w/index.php  or
       #   http://de.wikipedia.org/w/index.php  etc
-      base_url = SITE_BASE.gsub( "{lang}", lang )
+      base_url = SITE_BASE.sub( "{lang}", lang )
       params   = { action: 'raw',
                    title:  title }
 
@@ -33,6 +27,10 @@ module Wikiscript
 
 private
     def build_query( h )
+
+      ## todo/fix - check what to use for params encode
+      ##   e.g. escape_component or such?
+      ##   fix add params upstream to weblclient - why? why not?
       h.map do |k,v|
         "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
       end.join( '&' )
@@ -48,27 +46,12 @@ private
       ## fix: pass in uri (add to fetcher check for is_a? URI etc.)
       uri_string = "#{base_url}?#{query}"
 
-      response = @worker.get_response( uri_string )
-
-      if response.code == '200'
-        t = response.body
-        ###
-        # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-        # will mostly be ASCII
-        # - try to change encoding to UTF-8 ourselves
-        logger.debug "t.encoding.name (before): #{t.encoding.name}"
-        #####
-        # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+      response = Webclient.get( uri_string )
 
-        ## NB:
-        # for now "hardcoded" to utf8 - what else can we do?
-        # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-        t = t.force_encoding( Encoding::UTF_8 )
-        logger.debug "t.encoding.name (after): #{t.encoding.name}"
-        ## pp t
-        t
+      if response.status.ok?
+        response.text
       else
-        logger.error "fetch HTTP - #{response.code} #{response.message}"
+        logger.error "HTTP ERROR - #{response.status.code} #{response.status.message}"
         exit 1    ### exit for now on error - why? why not?
         ## nil
       end

data/lib/wikiscript/outline_reader.rb

@@ -0,0 +1,97 @@
+
+module Wikiscript
+
+class OutlineReader
+
+  def self.read( path )
+    txt = File.open( path, 'r:utf-8' ) { |f| f.read }
+    parse( txt )
+  end
+
+  def self.parse( txt )
+    new( txt ).parse
+  end
+
+  def initialize( txt )
+    @txt = txt
+  end
+
+
+ HEADING_RE = %r{\A
+                  (?<marker>={1,})       ## 1. leading ======
+                    [ ]*
+                  (?<text>[^=]+?)        ## 2. text  (note: for now no "inline" = allowed)
+                    [ ]*
+                    =*                   ## 3. (optional) trailing ====
+                  \z}x
+
+  def parse
+    outline = []   ## outline / page structure
+
+    start_para = true      ## start new para(graph) on new text line?
+
+    @txt.each_line do |line|
+
+##
+##  (auto-)sanitize first
+##    -  &nbsp;   =>  "vanilla" space
+##    -  1–2      => 1-2  - "vanilla" dash
+##    todo - move up into txt!!!
+        line = line.gsub( '&nbsp;', ' ' )
+        line = line.gsub( /[–]/, '-' )
+
+
+        line = line.strip      ## todo/fix: keep leading and trailing spaces - why? why not?
+
+        if line.empty?    ## todo/fix: keep blank line nodes?? and just remove comments and process headings?! - why? why not?
+          start_para = true
+          next
+        end
+
+        break if line == '__END__'
+
+        next if line.start_with?( '#' )   ## skip comments too
+        ## strip inline (until end-of-line) comments too
+        ##  e.g Eupen | KAS Eupen ## [de]
+        ##   => Eupen | KAS Eupen
+        ##  e.g bq   Bonaire,  BOE        # CONCACAF
+        ##   => bq   Bonaire,  BOE
+        line = line.sub( /#.*/, '' ).strip
+
+         ## note: like in wikimedia markup (and markdown) all optional trailing ==== too
+        if m=HEADING_RE.match( line )
+           start_para = true
+
+           heading_marker = m[:marker]
+           heading_level  = heading_marker.length   ## count number of = for heading level
+           heading        = m[:text].strip
+
+           outline << [:"h#{heading_level}", heading]
+        elsif line == '----'   ## make more generic/flexible - why? why not?
+            start_para = true
+            outline << [:hr]
+ ## The horizontal rule represents a paragraph-level thematic break. Do not use in article content,
+ ##  as rules are used only after main sections, and this is automatic.
+ ## HTML equivalent: <hr /> (which can be indented,
+ ##  whereas ---- always starts at the left margin.)
+        else    ## assume it's a (plain/regular) text line
+           if start_para
+             outline << [:p, [line]]
+             start_para = false
+           else
+             node = outline[-1]    ## get last entry
+             if node[0] == :p      ##  assert it's a p(aragraph) node!!!
+                node[1] << line    ## add line to p(aragraph)
+             else
+               puts "!! ERROR - invalid outline state / format - expected p(aragraph) node; got:"
+               pp node
+               exit 1
+             end
+           end
+        end
+    end
+    outline
+  end # method parse
+end # class OutlineReader
+
+end # module Wikiscript

data/lib/wikiscript/page.rb

@@ -1,10 +1,9 @@
-# encoding: utf-8
 
 module Wikiscript
 
   class Page
 
-    include LogUtils::Logging
+    include Logging
 
     attr_reader :title, :lang
 
@@ -16,7 +15,7 @@ module Wikiscript
     end
 
     def self.read( path )
-      text = File.open( path, 'r:utf-8' ).read
+      text = File.open( path, 'r:utf-8' ) { |f| f.read }
       o = new( text, title: "File:#{path}" )   ## use auto-generated File:<path> title path - why? why not?
       o
     end
@@ -33,11 +32,23 @@ module Wikiscript
     end
 
     def text
-      @text ||= get   # cache text (from request)
+      @text ||= get        # cache text (from request)
+    end
+
+    def nodes
+      @nodes ||= parse     # cache text (from parse)
+    end
+
+    def each   ## loop over all nodes / elements -note: nodes is a (flat) list (array) for now
+      nodes.each do |node|
+        yield( node )
+      end
     end
 
 
     def get    ## "force" refresh text (get/fetch/download)
+      @nodes = nil  ## note: reset cached parsed nodes too
+
       @text = Client.new.text( @title, lang: @lang )
       @text
     end
@@ -45,9 +56,9 @@ module Wikiscript
     alias_method :download, :get
 
 
-
     def parse   ## todo/change: use/find a different name e.g. doc/elements/etc. - why? why not?
-      PageReader.parse( text )
+      @nodes = PageReader.parse( text )
+      @nodes
     end
   end # class Page
 end # Wikiscript

data/lib/wikiscript/page_reader.rb

@@ -1,11 +1,10 @@
-# encoding: utf-8
 
 module Wikiscript
 
 class PageReader
 
-  def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ).read
+  def self.read( path )
+    txt = File.open( path, 'r:utf-8' ) { |f| f.read }
     parse( txt )
   end
 
@@ -54,7 +53,7 @@ class PageReader
          table_txt << line << "\n"
       else
         ## note: skip unknown line types for now
-        
+
         ## puts "** !!! ERROR !!! unknown line type in wiki page:"
         ## pp line
         ## exit 1

data/lib/wikiscript/table_reader.rb

@@ -1,11 +1,10 @@
-# encoding: utf-8
 
 module Wikiscript
 
 class TableReader
 
-  def self.read( path )   ## use - rename to read_file or from_file etc. - why? why not?
-    txt = File.open( path, 'r:utf-8' ).read
+  def self.read( path )
+    txt = File.open( path, 'r:utf-8' ) { |f| f.read }
     parse( txt )
   end
 

data/lib/wikiscript/version.rb

@@ -1,12 +1,12 @@
 
 module Wikiscript
-   VERSION = '0.3.1'
+   VERSION = '0.4.0'
 
    def self.banner
-     "wikiscript/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+     "wikiscript/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}] in (#{root})"
    end
 
    def self.root
-     "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
+     File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
    end
 end

data/lib/wikiscript.rb

@@ -1,26 +1,25 @@
-# encoding: utf-8
-
-## stdlibs
-
-require 'net/http'
-require 'uri'
-require 'cgi'
-require 'pp'
+## stdlibs via cocos
+require 'cocos'
 
 
 ## 3rd party gems/libs
 ## require 'props'
 
 require 'logutils'
-require 'fetcher'
 
-# our own code
+module Wikiscript
+  Logging = LogUtils::Logging
+end
 
-require 'wikiscript/version' # let it always go first
-require 'wikiscript/client'
-require 'wikiscript/table_reader'
-require 'wikiscript/page_reader'
-require 'wikiscript/page'
+
+
+# our own code
+require_relative 'wikiscript/version' # let it always go first
+require_relative 'wikiscript/client'
+require_relative 'wikiscript/table_reader'
+require_relative 'wikiscript/page_reader'
+require_relative 'wikiscript/outline_reader'
+require_relative 'wikiscript/page'
 
 
 

metadata

@@ -1,17 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: wikiscript
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Gerald Bauer
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-21 00:00:00.000000000 Z
+date: 2024-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logutils
+  name: cocos
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fetcher
+  name: logutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -42,64 +42,62 @@ dependencies:
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: hoe
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.16'
+        version: '4.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.16'
-description: wikiscript - scripts for wikipedia (get wikitext for page etc.)
-email: opensport@googlegroups.com
+        version: '4.1'
+description: wikiscript - scripts for wikipedia (get wikitext for page, parse tables
+  'n' links, etc.)
+email: gerald.bauer@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files:
 - CHANGELOG.md
-- LICENSE.md
 - Manifest.txt
-- NOTES.md
 - README.md
 files:
 - CHANGELOG.md
-- LICENSE.md
 - Manifest.txt
-- NOTES.md
 - README.md
 - Rakefile
 - lib/wikiscript.rb
 - lib/wikiscript/client.rb
+- lib/wikiscript/outline_reader.rb
 - lib/wikiscript/page.rb
 - lib/wikiscript/page_reader.rb
 - lib/wikiscript/table_reader.rb
 - lib/wikiscript/version.rb
-- test/helper.rb
-- test/test_link.rb
-- test/test_page.rb
-- test/test_page_de.rb
-- test/test_page_reader.rb
-- test/test_table_reader.rb
 homepage: https://github.com/wikiscript/wikiscript
 licenses:
 - Public Domain
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options:
 - "--main"
 - README.md
@@ -109,16 +107,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.2
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
-summary: wikiscript - scripts for wikipedia (get wikitext for page etc.)
+summary: wikiscript - scripts for wikipedia (get wikitext for page, parse tables 'n'
+  links, etc.)
 test_files: []

data/LICENSE.md

@@ -1,116 +0,0 @@
-CC0 1.0 Universal
-
-Statement of Purpose
-
-The laws of most jurisdictions throughout the world automatically confer
-exclusive Copyright and Related Rights (defined below) upon the creator and
-subsequent owner(s) (each and all, an "owner") of an original work of
-authorship and/or a database (each, a "Work").
-
-Certain owners wish to permanently relinquish those rights to a Work for the
-purpose of contributing to a commons of creative, cultural and scientific
-works ("Commons") that the public can reliably and without fear of later
-claims of infringement build upon, modify, incorporate in other works, reuse
-and redistribute as freely as possible in any form whatsoever and for any
-purposes, including without limitation commercial purposes. These owners may
-contribute to the Commons to promote the ideal of a free culture and the
-further production of creative, cultural and scientific works, or to gain
-reputation or greater distribution for their Work in part through the use and
-efforts of others.
-
-For these and/or other purposes and motivations, and without any expectation
-of additional consideration or compensation, the person associating CC0 with a
-Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
-and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
-and publicly distribute the Work under its terms, with knowledge of his or her
-Copyright and Related Rights in the Work and the meaning and intended legal
-effect of CC0 on those rights.
-
-1. Copyright and Related Rights. A Work made available under CC0 may be
-protected by copyright and related or neighboring rights ("Copyright and
-Related Rights"). Copyright and Related Rights include, but are not limited
-to, the following:
-
-  i. the right to reproduce, adapt, distribute, perform, display, communicate,
-  and translate a Work;
-
-  ii. moral rights retained by the original author(s) and/or performer(s);
-
-  iii. publicity and privacy rights pertaining to a person's image or likeness
-  depicted in a Work;
-
-  iv. rights protecting against unfair competition in regards to a Work,
-  subject to the limitations in paragraph 4(a), below;
-
-  v. rights protecting the extraction, dissemination, use and reuse of data in
-  a Work;
-
-  vi. database rights (such as those arising under Directive 96/9/EC of the
-  European Parliament and of the Council of 11 March 1996 on the legal
-  protection of databases, and under any national implementation thereof,
-  including any amended or successor version of such directive); and
-
-  vii. other similar, equivalent or corresponding rights throughout the world
-  based on applicable law or treaty, and any national implementations thereof.
-
-2. Waiver. To the greatest extent permitted by, but not in contravention of,
-applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
-unconditionally waives, abandons, and surrenders all of Affirmer's Copyright
-and Related Rights and associated claims and causes of action, whether now
-known or unknown (including existing as well as future claims and causes of
-action), in the Work (i) in all territories worldwide, (ii) for the maximum
-duration provided by applicable law or treaty (including future time
-extensions), (iii) in any current or future medium and for any number of
-copies, and (iv) for any purpose whatsoever, including without limitation
-commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes
-the Waiver for the benefit of each member of the public at large and to the
-detriment of Affirmer's heirs and successors, fully intending that such Waiver
-shall not be subject to revocation, rescission, cancellation, termination, or
-any other legal or equitable action to disrupt the quiet enjoyment of the Work
-by the public as contemplated by Affirmer's express Statement of Purpose.
-
-3. Public License Fallback. Should any part of the Waiver for any reason be
-judged legally invalid or ineffective under applicable law, then the Waiver
-shall be preserved to the maximum extent permitted taking into account
-Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
-is so judged Affirmer hereby grants to each affected person a royalty-free,
-non transferable, non sublicensable, non exclusive, irrevocable and
-unconditional license to exercise Affirmer's Copyright and Related Rights in
-the Work (i) in all territories worldwide, (ii) for the maximum duration
-provided by applicable law or treaty (including future time extensions), (iii)
-in any current or future medium and for any number of copies, and (iv) for any
-purpose whatsoever, including without limitation commercial, advertising or
-promotional purposes (the "License"). The License shall be deemed effective as
-of the date CC0 was applied by Affirmer to the Work. Should any part of the
-License for any reason be judged legally invalid or ineffective under
-applicable law, such partial invalidity or ineffectiveness shall not
-invalidate the remainder of the License, and in such case Affirmer hereby
-affirms that he or she will not (i) exercise any of his or her remaining
-Copyright and Related Rights in the Work or (ii) assert any associated claims
-and causes of action with respect to the Work, in either case contrary to
-Affirmer's express Statement of Purpose.
-
-4. Limitations and Disclaimers.
-
-  a. No trademark or patent rights held by Affirmer are waived, abandoned,
-  surrendered, licensed or otherwise affected by this document.
-
-  b. Affirmer offers the Work as-is and makes no representations or warranties
-  of any kind concerning the Work, express, implied, statutory or otherwise,
-  including without limitation warranties of title, merchantability, fitness
-  for a particular purpose, non infringement, or the absence of latent or
-  other defects, accuracy, or the present or absence of errors, whether or not
-  discoverable, all to the greatest extent permissible under applicable law.
-
-  c. Affirmer disclaims responsibility for clearing rights of other persons
-  that may apply to the Work or any use thereof, including without limitation
-  any person's Copyright and Related Rights in the Work. Further, Affirmer
-  disclaims responsibility for obtaining any necessary consents, permissions
-  or other rights required for any use of the Work.
-
-  d. Affirmer understands and acknowledges that Creative Commons is not a
-  party to this document and has no duty or obligation with respect to this
-  CC0 or use of the Work.
-
-For more information, please see
-<http://creativecommons.org/publicdomain/zero/1.0/>

data/NOTES.md

@@ -1,52 +0,0 @@
-# Notes
-
-
-## Alternatives
-
-Wikipedia
-
-- [wikipedia-client](https://rubygems.org/gems/wikipedia-client) by Ken Pratt et al - ruby client for the Wikipedia API
-  - <https://github.com/kenpratt/wikipedia-client>
-  - <https://www.rubydoc.info/gems/wikipedia-client>
-
-<!-- break -->
-
-- [infoboxer](https://rubygems.org/gems/infoboxer) by Victor Shepelev et al - pure-Ruby Wikipedia (and generic MediaWiki) client and parser, targeting information extraction  
-  - <https://github.com/molybdenum-99/infoboxer>
-  - <https://www.rubydoc.info/gems/infoboxer>
-
-<!-- break -->
-
-More
-
-- <https://github.com/molybdenum-99/reality>
-- https://github.com/molybdenum-99/mediawiktory
-
-
-Wikidata
-
-- [wikidata](https://rubygems.org/gems/wikidata) by Wil Gieseler
-  - <https://github.com/wilg/wikidata>
-  - <https://www.rubydoc.info/gems/wikidata>
-
-<!-- break -->
-
-- [wikidata-fetcher](https://rubygems.org/gems/wikidata-fetcher)
-  - <https://github.com/everypolitician/wikidata-fetcher>
-
-<!-- break -->
-
-- [mediawiki_api-wikidata](https://rubygems.org/gems/mediawiki_api-wikidata)
-  - <https://github.com/wmde/WikidataApiGem>
-
-
-
-**Python**
-
-- <https://pypi.org/project/wptools/> - Wikipedia tools (for Humans)
-  - <https://github.com/siznax/wptools/>
-
-
-## Wikipedia
-
-  - Wikipedia API reference: <http://en.wikipedia.org/w/api.php>

data/test/helper.rb

@@ -1,8 +0,0 @@
-## $:.unshift(File.dirname(__FILE__))
-
-## minitest setup
-require 'minitest/autorun'
-
-
-## our own code
-require 'wikiscript'

data/test/test_link.rb

@@ -1,31 +0,0 @@
-# encoding: utf-8
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_link.rb
-
-
-require 'helper'
-
-
-class TestLink < MiniTest::Test
-
-  def test_unlink
-    assert_equal 'Santiago (La Florida)',  Wikiscript.unlink( '[[Santiago]] ([[La Florida, Chile|La Florida]])' )
-  end
-
-  def test_parse_link
-    link, title = Wikiscript.parse_link( '[[La Florida, Chile|La Florida]]' )
-    assert_equal 'La Florida, Chile', link
-    assert_equal 'La Florida',        title
-
-    link, title = Wikiscript.parse_link( '[[ La Florida, Chile | La Florida ]]' )
-    assert_equal 'La Florida, Chile', link
-    assert_equal 'La Florida',        title
-
-    link, title = Wikiscript.parse_link( 'La Florida' )
-    assert  link  == nil
-    assert  title == nil
-  end
-
-end # class TestLink

data/test/test_page.rb

@@ -1,54 +0,0 @@
-# encoding: utf-8
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_page.rb
-
-
-require 'helper'
-
-
-class TestPage < MiniTest::Test
-
-  def setup
-    Wikiscript.lang = :en
-  end
-
-  def test_austria_en
-    page = Wikiscript::Page.get( 'Austria' )
-# [debug] GET /w/index.php?action=raw&title=Austria uri=http://en.wikipedia.org/w/index.php?action=raw&title=Austria, redirect_limit=5
-# [debug] 301 TLS Redirect location=https://en.wikipedia.org/w/index.php?action=raw&title=Austria
-# [debug] GET /w/index.php?action=raw&title=Austria uri=https://en.wikipedia.org/w/index.php?action=raw&title=Austria, redirect_limit=4
-# [debug] 200 OK
-
-    text = page.text
-
-    ## print first 600 chars
-    pp text[0..600]
-
-    ## check for some snippets
-    assert /{{Infobox country/ =~ text
-    assert /common_name = Austria/ =~ text
-    assert /capital = \[\[Vienna\]\]/ =~ text
-    # assert /The origins of modern-day Austria date back to the time/ =~ text
-  end
-
-  def test_sankt_poelten_en
-    page = Wikiscript::Page.get( 'Sankt_Pölten' )
-# [debug] GET /w/index.php?action=raw&title=Sankt_P%C3%B6lten uri=http://en.wikipedia.org/w/index.php?action=raw&title=Sankt_P%C3%B6lten, redirect_limit=5
-# [debug] 301 TLS Redirect location=https://en.wikipedia.org/w/index.php?action=raw&title=Sankt_P%C3%B6lten
-# [debug] GET /w/index.php?action=raw&title=Sankt_P%C3%B6lten uri=https://en.wikipedia.org/w/index.php?action=raw&title=Sankt_P%C3%B6lten, redirect_limit=4
-# [debug] 200 OK
-
-    text = page.text
-
-    ## print first 600 chars
-    pp text[0..600]
-
-    ## check for some snippets
-    assert /{{Infobox settlement/ =~ text
-    assert /name\s+=\s+Sankt Pölten/ =~ text
-    # assert /'''Sankt Pölten''' \(''St. Pölten''\) is the capital city of/ =~ text
-  end
-
-end # class TestPage

data/test/test_page_de.rb

@@ -1,38 +0,0 @@
-# encoding: utf-8
-
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_page_de.rb
-
-
-require 'helper'
-
-
-class TestPageDe < MiniTest::Test
-
-  def setup
-    Wikiscript.lang = :de
-  end
-
-  def test_st_poelten_de
-    page = Wikiscript::Page.get( 'St._Pölten' )
-#    [debug] GET /w/index.php?action=raw&title=St._P%C3%B6lten uri=http://de.wikipedia.org/w/index.php?action=raw&title=St._P%C3%B6lten, redirect_limit=5
-#    [debug] 301 TLS Redirect location=https://de.wikipedia.org/w/index.php?action=raw&title=St._P%C3%B6lten
-#    [debug] GET /w/index.php?action=raw&title=St._P%C3%B6lten uri=https://de.wikipedia.org/w/index.php?action=raw&title=St._P%C3%B6lten, redirect_limit=4
-#    [debug] 200 OK
-
-
-    text = page.text
-
-    ## print first 600 chars
-    pp text[0..600]
-
-    ## check for some snippets
-    assert /{{Infobox Gemeinde in Österreich/ =~ text
-    assert /Name\s+=\s+St\. Pölten/ =~ text
-    assert /'''St\. Pölten''' \(amtlicher Name,/ =~ text
-    ## assert /Die Stadt liegt am Fluss \[\[Traisen \(Fluss\)\|Traisen\]\]/ =~ text
-  end
-
-end # class TestPageDe

data/test/test_page_reader.rb

@@ -1,80 +0,0 @@
-# encoding: utf-8
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_page_reader.rb
-
-
-require 'helper'
-
-
-class TestPageReader < MiniTest::Test
-
-  def test_basic
-    el = Wikiscript.parse( <<TXT )
-=Heading 1==
-==Heading 2==
-===Heading 3===
-
-{|
-|-
-! header1
-! header2
-! header3
-|-
-| row1cell1
-| row1cell2
-| row1cell3
-|-
-| row2cell1
-| row2cell2
-| row2cell3
-|}
-TXT
-
-    pp el
-
-    assert_equal 4, el.size
-    assert_equal [:h1, 'Heading 1'],  el[0]
-    assert_equal [:h2, 'Heading 2'],  el[1]
-    assert_equal [:h3, 'Heading 3'],  el[2]
-    assert_equal [:table, [['header1', 'header2', 'header3'],
-                           ['row1cell1', 'row1cell2', 'row1cell3'],
-                           ['row2cell1', 'row2cell2', 'row2cell3']]],  el[3]
-  end
-
-  def test_parse
-    page = Wikiscript::Page.new( <<TXT )
-=Heading 1==
-==Heading 2==
-===Heading 3===
-
-{|
-|-
-! header1
-! header2
-! header3
-|-
-| row1cell1
-| row1cell2
-| row1cell3
-|-
-| row2cell1
-| row2cell2
-| row2cell3
-|}
-TXT
-
-    el = page.parse
-    pp el
-
-    assert_equal 4, el.size
-    assert_equal [:h1, 'Heading 1'],  el[0]
-    assert_equal [:h2, 'Heading 2'],  el[1]
-    assert_equal [:h3, 'Heading 3'],  el[2]
-    assert_equal [:table, [['header1', 'header2', 'header3'],
-                           ['row1cell1', 'row1cell2', 'row1cell3'],
-                           ['row2cell1', 'row2cell2', 'row2cell3']]],  el[3]
-  end
-
-end # class TestPageReader

data/test/test_table_reader.rb

@@ -1,109 +0,0 @@
-# encoding: utf-8
-
-###
-#  to run use
-#     ruby -I ./lib -I ./test test/test_table_reader.rb
-
-
-require 'helper'
-
-class TestTableReader < MiniTest::Test
-
-  def test_basic
-    table = Wikiscript.parse_table( <<TXT )
-{|
-|-
-! header1
-! header2
-! header3
-|-
-| row1cell1
-| row1cell2
-| row1cell3
-|-
-| row2cell1
-| row2cell2
-| row2cell3
-|}
-TXT
-
-    assert_equal 3, table.size      ## three rows
-    assert_equal ['header1', 'header2', 'header3'], table[0]
-    assert_equal ['row1cell1', 'row1cell2', 'row1cell3'], table[1]
-    assert_equal ['row2cell1', 'row2cell2', 'row2cell3'], table[2]
-  end
-
-  def test_basic_ii     ##  with optional (missing) row divider before headers
-    table = Wikiscript.parse_table( <<TXT )
-{|
-! header1 !! header2 !! header3
-|-
-| row1cell1 || row1cell2 || row1cell3
-|-
-| row2cell1 || row2cell2 || row2cell3
-|}
-TXT
-
-    assert_equal 3, table.size      ## three rows
-    assert_equal ['header1', 'header2', 'header3'], table[0]
-    assert_equal ['row1cell1', 'row1cell2', 'row1cell3'], table[1]
-    assert_equal ['row2cell1', 'row2cell2', 'row2cell3'], table[2]
-  end
-
-  def test_basic_iii  # with continuing header column lines
-    table = Wikiscript.parse_table( <<TXT )
-{|
-|-
-!
-header1
-!
-header2
-!
-header3
-|-
-|
-row1cell1
-|
-row1cell2
-|
-row1cell3
-|-
-|
-row2cell1
-|
-row2cell2
-|
-row2cell3
-|}
-TXT
-
-    assert_equal 3, table.size      ## three rows
-    assert_equal ['header1', 'header2', 'header3'], table[0]
-    assert_equal ['row1cell1', 'row1cell2', 'row1cell3'], table[1]
-    assert_equal ['row2cell1', 'row2cell2', 'row2cell3'], table[2]
-  end
-
-
-  def test_strip_attributes_and_emphases
-    table = Wikiscript.parse_table( <<TXT )
-{|
-|-
-! style="width:200px;"|Club
-! style="width:150px;"|City
-|-
-|[[Biu Chun Rangers]]||[[Sham Shui Po]]
-|-
-|bgcolor=#ffff44 |''[[Eastern Sports Club|Eastern]]''||[[Mong Kok]]
-|-
-|[[HKFC Soccer Section]]||[[Happy Valley, Hong Kong|Happy Valley]]
-|}
-TXT
-
-assert_equal 4, table.size      ## four rows
-assert_equal ['Club', 'City'], table[0]
-assert_equal ['[[Biu Chun Rangers]]',            '[[Sham Shui Po]]'], table[1]
-assert_equal ['[[Eastern Sports Club|Eastern]]', '[[Mong Kok]]'],     table[2]
-assert_equal ['[[HKFC Soccer Section]]',         '[[Happy Valley, Hong Kong|Happy Valley]]'],  table[3]
-  end
-
-end # class TestTableReader