wikiscript 0.1.0 → 0.1.1

data/Manifest.txt

@@ -7,4 +7,5 @@ lib/wikiscript/client.rb
 lib/wikiscript/page.rb
 lib/wikiscript/version.rb
 test/helper.rb
-test/test_austria.rb
+test/test_page.rb
+test/test_page_de.rb

data/Rakefile

@@ -18,7 +18,8 @@ Hoe.spec 'wikiscript' do
   self.history_file = 'HISTORY.md'
 
   self.extra_deps = [
-    ['logutils' ]
+    ['logutils' ],
+    ['fetcher']
   ]
 
   self.licenses = ['Public Domain']

data/lib/wikiscript.rb

@@ -1,17 +1,18 @@
+# encoding: utf-8
+
 ## stdlibs
 
 require 'net/http'
 require 'uri'
-require 'json'
+require 'cgi'
 require 'pp'
-require 'ostruct'
 
 
 ## 3rd party gems/libs
 ## require 'props'
 
 require 'logutils'
-
+require 'fetcher'
 
 # our own code
 
@@ -29,7 +30,18 @@ module Wikiscript
   def self.root
     "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
   end
-  
+
+
+  ## for now make lang a global - change why? why not??
+  def self.lang=(value)
+    @@lang = value.to_s     # use to_s - lets you pass ing :en, :de etc.
+  end
+
+  def self.lang
+    # note: for now always returns a string e.g. 'en', 'de' etc. not a symbol
+    @@lang ||= 'en'
+  end
+
 end # module Wikiscript
 
 

data/lib/wikiscript/client.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 
 module Wikiscript
 
@@ -5,52 +6,65 @@ module Wikiscript
 
     include LogUtils::Logging
 
-    SITE_BASE = 'http://en.wikipedia.org/w/index.php'
+    SITE_BASE = 'http://{lang}.wikipedia.org/w/index.php'
 
     ### API_BASE  = 'http://en.wikipedia.org/w/api.php'
 
     def initialize( opts={} )
-      @opts = opts
+      @opts   = opts
+      @worker = Fetcher::Worker.new
     end
 
+    ## change to: wikitext why? why not? or to raw? why? why not?
     def text( title )
-      ## todo/fix: urlencode title ???
-      ## fix: use params hash!!!
-      get( "action=raw&title=#{title}" )
+      ## todo/fix: convert spaces to _ if not present for wikipedia page title - why ?? why not ???
+      get( action: 'raw', title: title )
     end
 
 private
-    ### fix: reuse code from fetcher gem!!!!
-    ##  do NOT duplicate!!! also cleanup jogo gem!!!!
+    def site_base
+      ## replace lang w/ lang config if present e.g.
+      ##   http://{lang}.wikipedia.org/w/index.php
+      # becomes
+      #   http://en.wikipedia.org/w/index.php  or
+      #   http://de.wikipedia.org/w/index.php  etc
+
+      SITE_BASE.gsub( "{lang}", Wikiscript.lang )
+    end
+
+    def build_query( h )
+      h.map do |k,v|
+        "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end.join( '&' )
+    end
 
     def get( params )
-      uri = URI.parse( "#{SITE_BASE}?#{params}" )
-
-    
-      # new code: honor proxy env variable HTTP_PROXY
-      proxy = ENV['HTTP_PROXY']
-      proxy = ENV['http_proxy'] if proxy.nil?   # try possible lower/case env variable (for *nix systems) is this necessary??
-    
-      if proxy
-        proxy = URI.parse( proxy )
-        logger.debug "using net http proxy: proxy.host=#{proxy.host}, proxy.port=#{proxy.port}"
-        if proxy.user && proxy.password
-          logger.debug "  using credentials: proxy.user=#{proxy.user}, proxy.password=****"
-        else
-          logger.debug "  using no credentials"
-        end
-      else
-        logger.debug "using direct net http access; no proxy configured"
-        proxy = OpenStruct.new   # all fields return nil (e.g. proxy.host, etc.)
-      end
+      # note: lets us passing in params as hash e.g.
+      #   action: 'raw', title: 'Austria'
+      #   key and values will get CGI escaped
+      query = build_query( params )
 
-      http_proxy = Net::HTTP::Proxy( proxy.host, proxy.port, proxy.user, proxy.password )
+      ## uri = URI.parse( "#{SITE_BASE}?#{params}" )
+      ## fix: pass in uri (add to fetcher check for is_a? URI etc.)
+      uri_string = "#{site_base}?#{query}"
 
-      http = http_proxy.new( uri.host, uri.port )
-      response = http.request( Net::HTTP::Get.new( uri.request_uri ))
+      response = @worker.get_response( uri_string )
 
       if response.code == '200'
         t = response.body
+        ###
+        # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+        # will mostly be ASCII
+        # - try to change encoding to UTF-8 ourselves
+        logger.debug "t.encoding.name (before): #{t.encoding.name}"
+        #####
+        # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+        ## NB:
+        # for now "hardcoded" to utf8 - what else can we do?
+        # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+        t = t.force_encoding( Encoding::UTF_8 )
+        logger.debug "t.encoding.name (after): #{t.encoding.name}"
         ## pp t
         t
       else

data/lib/wikiscript/page.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 
 module Wikiscript
 

data/lib/wikiscript/version.rb

@@ -1,5 +1,5 @@
 
 module Wikiscript
-   VERSION = '0.1.0'
+   VERSION = '0.1.1'
 end
 

data/test/test_page.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+
+require 'helper'
+
+
+class TestPage < MiniTest::Unit::TestCase
+
+  def setup
+    Wikiscript.lang = :en
+  end
+
+  def test_austria_en
+    page = Wikiscript::Page.new( 'Austria' )
+    text = page.text
+
+    ## print first 600 chars
+    pp text[0..600]
+
+    ## check for some snippets
+    assert( /{{Infobox country/ =~ text ) 
+    assert( /common_name = Austria/ =~ text )
+    assert( /capital = \[\[Vienna\]\]/ =~ text )
+    assert( /The origins of modern-day Austria date back to the time/ =~ text )
+  end
+
+  def test_sankt_poelten_en
+    page = Wikiscript::Page.new( 'Sankt_Pölten' )
+    text = page.text
+
+    ## print first 600 chars
+    pp text[0..600]
+
+    ## check for some snippets
+    assert( /{{Infobox Town AT/ =~ text ) 
+    assert( /Name\s+=\s+Sankt Pölten/ =~ text )
+    assert( /'''Sankt Pölten''' \(''St. Pölten''\) is the capital city of/ =~ text )
+  end
+
+end # class TestPage
+

data/test/test_page_de.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+
+require 'helper'
+
+
+class TestPageDe < MiniTest::Unit::TestCase
+
+  def setup
+    Wikiscript.lang = :de
+  end
+
+  def test_st_poelten_de
+    page = Wikiscript::Page.new( 'St._Pölten' )
+    text = page.text
+
+    ## print first 600 chars
+    pp text[0..600]
+
+    ## check for some snippets
+    assert( /{{Infobox Gemeinde in Österreich/ =~ text ) 
+    assert( /Name\s+=\s+St\. Pölten/ =~ text )
+    assert( /'''St\. Pölten''' \(amtlicher Name,/ =~ text )
+    assert( /Die Stadt liegt am Fluss \[\[Traisen \(Fluss\)\|Traisen\]\]/ =~ text )
+  end
+
+end # class TestPageDe
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wikiscript
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-04 00:00:00.000000000 Z
+date: 2014-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &73243800 !ruby/object:Gem::Requirement
+  requirement: &79133900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *73243800
+  version_requirements: *79133900
+- !ruby/object:Gem::Dependency
+  name: fetcher
+  requirement: &79133630 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *79133630
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &73243290 !ruby/object:Gem::Requirement
+  requirement: &79133350 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +43,10 @@ dependencies:
         version: '4.0'
   type: :development
   prerelease: false
-  version_requirements: *73243290
+  version_requirements: *79133350
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &73242780 !ruby/object:Gem::Requirement
+  requirement: &79133100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +54,7 @@ dependencies:
         version: '3.11'
   type: :development
   prerelease: false
-  version_requirements: *73242780
+  version_requirements: *79133100
 description: wikiscript - scripts for wikipedia (get wikitext for page etc.)
 email: opensport@googlegroups.com
 executables: []
@@ -62,7 +73,8 @@ files:
 - lib/wikiscript/page.rb
 - lib/wikiscript/version.rb
 - test/helper.rb
-- test/test_austria.rb
+- test/test_page.rb
+- test/test_page_de.rb
 - .gemtest
 homepage: https://github.com/wikiscript/wikiscript.ruby
 licenses:
@@ -92,4 +104,5 @@ signing_key:
 specification_version: 3
 summary: wikiscript - scripts for wikipedia (get wikitext for page etc.)
 test_files:
-- test/test_austria.rb
+- test/test_page_de.rb
+- test/test_page.rb

data/test/test_austria.rb

@@ -1,24 +0,0 @@
-# encoding: utf-8
-
-
-require 'helper'
-
-
-class TestAustria < MiniTest::Unit::TestCase
-
-  def test_text
-    page = Wikiscript::Page.new( 'Austria' )
-    text = page.text
-
-    ## print first 600 chars
-    pp text[0..600]
-
-    ## check for some snippets
-    assert( /{{Infobox country/ =~ text ) 
-    assert( /common_name = Austria/ =~ text )
-    assert( /capital = \[\[Vienna\]\]/ =~ text )
-    assert( /The origins of modern-day Austria date back to the time/ =~ text )
-  end
-
-end # class TestAustria
-