infoboxer 0.3.1.pre → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ebc38c153d481aca588625caf2bcb576046afa9
-  data.tar.gz: 4f0ccc3b3130403a7a1e35adfc3977d954ac5a66
+  metadata.gz: 67bb7aed02bc7048e3508902a5b921be691b8bac
+  data.tar.gz: 0c1b8de5e72f824f29802d5a1e202e914ad3c241
 SHA512:
-  metadata.gz: 4593f04e93b2714f13f9abb68cb42cec84ad258d786c815a99ca95d150ba55d597106d1e9e71d3802177cce4a4ad5bf1de0b1f5fa4cc733af19d673fbc600945
-  data.tar.gz: 1dc93e4fc257a4cee3fd2c9eb263b24f1a9d8b755a3765bfdf71d1617f7ba9a4ee7d9323502dbee23e48dc85bb6899facb7eadcc62ccc386d958dfca6f28fe5e
+  metadata.gz: ee0babd55c7fe433dc3b55aa6988abf17ddee81424f56bd83b31dcb076dcdcfd22c966b50a26b979e5a917e096847bdaf9390063807694133734671201cb5003
+  data.tar.gz: b70854401485cb7981110c3da6a5f7d4b4623545b3b1af4b05c4b541514efa9727818a09c25d89dbe55b31a79c2e772adedf06f2b1211386f17242ddb11cde1a

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Infoboxer's change log
 
+## 0.3.1 (2017-12-04)
+
+* (Experimental) new representation of templates, much more readable;
+* More access to querying process and underlying `MediaWiktory::Wikipedia::Query`;
+* Finally, `limit` parameter for multi-page queries (category, search, prefixsearch).
+
 ## 0.3.1.pre (2017-09-16)
 
 * Introduce interwiki links following (and proper handling of interwikis, in general);

data/infoboxer.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   s.executables << 'infoboxer'
 
   s.add_dependency 'htmlentities'
-  s.add_dependency 'mediawiktory', '>= 0.1.0'
+  s.add_dependency 'mediawiktory', '= 0.1.2'
   s.add_dependency 'addressable'
   s.add_dependency 'terminal-table'
 end

data/lib/infoboxer.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 # Main client module for entire infoboxer functionality. If you're lucky,
 # there's no other classes/modules you need to instantiate or call
 # directly. You just do:

data/lib/infoboxer/core_ext.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 # @private
 class Object
   # Unfortunately, not in backports gem still :(

data/lib/infoboxer/definitions/en.wikipedia.org.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   MediaWiki::Traits.for('en.wikipedia.org') do
     templates do

data/lib/infoboxer/media_wiki.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'mediawiktory'
 require 'addressable/uri'
 
@@ -61,24 +59,24 @@ module Infoboxer
     # classes).
     #
     # @param titles [Array<String>] List of page titles to get.
-    # @param prop [Array<Symbol>] List of additional page properties to get, refer to
-    #   [MediaWiktory::Actions::Query#prop](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query#prop-instance_method)
-    #   for the list of available properties.
+    # @param processor [Proc] Optional block to preprocess MediaWiktory query. Refer to
+    #   [MediaWiktory::Actions::Query](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query)
+    #   for its API. Infoboxer assumes that the block returns new instance of `Query`, so be careful
+    #   while using it.
     #
     # @return [Hash{String => Hash}] Hash of `{requested title => raw MediaWiki object}`. Note that
     #   even missing (does not exist in current Wiki) or invalid (impossible title) still be present
     #   in response, just will have `"missing"` or `"invalid"` key, just like MediaWiki returns them.
-    def raw(*titles, prop: [])
+    def raw(*titles, &processor)
       # could emerge on "automatically" created page lists, should work
       return {} if titles.empty?
 
       titles.each_slice(50).map do |part|
-        response = @client
-                   .query
-                   .titles(*part)
-                   .prop(:revisions, :info, *prop).prop(:content, :timestamp, :url)
-                   .redirects
-                   .response
+        request = prepare_request(@client.query.titles(*part), &processor)
+        response = request.response
+
+        # If additional props are required, there may be additional pages, even despite each_slice(50)
+        response = response.continue while response.continue?
 
         sources = response['pages'].values.map { |page| [page['title'], page] }.to_h
         redirects =
@@ -102,9 +100,11 @@ module Infoboxer
     # `(titles.count / 50.0).ceil` requests)
     #
     # @param titles [Array<String>] List of page titles to get.
-    # @param prop [Array<Symbol>] List of additional page properties to get, refer to
-    #   [MediaWiktory::Actions::Query#prop](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query#prop-instance_method)
-    #   for the list of available properties.
+    # @param interwiki [Symbol] Identifier of other wiki, related to current, to fetch pages from.
+    # @param processor [Proc] Optional block to preprocess MediaWiktory query. Refer to
+    #   [MediaWiktory::Actions::Query](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query)
+    #   for its API. Infoboxer assumes that the block returns new instance of `Query`, so be careful
+    #   while using it.
     #
     # @return [Page, Tree::Nodes<Page>] array of parsed pages. Notes:
     #   * if you call `get` with only one title, one page will be
@@ -122,10 +122,10 @@ module Infoboxer
     #     and obtain meaningful results instead of `NoMethodError` or
     #     `SomethingNotFound`.
     #
-    def get(*titles, prop: [], interwiki: nil)
-      return interwikis(interwiki).get(*titles, prop: prop) if interwiki
+    def get(*titles, interwiki: nil, &processor)
+      return interwikis(interwiki).get(*titles, &processor) if interwiki
 
-      pages = get_h(*titles, prop: prop).values.compact
+      pages = get_h(*titles, &processor).values.compact
       titles.count == 1 ? pages.first : Tree::Nodes[*pages]
     end
 
@@ -142,14 +142,15 @@ module Infoboxer
     # you've received.
     #
     # @param titles [Array<String>] List of page titles to get.
-    # @param prop [Array<Symbol>] List of additional page properties to get, refer to
-    #   [MediaWiktory::Actions::Query#prop](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query#prop-instance_method)
-    #   for the list of available properties.
+    # @param processor [Proc] Optional block to preprocess MediaWiktory query. Refer to
+    #   [MediaWiktory::Actions::Query](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query)
+    #   for its API. Infoboxer assumes that the block returns new instance of `Query`, so be careful
+    #   while using it.
     #
     # @return [Hash<String, Page>]
     #
-    def get_h(*titles, prop: [])
-      raw_pages = raw(*titles, prop: prop)
+    def get_h(*titles, &processor)
+      raw_pages = raw(*titles, &processor)
                   .tap { |ps| ps.detect { |_, p| p['invalid'] }.tap { |_, i| i && fail(i['invalidreason']) } }
                   .reject { |_, p| p.key?('missing') }
       titles.map { |title| [title, make_page(raw_pages, title)] }.to_h
@@ -157,59 +158,59 @@ module Infoboxer
 
     # Receive list of parsed MediaWiki pages from specified category.
     #
-    # **NB**: currently, this API **always** fetches all pages from
-    # category, there is no option to "take first 20 pages". Pages are
-    # fetched in 50-page batches, then parsed. So, for large category
-    # it can really take a while to fetch all pages.
-    #
     # @param title [String] Category title. You can use namespaceless title (like
     #     `"Countries in South America"`), title with namespace (like
     #     `"Category:Countries in South America"`) or title with local
     #     namespace (like `"Catégorie:Argentine"` for French Wikipedia)
+    # @param limit [Integer, "max"]
+    # @param processor [Proc] Optional block to preprocess MediaWiktory query. Refer to
+    #   [MediaWiktory::Actions::Query](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query)
+    #   for its API. Infoboxer assumes that the block returns new instance of `Query`, so be careful
+    #   while using it.
     #
     # @return [Tree::Nodes<Page>] array of parsed pages.
     #
-    def category(title)
+    def category(title, limit: 'max', &processor)
       title = normalize_category_title(title)
 
-      list(@client.query.generator(:categorymembers).title(title).limit('max'))
+      list(@client.query.generator(:categorymembers).title(title), limit, &processor)
     end
 
     # Receive list of parsed MediaWiki pages for provided search query.
     # See [MediaWiki API docs](https://www.mediawiki.org/w/api.php?action=help&modules=query%2Bsearch)
     # for details.
     #
-    # **NB**: currently, this API **always** fetches all pages from
-    # category, there is no option to "take first 20 pages". Pages are
-    # fetched in 50-page batches, then parsed. So, for large search query
-    # it can really take a while to fetch all pages.
-    #
     # @param query [String] Search query. For old installations, look at
     #     https://www.mediawiki.org/wiki/Help:Searching
     #     for search syntax. For new ones (including Wikipedia), see at
     #     https://www.mediawiki.org/wiki/Help:CirrusSearch.
+    # @param limit [Integer, "max"]
+    # @param processor [Proc] Optional block to preprocess MediaWiktory query. Refer to
+    #   [MediaWiktory::Actions::Query](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query)
+    #   for its API. Infoboxer assumes that the block returns new instance of `Query`, so be careful
+    #   while using it.
     #
     # @return [Tree::Nodes<Page>] array of parsed pages.
     #
-    def search(query)
-      list(@client.query.generator(:search).search(query).limit('max'))
+    def search(query, limit: 'max', &processor)
+      list(@client.query.generator(:search).search(query), limit, &processor)
     end
 
     # Receive list of parsed MediaWiki pages with titles startin from prefix.
     # See [MediaWiki API docs](https://www.mediawiki.org/w/api.php?action=help&modules=query%2Bprefixsearch)
     # for details.
     #
-    # **NB**: currently, this API **always** fetches all pages from
-    # category, there is no option to "take first 20 pages". Pages are
-    # fetched in 50-page batches, then parsed. So, for large search query
-    # it can really take a while to fetch all pages.
-    #
     # @param prefix [String] Page title prefix.
+    # @param limit [Integer, "max"]
+    # @param processor [Proc] Optional block to preprocess MediaWiktory query. Refer to
+    #   [MediaWiktory::Actions::Query](http://www.rubydoc.info/gems/mediawiktory/MediaWiktory/Wikipedia/Actions/Query)
+    #   for its API. Infoboxer assumes that the block returns new instance of `Query`, so be careful
+    #   while using it.
     #
     # @return [Tree::Nodes<Page>] array of parsed pages.
     #
-    def prefixsearch(prefix)
-      list(@client.query.generator(:prefixsearch).search(prefix).limit('max'))
+    def prefixsearch(prefix, limit: 'max', &processor)
+      list(@client.query.generator(:prefixsearch).search(prefix), limit, &processor)
     end
 
     # @return [String]
@@ -225,14 +226,11 @@ module Infoboxer
       Page.new(self, Parser.paragraphs(source['revisions'].first['*'], traits), source)
     end
 
-    def list(query)
-      response = query
-                 .prop(:revisions, :info)
-                 .prop(:content, :timestamp, :url)
-                 .redirects
-                 .response
+    def list(query, limit, &processor)
+      request = prepare_request(query.limit(limit), &processor)
+      response = request.response
 
-      response = response.continue while response.continue?
+      response = response.continue while response.continue? && (limit == 'max' || response['pages'].count < limit)
 
       return Tree::Nodes[] if response['pages'].nil?
 
@@ -243,6 +241,11 @@ module Infoboxer
       Tree::Nodes[*pages]
     end
 
+    def prepare_request(request)
+      request = request.prop(:revisions, :info).prop(:content, :timestamp, :url).redirects
+      block_given? ? yield(request) : request
+    end
+
     def normalize_category_title(title)
       # FIXME: shouldn't it go to MediaWiktory?..
       namespace, titl = title.include?(':') ? title.split(':', 2) : [nil, title]

data/lib/infoboxer/media_wiki/page.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class MediaWiki
     # A descendant of {Tree::Document Document}, representing page,

data/lib/infoboxer/media_wiki/traits.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class MediaWiki
     # DSL for defining "traits" for some site.

data/lib/infoboxer/navigation.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   # Navigation is one of the things Infoboxer is proud about. It tries
   # to be logical, unobtrusive and compact.

data/lib/infoboxer/navigation/lookup.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require_relative 'selector'
 
 module Infoboxer

data/lib/infoboxer/navigation/sections.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Navigation
     # `Sections` module provides logical view on document strcture.
@@ -81,6 +79,18 @@ module Infoboxer
           end
         end
 
+        def subsections(*names)
+          sections = names.map { |name|
+            heading = lookup_children(:Heading, text_: name).first
+            next unless heading
+            body = heading.next_siblings
+                          .take_while { |n| !n.is_a?(Tree::Heading) || n.level > heading.level }
+
+            Section.new(heading, body)
+          }.compact
+          Tree::Nodes.new(sections)
+        end
+
         def lookup_children(*arg)
           if arg.include?(:Section)
             sections.find(*(arg - [:Section]))
@@ -138,7 +148,7 @@ module Infoboxer
           end
 
           body = heading.next_siblings
-                        .take_while { |n| !n.is_a?(Tree::Heading) || n.level < heading.level }
+                        .take_while { |n| !n.is_a?(Tree::Heading) || n.level > heading.level }
 
           section = Section.new(heading, body)
           @in_sections = Tree::Nodes[section, *heading.in_sections]

data/lib/infoboxer/navigation/selector.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Navigation
     module Lookup

data/lib/infoboxer/navigation/shortcuts.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Navigation
     # See {Shortcuts::Node Shortcuts::Node} for everything!

data/lib/infoboxer/navigation/wikipath.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require_relative '../wiki_path'
 
 module Infoboxer

data/lib/infoboxer/parser.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'ostruct'
 require 'logger'
 

data/lib/infoboxer/parser/context.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'strscan'
 
 module Infoboxer

data/lib/infoboxer/parser/html.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     module HTML

data/lib/infoboxer/parser/image.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     module Image

data/lib/infoboxer/parser/inline.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     module Inline

data/lib/infoboxer/parser/paragraphs.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     module Paragraphs

data/lib/infoboxer/parser/table.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     # http://en.wikipedia.org/wiki/Help:Table

data/lib/infoboxer/parser/template.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     module Template

data/lib/infoboxer/parser/util.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   class Parser
     module Util

data/lib/infoboxer/templates/set.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Templates
     # Base class for defining set of templates, used for some site/domain.

data/lib/infoboxer/tree.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   # Infoboxer provides you with tree structure of the Wikipedia page,
   # which you can introspect and navigate with ease. This tree structure

data/lib/infoboxer/tree/compound.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Base class for all nodes with children.

data/lib/infoboxer/tree/document.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents entire document.

data/lib/infoboxer/tree/gallery.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents gallery of images (contents of `<gallery>` special tag).

data/lib/infoboxer/tree/html.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     module HTMLTagCommons

data/lib/infoboxer/tree/image.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents image (or other media file).

data/lib/infoboxer/tree/inline.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents italic text.

data/lib/infoboxer/tree/list.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents item of ordered or unordered list.

data/lib/infoboxer/tree/node.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'htmlentities'
 
 module Infoboxer
@@ -162,7 +160,7 @@ module Infoboxer
       end
 
       def _eq(_other)
-        fail(NotImplementedError, "#_eq should be defined in subclasses (called for #{self.class})")
+        false
       end
 
       def decode(str)

data/lib/infoboxer/tree/nodes.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # List of nodes, which tries to be useful both as array, and as proxy
@@ -153,6 +151,12 @@ module Infoboxer
         map(&:text).join
       end
 
+      alias_method :to_s, :text
+
+      def unwrap
+        map { |n| n.respond_to?(:unwrap) ? n.unwrap : n }
+      end
+
       # Fetches pages by ALL wikilinks inside in ONE query to MediaWiki
       # API.
       #

data/lib/infoboxer/tree/paragraphs.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Base class for all "paragraph-level" nodes: {Paragraph}, {ListItem},

data/lib/infoboxer/tree/ref.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents footnote.

data/lib/infoboxer/tree/table.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'terminal-table'
 
 module Infoboxer

data/lib/infoboxer/tree/template.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require_relative 'linkable'
 
 module Infoboxer
@@ -119,7 +117,12 @@ module Infoboxer
       end
 
       def text
-        ''
+        res = unnamed_variables.map(&:text).join('|')
+        res.empty? ? '' : "{#{name}:#{res}}"
+      end
+
+      def unwrap
+        unnamed_variables.flat_map(&:children).unwrap
       end
 
       # See {Node#to_tree}

data/lib/infoboxer/tree/text.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Infoboxer
   module Tree
     # Represents plain text node.

data/lib/infoboxer/tree/wikilink.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require_relative 'linkable'
 
 module Infoboxer

data/lib/infoboxer/version.rb

@@ -1,9 +1,7 @@
-# encoding: utf-8
-
 module Infoboxer
   MAJOR = 0
   MINOR = 3
   PATCH = 1
-  PRE = 'pre'.freeze # set to `nil` for normal releases
+  PRE = nil
   VERSION = [MAJOR, MINOR, PATCH, PRE].compact.join('.')
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: infoboxer
 version: !ruby/object:Gem::Version
-  version: 0.3.1.pre
+  version: 0.3.1
 platform: ruby
 authors:
 - Victor Shepelev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-16 00:00:00.000000000 Z
+date: 2017-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: htmlentities
@@ -28,16 +28,16 @@ dependencies:
   name: mediawiktory
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -166,9 +166,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.10