wiki-yggdrasil 0.1.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb439b993ec8d953183a78347ba0823ea19f93351315965e22df8765d157ecaa
-  data.tar.gz: dec29e7e24a77c1a15bbd0a8af3bac4b5e552cd2de0e4d267e156e05a94a2556
+  metadata.gz: c9987a3fa945f2a041c202f1f46b19855885cb76cf9b19b5b853eb63f401d100
+  data.tar.gz: 41a39a82cb8bbc5c4bea0aecc96fefeedc3ce671799653f8018b3d55d138abe4
 SHA512:
-  metadata.gz: 240261bfd3dd92c15ac716e50ab7951750dd9738560fbed4cf5ff5a6e7ed6a8ef3d1d3fe093fdfab5de66074b2562ed7db383a63128782c31d9662c84b8f9602
-  data.tar.gz: 819209e6592bc880931476265e8c6e5a9d365923966c5229868f4e5ddd6ce5759a0c710bb0e25de3a69ce241154ced34a4032ef87a695b7de4ec2f3a87042780
+  metadata.gz: 5f3faa407a997c9ad97581426782ea00b43ccf43fdcfc245ed219035048610c37f9a16e02f23694db8b7ef3cb48f54f709251cc6331649e2ffe07d089d169629
+  data.tar.gz: 59a856c97b0284194c5a7abbdb52e725dc0b8b1550d02bb05b30b7ebc367b57fa31ddf9fccd714fdc905e02bd03b8b107cf2d230c6a39a925f8e38e2fb35d418

data/.ruby-version

@@ -0,0 +1 @@
+2.6.1

data/README.md

@@ -1,10 +1,6 @@
 # Wiki::Yggdrasil
 ![Travis CI Build](https://travis-ci.org/alex0112/wiki-yggdrasil.svg?branch=master)
 
-You.  You're up late at night again reading up on some obscure mathematical topic.  You find yourself with *so many* open tabs on Wikipedia.  Wouldn't it be nice if you could just pick an article, and then view a tree of the articles it references?
-
-Introducing Wiki::Yggdrasil.  Named after the tree in Norse mythology that drinks from the well of all wisdom, Wiki::Yggdrasil is here to help you drink just as deeply from the well of wisdom that is Wikipedia.
-
 Wiki::Yggdrasil takes a Wikipedia URI as an argument, and proceeds to spider out a dependency tree of referenced articles.
 
 ## Usage
@@ -12,16 +8,33 @@ Wiki::Yggdrasil takes a Wikipedia URI as an argument, and proceeds to spider out
 require 'wiki/yggdrasil'
 
 @tree               = Wiki::Yggdrasil::Yggdrasil.new(uri: 'http://en.wikipedia.org/wiki/Yggdrasil')
-referenced_articles = @tree.children(depth: 3) ## A hash of of articles linked by the parent
+referenced_articles = @tree.children(depth: 1) ## A hash of of articles linked by the parent
+```
+The preceeding code produces a structure as follows:
+```ruby
+{:name=>"Help:IPA/English", :children=>[]}
+{:name=>"Help:IPA/English", :children=>[]}
+{:name=>"Old Norse", :children=>[]}
+{:name=>"Help:IPA", :children=>[]}
+{:name=>"Trees in mythology", :children=>[]}
+{:name=>"Norse cosmology", :children=>[]}
+{:name=>"<i>Poetic Edda</i>", :children=>[]}
+{:name=>"<i>Prose Edda</i>", :children=>[]}
+{:name=>"Snorri Sturluson", :children=>[]}
+{:name=>"<i>Fraxinus excelsior</i>", :children=>[]}
+{:name=>"Æsir", :children=>[]}
+{:name=>"Thing (assembly)", :children=>[]}
+{:name=>"Urðarbrunnr", :children=>[]}
+{:name=>"Hvergelmir", :children=>[]}
+{:name=>"Mímisbrunnr", :children=>[]}
+{:name=>"Níðhöggr", :children=>[]}
+{:name=>"Veðrfölnir and eagle", :children=>[]}
+{:name=>"Dáinn, Dvalinn, Duneyrr and Duraþrór", :children=>[]}
+{:name=>"Sacred trees and groves in Germanic paganism and mythology", :children=>[]}
+{:name=>"Ragnarök", :children=>[]}
 ```
 
-## FAQ
-
-### This is taking a long time.  Is that normal?
-Yes.  This is normal. Any Yggdrasil object created with a depth of three or higher will likely take a few minutes to scrape the necessary information.
-
-### Why didn't you just use Wikipedia's API?
-Wikipedia's API doesn't have an endpoint that allows you to programatically view the summary section of each article and its children.  If it did that would obviously be the ideal choice.
+_Note: Any Yggdrasil object created with a depth of three or higher will likely take a few minutes to scrape the necessary information._
 
 ## Installation
 

data/lib/wiki/article.rb

@@ -1,5 +1,6 @@
 require 'nokogiri'
 require 'open-uri'
+
 module Wiki::Yggdrasil
   
   class Article
@@ -10,34 +11,52 @@ module Wiki::Yggdrasil
       @uri         = uri
       @summary     = nil
       @child_links = nil
+      @name        = nil
+      @checksum    = nil
     end
-
+    
     def summary
       @summary ||= Nokogiri::HTML(Nokogiri::HTML(open(self.uri)).to_s.split('<div id="toc" class="toc">')[0]).css('p') ## TODO: Cleanup
     end
 
-    def child_links
+    def checksum
+      Digest::MD5.hexdigest(@summary.to_s)
+    end
+
+    def child_links(help: false)
       formatted_links = format_links
-      validated_links =  formatted_links.select { |uri| Wiki::Yggdrasil::Article.is_valid_wiki_article?(uri: uri) }
+      validated_links = formatted_links.select { |uri| Wiki::Yggdrasil::Article.is_valid_wiki_article?(uri: uri) }
+      
       @child_links  ||= validated_links
     end
 
-    def scrape_all_summary_links
-      self.summary.css('p a')
+    def scrape_links(help_links: false) ## TODO test help_links param in spec
+      help_links ? self.summary.css('p a') : self.summary.css('p a[href!="/wiki/Help:IPA/English"]')
     end
 
-    def format_links(anchors: self.scrape_all_summary_links)
+    def name
+      @name ||= Nokogiri::HTML(open(self.uri)).css('#firstHeading').inner_html
+      ## TODO: Cleanup
+    end
+    
+    def format_links(anchors: self.scrape_links)
       uris = anchors.map do |anchor|
         anchor.nil? || anchor['href'].nil? ? next : 'https://en.wikipedia.org' << anchor['href'] ## nil href attributes are often self refs (but possibly not always). Ignore them.
+        ## TODO: take care of this in .scrape_links with a css selector (like the Help:IPA links)
       end
 
       uris.compact
     end
+
+    def self.remove_italic_tags(uri_list)
+      
+    end
     
     def self.is_valid_wiki_article?(uri:)
-      ## Is this URI a wikipedia article?
       uri =~ /.*wikipedia\.org\/wiki\/.+/ ? true : false
     end
     
   end
+
+
 end

data/lib/wiki/yggdrasil.rb

@@ -13,21 +13,31 @@ module Wiki
       end
 
       def children(depth: 4, article_children: self.root.child_links)
-        get_children = lambda do |depth, article_children|
-          article_children.each_with_object({}) do |uri, tree|
-            if (depth == 1)
-              tree[uri] = nil
-            else
-              article   = Wiki::Yggdrasil::Article.new(uri: uri)
-              @children = tree
-              tree[uri] = get_children.call(depth - 1, article.child_links)
-            end
+        @children ||= { name: self.root.name, children: recursive_scrape(depth: depth), index: 0, depth: 0 }
+      end
+
+      def recursive_scrape(depth: 1, children: @root.child_links)
+        children.each_with_index.map do |uri, index|
+          article = Wiki::Yggdrasil::Article.new(uri: uri)
+          if (depth == 1)
+            {
+              name: article.name,
+              index: index + 1,
+              level: depth,
+              children: [],
+            }
+          else
+            {
+              name: article.name,
+              index: index + 1,
+              level: depth,
+              children: recursive_scrape(depth - 1, article.child_links),
+            }
           end
         end
-
-        @children ||= get_children.call(depth, article_children)
       end
-
     end
+
   end
+
 end

data/lib/wiki/yggdrasil/version.rb

@@ -1,5 +1,5 @@
 module Wiki
   module Yggdrasil
-    VERSION = "0.1.0"
+    VERSION = "0.4.0"
   end
 end

data/wiki-yggdrasil.gemspec

@@ -30,9 +30,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "nokogiri", "~> 1.8.2"
-  
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_runtime_dependency "nokogiri", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wiki-yggdrasil
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.4.0
 platform: ruby
 authors:
 - alex0112
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-12 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -16,28 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.2
+        version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.2
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.16'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -76,6 +62,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".ruby-version"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt
@@ -107,8 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Scrape Wikipedia articles and generate a json tree