hermaeus 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21e1bdd247e29c25253a077f728526d215f91a14
-  data.tar.gz: 8a01184c5502a2a9f03d1c82ff0d20b7664bc27b
+  metadata.gz: ca26f9776acb76bf0097fa0c43b18d1563890084
+  data.tar.gz: fb9835383d6e290d90cf8cfa9191033031e469de
 SHA512:
-  metadata.gz: 86d808111b779e4a78ec52727e589e7dda9f9f102424fbd4ad1ac9aa3183c5a0989c63023133580970ec5a421e9c6ae3d8865948e396c0d4270041c789089e3f
-  data.tar.gz: 92660f2b99ea6cc07057606620be793ab62947edd64d4a4e2477e5cda3607fb2578b7475de8c0d194cdfe5f465a2bb33b6d12a0b92db8993cf428a9b167472c2
+  metadata.gz: 6111c2eee01f5c4122aa1542510aa8610a2a4542a053c470a7aa333be282dd7616c7f40f3121700a2b8ddb997adbf1cd368a0998641b15bd900603247159d4ed
+  data.tar.gz: f81f1158506b2eed59c30a29258cfafefdcf41b3b1f7fdca584a814c4fdf7d87b0d9b81159c80c92f8c9afdc83405f846fdd4cbd0ee97ec82e411be8572453f7

data/CHANGELOG.md

@@ -2,18 +2,29 @@
 
 ## v1
 
-### v1.0.2
+### v1.1
+
+Move the index page out of a magic string and into the configuration file.
+
+Update the configuration handler to account for the new information, and provide
+some more helpful error messages on validation failure.
+
+Update the Changelog structure slightly.
+
+### v1.0
+
+#### v1.0.2
 
 Fix a File.open call; it was accidentally being given a string permission set
 instead of numeric.
 
-### v1.0.1
+#### v1.0.1
 
 Deployment to a separate machine brought some hidden bugs to light. Fixed a
 syntax error in the initialization routines and added documentation about fresh
 installations to the README.
 
-### v1.0.0
+#### v1.0.0
 
 Added a storage backend (`Apocryphon` and `Archivist` classes) capable of
 formatting the retrieved text and storing it on disk.
@@ -21,16 +32,16 @@ formatting the retrieved text and storing it on disk.
 `mora` is confirmed to work in the wild, and so Hermaeus is ready for a 1.0
 release.
 
-## v0
+### v0
 
 Development versions used only for experimentation.
 
-### v0.2.0
+#### v0.2.0
 
 Completed the ability to retrieve texts from reddit and process them enough for
 demonstration purposes.
 
-### v0.1.0
+#### v0.1.0
 
 Initial version -- Gained the ability to connect to reddit and retrieve basic
 information.

data/data/config.toml

@@ -16,3 +16,7 @@ password = "hunter2"
 [archive]
 # The path where Hermaeus dumps output files.
 path = "archive"
+
+# Settings for the /r/teslore index page
+[index]
+path = "/r/teslore/wiki/archive"

data/lib/hermaeus.rb

@@ -18,8 +18,17 @@ module Hermaeus
 	def self.init
 		FileUtils.mkdir_p(Config::DIR)
 		if File.exist? Config::FILE
-			@cfg = Config.load
-			Config.validate @cfg
+			Config.load
+			begin
+				Config.validate!
+			rescue ConfigurationError => e
+				puts <<-EOS
+#{e.message}
+
+Edit your configuration file (#{File.join Config::DIR, "config.toml"}) to \
+continue.
+				EOS
+			end
 		else
 			File.open Config::FILE, "w+" do |file|
 				File.open File.expand_path(Config::SOURCE), "r", 0600 do |cfg|
@@ -35,7 +44,7 @@ for Hermaeus to function.
 
 	# Public: Connects Hermaeus to reddit.
 	def self.connect
-		@client = Client.new @cfg[:client]
+		@client = Client.new
 	end
 
 	# Public: Downloads Apocrypha posts.

data/lib/hermaeus/client.rb

@@ -4,6 +4,8 @@
 	redd
 ].each(&method(:require))
 
+include Enumerable
+
 require "hermaeus/config"
 require "hermaeus/version"
 
@@ -13,13 +15,10 @@ module Hermaeus
 	class Client
 		USER_AGENT = "Redd/Ruby:Hermaeus:#{Hermaeus::VERSION} (by /u/myrrlyn)"
 		# Public: Connects the Hermaeus::Client to reddit.
-		#
-		# info - A Hash with Symbol keys containing reddit connection information.
-		# It should be the `[:client]` section of the Hash returned by
-		# `Hermaeus::Config.load`.
-		def initialize client
-			Config.validate client: client
-			@client = Redd.it(client.delete(:type).to_sym, *client.values, user_agent: USER_AGENT)
+		def initialize
+			Config.validate!
+			cfg = Config.info[:client]
+			@client = Redd.it(cfg.delete(:type).to_sym, *cfg.values, user_agent: USER_AGENT)
 			@client.authorize!
 			@html_filter = HTMLEntities.new
 		end
@@ -28,7 +27,7 @@ module Hermaeus
 		#
 		# Wraps Client#scrape_index; see it for documentation.
 		def get_global_listing **opts
-			scrape_index "/r/teslore/wiki/compilation", opts
+			scrape_index Config.info[:index][:path], opts
 		end
 
 		# Public: Scrapes a Weekly Community Thread patch index.
@@ -46,6 +45,8 @@ module Hermaeus
 			ids.map! do |id|
 			 "t3_#{id}" unless id.match /^t3_/
 			end
+			# TODO: Ensure that this is safe (only query <= 100 IDs at a time), and
+			# call the scraper multiple times and reassemble output if necessary.
 			query = "/by_id/#{ids.join(",")}"
 			scrape_index query, opts
 		end
@@ -62,6 +63,7 @@ module Hermaeus
 		# Returns a String Array containing the reddit fullnames harvested from the
 		# input list. Input elements that do not match are stripped.
 		def get_fullnames data, **opts
+			# TODO: Move this regex to the configuration file.
 			regex = opts[:regex] || %r(/r/.+/(comments/)?(?<id>[0-9a-z]+)/.+)
 			data.map do |item|
 				m = item.match regex
@@ -89,25 +91,11 @@ module Hermaeus
 			ret = []
 			# reddit has finite limits on acceptable query sizes. Split the list into
 			# manageable portions
-			fullnames.fracture.each do |chunk|
+			fullnames.each_slice(100).each do |chunk|
 				# Assemble the list of reddit objects being queried
-				query = chunk.join(",")
-				# Ask reddit to procure our items
-				response = @client.get("/by_id/#{query}.json")
-				if response.success?
-					payload = response.body
-					# The payload should be a Listing even for a single-item query; the
-					# :children array will just have one element.
-					if payload[:kind] == "Listing"
-						payload[:data][:children].each do |item|
-							yield item[:data]
-						end
-					# else
-					end
-					ret << payload
-				end
-				# Keep the rate limiter happy
-				sleep 1
+				query = "/by_id/#{chunk.join(",")}.json"
+				response = scrape_posts query, &block
+				ret << response.body
 			end
 			ret
 		end
@@ -175,40 +163,40 @@ module Hermaeus
 			end
 			.flatten
 		end
-	end
-end
 
-class Array
-	# Public: Splits an Array into several arrays, each of which has a maximum
-	# size.
-	#
-	# size - The maximum length of each segment. Defaults to 100.
-	#
-	# Returns an Array of Arrays. Each element of the returned array is a section
-	# of the original array.
-	#
-	# Examples
-	#
-	# %w[a b c d e f g h i j k l m n o p q r s t u v w x y z].fracture 5
-	# => [
-	#   ["a", "b", "c", "d", "e"],
-	#   ["f", "g", "h", "i", "j"],
-	#   ["k", "l", "m", "n", "o"],
-	#   ["p", "q", "r", "s", "t"],
-	#   ["u", "v", "w", "x", "y"],
-	#   ["z"]
-	# ]
-	# %w[hello world].fracture 5 => [["hello", "world"]]
-	def fracture size = 100
-		if self.length < size
-			[self]
-		else
-			ret = []
-			self.each_with_index do |val, idx|
-				ret[idx / size] ||= []
-				ret[idx / size] << val
+		# Internal: Provides the actual functionality for collecting posts.
+		#
+		# query - The reddit API endpoint or path being queried.
+		# opts - Options for the reddit API call
+		# block - This method yields each post fetched to its block.
+		# tries - hidden parameter used to prevent infinite stalling on rate limits.
+		#
+		# Returns reddit's response to the query.
+		def scrape_posts query, tries = 0, **opts, &block
+			begin
+				# Ask reddit to procure our items
+				response = @client.get(query, opts)
+				if response.success?
+					payload = response.body
+					# The payload should be a Listing even for a single-item query; the
+					# :children array will just have one element.
+					if payload[:kind] == "Listing"
+						payload[:data][:children].each do |item|
+							yield item[:data]
+						end
+					end
+					return response
+				end
+			# If at first you don't succeed...
+			rescue Redd::Error::RateLimited => e
+				sleep e.time + 1
+				# Try try again.
+				if tries < 3
+					scrape_posts query, tries + 1
+				else
+					raise RuntimeError, "reddit rate limit will not unlock"
+				end
 			end
-			ret
 		end
 	end
 end

data/lib/hermaeus/config.rb

@@ -14,28 +14,34 @@ module Hermaeus
 		# List of allowed types a reddit client can take
 		ALLOWED_TYPES = %w[script web userless installed]
 
+		# Public: Accessor for the loaded and parsed information.
+		#
+		# Returns nil if the config file has yet to be processed.
+		def self.info
+			@info
+		end
+
 		# Public: Load a configuration file into memory
 		#
 		# Returns the configuration file represented as a Hash with Symbol keys
 		def self.load
-			Tomlrb.load_file FILE, symbolize_keys: true
+			@info = Tomlrb.load_file FILE, symbolize_keys: true
 		end
 
 		# Public: Performs validation checks on a configuration structure
 		#
-		# cfg - A Hash with Symbol keys to check for validity
-		#
 		# Returns true if the configuration argument is valid
 		#
 		# Raises a ConfigurationError if the configuration is invalid, with an
 		# error message describing the failure.
-		def self.validate cfg
-			unless cfg.has_key? :client
-				raise ConfigurationError.new <<-EOS
+		def self.validate!
+			# Validate the [client] section.
+			raise ConfigurationError.new(<<-EOS) unless @info.has_key? :client
 Hermaeus’ configuration file must contain a [client] section.
-				EOS
-			end
-			unless cfg[:client].has_key?(:type) && ALLOWED_TYPES.include?(cfg[:client][:type])
+			EOS
+
+			# Validate the [client] section’s type field.
+			unless @info[:client].has_key?(:type) && ALLOWED_TYPES.include?(@info[:client][:type])
 				raise ConfigurationError.new <<-EOS
 Hermaeus’ [client] section must include a type key whose value is one of:
 #{ALLOWED_TYPES.join(", ")}.
@@ -44,7 +50,9 @@ Hermaeus’ [client] section must include a type key whose value is one of:
 type = "one of the listed types"
 				EOS
 			end
-			unless cfg[:client].has_key?(:id) && cfg[:client].has_key?(:secret)
+
+			# Validate the [client] section’s id and secret fields.
+			unless @info[:client].has_key?(:id) && @info[:client].has_key?(:secret)
 				raise ConfigurationError.new <<-EOS
 Hermaeus’ [client] section must include keys for the ID and secret provided by
 reddit for your application.
@@ -54,8 +62,10 @@ id = "an ID from reddit"
 secret = "a secret from reddit"
 				EOS
 			end
-			if cfg[:client][:type] == "script"
-				client = cfg[:client]
+
+			# Validate the [client] section’s username and password fields, if needed.
+			if @info[:client][:type] == "script"
+				client = @info[:client]
 				unless client.has_key?(:username) && client.has_key?(:password)
 					raise ConfigurationError.new <<-EOS
 When configured for `type = "script"`, Hermaeus’ [client] section must include
@@ -67,7 +77,36 @@ password = "hunter2"
 					EOS
 				end
 			end
-		true
+
+			# Validate the [archive] section.
+			raise ConfigurationError.new(<<-EOS) unless @info.has_key? :archive
+Hermaeus’ configuration file must include an [archive] section to govern the
+storage of downloaded posts.
+			EOS
+
+			raise ConfigurationError.new(<<-EOS) unless @info[:archive].has_key? :path
+Hermaeus’ [archive] section must include a path field containing a relative or
+absolute path in which to store the downloaded posts.
+
+[archive]
+path = "./archive"
+# path = "/tmp/teslore/archive"
+			EOS
+
+			# Validate the [index] section.
+			raise ConfigurationError.new(<<-EOS) unless @info.has_key? :index
+Hermaeus’ configuration file must include and [index] section to govern the
+processing of the subreddit’s index page.
+			EOS
+
+			raise ConfigurationError.new(<<-EOS) unless @info[:index].has_key? :path
+Hermaeus’ [index] section must include a path field containing the reddit page
+at which the index resides.
+
+[index]
+path = "/r/teslore/wiki/archive"
+			EOS
+			true
 		end
 	end
 end

data/lib/hermaeus/version.rb

@@ -1,3 +1,3 @@
 module Hermaeus
-	VERSION = "1.0.2"
+	VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hermaeus
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - myrrlyn
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-12 00:00:00.000000000 Z
+date: 2016-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler