feedstock 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c87214f9b1fca1686eb03c1364b62f1d8c716e9998c695edb13273e4bc4f3bd
-  data.tar.gz: 34dce9c661854f0225e9036102b3c52cf205fd750a454bc42e54b538670fe2d1
+  metadata.gz: 60dc0bcb05928b59220fe1ed6ac24487428ceef5279f454bce047d3b3a94a56d
+  data.tar.gz: 91d7a161cdd3aedaf2316082b9f5bbae2fa48dcfd9d599cd65ee746673afc2b8
 SHA512:
-  metadata.gz: 0bcd9b33a1216473daedd4cefab2506e7cf2a763345c6b94d5fb37e0cf6876142cf1269a4229c53d8b5d1118663ad6f4350787e818aefee52d441b3b27fe1581
-  data.tar.gz: da15a4cfd0c7df236ce98fa6e580c268b05ad5ee4d9750fe8beaa0c88a48711790dd4e73dd2c91d5fee870b3e0846816e64b1c4220f7c579b3f3f78b8083f330
+  metadata.gz: 4513cbec520821710ed756544b1a1f7797498a5769395333e19a96c39a466cf83291863944973020ad33a9adb21b58ae2370c3bf08a72843683df1025432fc7c
+  data.tar.gz: 327995486920781894858903f4d510b958ee85b04a9aebf7139a4934f73c8f5af446b8eda6a06bf0e158692c4de278ebfc007b7f729f6bca953c67ea8eaff432

data/README.md

@@ -21,6 +21,150 @@ Feedstock is available as a gem:
 $ gem install feedstock
 ```
 
+## Usage
+
+Feedstock extracts information from a given document using a collection of
+_rules_.
+
+A collection of rules is expressed as a hash. The hash has two mandatory keys
+and one optional key.
+
+### Info
+
+The `"info"` key is mandatory. It must be associated with a hash. This document
+refers to this hash as the 'info hash'.
+
+#### Keys
+
+The keys in the info hash are strings (not symbols). When used with the default
+template, Feedstock will use the key as the name of the XML entity in the
+resulting feed. For example, if the key is `"id"`, the XML entity in the
+resulting feed will be `<id>`.
+
+#### Values
+
+The value associated with each key in the info hash can be either a string or a
+hash. 
+
+##### String
+
+If the value is a string, this defines a path to a node in the document. The
+path is expressed using CSS's selector syntax. Although a CSS selector can match
+more than one node, when used in the info hash, a path will only match the first
+matching node in the document.
+
+##### Hash
+
+If the value is a hash, this is the 'data hash'. The data hash defines the
+rules that Feedstock uses to extract data. It must contain one of two keys:
+
+- `"literal"`: The value associated with this key is used for the content of the
+  XML entity. This can be useful for elements that are not on the page or that
+  don't change.
+
+- `"path"`: The path to the node in the document expressed in CSS's selector
+  syntax.  As noted above, if the value of a key in the info hash is a string,
+  this is treated as a path. The reason to use a data hash with a `"path"` key
+  is when using one or more of the keys below. In the info hash, a path matches
+  only the first matching node in the document.
+
+The following keys may also be defined in a data hash:
+
+- `"attribute"`: The default is `nil`. If an attribute is provided, Feedstock
+  will extract the content of the attribute rather than the content of the node.
+  This is important for links, where the link itself is typically the content of
+  the `href` attribute rather than the content of the `<a>` element.
+
+- `"prefix"`: The default is `nil`. If a prefix is provided, the string value of
+  the prefix is appended to the beginning of the content extracted.
+
+- `"suffix"`: The default is `nil`. If a suffix is provided, the string value of
+  the suffix is appended to the end of the content extracted.
+
+- `"type"`: The default is `nil`. This causes Feedstock to extract only the text
+  in a node (stripping out all HTML). However, a user may specify `"datetime"`
+  or `"cdata"`. `"datetime"` content is parsed by [the Timeliness
+  library][Timeliness] (this is bundled with Feedstock) to return a string.
+  `"cdata"` content includes any HTML and is wrapped in `<![CDATA[` and `]]>`
+  tags.
+
+[Timeliness]: https://github.com/adzap/timeliness "The official repository for
+the Timeliness library"
+
+### Entry
+
+The `"entry"` key is mandatory. It must be associated with a hash. This document
+refers to this hash as the 'entry hash'.
+
+#### Keys
+
+The keys in the entry hash are strings (not symbols). When used with the default
+template, Feedstock will use the key as the name of the XML entity in the
+resulting feed. For example, if the key is `"id"`, the XML entity in the
+resulting feed will be `<id>`.
+
+#### Values
+
+The value associated with each key in the entry hash can be either a string or a
+hash. 
+
+##### String
+
+If the value is a string, this defines a path to a node in the document. The
+path is expressed using CSS's selector syntax. Unlike with the info hash, a
+the CSS selector will match all nodes.
+
+##### Hash
+
+If the value is a hash, we call this the "data hash". The data hash defines the
+rules that Feedstock uses to extract data. It must contain one of two keys:
+
+- `"literal"`: The value associated with this key is used for the content of the
+  XML entity. This can be useful for elements that are not on the page or that
+  don't change.
+
+- `"path"`: The path to the node in the document expressed in CSS's selector
+  syntax. Unlike with the info hash, the CSS selector will match all nodes. 
+
+The following keys may also be defined in a data hash:
+
+- `"attribute"`: The default is `nil`. If an attribute is provided, Feedstock
+  will extract the content of the attribute rather than the content of the node.
+  This is important for links, where the link itself is typically the content of
+  the `href` attribute rather than the content of the `<a>` element.
+
+- `"infix"`: The default is `nil`. If the entries hash has been provided (see
+  below), then the string value of the infix is inserted between the content of
+  each matching node. If the entries hash not been provided, this is ignored.
+
+- `"prefix"`: The default is `nil`. If a prefix is provided, the string value of
+  the prefix is appended to the beginning of the content extracted.
+
+- `"repeat"`: The default is `nil`. If repeat is set to `true`, Feedstock will
+  use the content provided by either `"literal"` or `"path"` repeatedly. Since
+  the value of `"literal"` implies `"repeat"`, it is not necessary to specify it
+  expressly.
+
+- `"suffix"`: The default is `nil`. If a suffix is provided, the string value of
+  the suffix is appended to the end of the content extracted.
+
+- `"type"`: The default is `nil`. This causes Feedstock to extract only the text
+  in a node (stripping out all HTML). However, a user may specify `"datetime"`
+  or `"cdata"`. `"datetime"` content is parsed by [the Timeliness
+  library][Timeliness] (this is bundled with Feedstock) to return a string.
+  `"cdata"` content includes any HTML and is wrapped in `<![CDATA[` and `]]>`
+  tags.
+
+### Entries
+
+The `"entries"` key is optional. It can be associated with a hash. This document
+refers to this hash as the 'entries hash'.
+
+If an entries hash is provided, it must contain the following key:
+
+- `"path"`: The path to the node in the document expressed in CSS's selector
+  syntax. This path is used as the root for the paths in the entry hash.
+
 ## Bugs
 
 Found a bug? I'd love to know about it. The best way is to report them in the
@@ -30,12 +174,13 @@ Found a bug? I'd love to know about it. The best way is to report them in the
 
 ## Versioning
 
-Tenter uses [Semantic Versioning 2.0.0][sv2].
+Feedstock uses [Semantic Versioning 2.0.0][sv2].
 
 [sv2]: http://semver.org/
 
 ## Licence
 
-Tenter is released into the public domain. See [LICENSE.md][lc] for more details.
+Feedstock is released into the public domain. See [LICENSE.md][lc] for more
+details.
 
 [lc]: https://github.com/pyrmont/feedstock/blob/master/LICENSE.md

data/feedstock.gemspec

@@ -19,13 +19,13 @@ Gem::Specification.new do |s|
   s.files = Dir["Gemfile", "default.xml", "LICENSE", "README.md",
                 "feedstock.gemspec", "lib/feedstock.rb", "lib/**/*.rb"]
   s.require_paths = ["lib"]
-  
+
   s.metadata["allowed_push_host"] = "https://rubygems.org"
 
   s.add_runtime_dependency "nokogiri"
   s.add_runtime_dependency "timeliness"
 
-  s.add_development_dependency "minitest" 
+  s.add_development_dependency "minitest"
   s.add_development_dependency "rake"
   s.add_development_dependency "warning"
 end

data/lib/feedstock.rb

@@ -22,14 +22,22 @@ module Feedstock
   end
 
   def self.download_page(url)
-    Nokogiri::HTML open(url)
+    Nokogiri::HTML URI.open(url)
   end
 
   def self.extract_entries(page, rules)
+    if rules["entries"]
+      extract_entries_wrapped page, rules
+    else
+      extract_entries_unwrapped page, rules
+    end
+  end
+
+  def self.extract_entries_unwrapped(page, rules)
     static  = Hash.new
     entries = Array.new
 
-    rules['entries'].each do |name, rule|
+    rules["entry"].each do |name, rule|
       if rule["literal"]
         static[name] = rule["literal"]
       elsif rule["repeat"]
@@ -49,6 +57,28 @@ module Feedstock
     entries
   end
 
+  def self.extract_entries_wrapped(page, rules)
+    entries = Array.new
+
+    page.css(rules["entries"]["path"]).each.with_index do |node, i|
+      rules["entry"].each do |name, rule|
+        entries[i] = Hash.new if entries[i].nil?
+
+        content = if rule["literal"]
+                    rule["literal"]
+                  elsif rule["repeat"]
+                    format_content page.at_css(rule["path"]), rule
+                  else
+                    format_content node.at_css(rule["path"]), rule
+                  end
+
+        entries[i].merge!({ name => content })
+      end
+    end
+
+    entries
+  end
+
   def self.extract_info(page, rules)
     info = Hash.new
 
@@ -84,8 +114,14 @@ module Feedstock
 
   def self.normalise_rules(rules)
     rules.keys.each do |category|
-      rules[category].each do |name, rule|
-        rules[category][name] = { "path" => rule } unless rule.is_a? Hash
+      case category
+      when "info", "entry"
+        rules[category].each do |name, rule|
+          rules[category][name] = { "path" => rule } unless rule.is_a? Hash
+        end
+      when "entries"
+        rule = rules[category]
+        rules[category] = { "path" => rule } unless rule.is_a? Hash
       end
     end
 

data/lib/feedstock/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Feedstock
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedstock
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Michael Camilleri
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-18 00:00:00.000000000 Z
+date: 2021-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -115,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A library for creating RSS feeds from webpages