feedstock 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1c87214f9b1fca1686eb03c1364b62f1d8c716e9998c695edb13273e4bc4f3bd
+  data.tar.gz: 34dce9c661854f0225e9036102b3c52cf205fd750a454bc42e54b538670fe2d1
+SHA512:
+  metadata.gz: 0bcd9b33a1216473daedd4cefab2506e7cf2a763345c6b94d5fb37e0cf6876142cf1269a4229c53d8b5d1118663ad6f4350787e818aefee52d441b3b27fe1581
+  data.tar.gz: da15a4cfd0c7df236ce98fa6e580c268b05ad5ee4d9750fe8beaa0c88a48711790dd4e73dd2c91d5fee870b3e0846816e64b1c4220f7c579b3f3f78b8083f330

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>

data/README.md

@@ -0,0 +1,41 @@
+# Feedstock
+
+Feedstock is a Ruby library for extracting information from a webpage and
+converting it into an Atom feed.
+
+## Rationale
+
+Feeds are great. But sometimes a website doesn't provide a feed or doesn't
+provide a feed for the specific content that you want. That's where Feedstock
+can help.
+
+Feedstock is a Ruby library that you can use to create an Atom feed. It takes a
+URL to the webpage to check and a hash of rules. The rules tell Feedstock how to
+extract and transform the data it finds on the webpage.
+
+## Installation
+
+Feedstock is available as a gem:
+
+```shell
+$ gem install feedstock
+```
+
+## Bugs
+
+Found a bug? I'd love to know about it. The best way is to report them in the
+[Issues section][ghi] on GitHub.
+
+[ghi]: https://github.com/pyrmont/feedstock/issues
+
+## Versioning
+
+Tenter uses [Semantic Versioning 2.0.0][sv2].
+
+[sv2]: http://semver.org/
+
+## Licence
+
+Tenter is released into the public domain. See [LICENSE.md][lc] for more details.
+
+[lc]: https://github.com/pyrmont/feedstock/blob/master/LICENSE.md

data/default.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+<%- info.each do |name, value| -%>
+  <%- case name -%>
+  <%- when "author" -%>
+  <author><name><%= value %></name></author>
+  <%- when "link" -%>
+  <<%= name %> href="<%= value %>" rel="alternate" />
+  <%- else -%>
+  <<%= name %>><%= value %></<%= name %>>
+  <%- end -%>
+<%- end -%>
+<%- entries.each do |entry| -%>
+  <entry>
+  <%- entry.each do |name, value| -%>
+    <%- case name -%>
+    <%- when "author" -%>
+    <author><name><%= value %></name></author>
+    <%- when "link" -%>
+    <<%= name %> href="<%= value %>" rel="alternate" />
+    <%- else -%>
+    <<%= name %>><%= value %></<%= name %>>
+    <%- end -%>
+  <%- end -%>
+  </entry>
+<%- end -%>
+</feed>

data/feedstock.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "./lib/feedstock/version"
+
+Gem::Specification.new do |s|
+  s.name = "feedstock"
+  s.version = Feedstock::VERSION
+  s.authors = ["Michael Camilleri"]
+  s.email = ["mike@inqk.net"]
+  s.summary = "A library for creating RSS feeds from webpages"
+  s.description = <<-desc.strip.gsub(/\s+/, " ")
+    Feedstock is a library for extracting information from a webpage and
+    transforming it into an Atom feed.
+  desc
+  s.homepage = "https://github.com/pyrmont/feedstock/"
+  s.licenses = "Unlicense"
+  s.required_ruby_version = ">= 2.5"
+
+  s.files = Dir["Gemfile", "default.xml", "LICENSE", "README.md",
+                "feedstock.gemspec", "lib/feedstock.rb", "lib/**/*.rb"]
+  s.require_paths = ["lib"]
+  
+  s.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "timeliness"
+
+  s.add_development_dependency "minitest" 
+  s.add_development_dependency "rake"
+  s.add_development_dependency "warning"
+end

data/lib/feedstock.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "erb"
+require "nokogiri"
+require "open-uri"
+require "timeliness"
+
+module Feedstock
+  def self.feed(url, rules, template_file = "#{__dir__}/../default.xml")
+    rules   = normalise_rules rules
+    page    = download_page url
+    info    = extract_info page, rules
+    entries = extract_entries page, rules
+    feed    = create_feed info, entries, template_file
+
+    feed
+  end
+
+  def self.create_feed(info, entries, template_file)
+    template = ERB.new File.read(template_file), trim_mode: "-"
+    template.result_with_hash info: info, entries: entries
+  end
+
+  def self.download_page(url)
+    Nokogiri::HTML open(url)
+  end
+
+  def self.extract_entries(page, rules)
+    static  = Hash.new
+    entries = Array.new
+
+    rules['entries'].each do |name, rule|
+      if rule["literal"]
+        static[name] = rule["literal"]
+      elsif rule["repeat"]
+        static[name] = format_content page.at_css(rule["path"]), rule
+      else
+        page.css(rule["path"]).each.with_index do |match, i|
+          entries[i] = Hash.new if entries[i].nil?
+          entries[i].merge!({ name => format_content(match, rule) })
+        end
+      end
+    end
+
+    unless static.empty?
+      entries.each{ |entry| entry.merge!(static) }
+    end
+
+    entries
+  end
+
+  def self.extract_info(page, rules)
+    info = Hash.new
+
+    rules["info"].each do |name, rule|
+      if rule["literal"]
+        info[name] = rule["literal"]
+      else
+        info[name] = format_content page.at_css(rule["path"]), rule
+      end
+    end
+
+    info
+  end
+
+  def self.format_content(match, rule)
+    return "" if match.nil?
+
+    text = if rule["attribute"]
+             match[rule["attribute"]]
+           else
+             match.content.strip
+           end
+
+    case rule["type"]
+    when "cdata"
+      "<![CDATA[#{wrap_content(match.inner_html, rule)}]]>"
+    when "datetime"
+      "#{Timeliness.parse(wrap_content(text, rule))&.iso8601}"
+    else
+      wrap_content text, rule
+    end
+  end
+
+  def self.normalise_rules(rules)
+    rules.keys.each do |category|
+      rules[category].each do |name, rule|
+        rules[category][name] = { "path" => rule } unless rule.is_a? Hash
+      end
+    end
+
+    rules
+  end
+
+  def self.wrap_content(content, rule)
+    return content unless rule["prepend"] || rule["append"]
+
+    "#{rule["prepend"]}#{content}#{rule["append"]}"
+  end
+end

data/lib/feedstock/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Feedstock
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: feedstock
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michael Camilleri
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timeliness
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: warning
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Feedstock is a library for extracting information from a webpage and
+  transforming it into an Atom feed.
+email:
+- mike@inqk.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- default.xml
+- feedstock.gemspec
+- lib/feedstock.rb
+- lib/feedstock/version.rb
+homepage: https://github.com/pyrmont/feedstock/
+licenses:
+- Unlicense
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A library for creating RSS feeds from webpages
+test_files: []