weneedfeed 0.7.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e666f142f72459d43fd7c6fe071740d1fce11dbac4605bcd1d7fbcc8766187ab
-  data.tar.gz: a6fdf6ce28e9c610d9f47fa747eaedc4d0198c59c70fd5b736d99dbeaf90d7df
+  metadata.gz: 7aeb8844141a877a1c8e17c1aa1549d030a9384d1fbc01219a75a45891823830
+  data.tar.gz: e707d7fdec8299c0b5cf76c0628ab58a5e8b1c1aec443fe2cc00524671df68b5
 SHA512:
-  metadata.gz: e07f3fe6d8d712e20384310e1bee1eca48065ccf2c0a400aaccd6e9d0f0d219b011f7944c34d11870bb5e3b83750f17de267d8eff260926916daaff1b9f08f35
-  data.tar.gz: 792f22dbcda78c4df3f0d738fe6dee28dae5a64eb075f8a1603da6d0c9d5ca68bba3337374f3f2d6dd93e798a6c1f57c4436a9ff34425ee84366d94b56c89903
+  metadata.gz: a3db194daf276a2abff61e6c7a92cc137404b6bfb640df7db77a770316e4f10f6cc57b72172bcdda94eceb1b4db726bfc0547719452d097dd33d320394e7f6ea
+  data.tar.gz: 14cfe2f49aa515f396d4ae598bb07eaeab91cec57d1f5201e68d04770bb6d762b86e346700030a0f7fdd2b9fd57b81e64ff3982c03bdbb34b02e73f34a83f03f

data/.github/workflows/test.yml

@@ -7,14 +7,24 @@ on:
       - master
 
 jobs:
-  build:
-    runs-on: ubuntu-18.04
+  rspec:
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+          ruby-version: 2.7.2
+      - run: bundle exec rspec --force-color
+  rubocop:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
       - uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
           ruby-version: 2.7.2
       - uses: r7kamura/rubocop-problem-matchers-action@v1
       - run: bundle exec rubocop --parallel
-      - run: bundle exec rspec --force-color

data/CHANGELOG.md

@@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 0.9.1 - 2020-12-03
+
+### Fixed
+
+- Fix escape bug in escaping HTML and generating XML.
+
+## 0.9.0 - 2020-12-03
+
+### Added
+
+- Add item_image_selector.
+
+## 0.8.0 - 2020-11-25
+
+### Added
+
+- Add page description for writing channel description.
+
+### Changed
+
+- Change channel link content from feeds index URL to page URL.
+
+## 0.7.2 - 2020-11-25
+
+### Fixed
+
+- Fix time parse error by HTML entities handling.
+
+## 0.7.1 - 2020-11-25
+
+### Fixed
+
+- Fix error when no title element found.
+
 ## 0.7.0 - 2020-11-23
 
 ### Added

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    weneedfeed (0.7.0)
+    weneedfeed (0.9.1)
       activesupport
       builder
       faraday
       faraday_middleware
       hibana (>= 0.2)
+      mimemagic
       nokogiri
       rack-capture (>= 0.4.0)
       thor
@@ -44,6 +45,7 @@ GEM
       tilt
     i18n (1.8.5)
       concurrent-ruby (~> 1.0)
+    mimemagic (0.3.5)
     mini_portile2 (2.4.0)
     minitest (5.14.2)
     multipart-post (2.1.1)
@@ -106,7 +108,7 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    zeitwerk (2.4.1)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby

data/README.md

@@ -27,24 +27,28 @@ gem install weneedfeed
 
 ## Schema
 
-You need to write a schema file named with `weneedfeed.yml` to use this gem.
+Weneedfeed requires `weneedfeed.yml` that describes URLs and selectors.
 
 ### Example
 
 ```yaml
 pages:
   - id: example1
-    title: Example feed with CSS Selector
+    title: Example 1
+    description: Example feed with CSS Selector
     url: http://example.com/1
     item_selector: li
     item_description_selector: p:nth-child(3)
+    item_image_selector: img
     item_link_selector: a
     item_time_selector: time
     item_title_selector: p:nth-child(2)
   - id: example2
-    title: Example feed with XPath
+    title: Example 2
+    description: Example feed with XPath
     url: http://example.com/2
     item_selector: //li
+    item_image_selector: .//img
     item_description_selector: .//p[3]
     item_link_selector: .//a
     item_time_selector: .//time
@@ -56,7 +60,7 @@ pages:
 Feed ID.
 
 - required
-- Used for feed URL.
+- Used for feed URL and `<link>` element in `<channel>` element.
 
 ### `title`
 
@@ -65,6 +69,13 @@ Feed title.
 - required
 - Used for RSS `<title>` element in `<channel>` element.
 
+### `description`
+
+Feed description.
+
+- optional
+- Used for RSS `<description>` element in `<channel>` element.
+
 ### `url`
 
 HTML source URL.
@@ -100,6 +111,13 @@ CSS or XPath selector to find element with description information in each item.
 - optional
 - Used for `<description>` in `<item>`.
 
+### `item_image_selector`
+
+CSS or XPath selector to find `<img>`element in each item.
+
+- optional
+- Used for `<enclosure>` in `<item>`.
+
 ### `item_time_selector`
 
 CSS or XPath selector to find element with datetime information in each item.

data/lib/weneedfeed/controllers/show_feed.rb

@@ -11,7 +11,9 @@ module Weneedfeed
         end
 
         scraping = ::Weneedfeed::Scraping.new(
+          description: page_schema.description,
           item_description_selector: page_schema.item_description_selector,
+          item_image_selector: page_schema.item_image_selector,
           item_link_selector: page_schema.item_link_selector,
           item_time_selector: page_schema.item_time_selector,
           item_title_selector: page_schema.item_title_selector,

data/lib/weneedfeed/item.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'mimemagic'
+
 module Weneedfeed
   class Item
     class << self
@@ -16,6 +18,7 @@ module Weneedfeed
     end
 
     # @param [String, nil] description_selector
+    # @param [String, nil] image_selector
     # @param [String, nil] link_selector
     # @param [Nokogiri::Node] node
     # @param [String] time_selector
@@ -23,6 +26,7 @@ module Weneedfeed
     # @param [String] url
     def initialize(
       description_selector:,
+      image_selector:,
       link_selector:,
       node:,
       time_selector:,
@@ -30,6 +34,7 @@ module Weneedfeed
       url:
     )
       @description_selector = description_selector
+      @image_selector = image_selector
       @link_selector = link_selector
       @node = node
       @time_selector = time_selector
@@ -44,6 +49,29 @@ module Weneedfeed
       @node.at(@description_selector)&.inner_html
     end
 
+    # @return [String, nil]
+    def image_mime_type
+      return unless image_url
+
+      ::MimeMagic.by_path(image_url)&.type
+    end
+
+    # @return [String, nil]
+    def image_path_or_url
+      return unless @image_selector
+
+      @node.at(@image_selector)&.[]('src')
+    end
+
+    def image_url
+      return unless image_path_or_url
+
+      ::URI.join(
+        @url,
+        image_path_or_url
+      ).to_s
+    end
+
     # @return [String]
     def link
       ::URI.join(
@@ -73,9 +101,9 @@ module Weneedfeed
       self.class.parse_time(string)
     end
 
-    # @return [String]
+    # @return [String, nil]
     def title
-      @node.at(@title_selector).inner_text
+      @node.at(@title_selector)&.inner_text
     end
 
     private
@@ -90,7 +118,7 @@ module Weneedfeed
       node = time_node
       return unless node
 
-      node['datetime'] || node.inner_html
+      node['datetime'] || node.inner_text
     end
   end
 end

data/lib/weneedfeed/page.rb

@@ -2,13 +2,18 @@
 
 module Weneedfeed
   class Page
+    # @return [String, nil]
+    attr_reader :description
+
     # @return [String]
     attr_reader :title
 
     # @return [String]
     attr_reader :url
 
+    # @param [String, nil] description
     # @param [String, nil] item_description_selector
+    # @param [String, nil] item_image_selector
     # @param [String] item_link_selector
     # @param [String, nil] item_time_selector
     # @param [String] item_title_selector
@@ -17,7 +22,9 @@ module Weneedfeed
     # @param [String] title
     # @param [String] url
     def initialize(
+      description:,
       item_description_selector:,
+      item_image_selector:,
       item_link_selector:,
       item_time_selector:,
       item_title_selector:,
@@ -26,7 +33,9 @@ module Weneedfeed
       title:,
       url:
     )
+      @description = description
       @item_description_selector = item_description_selector
+      @item_image_selector = item_image_selector
       @item_link_selector = item_link_selector
       @item_time_selector = item_time_selector
       @item_title_selector = item_title_selector
@@ -41,6 +50,7 @@ module Weneedfeed
       @node.search(@item_selector).map do |node|
         ::Weneedfeed::Item.new(
           description_selector: @item_description_selector,
+          image_selector: @item_image_selector,
           link_selector: @item_link_selector,
           node: node,
           time_selector: @item_time_selector,

data/lib/weneedfeed/page_schema.rb

@@ -2,8 +2,10 @@
 
 module Weneedfeed
   PageSchema = Struct.new(
+    :description,
     :id,
     :item_description_selector,
+    :item_image_selector,
     :item_link_selector,
     :item_time_selector,
     :item_title_selector,

data/lib/weneedfeed/schema.rb

@@ -24,8 +24,10 @@ module Weneedfeed
     def page_schemata
       @raw['pages'].map do |hash|
         ::Weneedfeed::PageSchema.new(
+          description: hash['description'],
           id: hash['id'],
           item_description_selector: hash['item_description_selector'],
+          item_image_selector: hash['item_image_selector'],
           item_link_selector: hash['item_link_selector'],
           item_time_selector: hash['item_time_selector'],
           item_title_selector: hash['item_title_selector'],

data/lib/weneedfeed/scraping.rb

@@ -14,7 +14,9 @@ module Weneedfeed
       end
     end
 
+    # @param [String, nil] description
     # @param [String, nil] item_description_selector
+    # @param [String, nil] item_image_selector
     # @param [String] item_link_selector
     # @param [String, nil] item_time_selector
     # @param [String] item_title_selector
@@ -22,7 +24,9 @@ module Weneedfeed
     # @param [String] title
     # @param [String] url
     def initialize(
+      description:,
       item_description_selector:,
+      item_image_selector:,
       item_link_selector:,
       item_time_selector:,
       item_title_selector:,
@@ -30,7 +34,9 @@ module Weneedfeed
       title:,
       url:
     )
+      @description = description
       @item_description_selector = item_description_selector
+      @item_image_selector = item_image_selector
       @item_link_selector = item_link_selector
       @item_time_selector = item_time_selector
       @item_title_selector = item_title_selector
@@ -42,8 +48,10 @@ module Weneedfeed
     # @return [Weneedfeed::Page]
     def call
       ::Weneedfeed::Page.new(
+        description: @description,
         node: parsed_body,
         item_description_selector: @item_description_selector,
+        item_image_selector: @item_image_selector,
         item_selector: @item_selector,
         item_link_selector: @item_link_selector,
         item_time_selector: @item_time_selector,
@@ -57,12 +65,16 @@ module Weneedfeed
 
     # @return [Nokogiri::Node]
     def parsed_body
-      ::Nokogiri::XML.parse(response.body)
+      if response.headers['Content-Type']&.include?('application/json')
+        ::Nokogiri::XML.parse(response.body)
+      else
+        ::Nokogiri::HTML.parse(response.body)
+      end
     end
 
     # @return [Faraday::Response]
     def response
-      self.class.faraday_connection.get(@url)
+      @response ||= self.class.faraday_connection.get(@url)
     end
   end
 end

data/lib/weneedfeed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Weneedfeed
-  VERSION = '0.7.0'
+  VERSION = '0.9.1'
 end

data/templates/show_feed.xml.erb

@@ -4,9 +4,9 @@
   xmlns:content="http://purl.org/rss/1.0/modules/content/">
   <channel>
     <title><![CDATA[<%= @page.title %>]]></title>
-    <link><%= "#{request.base_url}#{top_page_path}" %></link>
+    <link><%= "#{@page.url}" %></link>
     <atom:link href="<%= "#{request.base_url}#{request.path}" %>" rel="self"/>
-    <description><![CDATA[Recent content on <%= @page.title %>]]></description>
+    <description><![CDATA[<%= @page.description %>]]></description>
     <lastBuildDate><%= Time.now.rfc822 %></lastBuildDate>
     <% items.each do |item| %>
     <item>
@@ -18,6 +18,9 @@
       <description><![CDATA[<%= item.description %>]]></description>
       <content:encoded><![CDATA[<%= item.description %>]]></content:encoded>
       <guid isPermaLink="true"><%= item.link %></guid>
+      <% if item.image_url %>
+        <enclosure url=<%= item.image_url.encode(xml: :attr) %> length="0" type="<%= item.image_mime_type %>"/>
+      <% end %>
     </item>
     <% end %>
   </channel>

data/weneedfeed.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'faraday'
   spec.add_runtime_dependency 'faraday_middleware'
   spec.add_runtime_dependency 'hibana', '>= 0.2'
+  spec.add_runtime_dependency 'mimemagic'
   spec.add_runtime_dependency 'nokogiri'
   spec.add_runtime_dependency 'rack-capture', '>= 0.4.0'
   spec.add_runtime_dependency 'thor'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: weneedfeed
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Ryo Nakamura
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-23 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: mimemagic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement