weneedfeed 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fdaa4a0f724b7018d6b1b2b585aeaf970eff37538df7ca7aaf760fc217b56dc
-  data.tar.gz: 9f34a11d59aaada2d5afd689263f722c0143af5e59b9741e7d6f34d16d807c57
+  metadata.gz: 65814d68cfaf1936de388ce721e3aee49a0e4641c0ecd6ae2afb72c858fbd8f4
+  data.tar.gz: ed124803cfb9589dd3b8e44169454b07623a2460a7ac2412c1946d08e51d590f
 SHA512:
-  metadata.gz: 878189a82437b83ed25d0c25c5a6de0f4b840c1daf117ccd83d31be1ecbbe2db0d97289bd5ab93e327508562843ddea253948cad63324d3557ab28a0d5052268
-  data.tar.gz: 8c56392e937c331456b44c869a96316c892b5321fd85611325c91c3987de079d101144998a4399f4fa668c46181e3f0c51f79fe157bf333d211fcf5ae0aeec87
+  metadata.gz: 486e7af49032ebf8124d6360cd835ec5ede778c4dc6911259fcee753247166ef6c53e3468bb3c386a3e2d794b3939685795c230a4ff24e65a67e615c5b39a800
+  data.tar.gz: 636f09f3860a1a7a16bf425facd29988e7037ae35a9ab8a11b8a37118339c9bdf750336eb01b8c36e2bee0d8b81ed2b1a9846269096428b702c09b27b873617a

data/.rubocop.yml

@@ -9,3 +9,6 @@ Metrics:
 
 Style/Documentation:
   Enabled: false
+
+Style/TrailingCommaInArguments:
+  Enabled: false

data/CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## 0.6.0 - 2020-11-15
+
+### Changed
+
+- Change pages schema from Hash to Array.
+- Require hanami-router 2.0.0.alpha3 or later versions.
+- Ignore pubDate when item.time is not found.
+- Make some item_description_selector and item_time_selector optional.
+
 ## 0.5.0 - 2020-11-14
 
 ### Added

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    weneedfeed (0.5.0)
+    weneedfeed (0.6.0)
       faraday
+      hanami-router (= 2.0.0.alpha3)
       hibana
       nokogiri
       rack-capture (>= 0.4.0)
@@ -14,29 +15,28 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.1)
-    concurrent-ruby (1.1.7)
     crack (0.4.4)
     diff-lcs (1.4.4)
     faraday (1.1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords
-    hanami-router (1.3.2)
-      hanami-utils (~> 1.3)
-      http_router (= 0.11.2)
+    hanami-router (2.0.0.alpha3)
+      mustermann (~> 1.0)
+      mustermann-contrib (~> 1.0)
       rack (~> 2.0)
-    hanami-utils (1.3.6)
-      concurrent-ruby (~> 1.0)
-      transproc (~> 1.0)
+    hansi (0.2.0)
     hashdiff (1.0.1)
     hibana (0.1.1)
       hanami-router
       rack
       tilt
-    http_router (0.11.2)
-      rack (>= 1.0.0)
-      url_mount (~> 0.2.1)
     mini_portile2 (2.4.0)
     multipart-post (2.1.1)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    mustermann-contrib (1.1.1)
+      hansi (~> 0.2.0)
+      mustermann (= 1.1.1)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     parallel (1.19.2)
@@ -80,10 +80,7 @@ GEM
     ruby2_keywords (0.0.2)
     thor (1.0.1)
     tilt (2.0.10)
-    transproc (1.1.1)
     unicode-display_width (1.7.0)
-    url_mount (0.2.1)
-      rack
     webmock (3.9.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)

data/README.md

@@ -33,51 +33,79 @@ You need to write a schema file named with `weneedfeed.yml` to use this gem.
 
 ```yaml
 pages:
-  example1:
-    title: example site 1
+  - id: example1
+    title: Example feed with CSS Selector
     url: http://example.com/1
     item_selector: li
     item_description_selector: p:nth-child(3)
     item_link_selector: a
     item_time_selector: time
     item_title_selector: p:nth-child(2)
-  example2:
-    title: example site 2
+  - id: example2
+    title: Example feed with XPath
     url: http://example.com/2
     item_selector: //li
     item_description_selector: .//p[3]
-    item_link_selector: .//a/@href
+    item_link_selector: .//a
     item_time_selector: .//time
     item_title_selector: .//p[2]
 ```
 
+### `id`
+
+Feed ID.
+
+- required
+- Used for feed URL.
+
 ### `title`
 
-Feed title, used for RSS `<title>` element in `<channel>` element.
+Feed title.
+
+- required
+- Used for RSS `<title>` element in `<channel>` element.
 
 ### `url`
 
-URL to fetch HTML page for building feed.
+HTML source URL.
+
+- required
+- Used to fetch HTML page for building feed.
 
 ### `item_selector`
 
-CSS or XPath selector to search each item, equivalent unit to RSS `<item>` element.
+CSS or XPath selector to search each item.
 
-### `item_link_selector`
+- required
+- Equivalent unit to RSS `<item>` element.
 
-CSS or XPath selector to find `<a>` element in each item, used for `<link>` in `<item>`.
+### `item_link_selector`
 
-### `item_time_selector`
+CSS or XPath selector to find `<a>` element in each item.
 
-CSS or XPath selector to find element with datetime information in each item, used for `<pubDate>` in `<item>`. Its `datetime` attribute or its inner HTML is used to calculate datetime.
+- required
+- Used for `<link>` in `<item>`.
 
 ### `item_title_selector`
 
-CSS or XPath selector to find element with title information in each item, used for `<pubDate>` in `<item>`.
+CSS or XPath selector to find element with title information in each item.
+
+- required
+- Used for `<title>` in `<item>`.
 
 ### `item_description_selector`
 
-CSS or XPath selector to find element with description information in each item, used for `<description>` in `<item>`.
+CSS or XPath selector to find element with description information in each item.
+
+- optional
+- Used for `<description>` in `<item>`.
+
+### `item_time_selector`
+
+CSS or XPath selector to find element with datetime information in each item.
+
+- optional
+- Used for `<pubDate>` in `<item>`. Its `datetime` attribute or its inner HTML is used to calculate datetime.
 
 ## Usage
 

data/lib/weneedfeed.rb

@@ -7,8 +7,11 @@ module Weneedfeed
   autoload :Capture, 'weneedfeed/capture'
   autoload :Command, 'weneedfeed/command'
   autoload :Controllers, 'weneedfeed/controllers'
+  autoload :Helpers, 'weneedfeed/helpers'
   autoload :Item, 'weneedfeed/item'
   autoload :Page, 'weneedfeed/page'
+  autoload :PageSchema, 'weneedfeed/page_schema'
+  autoload :Schema, 'weneedfeed/schema'
   autoload :Scraping, 'weneedfeed/scraping'
   autoload :Views, 'weneedfeed/views'
 end

data/lib/weneedfeed/application.rb

@@ -6,12 +6,12 @@ module Weneedfeed
   class Application < ::Hibana::Application
     route do
       get '/', to: ::Weneedfeed::Controllers::ShowTopPage, as: :top_page
-      get '/feeds/:page_name.xml', to: ::Weneedfeed::Controllers::ShowFeed, as: :feed
+      get '/feeds/:page_id.xml', to: ::Weneedfeed::Controllers::ShowFeed, as: :feed
     end
 
     # @param [Hash] schema
     def initialize(schema:)
-      @schema = schema
+      @schema = ::Weneedfeed::Schema.new(schema)
       super()
     end
 
@@ -23,8 +23,8 @@ module Weneedfeed
 
     # @return [Array<String>]
     def paths
-      ['/'] + @schema['pages'].keys.map do |page_name|
-        "/feeds/#{page_name}.xml"
+      ['/'] + @schema.page_ids.map do |page_id|
+        "/feeds/#{page_id}.xml"
       end
     end
   end

data/lib/weneedfeed/controllers.rb

@@ -2,6 +2,7 @@
 
 module Weneedfeed
   module Controllers
+    autoload :Base, 'weneedfeed/controllers/base'
     autoload :ShowFeed, 'weneedfeed/controllers/show_feed'
     autoload :ShowTopPage, 'weneedfeed/controllers/show_top_page'
   end

data/lib/weneedfeed/controllers/base.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Weneedfeed
+  module Controllers
+    class Base < ::Hibana::Controller
+      include ::Weneedfeed::Helpers::Parameters
+
+      private
+
+      # @return [Weneedfeed::Schema]
+      def schema
+        request.env['weneedfeed.schema']
+      end
+    end
+  end
+end

data/lib/weneedfeed/controllers/show_feed.rb

@@ -2,31 +2,22 @@
 
 module Weneedfeed
   module Controllers
-    class ShowFeed < ::Hibana::Controller
+    class ShowFeed < ::Weneedfeed::Controllers::Base
       def call
-        env = request.env
-        page_name = env.dig(
-          'router.params',
-          :page_name
-        )
-        properties = env.dig(
-          'weneedfeed.schema',
-          'pages',
-          page_name
-        )
-        unless properties
+        page_schema = schema.find_page_schema(path_parameters[:page_id])
+        unless page_schema
           response.status = 404
           return
         end
 
         scraping = ::Weneedfeed::Scraping.new(
-          item_description_selector: properties['item_description_selector'],
-          item_link_selector: properties['item_link_selector'],
-          item_time_selector: properties['item_time_selector'],
-          item_title_selector: properties['item_title_selector'],
-          item_selector: properties['item_selector'],
-          title: properties['title'],
-          url: properties['url']
+          item_description_selector: page_schema.item_description_selector,
+          item_link_selector: page_schema.item_link_selector,
+          item_time_selector: page_schema.item_time_selector,
+          item_title_selector: page_schema.item_title_selector,
+          item_selector: page_schema.item_selector,
+          title: page_schema.title,
+          url: page_schema.url,
         )
         page = scraping.call
 

data/lib/weneedfeed/controllers/show_top_page.rb

@@ -2,18 +2,13 @@
 
 module Weneedfeed
   module Controllers
-    class ShowTopPage < ::Hibana::Controller
+    class ShowTopPage < ::Weneedfeed::Controllers::Base
       def call
-        pages = request.env.dig(
-          'weneedfeed.schema',
-          'pages'
-        ).sort_by do |_key, value|
-          value['title']
-        end
+        page_schemata = schema.page_schemata.sort_by(&:title)
         response.content_type = 'text/html'
         response.write(
           ::Weneedfeed::Views::ShowTopPage.new(
-            pages: pages,
+            page_schemata: page_schemata,
             partial_template_path: ::File.expand_path(
               'templates/show_top_page.html.erb',
               "#{__dir__}/../../.."

data/lib/weneedfeed/helpers.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Weneedfeed
+  module Helpers
+    autoload :Parameters, 'weneedfeed/helpers/parameters'
+  end
+end

data/lib/weneedfeed/helpers/parameters.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Weneedfeed
+  module Helpers
+    module Parameters
+      private
+
+      # @return [Hash]
+      def path_parameters
+        request.env['router.params']
+      end
+    end
+  end
+end

data/lib/weneedfeed/item.rb

@@ -15,8 +15,8 @@ module Weneedfeed
       end
     end
 
-    # @param [String] description_selector
-    # @param [String] link_selector
+    # @param [String, nil] description_selector
+    # @param [String, nil] link_selector
     # @param [Nokogiri::Node] node
     # @param [String] time_selector
     # @param [String] title_selector
@@ -39,6 +39,8 @@ module Weneedfeed
 
     # @return [String, nil]
     def description
+      return unless @description_selector
+
       @node.at(@description_selector)&.inner_html
     end
 
@@ -52,7 +54,12 @@ module Weneedfeed
 
     # @return [Time, nil]
     def time
-      self.class.parse_time(time_string)
+      return unless @time_selector
+
+      string = time_string
+      return unless string
+
+      self.class.parse_time(string)
     end
 
     # @return [String]
@@ -67,7 +74,7 @@ module Weneedfeed
       @node.at(@time_selector)
     end
 
-    # @return [String]
+    # @return [String, nil]
     def time_string
       node = time_node
       return unless node

data/lib/weneedfeed/page.rb

@@ -8,9 +8,9 @@ module Weneedfeed
     # @return [String]
     attr_reader :url
 
-    # @param [String] item_description_selector
+    # @param [String, nil] item_description_selector
     # @param [String] item_link_selector
-    # @param [String] item_time_selector
+    # @param [String, nil] item_time_selector
     # @param [String] item_title_selector
     # @param [String] item_selector
     # @param [Nokogiri::Node] node

data/lib/weneedfeed/page_schema.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Weneedfeed
+  PageSchema = Struct.new(
+    :id,
+    :item_description_selector,
+    :item_link_selector,
+    :item_time_selector,
+    :item_title_selector,
+    :item_selector,
+    :title,
+    :url,
+    keyword_init: true,
+  )
+end

data/lib/weneedfeed/schema.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Weneedfeed
+  class Schema
+    # @param [Hash] raw
+    def initialize(raw)
+      @raw = raw
+    end
+
+    # @param [String, nl] page_schema_id
+    # @return [Weneedfeed::PageSchema]
+    def find_page_schema(page_schema_id)
+      page_schemata.find do |page_schema|
+        page_schema.id == page_schema_id
+      end
+    end
+
+    # @return [Array<String>]
+    def page_ids
+      page_schemata.map(&:id)
+    end
+
+    # @return [Array<Weneedfeed::PageSchema>]
+    def page_schemata
+      @raw['pages'].map do |hash|
+        ::Weneedfeed::PageSchema.new(
+          id: hash['id'],
+          item_link_selector: hash['item_link_selector'],
+          item_time_selector: hash['item_time_selector'],
+          item_title_selector: hash['item_title_selector'],
+          item_selector: hash['item_selector'],
+          title: hash['title'],
+          url: hash['url'],
+        )
+      end
+    end
+  end
+end

data/lib/weneedfeed/scraping.rb

@@ -5,9 +5,9 @@ require 'nokogiri'
 
 module Weneedfeed
   class Scraping
-    # @param [String] item_description_selector
+    # @param [String, nil] item_description_selector
     # @param [String] item_link_selector
-    # @param [String] item_time_selector
+    # @param [String, nil] item_time_selector
     # @param [String] item_title_selector
     # @param [String] item_selector
     # @param [String] title

data/lib/weneedfeed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Weneedfeed
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

data/lib/weneedfeed/views/show_feed.rb

@@ -3,6 +3,8 @@
 module Weneedfeed
   module Views
     class ShowFeed < ::Hibana::View
+      include ::Weneedfeed::Helpers::Parameters
+
       # @param [Weneedfeed::Page] page
       def initialize(page:, **argv)
         super(**argv)
@@ -13,7 +15,7 @@ module Weneedfeed
 
       # @return [String]
       def top_page_path
-        request.path.delete_suffix(router.path(:feed, page_name: page_name))
+        request.path.delete_suffix(router.path(:feed, page_id: page_id))
       end
 
       # @return [Enumerable<Weneedfeed::Item>]
@@ -24,8 +26,8 @@ module Weneedfeed
       end
 
       # @return [String]
-      def page_name
-        request.env['router.params'][:page_name]
+      def page_id
+        path_parameters[:page_id]
       end
 
       # @return [Hanami::Router]

data/lib/weneedfeed/views/show_top_page.rb

@@ -3,10 +3,10 @@
 module Weneedfeed
   module Views
     class ShowTopPage < ::Hibana::View
-      # @param [Array<Weneedfeed::Page>] pages
-      def initialize(pages:, **argv)
+      # @param [Array<Hash>] page_schemata
+      def initialize(page_schemata:, **argv)
         super(**argv)
-        @pages = pages
+        @page_schemata = page_schemata
       end
 
       private
@@ -16,10 +16,10 @@ module Weneedfeed
         request.path.delete_suffix(router.path(:top_page))
       end
 
-      # @param [String] page_name
+      # @param [String] page_id
       # @return [String]
-      def feed_path(page_name:)
-        "#{base_path}#{router.path(:feed, page_name: page_name)}"
+      def feed_path(page_id:)
+        "#{base_path}#{router.path(:feed, page_id: page_id)}"
       end
 
       # @return [Hanami::Router]

data/templates/show_feed.xml.erb

@@ -12,7 +12,9 @@
     <item>
       <title><![CDATA[<%= item.title %>]]></title>
       <link><%= item.link %></link>
-      <pubDate><%= item.time.rfc822 %></pubDate>
+      <% if item.time %>
+        <pubDate><%= item.time.rfc822 %></pubDate>
+      <% end %>
       <description><![CDATA[<%= item.description %>]]></description>
       <content:encoded><![CDATA[<%= item.description %>]]></content:encoded>
       <guid isPermaLink="true"><%= item.link %></guid>

data/templates/show_top_page.html.erb

@@ -7,9 +7,9 @@
   </head>
   <body>
     <ul>
-      <% @pages.each do |page_name, hash| %>
+      <% @page_schemata.each do |page_schema| %>
       <li>
-        <a href="<%= feed_path(page_name: page_name) %>"><%= hash['title'] %></a>
+        <a href="<%= feed_path(page_id: page_schema.id) %>"><%= page_schema.title %></a>
       </li>
       <% end %>
     </ul>

data/weneedfeed.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'faraday'
+  spec.add_runtime_dependency 'hanami-router', '2.0.0.alpha3'
   spec.add_runtime_dependency 'hibana'
   spec.add_runtime_dependency 'nokogiri'
   spec.add_runtime_dependency 'rack-capture', '>= 0.4.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: weneedfeed
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Ryo Nakamura
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-14 00:00:00.000000000 Z
+date: 2020-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: hanami-router
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.alpha3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.0.alpha3
 - !ruby/object:Gem::Dependency
   name: hibana
   requirement: !ruby/object:Gem::Requirement
@@ -106,10 +120,15 @@ files:
 - lib/weneedfeed/capture.rb
 - lib/weneedfeed/command.rb
 - lib/weneedfeed/controllers.rb
+- lib/weneedfeed/controllers/base.rb
 - lib/weneedfeed/controllers/show_feed.rb
 - lib/weneedfeed/controllers/show_top_page.rb
+- lib/weneedfeed/helpers.rb
+- lib/weneedfeed/helpers/parameters.rb
 - lib/weneedfeed/item.rb
 - lib/weneedfeed/page.rb
+- lib/weneedfeed/page_schema.rb
+- lib/weneedfeed/schema.rb
 - lib/weneedfeed/scraping.rb
 - lib/weneedfeed/version.rb
 - lib/weneedfeed/views.rb