weneedfeed 0.1.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +6 -4
- data/.rspec +1 -0
- data/.rubocop.yml +3 -0
- data/CHANGELOG.md +70 -0
- data/Gemfile.lock +3 -1
- data/README.md +81 -22
- data/exe/weneedfeed +6 -0
- data/lib/weneedfeed.rb +1 -0
- data/lib/weneedfeed/application.rb +2 -2
- data/lib/weneedfeed/command.rb +63 -0
- data/lib/weneedfeed/controllers/show_feed.rb +5 -5
- data/lib/weneedfeed/controllers/show_top_page.rb +3 -1
- data/lib/weneedfeed/item.rb +25 -17
- data/lib/weneedfeed/page.rb +20 -20
- data/lib/weneedfeed/scraping.rb +20 -20
- data/lib/weneedfeed/version.rb +1 -1
- data/lib/weneedfeed/views/show_feed.rb +10 -0
- data/lib/weneedfeed/views/show_top_page.rb +18 -0
- data/templates/show_feed.xml.erb +5 -4
- data/templates/show_top_page.html.erb +1 -1
- data/weneedfeed.gemspec +5 -2
- metadata +24 -6
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 0fdaa4a0f724b7018d6b1b2b585aeaf970eff37538df7ca7aaf760fc217b56dc
         | 
| 4 | 
            +
              data.tar.gz: 9f34a11d59aaada2d5afd689263f722c0143af5e59b9741e7d6f34d16d807c57
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 878189a82437b83ed25d0c25c5a6de0f4b840c1daf117ccd83d31be1ecbbe2db0d97289bd5ab93e327508562843ddea253948cad63324d3557ab28a0d5052268
         | 
| 7 | 
            +
              data.tar.gz: 8c56392e937c331456b44c869a96316c892b5321fd85611325c91c3987de079d101144998a4399f4fa668c46181e3f0c51f79fe157bf333d211fcf5ae0aeec87
         | 
    
        data/.github/workflows/test.yml
    CHANGED
    
    | @@ -5,16 +5,18 @@ on: | |
| 5 5 | 
             
              push:
         | 
| 6 6 | 
             
                branches:
         | 
| 7 7 | 
             
                  - master
         | 
| 8 | 
            +
             | 
| 8 9 | 
             
            jobs:
         | 
| 9 10 | 
             
              build:
         | 
| 10 11 | 
             
                runs-on: ubuntu-18.04
         | 
| 11 12 | 
             
                steps:
         | 
| 12 13 | 
             
                  - uses: actions/checkout@v2
         | 
| 13 | 
            -
                  - uses:  | 
| 14 | 
            +
                  - uses: ruby/setup-ruby@v1
         | 
| 14 15 | 
             
                    with:
         | 
| 15 | 
            -
                       | 
| 16 | 
            -
             | 
| 17 | 
            -
                  -  | 
| 16 | 
            +
                      bundler-cache: true
         | 
| 17 | 
            +
                      ruby-version: 2.7.2
         | 
| 18 | 
            +
                  - uses: r7kamura/rubocop-problem-matchers-action@v1
         | 
| 19 | 
            +
                  - run: bundle exec rubocop --parallel
         | 
| 18 20 | 
             
                  - run: bundle exec rspec --force-color
         | 
| 19 21 |  | 
| 20 22 |  | 
    
        data/.rspec
    CHANGED
    
    
    
        data/.rubocop.yml
    CHANGED
    
    
    
        data/CHANGELOG.md
    ADDED
    
    | @@ -0,0 +1,70 @@ | |
| 1 | 
            +
            # Changelog
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            All notable changes to this project will be documented in this file.
         | 
| 4 | 
            +
             | 
| 5 | 
            +
            The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
         | 
| 6 | 
            +
            and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
         | 
| 7 | 
            +
             | 
| 8 | 
            +
            ## Unreleased
         | 
| 9 | 
            +
             | 
| 10 | 
            +
            ## 0.5.0 - 2020-11-14
         | 
| 11 | 
            +
             | 
| 12 | 
            +
            ### Added
         | 
| 13 | 
            +
             | 
| 14 | 
            +
            - Add datetime attribute support on time element.
         | 
| 15 | 
            +
             | 
| 16 | 
            +
            ### Changed
         | 
| 17 | 
            +
             | 
| 18 | 
            +
            - Change schema format about selectors.
         | 
| 19 | 
            +
             | 
| 20 | 
            +
            ## 0.4.1 - 2020-11-08
         | 
| 21 | 
            +
             | 
| 22 | 
            +
            ### Fixed
         | 
| 23 | 
            +
             | 
| 24 | 
            +
            - Fix error when item description is not found.
         | 
| 25 | 
            +
             | 
| 26 | 
            +
            ## 0.4.0 - 2020-11-08
         | 
| 27 | 
            +
             | 
| 28 | 
            +
            ### Added
         | 
| 29 | 
            +
             | 
| 30 | 
            +
            - Add CSS selector support.
         | 
| 31 | 
            +
             | 
| 32 | 
            +
            ### Changed
         | 
| 33 | 
            +
             | 
| 34 | 
            +
            - Change schema key from `xpath` to `selectors`.
         | 
| 35 | 
            +
            - Change item_link_selector target from href attribute to a element.
         | 
| 36 | 
            +
            - Change some argument names from `_xpath` to `_selector`.
         | 
| 37 | 
            +
            - Change `weneedfeed build` description.
         | 
| 38 | 
            +
            - Change `weneedfeed server` description.
         | 
| 39 | 
            +
            - Change channel link from feed URL to top page URL.
         | 
| 40 | 
            +
            - Sort top page feeds by its title.
         | 
| 41 | 
            +
             | 
| 42 | 
            +
            ## 0.3.0 - 2020-11-08
         | 
| 43 | 
            +
             | 
| 44 | 
            +
            ### Added
         | 
| 45 | 
            +
             | 
| 46 | 
            +
            - Add channel description.
         | 
| 47 | 
            +
            - Add item description.
         | 
| 48 | 
            +
            - Add `weneedfeed build` command.
         | 
| 49 | 
            +
            - Add `weneedfeed server` command.
         | 
| 50 | 
            +
             | 
| 51 | 
            +
            ### Changed
         | 
| 52 | 
            +
             | 
| 53 | 
            +
            - Surround channel title by CDATA.
         | 
| 54 | 
            +
            - Change channel children order.
         | 
| 55 | 
            +
             | 
| 56 | 
            +
            ## 0.2.0 - 2020-11-07
         | 
| 57 | 
            +
             | 
| 58 | 
            +
            ### Changed
         | 
| 59 | 
            +
             | 
| 60 | 
            +
            - Change required ruby version from 2.4.0 to 2.5.0.
         | 
| 61 | 
            +
             | 
| 62 | 
            +
            ### Fixed
         | 
| 63 | 
            +
             | 
| 64 | 
            +
            - Fix feed URL when SCRIPT_NAME is used.
         | 
| 65 | 
            +
             | 
| 66 | 
            +
            ## 0.1.0 - 2020-10-29
         | 
| 67 | 
            +
             | 
| 68 | 
            +
            ### Added
         | 
| 69 | 
            +
             | 
| 70 | 
            +
            - 1st release.
         | 
    
        data/Gemfile.lock
    CHANGED
    
    | @@ -1,11 +1,12 @@ | |
| 1 1 | 
             
            PATH
         | 
| 2 2 | 
             
              remote: .
         | 
| 3 3 | 
             
              specs:
         | 
| 4 | 
            -
                weneedfeed (0. | 
| 4 | 
            +
                weneedfeed (0.5.0)
         | 
| 5 5 | 
             
                  faraday
         | 
| 6 6 | 
             
                  hibana
         | 
| 7 7 | 
             
                  nokogiri
         | 
| 8 8 | 
             
                  rack-capture (>= 0.4.0)
         | 
| 9 | 
            +
                  thor
         | 
| 9 10 |  | 
| 10 11 | 
             
            GEM
         | 
| 11 12 | 
             
              remote: https://rubygems.org/
         | 
| @@ -77,6 +78,7 @@ GEM | |
| 77 78 | 
             
                  parser (>= 2.7.1.5)
         | 
| 78 79 | 
             
                ruby-progressbar (1.10.1)
         | 
| 79 80 | 
             
                ruby2_keywords (0.0.2)
         | 
| 81 | 
            +
                thor (1.0.1)
         | 
| 80 82 | 
             
                tilt (2.0.10)
         | 
| 81 83 | 
             
                transproc (1.1.1)
         | 
| 82 84 | 
             
                unicode-display_width (1.7.0)
         | 
    
        data/README.md
    CHANGED
    
    | @@ -3,7 +3,7 @@ | |
| 3 3 | 
             
            [](https://rubygems.org/gems/weneedfeed)
         | 
| 4 4 | 
             
            [](https://github.com/r7kamura/weneedfeed/actions?query=workflow%3Atest)
         | 
| 5 5 |  | 
| 6 | 
            -
            Generate feeds from URL and  | 
| 6 | 
            +
            Generate feeds from URL and selectors.
         | 
| 7 7 |  | 
| 8 8 | 
             
            ## Installation
         | 
| 9 9 |  | 
| @@ -17,6 +17,7 @@ And then execute: | |
| 17 17 |  | 
| 18 18 | 
             
            ```sh
         | 
| 19 19 | 
             
            bundle install
         | 
| 20 | 
            +
            ```
         | 
| 20 21 |  | 
| 21 22 | 
             
            Or install it yourself as:
         | 
| 22 23 |  | 
| @@ -24,38 +25,96 @@ Or install it yourself as: | |
| 24 25 | 
             
            gem install weneedfeed
         | 
| 25 26 | 
             
            ```
         | 
| 26 27 |  | 
| 27 | 
            -
            ##  | 
| 28 | 
            +
            ## Schema
         | 
| 28 29 |  | 
| 29 | 
            -
             | 
| 30 | 
            +
            You need to write a schema file named with `weneedfeed.yml` to use this gem.
         | 
| 31 | 
            +
             | 
| 32 | 
            +
            ### Example
         | 
| 30 33 |  | 
| 31 34 | 
             
            ```yaml
         | 
| 32 35 | 
             
            pages:
         | 
| 33 | 
            -
               | 
| 34 | 
            -
                title: example  | 
| 35 | 
            -
                url: http://example.com/
         | 
| 36 | 
            -
                 | 
| 37 | 
            -
             | 
| 38 | 
            -
             | 
| 39 | 
            -
             | 
| 40 | 
            -
             | 
| 41 | 
            -
             | 
| 36 | 
            +
              example1:
         | 
| 37 | 
            +
                title: example site 1
         | 
| 38 | 
            +
                url: http://example.com/1
         | 
| 39 | 
            +
                item_selector: li
         | 
| 40 | 
            +
                item_description_selector: p:nth-child(3)
         | 
| 41 | 
            +
                item_link_selector: a
         | 
| 42 | 
            +
                item_time_selector: time
         | 
| 43 | 
            +
                item_title_selector: p:nth-child(2)
         | 
| 44 | 
            +
              example2:
         | 
| 45 | 
            +
                title: example site 2
         | 
| 46 | 
            +
                url: http://example.com/2
         | 
| 47 | 
            +
                item_selector: //li
         | 
| 48 | 
            +
                item_description_selector: .//p[3]
         | 
| 49 | 
            +
                item_link_selector: .//a/@href
         | 
| 50 | 
            +
                item_time_selector: .//time
         | 
| 51 | 
            +
                item_title_selector: .//p[2]
         | 
| 42 52 | 
             
            ```
         | 
| 43 53 |  | 
| 44 | 
            -
             | 
| 54 | 
            +
            ### `title`
         | 
| 45 55 |  | 
| 46 | 
            -
             | 
| 47 | 
            -
             | 
| 56 | 
            +
            Feed title, used for RSS `<title>` element in `<channel>` element.
         | 
| 57 | 
            +
             | 
| 58 | 
            +
            ### `url`
         | 
| 59 | 
            +
             | 
| 60 | 
            +
            URL to fetch HTML page for building feed.
         | 
| 61 | 
            +
             | 
| 62 | 
            +
            ### `item_selector`
         | 
| 63 | 
            +
             | 
| 64 | 
            +
            CSS or XPath selector to search each item, equivalent unit to RSS `<item>` element.
         | 
| 65 | 
            +
             | 
| 66 | 
            +
            ### `item_link_selector`
         | 
| 67 | 
            +
             | 
| 68 | 
            +
            CSS or XPath selector to find `<a>` element in each item, used for `<link>` in `<item>`.
         | 
| 69 | 
            +
             | 
| 70 | 
            +
            ### `item_time_selector`
         | 
| 71 | 
            +
             | 
| 72 | 
            +
            CSS or XPath selector to find element with datetime information in each item, used for `<pubDate>` in `<item>`. Its `datetime` attribute or its inner HTML is used to calculate datetime.
         | 
| 73 | 
            +
             | 
| 74 | 
            +
            ### `item_title_selector`
         | 
| 75 | 
            +
             | 
| 76 | 
            +
            CSS or XPath selector to find element with title information in each item, used for `<pubDate>` in `<item>`.
         | 
| 77 | 
            +
             | 
| 78 | 
            +
            ### `item_description_selector`
         | 
| 79 | 
            +
             | 
| 80 | 
            +
            CSS or XPath selector to find element with description information in each item, used for `<description>` in `<item>`.
         | 
| 81 | 
            +
             | 
| 82 | 
            +
            ## Usage
         | 
| 83 | 
            +
             | 
| 84 | 
            +
            ### Build
         | 
| 85 | 
            +
             | 
| 86 | 
            +
            Run `weneedfeed build` to build static files.
         | 
| 87 | 
            +
             | 
| 88 | 
            +
            ```
         | 
| 89 | 
            +
            Usage:
         | 
| 90 | 
            +
              weneedfeed build --base-url=BASE_URL
         | 
| 91 | 
            +
             | 
| 92 | 
            +
            Options:
         | 
| 93 | 
            +
              --base-url=BASE_URL          # Base URL where to locate built files. (e.g. `"https://user.github.io/repo"`)
         | 
| 94 | 
            +
              [--schema-path=SCHEMA_PATH]  # Path to weneedfeed YAML schema file.
         | 
| 95 | 
            +
                                           # Default: weneedfeed.yml
         | 
| 96 | 
            +
             | 
| 97 | 
            +
            Build static files.
         | 
| 98 | 
            +
            ```
         | 
| 99 | 
            +
             | 
| 100 | 
            +
            ### Server
         | 
| 101 | 
            +
             | 
| 102 | 
            +
            Run `weneedfeed server` to run HTTP server.
         | 
| 103 | 
            +
             | 
| 104 | 
            +
            ```
         | 
| 105 | 
            +
            Usage:
         | 
| 106 | 
            +
              weneedfeed server
         | 
| 107 | 
            +
             | 
| 108 | 
            +
            Options:
         | 
| 109 | 
            +
              [--schema-path=SCHEMA_PATH]  # Path to weneedfeed YAML schema file.
         | 
| 110 | 
            +
                                           # Default: weneedfeed.yml
         | 
| 48 111 |  | 
| 49 | 
            -
             | 
| 50 | 
            -
              base_url: 'https://user.github.io/repo',
         | 
| 51 | 
            -
              schema_path: 'schema.yml'
         | 
| 52 | 
            -
            )
         | 
| 112 | 
            +
            Run HTTP server.
         | 
| 53 113 | 
             
            ```
         | 
| 54 114 |  | 
| 55 | 
            -
             | 
| 115 | 
            +
            ## GitHub Actions Integration
         | 
| 56 116 |  | 
| 57 | 
            -
            -  | 
| 58 | 
            -
            - output/feeds/example.xml
         | 
| 117 | 
            +
            Use [weneedfeed-action](https://github.com/r7kamura/weneedfeed-action) for invoking weneedfeed on GitHub Actions.
         | 
| 59 118 |  | 
| 60 119 | 
             
            ## Development
         | 
| 61 120 |  | 
    
        data/exe/weneedfeed
    ADDED
    
    
    
        data/lib/weneedfeed.rb
    CHANGED
    
    | @@ -5,6 +5,7 @@ require 'weneedfeed/version' | |
| 5 5 | 
             
            module Weneedfeed
         | 
| 6 6 | 
             
              autoload :Application, 'weneedfeed/application'
         | 
| 7 7 | 
             
              autoload :Capture, 'weneedfeed/capture'
         | 
| 8 | 
            +
              autoload :Command, 'weneedfeed/command'
         | 
| 8 9 | 
             
              autoload :Controllers, 'weneedfeed/controllers'
         | 
| 9 10 | 
             
              autoload :Item, 'weneedfeed/item'
         | 
| 10 11 | 
             
              autoload :Page, 'weneedfeed/page'
         | 
| @@ -5,8 +5,8 @@ require 'hibana' | |
| 5 5 | 
             
            module Weneedfeed
         | 
| 6 6 | 
             
              class Application < ::Hibana::Application
         | 
| 7 7 | 
             
                route do
         | 
| 8 | 
            -
                  get '/', to: ::Weneedfeed::Controllers::ShowTopPage
         | 
| 9 | 
            -
                  get '/feeds/:page_name.xml', to: ::Weneedfeed::Controllers::ShowFeed
         | 
| 8 | 
            +
                  get '/', to: ::Weneedfeed::Controllers::ShowTopPage, as: :top_page
         | 
| 9 | 
            +
                  get '/feeds/:page_name.xml', to: ::Weneedfeed::Controllers::ShowFeed, as: :feed
         | 
| 10 10 | 
             
                end
         | 
| 11 11 |  | 
| 12 12 | 
             
                # @param [Hash] schema
         | 
| @@ -0,0 +1,63 @@ | |
| 1 | 
            +
            # frozen_string_literal: true
         | 
| 2 | 
            +
             | 
| 3 | 
            +
            require 'rack'
         | 
| 4 | 
            +
            require 'thor'
         | 
| 5 | 
            +
            require 'yaml'
         | 
| 6 | 
            +
             | 
| 7 | 
            +
            module Weneedfeed
         | 
| 8 | 
            +
              class Command < ::Thor
         | 
| 9 | 
            +
                class << self
         | 
| 10 | 
            +
                  # @note Override for thor breaking change.
         | 
| 11 | 
            +
                  #   See https://github.com/erikhuda/thor/issues/244.
         | 
| 12 | 
            +
                  def exit_on_failure?
         | 
| 13 | 
            +
                    true
         | 
| 14 | 
            +
                  end
         | 
| 15 | 
            +
                end
         | 
| 16 | 
            +
             | 
| 17 | 
            +
                desc(
         | 
| 18 | 
            +
                  'build',
         | 
| 19 | 
            +
                  'Build static files.'
         | 
| 20 | 
            +
                )
         | 
| 21 | 
            +
             | 
| 22 | 
            +
                method_option(
         | 
| 23 | 
            +
                  :base_url,
         | 
| 24 | 
            +
                  desc: 'Base URL where to locate built files. (e.g. `"https://user.github.io/repo"`)',
         | 
| 25 | 
            +
                  required: true,
         | 
| 26 | 
            +
                  type: :string
         | 
| 27 | 
            +
                )
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                method_option(
         | 
| 30 | 
            +
                  :schema_path,
         | 
| 31 | 
            +
                  default: 'weneedfeed.yml',
         | 
| 32 | 
            +
                  desc: 'Path to weneedfeed YAML schema file.',
         | 
| 33 | 
            +
                  type: :string
         | 
| 34 | 
            +
                )
         | 
| 35 | 
            +
             | 
| 36 | 
            +
                # @param [String] base_url
         | 
| 37 | 
            +
                # @param [String] schema_path
         | 
| 38 | 
            +
                def build
         | 
| 39 | 
            +
                  ::Weneedfeed::Capture.call(
         | 
| 40 | 
            +
                    base_url: options[:base_url],
         | 
| 41 | 
            +
                    schema_path: options[:schema_path]
         | 
| 42 | 
            +
                  )
         | 
| 43 | 
            +
                end
         | 
| 44 | 
            +
             | 
| 45 | 
            +
                desc(
         | 
| 46 | 
            +
                  'server',
         | 
| 47 | 
            +
                  'Run HTTP server.'
         | 
| 48 | 
            +
                )
         | 
| 49 | 
            +
             | 
| 50 | 
            +
                method_option(
         | 
| 51 | 
            +
                  :schema_path,
         | 
| 52 | 
            +
                  default: 'weneedfeed.yml',
         | 
| 53 | 
            +
                  desc: 'Path to weneedfeed YAML schema file.',
         | 
| 54 | 
            +
                  type: :string
         | 
| 55 | 
            +
                )
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                def server
         | 
| 58 | 
            +
                  schema = ::YAML.load_file(options[:schema_path])
         | 
| 59 | 
            +
                  application = Weneedfeed::Application.new(schema: schema)
         | 
| 60 | 
            +
                  ::Rack::Handler.default.run(application)
         | 
| 61 | 
            +
                end
         | 
| 62 | 
            +
              end
         | 
| 63 | 
            +
            end
         | 
| @@ -20,11 +20,11 @@ module Weneedfeed | |
| 20 20 | 
             
                    end
         | 
| 21 21 |  | 
| 22 22 | 
             
                    scraping = ::Weneedfeed::Scraping.new(
         | 
| 23 | 
            -
                       | 
| 24 | 
            -
                       | 
| 25 | 
            -
                       | 
| 26 | 
            -
                       | 
| 27 | 
            -
                       | 
| 23 | 
            +
                      item_description_selector: properties['item_description_selector'],
         | 
| 24 | 
            +
                      item_link_selector: properties['item_link_selector'],
         | 
| 25 | 
            +
                      item_time_selector: properties['item_time_selector'],
         | 
| 26 | 
            +
                      item_title_selector: properties['item_title_selector'],
         | 
| 27 | 
            +
                      item_selector: properties['item_selector'],
         | 
| 28 28 | 
             
                      title: properties['title'],
         | 
| 29 29 | 
             
                      url: properties['url']
         | 
| 30 30 | 
             
                    )
         | 
    
        data/lib/weneedfeed/item.rb
    CHANGED
    
    | @@ -15,38 +15,38 @@ module Weneedfeed | |
| 15 15 | 
             
                  end
         | 
| 16 16 | 
             
                end
         | 
| 17 17 |  | 
| 18 | 
            -
                # @param [String]  | 
| 19 | 
            -
                # @param [String]  | 
| 18 | 
            +
                # @param [String] description_selector
         | 
| 19 | 
            +
                # @param [String] link_selector
         | 
| 20 20 | 
             
                # @param [Nokogiri::Node] node
         | 
| 21 | 
            -
                # @param [String]  | 
| 22 | 
            -
                # @param [String]  | 
| 21 | 
            +
                # @param [String] time_selector
         | 
| 22 | 
            +
                # @param [String] title_selector
         | 
| 23 23 | 
             
                # @param [String] url
         | 
| 24 24 | 
             
                def initialize(
         | 
| 25 | 
            -
                   | 
| 26 | 
            -
                   | 
| 25 | 
            +
                  description_selector:,
         | 
| 26 | 
            +
                  link_selector:,
         | 
| 27 27 | 
             
                  node:,
         | 
| 28 | 
            -
                   | 
| 29 | 
            -
                   | 
| 28 | 
            +
                  time_selector:,
         | 
| 29 | 
            +
                  title_selector:,
         | 
| 30 30 | 
             
                  url:
         | 
| 31 31 | 
             
                )
         | 
| 32 | 
            -
                  @ | 
| 33 | 
            -
                  @ | 
| 32 | 
            +
                  @description_selector = description_selector
         | 
| 33 | 
            +
                  @link_selector = link_selector
         | 
| 34 34 | 
             
                  @node = node
         | 
| 35 | 
            -
                  @ | 
| 36 | 
            -
                  @ | 
| 35 | 
            +
                  @time_selector = time_selector
         | 
| 36 | 
            +
                  @title_selector = title_selector
         | 
| 37 37 | 
             
                  @url = url
         | 
| 38 38 | 
             
                end
         | 
| 39 39 |  | 
| 40 40 | 
             
                # @return [String, nil]
         | 
| 41 41 | 
             
                def description
         | 
| 42 | 
            -
                  @node. | 
| 42 | 
            +
                  @node.at(@description_selector)&.inner_html
         | 
| 43 43 | 
             
                end
         | 
| 44 44 |  | 
| 45 45 | 
             
                # @return [String]
         | 
| 46 46 | 
             
                def link
         | 
| 47 47 | 
             
                  ::URI.join(
         | 
| 48 48 | 
             
                    @url,
         | 
| 49 | 
            -
                    @node. | 
| 49 | 
            +
                    @node.at(@link_selector)['href']
         | 
| 50 50 | 
             
                  ).to_s
         | 
| 51 51 | 
             
                end
         | 
| 52 52 |  | 
| @@ -55,16 +55,24 @@ module Weneedfeed | |
| 55 55 | 
             
                  self.class.parse_time(time_string)
         | 
| 56 56 | 
             
                end
         | 
| 57 57 |  | 
| 58 | 
            -
                # @return [String | 
| 58 | 
            +
                # @return [String]
         | 
| 59 59 | 
             
                def title
         | 
| 60 | 
            -
                  @node. | 
| 60 | 
            +
                  @node.at(@title_selector).inner_text
         | 
| 61 61 | 
             
                end
         | 
| 62 62 |  | 
| 63 63 | 
             
                private
         | 
| 64 64 |  | 
| 65 | 
            +
                # @return [Nokogiri::Node, nil]
         | 
| 66 | 
            +
                def time_node
         | 
| 67 | 
            +
                  @node.at(@time_selector)
         | 
| 68 | 
            +
                end
         | 
| 69 | 
            +
             | 
| 65 70 | 
             
                # @return [String]
         | 
| 66 71 | 
             
                def time_string
         | 
| 67 | 
            -
                   | 
| 72 | 
            +
                  node = time_node
         | 
| 73 | 
            +
                  return unless node
         | 
| 74 | 
            +
             | 
| 75 | 
            +
                  node['datetime'] || node.inner_html
         | 
| 68 76 | 
             
                end
         | 
| 69 77 | 
             
              end
         | 
| 70 78 | 
             
            end
         | 
    
        data/lib/weneedfeed/page.rb
    CHANGED
    
    | @@ -8,29 +8,29 @@ module Weneedfeed | |
| 8 8 | 
             
                # @return [String]
         | 
| 9 9 | 
             
                attr_reader :url
         | 
| 10 10 |  | 
| 11 | 
            -
                # @param [String]  | 
| 12 | 
            -
                # @param [String]  | 
| 13 | 
            -
                # @param [String]  | 
| 14 | 
            -
                # @param [String]  | 
| 15 | 
            -
                # @param [String]  | 
| 11 | 
            +
                # @param [String] item_description_selector
         | 
| 12 | 
            +
                # @param [String] item_link_selector
         | 
| 13 | 
            +
                # @param [String] item_time_selector
         | 
| 14 | 
            +
                # @param [String] item_title_selector
         | 
| 15 | 
            +
                # @param [String] item_selector
         | 
| 16 16 | 
             
                # @param [Nokogiri::Node] node
         | 
| 17 17 | 
             
                # @param [String] title
         | 
| 18 18 | 
             
                # @param [String] url
         | 
| 19 19 | 
             
                def initialize(
         | 
| 20 | 
            -
                   | 
| 21 | 
            -
                   | 
| 22 | 
            -
                   | 
| 23 | 
            -
                   | 
| 24 | 
            -
                   | 
| 20 | 
            +
                  item_description_selector:,
         | 
| 21 | 
            +
                  item_link_selector:,
         | 
| 22 | 
            +
                  item_time_selector:,
         | 
| 23 | 
            +
                  item_title_selector:,
         | 
| 24 | 
            +
                  item_selector:,
         | 
| 25 25 | 
             
                  node:,
         | 
| 26 26 | 
             
                  title:,
         | 
| 27 27 | 
             
                  url:
         | 
| 28 28 | 
             
                )
         | 
| 29 | 
            -
                  @ | 
| 30 | 
            -
                  @ | 
| 31 | 
            -
                  @ | 
| 32 | 
            -
                  @ | 
| 33 | 
            -
                  @ | 
| 29 | 
            +
                  @item_description_selector = item_description_selector
         | 
| 30 | 
            +
                  @item_link_selector = item_link_selector
         | 
| 31 | 
            +
                  @item_time_selector = item_time_selector
         | 
| 32 | 
            +
                  @item_title_selector = item_title_selector
         | 
| 33 | 
            +
                  @item_selector = item_selector
         | 
| 34 34 | 
             
                  @node = node
         | 
| 35 35 | 
             
                  @title = title
         | 
| 36 36 | 
             
                  @url = url
         | 
| @@ -38,13 +38,13 @@ module Weneedfeed | |
| 38 38 |  | 
| 39 39 | 
             
                # @return [Array<Weneedfeed::Item>]
         | 
| 40 40 | 
             
                def items
         | 
| 41 | 
            -
                  @node. | 
| 41 | 
            +
                  @node.search(@item_selector).map do |node|
         | 
| 42 42 | 
             
                    ::Weneedfeed::Item.new(
         | 
| 43 | 
            -
                       | 
| 44 | 
            -
                       | 
| 43 | 
            +
                      description_selector: @item_description_selector,
         | 
| 44 | 
            +
                      link_selector: @item_link_selector,
         | 
| 45 45 | 
             
                      node: node,
         | 
| 46 | 
            -
                       | 
| 47 | 
            -
                       | 
| 46 | 
            +
                      time_selector: @item_time_selector,
         | 
| 47 | 
            +
                      title_selector: @item_title_selector,
         | 
| 48 48 | 
             
                      url: @url
         | 
| 49 49 | 
             
                    )
         | 
| 50 50 | 
             
                  end
         | 
    
        data/lib/weneedfeed/scraping.rb
    CHANGED
    
    | @@ -5,27 +5,27 @@ require 'nokogiri' | |
| 5 5 |  | 
| 6 6 | 
             
            module Weneedfeed
         | 
| 7 7 | 
             
              class Scraping
         | 
| 8 | 
            -
                # @param [String]  | 
| 9 | 
            -
                # @param [String]  | 
| 10 | 
            -
                # @param [String]  | 
| 11 | 
            -
                # @param [String]  | 
| 12 | 
            -
                # @param [String]  | 
| 8 | 
            +
                # @param [String] item_description_selector
         | 
| 9 | 
            +
                # @param [String] item_link_selector
         | 
| 10 | 
            +
                # @param [String] item_time_selector
         | 
| 11 | 
            +
                # @param [String] item_title_selector
         | 
| 12 | 
            +
                # @param [String] item_selector
         | 
| 13 13 | 
             
                # @param [String] title
         | 
| 14 14 | 
             
                # @param [String] url
         | 
| 15 15 | 
             
                def initialize(
         | 
| 16 | 
            -
                   | 
| 17 | 
            -
                   | 
| 18 | 
            -
                   | 
| 19 | 
            -
                   | 
| 20 | 
            -
                   | 
| 16 | 
            +
                  item_description_selector:,
         | 
| 17 | 
            +
                  item_link_selector:,
         | 
| 18 | 
            +
                  item_time_selector:,
         | 
| 19 | 
            +
                  item_title_selector:,
         | 
| 20 | 
            +
                  item_selector:,
         | 
| 21 21 | 
             
                  title:,
         | 
| 22 22 | 
             
                  url:
         | 
| 23 23 | 
             
                )
         | 
| 24 | 
            -
                  @ | 
| 25 | 
            -
                  @ | 
| 26 | 
            -
                  @ | 
| 27 | 
            -
                  @ | 
| 28 | 
            -
                  @ | 
| 24 | 
            +
                  @item_description_selector = item_description_selector
         | 
| 25 | 
            +
                  @item_link_selector = item_link_selector
         | 
| 26 | 
            +
                  @item_time_selector = item_time_selector
         | 
| 27 | 
            +
                  @item_title_selector = item_title_selector
         | 
| 28 | 
            +
                  @item_selector = item_selector
         | 
| 29 29 | 
             
                  @title = title
         | 
| 30 30 | 
             
                  @url = url
         | 
| 31 31 | 
             
                end
         | 
| @@ -34,11 +34,11 @@ module Weneedfeed | |
| 34 34 | 
             
                def call
         | 
| 35 35 | 
             
                  ::Weneedfeed::Page.new(
         | 
| 36 36 | 
             
                    node: parsed_body,
         | 
| 37 | 
            -
                     | 
| 38 | 
            -
                     | 
| 39 | 
            -
                     | 
| 40 | 
            -
                     | 
| 41 | 
            -
                     | 
| 37 | 
            +
                    item_description_selector: @item_description_selector,
         | 
| 38 | 
            +
                    item_selector: @item_selector,
         | 
| 39 | 
            +
                    item_link_selector: @item_link_selector,
         | 
| 40 | 
            +
                    item_time_selector: @item_time_selector,
         | 
| 41 | 
            +
                    item_title_selector: @item_title_selector,
         | 
| 42 42 | 
             
                    title: @title,
         | 
| 43 43 | 
             
                    url: @url
         | 
| 44 44 | 
             
                  )
         | 
    
        data/lib/weneedfeed/version.rb
    CHANGED
    
    
| @@ -11,6 +11,11 @@ module Weneedfeed | |
| 11 11 |  | 
| 12 12 | 
             
                  private
         | 
| 13 13 |  | 
| 14 | 
            +
                  # @return [String]
         | 
| 15 | 
            +
                  def top_page_path
         | 
| 16 | 
            +
                    request.path.delete_suffix(router.path(:feed, page_name: page_name))
         | 
| 17 | 
            +
                  end
         | 
| 18 | 
            +
             | 
| 14 19 | 
             
                  # @return [Enumerable<Weneedfeed::Item>]
         | 
| 15 20 | 
             
                  def items
         | 
| 16 21 | 
             
                    @page.items.sort_by do |item|
         | 
| @@ -22,6 +27,11 @@ module Weneedfeed | |
| 22 27 | 
             
                  def page_name
         | 
| 23 28 | 
             
                    request.env['router.params'][:page_name]
         | 
| 24 29 | 
             
                  end
         | 
| 30 | 
            +
             | 
| 31 | 
            +
                  # @return [Hanami::Router]
         | 
| 32 | 
            +
                  def router
         | 
| 33 | 
            +
                    ::Weneedfeed::Application.router
         | 
| 34 | 
            +
                  end
         | 
| 25 35 | 
             
                end
         | 
| 26 36 | 
             
              end
         | 
| 27 37 | 
             
            end
         | 
| @@ -8,6 +8,24 @@ module Weneedfeed | |
| 8 8 | 
             
                    super(**argv)
         | 
| 9 9 | 
             
                    @pages = pages
         | 
| 10 10 | 
             
                  end
         | 
| 11 | 
            +
             | 
| 12 | 
            +
                  private
         | 
| 13 | 
            +
             | 
| 14 | 
            +
                  # @return [String]
         | 
| 15 | 
            +
                  def base_path
         | 
| 16 | 
            +
                    request.path.delete_suffix(router.path(:top_page))
         | 
| 17 | 
            +
                  end
         | 
| 18 | 
            +
             | 
| 19 | 
            +
                  # @param [String] page_name
         | 
| 20 | 
            +
                  # @return [String]
         | 
| 21 | 
            +
                  def feed_path(page_name:)
         | 
| 22 | 
            +
                    "#{base_path}#{router.path(:feed, page_name: page_name)}"
         | 
| 23 | 
            +
                  end
         | 
| 24 | 
            +
             | 
| 25 | 
            +
                  # @return [Hanami::Router]
         | 
| 26 | 
            +
                  def router
         | 
| 27 | 
            +
                    ::Weneedfeed::Application.router
         | 
| 28 | 
            +
                  end
         | 
| 11 29 | 
             
                end
         | 
| 12 30 | 
             
              end
         | 
| 13 31 | 
             
            end
         | 
    
        data/templates/show_feed.xml.erb
    CHANGED
    
    | @@ -3,16 +3,17 @@ | |
| 3 3 | 
             
              xmlns:atom="http://www.w3.org/2005/Atom"
         | 
| 4 4 | 
             
              xmlns:content="http://purl.org/rss/1.0/modules/content/">
         | 
| 5 5 | 
             
              <channel>
         | 
| 6 | 
            -
                < | 
| 7 | 
            -
                <link><%= "#{request.base_url}#{ | 
| 8 | 
            -
                <atom:link href="<%= request.base_url  | 
| 9 | 
            -
                < | 
| 6 | 
            +
                <title><![CDATA[<%= @page.title %>]]></title>
         | 
| 7 | 
            +
                <link><%= "#{request.base_url}#{top_page_path}" %></link>
         | 
| 8 | 
            +
                <atom:link href="<%= "#{request.base_url}#{request.path}" %>" rel="self"/>
         | 
| 9 | 
            +
                <description><![CDATA[Recent content on <%= @page.title %>]]></description>
         | 
| 10 10 | 
             
                <lastBuildDate><%= Time.now.rfc822 %></lastBuildDate>
         | 
| 11 11 | 
             
                <% items.each do |item| %>
         | 
| 12 12 | 
             
                <item>
         | 
| 13 13 | 
             
                  <title><![CDATA[<%= item.title %>]]></title>
         | 
| 14 14 | 
             
                  <link><%= item.link %></link>
         | 
| 15 15 | 
             
                  <pubDate><%= item.time.rfc822 %></pubDate>
         | 
| 16 | 
            +
                  <description><![CDATA[<%= item.description %>]]></description>
         | 
| 16 17 | 
             
                  <content:encoded><![CDATA[<%= item.description %>]]></content:encoded>
         | 
| 17 18 | 
             
                  <guid isPermaLink="true"><%= item.link %></guid>
         | 
| 18 19 | 
             
                </item>
         | 
    
        data/weneedfeed.gemspec
    CHANGED
    
    | @@ -8,10 +8,10 @@ Gem::Specification.new do |spec| | |
| 8 8 | 
             
              spec.authors       = ['Ryo Nakamura']
         | 
| 9 9 | 
             
              spec.email         = ['r7kamura@gmail.com']
         | 
| 10 10 |  | 
| 11 | 
            -
              spec.summary       = 'Generate feeds from URL and  | 
| 11 | 
            +
              spec.summary       = 'Generate feeds from URL and selectors.'
         | 
| 12 12 | 
             
              spec.homepage      = 'https://github.com/r7kamura/weneedfeed'
         | 
| 13 13 | 
             
              spec.license       = 'MIT'
         | 
| 14 | 
            -
              spec.required_ruby_version = Gem::Requirement.new('>= 2. | 
| 14 | 
            +
              spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
         | 
| 15 15 |  | 
| 16 16 | 
             
              spec.metadata['homepage_uri'] = spec.homepage
         | 
| 17 17 | 
             
              spec.metadata['source_code_uri'] = spec.homepage
         | 
| @@ -21,10 +21,13 @@ Gem::Specification.new do |spec| | |
| 21 21 | 
             
              spec.files = Dir.chdir(File.expand_path(__dir__)) do
         | 
| 22 22 | 
             
                `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
         | 
| 23 23 | 
             
              end
         | 
| 24 | 
            +
              spec.bindir = 'exe'
         | 
| 25 | 
            +
              spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
         | 
| 24 26 | 
             
              spec.require_paths = ['lib']
         | 
| 25 27 |  | 
| 26 28 | 
             
              spec.add_runtime_dependency 'faraday'
         | 
| 27 29 | 
             
              spec.add_runtime_dependency 'hibana'
         | 
| 28 30 | 
             
              spec.add_runtime_dependency 'nokogiri'
         | 
| 29 31 | 
             
              spec.add_runtime_dependency 'rack-capture', '>= 0.4.0'
         | 
| 32 | 
            +
              spec.add_runtime_dependency 'thor'
         | 
| 30 33 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: weneedfeed
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.5.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Ryo Nakamura
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 | 
            -
            bindir:  | 
| 9 | 
            +
            bindir: exe
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2020-11- | 
| 11 | 
            +
            date: 2020-11-14 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: faraday
         | 
| @@ -66,10 +66,25 @@ dependencies: | |
| 66 66 | 
             
                - - ">="
         | 
| 67 67 | 
             
                  - !ruby/object:Gem::Version
         | 
| 68 68 | 
             
                    version: 0.4.0
         | 
| 69 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 70 | 
            +
              name: thor
         | 
| 71 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 72 | 
            +
                requirements:
         | 
| 73 | 
            +
                - - ">="
         | 
| 74 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 75 | 
            +
                    version: '0'
         | 
| 76 | 
            +
              type: :runtime
         | 
| 77 | 
            +
              prerelease: false
         | 
| 78 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 79 | 
            +
                requirements:
         | 
| 80 | 
            +
                - - ">="
         | 
| 81 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 82 | 
            +
                    version: '0'
         | 
| 69 83 | 
             
            description: 
         | 
| 70 84 | 
             
            email:
         | 
| 71 85 | 
             
            - r7kamura@gmail.com
         | 
| 72 | 
            -
            executables: | 
| 86 | 
            +
            executables:
         | 
| 87 | 
            +
            - weneedfeed
         | 
| 73 88 | 
             
            extensions: []
         | 
| 74 89 | 
             
            extra_rdoc_files: []
         | 
| 75 90 | 
             
            files:
         | 
| @@ -77,6 +92,7 @@ files: | |
| 77 92 | 
             
            - ".gitignore"
         | 
| 78 93 | 
             
            - ".rspec"
         | 
| 79 94 | 
             
            - ".rubocop.yml"
         | 
| 95 | 
            +
            - CHANGELOG.md
         | 
| 80 96 | 
             
            - Gemfile
         | 
| 81 97 | 
             
            - Gemfile.lock
         | 
| 82 98 | 
             
            - LICENSE.txt
         | 
| @@ -84,9 +100,11 @@ files: | |
| 84 100 | 
             
            - Rakefile
         | 
| 85 101 | 
             
            - bin/console
         | 
| 86 102 | 
             
            - bin/setup
         | 
| 103 | 
            +
            - exe/weneedfeed
         | 
| 87 104 | 
             
            - lib/weneedfeed.rb
         | 
| 88 105 | 
             
            - lib/weneedfeed/application.rb
         | 
| 89 106 | 
             
            - lib/weneedfeed/capture.rb
         | 
| 107 | 
            +
            - lib/weneedfeed/command.rb
         | 
| 90 108 | 
             
            - lib/weneedfeed/controllers.rb
         | 
| 91 109 | 
             
            - lib/weneedfeed/controllers/show_feed.rb
         | 
| 92 110 | 
             
            - lib/weneedfeed/controllers/show_top_page.rb
         | 
| @@ -114,7 +132,7 @@ required_ruby_version: !ruby/object:Gem::Requirement | |
| 114 132 | 
             
              requirements:
         | 
| 115 133 | 
             
              - - ">="
         | 
| 116 134 | 
             
                - !ruby/object:Gem::Version
         | 
| 117 | 
            -
                  version: 2. | 
| 135 | 
            +
                  version: 2.5.0
         | 
| 118 136 | 
             
            required_rubygems_version: !ruby/object:Gem::Requirement
         | 
| 119 137 | 
             
              requirements:
         | 
| 120 138 | 
             
              - - ">="
         | 
| @@ -124,5 +142,5 @@ requirements: [] | |
| 124 142 | 
             
            rubygems_version: 3.1.2
         | 
| 125 143 | 
             
            signing_key: 
         | 
| 126 144 | 
             
            specification_version: 4
         | 
| 127 | 
            -
            summary: Generate feeds from URL and  | 
| 145 | 
            +
            summary: Generate feeds from URL and selectors.
         | 
| 128 146 | 
             
            test_files: []
         |