newsfetcher 0.84

data/lib/newsfetcher.rb

@@ -0,0 +1,86 @@
+# system
+
+require 'logger'
+require 'ostruct'
+
+# gems
+
+require 'addressable/uri'
+require 'addressable-prettify'
+require 'erb'
+require 'faraday'
+require 'feedjira'
+require 'loofah'
+require 'mail'
+require 'maildir'
+require 'nokogiri'
+require 'path'
+require 'rubypants'
+require 'sassc'
+require 'set_params'
+require 'simple-builder'
+require 'simple-command-parser'
+require 'simple-configurator'
+require 'simple-printer'
+
+module NewsFetcher
+
+  ConfigFileName = 'config.json'
+  ItemHistoryFileName = 'item_history.jsonl'
+  ResponseHistoryFileName = 'response_history.jsonl'
+  DefaultProfileDir = '~/.newsfetcher'
+  SubscriptionsDirName = 'subscriptions'
+  # see https://stackoverflow.com/questions/595616/what-is-the-correct-mime-type-to-use-for-an-rss-feed
+  # ordered by preference
+  FeedTypes = %w[
+    application/atom+xml
+    application/rss+xml
+    application/rdf+xml
+    text/atom+xml
+    text/rss+xml
+    text/rdf+xml
+    application/xml
+    text/xml
+  ]
+end
+
+require 'newsfetcher/extensions/string'
+require 'newsfetcher/extensions/mail'
+require 'newsfetcher/extensions/addressable-uri'
+
+require 'newsfetcher/error'
+require 'newsfetcher/fetcher'
+require 'newsfetcher/history'
+require 'newsfetcher/item'
+require 'newsfetcher/profile'
+require 'newsfetcher/scrubber'
+require 'newsfetcher/subscription'
+
+module NewsFetcher
+
+  DaySecs = 24 * 60 * 60
+  BaseConfig = Simple::Configurator.define(
+    max_threads: 100,
+    log_level: { default: :warn, converter: proc { |o| o.downcase.to_sym } },
+    max_age: 30 * DaySecs,
+    main_stylesheet: File.join(File.dirname(__FILE__), '../message/stylesheet.css'),
+    aux_stylesheets: nil,
+    delivery_method: { converter: :to_sym },
+    delivery_params: { default: {} },
+    mail_from: nil,
+    mail_to: nil,
+    mail_subject: '[<%= subscription_id %>] <%= item_title %>',
+    uri: proc { |o| Addressable::URI.parse(o) },
+    title: nil,
+    disabled: false,
+    update_interval: 60 * 60,
+    ignore_uris: { default: [], converter: proc { |o| [o].flatten.compact.map { |r| Regexp.new(r) } } },
+    ignore_moved: false,
+    root_folder: nil,
+    consolidate: false,
+  )
+
+end
+
+require 'newsfetcher/command'
+Path.new(__FILE__).dirname.glob('newsfetcher/commands/*.rb').each { |p| require p }

data/message/stylesheet.css

@@ -0,0 +1,48 @@
+body {
+  -webkit-text-size-adjust: 100%;
+  -ms-text-size-adjust: 100%;
+}
+
+a {
+  text-decoration: none;
+}
+
+img {
+  max-width: 100%;
+  width: 100% !important;
+  height: auto;
+}
+
+h1 {
+  font-size: 150%;
+}
+
+h2 {
+  font-size: 125%;
+}
+
+h3 {
+  font-size: 100%;
+}
+
+hr {
+  margin-top: -0.5em;
+  margin-bottom: -0.5em;
+  border: none;
+  text-align: center;
+}
+
+hr:after {
+  display: inline-block;
+  content: "—";   /* em dash */
+}
+
+.header {
+  margin: 1em 0 1em 0;
+  padding: 0.5em;
+  background-color: LightGray;
+}
+
+.blockquote {
+  margin: 1em;
+}

data/newsfetcher.gemspec

@@ -0,0 +1,42 @@
+require_relative 'lib/newsfetcher/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'newsfetcher'
+  s.version       = NewsFetcher::VERSION
+  s.summary       = 'Monitor RSS/Atom feeds and deliver new items by email'
+  s.author        = 'John Labovitz'
+  s.email         = 'johnl@johnlabovitz.com'
+  s.description   = %q{
+    NewsFetcher monitors RSS and Atom feeds, tracking which items have been delivered,
+    and sends new feed items as individual HTML emails using configurable mail delivery methods.
+  }
+  s.license       = 'MIT'
+  s.homepage      = 'http://github.com/jslabovitz/newsfetcher'
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
+  s.require_path  = 'lib'
+
+  s.add_dependency 'addressable', '~> 2.8'
+  s.add_dependency 'addressable-prettify', '~> 0.2'
+  s.add_dependency 'erb', '~> 4.0'
+  s.add_dependency 'faraday', '~> 2.14'
+  s.add_dependency 'feedjira', '~> 3.2'
+  s.add_dependency 'loofah', '~> 2.24'
+  s.add_dependency 'mail', '~> 2.8'
+  s.add_dependency 'maildir', '~> 2.2'
+  s.add_dependency 'ostruct', '0.6'
+  s.add_dependency 'path', '~> 2.1'
+  s.add_dependency 'rubypants', '~> 0.7'
+  s.add_dependency 'sassc', '~> 2.4'
+  s.add_dependency 'set_params', '~> 0.3'
+  s.add_dependency 'simple-builder', '~> 0.4'
+  s.add_dependency 'simple-command-parser', '~> 1.1'
+  s.add_dependency 'simple-configurator', '~> 0.1'
+  s.add_dependency 'simple-printer', '~> 0.4'
+
+  s.add_development_dependency 'minitest', '~> 5.26'
+  s.add_development_dependency 'minitest-power_assert', '~> 0.3'
+  s.add_development_dependency 'rake', '~> 13.3'
+  s.add_development_dependency 'simple-rake-tasks', '~> 0.1'
+end

data/test/history_test.rb

@@ -0,0 +1,60 @@
+$VERBOSE = false
+
+require 'minitest/autorun'
+require 'minitest/power_assert'
+
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'newsfetcher'
+
+module NewsFetcher
+
+  class TestHistory < Minitest::Test
+
+    def setup
+      @file = Path.new('test/tmp/history.jsonl')
+      @file.unlink if @file.exist?
+      @history = History.new(file: @file, index_key: :name)
+      @times = [
+        Time.now - 2,
+        Time.now - 1,
+        Time.now - 0,
+      ]
+      @history << { time: @times[0], name: 'a' }
+      @history << { time: @times[1], name: 'b' }
+      @history << { time: @times[2], name: 'c' }
+    end
+
+    def test_add
+      assert { @history.file.exist? }
+      assert { @history.size == 3 }
+    end
+
+    def test_prune
+      @history.prune(before: @times[1])
+      assert { @history.size == 2 }
+      assert { @history.entries[0].time == @times[1] }
+      assert { @history.entries[1].time == @times[2] }
+    end
+
+    def test_reset
+      @history.reset
+      assert { @history.size == 0 }
+      assert { !@history.file.exist? }
+    end
+
+    def test_last_entry
+      last = @history.last_entry
+      assert { last != nil }
+      assert { last.time == @times[2] }
+    end
+
+    def test_index
+      entry = @history['a']
+      assert { entry != nil }
+      assert { entry.time == @times[0] }
+      assert { entry.name == 'a' }
+    end
+
+  end
+
+end

data/test/main_test.rb

@@ -0,0 +1,66 @@
+$VERBOSE = false
+# avoid annoying warning
+class Object
+  def tainted?; false; end
+end
+
+require 'minitest/autorun'
+require 'minitest/power_assert'
+
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'newsfetcher'
+
+module NewsFetcher
+
+  class MainTest < Minitest::Test
+
+    def setup
+      @tmp_dir = Path.new('test/tmp')
+      @tmp_dir.rmtree if @tmp_dir.exist?
+      @dir = @tmp_dir / 'newsfetcher'
+      @dir.mkpath
+      @msgs_dir = @tmp_dir / 'msgs'
+      config_file = @dir / ConfigFileName
+      config = BaseConfig.make(
+        mail_from: 'newsfetcher',
+        mail_to: 'johnl',
+        log_level: :debug,
+        root_folder: 'News',
+        delivery_method: :maildir,
+        delivery_params: { dir: @msgs_dir.to_s },
+      )
+      config.save(config_file)
+    end
+
+    def test_run
+      config_file = @dir / ConfigFileName
+      config = BaseConfig.load(config_file)
+      @profile = Profile.new(dir: @dir, config: config)
+      # add web sites
+      initial_subscriptions = [
+        ['https://johnlabovitz.com', 'mine'],
+        ['http://nytimes.com', 'world'],
+        ['http://elpais.com', 'world', 'elpais'],
+      ].map do |uri, path, id|
+        feeds = Fetcher.get(uri).find_feeds
+        assert { feeds.count == 1 }
+        feed = feeds.first
+        assert { feed }
+        uri = feed[:uri]
+        assert { uri }
+        dir = @profile.add_subscription(uri: uri, id: id, path: path)
+        assert { dir.exist? }
+        dir
+      end
+      subscriptions = @profile.find_subscriptions
+      assert { subscriptions.count == initial_subscriptions.count }
+      # update
+      subscriptions.each(&:update)
+      # find
+      subscription = @profile.find_subscriptions(ids: %w[world/nytimes-services-nyt-homepage]).first
+      assert { subscription.item_history_file.exist? }
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,374 @@
+--- !ruby/object:Gem::Specification
+name: newsfetcher
+version: !ruby/object:Gem::Version
+  version: '0.84'
+platform: ruby
+authors:
+- John Labovitz
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: addressable-prettify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: erb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: feedjira
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: loofah
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.24'
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: maildir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: path
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: rubypants
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: sassc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: set_params
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: simple-builder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: simple-command-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: simple-configurator
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: simple-printer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.26'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.26'
+- !ruby/object:Gem::Dependency
+  name: minitest-power_assert
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.3'
+- !ruby/object:Gem::Dependency
+  name: simple-rake-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: "\n    NewsFetcher monitors RSS and Atom feeds, tracking which items
+  have been delivered,\n    and sends new feed items as individual HTML emails using
+  configurable mail delivery methods.\n  "
+email: johnl@johnlabovitz.com
+executables:
+- newsfetcher
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO.md
+- bin/newsfetcher
+- lib/newsfetcher.rb
+- lib/newsfetcher/command.rb
+- lib/newsfetcher/commands/add.rb
+- lib/newsfetcher/commands/dir.rb
+- lib/newsfetcher/commands/disable.rb
+- lib/newsfetcher/commands/discover.rb
+- lib/newsfetcher/commands/edit.rb
+- lib/newsfetcher/commands/enable.rb
+- lib/newsfetcher/commands/fix.rb
+- lib/newsfetcher/commands/get.rb
+- lib/newsfetcher/commands/init.rb
+- lib/newsfetcher/commands/remove.rb
+- lib/newsfetcher/commands/reset.rb
+- lib/newsfetcher/commands/show.rb
+- lib/newsfetcher/commands/update.rb
+- lib/newsfetcher/error.rb
+- lib/newsfetcher/extensions/addressable-uri.rb
+- lib/newsfetcher/extensions/mail.rb
+- lib/newsfetcher/extensions/string.rb
+- lib/newsfetcher/fetcher.rb
+- lib/newsfetcher/history.rb
+- lib/newsfetcher/item.rb
+- lib/newsfetcher/profile.rb
+- lib/newsfetcher/scrubber.rb
+- lib/newsfetcher/subscription.rb
+- lib/newsfetcher/version.rb
+- message/stylesheet.css
+- newsfetcher.gemspec
+- test/history_test.rb
+- test/main_test.rb
+homepage: http://github.com/jslabovitz/newsfetcher
+licenses:
+- MIT
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.2
+specification_version: 4
+summary: Monitor RSS/Atom feeds and deliver new items by email
+test_files:
+- test/history_test.rb
+- test/main_test.rb