newsfetcher 0.84

data/lib/newsfetcher/fetcher.rb

@@ -0,0 +1,99 @@
+module NewsFetcher
+
+  class Fetcher
+
+    attr_reader   :uri
+    attr_accessor :timeout
+    attr_accessor :max_redirects
+    attr_accessor :moved
+    attr_accessor :actual_uri
+
+    include SetParams
+
+    def self.get(uri)
+      new(uri: uri).tap(&:get)
+    end
+
+    def initialize(**params)
+      super({
+        timeout: 30,
+        max_redirects: 5,
+      }.merge(params))
+    end
+
+    def uri=(uri)
+      @uri = Addressable::URI.parse(uri)
+    end
+
+    def success?
+      @response.success?
+    end
+
+    def response_status
+      @response&.status
+    end
+
+    def response_reason
+      @response&.reason_phrase
+    end
+
+    def content
+      @response&.body
+    end
+
+    def parse_feed
+      raise Error, "No response yet" unless @response
+      raise Error, "Can't parse feed in failed response" unless @response.success?
+      Feedjira.configure { |c| c.strip_whitespace = true }
+      begin
+        feedjira = Feedjira.parse(content)
+      rescue Feedjira::NoParserAvailable, Date::Error => e
+        raise Error, "Can't parse XML feed from #{@uri}: #{e}"
+      end
+      {
+        title: feedjira.title,
+        items: feedjira.entries.map { |e| Item.new(e) },
+      }
+    end
+
+    def find_feeds
+      raise Error, "No response yet" unless @response
+      raise Error, "Can't find feeds in failed response" unless @response.success?
+      html = Nokogiri::HTML::Document.parse(content)
+      html.xpath('//link[@rel="alternate"]').select { |link| FeedTypes.include?(link['type']) }.map do |link|
+        {
+          uri: @uri.join(link['href']),
+          type: link['type'],
+        }
+      end
+    end
+
+    def get
+      @actual_uri = @uri
+      redirects = 0
+      while redirects < @max_redirects do
+        @response = nil
+        connection = Faraday.new(
+          url: @actual_uri,
+          request: { timeout: @timeout },
+          ssl: { verify: false })
+        begin
+          @response = connection.get
+        rescue Faraday::Error => e
+          raise Error, "Error: #{e.message} (#{e.class})"
+        end
+        if (300...400).include?(@response.status)
+          @moved = (@response.status == 302)
+          location = @response.headers[:location] or raise Error, "No Location header found in redirect"
+          @actual_uri = @actual_uri.join(Addressable::URI.parse(location))
+          redirects += 1
+          next
+        end
+        return @response.success?
+      end
+      raise Error, "Too many redirects"
+    end
+
+  end
+
+end

data/lib/newsfetcher/history.rb

@@ -0,0 +1,101 @@
+module NewsFetcher
+
+  class History
+
+    attr_accessor :file
+    attr_accessor :entries
+
+    def initialize(file:, index_key: nil)
+      @file = Path.new(file)
+      @entries = []
+      @index = {}
+      @index_key = index_key
+      if @file.exist?
+        load_entries
+      else
+        @file.touch
+      end
+    end
+
+    def load_entries
+      @file.readlines.map do |line|
+        add_entry(Entry.from_json(line))
+      end
+    end
+
+    def <<(entry)
+      entry = Entry.new(**entry) if entry.kind_of?(Hash)
+      add_entry(entry)
+      @file.open('a') { |io| io.puts entry.to_json }
+    end
+
+    def [](key)
+      raise "No index_key defined" unless @index_key
+      @index[key]
+    end
+
+    def size
+      @entries.size
+    end
+
+    def last_entry
+      @entries.last
+    end
+
+    def prune(before:)
+      old = @entries.select { |e| e.time < before }
+      old.each { |e| delete_entry(e) }
+      save_entries
+      old
+    end
+
+    def reset
+      @entries = []
+      @index = {}
+      @file.unlink
+    end
+
+    private
+
+    def save_entries
+      new_file = @file.add_extension('.new')
+      new_file.open('w') do |io|
+        @entries.each { |e| io.puts e.to_json }
+      end
+      @file.unlink
+      new_file.rename(@file)
+    end
+
+    def add_entry(entry)
+      @entries << entry
+      @index[entry[@index_key]] = entry if @index_key
+    end
+
+    def delete_entry(entry)
+      @entries.delete(entry)
+      @index.delete(entry[@index_key]) if @index_key
+    end
+
+    class Entry < OpenStruct
+
+      def self.from_json(data)
+        json = JSON.parse(data, symbolize_names: true)
+        self.parse_json(json)
+        new(**json)
+      end
+
+      def self.parse_json(json)
+        json[:time] = Time.at(json[:time])
+      end
+
+      def to_json(*opts)
+        h = to_h
+        h[:time] = h[:time].to_i
+        h.to_json(*opts)
+      end
+
+    end
+
+  end
+
+end

data/lib/newsfetcher/item.rb

@@ -0,0 +1,60 @@
+module NewsFetcher
+
+  class Item
+
+    attr_accessor :id
+    attr_accessor :date
+    attr_accessor :title
+    attr_accessor :uri
+    attr_accessor :author
+    attr_accessor :content
+
+    include SetParams
+    include Simple::Printer::Printable
+
+    def initialize(entry)
+      if entry.url
+        begin
+          uri = Addressable::URI.parse(entry.url.strip)
+        rescue Addressable::URI::InvalidURIError => e
+          raise Error, "Can't parse URL for entry: #{entry.url.inspect}"
+        end
+      else
+        uri = nil
+      end
+      id = entry.entry_id || uri or raise Error, "Can't determine ID or URL for entry"
+      super(
+        id: id.to_s,
+        uri: uri,
+        date: entry.published || Time.now,
+        title: entry.title,
+        author: entry.respond_to?(:author) ? entry.author&.sub(/^by\s+/i, '') : nil,
+        content: (entry.content || entry.summary)&.to_s,
+      )
+    end
+
+    def printable
+      [
+        [:id, 'ID'],
+        :date,
+        :title,
+        :uri,
+        :author,
+      ]
+    end
+
+    def eql?(other)
+      @id.eql?(other.id)
+    end
+
+    def ==(other)
+      @id == other&.id
+    end
+
+    def age
+      Time.now - @date
+    end
+
+  end
+
+end

data/lib/newsfetcher/profile.rb

@@ -0,0 +1,89 @@
+module NewsFetcher
+
+  class Profile
+
+    attr_reader   :dir
+    attr_accessor :config
+
+    include SetParams
+
+    def initialize(params={})
+      super
+      setup_logger
+      setup_styles
+    end
+
+    def setup_logger
+      $logger = Logger.new(STDERR,
+        level: @config.log_level,
+        formatter: proc { |severity, timestamp, progname, msg|
+          "%s %5s: %s\n" % [timestamp.strftime('%FT%T%:z'), severity, msg]
+        },
+      )
+    end
+
+    def setup_styles
+      raise Error, "dir not set" unless @dir
+      @styles = [@config.main_stylesheet, @config.aux_stylesheets].flatten.compact.map do |file|
+        file = Path.new(file)
+        file = @dir / file if file.relative?
+        SassC::Engine.new(file.read, syntax: :scss, style: :compressed).render
+      end
+    end
+
+    def dir=(dir)
+      @dir = Path.new(dir).expand_path
+    end
+
+    def id
+      @dir.basename.to_s
+    end
+
+    def config_file
+      @dir / ConfigFileName
+    end
+
+    def subscriptions_dir
+      @dir / SubscriptionsDirName
+    end
+
+    def all_ids
+      subscriptions_dir.glob("**/#{ConfigFileName}").map { |p| p.dirname.relative_to(subscriptions_dir).to_s }
+    end
+
+    def find_subscriptions(ids: nil, status: nil, sort: nil)
+      status = [status].flatten.compact if status
+      sort ||= :id
+      ids = all_ids if ids.nil? || ids.empty?
+      subscriptions = ids.map do |id|
+        subscription_dir = subscriptions_dir / id
+        begin
+          subscription_config = @config.load(subscription_dir / ConfigFileName)
+        rescue Config::Error => e
+          raise "#{id}: #{e}"
+        end
+        Subscription.new(id: id, dir: subscription_dir, config: subscription_config, styles: @styles)
+      end
+      subscriptions.
+        select { |s| status.nil? || status.include?(s.status) }.
+        sort_by { |s| s.send(sort).to_s }
+    end
+
+    def add_subscription(uri:, id: nil, path: nil)
+      uri = Addressable::URI.parse(uri)
+      raise Error, "Bad URI: #{uri}" unless uri.absolute?
+      id ||= uri.make_subscription_id
+      id = "#{path}/#{id}" if path
+      subscription_dir = subscriptions_dir / id
+      raise Error, "Subscription already exists in #{subscription_dir}" if subscription_dir.exist?
+      subscription_dir.mkpath
+      config = @config.make(uri: uri)
+      config_file = subscription_dir / ConfigFileName
+      config.save(config_file)
+      $logger.info { "Saved new subscription to #{config_file}" }
+      subscription_dir
+    end
+
+  end
+
+end

data/lib/newsfetcher/scrubber.rb

@@ -0,0 +1,70 @@
+module NewsFetcher
+
+  module Scrubber
+    
+    def self.scrub_html(html)
+      Loofah.fragment(html).
+        scrub!(:prune).
+        scrub!(RemoveExtras).
+        scrub!(RemoveVoxFooter).
+        scrub!(RemoveStyling).
+        scrub!(ReplaceBlockquote).
+        to_html
+    end
+
+    def self.text_to_html(text)
+      Simple::Builder.build_html do |html|
+        text.split("\n").each_with_index do |line, i|
+          html.br unless i == 0
+          html.text(line)
+        end
+      end.to_html
+    end
+
+    RemoveVoxFooter = Loofah::Scrubber.new do |node|
+      if node.text == 'Help keep Vox free for all'
+        n = node
+        while (n = n.previous)
+          if n.name == 'hr'
+            n.remove
+            break
+          end
+        end
+        while node.next
+          node.next.remove
+        end
+        node.remove
+        Loofah::Scrubber::STOP
+      end
+    end
+
+    RemoveExtras = Loofah::Scrubber.new do |node|
+      if node.name == 'div' && node['class'] == 'feedflare'
+        node.remove
+      elsif node.name == 'img' && node['height'] == '1' && node['width'] == '1'
+        node.remove
+      elsif node.name == 'form'
+        node.replace(node.children)
+      end
+    end
+
+    RemoveStyling = Loofah::Scrubber.new do |node|
+      if %w{font big small}.include?(node.name)
+        node.replace(node.children)
+      else
+        node.remove_attribute('style') if node['style']
+        node.remove_attribute('class') if node['class']
+        node.remove_attribute('id') if node['id']
+      end
+    end
+
+    ReplaceBlockquote = Loofah::Scrubber.new do |node|
+      if node.name == 'blockquote'
+        node.name = 'div'
+        node['class'] = 'blockquote'
+      end
+    end
+
+  end
+
+end

data/lib/newsfetcher/subscription.rb

@@ -0,0 +1,274 @@
+module NewsFetcher
+
+  class Subscription
+
+    attr_accessor :id
+    attr_accessor :dir
+    attr_accessor :config
+    attr_accessor :styles
+    attr_accessor :items
+    attr_accessor :title
+
+    include SetParams
+    include Simple::Printer::Printable
+
+    def initialize(**params)
+      @title = nil
+      @items = []
+      super
+      @item_history = History.new(file: item_history_file, index_key: :id)
+      @response_history = History.new(file: response_history_file)
+    end
+
+    def inspect
+      to_s
+    end
+
+    def printable
+      [
+        [:id, 'ID'],
+        { label: 'URI', value: @config.uri },
+        :dir,
+        :title,
+        :status,
+        { label: 'Age', value: (a = age) ? '%d days' % (a / DaySecs) : 'never' },
+        { label: 'Disabled', value: @config.disabled },
+        { label: 'Last response', value: format_response_history_entry(@response_history.last_entry) },
+        :items,
+      ]
+    end
+
+    def format_response_history_entry(entry)
+      if entry
+        '%s (%s) at %s' % [entry.status, entry.reason, entry.time]
+      else
+        'none'
+      end
+    end
+
+    def config_file
+      raise Error, "dir not set" unless @dir
+      @dir / ConfigFileName
+    end
+
+    def item_history_file
+      raise Error, "dir not set" unless @dir
+      @dir / ItemHistoryFileName
+    end
+
+    def response_history_file
+      raise Error, "dir not set" unless @dir
+      @dir / ResponseHistoryFileName
+    end
+
+    def age
+      entry = @item_history.last_entry or return nil
+      Time.now - entry.time
+    end
+
+    def status
+      if (a = age)
+        if a > @config.max_age
+          :dormant
+        else
+          :active
+        end
+      else
+        :new
+      end
+    end
+
+    def disabled?
+      !!@config.disabled
+    end
+
+    def make_dotted_folder
+      components = @id.split('/')
+      components.pop if @config.consolidate && components.length > 1
+      components.unshift(@config.root_folder) if @config.root_folder
+      components.join('.')
+    end
+
+    def update_item_history
+      @items.each do |item|
+        @item_history << { time: item.date, id: item.id }
+      end
+    end
+
+    def prune_item_history
+      @item_history.prune(before: Time.now - @config.max_age).each do |entry|
+        $logger.info { "pruned #{entry.id.inspect} (#{entry.time})"}
+      end
+    end
+
+    def prune_response_history
+      @response_history.prune(before: Time.now - @config.max_age).each do |entry|
+        $logger.info { "pruned response from #{entry.time}" }
+      end
+    end
+
+    def update
+      $logger.debug { "#{@id}: updating" }
+      begin
+        prune_item_history
+        prune_response_history
+        if recently_updated?
+          $logger.info { "#{@id}: too soon to update" }
+          return
+        end
+        get
+        reject_items
+        update_item_history
+        deliver
+      rescue Error => e
+        $logger.error { "#{@id}: #{e}" }
+      end
+    end
+
+    def recently_updated?
+      (entry = @response_history.last_entry) &&
+        # (200...400).include?(entry.status) &&
+        Time.now - entry.time < @config.update_interval
+    end
+
+    def get
+      fetcher = Fetcher.get(@config.uri)
+      @response_history << {
+        time: Time.now,
+        status: fetcher.response_status,
+        reason: fetcher.response_reason,
+      }
+      if fetcher.success?
+        if fetcher.moved && !@config.ignore_moved
+          $logger.warn { "#{@id}: URI #{@config.uri} moved to #{fetcher.actual_uri}" }
+        end
+        feed = fetcher.parse_feed
+        @title = @config.title || feed[:title]
+        @items = feed[:items]
+      else
+        $logger.warn { "#{@id}: HTTP error #{fetcher.response_status} (#{fetcher.response_reason})" }
+      end
+    end
+
+    def reject_items
+      @items.reject! do |item|
+        if (reason = reject_item?(item))
+          $logger.debug { "#{@id}: removing item: #{reason} #{item.id}" }
+          true
+        end
+      end
+    end
+
+    def reject_item?(item)
+      if item.age > @config.max_age
+        'outdated item'
+      elsif @item_history[item.id]
+        'seen item'
+      elsif @config.ignore_uris.find { |r| item.uri.to_s =~ r }
+        'ignored item'
+      end
+    end
+
+    def deliver
+      $logger.debug { "#{@id}: no items to deliver" } if @items.empty?
+      @items.sort_by(&:date).each do |item|
+        deliver_item(item)
+      end
+    end
+
+    def reset
+      @item_history.reset
+    end
+
+    def enable
+      @config.disabled = false
+      @config.save(config_file)
+    end
+
+    def disable
+      @config.disabled = true
+      @config.save(config_file)
+    end
+
+    def fix
+    end
+
+    def deliver_item(item)
+      folder = make_dotted_folder
+      fields = {
+        subscription_id: @id,
+        item_title: item.title,
+        subscription_folder: folder,
+      }
+      mail = Mail.new
+      mail.date =         item.date
+      mail.from =         ERB.new(@config.mail_from).result_with_hash(fields)
+      mail.to =           ERB.new(@config.mail_to).result_with_hash(fields)
+      mail.subject =      ERB.new(@config.mail_subject).result_with_hash(fields)
+      mail.content_type = 'text/html'
+      mail.charset =      'utf-8'
+      mail.body =         build_item_html(item)
+      delivery_method = @config.delivery_method&.to_sym
+      delivery_params = @config.delivery_params
+      $logger.info {
+        "#{@id}: Sending item to %s in folder %s via %s: %p" % [
+          mail.to.join(', '),
+          folder,
+          delivery_method || '<default>',
+          mail.subject,
+        ]
+      }
+      if delivery_method == :maildir
+        delivery_method = Mail::Maildir
+        delivery_dir = Path.new(delivery_params[:dir]) / ".#{folder}"
+        delivery_params = delivery_params.merge(dir: delivery_dir.to_s)
+      end
+      mail.delivery_method(delivery_method, **delivery_params) if delivery_method
+      mail.deliver!
+    end
+
+    def build_item_html(item)
+      Simple::Builder.build_html4_document do |html|
+        html.html do
+          html.head do
+            html.meta(name: 'x-apple-disable-message-reformatting')
+            html.meta(name: 'viewport', content: 'width=device-width, initial-scale=1')
+            @styles.each do |style|
+              html.style { html << style }
+            end
+          end
+          html.body do
+            html.div(class: 'header') do
+              html << ('%s [%s]' % [@title, @id]).to_html
+            end
+            if item.title
+              html.h1 do
+                html << item.title.to_html
+              end
+            end
+            html.h2 do
+              html << [
+                item.date.strftime('%e %B %Y'),
+                item.author,
+              ].compact.join(' • ').to_html
+            end
+            if item.uri
+              html.h3 do
+                html.a(item.uri.prettify, href: item.uri)
+              end
+            end
+            if item.content
+              if item.content.html?
+                html << Scrubber.scrub_html(item.content)
+              else
+                html << Scrubber.text_to_html(item.content)
+              end
+            end
+          end
+        end
+      end.to_html
+    end
+
+  end
+
+end

data/lib/newsfetcher/version.rb

@@ -0,0 +1,5 @@
+module NewsFetcher
+
+  VERSION = '0.84'
+
+end