rss_notifier 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 942180ec87b2c3e1b2ec875c5cf0086938a6a617
-  data.tar.gz: 38640885c7f8676021c4211d8f7bf7aef6375535
+  metadata.gz: 30348630c71ce8a624dcda3085e681c23888b2db
+  data.tar.gz: 5de199efd8829d5ac06753b6daef97457f32eacc
 SHA512:
-  metadata.gz: 0a914cf858609759f4edb975096fc4d79a825387e66c897044d17be3787237b1414cde51c977f61a236d01e57ce2228c721e1b537707ee2a769bd1a6175d8d7f
-  data.tar.gz: cbbe8e615516e90bb8c6dc397e17c572f559cba3f30adf1f8be9efec2a7a80515a78d2982c937a720e70aa384b43400a0b99358674c19f8ab0ab2d147e01ad22
+  metadata.gz: ffb126f61c05742ccef948092f0d016cb29336983114613a5d238ee5b29e1d6ea285284466eafe2cf31d087c93436c6a0f5a8238465211d40fe455cbf28f4ab0
+  data.tar.gz: 91e4ea57ce1661425a3d173a43c9395c25836009d9fe53c31c12aa163287701b1861eee45807d697a4eb789a79509d925d00e23a23b2a655e9e82707c609dc5d

data/README.md

@@ -49,16 +49,20 @@ If you want it to run in the background, use `tmux`.
 ## Craigslist
 
 1. Create a search url on craigslist, e.g. http://sfbay.craigslist.org/search/bik?query=trek+520
-2. Make it RSS by clicking `RSS` on the bottom-right of the page, e.g. https://sfbay.craigslist.org/search/bik?format=rss&query=trek%20520
+2. Make it RSS by clicking orange button `RSS` on the bottom-right of the page, e.g. https://sfbay.craigslist.org/search/bik?format=rss&query=trek%20520
 3. Install gem and initialize it.
 4. Add RSS link to the `config.yml`.
-5. Start `rss_notifier`.
+5. Start without notification: `$ rss_notifier start`, other you'll flood your inbox.
+6. Start with notification: `$ rss_notifier start --notify`. I use `tmux` to run it in the background.
 
 
 ## Development
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## TODO
+
+- Add cache handling
 
 ## Contributing
 

data/lib/rss_notifier/app.rb

@@ -6,6 +6,7 @@ require 'rss_notifier/models'
 require 'pathname'
 require 'rss'
 require 'http'
+require 'htmlentities'
 
 module RssNotifier
   class App
@@ -42,6 +43,7 @@ module RssNotifier
       @options = {
         notify: notify
       }
+      @html_coder = HTMLEntities.new
 
       setup_notify(@config.notify)
     end
@@ -85,8 +87,25 @@ module RssNotifier
         rss_raw = nil
         response = nil
         begin
-          response = HTTP.get(url)
-          # require 'pry'; binding.pry
+          headers = {}
+          max_cache_duration = 20*60 # 20 minutes
+          if feed.last_modified
+            headers['If-Modified-Since'] = feed.last_modified.utc.httpdate
+          end
+          headers['Cache-Control'] = 'no-cache'
+          response = HTTP.headers(headers).get(url)
+
+          if response.code == 304
+            RssNotifier.logger.info "Not modified since #{feed.last_modified}: #{name} | #{url}"
+            next
+          elsif response.code != 200
+            RssNotifier.logger.warn "got non 200 code: #{response.code}:"
+            puts response.body.to_s
+            next
+          end
+
+          feed.last_modified = Time.parse(response.headers['Last-Modified'])
+
           raw_encoded = response.to_s.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
           rss_raw = RSS::Parser.parse(raw_encoded)
         rescue => e
@@ -94,11 +113,12 @@ module RssNotifier
           puts e.backtrace
           next
         end
+
         rss_raw.items.each do |raw_item|
           item = feed.find_or_create_item(link: raw_item.link)
           item.update({
-            title: raw_item.title,
-            description: raw_item.description,
+            title: decode_html(raw_item.title),
+            description: decode_html(raw_item.description),
             date: raw_item.date
           })
           changed_items << item if item.changed?
@@ -123,6 +143,12 @@ module RssNotifier
       end
     end
 
+    def decode_html(encoded)
+      @html_coder.decode(encoded)
+    rescue => e
+      encoded
+    end
+
     def notify!(item)
       @notify.each do |notify|
         begin

data/lib/rss_notifier/cli.rb

@@ -1,4 +1,5 @@
 require 'thor'
+require 'colorize'
 
 module RssNotifier
   class Cli < Thor
@@ -20,7 +21,7 @@ module RssNotifier
     method_options :notify => :boolean
     def start
       unless options.notify?
-        RssNotifier.logger.warn "Notifcation is disabled. To enable, run $ rss_notifier start --notify"
+        RssNotifier.logger.warn "Notifcation is disabled. To enable, run $ rss_notifier start --notify".red
       end
 
       app = RssNotifier::App.new(notify: options.notify?)

data/lib/rss_notifier/models/feed.rb

@@ -2,12 +2,6 @@ require 'virtus'
 require 'inflecto'
 require 'json'
 
-# {
-#   rss_url => {
-#     'Last-Modified' => 'blah',
-#     'items' => { item_url => Item, ... },
-#   }
-# }
 module RssNotifier
   module Models
     class Feed
@@ -15,6 +9,7 @@ module RssNotifier
 
       attribute :url, String
       attribute :name, String
+      attribute :last_modified, Time
       attribute :items, Array['RssNotifier::Models::Item']
 
       # updated_at in DB
@@ -56,6 +51,7 @@ module RssNotifier
           'url' => url,
           'name' => name,
           'updated_at' => updated_at,
+          'last_modified' => last_modified,
           'items' => items.map(&:to_db_object),
         }
       end

data/lib/rss_notifier/version.rb

@@ -1,3 +1,3 @@
 module RssNotifier
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/rss_notifier.gemspec

@@ -24,6 +24,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'virtus', '~> 1.0'
   spec.add_dependency 'inflecto', '~> 0.0'
   spec.add_dependency 'sendgrid-ruby', '~> 1.1'
+  spec.add_dependency 'htmlentities', '~> 4.3'
+  spec.add_dependency 'colorize', '~> 0.0'
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rss_notifier
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Zhomart Mukhamejanov
@@ -80,6 +80,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement