pluto-feedfetcher 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8f4eed6539db8183401c2da47b91e42ecc59aaa3
+  data.tar.gz: 4cbf9edd2c859a5b6b643e901789b3c84e0dce9d
+SHA512:
+  metadata.gz: 493a4e41cdc0628aab781058f442a316d254a358925893e3f17db4e1be2221472b4199351e485390cb4844e0b14b9e495f42746bd9bd43324c7f2bbff9b4bdd5
+  data.tar.gz: d199ce23f2b41e4cd140c4b23e82193bd1fa7adcf8f92e2ccd02e325865b514ad3fd3cea9ba2c952c8ecd1fc90ebe1b1a7d823ec1189d6dec532440b36aec1ff

data/HISTORY.md

@@ -0,0 +1,5 @@
+
+### 0.1.0 / 2015-01-11
+
+* Everything is new. First release
+

data/Manifest.txt

@@ -0,0 +1,8 @@
+HISTORY.md
+Manifest.txt
+README.md
+Rakefile
+lib/pluto/feedfetcher.rb
+lib/pluto/feedfetcher/basic.rb
+lib/pluto/feedfetcher/cond_get_with_cache.rb
+lib/pluto/feedfetcher/version.rb

data/README.md

@@ -0,0 +1,25 @@
+# pluto-feedfetcher gem - fetch web feeds (w/ conditional HTTP get e.g. use etags, if-modified-since etc.)
+
+* home  :: [github.com/feedreader/pluto.feed.fetcher](https://github.com/feedreader/pluto.feed.fetcher)
+* bugs  :: [github.com/feedreader/pluto.feed.fetcher/issues](https://github.com/feedreader/pluto.feed.fetcher/issues)
+* gem   :: [rubygems.org/gems/pluto-feedfetcher](https://rubygems.org/gems/pluto-feedfetcher)
+* rdoc  :: [rubydoc.info/gems/pluto-feedfetcher](http://rubydoc.info/gems/pluto-feedfetcher)
+* forum :: [groups.google.com/group/feedreader](http://groups.google.com/group/feedreader)
+
+
+
+## Usage
+
+TBD
+
+
+## License
+
+The `pluto-feedfetcher` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+## Questions? Comments?
+
+Send them along to the [Planet Pluto and Friends Forum/Mailing List](http://groups.google.com/group/feedreader).
+Thanks!
+

data/Rakefile

@@ -0,0 +1,31 @@
+require 'hoe'
+require './lib/pluto/feedfetcher/version.rb'
+
+Hoe.spec 'pluto-feedfetcher' do
+
+  self.version = PlutoFeedFetcher::VERSION
+
+  self.summary = "pluto-feedfetcher - fetch web feeds (w/ conditional HTTP get e.g. use etags, if-modified-since etc.)"
+  self.description = summary
+
+  self.urls    = ['https://github.com/feedreader/pluto.feed.fetcher']
+
+  self.author  = 'Gerald Bauer'
+  self.email   = 'feedreader@googlegroups.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'HISTORY.md'
+
+  self.extra_deps = [
+    ['pluto-models',  '>= 1.3.2'],
+    ['fetcher',       '>= 0.4.4'],
+  ]
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+    required_ruby_version: '>= 1.9.2'
+  }
+
+end

data/lib/pluto/feedfetcher/basic.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+
+module Pluto
+
+class FeedFetcherBasic
+
+  include LogUtils::Logging
+  include Models   # for easy convenience access for Activity etc.
+
+  def initialize
+    @worker  = Fetcher::Worker.new
+  end
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+
+  def fetch( feed_rec )
+    # simple/basic feed fetcher; use for debugging (only/mostly)
+    #  -- Note: will NOT change db records in any way
+
+    feed_url = feed_rec.feed_url
+    feed_key = feed_rec.key
+
+    feed_xml = fix_me_fetch_utf8( feed_url )
+
+    logger.debug "feed_xml:"
+    logger.debug feed_xml[ 0..500 ] # get first 500 chars
+
+
+####  todo/fix: make it generic - move out of this method (re)use - for all fetcher??
+    #  if opts.verbose?  # also write a copy to disk
+    if debug?
+      logger.debug "saving feed to >./#{feed_key}.xml<..."
+      File.open( "./#{feed_key}.xml", 'w' ) do |f|
+        f.write( feed_xml )
+      end
+    end
+
+
+
+    ### todo/fix:
+    ###  return feed_xml !!! - move FeedUtils::Parser.parse to update or someting !!!
+
+    puts "Before parsing feed >#{feed_key}<..."
+
+    
+    feed_xml
+    
+    ## fix/todo: check for feed.nil?   -> error parsing!!!
+    #    or throw exception
+    # feed = FeedUtils::Parser.parse( feed_xml )
+    # feed
+  end
+
+
+###########
+#  todo/fix: use "standard" fetch method e.g. Fetcher.read_utf8!() - clean up/remove (duplicate) here??
+private
+  def fix_me_fetch_utf8( url )
+    response = @worker.get( url )
+
+    ## if debug?
+      puts "http status #{response.code} #{response.message}"
+
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    xml = response.body
+
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    xml = xml.force_encoding( Encoding::UTF_8 )
+    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
+
+    xml
+  end
+
+end # class FeedFetcherBasic
+
+end # module Pluto

data/lib/pluto/feedfetcher/cond_get_with_cache.rb

@@ -0,0 +1,196 @@
+# encoding: utf-8
+
+module Pluto
+
+class FeedFetcherCondGetWithCache
+
+  include LogUtils::Logging
+  include Models   # for easy convenience access for Activity etc.
+
+  def initialize
+    @worker  = Fetcher::Worker.new
+  end
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+
+  def fetch( feed_rec )
+    #############
+    # try smart http update; will update db records
+
+    feed_url = feed_rec.feed_url
+    feed_key = feed_rec.key
+
+    ### todo/fix: normalize/unifiy feed_url
+    ##  - same in fetcher - use shared utitlity method or similar
+
+    @worker.use_cache = true
+    @worker.cache[ feed_url ] = {
+      'etag'          => feed_rec.http_etag,
+      'last-modified' => feed_rec.http_last_modified
+    }
+
+    begin
+      response = @worker.get( feed_url )
+    rescue SocketError => e
+      ## catch socket error for unknown domain names (e.g. pragdave.blogs.pragprog.com)
+      ###  will result in SocketError -- getaddrinfo: Name or service not known
+      puts "*** error: fetching feed '#{feed_key}' - #{e.to_s}"
+      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - #{e.to_s}" )
+
+      ### todo/fix: update feed rec in db
+      @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+      return nil
+    end
+
+    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+
+    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
+      puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+      puts "no change; request returns not modified (304); skipping parsing feed"
+      return nil   # no updates available; nothing to do
+    end
+
+    feed_fetched = Time.now
+
+    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
+
+      puts "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+      
+      feed_attribs = {
+        http_code:          response.code.to_i,
+        http_server:        response.header[ 'server' ],
+        http_etag:          nil,
+        http_last_modified: nil,
+        body:               nil,
+        md5:                nil,
+        fetched:            feed_fetched 
+      }
+      feed_rec.update_attributes!( feed_attribs )
+      
+      ## add log error activity -- in future add to error log - better - why? why not?
+      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}" )
+
+      return nil  #  sorry; no feed for parsing available
+    end
+
+    puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+
+    feed_xml = response.body
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "feed_xml.encoding.name (before): #{feed_xml.encoding.name}"
+
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    # try Converting ASCII-8BIT to UTF-8 based domain-specific guesses
+    begin
+      # Try it as UTF-8 directly
+      #   Note: make a copy/dup - otherwise convert fails (because string is already changed/corrupted)
+      feed_xml_cleaned = feed_xml.dup.force_encoding( Encoding::UTF_8 )
+      unless feed_xml_cleaned.valid_encoding?
+
+         puts "*** warn: feed '#{feed_key}' charset encoding not valid utf8 - trying latin1"
+         Activity.create!( text: "*** warn: feed '#{feed_key}' charset encoding not valid utf8 - trying latin1" )
+         # Some of it might be old Windows code page
+         # -- (Windows Code Page CP1252 is ISO_8859_1 is Latin-1 - check ??)
+
+         # tell ruby the encoding
+         # encode to utf-8
+         ## use all in code encode ?? e.g. feed_xml_cleaned = feed_xml.encode( Encoding::UTF_8, Encoding::ISO_8859_1 )
+         feed_xml_cleaned = feed_xml.dup.force_encoding( Encoding::ISO_8859_1 ).encode( Encoding::UTF_8 ) 
+      end
+      feed_xml = feed_xml_cleaned
+    rescue EncodingError => e
+      puts "*** warn: feed '#{feed_key}' charset encoding to utf8 failed; throwing out invalid bits - #{e.to_s}"
+      Activity.create!( text: "*** warn: feed '#{feed_key}' charset encoding to utf8 failed; throwing out invalid bits - #{e.to_s}" )
+
+      # Force it to UTF-8, throwing out invalid bits
+      ## todo: check options - add ?? or something to mark invalid chars ???
+      feed_xml.encode!( Encoding::UTF_8, :invalid => :replace, :undef => :replace )
+    end
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    ### old "simple" version
+    ## feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
+
+
+    logger.debug "feed_xml.encoding.name (after): #{feed_xml.encoding.name}"
+
+    ## check for md5 hash for response.body
+
+    last_feed_md5 = feed_rec.md5
+    feed_md5 = Digest::MD5.hexdigest( feed_xml )
+    
+    if last_feed_md5 && last_feed_md5 == feed_md5
+      # not all servers handle conditional gets, so while not much can be
+      # done about the bandwidth, but if the response body is identical
+      # the downstream processing (parsing, caching, ...) can be avoided.
+      #  - thanks to planet mars -fido.rb for the idea, cheers.
+      
+      puts "no change; md5 digests match; skipping parsing feed"
+      return nil   # no updates available; nothing to do
+    end
+
+    feed_attribs = {
+      http_code:          response.code.to_i,
+      http_server:        response.header[ 'server' ],
+      http_etag:          response.header[ 'etag' ],
+      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
+      body:               feed_xml,
+      md5:                feed_md5,
+      fetched:            feed_fetched
+    }
+
+    ## if debug?
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    ### note: might crash w/ encoding errors when saving in postgress
+    ##  e.g. PG::CharacterNotInRepertoire: ERROR: ...
+    ##  catch error, log it and stop for now
+    #
+    #  in the future check for different charset than utf-8 ?? possible?? how to deal with non-utf8 charsets??
+
+    begin
+      feed_rec.update_attributes!( feed_attribs )
+    rescue Exception => e
+      # log db error; and continue
+      puts "*** error: updating feed database record '#{feed_key}' - #{e.to_s}"
+      Activity.create!( text: "*** error: updating feed database record '#{feed_key}' - #{e.to_s}" )
+      return nil  #  sorry; corrupt feed; parsing not possible; fix char encoding - make it an option in config??
+    end
+
+
+    logger.debug "feed_xml:"
+    logger.debug feed_xml[ 0..300 ] # get first 300 chars
+      
+    puts "Before parsing feed >#{feed_key}<..."
+
+    ### move to feedutils
+    ### logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
+
+
+    ### todo/fix:
+    ###  return feed_xml !!! - move FeedUtils::Parser.parse to update or someting !!!
+
+    feed_xml
+    ## fix/todo: check for feed.nil?   -> error parsing!!!
+    #    or throw exception
+    ## feed = FeedUtils::Parser.parse( feed_xml )
+    ## feed
+  end
+
+
+end # class FeedFetcherCondGetWithCache
+
+end # module Pluto

data/lib/pluto/feedfetcher/version.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+module PlutoFeedFetcher
+
+  MAJOR = 0
+  MINOR = 1
+  PATCH = 0
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    ### todo: add RUBY_PATCHLEVEL or RUBY_PATCH_LEVEL  e.g. -p124
+    "pluto-feedfetcher/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    "#{File.expand_path( File.dirname(File.dirname(File.dirname(File.dirname(__FILE__)))) )}"
+  end
+
+end # module PlutoFeedFetcher
+

data/lib/pluto/feedfetcher.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+
+require 'pluto/models'
+
+
+# more 3rd party gems
+require 'fetcher'    # fetch (text) documents
+
+
+# our own code
+require 'pluto/feedfetcher/version'   # Note: let version always go first
+require 'pluto/feedfetcher/basic'
+require 'pluto/feedfetcher/cond_get_with_cache'
+
+
+
+# say hello
+puts PlutoFeedFetcher.banner   if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG) 
+

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: pluto-feedfetcher
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Gerald Bauer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pluto-models
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+- !ruby/object:Gem::Dependency
+  name: fetcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+description: pluto-feedfetcher - fetch web feeds (w/ conditional HTTP get e.g. use
+  etags, if-modified-since etc.)
+email: feedreader@googlegroups.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- HISTORY.md
+- Manifest.txt
+- README.md
+files:
+- HISTORY.md
+- Manifest.txt
+- README.md
+- Rakefile
+- lib/pluto/feedfetcher.rb
+- lib/pluto/feedfetcher/basic.rb
+- lib/pluto/feedfetcher/cond_get_with_cache.rb
+- lib/pluto/feedfetcher/version.rb
+homepage: https://github.com/feedreader/pluto.feed.fetcher
+licenses:
+- Public Domain
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--main"
+- README.md
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: pluto-feedfetcher - fetch web feeds (w/ conditional HTTP get e.g. use etags,
+  if-modified-since etc.)
+test_files: []