pluto-update 1.1.1 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b4f3fca682131a8aa9d60081466b12c4ec0b6cc9
-  data.tar.gz: 040a6c5e4fa0dede47c35da2217d890bf2791cae
+  metadata.gz: 39836180edc940c1dd06d5161c18d7991c551715
+  data.tar.gz: 74c2f94f4da7e426bdc7c806672f9625dc35d081
 SHA512:
-  metadata.gz: 9d95b94f9e0ef5b9554c14af12be6bc04641405209299a4b069665710e90b5bf6b98138c3c25b6ed5e6f0d8f0bf401c6fb5009c95c5378f83c062bf4640c6979
-  data.tar.gz: 275a2b93af9508f35f3c57f76cc3fa7db2024a4f5f0362427ac56e3c662196fb97579900bbeae3997399b6fc44eb07f1e3e5bd9195ff74c6c738da29933d07d4
+  metadata.gz: adf0ec25e897b3eb3d0f1f44fe9178b4303eec1b535bf9ac98390002e87f3cc86e0208b3284a43215842053fdcc5db5ebe5eb99417badb0085aa806e0ee1a7ba
+  data.tar.gz: 7b8035080c0d059e14c455a4c9591c50b5966904429e9149e2c35512e671d869b15c6322cad6321dbf3293cd89d2c10e47cd652da156f0544b09d111d2e5a48e

data/lib/pluto/update/fetcher.rb

@@ -144,13 +144,44 @@ class Fetcher
     # - try to change encoding to UTF-8 ourselves
     logger.debug "feed_xml.encoding.name (before): #{feed_xml.encoding.name}"
 
+
     #####
     # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
 
+    # try Converting ASCII-8BIT to UTF-8 based domain-specific guesses
+    begin
+      # Try it as UTF-8 directly
+      #   Note: make a copy/dup - otherwise convert fails (because string is already changed/corrupted)
+      feed_xml_cleaned = feed_xml.dup.force_encoding( Encoding::UTF_8 )
+      unless feed_xml_cleaned.valid_encoding?
+
+         puts "*** warn: feed '#{feed_key}' charset encoding not valid utf8 - trying latin1"
+         Activity.create!( text: "*** warn: feed '#{feed_key}' charset encoding not valid utf8 - trying latin1" )
+         # Some of it might be old Windows code page
+         # -- (Windows Code Page CP1252 is ISO_8859_1 is Latin-1 - check ??)
+
+         # tell ruby the encoding
+         # encode to utf-8
+         ## use all in code encode ?? e.g. feed_xml_cleaned = feed_xml.encode( Encoding::UTF_8, Encoding::ISO_8859_1 )
+         feed_xml_cleaned = feed_xml.dup.force_encoding( Encoding::ISO_8859_1 ).encode( Encoding::UTF_8 ) 
+      end
+      feed_xml = feed_xml_cleaned
+    rescue EncodingError => e
+      puts "*** warn: feed '#{feed_key}' charset encoding to utf8 failed; throwing out invalid bits - #{e.to_s}"
+      Activity.create!( text: "*** warn: feed '#{feed_key}' charset encoding to utf8 failed; throwing out invalid bits - #{e.to_s}" )
+
+      # Force it to UTF-8, throwing out invalid bits
+      ## todo: check options - add ?? or something to mark invalid chars ???
+      feed_xml.encode!( Encoding::UTF_8, :invalid => :replace, :undef => :replace )
+    end
+
     ## NB:
     # for now "hardcoded" to utf8 - what else can we do?
     # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
+    ### old "simple" version
+    ## feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
+
+
     logger.debug "feed_xml.encoding.name (after): #{feed_xml.encoding.name}"
 
     ## check for md5 hash for response.body
@@ -186,7 +217,7 @@ class Fetcher
 
     ### note: might crash w/ encoding errors when saving in postgress
     ##  e.g. PG::CharacterNotInRepertoire: ERROR: ...
-    ##  catch error, log it and continue for now
+    ##  catch error, log it and stop for now
     #
     #  in the future check for different charset than utf-8 ?? possible?? how to deal with non-utf8 charsets??
 
@@ -196,6 +227,7 @@ class Fetcher
       # log db error; and continue
       puts "*** error: updating feed database record '#{feed_key}' - #{e.to_s}"
       Activity.create!( text: "*** error: updating feed database record '#{feed_key}' - #{e.to_s}" )
+      return nil  #  sorry; corrupt feed; parsing not possible; fix char encoding - make it an option in config??
     end
 
 

data/lib/pluto/update/subscriber.rb

@@ -75,7 +75,9 @@ class Subscriber
         title:    feed_hash[ 'title' ] || feed_hash[ 'name' ],
         title2:   feed_hash[ 'title2' ],
         includes: feed_hash[ 'includes' ] || feed_hash[ 'include' ],
-        excludes: feed_hash[ 'excludes' ] || feed_hash[ 'exclude' ]
+        excludes: feed_hash[ 'excludes' ] || feed_hash[ 'exclude' ],
+ ## todo/future: add option for adding encoding - might not always be utf8 !!!!
+ ##       encoding: feed_hash[ 'encoding' ] || feed_hash[ 'charset' ] || 'utf8',  ## default to utf8 
       }
 
       puts "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..."

data/lib/pluto/update/version.rb

@@ -5,7 +5,7 @@ module PlutoUpdate
 
   MAJOR = 1
   MINOR = 1
-  PATCH = 1
+  PATCH = 2
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pluto-update
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors:
 - Gerald Bauer