feed_processor_utils 0.0.0

data/lib/feed_processor_utils/config/feed_post_builder.yml

@@ -0,0 +1,44 @@
+:attributes:
+  :title:
+    :keys:
+    - :atom_title
+    - :title
+  :description:
+    :keys:
+    - :meta_desc
+  :pics:
+    :keys:
+    - :image
+    - :og_image
+  :time:
+    :keys:
+    - :atom_updated
+    - :atom_published
+    - :date
+    :helper: :time_with_default
+  :url:
+    :keys:
+    - atom_link
+    - link
+  :text:
+    :keys:
+    - :atom_text
+    - :content
+    :helper: :clean_text
+:replacements:
+- - !ruby/regexp /\<a.+href.+data:.+base64,.+\<\/a\>/
+  - ''
+- - !ruby/regexp /\<script.+\<\/script\>/
+  - ''
+- - !ruby/regexp /<img\/?[^>]+?>/
+  - ''
+- - !ruby/regexp /(<\s*p\s*\w*>\s*&nbsp;\s*<\s*\/p\s*>)+/
+  - <br />
+- - !ruby/regexp /(\s*\n\s*)+/
+  - <br />
+- - !ruby/regexp /(\<\/?\s*br\s*\/?\>)+/
+  - <br />
+- - !ruby/regexp /<\s*\/\s*p\s*><br \/>/
+  - </p>
+- - !ruby/regexp /<a /
+  - ! '<a rel="nofollow" '

data/lib/feed_processor_utils/config/html_parser.yml

@@ -0,0 +1,51 @@
+:real_title:
+  :selectors:
+  - title
+  :fallback_text: true
+:page_title:
+  :selectors:
+  - meta[name='og:title']
+  - meta[property='og:title']
+  - title
+  :attribute: content
+  :fallback_text: true
+:meta_desc:
+  :selectors:
+  - meta[name='og:description']
+  - meta[property='og:description']
+  - meta[name='description']
+  - meta[name='Description']
+  :attribute: content
+:video_url:
+  :collection: true
+  :selectors:
+  - embed
+  - iframe
+  :attribute: src
+:video_site:
+  :selectors:
+  - .video_holder img.video_thumbnail
+  :attribute: video-site
+:video_id:
+  :selectors:
+  - .video_holder img.video_thumbnail
+  :attribute: data-video
+:image:
+  :selectors:
+  - img.entry-image
+  :attribute: src
+:lazy_image_tags:
+  :collection: true
+  :selectors:
+  - img
+  :attribute: data-src
+:image_tags:
+  :collection: true
+  :selectors:
+  - img
+  :attribute: src
+:og_image:
+  :attribute: content
+  :selectors:
+  - meta[property='og:image']
+  

data/lib/feed_processor_utils/feed_post_builder.rb

@@ -0,0 +1,115 @@
+require 'newer_image_size'
+require 'image_size'
+
+module FeedProcessorUtils
+  class FeedPostBuilder
+
+    @@config_file = File.join(File.dirname(__FILE__), 'config/feed_post_builder.yml')
+
+    def self.sanitize(text)
+      return nil unless text 
+      replacements.each do |pattern, replacement|
+        text.gsub!(pattern, replacement)
+      end
+      text
+    end
+
+    def self.ensure_absolute(url, host)
+      url[0] == "/" ? "http://"+ host + url : url 
+    end
+
+    def self.match_based64?(uri)
+      /data:\w+\/\w+;base64,/.match(uri)
+    end
+
+    def self.longest_content(*strings)
+      strings.sort_by! do |string|
+        string.to_s.gsub(/[\n\r]/,' ').gsub(/&nbsp;/, " ").gsub(/(<\/?[^<>]+>)/,' ').gsub(/\s{2,}/, ' ').gsub(/ / ,' ').split(' ').size
+      end
+      strings.last
+    end
+
+    def self.get_image_dimensions(uri)
+      match = match_based64?(uri)
+      if match
+        data = Base64.decode64(uri.split(match[0])[1])
+      else
+        begin
+          file = open(uri)
+        rescue => e
+          return [0,0]
+        end
+        data = file.read
+      end
+
+      if file && data.encoding.name == "UTF-8"
+        NewerImageSize.new(file).size
+      else
+        ImageSize.new(data).get_size
+      end
+
+    end
+
+    def self.dimensions_ok?(dimensions)
+      ratio = (0.5..2.5).cover?(dimensions.first.to_f / dimensions.last.to_f) unless dimensions.last.to_i == 0
+      ratio ||= false
+
+      ratio && have_minimum_size?(dimensions)
+    end
+
+    def self.have_minimum_size?(dimensions)
+      dimensions.first.to_i >= 300 && dimensions.last.to_i >= 150
+    end
+    
+    def self.get_images(item_data, is_news)
+      image_urls = []
+      domain = URI.parse(item_data[:url] || item_data[:id])
+      [:og_image, :image].each do |key|
+        url = item_data[key]
+        if url
+          url = ensure_absolute(url.to_s, domain.to_s)
+          if have_minimum_size?(get_image_dimensions(url))
+            break if image_urls << url
+          end
+        end
+      end
+      nominated_images = if item_data[:lazy_image_tags].present?
+        item_data[:lazy_image_tags]
+      else
+        item_data[:images_in_text]
+      end
+      with_size = nominated_images.map do |url|
+        {url: url, dim: get_image_dimensions(url)}
+      end
+      largest_img = with_size.sort_by do |img|
+        dim = img[:dim]
+        dim ? dim[0] * dim[1] : 0
+      end.last
+      if largest_img
+        if is_news
+          image_urls << largest_img[:url]
+        else
+          image_urls.unshift(largest_img[:url]) if largest_img[:dim] && dimensions_ok?(largest_img[:dim])
+        end
+      end
+      image_urls
+    end
+
+    def self.get_videos(item_data)
+      item_data[:videos_in_text].map do |video|
+        video = "http:" + video if video[0..1] == "//" # sometimes the protocol is omitted from video url
+        video
+      end
+    end
+
+
+    def self.config
+      @@config ||= YAML.load(File.read(@@config_file))
+    end
+
+    def self.replacements
+      config[:replacements]
+    end
+
+  end
+end

data/lib/feed_processor_utils/html_parser.rb

@@ -0,0 +1,61 @@
+require 'open-uri'
+
+module FeedProcessorUtils
+
+  class HTMLParser
+
+    @@default_config = File.join(File.dirname(__FILE__), "config/html_parser.yml")
+
+    def initialize(config_file = nil)
+      config_file ||= @@default_config
+      @config = YAML.load(File.read(config_file))
+    end
+
+    def parse_data(input)
+      input_doc = Nokogiri::HTML(input)
+      Hash[
+        fields.map do |field_name, parsing_data|
+          [field_name, extract_field(input_doc, parsing_data)]
+        end
+      ]
+    end
+
+    def parse_url(url)
+      input = open(url).read
+      parse_data(input)
+    end
+
+    private
+
+    def extract_field(input_doc, parsing_data)
+      if parsing_data[:collection]
+        collection = []
+        parsing_data[:selectors].each do |selector|
+          elements = input_doc.css(selector)
+          elements.each do |element|
+            if element[parsing_data[:attribute]]
+              collection << element[parsing_data[:attribute]]
+            elsif parsing_data[:fallback_text]
+              collection << element.text
+            end
+          end
+        end
+        collection
+      else
+        parsing_data[:selectors].each do |selector|
+          element = input_doc.at_css(selector)
+          if element
+            return element[parsing_data[:attribute]] if element[parsing_data[:attribute]]
+            return element.text if parsing_data[:fallback_text]
+          end
+        end
+        nil
+      end
+    end
+
+    def fields
+      @config
+    end
+
+  end
+end

data/lib/feed_processor_utils.rb

@@ -0,0 +1,2 @@
+require 'feed_processor_utils/feed_post_builder'
+require 'feed_processor_utils/html_parser'

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: feed_processor_utils
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- FTBpro
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: imagesize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: newer_image_size
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: utility classes to work with feeds
+email: gashaw@ftbpro.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/feed_processor_utils.rb
+- lib/feed_processor_utils/feed_post_builder.rb
+- lib/feed_processor_utils/html_parser.rb
+- lib/feed_processor_utils/config/html_parser.yml
+- lib/feed_processor_utils/config/feed_post_builder.yml
+homepage: http://rubygems.org/gems/feed_processor_utils
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Feed Processing toolbox
+test_files: []