consummo 0.4.4

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.3.0
+before_install: gem install bundler -v 1.11.2
+addons:
+    code_climate:
+        repo_token: b88e21fff712be545a536cef38d27e541c56f41542da3524bd3e11d19d156d84

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in consummo.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Clayton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# Consummo
+
+Consummo is an engine for consuming, enriching and producing pieces of content from RSS feeds.
+
+## Quick Start
+
+```ruby
+
+  # Create a Feed
+  feed = Feed.new(uri: "http://feedjira.com/blog/feed.xml")
+
+  # Produce Items from the feed
+  items = ItemProducer.new(feeds: [feed]).produce
+  # items => [SimpleItem, SimpleItem, SimpleItem]
+
+  # define our content enrichers
+  enrichers = [FacebookLikeEnricher.new]
+
+  # Consume items
+  enriched_items = ItemConsumer.new(items: items, enrichers: enrichers).consume
+  # enriched_items => [SimpleItem, SimpleItem, SimpleItem]
+
+```
+
+## Using with Rails
+
+If you're using consummo with rails, you'll probably want to create `ActiveRecord` backed objects for:
+
+- `FeedItem`
+- `Feed`
+
+When producing and consuming `FeedItems` you'll probably want to persist them to a datastore when producing and/or consuming.
+
+## The Consummo Domain
+
+### Feeds
+A `Feed` is a simple data structure that represents a URI feed. It has a `uri` attribute.
+
+### Feed Items
+A `Feed Item` represents a singular piece of content produced from a `Feed`. It has attributes like `title` and `url`.
+
+### Producers
+An `Item Producer` takes a list of `Feeds` along with a `Fetcher` and fetches items from the feed.
+
+### Consumers
+An `Item Consumer` takes unenriched `Feed Items` and passes them through `Item Enrichers`.
+
+### Item Enricher
+An `Item Enricher` enriches the details and data from a `Feed Item`. For example, the `FacebookLikeEnricher` will determine the number of Facebook Likes for a particular `Feed Item` (using the item's url) and add that attribute to the `Feed Item`.
+
+The intent of `Item Enrichers` is that they are extensible and easy to implement such that multiple custom enrichments are possible.
+
+## Custom Enrichment
+
+Enrichers follow a very simple interface:
+
+```ruby
+  class SimpleEnricher
+    def enrich(item)
+      { "simple" => "enrichment" }
+    end
+  end
+```
+
+An Enricher should be able to `enrich` something that looks like an `item` (`SimpleItem`) and return a hash of key/value pairs.
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'consummo'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install consummo
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/consummo.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "consummo"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/consummo.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'consummo/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "consummo"
+  spec.version       = Consummo::VERSION
+  spec.authors       = ["Clayton Lengel-Zigich"]
+  spec.email         = ["clayton@claytonlz.com"]
+
+  spec.summary       = %q{An engine for producing, consuming and enriching items from RSS Feeds.}
+  spec.description   = %q{Consummo is a small ruby library that provides item producers, consumers and enrichers to take items from RSS feeds and turn them into persistable feed items. Consummo was extracted from https://consummo.io where it was used to find and process items in a new curation workflow.}
+  spec.homepage      = "https://github.com/clayton/consummo"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "feedjira", "~> 2.0"
+  spec.add_runtime_dependency "activesupport", "~> 4.2"
+  spec.add_runtime_dependency "odyssey", "~> 0.2"
+  spec.add_runtime_dependency "httparty", "~> 0.13"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "vcr", "~> 3.0"
+  spec.add_development_dependency "simplecov", "~> 0.9"
+  spec.add_development_dependency "codeclimate-test-reporter", "~> 0.5"
+end

data/lib/consummo/content_scorer.rb

@@ -0,0 +1,6 @@
+class ContentScorer
+  def score(text)
+    return 0 if text.blank?
+    Odyssey.flesch_kincaid_re(text, false)
+  end
+end

data/lib/consummo/facebook_like_enricher.rb

@@ -0,0 +1,13 @@
+class FacebookLikeEnricher < SocialMediaMetricEnricher
+  def metric_endpoint
+    "http://graph.facebook.com/?id="
+  end
+
+  def metric_name
+    "facebook_likes"
+  end
+
+  def metric_key
+    "shares"
+  end
+end

data/lib/consummo/feed.rb

@@ -0,0 +1,15 @@
+class Feed
+  attr_accessor :uri
+
+  def self.create(args={})
+    Feed.new(args)
+  end
+
+  def initialize(attrs={})
+    self.uri = attrs[:uri]
+  end
+
+  def id
+    return self.to_s
+  end
+end

data/lib/consummo/feed_client.rb

@@ -0,0 +1,9 @@
+require 'feedjira'
+
+class FeedClient
+  def fetch_and_parse(uri)
+    feed = Feedjira::Feed.fetch_and_parse(uri)
+    return [] if feed.is_a?(Fixnum)
+    feed.entries
+  end
+end

data/lib/consummo/feed_fetcher.rb

@@ -0,0 +1,22 @@
+class FeedFetcher
+  def initialize(client: FeedClient.new, factory: FeedItemFactory.new)
+    @client  = client
+    @factory = factory
+  end
+
+  def fetch(feed, last_modified = nil)
+    @client.fetch_and_parse(feed.uri).map do |entry|
+      next if old_entry?(entry, last_modified)
+      item = @factory.build(entry)
+      item.attributes = {:feed_id => feed.id}
+
+      item
+    end.compact
+  end
+
+private
+  def old_entry?(entry, last_modified)
+    return false if last_modified.nil?
+    last_modified > entry.last_modified
+  end
+end

data/lib/consummo/feed_item.rb

@@ -0,0 +1,11 @@
+class FeedItem
+  attr_accessor :title
+
+  def initialize(attrs={})
+    self.title = attrs[:title]
+  end
+
+  def self.create(attributes={})
+    # 
+  end
+end

data/lib/consummo/feed_item_factory.rb

@@ -0,0 +1,24 @@
+class FeedItemFactory
+  def build(entry)
+    item = SimpleItem.new({
+      :title        => entry.title,
+      :hinted_title => entry.title,
+      :url          => entry.url,
+      :author       => entry.author,
+      :summary      => entry.summary,
+      :published_at => sanitize_published_at(entry.published),
+      :guid         => entry.id
+    })
+
+    item.attributes = {:categories   => entry.categories.join(";")} unless entry.is_a?(Feedjira::Parser::ITunesRSSItem)
+
+    item
+  end
+
+private
+  def sanitize_published_at(published)
+    return Time.now if published.blank?
+    return Time.now if Time.parse(published.to_s) < Time.parse('1979-01-01-01 00:00:00')
+    published
+  end
+end

data/lib/consummo/feed_processor.rb

@@ -0,0 +1,19 @@
+class FeedProcessor
+  def initialize(feeds = [], enrichers: [])
+    @feeds = feeds
+    @enrichers = enrichers
+  end
+
+  def process
+    producer = ItemProducer.new(feeds: @feeds)
+    consumer = ItemConsumer.new(items: producer.produce, enrichers: @enrichers)
+    persist(consumer.consume)
+  end
+
+private
+  def persist(items)
+    items.each do |item|
+      FeedItem.create(item.attributes)
+    end
+  end
+end

data/lib/consummo/http_client.rb

@@ -0,0 +1,6 @@
+class HttpClient
+  def get(uri)
+    response = HTTParty.get(uri)
+    response.to_h
+  end
+end

data/lib/consummo/item_consumer.rb

@@ -0,0 +1,15 @@
+class ItemConsumer
+  def initialize(items: [], enrichers: [])
+    @items = items
+    @enrichers = enrichers
+  end
+
+  def consume
+    @items.flat_map do |item|
+      @enrichers.each do |enricher|
+        item.attributes = enricher.enrich(item)
+      end
+      item
+    end
+  end
+end

data/lib/consummo/item_producer.rb

@@ -0,0 +1,12 @@
+class ItemProducer
+  def initialize(feeds: [], fetcher: FeedFetcher.new)
+    @feeds = feeds
+    @fetcher = fetcher
+  end
+
+  def produce
+    @feeds.flat_map do |feed|
+      @fetcher.fetch(feed)
+    end.shuffle
+  end
+end

data/lib/consummo/keyword_hint_enricher.rb

@@ -0,0 +1,22 @@
+class KeywordHintEnricher
+  def initialize(keywords:[])
+    @keywords = keywords
+  end
+
+  def enrich(item, keywords=[])
+    modified = item.title
+    list = keywords.empty? ? @keywords : keywords
+
+    for keyword in list do
+      modified = make_replacements(modified, keyword)
+    end
+    {hinted_title: modified}
+  end
+
+private
+  def make_replacements(text, keyword)
+    return nil if text.blank?
+    return text unless text.match(/#{keyword}/i)
+    text.gsub(/(#{keyword})/i, '<strong>\1</strong>')
+  end
+end

data/lib/consummo/readability_enricher.rb

@@ -0,0 +1,12 @@
+require "odyssey"
+
+class ReadabilityEnricher
+  def initialize(scorer: ContentScorer.new)
+    @scorer = scorer
+  end
+
+  def enrich(item)
+    score = @scorer.score(item.content)
+    {:readability_score => score}
+  end
+end

data/lib/consummo/simple_enricher.rb

@@ -0,0 +1,5 @@
+class SimpleEnricher
+  def enrich(item)
+    { "simple" => "enrichment" }
+  end
+end

data/lib/consummo/simple_item.rb

@@ -0,0 +1,29 @@
+class SimpleItem
+  def initialize(attributes={})
+    @attributes = attributes
+  end
+
+  def attributes
+    @attributes
+  end
+
+  def attributes=(attrs)
+    @attributes.merge!(attrs)
+  end
+
+  def title
+    @attributes[:title]
+  end
+
+  def url
+    @attributes[:url]
+  end
+
+  def guid
+    @attributes[:guid] || @attributes[:url] || nil
+  end
+
+  def content
+    @attributes[:content] || ""
+  end
+end

data/lib/consummo/social_media_metric_enricher.rb

@@ -0,0 +1,26 @@
+class SocialMediaMetricEnricher
+  def initialize(client: HttpClient.new)
+    @client = client
+  end
+
+  def enrich(item)
+    begin
+      result = @client.get("#{metric_endpoint}#{URI.encode(item.url)}")
+      {metric_name.to_sym => result[metric_key]}
+    rescue Exception => e
+      {}
+    end
+  end
+
+  def metric_endpoint
+    "http://example.com?id="
+  end
+
+  def metric_name
+    "shares"
+  end
+
+  def metric_key
+    metric_name
+  end
+end

data/lib/consummo/twitter_share_enricher.rb

@@ -0,0 +1,13 @@
+class TwitterShareEnricher < SocialMediaMetricEnricher
+  def metric_endpoint
+    "http://urls.api.twitter.com/1/urls/count.json?url="
+  end
+
+  def metric_name
+    "twitter_shares"
+  end
+
+  def metric_key
+    "count"
+  end
+end

data/lib/consummo/url_enricher.rb

@@ -0,0 +1,10 @@
+class UrlEnricher
+  def initialize(resolver: UrlResolver.new)
+    @resolver = resolver
+  end
+
+  def enrich(item)
+    resolved = @resolver.resolve(item.url)
+    {url: resolved.to_s}
+  end
+end

data/lib/consummo/url_resolver.rb

@@ -0,0 +1,13 @@
+require 'httparty'
+
+class UrlResolver
+  def resolve(url)
+    return url if url.blank?
+    begin
+      response = HTTParty.head(url, follow_redirects: true)
+      response.request.last_uri
+    rescue Errno::ECONNREFUSED, URI::InvalidURIError, HTTParty::RedirectionTooDeep
+      url
+    end
+  end
+end

data/lib/consummo/version.rb

@@ -0,0 +1,3 @@
+module Consummo
+  VERSION = "0.4.4"
+end

data/lib/consummo/word_count_enricher.rb

@@ -0,0 +1,6 @@
+class WordCountEnricher
+  def enrich(item)
+    wc = item.content.scan(/[[:alpha:]]+/).count
+    {:word_count => wc}
+  end
+end

data/lib/consummo.rb

@@ -0,0 +1,31 @@
+require 'active_support'
+require 'active_support/core_ext/object/blank'
+
+require "consummo/version"
+
+require "consummo/social_media_metric_enricher"
+require "consummo/content_scorer"
+require "consummo/facebook_like_enricher"
+require "consummo/feed"
+require "consummo/feed_client"
+require "consummo/feed_fetcher"
+require "consummo/feed_item"
+require "consummo/feed_item_factory"
+require "consummo/feed_processor"
+require "consummo/http_client"
+require "consummo/item_consumer"
+require "consummo/item_producer"
+require "consummo/keyword_hint_enricher"
+require "consummo/readability_enricher"
+require "consummo/simple_item"
+require "consummo/social_media_metric_enricher"
+require "consummo/twitter_share_enricher"
+require "consummo/url_enricher"
+require "consummo/url_resolver"
+require "consummo/version"
+require "consummo/word_count_enricher"
+
+
+module Consummo
+  # Your code goes here...
+end