rssly 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3e7668f11f0cec7ba5f1da6674e516a46df57bef
+  data.tar.gz: 4969418e2ef5a9d8e55433988a02b8342bc1ee41
+SHA512:
+  metadata.gz: 788e43bd09c3f51e61a7693efaf292ffe434979444b543828ded0141390fde2273ba61d4396033c38b8785c7ccb41837c349649609c4d5d06c2375ce2e460f21
+  data.tar.gz: ae3af50267214d349159fb09a9412779ef81f2b05a5d5fbbb6b1a265dc0473c10d44032eee981a9c8ab5c5a3be068b99b930c113d1868119a37da15c664a1654

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.ruby-version

@@ -0,0 +1 @@
+2.1.1

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rssly.gemspec
+gemspec
+
+gem 'activesupport'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Nathan Kot
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,26 @@
+# Rssly
+
+A tool for serializing multiple RSS feeds into different formats.
+
+Design-wise, this gem is responsible for:
+
+- Encapsulating RSS Feed URL's
+- Creating a collection of articles from either Feed URL's or persisted data
+- Filtering that collection of articles
+- Sorting that collection of articles
+- Transforming the collection of articles into different formats
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rssly'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rssly
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/rssly

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# vim: set ft=ruby:
+require 'rssly/cli'
+Rssly::CLI.start

data/lib/rssly/article.rb

@@ -0,0 +1,79 @@
+require 'addressable/uri'
+require 'readability'
+require 'open-uri'
+require 'sanitize'
+require 'ots'
+
+module Rssly
+  # Represents a single rss article
+  class Article
+    class << self
+      def create_from_feedjira_entry(entry)
+        new(title: entry.title, url: entry.url, published: entry.published)
+      end
+    end
+
+    SUMMARY_RATIO = 35
+
+    attr_accessor :title
+    attr_accessor :url
+    attr_accessor :summary
+    attr_accessor :topics
+    attr_accessor :published
+
+    def initialize(
+      title: nil,
+      url: nil,
+      published: nil,
+      summary: nil,
+      topics: nil
+    )
+      self.title = title
+      self.url = url
+      self.published = published
+      self.summary = summary
+      self.topics = topics
+    end
+
+    def title
+      @title ||= extracted.title
+    end
+
+    def summary
+      @summary ||= summarized.summarize(
+        percent: SUMMARY_RATIO
+      ).reduce('') do |text, o|
+        text << o[:sentence].strip + ' '
+      end.strip
+    end
+
+    def topics
+      @topics ||= summarized.topics
+    end
+
+    def url=(url)
+      normalized = Addressable::URI.parse(url).normalize
+      @url = normalized.scheme + '://' + normalized.host + normalized.path
+    end
+
+    def published
+      @published ||= Time.now
+    end
+
+    private
+
+    def summarized
+      @summarized ||= begin
+                        text = Sanitize.clean(extracted.content)
+                        OTS.parse(text)
+                      end
+    end
+
+    def extracted
+      @extracted ||= begin
+                       source = open(url).read
+                       Readability::Document.new(source)
+                     end
+    end
+  end
+end

data/lib/rssly/cli.rb

@@ -0,0 +1,32 @@
+require 'thor'
+require 'rssly'
+
+module Rssly
+  # The rssly CLI
+  class CLI < Thor
+    desc 'fetch URL1 URL2', 'Fetch feeds and output them in a given format.'
+
+    option :format, default: 'text', desc: <<-EOT
+      Choose from one of the following serializers:
+
+      `text` : Output articles in a simple text layout.
+      `html` : Output articles in an html format.
+    EOT
+
+    def fetch(*feed_urls)
+      feeds = feed_urls.map do |url|
+        Rssly::Feed.new url: url
+      end
+
+      collection = Rssly::Collection.create_from_feeds(*feeds)
+
+      serializer = case options[:format]
+                   when 'text' then Rssly::Serializers::Text
+                   when 'html' then Rssly::Serializers::HTML
+                   else fail "Serializer #{options[:format]} not found."
+                   end
+
+      puts serializer.new(collection).perform
+    end
+  end
+end

data/lib/rssly/collection/age_filter.rb

@@ -0,0 +1,15 @@
+module Rssly
+  class Collection
+    # Filters articles out by age
+    module AgeFilter
+      def self.parse(articles, min: nil, max: nil)
+        articles.select do |article|
+          if min && article.published < min then false
+          elsif max && article.published > max then false
+          else true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rssly/collection/unique_filter.rb

@@ -0,0 +1,21 @@
+require 'bloomfilter-rb'
+
+module Rssly
+  class Collection
+    # Filters out any articles that have the same url in the set.
+    module UniqueFilter
+      def self.parse(articles, **args)
+        bf = BloomFilter::Redis.new(
+          size: 10000
+        )
+
+        articles.select do |article|
+          unless bf.include?(article.url)
+            bf.insert article.url
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rssly/collection.rb

@@ -0,0 +1,37 @@
+module Rssly
+  # Represents a collection of articles
+  class Collection
+    class << self
+      def create_from_feeds(*feeds)
+        instance = new(
+          feeds.map { |f| f.articles }.flatten
+        )
+
+        instance.filter(UniqueFilter)
+        instance
+      end
+    end
+
+    attr_accessor :articles
+
+    def initialize(items)
+      self.articles = items
+    end
+
+    def each(&block)
+      articles.each(&block)
+    end
+
+    def length
+      articles.length
+    end
+
+    def articles
+      @articles ||= []
+    end
+
+    def filter(filter, **opts)
+      self.articles = filter.parse(articles, **opts)
+    end
+  end
+end

data/lib/rssly/deserializers.rb

@@ -0,0 +1,5 @@
+module Rssly
+  module Deserializers
+
+  end
+end

data/lib/rssly/feed.rb

@@ -0,0 +1,32 @@
+require 'feedjira'
+
+module Rssly
+  # Represents a single rss feed and it's metadata
+  class Feed
+    class << self
+    end
+
+    attr_accessor :url
+    attr_accessor :title
+    attr_accessor :articles
+
+    def initialize(title: nil, url: nil)
+      self.url = url
+      self.title = title
+    end
+
+    def articles
+      @articles ||= fetch_articles
+    end
+
+    private
+
+    def fetch_articles
+      result = Feedjira::Feed.fetch_and_parse @url
+      self.title = result.title
+      result.entries.map do |obj|
+        Article.create_from_feedjira_entry(obj)
+      end
+    end
+  end
+end

data/lib/rssly/serializers.rb

@@ -0,0 +1,56 @@
+module Rssly
+  module Serializers
+    # The base serializer class
+    class Serializer
+      def initialize(collection)
+        @collection = collection
+      end
+
+      def perform
+        fail 'Serializer#perform must be implemented'
+      end
+    end
+
+    # Transforms a collection into a text representation
+    class Text < Serializer
+      ARTICLE_TEMPLATE = <<-EOT
+      ====== %{title} ======
+      Seen @ %{published}
+
+      %{summary}
+
+      Read on: %{url}
+
+      --
+
+      EOT
+
+      def perform
+        @collection.articles.reduce('') do |response, article|
+          response + ARTICLE_TEMPLATE % {
+            title: article.title,
+            summary: article.summary,
+            published: article.published,
+            url: article.url
+          }
+        end
+      end
+    end
+
+    # Transforms a collection into a html representation
+    class HTML < Text
+      ARTICLE_TEMPLATE = <<-EOT
+      <article>
+        <h1 class="title">%{title}</h1>
+        <date>%{published}</date>
+
+        <p>%{summary}</p>
+
+        <a class="read more" href="%{url}">
+          Read on.
+        </a>
+      </article>
+      EOT
+    end
+  end
+end

data/lib/rssly/version.rb

@@ -0,0 +1,3 @@
+module Rssly
+  VERSION = "0.0.1"
+end

data/lib/rssly.rb

@@ -0,0 +1,12 @@
+require 'active_support'
+require "rssly/version"
+require "rssly/article"
+require "rssly/feed"
+require "rssly/deserializers"
+require "rssly/serializers"
+require "rssly/collection"
+require "rssly/collection/unique_filter"
+require "rssly/collection/age_filter"
+
+module Rssly
+end

data/rssly.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rssly/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rssly"
+  spec.version       = Rssly::VERSION
+  spec.authors       = ["Nathan Kot"]
+  spec.email         = ["me@nathankot.com"]
+  spec.homepage      = ""
+  spec.license       = "MIT"
+  spec.description   = "Collect and serialize rss feeds."
+  spec.summary       = <<-EOT
+    Rssly is a tool to gather rssly feeds, filter and serialize them into
+    various outputs.
+  EOT
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "feedbag", "~> 0.9"
+  spec.add_dependency "feedjira", "~> 1.1"
+  spec.add_dependency "ruby-readability", "~> 0.6"
+  spec.add_dependency "sanitize", "~> 2.1"
+  spec.add_dependency "ots", "~> 0.5"
+  spec.add_dependency "addressable", "~> 2.3"
+  spec.add_dependency "bloomfilter-rb", "~> 2.1"
+  spec.add_dependency "thor"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+
+end

data/spec/article_spec.rb

@@ -0,0 +1,61 @@
+require 'spec_helper'
+require 'rssly'
+require 'feedjira'
+require 'addressable/uri'
+
+describe 'Rssly::Article' do
+
+  ARTICLE_TEST_FEED_URL = 'news.ycombinator.com/rss'
+  ARTICLE_TEST_URL = \
+    'http://blog.ycombinator.com/meet-the-people-taking-over-hacker-news'
+
+  describe 'class' do
+    describe '#create_from_feedjira_entry' do
+      before do
+        @entry = Feedjira::Feed.fetch_and_parse(ARTICLE_TEST_FEED_URL).entries.first
+        allow(@entry).to receive(:published).and_return(Time.now)
+        @article = Rssly::Article.create_from_feedjira_entry @entry
+      end
+
+      it 'should set the title' do
+        expect(@article.title).to eq(@entry.title)
+      end
+
+      it 'should set the url' do
+        expect(@article.url).to eq(@entry.url)
+      end
+
+      it 'should set the publish date' do
+        expect(@article.published).to eq(@entry.published)
+      end
+    end
+  end
+
+  describe 'instance' do
+    before do
+      @article = Rssly::Article.new(url: ARTICLE_TEST_URL)
+    end
+
+    it 'should standardize the url' do
+      article = Rssly::Article.new url: 'http://notstandard.com/123123/?test=hello/'
+      expect(article.url).to eq('http://notstandard.com/123123/')
+    end
+
+    it 'should standardize a file url' do
+      article = Rssly::Article.new url: 'http://notstandard.com/123123.html?hello'
+      expect(article.url).to eq('http://notstandard.com/123123.html')
+    end
+
+    it 'should fetch the title if not given' do
+      expect(@article.title).not_to eq(nil)
+    end
+
+    it 'should fetch the summary if not given' do
+      expect(@article.summary).not_to eq(nil)
+    end
+
+    it 'should always have a published date' do
+      expect(@article.published).not_to eq(nil)
+    end
+  end
+end

data/spec/collection_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+require 'rssly'
+require 'active_support/core_ext'
+
+describe 'Rssly::Collection' do
+
+  COLLECTION_TEST_URL = 'news.ycombinator.com/rss'
+  COLLECTION_ARTICLE_TEST_URL = \
+    'http://blog.ycombinator.com/meet-the-people-taking-over-hacker-news'
+
+  before do
+    @feed = Rssly::Feed.new(url: COLLECTION_TEST_URL)
+    @feed2 = Rssly::Feed.new(url: COLLECTION_TEST_URL)
+  end
+
+  describe 'class' do
+    describe '#create_from_feeds' do
+      before do
+        @feed_count = @feed.articles.length
+        @collection = Rssly::Collection.create_from_feeds(
+          @feed, @feed2
+        )
+      end
+
+      it 'filters out duplicate articles' do
+        expect(@collection.length).to eq(@feed_count)
+      end
+    end
+  end
+
+  describe 'instance' do
+    before do
+      @articles = []
+      3.times do
+        @articles << Rssly::Article.new(url: COLLECTION_ARTICLE_TEST_URL)
+      end
+
+      @collection = Rssly::Collection.new(@articles)
+    end
+
+    it 'should respond to #each' do
+      expect(@collection.respond_to?(:each)).to eq(true)
+    end
+
+    it 'should respond to #length' do
+      expect(@collection.length).to eq(3)
+    end
+
+    describe 'filtering' do
+      it 'can filter by age' do
+        @collection.filter(
+          Rssly::Collection::AgeFilter, min: 1.hour.from_now
+        )
+
+        expect(@collection.length).to eq(0)
+
+        @collection.filter(
+          Rssly::Collection::AgeFilter, max: 1.hour.ago
+        )
+
+        expect(@collection.length).to eq(0)
+      end
+    end
+  end
+end

data/spec/feed_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+require 'rssly'
+
+describe 'Rssly::Feed' do
+
+  FEED_TEST_URL = 'news.ycombinator.com/rss'
+
+  before do
+    @feed = Rssly::Feed.new(url: FEED_TEST_URL)
+  end
+
+  it 'should set the @url' do
+    expect(@feed.url).to eq(FEED_TEST_URL)
+  end
+
+  describe '#articles' do
+    before do
+      allow(Rssly::Article).to receive(:create_from_feedjira_entry) do |entry|
+        entry
+      end
+    end
+
+    it 'should call Article#create_from_feedjira_entry' do
+      expect(Rssly::Article).to receive(:create_from_feedjira_entry)
+      @feed.articles
+    end
+
+    it 'articles should have titles' do
+      expect(@feed.articles.first.title).to_not eq(nil)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,227 @@
+--- !ruby/object:Gem::Specification
+name: rssly
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nathan Kot
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: feedbag
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: feedjira
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: ruby-readability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: sanitize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: ots
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Collect and serialize rss feeds.
+email:
+- me@nathankot.com
+executables:
+- rssly
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .ruby-version
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rssly
+- lib/rssly.rb
+- lib/rssly/article.rb
+- lib/rssly/cli.rb
+- lib/rssly/collection.rb
+- lib/rssly/collection/age_filter.rb
+- lib/rssly/collection/unique_filter.rb
+- lib/rssly/deserializers.rb
+- lib/rssly/feed.rb
+- lib/rssly/serializers.rb
+- lib/rssly/version.rb
+- rssly.gemspec
+- spec/article_spec.rb
+- spec/collection_spec.rb
+- spec/feed_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Rssly is a tool to gather rssly feeds, filter and serialize them into various
+  outputs.
+test_files:
+- spec/article_spec.rb
+- spec/collection_spec.rb
+- spec/feed_spec.rb
+- spec/spec_helper.rb