celes-web 1.0.0

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/.rvmrc

@@ -0,0 +1 @@
+rvm use --create 1.9.3@celes

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in edgar.gemspec
+gemspec
+
+gem 'rspec'
+gem 'nokogiri'

data/Gemfile.lock

@@ -0,0 +1,32 @@
+PATH
+  remote: .
+  specs:
+    celes-web (1.0.0)
+      nokogiri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    mini_portile (0.5.2)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    rake (10.1.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.4)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.3)
+  celes-web!
+  nokogiri
+  rake
+  rspec

data/README.md

@@ -0,0 +1,44 @@
+Celes
+=====
+
+A simple Ruby gem for parsing snippets of text and images based on the HTML content at a provided URL.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'celes-web'
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install 'celes-web'
+
+## Usage
+
+Require 'celes' and then construct a Celes object providing the URL to parse
+
+    require 'celes'
+    c = Celes.new(url: 'http://www.yahoo.com')
+
+Calling '#snippets' without any options will get the text snippets that are at least 40 characters long
+
+    c.snippets #=> ['Array', 'of', 'Strings']
+
+Calling '#images' without any options will get the source field for any images on the page
+
+    c.images #=> ['Array', 'of', 'Strings']
+
+### Options
+
+Use 'min_snippet_length' to define the minimum length of text in the document node to be considered a "snippet".  The default is 40 characters.
+
+    c = Celes.new(url: 'http://www.yahoo.com', min_snippet_length: 100)
+
+Use 'snip_length' to truncate longer snippets down to a shorter length.  By default, Celes will truncate snippets to 140 characters.
+
+    c = Celes.new(url: 'http://www.yahoo.com', snip_length: 45)
+

data/celes.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "celes-web"
+  spec.version       = '1.0.0'
+  spec.authors       = ["Andy Schrage"]
+  spec.email         = ["ajschrag@mtu.edu"]
+  spec.description   = %q{A simple Ruby gem for providing a snipet of text and images based on a url.}
+  spec.summary       = %q{A simple Ruby gem for providing a snipet of text and thumbnails based on a url.}
+  spec.homepage      = "https://github.com/Swimminschrage/celes"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rspec'
+  spec.add_dependency 'nokogiri'
+end

data/lib/celes.rb

@@ -0,0 +1,71 @@
+require 'uri'
+require 'net/http'
+require 'nokogiri'
+
+class Celes
+  attr_reader :url
+
+  def initialize(options = {})
+    raise 'No URL provided' unless options[:url]
+
+    @url = options[:url]
+    @uri = URI(@url)
+
+    parse_options(options)
+  end
+
+  def snippets
+    return @snippets if @snippets
+    make_request
+    @snippets
+  end
+
+  def images
+    return @images if @images
+    make_request
+    @images
+  end
+
+  private
+
+  def parse_options(options)
+    # Init defaults as necessary
+    @snip_length = options[:snip_length] || 140
+    @min_snippet_length = options[:min_snippet_length] || 40
+
+    # Normalize bad options
+    @min_snippet_length = 0 if @min_snippet_length < 0
+    @snip_length = 0 if @snip_length < 0
+  end
+
+  def make_request
+    resp = get_html_content
+    if resp
+      parse_html resp
+    else
+      raise "Unable to reach #{@url.to_s}"
+    end
+  end
+
+  def get_html_content
+    resp = Net::HTTP.get_response(@uri)
+    case resp
+      when Net::HTTPSuccess then resp.body
+      when Net::HTTPRedirection then
+        @url = resp['location']
+        @uri = URI(@url)
+        get_html_content
+    end
+  end
+
+  def parse_html (response)
+    doc = Nokogiri::HTML.parse(response)
+
+    @images = doc.xpath('/html/body//img/@src').map{ |x| x.value }
+    @snippets = doc.xpath('/html/body//*[self::p or self::span]/text()').map{ |x| x.to_s}.keep_if do |str|
+      str.strip.length >= @min_snippet_length
+    end
+
+    @snippets = @snippets.map { |x| x[0, @snip_length] }
+  end
+end

data/spec/celes_spec.rb

@@ -0,0 +1,142 @@
+require 'spec_helper'
+require_relative '../lib/celes'
+require 'uri'
+
+describe Celes do
+  let(:url) {'http://www.theredheadproject.com'}
+  let(:celes) { Celes.new(url: url) }
+  subject{celes}
+
+  it {should respond_to(:images)}
+  it {should respond_to(:snippets)}
+  it {should respond_to(:url)}
+
+  it 'should error out when provided no url' do
+    expect{Celes.new}.to raise_error(RuntimeError)
+  end
+
+  describe 'when provided a valid url' do
+    its(:url) {should be == url}
+
+    context 'that causes a redirect' do
+      let(:url) {'http://google.com'} # Should redirect the user to http://www.google.com
+      let(:celes) { Celes.new(url: url) }
+      subject{celes}
+
+      its(:get_html_content) {should_not be nil}
+    end
+
+    describe 'that contains images and text snippets' do
+      before do
+        subject.stub(:get_html_content) { '<html><body><p>This is a test for some text that should be long enough</p><img src="myimg.png" width="100" height="100"/></body></html>' }
+      end
+
+      it 'should have only 1 image' do
+        expect(subject.images).to be_an Array
+        expect(subject.images.size).to be 1
+        expect(subject.images[0]).to be == 'myimg.png'
+      end
+
+      it 'should have only 1 snippet' do
+        expect(subject.snippets).to be_an Array
+        expect(subject.snippets.size).to be 1
+        expect(subject.snippets[0]).to be == 'This is a test for some text that should be long enough'
+      end
+    end
+
+    # Marking these tests as pending until I can figure out how to get Nokogiri to parse html that has img and p tags
+    # in the head.
+    describe 'only parses images and text snippets within the body' do
+      let(:url) {'http://www.theredheadproject.com'}
+      let(:nobody) { Celes.new(url: url) }
+      subject {nobody}
+      before do
+        nobody.stub(:get_html_content).and_return '<html><head><p>This is a test for some text that should be long enough</p><img src="myimg.png" width="100" height="100"/></head><body><p>Body Text</p></body></html>'
+      end
+
+      xit 'should have no images' do
+        expect(subject.images).to be_an Array
+        subject.images.each { |img| puts img }
+        expect(subject.images.size).to be 0
+
+      end
+
+      xit 'should have no snippets' do
+        expect(subject.snippets).to be_an Array
+        expect(subject.snippets.size).to be 0
+      end
+    end
+
+    describe 'can take options' do
+      context ':min_snippet_length' do
+        it 'defaults to 40 characters' do
+          c = Celes.new(url: url)
+          c.stub(:get_html_content).and_return '<p>Not 40 characters</p><p>Definitely 40 characters long test here!</p>'
+
+          expect(c.snippets.size).to be 1
+          expect(c.snippets[0]).to eq 'Definitely 40 characters long test here!'
+        end
+
+        it 'less than 0 defaults to 0' do
+          c = Celes.new(url: url, min_snippet_length: -10)
+          c.stub(:get_html_content).and_return '<p>Not 40 characters</p><p>Definitely 40 characters long test here!</p>'
+
+          expect(c.snippets.size).to be 2
+          expect(c.snippets[0]).to eq 'Not 40 characters'
+          expect(c.snippets[1]).to eq 'Definitely 40 characters long test here!'
+        end
+
+        it 'can be set to any int 0 or greater' do
+          c = Celes.new(url: url, min_snippet_length: 17)
+          c.stub(:get_html_content).and_return '<p>Not 40 characters</p><p>Definitely 40 characters long test here!</p>'
+
+          expect(c.snippets.size).to be 2
+          expect(c.snippets[0]).to eq 'Not 40 characters'
+          expect(c.snippets[1]).to eq 'Definitely 40 characters long test here!'
+        end
+
+        it 'can be set to 0' do
+          c = Celes.new(url: url, min_snippet_length: 0)
+          c.stub(:get_html_content).and_return '<p>A</p><p>Not 40 characters</p><p>Definitely 40 characters long test here!</p>'
+
+          expect(c.snippets.size).to be 3
+          expect(c.snippets[0]).to eq 'A'
+        end
+      end
+
+      context ':snip_length' do
+        let(:text59) { 'C' * 59 }
+        let(:text140) { 'A' * 140 }
+        let(:text150) { 'B' * 150 }
+
+        before {Celes.any_instance.stub(:get_html_content).and_return "<p>#{text59}</p><p>#{text140}</p><p>#{text150}</p>"}
+
+        it 'defaults to 140 characters' do
+          c = Celes.new(url: url)
+
+          expect(c.snippets.size).to be 3
+          expect(c.snippets[0]).to eq text59
+          expect(c.snippets[1]).to eq text140
+          expect(c.snippets[2]).to eq text150[0, 140]
+        end
+
+        it 'defaults to 0 when a negative value is provided' do
+          c = Celes.new(url: url, snip_length: -100)
+
+          expect(c.snippets.size).to be 3
+          expect(c.snippets[0]).to be_empty
+          expect(c.snippets[1]).to be_empty
+        end
+
+        it 'can be set to be any value greater than 0' do
+          c = Celes.new(url: url, snip_length: 60)
+
+          expect(c.snippets.size).to be 3
+          expect(c.snippets[0]).to eq text59
+          expect(c.snippets[1]).to eq text140[0, 60]
+          expect(c.snippets[2]).to eq text150[0, 60]
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: celes-web
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Andy Schrage
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A simple Ruby gem for providing a snipet of text and images based on
+  a url.
+email:
+- ajschrag@mtu.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Gemfile.lock
+- README.md
+- celes.gemspec
+- lib/celes.rb
+- spec/celes_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/Swimminschrage/celes
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A simple Ruby gem for providing a snipet of text and thumbnails based on
+  a url.
+test_files:
+- spec/celes_spec.rb
+- spec/spec_helper.rb