shiner 0.0.2

data/lib/shiner.rb

@@ -0,0 +1,57 @@
+#require "shiner/version"
+
+require 'rubygems'
+require 'bundler/setup'
+require 'pp'
+
+require 'tactful_tokenizer'
+require 'classifier'
+
+module Shiner
+  def self.shine(string, options={:max_length => 188})
+    best = string_to_best_sentences(string, options)
+    best[:sentences].collect{|sentence| sentence[:sentence]}.join(' ')
+  end
+  
+  def self.string_to_best_sentences(string, options={})
+    sentences = string_to_scored_sentences(string)
+    options[:max_sentences]||=sentences.size
+    batches = []
+    sentences.each_index{|index|
+      batch={:sentences => sentences[index, options[:max_sentences]]}
+      batch[:sentences].pop while options[:max_length] && batch[:sentences].collect{|sentence| sentence[:sentence]}.join(' ').length > options[:max_length]
+      batch[:score] = batch[:sentences].collect{|sentence| sentence[:score]}.sum
+      batches << batch
+    }
+    best = batches.sort_by{|batch| batch[:score]}.last
+  end
+  
+  def self.string_to_scored_sentences(string)
+    sentences=[]
+    string_to_sentences(string).each{ |sentence|
+      classifications = classifier.classifications(sentence)
+      sentences << {:sentence => sentence, :classifications => classifications, 
+        :score => (classifications['Interesting'] - classifications['Uninteresting']) / (-classifications['Uninteresting']) }
+    }
+    #sentences = sentences.sort_by{|sentence| sentence[:score] }
+    sentences
+  end
+  
+  def self.classifier
+    @classifier ||= (
+        b = Classifier::Bayes.new 'Interesting', 'Uninteresting'
+        File.read(File.dirname(__FILE__) + '/../data/interesting.txt').split("\n").each { |i|
+          b.train_interesting i
+        }
+        File.read(File.dirname(__FILE__) + '/../data/uninteresting.txt').split("\n").each { |i|
+          b.train_uninteresting i
+        }
+        b
+      )
+  end
+  
+  def self.string_to_sentences(string)
+    @tactful_tokenizer ||= TactfulTokenizer::Model.new
+    @tactful_tokenizer.tokenize_text(string)
+  end
+end

data/lib/shiner/version.rb

@@ -0,0 +1,3 @@
+module Shiner
+  VERSION = "0.0.2"
+end

data/shiner.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/shiner/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jason Ling Xiaowei"]
+  gem.email         = ["jason@jeyel.com"]
+  gem.description   = 'Extract the most interesting sentences from an article.'
+  gem.summary       = 'Extract the most interesting sentences from an article.'
+  gem.homepage      = "https://github.com/jlxw/shiner"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "shiner"
+  gem.require_paths = ["lib"]
+  gem.version       = Shiner::VERSION
+
+  gem.add_dependency "tactful_tokenizer"
+  gem.add_dependency "classifier"
+end

data/test/test_shiner.rb

@@ -0,0 +1,63 @@
+require 'test/unit'
+require 'shiner'
+#require 'active_support/testing/assertions'
+
+
+class ShinerTest < Test::Unit::TestCase
+  def test_truth
+    doc = <<-EOF
+Facebook shares have dipped below $29 for the first time since their flotation to a new low.
+
+The shares were launched at $38 less than two weeks ago during its high-profile flotation, which valued the firm at $104bn (�66bn).
+
+Since then, Facebook's shares have lost almost a quarter of their value.
+
+Russia's biggest social network, VKontakte, has now postponed its stock market launch, fearing a repeat of Facebook's problems.
+
+"The IPO of FB [Facebook] destroyed the faith of many private investors in social networks," said chief executive Pavel Durov in a message posted on Twitter.
+
+Facebook shares closed down 9.6% to $28.82, just above its low of $28.78.
+
+One reason for the fall in shares since its initial public offering (IPO) is that Tuesday was the first day that options on Facebook stock began trading.
+
+Options are a form of derivative, that allow bets on the future direction of the stock. It appears that most investors are betting Facebook shares will head lower.
+
+"Facebook's stock market debut is already going down as one of the most troubled of recent years," says BBC technology correspondent Rory Cellan-Jones.
+
+The situation is a remarkable turnaround from recent weeks, when the eight-year-old firm's share sale was over-subscribed.
+Lawsuits
+
+The social networking site has transformed the way in which hundreds of millions of people around the world communicate. It is also transforming the way companies advertise to existing and potential customers.
+
+But Facebook's 900 million users helped the company generate just $1bn in profit last year, and there are concerns about its ability to increase profits in the future.
+
+The flotation was disrupted on its first day of trading by technical glitches on the Nasdaq stock exchange. The share price has since slumped amid worries that the company was over-valued by advisers marketing the float.
+
+Now, a group of investors has issued a class-action lawsuit alleging that Facebook revenues were revised down because of a surge in the number of people using mobile devices for apps and connection to websites.
+
+The suit targets Facebook, its founder Mark Zuckerberg and the banks behind the flotation, including lead underwriter Morgan Stanley.
+
+The share sale in New York raised $16bn for Facebook.
+
+Recent reports suggested that Facebook is to launch its own smartphone by next year.
+EOF
+    result = Shiner.shine(doc, :max_length => 388)
+    pp [result, result.size]
+    assert result.size > 100
+    assert result.size < 388
+    result = Shiner.shine(doc, :max_length => 888)
+    pp [result, result.size]
+    assert result.size > 100
+    assert result.size < 888
+    result = Shiner.shine(doc, :max_length => 1888)
+    pp [result, result.size]
+    assert result.size > 100
+    assert result.size < 1888
+    result = Shiner.shine(doc, :max_sentences => 1)
+    pp [result, result.size]
+    result = Shiner.shine(doc, :max_sentences => 2)
+    pp [result, result.size]
+    result = Shiner.shine(doc, :max_sentences => 3)
+    pp [result, result.size]
+  end
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: shiner
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Jason Ling Xiaowei
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-05-30 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: tactful_tokenizer
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: classifier
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: Extract the most interesting sentences from an article.
+email: 
+- jason@jeyel.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- data/README
+- data/interesting.txt
+- data/uninteresting.txt
+- lib/shiner.rb
+- lib/shiner/version.rb
+- shiner.gemspec
+- test/test_shiner.rb
+homepage: https://github.com/jlxw/shiner
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.13
+signing_key: 
+specification_version: 3
+summary: Extract the most interesting sentences from an article.
+test_files: 
+- test/test_shiner.rb