pismo 0.2.2 → 0.2.3

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.2.3

data/lib/pismo.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'open-uri'
 require 'nokogiri'
 require 'fast_stemmer'
@@ -33,9 +35,12 @@ class Nokogiri::HTML::Document
       elsif query.is_a?(Array)
         result = query[1].call(self.search(query.first).first).strip rescue nil
       end
+
       if result
-        result.gsub!(/\342\200\231/, '\'')
-        result.gsub!(/\342\200\224/, '-')
+      #  result.gsub!(/\342\200\231/, '\'')
+      #  result.gsub!(/\342\200\224/, '-')
+        result.gsub!('’', '\'')
+        result.gsub!('—', '-')
         return result
       end
     end

data/lib/pismo/internal_attributes.rb

@@ -3,6 +3,7 @@ module Pismo
   module InternalAttributes
     # Returns the title of the page/content - attempts to strip site name, etc, if possible
     def title
+      # TODO: Memoizations
       title = @doc.match( 'h2.title',
                           '.entry h2',                                                      # Common style
                           '.entryheader h1',                                                # Ruby Inside/Kubrick
@@ -147,12 +148,13 @@ module Pismo
       
       words = {}
       
-      # Convert doc to lowercase, scrub out most HTML tags
-      body.downcase.gsub(/\<[^\>]{1,100}\>/, '').gsub(/\&\w+\;/, '').scan(/\b[a-z][a-z\'\#\.]*\b/).each do |word|
+      # Convert doc to lowercase, scrub out most HTML tags, then keep track of words
+      cached_title = title
+      body.downcase.gsub(/\<[^\>]{1,100}\>/, '').gsub(/\&\w+\;/, '').scan(/\b[a-z][a-z\'\+\#\.]*\b/).each do |word|
         next if word.length > options[:word_length_limit]
         word.gsub!(/\'\w+/, '')
         words[word] ||= 0
-        words[word] += 1
+        words[word] += (cached_title =~ /#{word}/i ? 5 : 1)
       end
 
       # Stem the words and stop words if necessary
@@ -168,8 +170,8 @@ module Pismo
       @body ||= Readability::Document.new(@doc.to_s).content.strip
       
       # HACK: Remove annoying DIV that readability leaves around
-      @body.gsub!(/\A\<div\>/, '')
-      @body.gsub!(/\<\/div\>\Z/, '')
+      @body.sub!(/\A\<div\>/, '')
+      @body.sub!(/\<\/div\>\Z/, '')
       
       return @body
     end

data/lib/pismo/readability.rb

@@ -174,13 +174,13 @@ module Readability
     def score_node(elem)
       content_score = class_weight(elem)
       case elem.name.downcase
-        when "div":
+        when "div"
           content_score += 5
-        when "blockquote":
+        when "blockquote"
           content_score += 3
-        when "form":
+        when "form"
           content_score -= 3
-        when "th":
+        when "th"
           content_score -= 5
       end
       { :content_score => content_score, :elem => elem }

data/lib/pismo/stopwords.txt

@@ -890,4 +890,7 @@ Erin
 Nevaeh
 Brooklyn
 Marissa
-yeah
+yeah
+covering
+kid
+notably

data/pismo.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{pismo}
-  s.version = "0.2.2"
+  s.version = "0.2.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Peter Cooper"]
-  s.date = %q{2010-03-26}
+  s.date = %q{2010-05-03}
   s.default_executable = %q{pismo}
   s.description = %q{Pismo extracts and retrieves content-related metadata from HTML pages - you can use the resulting data in an organized way, such as a summary/first paragraph, body text, keywords, RSS feed URL, favicon, etc.}
   s.email = %q{git@peterc.org}
@@ -39,6 +39,7 @@ Gem::Specification.new do |s|
      "test/corpus/factor.html",
      "test/corpus/huffington.html",
      "test/corpus/metadata_expected.yaml",
+     "test/corpus/metadata_expected.yaml.old",
      "test/corpus/rubyinside.html",
      "test/corpus/rww.html",
      "test/corpus/spolsky.html",

data/test/corpus/metadata_expected.yaml

@@ -1,10 +1,4 @@
 --- 
-:rubyinside: 
-  :title: "CoffeeScript: A New Language With A Pure Ruby Compiler"
-  :author: Peter Cooper
-  :lede: CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it "JavaScript's less ostentatious kid brother" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.
-  :body: "CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it \"JavaScript's less ostentatious kid brother\" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.\nTo get a feel for the language, check out this example code (CoffeeScript on the left, resulting JavaScript on the right):\nAs a Ruby project, you can get the CoffeeScript compiler installed with a simple gem install coffee-script or check out the code from/on GitHub. The code is worth a look as it's notably quite vanilla with hand crafted Ruby covering the lexer and code generation and Racc built code for the parser.\n"
-  :feed: http://www.rubyinside.com/feed/
 :rww: 
   :title: "Cartoon: Apple Tablet: Now With Barometer and Bird Call Generator"
   :feed: http://www.readwriteweb.com/rss.xml
@@ -38,44 +32,9 @@
 :techcrunch: 
   :title: Googlle Gets A Sexy New Logo; Remains Sketchy
   :author: MG Siegler
-  :keywords:
-  - - googlle
-    - 7
-  - - google
-    - 6
-  - - site
-    - 3
-  - - logo
-    - 2
-  - - font
-    - 2
-  - - india
-    - 2
-  - - surprised
-    - 1
-  - - week
-    - 1
-  - - switched
-    - 1
-  - - school
-    - 1
-  - - things
-    - 1
-  - - removing
-    - 1
-  - - steve
-    - 1
-  - - decided
-    - 1
-  - - advantage
-    - 1
-  - - wasn
-    - 1
-  - - accepting
-    - 1
-  - - red
-    - 1
-  - - copy
-    - 1
-  - - wouldn
-    - 1
+:rubyinside: 
+  :title: "CoffeeScript: A New Language With A Pure Ruby Compiler"
+  :author: Peter Cooper
+  :lede: CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it "JavaScript's less ostentatious kid brother" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.
+  :body: "CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it \"JavaScript's less ostentatious kid brother\" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.\nTo get a feel for the language, check out this example code (CoffeeScript on the left, resulting JavaScript on the right):\nAs a Ruby project, you can get the CoffeeScript compiler installed with a simple gem install coffee-script or check out the code from/on GitHub. The code is worth a look as it's notably quite vanilla with hand crafted Ruby covering the lexer and code generation and Racc built code for the parser.\n"
+  :feed: http://www.rubyinside.com/feed/

data/test/corpus/metadata_expected.yaml.old

@@ -0,0 +1,122 @@
+--- 
+:rww: 
+  :title: "Cartoon: Apple Tablet: Now With Barometer and Bird Call Generator"
+  :feed: http://www.readwriteweb.com/rss.xml
+:briancray: 
+  :title: 5 great examples of popular blog posts that you should know
+  :feed: http://feeds.feedburner.com/briancray/blog
+  :lede: "This is a mock post. While there is a place for all of these posts, I'm trying to make a point that original blogs are being shut out by formulaic blogs."
+:huffington:
+  :title: Afghans Losing Hope After 8 Years Of War
+  :author: TODD PITMAN
+  :feed: http://feeds.huffingtonpost.com/huffingtonpost/raw_feed
+  :lede: "KABUL - The man on the motorcycle was going the wrong way down a one-way street, gesturing indignantly for the phalanx of traffic-clogged cars in front of him to move."
+:bbcnews:
+  :title: Gay Muslims made homeless by family violence
+  :author: Poonam Taneja
+  :description: A charity is dealing with more gay Muslims made homeless after fleeing forced marriages and so-called "honour" violence.
+  :lede: A UK charity is dealing with an increasing number of young gay Muslims becoming homeless after fleeing forced marriages and so-called honour violence.
+  :feed: http://newsrss.bbc.co.uk/rss/newsonline_uk_edition/england/rss.xml
+:factor:
+  :title: Factor's bootstrap process explained
+  :lede: "Separation of concerns between Factor VM and library codeThe Factor VM implements an abstract machine consisting of a data heap of objects, a code heap of machine code blocks, and a set of stacks. The VM loads an image file on startup, which becomes the data and code heap. "
+:youtube:
+  :title: YMO - Rydeen (Official Video)
+  :author: ymo1965
+:spolsky:
+  :title: The Absolute Minimum Every Software Developer Absolutely, Positively Must Know About Unicode and Character Sets (No Excuses!)
+  :description: Haven't mastered the basics of Unicode and character sets? Please don't write another line of code until you've read this article.
+  :author: Joel Spolsky
+  :favicon: /favicon.ico
+  :feed: http://www.joelonsoftware.com/rss.xml
+:techcrunch: 
+  :title: Googlle Gets A Sexy New Logo; Remains Sketchy
+  :author: MG Siegler
+  :keywords:
+  - - googlle
+    - 35
+  - - logo
+    - 10
+  - - google
+    - 6
+  - - site
+    - 3
+  - - font
+    - 2
+  - - india
+    - 2
+  - - surprised
+    - 1
+  - - week
+    - 1
+  - - switched
+    - 1
+  - - school
+    - 1
+  - - things
+    - 1
+  - - removing
+    - 1
+  - - steve
+    - 1
+  - - decided
+    - 1
+  - - advantage
+    - 1
+  - - wasn
+    - 1
+  - - accepting
+    - 1
+  - - red
+    - 1
+  - - copy
+    - 1
+  - - wouldn
+    - 1
+:rubyinside: 
+  :title: "CoffeeScript: A New Language With A Pure Ruby Compiler"
+  :author: Peter Cooper
+  :lede: CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it "JavaScript's less ostentatious kid brother" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.
+  :body: "CoffeeScript (GitHub repo) is a new programming language with a pure Ruby compiler. Creator Jeremy Ashkenas calls it \"JavaScript's less ostentatious kid brother\" - mostly because it compiles into JavaScript and shares most of the same constructs, but with a different, tighter syntax.\nTo get a feel for the language, check out this example code (CoffeeScript on the left, resulting JavaScript on the right):\nAs a Ruby project, you can get the CoffeeScript compiler installed with a simple gem install coffee-script or check out the code from/on GitHub. The code is worth a look as it's notably quite vanilla with hand crafted Ruby covering the lexer and code generation and Racc built code for the parser.\n"
+  :feed: http://www.rubyinside.com/feed/
+  :keywords:
+  - - ruby
+    - 15
+  - - coffeescript
+    - 15
+  - - compiler
+    - 10
+  - - language
+    - 10
+  - - coffee
+    - 5
+  - - pure
+    - 5
+  - - code
+    - 5
+  - - script
+    - 5
+  - - javascript
+    - 3
+  - - github
+    - 2
+  - - syntax
+    - 1
+  - - programming
+    - 1
+  - - brother
+    - 1
+  - - constructs
+    - 1
+  - - vanilla
+    - 1
+  - - parser
+    - 1
+  - - lexer
+    - 1
+  - - project
+    - 1
+  - - installed
+    - 1
+  - - simple
+    - 1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: pismo
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors: 
 - Peter Cooper
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-03-26 00:00:00 +00:00
+date: 2010-05-03 00:00:00 +01:00
 default_executable: pismo
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -102,6 +102,7 @@ files:
 - test/corpus/factor.html
 - test/corpus/huffington.html
 - test/corpus/metadata_expected.yaml
+- test/corpus/metadata_expected.yaml.old
 - test/corpus/rubyinside.html
 - test/corpus/rww.html
 - test/corpus/spolsky.html