gistgen 0.1.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+.DS_Store
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in gistgen.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/gistgen.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "gistgen/version"
+
+Gem::Specification.new do |s|
+  s.name        = "gistgen"
+  s.version     = Gistgen::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Quan Nguyen"]
+  s.email       = ["mquannie@gmail.com"]
+  s.homepage    = "http://github.com/mquan/gistgen"
+  s.summary     = %q{generate different types of summaries for a text}
+  s.description = %q{gistgen has several modules to generate summaries from wikipedia and crunchbase}
+  s.rubyforge_project = "gistgen"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_dependency "mechanize"
+  s.add_dependency "json"
+  s.add_dependency "htmlentities"  
+end

data/lib/gistgen.rb

@@ -0,0 +1,8 @@
+$: << File.dirname(__FILE__)
+require 'gistgen/wiki_abstract'
+require 'gistgen/crunch_view'
+require 'gistgen/hacker_news'
+require 'gistgen/reddit'
+
+module Gistgen
+end

data/lib/gistgen/crunch_view.rb

@@ -0,0 +1,44 @@
+require 'json'
+require 'htmlentities'
+
+require "gistgen/page"
+require "gistgen/string"
+require "gistgen/url"
+
+module Gistgen
+  class CrunchView    
+    def initialize(name)
+      begin
+        res = Gistgen::Page.get_page("http://api.crunchbase.com/v/1/company/#{name}.js")
+        @json = JSON.parse(res)
+        @json = (@json and @json['overview'])? @json : nil
+      rescue
+        nil
+      end   
+    end
+    
+    def overview(length=500)
+      begin
+        text = @json['overview'].gsub(/\u003C(.*?)\\u003E/,'').gsub(/<(.*?)>/,'').gsub("\n",'')
+        text = HTMLEntities.new.decode(text) #decode_html
+        text.extract_passage(0, length)
+      rescue
+        nil
+      end
+    end
+    
+    def permalink
+      begin
+        Gistgen::URL.standardize("http://www.crunchbase.com/company/#{@json['permalink']}")
+      rescue
+        nil
+      end
+    end
+
+    def homepage
+      (@json)? Gistgen::URL.standardize(@json['homepage_url']) : nil
+    end
+    
+  end
+end
+

data/lib/gistgen/hacker_news.rb

@@ -0,0 +1,61 @@
+require 'json'
+
+require 'gistgen/page'
+require 'gistgen/url'
+
+module Gistgen
+  class HackerNews
+    #http://api.ihackernews.com/        
+    def self.frontpage
+      begin
+        res = Gistgen::Page.get_page("http://api.ihackernews.com/page")
+        Gistgen::HackerNews.get_hash(res)
+      rescue
+        nil
+      end
+    end
+    
+    #don't use this too often (low score shouldn't be added)
+    def self.new_posts
+      begin
+        res = Gistgen::Page.get_page("http://api.ihackernews.com/new")
+        Gistgen::HackerNews.get_hash(res)
+      rescue
+        nil
+      end
+    end
+    
+    def self.get_score(hn_url)
+      id = hn_url.match(/\d+$/)[0]
+      begin
+        res = Gistgen::Page.get_page("http://api.ihackernews.com/post/#{id}")
+        json = JSON.parse(res)
+        json['points']
+      rescue
+        nil
+      end
+    end
+    
+    def self.get_hash(res)
+      json = JSON.parse(res)
+      json['items'].map do |i| 
+        {"title" => i['title'], 
+          "url" => Gistgen::URL.standardize(i['url']), 
+          "score" => i['points'], 
+          "time" => Gistgen::HackerNews.parse_time(i['postedAgo']),
+          "discussion_url" => "http://news.ycombinator.com/item?id=#{i['id']}"
+        }
+      end
+    end
+    
+    def self.parse_time(time_ago)
+      begin
+        tmp = time_ago.split(' ')
+        time = tmp[0].to_i.send(tmp[1]).ago
+      rescue
+        Time.now.utc
+      end
+    end
+    
+  end
+end

data/lib/gistgen/page.rb

@@ -0,0 +1,18 @@
+require 'mechanize'
+
+module Gistgen
+  class Page
+    #make http request and return the html page
+    def self.get_page(url, user_agent='gistgen gem request')
+      begin
+        agent = Mechanize.new
+        agent.user_agent = user_agent
+        agent.get(url)
+        agent.page.body
+      rescue
+        nil
+      end
+    end
+    
+  end
+end

data/lib/gistgen/reddit.rb

@@ -0,0 +1,48 @@
+require 'json'
+
+require 'gistgen/page'
+require 'gistgen/url'
+
+module Gistgen
+  class Reddit
+    #http://code.reddit.com/wiki/API
+    #reddit api is so nice, you just pick a page and add .json to get just the data
+    #ex: http://www.reddit.com/.json
+    def self.fetch(url)
+      res = Gistgen::Page.get_page("#{url.gsub(/\/$/,'')}/.json")
+      Gistgen::Reddit.get_hash(res)
+    end
+    
+    def self.get_score(reddit_url)
+      begin
+        res = Gistgen::Page.get_page("#{reddit_url.gsub(/\/$/,'')}/.json")
+        score = res.scan(/"score"\s*:\s*(\d+)/)[0].join('').to_i #reddit nested comments is too deep for json
+      rescue
+        nil
+      end
+    end
+    
+    def self.get_hash(res)
+      json = JSON.parse(res)
+      items = json['data']['children']
+      items.map do |i|
+        post = i['data']
+        {"title" => post['title'], 
+          "url" => Gistgen::URL.standardize(post['url']), 
+          "score" => post['score'], 
+          "time" => Time.at(post['created_utc']),
+          "discussion_url" => "http://reddit.com#{post['permalink']}"
+        }
+      end
+    end
+    
+    #ban digg: they link to their url shortener
+    #http://developers.digg.com/documentation
+    #require 'uri'
+    #def self.get_diggs(url)
+    #  res = Gistgen::Page.get_page(URI.escape(url)) #need to encode url
+    #  json = JSON.parse(res)
+    #  json['stories'][0]['diggs']
+    #end
+  end
+end

data/lib/gistgen/string.rb

@@ -0,0 +1,39 @@
+class String
+  #return a passage of size <= length from start_indexth sentence
+  def extract_passage(start_index=0,length=500)
+    sentences = self.split_sentences
+    stop = ((start_index+1)...sentences.size).detect { |i| (sentences[start_index..i].join('. ')).size > length }
+    stop = (stop and stop <= sentences.size)? stop-1 : sentences.size - 1
+    passages = sentences[start_index...stop].join('. ').split("\n")
+    (passages.size > 0)? passages[0].gsub(/^[^\w]+/,'').limit(length) : ''
+  end
+  
+  #split text into sentences, take into account Mr.|Ms. endings are not end of sentence
+  def split_sentences
+    #break text first by paragraph then into chunks delimited by a period
+    #but these are not quite sentences yet
+    chunks = (self.split(/\n+/).map { |p| "#{p}\n".split(/\.(?:[^\w])/) }).flatten.compact
+  
+    #if a sentence is split at Mr.|Ms.|Dr.|Mrs. 
+    #then recombine it with its remaining part and nil it to delete later
+    tmp=''
+    sentences = chunks.map { |c|
+      ss = (tmp != '')? "#{tmp}. #{c}" : c
+      if c.match(/(?:Dr|Mr|Ms|Mrs)$/) #what about John F. Kennedy ([A-Z])
+        tmp = ss
+        ss=nil
+      else
+        tmp = ''
+      end
+      ss
+    } 
+    sentences.compact #delete nil elements
+  end
+  
+  #constraint a string to a fixed length or less
+  #discard everything after the last punctuation that occurs right before lenght limit
+  #the regexp look ahead for any punctuation
+  def limit(length)
+    (self.length > length)? self[0...length].gsub(/(?![\s\S]+?[,:;)\/\\\|])([,:;)\/\\\|].*)/,'') : self
+  end
+end

data/lib/gistgen/url.rb

@@ -0,0 +1,32 @@
+module Gistgen
+  class URL
+    def self.standardize(url)
+      protocol = url.split('.')[0].match(/^(.*):\/\//)
+      u1 = (!protocol)? "http://#{url}" : url
+      #raise error if protocol && protocol[0] != 'http'
+
+      #remove www subdomain if exist
+      u2 = u1.gsub(/^(http|https):\/\/www\./ix,'http://')
+
+      #make sure google.com and google.com/ are the same thing
+      u3 = (u2.match(/^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?$/ix))? "#{u2}/" : u2
+    end
+    
+    def self.is_valid?(url)
+      url.match(/^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}((:[0-9]{1,5})?\/.*)?$/ix)
+    end
+    
+    def self.is_image?(url)
+      url.match(/\.(?:jpg|jpeg|png|gif|tiff|raw|bmp|webp|ai|psd|svg)$/i)
+    end
+    
+    def self.is_multimedia?(url)
+      url.match(/\.(?:js|css|mp3|swf|wmv|mov|doc|pdf|ppt|xls|xlsx|docx|eps|ps|ttf|xml)$/i)
+    end
+    
+    #return true if it's a base url (nothing after first '/') and does not have subdomain
+    def self.is_root?(url)
+      url.match(/^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?\/$/ix) #and url.split('.').size == 2
+    end
+  end
+end

data/lib/gistgen/version.rb

@@ -0,0 +1,3 @@
+module Gistgen
+  VERSION = "0.1.1"
+end

data/lib/gistgen/wiki_abstract.rb

@@ -0,0 +1,34 @@
+require 'uri'
+require 'json'
+
+require "gistgen/page"
+require "gistgen/string"
+
+module Gistgen
+  class WikiAbstract
+    
+    def self.permalink(name)
+      "http://en.wikipedia.org/wiki/#{name}"
+    end
+    
+    def self.search(query, length=500)
+      q = URI.escape(query)
+      res = Gistgen::Page.get_page("http://en.wikipedia.org/w/api.php?format=json&action=query&prop=revisions&titles=#{q}&rvprop=content&rvsection=0")
+      json = JSON.parse(res)
+      rev=0
+      json['query']['pages'].each_key { |k| rev=k }
+      all_text = ''
+      text = json['query']['pages'][rev]['revisions'][0]['*'].to_s
+      if text.include?('#REDIRECT')
+        new_q = text.match(/\[\[(.*?)\]\]/)[0].gsub('[','').gsub(']','')
+        all_text = WikiAbstract.search(new_q)
+      else
+        t = text[text.index("'''")...text.size].gsub(/^\s+/,'')
+        all_text = t.gsub(/<ref>(.*?)<\/ref>/i,'').gsub(/<small>(.*?)<\/small>/,'').gsub(/<(.*?)>/,'').gsub(/\{\{(.*?)\}\}/,'').gsub(/\(stylized(.*?)\)/,'').gsub(/\[\[([^\]\]]*?)\|/,'')
+        ["[","]","'''"].each { |g,clean| all_text.gsub!(g,'') }
+      end
+      all_text.extract_passage(0, length)
+    end
+    
+  end
+end

data/tests/crunch_view_test.rb

@@ -0,0 +1,5 @@
+require "#{File.dirname(__FILE__)}/../lib/gistgen"
+
+cv = Gistgen::CrunchView.new('google')
+puts cv.permalink
+puts cv.overview

data/tests/hubs_test.rb

@@ -0,0 +1,8 @@
+require "#{File.dirname(__FILE__)}/../lib/gistgen"
+
+puts Gistgen::HackerNews.get_score('http://news.ycombinator.com/item?id=2458202')
+puts Gistgen::HackerNews.frontpage
+puts Gistgen::HackerNews.new_posts
+
+puts Gistgen::Reddit.fetch("http://reddit.com/")
+puts Gistgen::Reddit.get_score('http://www.reddit.com/r/funny/comments/gu7jw/dictator/')

data/tests/wiki_test.rb

@@ -0,0 +1,3 @@
+require "#{File.dirname(__FILE__)}/../lib/gistgen"
+
+puts Gistgen::WikiAbstract.search('google.com')

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification 
+name: gistgen
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Quan Nguyen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-01 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: gistgen has several modules to generate summaries from wikipedia and crunchbase
+email: 
+- mquannie@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- Rakefile
+- gistgen.gemspec
+- lib/gistgen.rb
+- lib/gistgen/crunch_view.rb
+- lib/gistgen/hacker_news.rb
+- lib/gistgen/page.rb
+- lib/gistgen/reddit.rb
+- lib/gistgen/string.rb
+- lib/gistgen/url.rb
+- lib/gistgen/version.rb
+- lib/gistgen/wiki_abstract.rb
+- tests/crunch_view_test.rb
+- tests/hubs_test.rb
+- tests/wiki_test.rb
+homepage: http://github.com/mquan/gistgen
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: gistgen
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: generate different types of summaries for a text
+test_files: []
+