cookler 0.2.0

data/lib/cookler.rb

@@ -0,0 +1,82 @@
+require 'anemone'
+require 'mongo'
+
+require 'cookler/utils'
+
+class Cookler
+
+  def initialize(targets, max_results=nil)
+      @targets      = targets
+      @max_results  = (max_results.nil? ? 15 : max_results)
+      @conn         = Mongo::Connection.new
+      @db           = @conn['cookler-db']
+      @pages        = @db['pages']
+      @stats        = @db['stats']
+      @depth_limit  = 3
+      @threads      = []
+  end
+      
+  def self.analyze(targets, max_results=nil)
+    cookler = self.new(targets, max_results)
+    cookler.run
+  end
+    
+  def run
+    
+    total_words = []
+    
+    @pages.remove
+    @stats.remove
+    
+    general_stime = Time.now.to_f
+    @targets.each do |target|
+      @threads << Thread.new(target) do |current_target|
+        current_target_stime = Time.now.to_f
+        current_target_words = _get_words_on_pages current_target
+        total_words += current_target_words
+        current_target_wstats = Hash[current_target_words.group_by{ |word| word.to_s.downcase}.map{ |word, instances| [word, instances.length] }.sort_by(&:last).reverse]
+        @stats.insert({:type => current_target[0], :duration => (Time.now.to_f - current_target_stime).to_s, :wstats => current_target_wstats})
+      end
+    end
+    @threads.each { |aThread|  aThread.join }
+    
+    general_wstats = Hash[total_words.group_by{ |word| word.to_s.downcase}.map{ |word, instances| [word, instances.length] }.sort_by(&:last).reverse]
+    @stats.insert({:type => :general, :duration => (Time.now.to_f - general_stime).to_s, :wstats => general_wstats})
+  end
+  
+  private
+  
+  def _get_words_on_pages(target)
+    
+    nbResults = 0
+    words = []
+    
+    anemone = Anemone::Core.new(target[1][0], {:depth_limit => @depth_limit})
+    anemone.storage = Anemone::Storage.MongoDB
+    anemone.focus_crawl do |page|
+      if nbResults == @max_results
+        anemone.kill_threads
+        return words
+      end
+      if target[1][3] == true
+        page.links.delete_if { |x| (x.to_s =~ target[1][1]).nil? }
+      end
+      page.links
+    end
+    anemone.on_pages_like(target[1][1]) do |page|
+      if not page.doc.nil?
+        page.doc.css('script, style, br').each { |x| x.replace ' ' }
+        title = page.doc.at('title').inner_html.gsub(/\r|\t|\n?/, "") rescue page.url.path
+        data = page.doc.at(target[1][2]).inner_text
+        @pages.insert({:type => target[0], :url => page.url.path, :title => title, :data => data})
+        words += data.removeaccents.to_s.gsub(/[0-9]+/, " ").scan(/[\w']+/)
+        nbResults += 1
+      end
+    end
+    anemone.run
+    anemone.kill_threads
+    return words
+  end
+
+end
+

data/lib/cookler/utils.rb

@@ -0,0 +1,46 @@
+# Monkey patch a new method kill_threads in Anemone::Core to kill all running threads:
+module Anemone
+    class Core
+        def kill_threads
+            @tentacles.each { |thread| Thread.kill(thread)  if thread.alive? }
+        end
+    end
+end
+
+# Patch to remove the accents from the string. (Uses String::ACCENTS_MAPPING as the source map).
+class String
+  
+  ACCENTS_MAPPING = {
+    'E' => [200,201,202,203],
+    'e' => [232,233,234,235],
+    'A' => [192,193,194,195,196,197],
+    'a' => [224,225,226,227,228,229,230],
+    'C' => [199],
+    'c' => [231],
+    'O' => [210,211,212,213,214,216],
+    'o' => [242,243,244,245,246,248],
+    'I' => [204,205,206,207],
+    'i' => [236,237,238,239],
+    'U' => [217,218,219,220],
+    'u' => [249,250,251,252],
+    'N' => [209],
+    'n' => [241],
+    'Y' => [221],
+    'y' => [253,255],
+    'AE' => [306],
+    'ae' => [346],
+    'OE' => [188],
+    'oe' => [189]
+  }
+  
+  def removeaccents    
+    str = String.new(self)
+    String::ACCENTS_MAPPING.each {|letter,accents|
+      packed = accents.pack('U*')
+      rxp = Regexp.new("[#{packed}]", nil)
+      str.gsub!(rxp, letter)
+    }
+    
+    str
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: cookler
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Alexandre MOOTASSEM
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-19 00:00:00.000000000Z
+dependencies: []
+description: ! "    Cookler is a Ruby library that provides spidering and analysing
+  features.\n    It helps you to quickly write a program to retrieve content and statistics
+  of any website.\n    The API is based on Anemone (http://anemone.rubyforge.org)
+  and use MongoDB as storage solution.\n    Cookler has a multi-threaded and easy-to-use
+  design. The retrieved data and generated stats are\n    stored in 'cookler-db'.\n"
+email: alexandre.mootassem@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cookler.rb
+- lib/cookler/utils.rb
+homepage: https://github.com/gastounage/cookler
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.8
+signing_key: 
+specification_version: 3
+summary: Spidering and Analyzing Library
+test_files: []