webminer 0.0.1 → 0.0.2

data/lib/webminer.rb

@@ -5,17 +5,18 @@ require 'rss/1.0'
 require 'rss/2.0'
 require 'cgi'
 require 'iconv'
+require 'ruby-debug'
+require 'mongo_mapper'
 
 #Main class, just a place holder
 
 class WebMiner
 
   def self.create_story(item, topic)
-
     params=CGI::parse(item.link)
 
     link = params["url"][0]
-
+    puts Story.class
     if Story.where(:link => link).exists?
       return nil
     end
@@ -58,9 +59,10 @@ class WebMiner
 
     begin
       story.save
-      puts "Saved "+story.title
-    rescue SQLite3::ConstraintException,ActiveRecord::RecordNotUnique => ex
-      puts "Skipped duplicated story: "+story.title
+      puts "Saved #{story.title}"
+    rescue ex
+      puts "Exception saving #{story.title}"
+      pp ex
     end
     
     #doc.css 'div[class="storyText"]' cbsnews.com
@@ -172,7 +174,7 @@ class WebMiner
     $options['From']='anon@anon.net'
     topics.keys.each do |topic|
       url = topics[topic]
-      sleep 5+prng.rand*2
+      sleep 20+prng.rand*2
       topic_threads << Thread.new(url) do
         #uri = URI.parse(url)
 
@@ -215,7 +217,7 @@ class WebMiner
             print "Skipping\n"
           end
           hash = new_hash
-          sleep 30+prng.rand*2
+          sleep 180+prng.rand*10
         end #while
       end #thread
     end #each

data/lib/webminer/util.rb

@@ -1,3 +1,5 @@
+require 'gsl'
+
 class WebMiner::Util
 
   def self.strip_js(raw_text)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webminer
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -12,10 +12,39 @@ cert_chain: []
 date: 2012-03-25 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name:
-  - mongo
-  - mongo_mapper
-  - nokogiri
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongo_mapper
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -64,3 +93,4 @@ signing_key:
 specification_version: 3
 summary: I mine the web
 test_files: []
+has_rdoc: