webminer 0.0.0 → 0.0.1

data/lib/webminer.rb

@@ -229,6 +229,5 @@ class WebMiner
 
 end
 
-require 'webminer/constants'
 require 'webminer/util'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webminer
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -10,8 +10,28 @@ autorequire:
 bindir: bin
 cert_chain: []
 date: 2012-03-25 00:00:00.000000000Z
-dependencies: []
-description: I really just mine the web
+dependencies:
+- !ruby/object:Gem::Dependency
+  name:
+  - mongo
+  - mongo_mapper
+  - nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: use in conjunction with https://github.com/yushen/tflogr, run rails r
+  script/webminer_script.rb
 email: yushen83@gmail.com
 executables: []
 extensions: []
@@ -19,7 +39,6 @@ extra_rdoc_files: []
 files:
 - lib/webminer.rb
 - lib/webminer/util.rb
-- lib/webminer/constants.rb
 homepage: https://github.com/yushen
 licenses: []
 post_install_message: 

data/lib/webminer/constants.rb

@@ -1,57 +0,0 @@
-class WebMiner::Constants
-  def self.get_parser_dictionary
-    return {
-    "google.com" => 'div[id="hostednews-article"]',
-    "cbsnews.com" => 'div[id="contentBody"]',
-    "reuters.com" => 'span[id="articleText"]',
-    "latimes.com" => 'div[id="story-body-text"]',
-    "csmonitor.com" => 'div[id="mainColumn"]',
-    "npr.org" => 'div[id="storytext"]',
-    "usatoday.com" => 'div[id="mainstory"]',
-    "content.usatoday.com" => 'div[id="mainstory"]',
-    "guardian.co.uk" => 'div[id="article-body-blocks"]',
-    "nytimes.com" => 'div[id="article"]',
-    "bloomberg.com" => 'div[id="story_content"]',
-    "online.wsj.com" => 'div[id="article_story_body"]',
-    "asia.wsj.com" => 'div[id="article_story_body"]',
-    "businessweek.com" => 'div[id="story-body"]',
-    "cnn.com" => 'div[id="cnnContentContainer"]',
-    "edition.cnn.com" => 'cnn_storyarea[id="cnnContentContainer"]',
-    "money.cnn.com" => 'div[id="storytext"]',
-    "abcnews.go.com" => 'div[id="innerbody"]',
-    "foxnews.com" => 'div[id="introduction"]',
-    "businessweek.com" => 'div[id="story-body"]',
-    "entertainment.msnbc.msn.com" => 'div[id="vine-t"] article',
-    "washingtonpost.com" => 'div[id="article_body"]',
-    #    "bbc.co.uk" => 'div[id="main-content"]',
-    "huffingtonpost.com" => 'div[id="entry_12345"]',
-    "telegraph.co.uk" => 'div[id="mainBodyArea"]',
-    "chicagotribune.com" => 'div[id="story-body-text"]',
-    "foxbusiness.com" => 'div[id="introduction"]',
-    "thedailybeast.com" => 'div[id="main"] article',
-    "economictimes.indiatimes.com" => 'div[id="storydiv"]',
-    "forbes.com" => 'div[id="leftRail"]',
-    "arstechnica.com" => 'div[id="story"]',
-    "theregister.co.uk"=> 'div[id="body"]',
-    "ingame.msnbc.msn.com"=> 'div[id="vine-t"] article',
-    "informationweek.com"=> 'span[id="articleBody"]',
-    "newyorker.com"=> 'div[id="articletext"]',
-    "kotaku.com"=> 'div[id="page"]',
-    "slashgear.com"=> 'span[id="intelliTxt"]',
-    "pcworld.com"=> 'div[id="articleText"]',
-    "news.cnet.com"=> 'div[id="article"]',
-    "english.aljazeera.net"=> 'td[id="tdTextContent"]',
-    "dailymail.co.uk"=> 'div[id="js-article-text"]',
-    "rttnews.com"=> 'div[id=""]',
-    "ft.com"=> 'div[id="storyContent"]',
-    "politico.com"=> 'div[id="mainContent"]',
-    "boston.com"=> 'div[id="page1"]',
-    "sfgate.com"=> 'div[id="fontprefs_bottom"]',
-    "oregonlive.com"=> 'div[id="article"]'
-    #""=> 'div[id=""]',
-
-    #    "wired.com"=> 'div[id=""]'?
-    #http://latimesblogs.latimes.com ?
-  }
-  end
-end