liveblog-indexer 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 711ba6e2a290dfc67dfc1703ffcddf417c7c90d3
-  data.tar.gz: 5525552551eee1953370b99034d2a712bb308ac2
+  metadata.gz: f82780521af1cc06c3f824c15200ce00745a99b7
+  data.tar.gz: 66819da8961da8a60b3e7441318d9f898274893f
 SHA512:
-  metadata.gz: 8648c96fdf3f0bf7c0b1ccc72f99bef2759db323a159e4361fc479f7d0d5847b2ad0f04361533e161f1a0cd6eb5b85544823f0159cd270adb29e6ac00cce10ed
-  data.tar.gz: 075d272a0af388c4addbcfb141ac0de8143d60b5b7ada4575d93a9ba84cc8e5b24b3c08490171196708012dbbed65dd5f2a588bdde538290f20ba121f79ced1a
+  metadata.gz: dbea6b4324aaad28c221b24d57b9bb06801d74678dbd0953dc54d3602f39e115b74e12cb66e90c2a86ef769d2432313b533afc3fd19b6eda5f9fcec189e112ac
+  data.tar.gz: 0b9dbb25e21b8adf798a4bc766ae338821b2b4e79f7e44ad9c76e4311ae868233a407ceb55669c80ae90c688134e7d474fe63cf36b8d173266c5433c74e50bff

data/lib/liveblog-indexer.rb

@@ -19,7 +19,12 @@ class LiveBlogIndexer
     end
 
     @xws = XWS.new
-    @url_index = {}    
+        
+    @url_index = if urls_indexed and File.exists? urls_indexed then
+      JSON.parse(File.read(urls_indexed))
+    else
+      {}
+    end
 
   end
 
@@ -40,11 +45,10 @@ class LiveBlogIndexer
 
       url = "%s/#%s" % [link[/^https?:\/\/[^\/]+(.*)(?=\/$)/,1], \
                                                        section.attributes[:id]]
-
       h = @xws.scan section.element('details')
       
       h.each do |k, v|
-
+        
         word, count = k, v
 
         keyword = @master[word]
@@ -61,6 +65,8 @@ class LiveBlogIndexer
         end # /keyword
       end # /h
     end # /section
+    
+    true
   end # /add_index
   
   def crawl(location)
@@ -81,14 +87,16 @@ class LiveBlogIndexer
   private
   
   def index_file(location)
-
+    
+    return if @url_index.has_key? location
+    
     puts 'indexing : ' + location.inspect
     doc = Rexle.new(RXFHelper.read(location).first)
     summary = doc.root.element 'summary'
     return unless summary
     
     result = add_index doc
-    return unless result
+    return  unless result
 
     prev_day = summary.text 'prev_day'
 
@@ -98,9 +106,6 @@ class LiveBlogIndexer
       index_file(url) 
     end
   end
-  
-  def save_urlsindex(filepath)
-    
-  end
+
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: liveblog-indexer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - James Robertson