liveblog-indexer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56937c6f8eb2b40f417e73a5512fa60593647517
-  data.tar.gz: a87152912b799beab92e305565065ee391364b32
+  metadata.gz: 711ba6e2a290dfc67dfc1703ffcddf417c7c90d3
+  data.tar.gz: 5525552551eee1953370b99034d2a712bb308ac2
 SHA512:
-  metadata.gz: 9e99db1533e3279c4d4b70ddb58609fb5e234e8e9124d401bff3970d5e8d2ffa5671556d1ec0e6706c0212868bde0c469f5e07e5a85609838aa5494df7bf9426
-  data.tar.gz: 9d58190002e8cb8b490c61879f47901fe07c0ec374546dd52fae46d0c5c95ec8a6dce1c92eff850e957c0d776fe99f59577eb5c32f1bf46c2512a516140f385e
+  metadata.gz: 8648c96fdf3f0bf7c0b1ccc72f99bef2759db323a159e4361fc479f7d0d5847b2ad0f04361533e161f1a0cd6eb5b85544823f0159cd270adb29e6ac00cce10ed
+  data.tar.gz: 075d272a0af388c4addbcfb141ac0de8143d60b5b7ada4575d93a9ba84cc8e5b24b3c08490171196708012dbbed65dd5f2a588bdde538290f20ba121f79ced1a

data/lib/liveblog-indexer.rb

@@ -4,13 +4,14 @@
 
 require 'xws'
 require 'json'
-require 'polyrex'
+require 'rxfhelper'
 
 
 class LiveBlogIndexer
 
-  def initialize(filepath=nil)
+  def initialize(filepath='wordindex.json', urls_indexed: 'urls-indexed.json')
 
+    @wordindex_filepath, @urls_index_filepath = filepath, urls_indexed
     @master = if filepath and File.exists? filepath then
       JSON.parse(File.read(filepath))
     else
@@ -18,17 +19,26 @@ class LiveBlogIndexer
     end
 
     @xws = XWS.new
+    @url_index = {}    
 
   end
 
   def add_index(src)
 
-    doc = Rexle.new(RXFHelper.read(src).first )
+    doc = if src.is_a? String then
+      Rexle.new(RXFHelper.read(src).first )
+    else
+      src
+    end
+    
+    link = doc.root.element('summary/link/text()')
+    return unless link
+    
     sections = doc.root.xpath 'records/section'
 
     sections.each do |section|
 
-      url = "%s/#%s" % [src[/^https?:\/\/[^\/]+(.*)(?=\/formatted.xml$)/,1], \
+      url = "%s/#%s" % [link[/^https?:\/\/[^\/]+(.*)(?=\/$)/,1], \
                                                        section.attributes[:id]]
 
       h = @xws.scan section.element('details')
@@ -52,25 +62,45 @@ class LiveBlogIndexer
       end # /h
     end # /section
   end # /add_index
-
+  
+  def crawl(location)
+    
+    index_file location
+    save @wordindex_filepath
+    File.write @urls_index_filepath, @url_index.to_json
+    
+  end  
 
   def save(filepath=nil)
 
     File.write filepath, @master.to_json
     puts 'saved ' + File.basename(filepath)
 
-    px = Polyrex.new 'words/key[word]/location[url, wordcount]'
-
-    @master.each do |k,rows|
-      px.create.key(word: k) do |create|
-        rows.each do |k, v|
-          create.location url: k, wordcount: v
-        end
-      end
+  end
+  
+  private
+  
+  def index_file(location)
+
+    puts 'indexing : ' + location.inspect
+    doc = Rexle.new(RXFHelper.read(location).first)
+    summary = doc.root.element 'summary'
+    return unless summary
+    
+    result = add_index doc
+    return unless result
+
+    prev_day = summary.text 'prev_day'
+
+    if prev_day then
+      url = prev_day + 'formatted.xml'
+      @url_index[url] = {last_indexed: Time.now}
+      index_file(url) 
     end
-    pxfilepath = File.join(File.dirname(filepath), 'search-index.xml')
-    px.save pxfilepath, options: {pretty: true}
-    puts 'saved ' + File.basename(pxfilepath)
+  end
+  
+  def save_urlsindex(filepath)
+    
   end
 
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: liveblog-indexer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -31,7 +31,7 @@ cert_chain:
   mOnsi2V1CXpq2biJtSgD7mBx4cO9FXgbK3Xnsv45ygAPo6jj4Eb34udqz+0v88Ys
   KoTqNQOniHAW2w==
   -----END CERTIFICATE-----
-date: 2015-11-12 00:00:00.000000000 Z
+date: 2015-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: xws
@@ -54,25 +54,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.1.1
 - !ruby/object:Gem::Dependency
-  name: polyrex
+  name: rxfhelper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.11
+        version: 0.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.11
+        version: 0.2.3
 description: 
 email: james@r0bertson.co.uk
 executables: []