indexer101 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c12f1a8d9fc5dcde5c95bd80e0116236f4d70b9b0f835899b8339d765f5771b8
-  data.tar.gz: d3aa8b7f9146bbe35a28496bc01654ffe6d638289771fde4fb62dd6aec371682
+  metadata.gz: '059a4e8a4809e20f53e1d35100dad97936de2da80e2477650fb7efd0841c1cc2'
+  data.tar.gz: 215873766f61840776a8d3540556a034536dd71621593b928af8cb100861f973
 SHA512:
-  metadata.gz: ad823a882a9052f38de0acb790ee88b5dae6bd5bd19abbea22aa858a665f7d795d8cc373e4b4cc6ff0c34c0d552822484258c2203df0c25cb362f9d54ba56c78
-  data.tar.gz: d20146a83a706cac482fdfde143b99f4649e1b4f77e9bae9bd000120c4a82c533600777d762b75e3cefd9f36e011aba07cc13f0160b5fb599438cbca23296ce7
+  metadata.gz: 54afb23966c1b323821dca57198fb0b45a88e5f0de514fd48361ec1fbb59c1903152d2854c30fb73b03b78f96b582d174ae6211cd1174105c465bff92665be55
+  data.tar.gz: 864f8fa54173d1e769e70fbc688bd48c073aefb9aaf4b076d3e327f0c5c4c686303c1b1ab2a64295868aea8eced05bae276949edda6f2a2edb57427b2185ed58

data/lib/indexer101.rb

@@ -118,7 +118,9 @@ class Indexer101
     
   end
   
-  def scan_dxindex(*locations)
+  # scan levels: 0 = tags only; 1 = all words in title (including tags)
+  #  
+  def scan_dxindex(*locations, level: 0)
     
     t = Time.now
     threads = locations.flatten.map do |location|
@@ -132,18 +134,41 @@ class Indexer101
     puts ("dxindex documents loaded in " + ("%.2f" % t2).brown \
           + " seconds").info
     
-    a.each.with_index do |dx, i|
+
+    id = 1
+    
+    a.each do |dx|
+
+      id2 = id
       
       @indexer.uri_index.merge! Hash[dx.all.reverse.map.with_index \
-                      {|x,j| [(i+1)*10000 + (j+1), [x.title, x.url].join(' ')]}]
+        {|x,i| [id+i, [Time.parse(x.created), x.title, x.url]]}]
               
-      dx.all.reverse.each.with_index do |x,j|
-        x.title.scan(/#(\w+)/).flatten(1).each do |keyword|
-          @indexer.index[keyword.to_sym] ||= []
-          @indexer.index[keyword.to_sym] << (i+1)*10000 + (j+1)
+      dx.all.reverse.each do |x|
+                
+        case level
+        when 0 
+          
+          x.title.scan(/(\#\w+)/).flatten(1).each do |keyword|
+            @indexer.index[keyword.downcase.to_sym] ||= []
+            @indexer.index[keyword.downcase.to_sym] << id2
+          end
+          
+        when 1
+          
+          x.title.split(/[\s:"!\?\(\)£]+(?=[\w#_'-]+)/).each do |keyword|
+            @indexer.index[keyword.downcase.to_sym] ||= []
+            @indexer.index[keyword.downcase.to_sym] << id2
+          end
+
         end
+                
+        id2 += 1
+        
       end    
       
+      id = id2
+      
     end
     
   end
@@ -182,23 +207,32 @@ class Indexer101
     
     t = Time.now
     
-    results = keywords.flatten(1).flat_map do |x|
+    r = keywords.flatten(1).map do |x|
       
       a = []
       a += @indexer.index[x.to_sym].reverse if @indexer.index.has_key? x.to_sym
       
       if x.length > 3 then
-        a += @indexer.index.keys.reverse.grep(/^#{x}/).flat_map\
-            {|y| @indexer.index[y]}
-        a += @indexer.index.keys.reverse.grep(/#{x}/).flat_map\
-            {|y| @indexer.index[y]} 
+        a += @indexer.index.keys.grep(/^#{x}/i).flat_map\
+            {|y| @indexer.index[y].reverse}
+        a += @indexer.index.keys.grep(/#{x}/i).flat_map\
+            {|y| @indexer.index[y].reverse} 
       end
       
       puts ('a: ' + a.inspect).debug if @debug
-      a.uniq.map {|y| @indexer.uri_index[y].split(/\s+(?=https?[^\s]+$)/,2) }
       
+      a.uniq.map {|y| @indexer.uri_index[y]}
+      
+    end
+    
+    # group by number of results found, sort by count, then by date
+    a3 = r.flatten(1).group_by(&:last).to_a.sort do |x, x2|
+      -([x.last.length, x.last.first] <=> [x2.last.length, x2.last.first])
     end
     
+    # fetch the 1st record from each group item
+    results = a3.map {|x| x.last.first}
+    
     t2 = Time.now - t
     puts ("found %s results" % results.length).info
     puts ("search took " + ("%.3f" % t2).brown + " seconds").info

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexer101
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - James Robertson