indexer101 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3987fe6c70a6da37dd0cbcb50bf3368601ee1b6ddc6a90da039d22dfa14f28b
-  data.tar.gz: f7a79a0f09d97948a5576c253c8f2d2e5f58079fdc0830ee4fbeee28c208c3af
+  metadata.gz: c12f1a8d9fc5dcde5c95bd80e0116236f4d70b9b0f835899b8339d765f5771b8
+  data.tar.gz: d3aa8b7f9146bbe35a28496bc01654ffe6d638289771fde4fb62dd6aec371682
 SHA512:
-  metadata.gz: fe696d43cc8c49962e3bb44a6f55497dd178b527d31cde3e395c2b2c1ef984add180fc503ac7f97fb2f4a716e37f785535972e149b3f6ec0cbeaa5698d64b1c9
-  data.tar.gz: e591fdfbec8eb9e579473075dabe554c04f163ffa23548746833d27053e5a55e7ba3514b2cd154acf43e8a4d2124f5cd753dd0bfcaa30697cd04d0e487bec318
+  metadata.gz: ad823a882a9052f38de0acb790ee88b5dae6bd5bd19abbea22aa858a665f7d795d8cc373e4b4cc6ff0c34c0d552822484258c2203df0c25cb362f9d54ba56c78
+  data.tar.gz: d20146a83a706cac482fdfde143b99f4649e1b4f77e9bae9bd000120c4a82c533600777d762b75e3cefd9f36e011aba07cc13f0160b5fb599438cbca23296ce7

data/lib/indexer101.rb

@@ -5,24 +5,33 @@
 require 'c32'
 require 'thread'
 require 'thwait'
+require 'dynarex'
 
 
 class Indexer101
   using ColouredText
     
   class Index
-    
+
     attr_reader :h
-    attr_accessor :index    
+    attr_accessor :uri_index, :index    
     
     def initialize()          
+
+      @uri_index = {} # contains each URI long with the title
+      @index = {} # contains eack keyword
+      @h = {} # nested keywords constructed from shared string keys
+
     end
     
     def build(a)
       
       threads = []
-      threads << Thread.new do
-        @index = Hash[a.map(&:to_sym).zip([''] * a.length)]
+      
+      if @index.empty? then
+        threads << Thread.new do
+          @index = Hash[a.map(&:to_sym).zip([''] * a.length)]
+        end
       end
       
       threads << Thread.new { @h = group a }
@@ -70,14 +79,14 @@ class Indexer101
     
   end
 
-  def build(a)
+  def build(a=@indexer.index.keys)
     
     t = Time.now
     @indexer.build(a)    
     t2 = Time.now - t
     
     puts "%d words indexed".info % a.length
-    puts "index built in %.2f seconds".info % t2 
+    puts ("index built in " + ("%.3f" % t2).brown + " seconds").info
     
     self
   end
@@ -97,7 +106,7 @@ class Indexer101
     t2 = Time.now - t
     
     puts "index contains %d words".info % @indexer.index.length
-    puts "index read in %.2f seconds".info % t2 
+    puts "index read in " + ("%.2f" % t2).brown + " seconds".info
     
   end
   
@@ -108,8 +117,45 @@ class Indexer101
     end 
     
   end
+  
+  def scan_dxindex(*locations)
+    
+    t = Time.now
+    threads = locations.flatten.map do |location|
+      Thread.new {Thread.current[:v] = Dynarex.new location}      
+    end
+    
+    ThreadsWait.all_waits(*threads)
+    
+    a = threads.map {|x| x[:v]}
+    t2 = Time.now - t
+    puts ("dxindex documents loaded in " + ("%.2f" % t2).brown \
+          + " seconds").info
+    
+    a.each.with_index do |dx, i|
+      
+      @indexer.uri_index.merge! Hash[dx.all.reverse.map.with_index \
+                      {|x,j| [(i+1)*10000 + (j+1), [x.title, x.url].join(' ')]}]
+              
+      dx.all.reverse.each.with_index do |x,j|
+        x.title.scan(/#(\w+)/).flatten(1).each do |keyword|
+          @indexer.index[keyword.to_sym] ||= []
+          @indexer.index[keyword.to_sym] << (i+1)*10000 + (j+1)
+        end
+      end    
+      
+    end
+    
+  end
+  
+  def uri_index()
+    @indexer.uri_index
+  end
 
-  def search(s, limit: 10)
+  # enter a few starting characters and lookup will suggest a few keywords
+  # useful for an auto suggest feature
+  #
+  def lookup(s, limit: 10)
 
     t = Time.now
     a = scan_path s
@@ -124,7 +170,39 @@ class Indexer101
     
     results = scan_leaves(r).sort_by(&:length).take(limit)
     t2 = Time.now - t
-    puts "search took %.2f seconds" % t2 if @debug
+    puts ("lookup took " + ("%.3f" % t2).brown + " seconds").info
+    
+    return results
+    
+  end
+  
+  # enter the exact keywords to search from the index
+  #
+  def search(*keywords)
+    
+    t = Time.now
+    
+    results = keywords.flatten(1).flat_map do |x|
+      
+      a = []
+      a += @indexer.index[x.to_sym].reverse if @indexer.index.has_key? x.to_sym
+      
+      if x.length > 3 then
+        a += @indexer.index.keys.reverse.grep(/^#{x}/).flat_map\
+            {|y| @indexer.index[y]}
+        a += @indexer.index.keys.reverse.grep(/#{x}/).flat_map\
+            {|y| @indexer.index[y]} 
+      end
+      
+      puts ('a: ' + a.inspect).debug if @debug
+      a.uniq.map {|y| @indexer.uri_index[y].split(/\s+(?=https?[^\s]+$)/,2) }
+      
+    end
+    
+    t2 = Time.now - t
+    puts ("found %s results" % results.length).info
+    puts ("search took " + ("%.3f" % t2).brown + " seconds").info
+    puts
     
     return results
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexer101
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - James Robertson
@@ -35,28 +35,28 @@ cert_chain:
   08cN0E9zjqKINgH/PsZTot+ohuVRLwn6WmHHhb18oUrxt3a0u4/3TNcWOcMeR0F2
   GeYL+mKGct5bfjn8IZnAJVKY
   -----END CERTIFICATE-----
-date: 2019-11-11 00:00:00.000000000 Z
+date: 2019-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: c32
+  name: dynarex
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.21
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.0
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.21
 description: 
 email: james@jamesrobertson.eu
 executables: []