indexer101 0.1.0 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3987fe6c70a6da37dd0cbcb50bf3368601ee1b6ddc6a90da039d22dfa14f28b
-  data.tar.gz: f7a79a0f09d97948a5576c253c8f2d2e5f58079fdc0830ee4fbeee28c208c3af
+  metadata.gz: 71b93d7302d137cd83ccd852e8506e349cbe3fc52723eed291c55875597606a3
+  data.tar.gz: 03d8a4f0800dac1ee5d7d7923742d7122898671ea72c9913b46c1d30e356dd35
 SHA512:
-  metadata.gz: fe696d43cc8c49962e3bb44a6f55497dd178b527d31cde3e395c2b2c1ef984add180fc503ac7f97fb2f4a716e37f785535972e149b3f6ec0cbeaa5698d64b1c9
-  data.tar.gz: e591fdfbec8eb9e579473075dabe554c04f163ffa23548746833d27053e5a55e7ba3514b2cd154acf43e8a4d2124f5cd753dd0bfcaa30697cd04d0e487bec318
+  metadata.gz: f3f8d541e4386651382c21f3936cfaa400f94cdff79729a95bfe7c10949ef628b4e3eddb1b873aecd9e1fc6ec97417ffcc79ab4d8639cec7e5b41ab93ecf2b2f
+  data.tar.gz: 52df9975cfbb4c4b7c1520d6d7e5efa3acc716ee70b11a15762534dbbe8acd70eee858428bb7982d718101c4395df33eb94fb53012e4c76e3839698531a0263c

data/lib/indexer101.rb

@@ -5,24 +5,34 @@
 require 'c32'
 require 'thread'
 require 'thwait'
+require 'dynarex'
+require 'dxlite'
 
 
 class Indexer101
   using ColouredText
     
   class Index
-    
+
     attr_reader :h
-    attr_accessor :index    
+    attr_accessor :uri_index, :index    
     
     def initialize()          
+
+      @uri_index = {} # contains each URI long with the title
+      @index = {} # contains eack keyword
+      @h = {} # nested keywords constructed from shared string keys
+
     end
     
     def build(a)
       
       threads = []
-      threads << Thread.new do
-        @index = Hash[a.map(&:to_sym).zip([''] * a.length)]
+      
+      if @index.empty? then
+        threads << Thread.new do
+          @index = Hash[a.map(&:to_sym).zip([''] * a.length)]
+        end
       end
       
       threads << Thread.new { @h = group a }
@@ -70,14 +80,14 @@ class Indexer101
     
   end
 
-  def build(a)
+  def build(a=@indexer.index.keys)
     
     t = Time.now
     @indexer.build(a)    
     t2 = Time.now - t
     
     puts "%d words indexed".info % a.length
-    puts "index built in %.2f seconds".info % t2 
+    puts ("index built in " + ("%.3f" % t2).brown + " seconds").info
     
     self
   end
@@ -97,7 +107,7 @@ class Indexer101
     t2 = Time.now - t
     
     puts "index contains %d words".info % @indexer.index.length
-    puts "index read in %.2f seconds".info % t2 
+    puts "index read in " + ("%.2f" % t2).brown + " seconds".info
     
   end
   
@@ -108,8 +118,86 @@ class Indexer101
     end 
     
   end
+  
+  # scan levels: 0 = tags only; 1 = all words in title (including tags)
+  #  
+  def scan_dxindex(*locations, level: 0)
+    
+    t = Time.now
+    threads = locations.flatten.map do |location|
+      
+      Thread.new {
+        
+        Thread.current[:v] = case File.extname(location)
+        when '.xml'
+          Dynarex.new location, debug: @debug
+        when '.json'
+          DxLite.new location, debug: @debug
+        end
+      }      
+    end
+    
+    ThreadsWait.all_waits(*threads)
+    
+    a = threads.map {|x| x[:v]}
+    puts '_a: ' + a.inspect if @debug
+    t2 = Time.now - t
+    puts ("dxindex documents loaded in " + ("%.2f" % t2).brown \
+          + " seconds").info
+    
+
+    id = 1
+    
+    a.each do |dx|
+
+      id2 = id
+      
+      if @debug then
+        puts 'dx: ' + dx.class.inspect
+        puts 'dx.all: ' + dx.all.inspect
+      end
+      
+      @indexer.uri_index.merge! Hash[dx.all.reverse.map.with_index \
+        {|x,i| [id+i, [Time.parse(x.created), x.title, x.url]]}]
+              
+      dx.all.reverse.each do |x|
+                
+        case level
+        when 0 
+          
+          x.title.scan(/(\#\w+)/).flatten(1).each do |keyword|
+            @indexer.index[keyword.downcase.to_sym] ||= []
+            @indexer.index[keyword.downcase.to_sym] << id2
+          end
+          
+        when 1
+          
+          # \u{A3} = £ <- represented as Unicode to avoid ASCII to UTF-8 error
+          x.title.split(/[\s:"!\?\(\)\u{A3}]+(?=[\w#_'-]+)/).each do |keyword|
+            @indexer.index[keyword.downcase.to_sym] ||= []
+            @indexer.index[keyword.downcase.to_sym] << id2
+          end
+
+        end
+                
+        id2 += 1
+        
+      end    
+      
+      id = id2
+      
+    end
+    
+  end
+  
+  def uri_index()
+    @indexer.uri_index
+  end
 
-  def search(s, limit: 10)
+  # enter a few starting characters and lookup will suggest a few keywords
+  # useful for an auto suggest feature
+  #
+  def lookup(s, limit: 10)
 
     t = Time.now
     a = scan_path s
@@ -124,7 +212,48 @@ class Indexer101
     
     results = scan_leaves(r).sort_by(&:length).take(limit)
     t2 = Time.now - t
-    puts "search took %.2f seconds" % t2 if @debug
+    puts ("lookup took " + ("%.3f" % t2).brown + " seconds").info
+    
+    return results
+    
+  end
+  
+  # enter the exact keywords to search from the index
+  #
+  def search(*keywords, minchars: 3)
+    
+    t = Time.now
+    
+    r = keywords.flatten(1).map do |x|
+      
+      a = []
+      a += @indexer.index[x.to_sym].reverse if @indexer.index.has_key? x.to_sym
+      
+      if x.length >= minchars then
+        a += @indexer.index.keys.grep(/^#{x}/i).flat_map\
+            {|y| @indexer.index[y].reverse}
+        a += @indexer.index.keys.grep(/#{x}/i).flat_map\
+            {|y| @indexer.index[y].reverse} 
+      end
+      
+      puts ('a: ' + a.inspect).debug if @debug
+      
+      a.uniq.map {|y| @indexer.uri_index[y]}
+      
+    end
+    
+    # group by number of results found, sort by count, then by date
+    a3 = r.flatten(1).group_by(&:last).to_a.sort do |x, x2|
+      -([x.last.length, x.last.first] <=> [x2.last.length, x2.last.first])
+    end
+    
+    # fetch the 1st record from each group item
+    results = a3.map {|x| x.last.first}
+    
+    t2 = Time.now - t
+    puts ("found %s results" % results.length).info
+    puts ("search took " + ("%.3f" % t2).brown + " seconds").info
+    puts
     
     return results
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexer101
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.4
 platform: ruby
 authors:
 - James Robertson
@@ -11,52 +11,72 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
-  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMTkxMTExMjI1NjA5WhcN
-  MjAxMTEwMjI1NjA5WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
-  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDEF92R
-  JWEWjPhgU6nIiCKKXtWI9EE8DbqPtU+CnYuUN2BWN7c4dsbaYrU1tVMzxx22G+Of
-  apTIeIGFrq/oqub6nhx+UbnkMAqpmbJS8zTgnwEhFsGHGI2CD9+4OXh5rl36SqqP
-  IGxY7w45KDvuPWA/Htb2aC20cHclJebOjeaMNogpYDByVEjjxtZeiOmIJtJlQSf+
-  YpUviQVTkFJqbSK0WkKsPLQZu29P1zHETkdBvIGlGGDdo13haBW2Hdj9a2INeWcz
-  B+v6nAdnv9fTTvH0GX51XDa/EIisWNELaopHk1Hcx97pZdm92gaICQfdgUHje65s
-  oUDjyynUKE8dq+LAPEq5B1wj3e4BsIOFI5PSvPCMWI5jpbJWBXV6owex9qln22UA
-  lDUEP3COb9/+r6bGNiCBUvUwyy7l/RdmRXPSOYnP8jPcwD/qSVypJYObM+2q68qc
-  5Eg0WqCVdFgpIXzBJPVcxeDJiN6EKmFWr5vJkMwIGz9hhIcitQjjSguk4r8CAwEA
-  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUN0C2lCCv
-  /ATnmYSeZRQfUNpplfwwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjEwMjIyMDAwNzQ2WhcN
+  MjIwMjIyMDAwNzQ2WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDqAwJO
+  ET6LAnOZB9q04zgLVFB0gJTcuLvfwjf7vpH9gCa5uqshSJnfi0owkeS2Hao0OwD+
+  vJrRRYbPfmXHAhEV8l9bSE6Ul1uTT9A+XS7g724sgOm5tCKFkLu+rcDy25MSjqpD
+  q+cPG4SN3ZUGK5eR9tp//dzdrjCV2wsOaoYKPajVY698p+sRf1zsHsSMxYnJPD/8
+  IkeNC+3VdsJFQ7wAoSk4hSpDuIi1xknA61/elDy5O07r1M25PJMntBE6QpJZblvw
+  v5u8U7+nK9P82KFfUwAjqkrhizt90M+0eK6dG44PnqafnxF84K2v7Qr1W6hKMIeL
+  DcqKPjAop+DO8WPCtfKFcFQKGRSe+H2rej3h34eGPH/GVoh8/h+ZuoDUfQRfoWLb
+  zUTM4uC1XwMlRjg8W4uYgV8SWZ5eii3tpmUz7moKQ4k9DSNNpGO+/bk/IcMVbRkb
+  xG/LzBAa6JQyAtVw9AMq8WHZKtJeNrdthoJSczsurthCHb0nY7VUQ/pp9JsCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUVn1uBRaL
+  Zh8+3WB077Lz84bokZowJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
   c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
-  BgkqhkiG9w0BAQsFAAOCAYEAXrKEIca0q3x//SBOsv17jkonBSlzwVLIBuXKXX4R
-  f0q3kJw7vWBPJfIVpAcGn6mbx2ziWw3XvG/SMuwIfzitz0pKTapvwbKZFPscqy82
-  KcDxGtJt1cjDHzl0Bm+mgN/MgY+PAj3TcT/osuCc8iTu4+Ib6UxkmOP/uy2svBLs
-  Su2XzGoqd5SKxPpj7IwMOBSVQWrqgWN2B+gdkN0CqjUPVMmiEKuMNjz9Idu36nG/
-  QOPsTlpqBxR+yFbPEP0DlN5X8BRRAsraQZ+LPi7W/bU1fjkvIJxXuUkQD9dMF99+
-  me+6s7PoCJ1yMmc5XfMFmv4WYswC+VqI1EeG0EneAsxRo8MmhWZcokqRE/KUk+ym
-  NlDFqcKPZDaMsdO5dkYvEeguet/iG3XS1u7WKAcZgfhPdiIbue7cAhz9eQpNybe7
-  08cN0E9zjqKINgH/PsZTot+ohuVRLwn6WmHHhb18oUrxt3a0u4/3TNcWOcMeR0F2
-  GeYL+mKGct5bfjn8IZnAJVKY
+  BgkqhkiG9w0BAQsFAAOCAYEAUzwCxgrA0YiJvXoi1ZHlhhz+ROzRn6XSQZZVd0Ym
+  gQVkUcvb/iiMnE0PZdivLiorRjhdR5tIPAYhmuN7Mr5IscQFdb1cndyC+qUzy6zP
+  HJGDGqqHDtiYpWlQ3/VAD4V+mAYj67CTj8gM2Y0OfjOIzKLf4jeLzcR0XjxCS9bH
+  g3cF/0FdJ5ydwo6r9QW/mE5yej6yuWBD9NFjWVbV/TAY3rWWDtw9g1WG31HjZVRB
+  lYndPAx0WIUBse5IRDGTiQ1JuMI5vBrxYJCb1Je506nR2rktACDRVSe/DTM4sxZn
+  oP3LBd1hPOAhNya8tD4FmUjQg4tvuWwIKh55XorZVEkzTWGgAJSnu7XTxtPcjxFA
+  U/3nmRr1BTYMN96T+3L81oqJTW5CxAAlsR97O7H8eZhwnNdG9HjgAk4PwiMLOhPb
+  Ely2/UitUG79uLcra+83gWVYzYiqBYC2d5HR4vCpTeecqYFXjWo9E3LMrvyB5Unk
+  zn10wjI+T1ysW7U6t+VJft8s
   -----END CERTIFICATE-----
-date: 2019-11-11 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: c32
+  name: dynarex
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.8.25
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.25
+- !ruby/object:Gem::Dependency
+  name: dxlite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.2.0
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
 description: 
 email: james@jamesrobertson.eu
 executables: []
@@ -83,7 +103,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.10
 signing_key: 
 specification_version: 4
 summary: Experimental gem to search a list of words 1 character at a time. Intended