indexer101 0.2.0 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c12f1a8d9fc5dcde5c95bd80e0116236f4d70b9b0f835899b8339d765f5771b8
-  data.tar.gz: d3aa8b7f9146bbe35a28496bc01654ffe6d638289771fde4fb62dd6aec371682
+  metadata.gz: 58debc39c10c1dff7e46220800a6f063ee8ae0c2b7e47c5dc1161b1c2555d64d
+  data.tar.gz: e7b468fb8b030e9a6710625f7e3f6061e30b7d0ae0f371d5977c01f027ab3b32
 SHA512:
-  metadata.gz: ad823a882a9052f38de0acb790ee88b5dae6bd5bd19abbea22aa858a665f7d795d8cc373e4b4cc6ff0c34c0d552822484258c2203df0c25cb362f9d54ba56c78
-  data.tar.gz: d20146a83a706cac482fdfde143b99f4649e1b4f77e9bae9bd000120c4a82c533600777d762b75e3cefd9f36e011aba07cc13f0160b5fb599438cbca23296ce7
+  metadata.gz: 480085f3742927accef9c583ef778bedaef2d08c113b87a4fb109fd910ef3b662324b3a35b7cce4c86f3a8d59043f49f03a69e3bfd5d0620a23ba354212e764d
+  data.tar.gz: 62f4cd3287838fb3e25d8c54ba74259b03f40d4bafbcb51198b6ad47c3d67adc0f6dcfcf72eddc771be60356253e05b644152b4b5d3b20bb0662226c4299f12e

data/lib/indexer101.rb

@@ -6,6 +6,7 @@ require 'c32'
 require 'thread'
 require 'thwait'
 require 'dynarex'
+require 'dxlite'
 
 
 class Indexer101
@@ -118,32 +119,81 @@ class Indexer101
     
   end
   
-  def scan_dxindex(*locations)
+  # scan levels: 0 = tags only; 1 = all words in title (including tags)
+  #  
+  def scan_dxindex(*locations, level: 0)
     
     t = Time.now
     threads = locations.flatten.map do |location|
-      Thread.new {Thread.current[:v] = Dynarex.new location}      
+      
+      Thread.new {
+
+        if location.is_a?(Dynarex) or location.is_a?(DxLite) then
+      
+          Thread.current[:v] = location
+      
+        elsif location.is_a? String
+      
+          case File.extname(location)
+          when '.xml'
+            Thread.current[:v] = Dynarex.new location, debug: @debug
+          when '.json'
+            Thread.current[:v] = DxLite.new location, debug: @debug
+          end
+      
+        end
+      }      
     end
     
     ThreadsWait.all_waits(*threads)
     
     a = threads.map {|x| x[:v]}
+    puts '_a: ' + a.inspect if @debug
     t2 = Time.now - t
     puts ("dxindex documents loaded in " + ("%.2f" % t2).brown \
           + " seconds").info
     
-    a.each.with_index do |dx, i|
+
+    id = 1
+    
+    a.each do |dx|
+
+      id2 = id
+      
+      if @debug then
+        puts 'dx: ' + dx.class.inspect
+        puts 'dx.all: ' + dx.all.inspect
+      end
       
       @indexer.uri_index.merge! Hash[dx.all.reverse.map.with_index \
-                      {|x,j| [(i+1)*10000 + (j+1), [x.title, x.url].join(' ')]}]
+        {|x,i| [id+i, [Time.parse(x.created), x.title, x.url]]}]
               
-      dx.all.reverse.each.with_index do |x,j|
-        x.title.scan(/#(\w+)/).flatten(1).each do |keyword|
-          @indexer.index[keyword.to_sym] ||= []
-          @indexer.index[keyword.to_sym] << (i+1)*10000 + (j+1)
+      dx.all.reverse.each do |x|
+                
+        case level
+        when 0 
+          
+          x.title.scan(/(\#\w+)/).flatten(1).each do |keyword|
+            @indexer.index[keyword.downcase.to_sym] ||= []
+            @indexer.index[keyword.downcase.to_sym] << id2
+          end
+          
+        when 1
+          
+          # \u{A3} = £ <- represented as Unicode to avoid ASCII to UTF-8 error
+          x.title.split(/[\s:"!\?\(\)\u{A3}]+(?=[\w#_'-]+)/).each do |keyword|
+            @indexer.index[keyword.downcase.to_sym] ||= []
+            @indexer.index[keyword.downcase.to_sym] << id2
+          end
+
         end
+                
+        id2 += 1
+        
       end    
       
+      id = id2
+      
     end
     
   end
@@ -178,27 +228,36 @@ class Indexer101
   
   # enter the exact keywords to search from the index
   #
-  def search(*keywords)
+  def search(*keywords, minchars: 3)
     
     t = Time.now
     
-    results = keywords.flatten(1).flat_map do |x|
+    r = keywords.flatten(1).map do |x|
       
       a = []
       a += @indexer.index[x.to_sym].reverse if @indexer.index.has_key? x.to_sym
       
-      if x.length > 3 then
-        a += @indexer.index.keys.reverse.grep(/^#{x}/).flat_map\
-            {|y| @indexer.index[y]}
-        a += @indexer.index.keys.reverse.grep(/#{x}/).flat_map\
-            {|y| @indexer.index[y]} 
+      if x.length >= minchars then
+        a += @indexer.index.keys.grep(/^#{x}/i).flat_map\
+            {|y| @indexer.index[y].reverse}
+        a += @indexer.index.keys.grep(/#{x}/i).flat_map\
+            {|y| @indexer.index[y].reverse} 
       end
       
       puts ('a: ' + a.inspect).debug if @debug
-      a.uniq.map {|y| @indexer.uri_index[y].split(/\s+(?=https?[^\s]+$)/,2) }
       
+      a.uniq.map {|y| @indexer.uri_index[y]}
+      
+    end
+    
+    # group by number of results found, sort by count, then by date
+    a3 = r.flatten(1).group_by(&:last).to_a.sort do |x, x2|
+      -([x.last.length, x.last.first] <=> [x2.last.length, x2.last.first])
     end
     
+    # fetch the 1st record from each group item
+    results = a3.map {|x| x.last.first}
+    
     t2 = Time.now - t
     puts ("found %s results" % results.length).info
     puts ("search took " + ("%.3f" % t2).brown + " seconds").info

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: indexer101
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.5
 platform: ruby
 authors:
 - James Robertson
@@ -11,31 +11,31 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
-  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMTkxMTExMjI1NjA5WhcN
-  MjAxMTEwMjI1NjA5WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
-  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDEF92R
-  JWEWjPhgU6nIiCKKXtWI9EE8DbqPtU+CnYuUN2BWN7c4dsbaYrU1tVMzxx22G+Of
-  apTIeIGFrq/oqub6nhx+UbnkMAqpmbJS8zTgnwEhFsGHGI2CD9+4OXh5rl36SqqP
-  IGxY7w45KDvuPWA/Htb2aC20cHclJebOjeaMNogpYDByVEjjxtZeiOmIJtJlQSf+
-  YpUviQVTkFJqbSK0WkKsPLQZu29P1zHETkdBvIGlGGDdo13haBW2Hdj9a2INeWcz
-  B+v6nAdnv9fTTvH0GX51XDa/EIisWNELaopHk1Hcx97pZdm92gaICQfdgUHje65s
-  oUDjyynUKE8dq+LAPEq5B1wj3e4BsIOFI5PSvPCMWI5jpbJWBXV6owex9qln22UA
-  lDUEP3COb9/+r6bGNiCBUvUwyy7l/RdmRXPSOYnP8jPcwD/qSVypJYObM+2q68qc
-  5Eg0WqCVdFgpIXzBJPVcxeDJiN6EKmFWr5vJkMwIGz9hhIcitQjjSguk4r8CAwEA
-  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUN0C2lCCv
-  /ATnmYSeZRQfUNpplfwwJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjEwMjIyMDAwNzQ2WhcN
+  MjIwMjIyMDAwNzQ2WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDqAwJO
+  ET6LAnOZB9q04zgLVFB0gJTcuLvfwjf7vpH9gCa5uqshSJnfi0owkeS2Hao0OwD+
+  vJrRRYbPfmXHAhEV8l9bSE6Ul1uTT9A+XS7g724sgOm5tCKFkLu+rcDy25MSjqpD
+  q+cPG4SN3ZUGK5eR9tp//dzdrjCV2wsOaoYKPajVY698p+sRf1zsHsSMxYnJPD/8
+  IkeNC+3VdsJFQ7wAoSk4hSpDuIi1xknA61/elDy5O07r1M25PJMntBE6QpJZblvw
+  v5u8U7+nK9P82KFfUwAjqkrhizt90M+0eK6dG44PnqafnxF84K2v7Qr1W6hKMIeL
+  DcqKPjAop+DO8WPCtfKFcFQKGRSe+H2rej3h34eGPH/GVoh8/h+ZuoDUfQRfoWLb
+  zUTM4uC1XwMlRjg8W4uYgV8SWZ5eii3tpmUz7moKQ4k9DSNNpGO+/bk/IcMVbRkb
+  xG/LzBAa6JQyAtVw9AMq8WHZKtJeNrdthoJSczsurthCHb0nY7VUQ/pp9JsCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUVn1uBRaL
+  Zh8+3WB077Lz84bokZowJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
   c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
-  BgkqhkiG9w0BAQsFAAOCAYEAXrKEIca0q3x//SBOsv17jkonBSlzwVLIBuXKXX4R
-  f0q3kJw7vWBPJfIVpAcGn6mbx2ziWw3XvG/SMuwIfzitz0pKTapvwbKZFPscqy82
-  KcDxGtJt1cjDHzl0Bm+mgN/MgY+PAj3TcT/osuCc8iTu4+Ib6UxkmOP/uy2svBLs
-  Su2XzGoqd5SKxPpj7IwMOBSVQWrqgWN2B+gdkN0CqjUPVMmiEKuMNjz9Idu36nG/
-  QOPsTlpqBxR+yFbPEP0DlN5X8BRRAsraQZ+LPi7W/bU1fjkvIJxXuUkQD9dMF99+
-  me+6s7PoCJ1yMmc5XfMFmv4WYswC+VqI1EeG0EneAsxRo8MmhWZcokqRE/KUk+ym
-  NlDFqcKPZDaMsdO5dkYvEeguet/iG3XS1u7WKAcZgfhPdiIbue7cAhz9eQpNybe7
-  08cN0E9zjqKINgH/PsZTot+ohuVRLwn6WmHHhb18oUrxt3a0u4/3TNcWOcMeR0F2
-  GeYL+mKGct5bfjn8IZnAJVKY
+  BgkqhkiG9w0BAQsFAAOCAYEAUzwCxgrA0YiJvXoi1ZHlhhz+ROzRn6XSQZZVd0Ym
+  gQVkUcvb/iiMnE0PZdivLiorRjhdR5tIPAYhmuN7Mr5IscQFdb1cndyC+qUzy6zP
+  HJGDGqqHDtiYpWlQ3/VAD4V+mAYj67CTj8gM2Y0OfjOIzKLf4jeLzcR0XjxCS9bH
+  g3cF/0FdJ5ydwo6r9QW/mE5yej6yuWBD9NFjWVbV/TAY3rWWDtw9g1WG31HjZVRB
+  lYndPAx0WIUBse5IRDGTiQ1JuMI5vBrxYJCb1Je506nR2rktACDRVSe/DTM4sxZn
+  oP3LBd1hPOAhNya8tD4FmUjQg4tvuWwIKh55XorZVEkzTWGgAJSnu7XTxtPcjxFA
+  U/3nmRr1BTYMN96T+3L81oqJTW5CxAAlsR97O7H8eZhwnNdG9HjgAk4PwiMLOhPb
+  Ely2/UitUG79uLcra+83gWVYzYiqBYC2d5HR4vCpTeecqYFXjWo9E3LMrvyB5Unk
+  zn10wjI+T1ysW7U6t+VJft8s
   -----END CERTIFICATE-----
-date: 2019-11-12 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dynarex
@@ -46,7 +46,7 @@ dependencies:
         version: '1.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.21
+        version: 1.8.25
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,7 +56,27 @@ dependencies:
         version: '1.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.21
+        version: 1.8.25
+- !ruby/object:Gem::Dependency
+  name: dxlite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
 description: 
 email: james@jamesrobertson.eu
 executables: []
@@ -83,7 +103,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.10
 signing_key: 
 specification_version: 4
 summary: Experimental gem to search a list of words 1 character at a time. Intended