StrIdx 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52d40e64a5ace0231828bdbbe6fd94475ab5986d0c1fb7e35e0ce18463a97ae0
-  data.tar.gz: e1cdcc2ed9f377b2acb049a9fb6de22f24acdbd6e3552748b1307342c10b6cf7
+  metadata.gz: f7655b6bd71bca58c86ad607fd197933fc19b97b3ae1c76e322ec0432025dad7
+  data.tar.gz: 2421892aa6fe750213d08e2254019d87ce7abb10496cdf1635a61815b8b8b0d7
 SHA512:
-  metadata.gz: f3c27923a568fe5916c17e91766066362a965abf9568b21a4daa269cd16a8a4778248ae935b26502aa482aad4807908d401989e7ebfb88d1fbdb011b0c240b60
-  data.tar.gz: f94dda8d71931c18ae3dc6b58204edda7ffd649bc7452a74fdba4929d6092183e99bf99d7b3632be5bafccfd0be7a877f5513c8c3e72e814dcca08bd79a9b217
+  metadata.gz: 0a0ed3f51b95b72a553cf97e1a852f63e8b6d1cbbba56fdab55ed5037ef4d658b8649f2cf92d35b5eea4e6657a18a3a1460a3a57069cda0889a1987f5d1611ee
+  data.tar.gz: f0d4753ee43cb205fa86468dad92a66644e12cf889d8018283cb421e4a5d670386b8666af64b89d4d475a59f10701de2223c31d03ac5d366f2ee255c77190cf8

data/CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 3.14)
+
+project(my_project)
+# https://github.com/google/googletest/issues/4000
+include(FetchContent)
+FetchContent_Declare(
+  googletest
+  URL https://github.com/google/googletest/archive/58d77fa8070e8cec2dc1ed015d66b454c8d78850.zip # release-1.12.1
+)
+
+set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(googletest)
+
+enable_testing()
+
+add_executable(
+  stridx_test
+  unittest.cpp
+)
+target_link_libraries(
+  stridx_test
+  GTest::gtest_main
+)
+
+include(GoogleTest)
+gtest_discover_tests(stridx_test)
+

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+

data/README.md

@@ -37,9 +37,46 @@ and candidate is "./drivers/char/hw_random/nomadik-rng.c", then scores are calcu
     score = score1/(11*11)*0.97 + score1/(11*38)*0.03 = 0.342944
 ```
 
-# Ruby interface
+# Interfaces
+
+## Commandline
+Install instructions (for Ubuntu Linux):
+```
+apt update
+apt install ruby ruby-dev build-essential
+gem install StrIdx
+```
+
+Start indexing server (on background):
+```
+stridx.rb start -- ~/Documents/ ~/Pictures/
+```
+
+Add bash keybindings (Ctrl-t):
+```
+eval "$(stridx.rb bash)"
+```
+
+Search by pressing <kbd>ctrl</kbd>+<kbd>t</kbd>.  Keys: <kbd>up</kbd>, <kbd>down</kbd>, select with <kbd>enter</kbd>
+
+![screencast](https://github.com/SamiSieranoja/stridx/assets/46612258/b2fd4fa2-37ad-4423-bd5f-d54b24ff6df5)
+
+
+Stop server:
+```
+stridx.rb stop
+```
+
+Start indexing server (on foreground, to debug):
+```
+stridx.rb run -- ~/Documents/ ~/Pictures/
+```
+
+
+## Ruby
 Install:
 ```
+apt install ruby ruby-dev build-essential
 gem install StrIdx
 ```
 
@@ -114,7 +151,7 @@ Search time: 0.0488 seconds
 ```
 
 
-# C++ API
+## C++
 See demo.cpp
 ```cpp
 #include "stridx.hpp"

data/exe/stridx.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + "/.."
+
+if ARGV[0] == "tty"
+  require "stridx-tty.rb"
+  StrIdxTTY.run
+elsif ARGV[0] == "bash"
+  puts %q/
+  bind -m emacs-standard '"\er": redraw-current-line';
+  bind -m emacs-standard '"\C-t": " \C-b\C-k \C-u`stridx.rb tty`\e\C-e\er\C-a\C-y\C-h\C-e\e \C-y\ey\C-x\C-x\C-f"'
+/
+else
+  require "daemons"
+  Daemons.run(File.dirname(__FILE__) + "/../runserver.rb")
+end

data/gem_install

@@ -0,0 +1,4 @@
+gem uninstall --force -x StrIdx
+gem build stridx.gemspec
+gem install $(ls -1tr StrIdx*gem | tail -n 1)
+

data/py_example.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+from stridx import StringIndex
+e=StringIndex()
+e.set_value(3)
+e.add("./rust/alloc/vec/spec_extend.rs",0)
+e.add("./virt/kvm/dirty_ring.c",1)
+e.add("./Documentation/staging/static-keys.rst",2)
+e.add("./Documentation/staging/lzo.rst",3)
+
+
+
+results = e.find("rstalloc")
+for x in results:
+	print(x)
+
+# print(e.get_value())
+

data/py_interf.cpp

@@ -0,0 +1,182 @@
+
+#include <stdio.h>
+#include <limits.h>
+#include <cstring>
+#include <pthread.h>
+
+#include <stdio.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <cfloat>
+#include <cmath>
+#include <bits/stdc++.h>
+
+using std::ios;
+using std::sort;
+using std::string;
+using std::vector;
+
+#include <Python.h>
+#include <cstring>
+
+#include "stridx.hpp"
+
+extern "C" {
+
+// Define a structure for the custom object
+typedef struct {
+  PyObject_HEAD int value;
+  StrIdx::StringIndex *idx;
+} StrIdxObject;
+
+// Method to allocate memory for the object
+static PyObject *StrIdxObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
+  StrIdxObject *self;
+
+  self = (StrIdxObject *)type->tp_alloc(type, 0);
+  if (self != NULL) {
+    self->value = 0;
+    self->idx = new StrIdx::StringIndex();
+  }
+
+  return (PyObject *)self;
+}
+
+// Method to deallocate memory for the object
+static void StrIdxObject_dealloc(StrIdxObject *self) { Py_TYPE(self)->tp_free((PyObject *)self); }
+
+// Method to set the value of the object
+static PyObject *StrIdxObject_set_value(StrIdxObject *self, PyObject *args) {
+  int value;
+
+  if (!PyArg_ParseTuple(args, "i", &value)) {
+    return NULL;
+  }
+
+  self->value = value;
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+static PyObject *StrIdxObject_add(StrIdxObject *self, PyObject *args) {
+  char *value;
+  int file_id;
+  std::string str;
+  if (!PyArg_ParseTuple(args, "si", &value, &file_id)) {
+    return NULL;
+  }
+  str = value;
+
+  printf("char[]*: %s %i\n", value, file_id);
+  self->idx->addStrToIndex(str, file_id);
+  // self->idx->addStrToIndexThreaded(str, file_id);
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+static PyObject *StrIdxObject_find(StrIdxObject *self, PyObject *args) {
+  char *value;
+  std::string str;
+  if (!PyArg_ParseTuple(args, "s", &value)) {
+    return NULL;
+  }
+  str = value;
+
+  printf("char*: %s\n", value);
+  const std::vector<std::pair<float, int>> &results = self->idx->findSimilar(str, 2);
+
+  int limit = 15;
+  int i = 0;
+
+  printf("res=%d\n", results.size());
+  if (results.size() < limit) {
+    limit = results.size();
+  }
+  PyObject *pyarr = PyList_New(limit);
+
+  for (const auto &[score,fileId] : results) {
+    PyObject *arr2 = PyList_New(2);
+    // PyList_SetItem(arr2, 0, Py_BuildValue("i", res.second));
+    // PyList_SetItem(arr2, 1, Py_BuildValue("d", res.first));
+    PyList_SetItem(arr2, 0, Py_BuildValue("i", fileId));
+    PyList_SetItem(arr2, 1, Py_BuildValue("d", score));
+    PyList_SetItem(pyarr, i, arr2);
+    i++;
+    if (i >= limit) {
+      break;
+    }
+  }
+
+  // Py_INCREF(Py_None);
+  return pyarr;
+}
+
+// Method to get the value of the object
+static PyObject *StrIdxObject_get_value(StrIdxObject *self) {
+  return PyLong_FromLong(self->value);
+}
+
+// Define methods of the class
+static PyMethodDef StrIdxObject_methods[] = {
+    {"set_value", (PyCFunction)StrIdxObject_set_value, METH_VARARGS,
+     "Set the value of the object"},
+    {"add", (PyCFunction)StrIdxObject_add, METH_VARARGS, "Set the value of the object"},
+    {"find", (PyCFunction)StrIdxObject_find, METH_VARARGS, "Find similar strings"},
+    {"get_value", (PyCFunction)StrIdxObject_get_value, METH_NOARGS, "Get the value of the object"},
+    {NULL} /* Sentinel */
+};
+
+// Define the type object for the class
+static PyTypeObject StrIdxType = {
+    PyVarObject_HEAD_INIT(NULL, 0).tp_name = "stridx.StrIdx",
+    .tp_basicsize = sizeof(StrIdxObject),
+    .tp_dealloc = (destructor)StrIdxObject_dealloc,
+    .tp_doc = PyDoc_STR("Fuzzy string index"),
+    .tp_methods = StrIdxObject_methods,
+    .tp_new = StrIdxObject_new,
+    // .tp_repr = (reprfunc)myobj_repr,
+};
+
+// PyVarObject_HEAD_INIT(NULL, 0)
+// .tp_name = "stridx.StrIdx",
+// .tp_doc = "StrIdx class",
+// .tp_basicsize = sizeof(StrIdxObject),
+// .tp_itemsize = 0,
+// .tp_flags = Py_TPFLAGS_DEFAULT,
+// .tp_new = StrIdxObject_new,
+// .tp_dealloc = (destructor)StrIdxObject_dealloc,
+// .tp_methods = StrIdxObject_methods,
+// };
+
+
+// Define python accessible methods
+static PyMethodDef StrIdxMethods[] = {
+    {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT, "stridx", NULL, -1, StrIdxMethods, NULL, NULL, NULL, NULL};
+
+PyMODINIT_FUNC PyInit_stridx(void) {
+  PyObject *m;
+  m = PyModule_Create(&moduledef);
+
+  // Initialize the type object
+  if (PyType_Ready(&StrIdxType) < 0) {
+    return NULL;
+  }
+
+  Py_INCREF(&StrIdxType);
+  if (PyModule_AddObject(m, "StringIndex", (PyObject *)&StrIdxType) < 0) {
+    Py_DECREF(&StrIdxType);
+    Py_DECREF(m);
+    return NULL;
+  }
+
+  if (!m) {
+    return NULL;
+  }
+  return m;
+}
+} // END extern "C"

data/runserver.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$:.unshift File.dirname(__FILE__)
+
+require "server.rb"
+# StrIdx::Server.start ARGV, daemonize: true
+StrIdx::Server.start ARGV
+

data/server.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+require "socket"
+require "stridx"
+
+module StrIdx
+  class Server
+    def recursively_find_files(directories)
+      filelist = []
+
+      for d in directories
+        filelist = filelist + Dir.glob("#{d}/**/*").select { |e|
+          File.file?(e)
+          # File.file?(e) or File.directory?(e)
+        }
+      end
+      return filelist
+    end
+
+    def self.start(dir_list, daemonize: false)
+      Server.new(dir_list, daemonize: daemonize)
+    end
+
+    def self.stop
+      sock_dir = File.expand_path("~/.stridx")
+      sockfn = "#{sock_dir}/sock"
+      client = UNIXSocket.new(sockfn)
+      client.puts "stop"
+      response = client.recv(200 * 200)
+      client.close
+    end
+
+    def initialize(dir_list, daemonize: false)
+      idx = StrIdx::StringIndex.new
+      idx.setDirSeparator("/")
+
+      t = Time.new
+
+      dirs = dir_list.select { |x| File.directory?(x) }
+      puts "Scanning files in directories:#{dirs.join(",")}"
+      flist = recursively_find_files(dirs)
+
+      i = 0
+      for x in flist
+        idx.add(x, i)
+        i += 1
+      end
+
+      idx.waitUntilDone()
+      idx_time = Time.new
+      puts "\nIndexing time (#{flist.size} files): #{(idx_time - t).round(4)} seconds"
+
+      sock_dir = File.expand_path("~/.stridx")
+      Dir.mkdir(sock_dir) if !Dir.exist?(sock_dir)
+      sockfn = "#{sock_dir}/sock"
+      File.unlink(sockfn) if File.exist?(sockfn)
+
+      puts "Indexing done, starting server"
+      if (daemonize)
+        require "daemons"
+        Daemons.daemonize
+        # exit if fork() # Daemonize
+      end
+
+      # exit if fork() # Daemonize
+      # $PROGRAM_NAME = "stridx-daemon"
+
+      t = Thread.new {
+        serv = UNIXServer.new(sockfn)
+
+        loop do
+          # Accept a new client connection
+          client = serv.accept
+
+          # puts "Client connected!"
+
+          # Read data from the client
+          data = client.recv(1024)
+
+          if data.match(/^stop$/)
+            puts "Got stop signal. Shutting down server."
+            client.close
+            break
+          end
+
+          # puts "Received from client: #{data}"
+          if data.match(/^find:(.*)/)
+            query = Regexp.last_match(1)
+            res = idx.find(query)
+            response = res.collect { |x| flist[x[0]] }.join("\n")
+
+            # Send a response back to the client
+            client.puts response
+          end
+          # Close the client connection
+          client.close
+        end
+      }
+
+      t.join
+    end
+  end
+end

data/setup.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import numpy
+
+import setuptools
+from setuptools import setup, Extension
+
+__version__ = "0.1"
+
+cargs = ['-fpermissive']
+
+
+with open('README.md', 'r', encoding='utf-8') as f:
+    long_description = f.read()
+
+module1 = Extension('stridx', sources=['py_interf.cpp'], include_dirs=['.'], extra_compile_args=cargs,
+language="c++",
+)
+                        
+ext_modules = [module1]
+                      
+setup(
+    name='stridx',
+    version='1.0',
+    setup_requires=['wheel'],
+    python_requires='>=3',
+    provides=['stridx'],
+    description='Fast fuzzy string similarity search and indexing (for filenames) ',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    ext_modules=[module1]
+)
+

data/stridx-tty.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+
+require "tty-prompt"
+require "tty-cursor"
+require "tty-reader"
+require "pastel"
+
+require "socket"
+
+class StrIdxTTY
+  def self.run
+    stty = StrIdxTTY.new
+    selected = stty.search
+    STDOUT.write selected
+  end
+
+  def initialize()
+    @lines = []
+    @selected = ""
+    @idx = 0
+
+    @reader = TTY::Reader.new(output: STDERR)
+    @pastel = Pastel.new()
+    @cursor = TTY::Cursor
+
+    sock_dir = File.expand_path("~/.stridx")
+    sockfn = "#{sock_dir}/sock"
+
+    error = true
+    while error
+      begin
+        # Create a new UNIXSocket
+        client = UNIXSocket.new(sockfn)
+      rescue Errno::ECONNREFUSED => e
+        out "Waiting for server to start\n"
+        sleep 2
+        error = true
+      else
+        error = false
+        client.close
+        #... executes when no error
+      end
+    end
+  end
+
+  def out(x)
+    STDERR.write x
+  end
+
+  def search
+    out "\n" * 20
+    out @cursor.clear_screen
+    out "\n" * 20
+    @cursor.move_to(0, 0)
+    @reader.on(:keypress) { |event|
+      handle_event(event)
+    }
+    @reader.read_line(">> ")
+
+    out @cursor.clear_screen
+    return @selected.strip
+  end
+
+  def get_res_from_server(query)
+    # Define the socket file path
+    sock_dir = File.expand_path("~/.stridx")
+    sockfn = "#{sock_dir}/sock"
+
+    # Create a new UNIXSocket
+    client = UNIXSocket.new(sockfn)
+
+    # Send data to the server
+    client.puts "find:#{query}"
+
+    # Read response from the server
+    response = client.recv(200 * 200)
+
+    # Close the client connection
+    client.close
+    return response.lines
+  end
+
+  def draw_list()
+    @selected = @list[@idx]
+    i = 0
+    for x in @list
+      out @cursor.up(1)
+      out @cursor.clear_line
+      if i == @idx
+        out @pastel.lookup(:bold)
+      end
+      out x.strip
+      out @pastel.lookup(:reset)
+      i += 1
+    end
+  end
+
+  def update_search(event)
+    query = event.line[3..-1]
+    if query.size > 2
+      @list = get_res_from_server(query)
+      draw_list
+    end
+  end
+
+  def handle_event(event)
+    out @cursor.save
+    if event.key.name == :alpha
+      update_search(event)
+    elsif event.key.name == :up
+      @idx += 1 if @idx < @list.size - 1
+      draw_list
+    elsif event.key.name == :down
+      @idx -= 1 if @idx > 0
+      draw_list
+    elsif event.key.name == :backspace
+      update_search(event)
+    end
+
+    out @cursor.restore
+  end
+end

data/stridx.gemspec

@@ -0,0 +1,37 @@
+Gem::Specification.new do |spec|
+  spec.name = "StrIdx"
+  spec.version = "0.1.4"
+  spec.authors = ["Sami Sieranoja"]
+  spec.email = ["sami.sieranoja@gmail.com"]
+
+  spec.summary = %q{StrIdx}
+  spec.description = %q{ Fast fuzzy string similarity search and indexing (for filenames)}
+  spec.homepage = "https://github.com/SamiSieranoja/stridx"
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(refcode|spec|features)/})
+  end
+  # spec.files << "thread_pool.hpp"
+  # spec.files << "exe/stridx.rb"
+  # spec.files << "server.rb"
+  # spec.files << "stridx-tty.rb"
+
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib", "ext"]
+
+  spec.add_development_dependency "bundler", "~> 2.4.21"
+  spec.add_development_dependency "rake", "~> 13.1.0"
+
+  spec.add_runtime_dependency "tty-cursor", "~> 0.7.1"
+  spec.add_runtime_dependency "tty-prompt", "~> 0.23.1"
+  spec.add_runtime_dependency "tty-reader", "~> 0.9.0"
+  spec.add_runtime_dependency "tty-screen", "~> 0.8.2"
+  spec.add_runtime_dependency "pastel", "~> 0.8.0"
+  spec.add_runtime_dependency "daemons", "~> 1.4.1"
+
+  spec.extensions = ["rubyext/extconf.rb"]
+  spec.licenses = ["LGPL-2.0+"]
+end

data/stridx.hpp

@@ -33,16 +33,16 @@ public:
   Output(int verb) : verboseLevel(verb) {}
   Output() : Output(3) {}
   ~Output() = default;
-  void print() {}
+  static void print() {}
 
   // When calling as print("xxx ",3, " yyy") outputs "xxx 3 yyy"
-  template <typename T, typename... Types> void print(T var1, Types... var2) {
+  template <typename T, typename... Types> static void print(T var1, Types... var2) {
     std::cout << var1;
     print(var2...);
   }
 
   // When calling as printl("xxx ",3, " yyy") outputs "xxx 3 yyy\n"
-  template <typename... Types> void printl(Types... var2) {
+  template <typename... Types> static void printl(Types... var2) {
     print(var2...);
     print("\n");
   }
@@ -79,7 +79,7 @@ std::vector<std::string> splitString(const std::string &input, const char &separ
 }
 
 // Convert int64_t to binary string
-[[nodiscard]] std::string int64ToBinaryString(int64_t num) {
+[[nodiscard]] std::string int64ToBinaryString(const int64_t &num) {
   std::string result;
   for (int i = 63; i >= 0; --i) {
     result += ((num >> i) & 1) ? '1' : '0';
@@ -88,7 +88,7 @@ std::vector<std::string> splitString(const std::string &input, const char &separ
 }
 
 // Debug. Convert a (8 char) string represented as int64_t to std::string
-[[nodiscard]] std::string int64ToStr(int64_t key) {
+[[nodiscard]] std::string int64ToStr(const int64_t &key) {
   int nchars = 8;
   std::string str;
   int multip = nchars * 8;
@@ -108,7 +108,7 @@ void printVector(const std::vector<int> &vec) {
 }
 
 // Debug
-[[nodiscard]] std::string charToBinaryString(char chr) {
+[[nodiscard]] std::string charToBinaryString(const char &chr) {
   std::string result;
   for (int i = 7; i >= 0; --i) {
     result += ((chr >> i) & 1) ? '1' : '0';
@@ -122,8 +122,7 @@ enum class segmentType { Dir, File };
 // A segment of a file path
 // e.g. if path is /foo/bar/baz.txt
 // segments are [{root}, foo, bar, baz.txt]
-class PathSegment {
-public:
+struct PathSegment {
   std::string str;
   int fileId; // (if FILE)
   Candidate *cand;
@@ -135,7 +134,7 @@ public:
   PathSegment(std::string _str) : str(_str), parent(nullptr) {}
   PathSegment(std::string _str, int _fileId)
       : str(_str), fileId(_fileId), cand(nullptr), parent(nullptr) {}
-  [[nodiscard]] int size() {
+  [[nodiscard]] int size() const {
     int sz = str.size();
     PathSegment *cur = parent;
     // Sum up length of parent segments (+1 for divisors)
@@ -148,8 +147,7 @@ public:
 };
 
 // Candidate for result in string (filename) search
-class Candidate {
-public:
+struct Candidate {
   std::vector<float> v_charscore;
   PathSegment *seg;
   int fileId;
@@ -162,25 +160,17 @@ public:
   int candLen; // Length of candidate
 
   Candidate(){};
-  Candidate(int _fileId, std::string _str, int _len) : fileId(_fileId), str(_str), len(_len) {
-    // Initialize v_charscores with zeros
-    v_charscore.resize(len, 0);
-    candLen = str.size();
-    seg = nullptr;
-  }
-
   Candidate(PathSegment *_seg, int _len) : seg(_seg), len(_len) {
     // Initialize v_charscores with zeros
     v_charscore.resize(len, 0);
     candLen = seg->size();
   }
 
-  [[nodiscard]] float getScore() {
+  [[nodiscard]] float getScore() const {
     int i = 0;
     float score = 0.0;
-    candLen = seg->size();
 
-    for (float &charscore : v_charscore) {
+    for (const float &charscore : v_charscore) {
       score += charscore;
       i++;
     }
@@ -193,7 +183,7 @@ public:
     return score;
   }
 
-  [[nodiscard]] float operator[](int idx) { return v_charscore[idx]; }
+  [[nodiscard]] float operator[](int idx) const { return v_charscore[idx]; }
 };
 
 // This seems to give 10x speed improvement over std::unordered_map
@@ -210,19 +200,17 @@ private:
   int numStrings = 0;
 
   std::vector<SegMap *> dirmaps;
+  std::array<std::mutex, 9> mts_d; // for dirmaps
   std::vector<SegMap *> filemaps;
+  std::array<std::mutex, 9> mts_f; // for filemaps
 
   std::vector<PathSegment *> segsToClean;
 
-  std::unordered_map<int, std::string> strlist;
   std::unordered_map<int, PathSegment *> seglist;
   PathSegment *root;
   int dirId = 0;
   float dirWeight = 0.7; // Give only 70% of score if match is for a directory
 
-  std::array<std::mutex, 9> mts_f;
-  std::array<std::mutex, 9> mts_d;
-
   std::unique_ptr<ThreadPool> pool;
   Output out{1}; // verbose level = 1
 
@@ -279,9 +267,11 @@ public:
   void addStrToIndexThreaded(std::string filePath, int fileId) {
     pool->enqueue([=] { addStrToIndex(filePath, fileId, dirSeparator); });
   }
-  void waitUntilReady() { pool->waitUntilDone(); }
+  void waitUntilReady() const { pool->waitUntilDone(); }
 
-  void waitUntilDone() { pool->waitUntilDone(); }
+  void waitUntilDone() const { pool->waitUntilDone(); }
+
+  int size() const { return seglist.size(); }
 
   /**
    * Add a string to the index to be searched for afterwards
@@ -291,8 +281,14 @@ public:
    * @param separator Can be used to split filePath to components (e.g. 'home','user'...). Usually
    * one of {'\\', '/', '\0' (no separation)}.
    */
+
   void addStrToIndex(std::string filePath, int fileId, const char &separator) {
-    out.printv(3, "Add file:", filePath, ",", fileId, ",", separator);
+    out.printv(3, "Add file:", filePath, ",", fileId, ",", separator, ",",dirSeparator);
+
+    // If a string with this index has beeen added already
+    if (seglist.find(fileId) != seglist.end()) {
+      return;
+    }
 
     std::vector<std::string> segs;
     numStrings += 1;
@@ -345,6 +341,17 @@ public:
     }
   }
 
+  std::string getString(int id) {
+    std::string s = "";
+    PathSegment *seg = seglist[id];
+    s += seg->str;
+    while (seg->parent->parent != nullptr) {
+      seg = seg->parent;
+      s = seg->str + dirSeparator + s;
+    }
+    return s;
+  }
+
   /**
   The search will find filepaths similar to the input string
 
@@ -423,7 +430,7 @@ public:
   }
 
   // Return int64_t representation of the first nchars in str, starting from index i
-  [[nodiscard]] int64_t getKeyAtIdx(std::string str, int i, int nchars) {
+  [[nodiscard]] int64_t getKeyAtIdx(const std::string &str, int i, int nchars) const {
     int64_t key = 0;
     for (int i_char = 0; i_char < nchars; i_char++) {
       key = key | static_cast<int64_t>(str[i + i_char]);
@@ -519,7 +526,7 @@ private:
   // Find pathsegments from <map> that include the substring of <str> which starts at index <i> and
   // is of length <nchars>.
   [[nodiscard]] std::vector<PathSegment *> findSimilarForNgram(std::string str, int i, int nchars,
-                                                               SegMap &map) {
+                                                               SegMap &map) const {
 
     assert(i + nchars <= static_cast<int>(str.size()));
     std::vector<PathSegment *> res;

data/test.rb

@@ -1,8 +1,13 @@
 #!/usr/bin/env ruby
 
+$:.unshift File.dirname(__FILE__)
+
 require "stridx"
 idx = StrIdx::StringIndex.new
 
+# "/" for unix-style file paths
+idx.setDirSeparator("/") #(comment out if not file paths)
+
 t = Time.new
 fn = File.expand_path("flist.txt")
 lines = IO.read(fn).lines.collect { |x| x.strip }
@@ -14,12 +19,12 @@ end
 
 idx_time = Time.new 
 # Time to start the threadpool to process indexing
-puts "\nIndexing launch time (#{lines.size} files}): #{(idx_time - t).round(4)} seconds"
+puts "\nIndexing launch time (#{lines.size} files): #{(idx_time - t).round(4)} seconds"
 
 idx.waitUntilDone() # Not necessary, will be called by idx.find
 idx_time = Time.new 
 # Time when all threads have completed
-puts "\nIndexing completed time (#{lines.size} files}): #{(idx_time - t).round(4)} seconds"
+puts "\nIndexing completed time (#{lines.size} files): #{(idx_time - t).round(4)} seconds"
 
 query = "rngnomadriv"
 res = idx.find(query)

data/unit_tests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cmake -S . -B build
+cmake --build build
+cd build && ctest

data/unittest.cpp

@@ -0,0 +1,147 @@
+
+#include <gtest/gtest.h>
+#include "stridx.hpp"
+#include <cmath>
+#include <memory>
+
+TEST(SplitString, MatchSize) {
+  std::vector<std::string> svec = StrIdx::splitString("foo/bar/test1.txt", '/');
+  EXPECT_EQ(svec.size(), 3);
+  if (svec.size() == 3) {
+    EXPECT_EQ(svec[0].size(), 3);
+    EXPECT_EQ(svec[2].size(), 9);
+  }
+}
+
+std::vector<std::string> flist{"./drivers/char/hw_random/nomadik-rng.c",
+                               "./drivers/pinctrl/nomadik",
+                               "./drivers/clk/clk-nomadik.c",
+                               "./drivers/gpio/gpio-nomadik.c",
+                               "./drivers/i2c/busses/i2c-nomadik.c",
+                               "./drivers/clocksource/nomadik-mtu.c",
+                               "./drivers/gpu/drm/pl111/pl111_nomadik.h",
+                               "./drivers/gpu/drm/pl111/pl111_nomadik.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik.c",
+                               "./drivers/input/keyboard/nomadik-ske-keypad.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c",
+                               "./drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c",
+                               "./drivers/char/hw_random/omap-rng.c",
+                               "./drivers/char/hw_random/omap3-rom-rng.c",
+                               "./include/dt-bindings/pinctrl/nomadik.h",
+                               "./Documentation/devicetree/bindings/arm/ste-nomadik.txt"};
+
+std::vector<float> target_scores{0.342944, 0.271396, 0.271126, 0.270893, 0.270431, 0.270355,
+                                 0.270088, 0.270088, 0.26987,  0.269776, 0.269574, 0.269538,
+                                 0.236358, 0.236074, 0.224804, 0.224238};
+
+void scoreTest(bool threaded) {
+
+  StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
+  std::string query = "rngnomadriv";
+
+  int i = 1;
+  for (const auto &str : flist) {
+    if (threaded) {
+      idx.addStrToIndexThreaded(str, i);
+    } else {
+      idx.addStrToIndex(str, i);
+    }
+    i++;
+  }
+  const std::vector<std::pair<float, int>> &results = idx.findSimilar(query);
+
+  std::cout << results[0].first;
+  EXPECT_EQ(results[0].second, 1);
+  if (results.size() == 16) {
+    int i = 0;
+    for (const auto &res : results) {
+      // Check if first five digits of the scores match
+      EXPECT_EQ(std::floor(res.first * 1e5), std::floor(1e5 * target_scores[i]));
+      i++;
+    }
+  }
+}
+
+TEST(IndexSearch, MatchingScoresSingleThread) { scoreTest(false); }
+TEST(IndexSearch, MatchingScoresThreaded) { scoreTest(true); }
+
+class IndexTest : public testing::Test {
+protected:
+  std::unique_ptr<StrIdx::StringIndex> idx = std::make_unique<StrIdx::StringIndex>('/');
+
+  IndexTest() {}
+
+  void SetUp() override {
+    // Code here will be called immediately after the constructor (right
+    // before each test).
+    idx = std::make_unique<StrIdx::StringIndex>('/');
+  }
+
+  void TearDown() override {
+    // Code here will be called immediately after each test (right
+    // before the destructor).
+  }
+};
+
+TEST_F(IndexTest, BinaryRepresentation1) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 0, 8);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  //                a       b       c       d ...
+  EXPECT_TRUE(s == "0110000101100010011000110110010001100101011001100110011101101000");
+}
+
+TEST_F(IndexTest, BinaryRepresentation2) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 0, 1);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  EXPECT_TRUE(
+      s == "0000000000000000000000000000000000000000000000000000000001100001"); // 01100001 == "a"
+}
+TEST_F(IndexTest, BinaryRepresentation3) {
+  int64_t num = idx->getKeyAtIdx("abcdefgh", 7, 1);
+  std::string s = StrIdx::int64ToBinaryString(num);
+  EXPECT_TRUE(
+      s == "0000000000000000000000000000000000000000000000000000000001101000"); // 01101000 == "h"
+}
+
+TEST_F(IndexTest, AccessString) {
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 0);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadiksdf.c", 2);
+  idx->addStrToIndex("./drivers//i2c///busses////aa-i2c-nomadiksdf.c", 3);
+  idx->addStrToIndex("/test/foo/bar.txt", 4);
+  idx->addStrToIndex("bar.txt", 5);
+
+  EXPECT_EQ(idx->size(), 5);
+  EXPECT_STREQ(idx->getString(0).c_str(), "./drivers/i2c/busses/i2c-nomadik.c");
+
+  // TODO: does not work yet
+  //  EXPECT_STREQ(idx->getString(3).c_str(), "./drivers//i2c///busses////aa-i2c-nomadiksdf.c");
+
+  // TODO: does not work yet
+  //  EXPECT_STREQ(idx->getString(4).c_str(), "/test/foo/bar.txt");
+  EXPECT_STREQ(idx->getString(5).c_str(), "bar.txt");
+}
+
+TEST_F(IndexTest, Size1) {
+  // Should not add different files with same id
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 0);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadiksdf.c", 0);
+
+  // Should not be added because essentially same as 0:
+  idx->addStrToIndex("./drivers//i2c///busses////i2c-nomadik.c", 44);
+
+  // Should not add same file with different id
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 1);
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 2);
+
+  EXPECT_EQ(idx->size(), 1);
+
+  // Test no-overwriting
+  EXPECT_STREQ(idx->getString(0).c_str(), "./drivers/i2c/busses/i2c-nomadik.c");
+}
+
+TEST_F(IndexTest, Size2) {
+  idx->addStrToIndex("./drivers/i2c/busses/i2c-nomadik.c", 22);
+  idx->addStrToIndex("./Documentation/devicetree/bindings/arm/ste-nomadik.txt", 1);
+  idx->addStrToIndex("./Documentation/devicetree/bindings/arm/ste-nomadik33.txt", 3);
+  EXPECT_EQ(idx->size(), 3);
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: StrIdx
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Sami Sieranoja
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-05-24 00:00:00.000000000 Z
+date: 2024-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,24 +38,123 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 13.1.0
+- !ruby/object:Gem::Dependency
+  name: tty-cursor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: tty-prompt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.23.1
+- !ruby/object:Gem::Dependency
+  name: tty-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: tty-screen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+- !ruby/object:Gem::Dependency
+  name: pastel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.4.1
 description: " Fast fuzzy string similarity search and indexing (for filenames)"
 email:
 - sami.sieranoja@gmail.com
-executables: []
+executables:
+- stridx.rb
 extensions:
 - rubyext/extconf.rb
 extra_rdoc_files: []
 files:
+- CMakeLists.txt
+- Gemfile
 - LICENSE
 - Makefile
 - README.md
 - demo.cpp
+- exe/stridx.rb
 - flist.txt
+- gem_install
+- py_example.py
+- py_interf.cpp
 - rubyext/extconf.rb
 - rubyext/ruby_interf.cpp
+- runserver.rb
+- server.rb
+- setup.py
+- stridx-screencast.mp4
+- stridx-tty.rb
+- stridx.gemspec
 - stridx.hpp
 - test.rb
 - thread_pool.hpp
+- unit_tests.sh
+- unittest.cpp
 - unordered_dense.h
 homepage: https://github.com/SamiSieranoja/stridx
 licenses:
@@ -63,7 +162,7 @@ licenses:
 metadata:
   source_code_uri: https://github.com/SamiSieranoja/stridx
   homepage_uri: https://github.com/SamiSieranoja/stridx
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -80,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.3.26
-signing_key: 
+signing_key:
 specification_version: 4
 summary: StrIdx
 test_files: []