StrIdx 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2155c54900f595ae954d2074840f425195a66d6bf464e35dc68a1bb10236255c
-  data.tar.gz: fc2b50bcf6083b806643886a275e2a03ce3267cbd66dd81a0d63f5f68f8e4268
+  metadata.gz: f7655b6bd71bca58c86ad607fd197933fc19b97b3ae1c76e322ec0432025dad7
+  data.tar.gz: 2421892aa6fe750213d08e2254019d87ce7abb10496cdf1635a61815b8b8b0d7
 SHA512:
-  metadata.gz: ac27e251a448f1ca17d4672d12ac165412f2c7fbd71359e77d21f8a8c30521797befe7f2555d181be2af0cace345636558b4a8d93d8e141f707e8b28ffe5687b
-  data.tar.gz: f8e65efbed5079c0a2e37504aa955e0855d0ef088ad07d1d2566dee6c5f51b79d9ba71ef102e5aaeef8d1b7d548eefa2fab3930e21b7de05b940da882c9b86f3
+  metadata.gz: 0a0ed3f51b95b72a553cf97e1a852f63e8b6d1cbbba56fdab55ed5037ef4d658b8649f2cf92d35b5eea4e6657a18a3a1460a3a57069cda0889a1987f5d1611ee
+  data.tar.gz: f0d4753ee43cb205fa86468dad92a66644e12cf889d8018283cb421e4a5d670386b8666af64b89d4d475a59f10701de2223c31d03ac5d366f2ee255c77190cf8

data/CMakeLists.txt

@@ -0,0 +1,27 @@
+cmake_minimum_required(VERSION 3.14)
+
+project(my_project)
+# https://github.com/google/googletest/issues/4000
+include(FetchContent)
+FetchContent_Declare(
+  googletest
+  URL https://github.com/google/googletest/archive/58d77fa8070e8cec2dc1ed015d66b454c8d78850.zip # release-1.12.1
+)
+
+set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+FetchContent_MakeAvailable(googletest)
+
+enable_testing()
+
+add_executable(
+  stridx_test
+  unittest.cpp
+)
+target_link_libraries(
+  stridx_test
+  GTest::gtest_main
+)
+
+include(GoogleTest)
+gtest_discover_tests(stridx_test)
+

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+

data/Makefile

@@ -1,7 +1,7 @@
 all: demo
 
-demo: *.hpp *.cpp
-	g++  -Wall -O3 -fopenmp -lstdc++ demo.cpp -o demo
+demo: *.hpp *.cpp Makefile
+	g++  -Wall -O3 -lstdc++ demo.cpp -o demo
 	
 clean: 
 	rm demo

data/README.md

@@ -37,12 +37,58 @@ and candidate is "./drivers/char/hw_random/nomadik-rng.c", then scores are calcu
     score = score1/(11*11)*0.97 + score1/(11*38)*0.03 = 0.342944
 ```
 
-# Ruby interface
+# Interfaces
+
+## Commandline
+Install instructions (for Ubuntu Linux):
+```
+apt update
+apt install ruby ruby-dev build-essential
+gem install StrIdx
+```
+
+Start indexing server (on background):
+```
+stridx.rb start -- ~/Documents/ ~/Pictures/
+```
+
+Add bash keybindings (Ctrl-t):
+```
+eval "$(stridx.rb bash)"
+```
+
+Search by pressing <kbd>ctrl</kbd>+<kbd>t</kbd>.  Keys: <kbd>up</kbd>, <kbd>down</kbd>, select with <kbd>enter</kbd>
+
+![screencast](https://github.com/SamiSieranoja/stridx/assets/46612258/b2fd4fa2-37ad-4423-bd5f-d54b24ff6df5)
+
+
+Stop server:
+```
+stridx.rb stop
+```
+
+Start indexing server (on foreground, to debug):
+```
+stridx.rb run -- ~/Documents/ ~/Pictures/
+```
+
+
+## Ruby
 Install:
 ```
+apt install ruby ruby-dev build-essential
 gem install StrIdx
 ```
 
+Or, for development version:
+```
+git clone https://github.com/SamiSieranoja/stridx.git
+cd stridx
+cd rubyext;  ruby extconf.rb ; make ; cd ..
+gem build stridx.gemspec
+gem install $(ls -1tr StrIdx*gem | tail -n 1)
+```
+
 Usage example (see test.rb):
 ```ruby
 require "stridx"
@@ -58,7 +104,7 @@ for x in lines
 end
 
 idx_time = Time.new 
-puts "\nIndexing time (#{lines.size} files}): #{(idx_time - t).round(4)} seconds"
+puts "\nIndexing time (#{lines.size} files): #{(idx_time - t).round(4)} seconds"
 
 query = "rngnomadriv"
 res = idx.find(query)
@@ -105,7 +151,7 @@ Search time: 0.0488 seconds
 ```
 
 
-# C++ API
+## C++
 See demo.cpp
 ```cpp
 #include "stridx.hpp"

data/demo.cpp

@@ -1,3 +1,12 @@
+
+#include <condition_variable>
+#include <functional>
+#include <iostream>
+#include <mutex>
+#include <queue>
+#include <thread>
+#include <algorithm>
+
 #include "stridx.hpp"
 
 #include <iostream>
@@ -28,7 +37,7 @@ std::vector<std::string> readLinesFromFile(const std::string &filename) {
 }
 
 int main() {
-  StringIndex idx;
+  StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
   // idx.addStrToIndex("./gdk/x11/gdkasync.c", 0 /*id*/, '/' /*separator*/);
   // idx.addStrToIndex("./gdk/x11/gdksettings.c", 1, '/');
   // idx.addStrToIndex("./gdk/x11/gdkx11devicemanager-xi2.h", 2, '/');
@@ -37,26 +46,39 @@ int main() {
   std::string fn_filePaths = "flist.txt";
   std::vector<std::string> v_filePaths = readLinesFromFile(fn_filePaths);
 
+	// Launch indexing to be run on background
+  cout << "File paths: " << v_filePaths.size() << std::endl;
+  cout << "Start indexing in the background" << std::endl;
   auto start = std::chrono::high_resolution_clock::now();
   int id = 0;
   for (const auto &filePath : v_filePaths) {
-    idx.addStrToIndex(filePath, id, '/' /*dir separator*/);
-    // idx.addStrToIndex(filePath, id, '\0' /*dir separator*/);
+    idx.addStrToIndexThreaded(filePath, id);
     id++;
   }
-  
+
+  auto idx_time_launch = std::chrono::high_resolution_clock::now();
+  std::chrono::duration<double, std::milli> duration_launch = idx_time_launch - start;
+  cout << "Indexing launch time (seconds): " << duration_launch.count() / 1000 << "\n";
+
+	// Wait until indexing has finished
+  idx.waitUntilDone();
+
   auto idx_time = std::chrono::high_resolution_clock::now();
   std::chrono::duration<double, std::milli> duration = idx_time - start;
-  cout << "Indexing creation time for " << v_filePaths.size() << " file paths (seconds): " << duration.count() / 1000 << "\n";
+  cout << "Indexing finished time for " << v_filePaths.size()
+       << " file paths (seconds): " << duration.count() / 1000 << "\n";
 
   // Find matching filepaths from the index for the query string "rngnomadriv"
   start = std::chrono::high_resolution_clock::now();
   std::string query = "rngnomadriv";
+  for (int i = 0; i < 99; i++) {
+    const vector<pair<float, int>> &results = idx.findSimilar(query, 2);
+  }
+
   const vector<pair<float, int>> &results = idx.findSimilar(query, 2);
   auto search_time = std::chrono::high_resolution_clock::now();
   duration = search_time - start;
-  cout << "Search time (seconds): " << duration.count() / 1000
-       << "\n";
+  cout << "Search time for 100 queries (seconds): " << duration.count() / 1000 << "\n";
 
   int i = 0;
   std::cout << "query string: " << query << "\n";
@@ -73,4 +95,4 @@ int main() {
 }
 
 // Compile:
-// g++  -Wall -Wno-unused-variable -O3 -fopenmp -lstdc++ demo.cpp -o demo
+// g++  -Wall -Wno-unused-variable -O3 -lstdc++ demo.cpp -o demo

data/exe/stridx.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + "/.."
+
+if ARGV[0] == "tty"
+  require "stridx-tty.rb"
+  StrIdxTTY.run
+elsif ARGV[0] == "bash"
+  puts %q/
+  bind -m emacs-standard '"\er": redraw-current-line';
+  bind -m emacs-standard '"\C-t": " \C-b\C-k \C-u`stridx.rb tty`\e\C-e\er\C-a\C-y\C-h\C-e\e \C-y\ey\C-x\C-x\C-f"'
+/
+else
+  require "daemons"
+  Daemons.run(File.dirname(__FILE__) + "/../runserver.rb")
+end

data/gem_install

@@ -0,0 +1,4 @@
+gem uninstall --force -x StrIdx
+gem build stridx.gemspec
+gem install $(ls -1tr StrIdx*gem | tail -n 1)
+

data/py_example.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+from stridx import StringIndex
+e=StringIndex()
+e.set_value(3)
+e.add("./rust/alloc/vec/spec_extend.rs",0)
+e.add("./virt/kvm/dirty_ring.c",1)
+e.add("./Documentation/staging/static-keys.rst",2)
+e.add("./Documentation/staging/lzo.rst",3)
+
+
+
+results = e.find("rstalloc")
+for x in results:
+	print(x)
+
+# print(e.get_value())
+

data/py_interf.cpp

@@ -0,0 +1,182 @@
+
+#include <stdio.h>
+#include <limits.h>
+#include <cstring>
+#include <pthread.h>
+
+#include <stdio.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <cfloat>
+#include <cmath>
+#include <bits/stdc++.h>
+
+using std::ios;
+using std::sort;
+using std::string;
+using std::vector;
+
+#include <Python.h>
+#include <cstring>
+
+#include "stridx.hpp"
+
+extern "C" {
+
+// Define a structure for the custom object
+typedef struct {
+  PyObject_HEAD int value;
+  StrIdx::StringIndex *idx;
+} StrIdxObject;
+
+// Method to allocate memory for the object
+static PyObject *StrIdxObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
+  StrIdxObject *self;
+
+  self = (StrIdxObject *)type->tp_alloc(type, 0);
+  if (self != NULL) {
+    self->value = 0;
+    self->idx = new StrIdx::StringIndex();
+  }
+
+  return (PyObject *)self;
+}
+
+// Method to deallocate memory for the object
+static void StrIdxObject_dealloc(StrIdxObject *self) { Py_TYPE(self)->tp_free((PyObject *)self); }
+
+// Method to set the value of the object
+static PyObject *StrIdxObject_set_value(StrIdxObject *self, PyObject *args) {
+  int value;
+
+  if (!PyArg_ParseTuple(args, "i", &value)) {
+    return NULL;
+  }
+
+  self->value = value;
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+static PyObject *StrIdxObject_add(StrIdxObject *self, PyObject *args) {
+  char *value;
+  int file_id;
+  std::string str;
+  if (!PyArg_ParseTuple(args, "si", &value, &file_id)) {
+    return NULL;
+  }
+  str = value;
+
+  printf("char[]*: %s %i\n", value, file_id);
+  self->idx->addStrToIndex(str, file_id);
+  // self->idx->addStrToIndexThreaded(str, file_id);
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+static PyObject *StrIdxObject_find(StrIdxObject *self, PyObject *args) {
+  char *value;
+  std::string str;
+  if (!PyArg_ParseTuple(args, "s", &value)) {
+    return NULL;
+  }
+  str = value;
+
+  printf("char*: %s\n", value);
+  const std::vector<std::pair<float, int>> &results = self->idx->findSimilar(str, 2);
+
+  int limit = 15;
+  int i = 0;
+
+  printf("res=%d\n", results.size());
+  if (results.size() < limit) {
+    limit = results.size();
+  }
+  PyObject *pyarr = PyList_New(limit);
+
+  for (const auto &[score,fileId] : results) {
+    PyObject *arr2 = PyList_New(2);
+    // PyList_SetItem(arr2, 0, Py_BuildValue("i", res.second));
+    // PyList_SetItem(arr2, 1, Py_BuildValue("d", res.first));
+    PyList_SetItem(arr2, 0, Py_BuildValue("i", fileId));
+    PyList_SetItem(arr2, 1, Py_BuildValue("d", score));
+    PyList_SetItem(pyarr, i, arr2);
+    i++;
+    if (i >= limit) {
+      break;
+    }
+  }
+
+  // Py_INCREF(Py_None);
+  return pyarr;
+}
+
+// Method to get the value of the object
+static PyObject *StrIdxObject_get_value(StrIdxObject *self) {
+  return PyLong_FromLong(self->value);
+}
+
+// Define methods of the class
+static PyMethodDef StrIdxObject_methods[] = {
+    {"set_value", (PyCFunction)StrIdxObject_set_value, METH_VARARGS,
+     "Set the value of the object"},
+    {"add", (PyCFunction)StrIdxObject_add, METH_VARARGS, "Set the value of the object"},
+    {"find", (PyCFunction)StrIdxObject_find, METH_VARARGS, "Find similar strings"},
+    {"get_value", (PyCFunction)StrIdxObject_get_value, METH_NOARGS, "Get the value of the object"},
+    {NULL} /* Sentinel */
+};
+
+// Define the type object for the class
+static PyTypeObject StrIdxType = {
+    PyVarObject_HEAD_INIT(NULL, 0).tp_name = "stridx.StrIdx",
+    .tp_basicsize = sizeof(StrIdxObject),
+    .tp_dealloc = (destructor)StrIdxObject_dealloc,
+    .tp_doc = PyDoc_STR("Fuzzy string index"),
+    .tp_methods = StrIdxObject_methods,
+    .tp_new = StrIdxObject_new,
+    // .tp_repr = (reprfunc)myobj_repr,
+};
+
+// PyVarObject_HEAD_INIT(NULL, 0)
+// .tp_name = "stridx.StrIdx",
+// .tp_doc = "StrIdx class",
+// .tp_basicsize = sizeof(StrIdxObject),
+// .tp_itemsize = 0,
+// .tp_flags = Py_TPFLAGS_DEFAULT,
+// .tp_new = StrIdxObject_new,
+// .tp_dealloc = (destructor)StrIdxObject_dealloc,
+// .tp_methods = StrIdxObject_methods,
+// };
+
+
+// Define python accessible methods
+static PyMethodDef StrIdxMethods[] = {
+    {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT, "stridx", NULL, -1, StrIdxMethods, NULL, NULL, NULL, NULL};
+
+PyMODINIT_FUNC PyInit_stridx(void) {
+  PyObject *m;
+  m = PyModule_Create(&moduledef);
+
+  // Initialize the type object
+  if (PyType_Ready(&StrIdxType) < 0) {
+    return NULL;
+  }
+
+  Py_INCREF(&StrIdxType);
+  if (PyModule_AddObject(m, "StringIndex", (PyObject *)&StrIdxType) < 0) {
+    Py_DECREF(&StrIdxType);
+    Py_DECREF(m);
+    return NULL;
+  }
+
+  if (!m) {
+    return NULL;
+  }
+  return m;
+}
+} // END extern "C"

data/rubyext/extconf.rb

@@ -1,15 +1,13 @@
 #!/usr/bin/env ruby
-#
 
 require 'mkmf'
 
 module_name = "stridx"
 extension_name = 'stridx'
 
-$CXXFLAGS << " -Wall -Wno-unused-variable -O3 -fopenmp" 
+$CXXFLAGS << " -Wall -Wno-unused-variable -O3" 
 
 have_library( 'stdc++');
-have_library( 'gomp' );
 
 dir_config(extension_name)       # The destination
 create_makefile(extension_name)  # Create Makefile

data/rubyext/ruby_interf.cpp

@@ -7,7 +7,7 @@
 
 extern "C" {
 
-void str_idx_free(void *data) { delete (StringIndex *)data; }
+void str_idx_free(void *data) { delete (StrIdx::StringIndex *)data; }
 
 // Wrap StringIndex class inside a ruby variable
 static const rb_data_type_t str_idx_type = {
@@ -26,7 +26,7 @@ static const rb_data_type_t str_idx_type = {
 };
 
 VALUE str_idx_alloc(VALUE self) {
-  void *data = new StringIndex();
+  void *data = new StrIdx::StringIndex();
   return TypedData_Wrap_Struct(self, &str_idx_type, data);
 }
 
@@ -36,18 +36,27 @@ VALUE StringIndexAddSegments(VALUE self, VALUE str, VALUE fileId) {
 
   void *data;
   TypedData_Get_Struct(self, int, &str_idx_type, data);
-  ((StringIndex *)data)->addStrToIndex(s1, fid);
+  // ((StringIndex *)data)->addStrToIndex(s1, fid);
+  ((StrIdx::StringIndex *)data)->addStrToIndexThreaded(s1, fid);
 
   return self;
 }
 
+VALUE StringIndexWaitUntilDone(VALUE self) {
+  void *data;
+  TypedData_Get_Struct(self, int, &str_idx_type, data);
+  ((StrIdx::StringIndex *)data)->waitUntilDone();
+  return self;
+}
+  
+
 VALUE StringIndexFind(VALUE self, VALUE str) {
   VALUE ret;
   std::string s1 = StringValueCStr(str);
 
   void *data;
   TypedData_Get_Struct(self, int, &str_idx_type, data);
-  StringIndex *idx = (StringIndex *)data;
+  StrIdx::StringIndex *idx = (StrIdx::StringIndex *)data;
 
   ret = rb_ary_new();
   const std::vector<std::pair<float, int>> &results = idx->findSimilar(s1, 2);
@@ -80,7 +89,7 @@ VALUE StringIndexSetDirSeparator(VALUE self, VALUE str) {
 
   void *data;
   TypedData_Get_Struct(self, int, &str_idx_type, data);
-  StringIndex *idx = (StringIndex *)data;
+  StrIdx::StringIndex *idx = (StrIdx::StringIndex *)data;
   idx->setDirSeparator(c);
 
   return self;
@@ -93,8 +102,12 @@ void Init_stridx(void) {
 
   rb_define_alloc_func(classStringIndex, str_idx_alloc);
   rb_define_method(classStringIndex, "add", StringIndexAddSegments, 2);
+  rb_define_method(classStringIndex, "waitUntilDone", StringIndexWaitUntilDone, 0);
   rb_define_method(classStringIndex, "find", StringIndexFind, 1);
+  
   rb_define_method(classStringIndex, "setDirSeparator", StringIndexSetDirSeparator, 1);
+  
+  
 }
 
 } // End extern "C"

data/runserver.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+$:.unshift File.dirname(__FILE__)
+
+require "server.rb"
+# StrIdx::Server.start ARGV, daemonize: true
+StrIdx::Server.start ARGV
+

data/server.rb

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+require "socket"
+require "stridx"
+
+module StrIdx
+  class Server
+    def recursively_find_files(directories)
+      filelist = []
+
+      for d in directories
+        filelist = filelist + Dir.glob("#{d}/**/*").select { |e|
+          File.file?(e)
+          # File.file?(e) or File.directory?(e)
+        }
+      end
+      return filelist
+    end
+
+    def self.start(dir_list, daemonize: false)
+      Server.new(dir_list, daemonize: daemonize)
+    end
+
+    def self.stop
+      sock_dir = File.expand_path("~/.stridx")
+      sockfn = "#{sock_dir}/sock"
+      client = UNIXSocket.new(sockfn)
+      client.puts "stop"
+      response = client.recv(200 * 200)
+      client.close
+    end
+
+    def initialize(dir_list, daemonize: false)
+      idx = StrIdx::StringIndex.new
+      idx.setDirSeparator("/")
+
+      t = Time.new
+
+      dirs = dir_list.select { |x| File.directory?(x) }
+      puts "Scanning files in directories:#{dirs.join(",")}"
+      flist = recursively_find_files(dirs)
+
+      i = 0
+      for x in flist
+        idx.add(x, i)
+        i += 1
+      end
+
+      idx.waitUntilDone()
+      idx_time = Time.new
+      puts "\nIndexing time (#{flist.size} files): #{(idx_time - t).round(4)} seconds"
+
+      sock_dir = File.expand_path("~/.stridx")
+      Dir.mkdir(sock_dir) if !Dir.exist?(sock_dir)
+      sockfn = "#{sock_dir}/sock"
+      File.unlink(sockfn) if File.exist?(sockfn)
+
+      puts "Indexing done, starting server"
+      if (daemonize)
+        require "daemons"
+        Daemons.daemonize
+        # exit if fork() # Daemonize
+      end
+
+      # exit if fork() # Daemonize
+      # $PROGRAM_NAME = "stridx-daemon"
+
+      t = Thread.new {
+        serv = UNIXServer.new(sockfn)
+
+        loop do
+          # Accept a new client connection
+          client = serv.accept
+
+          # puts "Client connected!"
+
+          # Read data from the client
+          data = client.recv(1024)
+
+          if data.match(/^stop$/)
+            puts "Got stop signal. Shutting down server."
+            client.close
+            break
+          end
+
+          # puts "Received from client: #{data}"
+          if data.match(/^find:(.*)/)
+            query = Regexp.last_match(1)
+            res = idx.find(query)
+            response = res.collect { |x| flist[x[0]] }.join("\n")
+
+            # Send a response back to the client
+            client.puts response
+          end
+          # Close the client connection
+          client.close
+        end
+      }
+
+      t.join
+    end
+  end
+end

data/setup.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import numpy
+
+import setuptools
+from setuptools import setup, Extension
+
+__version__ = "0.1"
+
+cargs = ['-fpermissive']
+
+
+with open('README.md', 'r', encoding='utf-8') as f:
+    long_description = f.read()
+
+module1 = Extension('stridx', sources=['py_interf.cpp'], include_dirs=['.'], extra_compile_args=cargs,
+language="c++",
+)
+                        
+ext_modules = [module1]
+                      
+setup(
+    name='stridx',
+    version='1.0',
+    setup_requires=['wheel'],
+    python_requires='>=3',
+    provides=['stridx'],
+    description='Fast fuzzy string similarity search and indexing (for filenames) ',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    ext_modules=[module1]
+)
+