StrIdx 0.1.3 → 0.1.5

data/gem_install

@@ -0,0 +1,4 @@
+gem uninstall --force -x StrIdx
+gem build stridx.gemspec
+gem install $(ls -1tr StrIdx*gem | tail -n 1)
+

data/py_example.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+
+from stridx import StringIndex
+e=StringIndex()
+e.set_value(3)
+e.add("./rust/alloc/vec/spec_extend.rs",0)
+e.add("./virt/kvm/dirty_ring.c",1)
+e.add("./Documentation/staging/static-keys.rst",2)
+e.add("./Documentation/staging/lzo.rst",3)
+
+
+
+results = e.find("rstalloc")
+for x in results:
+	print(x)
+
+# print(e.get_value())
+

data/py_interf.cpp

@@ -0,0 +1,182 @@
+
+#include <stdio.h>
+#include <limits.h>
+#include <cstring>
+#include <pthread.h>
+
+#include <stdio.h>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <cfloat>
+#include <cmath>
+#include <bits/stdc++.h>
+
+using std::ios;
+using std::sort;
+using std::string;
+using std::vector;
+
+#include <Python.h>
+#include <cstring>
+
+#include "stridx.hpp"
+
+extern "C" {
+
+// Define a structure for the custom object
+typedef struct {
+  PyObject_HEAD int value;
+  StrIdx::StringIndex *idx;
+} StrIdxObject;
+
+// Method to allocate memory for the object
+static PyObject *StrIdxObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds) {
+  StrIdxObject *self;
+
+  self = (StrIdxObject *)type->tp_alloc(type, 0);
+  if (self != NULL) {
+    self->value = 0;
+    self->idx = new StrIdx::StringIndex();
+  }
+
+  return (PyObject *)self;
+}
+
+// Method to deallocate memory for the object
+static void StrIdxObject_dealloc(StrIdxObject *self) { Py_TYPE(self)->tp_free((PyObject *)self); }
+
+// Method to set the value of the object
+static PyObject *StrIdxObject_set_value(StrIdxObject *self, PyObject *args) {
+  int value;
+
+  if (!PyArg_ParseTuple(args, "i", &value)) {
+    return NULL;
+  }
+
+  self->value = value;
+
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+static PyObject *StrIdxObject_add(StrIdxObject *self, PyObject *args) {
+  char *value;
+  int file_id;
+  std::string str;
+  if (!PyArg_ParseTuple(args, "si", &value, &file_id)) {
+    return NULL;
+  }
+  str = value;
+
+  printf("char[]*: %s %i\n", value, file_id);
+  self->idx->addStrToIndex(str, file_id);
+  // self->idx->addStrToIndexThreaded(str, file_id);
+  Py_INCREF(Py_None);
+  return Py_None;
+}
+
+static PyObject *StrIdxObject_find(StrIdxObject *self, PyObject *args) {
+  char *value;
+  std::string str;
+  if (!PyArg_ParseTuple(args, "s", &value)) {
+    return NULL;
+  }
+  str = value;
+
+  printf("char*: %s\n", value);
+  const std::vector<std::pair<float, int>> &results = self->idx->findSimilar(str, 2);
+
+  int limit = 15;
+  int i = 0;
+
+  printf("res=%d\n", results.size());
+  if (results.size() < limit) {
+    limit = results.size();
+  }
+  PyObject *pyarr = PyList_New(limit);
+
+  for (const auto &[score,fileId] : results) {
+    PyObject *arr2 = PyList_New(2);
+    // PyList_SetItem(arr2, 0, Py_BuildValue("i", res.second));
+    // PyList_SetItem(arr2, 1, Py_BuildValue("d", res.first));
+    PyList_SetItem(arr2, 0, Py_BuildValue("i", fileId));
+    PyList_SetItem(arr2, 1, Py_BuildValue("d", score));
+    PyList_SetItem(pyarr, i, arr2);
+    i++;
+    if (i >= limit) {
+      break;
+    }
+  }
+
+  // Py_INCREF(Py_None);
+  return pyarr;
+}
+
+// Method to get the value of the object
+static PyObject *StrIdxObject_get_value(StrIdxObject *self) {
+  return PyLong_FromLong(self->value);
+}
+
+// Define methods of the class
+static PyMethodDef StrIdxObject_methods[] = {
+    {"set_value", (PyCFunction)StrIdxObject_set_value, METH_VARARGS,
+     "Set the value of the object"},
+    {"add", (PyCFunction)StrIdxObject_add, METH_VARARGS, "Set the value of the object"},
+    {"find", (PyCFunction)StrIdxObject_find, METH_VARARGS, "Find similar strings"},
+    {"get_value", (PyCFunction)StrIdxObject_get_value, METH_NOARGS, "Get the value of the object"},
+    {NULL} /* Sentinel */
+};
+
+// Define the type object for the class
+static PyTypeObject StrIdxType = {
+    PyVarObject_HEAD_INIT(NULL, 0).tp_name = "stridx.StrIdx",
+    .tp_basicsize = sizeof(StrIdxObject),
+    .tp_dealloc = (destructor)StrIdxObject_dealloc,
+    .tp_doc = PyDoc_STR("Fuzzy string index"),
+    .tp_methods = StrIdxObject_methods,
+    .tp_new = StrIdxObject_new,
+    // .tp_repr = (reprfunc)myobj_repr,
+};
+
+// PyVarObject_HEAD_INIT(NULL, 0)
+// .tp_name = "stridx.StrIdx",
+// .tp_doc = "StrIdx class",
+// .tp_basicsize = sizeof(StrIdxObject),
+// .tp_itemsize = 0,
+// .tp_flags = Py_TPFLAGS_DEFAULT,
+// .tp_new = StrIdxObject_new,
+// .tp_dealloc = (destructor)StrIdxObject_dealloc,
+// .tp_methods = StrIdxObject_methods,
+// };
+
+
+// Define python accessible methods
+static PyMethodDef StrIdxMethods[] = {
+    {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef moduledef = {
+    PyModuleDef_HEAD_INIT, "stridx", NULL, -1, StrIdxMethods, NULL, NULL, NULL, NULL};
+
+PyMODINIT_FUNC PyInit_stridx(void) {
+  PyObject *m;
+  m = PyModule_Create(&moduledef);
+
+  // Initialize the type object
+  if (PyType_Ready(&StrIdxType) < 0) {
+    return NULL;
+  }
+
+  Py_INCREF(&StrIdxType);
+  if (PyModule_AddObject(m, "StringIndex", (PyObject *)&StrIdxType) < 0) {
+    Py_DECREF(&StrIdxType);
+    Py_DECREF(m);
+    return NULL;
+  }
+
+  if (!m) {
+    return NULL;
+  }
+  return m;
+}
+} // END extern "C"

data/rubyext/ruby_interf.cpp

@@ -59,8 +59,8 @@ VALUE StringIndexFind(VALUE self, VALUE str) {
   StrIdx::StringIndex *idx = (StrIdx::StringIndex *)data;
 
   ret = rb_ary_new();
-  const std::vector<std::pair<float, int>> &results = idx->findSimilar(s1, 2);
-  int limit = 15;
+  const std::vector<std::pair<float, int>> &results = idx->findSimilar(s1);
+  int limit = 40;
   int i = 0;
   for (const auto &res : results) {
     VALUE arr = rb_ary_new();
@@ -75,6 +75,60 @@ VALUE StringIndexFind(VALUE self, VALUE str) {
   return ret;
 }
 
+VALUE StringIndexFindFilesAndDirs(VALUE self, VALUE str) {
+  VALUE ret;
+  std::string s1 = StringValueCStr(str);
+
+  void *data;
+  TypedData_Get_Struct(self, int, &str_idx_type, data);
+  StrIdx::StringIndex *idx = (StrIdx::StringIndex *)data;
+
+  ret = rb_ary_new();
+  const std::vector<std::pair<float, std::string>> &results = idx->findFilesAndDirectories(s1);
+  int limit = 40;
+  int i = 0;
+  for (const auto &res : results) {
+    VALUE arr = rb_ary_new();
+    rb_ary_push(arr, rb_str_new_cstr(res.second.c_str()));
+    rb_ary_push(arr, DBL2NUM(res.first));
+    rb_ary_push(ret, arr);
+    i++;
+    if (i >= limit) {
+      break;
+    }
+  }
+  return ret;
+}
+
+VALUE StringIndexFindDirs(VALUE self, VALUE str) {
+  VALUE ret;
+  std::string s1 = StringValueCStr(str);
+
+  void *data;
+  TypedData_Get_Struct(self, int, &str_idx_type, data);
+  StrIdx::StringIndex *idx = (StrIdx::StringIndex *)data;
+
+  ret = rb_ary_new();
+  const std::vector<std::pair<float, std::string>> &results = idx->findFilesAndDirectories(s1,false,true);
+  int limit = 40;
+  int i = 0;
+  for (const auto &res : results) {
+    VALUE arr = rb_ary_new();
+    rb_ary_push(arr, rb_str_new_cstr(res.second.c_str()));
+    rb_ary_push(arr, DBL2NUM(res.first));
+    rb_ary_push(ret, arr);
+    i++;
+    if (i >= limit) {
+      break;
+    }
+  }
+  return ret;
+}
+
+
+
+
+
 VALUE StringIndexSetDirSeparator(VALUE self, VALUE str) {
   char c = '/';
   if (TYPE(str) == T_STRING) {
@@ -104,6 +158,8 @@ void Init_stridx(void) {
   rb_define_method(classStringIndex, "add", StringIndexAddSegments, 2);
   rb_define_method(classStringIndex, "waitUntilDone", StringIndexWaitUntilDone, 0);
   rb_define_method(classStringIndex, "find", StringIndexFind, 1);
+  rb_define_method(classStringIndex, "findFilesAndDirs", StringIndexFindFilesAndDirs, 1);
+  rb_define_method(classStringIndex, "findDirs", StringIndexFindDirs, 1);
   
   rb_define_method(classStringIndex, "setDirSeparator", StringIndexSetDirSeparator, 1);
   

data/runserver.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+$:.unshift File.dirname(__FILE__)
+
+def kill_signal
+  puts "\nShutting down..."
+  File.delete(File.expand_path("~/.stridx/sock"))
+end
+
+# https://gist.github.com/sauloperez/6592971
+# Trap ^C 
+Signal.trap("INT") { 
+  kill_signal 
+  exit
+}
+
+# Trap `Kill `
+Signal.trap("TERM") {
+  kill_signal
+  exit
+}
+
+require "server.rb"
+# StrIdx::Server.start ARGV, daemonize: true
+StrIdx::Server.start ARGV
+
+
+

data/server.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+require "socket"
+require "stridx"
+
+module StrIdx
+  class Server
+    def recursively_find_files(directories)
+      filelist = []
+
+      for d in directories
+        filelist = filelist + Dir.glob("#{d}/**/*").select { |e|
+          File.file?(e)
+          # File.file?(e) or File.directory?(e)
+        }
+      end
+      return filelist
+    end
+
+    def self.start(dir_list, daemonize: false)
+      Server.new(dir_list, daemonize: daemonize)
+    end
+
+    def self.stop
+      sock_dir = File.expand_path("~/.stridx")
+      sockfn = "#{sock_dir}/sock"
+      client = UNIXSocket.new(sockfn)
+      client.puts "stop"
+      response = client.recv(200 * 200)
+      client.close
+    end
+
+    def initialize(dir_list, daemonize: false)
+      idx = StrIdx::StringIndex.new
+      idx.setDirSeparator("/")
+
+      t = Time.new
+
+      dirs = dir_list.select { |x| File.directory?(x) }
+      puts "Scanning files in directories:#{dirs.join(",")}"
+      flist = recursively_find_files(dirs)
+
+      i = 0
+      for x in flist
+        idx.add(x, i)
+        i += 1
+      end
+
+      idx.waitUntilDone()
+      idx_time = Time.new
+      puts "\nIndexing time (#{flist.size} files): #{(idx_time - t).round(4)} seconds"
+
+      sock_dir = File.expand_path("~/.stridx")
+      Dir.mkdir(sock_dir) if !Dir.exist?(sock_dir)
+      sockfn = "#{sock_dir}/sock"
+      File.unlink(sockfn) if File.exist?(sockfn)
+
+      puts "Indexing done, starting server"
+      if (daemonize)
+        require "daemons"
+        Daemons.daemonize
+        # exit if fork() # Daemonize
+      end
+
+      # exit if fork() # Daemonize
+      # $PROGRAM_NAME = "stridx-daemon"
+
+      t = Thread.new {
+        serv = UNIXServer.new(sockfn)
+
+        loop do
+          # Accept a new client connection
+          client = serv.accept
+
+          # puts "Client connected!"
+
+          # Read data from the client
+          data = client.recv(1024)
+
+          if data.match(/^stop$/)
+            puts "Got stop signal. Shutting down server."
+            client.close
+            break
+          end
+
+          # puts "Received from client: #{data}"
+          if data.match(/^find:(.*)/)
+            query = Regexp.last_match(1)
+            # TODO: not sure which is best as default:
+            # res = idx.find(query)
+            # res = idx.findDirs(query)
+            res = idx.findFilesAndDirs(query)
+            # response = res.collect { |x| flist[x[0]] }.join("\n")
+            response = res.collect { |x| "/"+x[0] }.join("\n")
+
+            # Send a response back to the client
+            client.puts response
+          end
+          # Close the client connection
+          client.close
+        end
+      }
+
+      t.join
+      File.delete(sockfn)
+    end
+  end
+end

data/setup.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import numpy
+
+import setuptools
+from setuptools import setup, Extension
+
+__version__ = "0.1"
+
+cargs = ['-fpermissive']
+
+
+with open('README.md', 'r', encoding='utf-8') as f:
+    long_description = f.read()
+
+module1 = Extension('stridx', sources=['py_interf.cpp'], include_dirs=['.'], extra_compile_args=cargs,
+language="c++",
+)
+                        
+ext_modules = [module1]
+                      
+setup(
+    name='stridx',
+    version='1.0',
+    setup_requires=['wheel'],
+    python_requires='>=3',
+    provides=['stridx'],
+    description='Fast fuzzy string similarity search and indexing (for filenames) ',
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    ext_modules=[module1]
+)
+

data/stridx-tty.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+
+require "tty-prompt"
+require "tty-cursor"
+require "tty-reader"
+require "pastel"
+
+require "socket"
+
+class StrIdxTTY
+  def self.run
+    stty = StrIdxTTY.new
+    selected = stty.search
+    STDOUT.write selected
+  end
+
+  def initialize()
+    @lines = []
+    @selected = ""
+    @idx = 0
+
+    @reader = TTY::Reader.new(output: STDERR)
+    @pastel = Pastel.new()
+    @cursor = TTY::Cursor
+
+    sock_dir = File.expand_path("~/.stridx")
+    sockfn = "#{sock_dir}/sock"
+
+    error = true
+    while error
+      begin
+        # Create a new UNIXSocket
+        client = UNIXSocket.new(sockfn)
+      rescue Errno::ECONNREFUSED => e
+        out "Waiting for server to start\n"
+        sleep 2
+        error = true
+      else
+        error = false
+        client.close
+        #... executes when no error
+      end
+    end
+  end
+
+  def out(x)
+    STDERR.write x
+  end
+
+  def search
+    out "\n" * 20
+    out @cursor.clear_screen
+    out "\n" * 20
+    @cursor.move_to(0, 0)
+    @reader.on(:keypress) { |event|
+      handle_event(event)
+    }
+    @reader.read_line(">> ")
+
+    out @cursor.clear_screen
+    return @selected.strip
+  end
+
+  def get_res_from_server(query)
+    # Define the socket file path
+    sock_dir = File.expand_path("~/.stridx")
+    sockfn = "#{sock_dir}/sock"
+
+    # Create a new UNIXSocket
+    client = UNIXSocket.new(sockfn)
+
+    # Send data to the server
+    client.puts "find:#{query}"
+
+    # Read response from the server
+    response = client.recv(200 * 200)
+
+    # Close the client connection
+    client.close
+    return response.lines
+  end
+
+  def draw_list()
+    @selected = @list[@idx]
+    i = 0
+    for x in @list
+      out @cursor.up(1)
+      out @cursor.clear_line
+      if i == @idx
+        out @pastel.lookup(:bold)
+      end
+      out x.strip
+      out @pastel.lookup(:reset)
+      i += 1
+    end
+  end
+
+  def update_search(event)
+    query = event.line[3..-1]
+    if query.size > 2
+      @list = get_res_from_server(query)
+      draw_list
+    end
+  end
+
+  def handle_event(event)
+    out @cursor.save
+    if event.key.name == :alpha
+      update_search(event)
+    elsif event.key.name == :up
+      @idx += 1 if @idx < @list.size - 1
+      draw_list
+    elsif event.key.name == :down
+      @idx -= 1 if @idx > 0
+      draw_list
+    elsif event.key.name == :backspace
+      update_search(event)
+    end
+
+    out @cursor.restore
+  end
+end

data/stridx.gemspec

@@ -0,0 +1,33 @@
+Gem::Specification.new do |spec|
+  spec.name = "StrIdx"
+  spec.version = "0.1.5"
+  spec.authors = ["Sami Sieranoja"]
+  spec.email = ["sami.sieranoja@gmail.com"]
+
+  spec.summary = %q{StrIdx}
+  spec.description = %q{ Fast fuzzy string similarity search and indexing (for filenames)}
+  spec.homepage = "https://github.com/SamiSieranoja/stridx"
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(refcode|spec|features)/})
+  end
+
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib", "ext"]
+
+  spec.add_development_dependency "bundler", "~> 2.4.21"
+  spec.add_development_dependency "rake", "~> 13.1.0"
+
+  spec.add_runtime_dependency "tty-cursor", "~> 0.7.1"
+  spec.add_runtime_dependency "tty-prompt", "~> 0.23.1"
+  spec.add_runtime_dependency "tty-reader", "~> 0.9.0"
+  spec.add_runtime_dependency "tty-screen", "~> 0.8.2"
+  spec.add_runtime_dependency "pastel", "~> 0.8.0"
+  spec.add_runtime_dependency "daemons", "~> 1.4.1"
+
+  spec.extensions = ["rubyext/extconf.rb"]
+  spec.licenses = ["LGPL-2.0+"]
+end