StrIdx 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7655b6bd71bca58c86ad607fd197933fc19b97b3ae1c76e322ec0432025dad7
-  data.tar.gz: 2421892aa6fe750213d08e2254019d87ce7abb10496cdf1635a61815b8b8b0d7
+  metadata.gz: e79f1bebc1e56e0a1966ae5ff69b50e4048953dd338807a6c9dba1c53b1c8f34
+  data.tar.gz: 28eea5841ce96f9460975720a43a2655f4902a9c44458a8aae01ce0b3077e67d
 SHA512:
-  metadata.gz: 0a0ed3f51b95b72a553cf97e1a852f63e8b6d1cbbba56fdab55ed5037ef4d658b8649f2cf92d35b5eea4e6657a18a3a1460a3a57069cda0889a1987f5d1611ee
-  data.tar.gz: f0d4753ee43cb205fa86468dad92a66644e12cf889d8018283cb421e4a5d670386b8666af64b89d4d475a59f10701de2223c31d03ac5d366f2ee255c77190cf8
+  metadata.gz: e82430abd644e876dd758ceaffb3f759e19fde2a44cc85872f01721943c42d85d2032cbde91d235d44b7921833ed044241692247c88b56f54c6ea7248f5e584e
+  data.tar.gz: 642e16d0e6291474b007e0341bf6571576299f353bf1f5ff13a5c9966d9b1b2450de052f3ebad6f44c9ab9b341686dd41b4adf35922406e3df358c8cbabc9c31

data/Makefile

@@ -1,6 +1,7 @@
 all: demo
 
 demo: *.hpp *.cpp Makefile
+	#g++  -Wall -O0 -g -lstdc++ demo.cpp -o demo
 	g++  -Wall -O3 -lstdc++ demo.cpp -o demo
 	
 clean: 

data/README.md

@@ -5,8 +5,14 @@ The fuzziness means that candidate filepaths do not need to include exact match
 
 The library can be applied for UTF-8 data also, although there is a small bias in scoring for multibyte characters.
 
+1. [String similarity calculation](#stringsim)
+2. [Interfaces](#interfaces)
+    1. [Command line](#comm)
+    2. [Ruby](#ruby)
+    3. [C++](#cpp)
 
-## String similarity calculation
+
+## String similarity calculation <a name="stringsim"/>
 
 Once the index has been created, the contents can be searched to find the best matching strings. 
 
@@ -39,7 +45,7 @@ and candidate is "./drivers/char/hw_random/nomadik-rng.c", then scores are calcu
 
 # Interfaces
 
-## Commandline
+## Commandline <a name="comm"/>
 Install instructions (for Ubuntu Linux):
 ```
 apt update
@@ -55,7 +61,14 @@ stridx.rb start -- ~/Documents/ ~/Pictures/
 Add bash keybindings (Ctrl-t):
 ```
 eval "$(stridx.rb bash)"
+
+```
+
+To autostart server, add following line to .bashrc:
 ```
+stridx.rb start -- ~/Documents/ ~/Pictures/
+```
+
 
 Search by pressing <kbd>ctrl</kbd>+<kbd>t</kbd>.  Keys: <kbd>up</kbd>, <kbd>down</kbd>, select with <kbd>enter</kbd>
 
@@ -151,7 +164,7 @@ Search time: 0.0488 seconds
 ```
 
 
-## C++
+## C++ <a name="cpp"/>
 See demo.cpp
 ```cpp
 #include "stridx.hpp"

data/demo.cpp

@@ -1,4 +1,9 @@
 
+#include <sys/resource.h>
+       #include <malloc.h>
+
+#include "mem_info.h"
+
 #include <condition_variable>
 #include <functional>
 #include <iostream>
@@ -19,7 +24,7 @@ using std::cout;
 using std::pair;
 using std::vector;
 
-std::vector<std::string> readLinesFromFile(const std::string &filename) {
+std::vector<std::string> readLinesFromFile(const std::string &filename, int limit = 0) {
   std::vector<std::string> lines;
   std::ifstream file(filename);
   if (!file.is_open()) {
@@ -28,8 +33,10 @@ std::vector<std::string> readLinesFromFile(const std::string &filename) {
   }
 
   std::string line;
-  while (std::getline(file, line)) {
+  int i=0;
+  while (std::getline(file, line) && ( limit == 0 || i < limit) ) {
     lines.push_back(line);
+    i++;
   }
 
   file.close();
@@ -37,59 +44,134 @@ std::vector<std::string> readLinesFromFile(const std::string &filename) {
 }
 
 int main() {
-  StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
-  // idx.addStrToIndex("./gdk/x11/gdkasync.c", 0 /*id*/, '/' /*separator*/);
-  // idx.addStrToIndex("./gdk/x11/gdksettings.c", 1, '/');
-  // idx.addStrToIndex("./gdk/x11/gdkx11devicemanager-xi2.h", 2, '/');
-
-  // Add the file paths of 89828 files in linux-6.9-rc6 to the index
-  std::string fn_filePaths = "flist.txt";
-  std::vector<std::string> v_filePaths = readLinesFromFile(fn_filePaths);
-
-	// Launch indexing to be run on background
-  cout << "File paths: " << v_filePaths.size() << std::endl;
-  cout << "Start indexing in the background" << std::endl;
-  auto start = std::chrono::high_resolution_clock::now();
-  int id = 0;
-  for (const auto &filePath : v_filePaths) {
-    idx.addStrToIndexThreaded(filePath, id);
-    id++;
-  }
+  {
+    StrIdx::StringIndex idx('/'); // Separate directories using unix style "/" char
+    // idx.addStrToIndex("./gdk/x11/gdkasync.c", 0 /*id*/, '/' /*separator*/);
+    // idx.addStrToIndex("./gdk/x11/gdksettings.c", 1, '/');
+    // idx.addStrToIndex("./gdk/x11/gdkx11devicemanager-xi2.h", 2, '/');
+
+    // Add the file paths of 89828 files in linux-6.9-rc6 to the index
+    std::string fn_filePaths = "flist2.txt";
+    std::vector<std::string> v_filePaths = readLinesFromFile(fn_filePaths);
+    // std::vector<std::string> v_filePaths = readLinesFromFile(fn_filePaths,10000);
+
+    // int* a = new int[10];
+    // delete(a);
+    // delete(a);
+
+    // Launch indexing to be run on background
+    cout << "File paths: " << v_filePaths.size() << std::endl;
+    cout << "Start indexing in the background" << std::endl;
+    auto start = std::chrono::high_resolution_clock::now();
+    int id = 0;
+    for (const auto &filePath : v_filePaths) {
+      // idx.addStrToIndexThreaded(filePath, id);
+      idx.addStrToIndex(filePath, id);
+      id++;
+    }
+
+    std::cout << "========\n";
+    for (int i = 0; i < id; i++) {
+      // std::cout << idx.getString(2) << "{}";
+      idx.getString(i);
+    }
+    std::cout << "========\n";
+
+    auto idx_time_launch = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double, std::milli> duration_launch = idx_time_launch - start;
+    cout << "Indexing launch time (seconds): " << duration_launch.count() / 1000 << "\n";
+
+    // Wait until indexing has finished
+    idx.waitUntilDone();
+
+    auto idx_time = std::chrono::high_resolution_clock::now();
+    std::chrono::duration<double, std::milli> duration = idx_time - start;
+    cout << "Indexing finished time for " << v_filePaths.size()
+         << " file paths (seconds): " << duration.count() / 1000 << "\n";
+
+    cout << "DEBUG" << std::endl;
+    // idx.cm.debug();
+    cout << "END DEBUG" << std::endl;
+
+    // Find matching filepaths from the index for the query string "rngnomadriv"
+    start = std::chrono::high_resolution_clock::now();
+    std::string query = "rngnomadriv";
+    // std::string query = "rngnomaindriv";
+    // std::string query = "time.rs";
+    // std::string query = "irqbypass.c";
+    for (int i = 0; i < 99; i++) {
+      // const vector<pair<float, int>> &results = idx.findSimilar(query);
+    }
 
-  auto idx_time_launch = std::chrono::high_resolution_clock::now();
-  std::chrono::duration<double, std::milli> duration_launch = idx_time_launch - start;
-  cout << "Indexing launch time (seconds): " << duration_launch.count() / 1000 << "\n";
+    // idx.findSim(query);
+
+    // auto res = idx.findDirectories(query);
+
+    // const vector<pair<float, int>> &results = idx.findSimilar(query);
+    // const vector<pair<float, int>> &results = idx.findDirectories(query);
+    // const vector<pair<float, std::string>> &results = idx.findFilesAndDirectories(query, true,
+    // false);
+    vector<pair<float, std::string>> results = idx.findFilesAndDirectories(query, true, false);
+
+    auto search_time = std::chrono::high_resolution_clock::now();
+    duration = search_time - start;
+    cout << "Search time for 100 queries (seconds): " << duration.count() / 1000 << "\n";
+
+    int i = 0;
+    std::cout << "query string: " << query << "\n";
+    std::cout << "Top 20 matches[1]:\n";
+    bool isDir = true;
+    for (const auto &res : results) {
+      // std::cout << res.second << " " << res.first << " " << v_filePaths[res.second] << "\n";
+      std::cout << res.first << " " << res.second << "\n";
+      i++;
+      if (i > 40) {
+        break;
+      }
+    }
+    
+    {
+
+      auto results = idx.findFiles(query);
+      int i = 0;
+      std::cout << "query string: " << query << "\n";
+      std::cout << "Top 20 matchesfff:\n";
+      bool isDir = true;
+      for (const auto &res : results) {
+        std::cout << res.second << " " << res.first << " " << v_filePaths[res.second] << "\n";
+        // std::cout << res.first << " " << res.second << "\n";
+        i++;
+        if (i > 40) {
+          break;
+        }
+      }
+    }
 
-	// Wait until indexing has finished
-  idx.waitUntilDone();
+    std::cout << "========\n";
+    for (int i = 0; i < id; i++) {
+      // std::cout << idx.getString(2) << "{}";
+      // idx.getString(i);
+    }
+    std::cout << "========\n";
 
-  auto idx_time = std::chrono::high_resolution_clock::now();
-  std::chrono::duration<double, std::milli> duration = idx_time - start;
-  cout << "Indexing finished time for " << v_filePaths.size()
-       << " file paths (seconds): " << duration.count() / 1000 << "\n";
+    std::cout << "Size of CharNode: " << sizeof(StrIdx::CharNode) << " bytes" << std::endl;
+    std::cout << "Size of int: " << sizeof(int) << " bytes" << std::endl;
 
-  // Find matching filepaths from the index for the query string "rngnomadriv"
-  start = std::chrono::high_resolution_clock::now();
-  std::string query = "rngnomadriv";
-  for (int i = 0; i < 99; i++) {
-    const vector<pair<float, int>> &results = idx.findSimilar(query, 2);
+    StrIdx::out.printl("MEMSTAT current:", getCurrentRSS(), " peak:", getPeakRSS());
+  // std::this_thread::sleep_for(std::chrono::milliseconds(7000));
+    
   }
 
-  const vector<pair<float, int>> &results = idx.findSimilar(query, 2);
-  auto search_time = std::chrono::high_resolution_clock::now();
-  duration = search_time - start;
-  cout << "Search time for 100 queries (seconds): " << duration.count() / 1000 << "\n";
+  // Force memory dealloc to properly benchmark
+  // https://www.reddit.com/r/C_Programming/comments/13dn8d7/is_malloc_trim_safe_to_use/
+  malloc_trim(0); 
+  
+  StrIdx::out.printl("MEMSTAT current:", getCurrentRSS(), " peak:", getPeakRSS());
 
-  int i = 0;
-  std::cout << "query string: " << query << "\n";
-  std::cout << "Top 20 matches:\n";
-  for (const auto &res : results) {
-    std::cout << res.second << " " << res.first << " " << v_filePaths[res.second] << "\n";
-    i++;
-    if (i > 20) {
-      break;
-    }
-  }
+  // std::this_thread::sleep_for(std::chrono::milliseconds(7000));
+  struct rusage usage;
+  getrusage(RUSAGE_SELF, &usage);
+  std::cout << "Maximum resident set size: " << usage.ru_maxrss << " kilobytes" << std::endl;
 
   return 0;
 }

data/exe/stridx.rb

@@ -1,16 +1,135 @@
 #!/usr/bin/env ruby
+require "fileutils"
 
 $:.unshift File.dirname(__FILE__) + "/.."
+require "server.rb"
 
-if ARGV[0] == "tty"
+CUR_FILE = File.basename(__FILE__)
+PID_FILE = File.expand_path("~/.config/stridx/index.pid")
+LOCK_FILE = File.expand_path("~/.config/stridx/index.lock")
+
+
+
+# To prevent against race condition when two process started at the same time
+def obtain_lock_or_exit
+  @lockfile = File.open(LOCK_FILE, File::RDWR | File::CREAT, 0644)
+
+  unless @lockfile.flock(File::LOCK_NB | File::LOCK_EX)
+    puts "Another instance is already running."
+    exit 1
+  end
+
+  # Optionally truncate and write PID for info/logging
+  @lockfile.truncate(0)
+  @lockfile.write("#{Process.pid}\n")
+  @lockfile.flush
+end
+
+def running?
+  return false unless File.exist?(PID_FILE)
+
+  pid = File.read(PID_FILE).to_i
+
+  begin
+    # Check if process exists
+    Process.kill(0, pid)
+
+    # Handle race condition: if the daemon was previously killed with "kill -9",
+    # the PID file may remain. A new, unrelated process could later reuse the same PID,
+    # causing a false positive when checking for an existing instance and preventing the daemon from starting.
+
+    # ./daemon.rb      # Starts daemon
+    # kill -9 $(cat /tmp/daemon_example.pid)  # Force kill
+    # echo $$ > /tmp/daemon_example.pid       # Simulate reused PID (use another terminal)
+    # ./daemon.rb      # Old version would fail here; fixed version should detect mismatch
+
+    # Check if command line matches this script
+    cmdline = File.read("/proc/#{pid}/cmdline").split("\0")
+
+    correct_process = cmdline.any? { |arg| arg.include?(CUR_FILE) }
+    puts correct_process
+    if correct_process == false
+      puts "Old pidfile points to wrong process"
+      return false
+    end
+
+    return true
+  rescue Errno::ESRCH, Errno::ENOENT
+    return false
+  rescue Errno::EACCES
+    # Process exists, but inaccessible — might still be ours
+    return true
+  end
+end
+
+# Old version without /proc check
+def running_old?
+  return false unless File.exist?(PID_FILE)
+  pid = File.read(PID_FILE).to_i
+  Process.kill(0, pid)
+  true
+rescue Errno::ESRCH, Errno::EPERM
+  false
+end
+
+def start(daemonize: false)
+  if running?
+    puts "Daemon is already running."
+    exit 1
+  end
+
+  if daemonize
+    # Daemonize the process
+    Process.daemon(true, true)  # Don't change directory, close stdio
+
+    # Save PID
+    File.write(PID_FILE, Process.pid)
+    puts "Daemon started with PID #{Process.pid}"
+
+    trap("TERM") do
+      puts "Daemon stopping..."
+      File.delete(PID_FILE) if File.exist?(PID_FILE)
+      exit
+    end
+
+    pid_dir_path = File.expand_path("~/.config/stridx/")
+    FileUtils.mkdir_p(pid_dir_path)
+  end
+
+  StrIdx::Server.start ARGV
+end
+
+def stop
+  unless File.exist?(PID_FILE)
+    puts "No PID file found. Daemon not running?"
+    exit 1
+  end
+
+  pid = File.read(PID_FILE).to_i
+  puts "Stopping daemon with PID #{pid}..."
+  Process.kill("TERM", pid)
+  File.delete(PID_FILE) rescue nil
+rescue Errno::ESRCH
+  puts "Process not found. Cleaning up PID file."
+  File.delete(PID_FILE) rescue nil
+end
+
+# Entry point
+case ARGV.first
+when "stop"
+  stop
+when "tty"
   require "stridx-tty.rb"
   StrIdxTTY.run
-elsif ARGV[0] == "bash"
+when "bash"
   puts %q/
   bind -m emacs-standard '"\er": redraw-current-line';
   bind -m emacs-standard '"\C-t": " \C-b\C-k \C-u`stridx.rb tty`\e\C-e\er\C-a\C-y\C-h\C-e\e \C-y\ey\C-x\C-x\C-f"'
 /
-else
-  require "daemons"
-  Daemons.run(File.dirname(__FILE__) + "/../runserver.rb")
+when "run"
+  obtain_lock_or_exit
+  start(daemonize: false)
+when "start"
+  obtain_lock_or_exit
+  start(daemonize: true)
 end